...
 
Commits (12)
......@@ -12,3 +12,5 @@ doc/en/dev_api.md
.vagrant
duralex
sedlex
dist
gitlab_token.json
......@@ -3,6 +3,8 @@ Vagrant.require_version ">= 2.0.0"
Vagrant.configure(2) do |config|
config.vm.box = "ubuntu/xenial64"
config.vm.synced_folder "./duralex", "/srv/duralex"
config.vm.synced_folder "./sedlex", "/srv/sedlex"
config.vm.provider :virtualbox do |vbox, override|
vbox.customize ["modifyvm", :id, "--memory", 4096]
......@@ -11,6 +13,7 @@ Vagrant.configure(2) do |config|
# Allow symlinks in the /vagrant shared folder on Windows.
# https://github.com/npm/npm/issues/7308#issuecomment-84214837
vbox.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/vagrant", "1"]
end
config.vm.network :private_network, ip: "192.168.50.42"
......
......@@ -20,5 +20,5 @@ app.use(bodyParser.json());
app.use('/api/v1/ping', require('./ping'));
app.listen(3001, () => {
console.log('API server listening on port 3001')
console.log('API server listening on port 3001');
});
{
"name": "amenda-gitlab-bot",
"version": "1.0.0",
"description": "",
"main": "index.js",
"dependencies": {
"babel-cli": "^6.26.0",
"babel-plugin-transform-async-to-generator": "^6.24.1",
"babel-polyfill": "^6.26.0",
"babel-preset-es2015": "^6.24.1",
"express": "^4.15.5",
"express-winston": "^2.4.0",
"nodemon": "^1.12.1",
"superagent": "^3.7.0",
"superagent-use": "^0.1.0",
"winston": "^3.0.0-rc1"
},
"devDependencies": {},
"scripts": {
"build": "rm -rf dist && babel src --out-dir dist --source-maps inline",
"start": "nodemon -L -w src -w /etc/amenda --exec 'npm run build && nodejs ./dist/index.js'",
"test": "echo \"Error: no test specified\" && exit 1"
},
"babel": {
"presets": [
"es2015"
],
"plugins": [
"transform-async-to-generator",
"add-module-exports"
]
}
}
const request = require('superagent-use')(require('superagent'));
const logger = require('./logger');
import fs from 'fs';
request.use((req) => req.set('Accept', 'application/json'));
export default class GitLabAPI {
constructor(url) {
this.url = url;
}
async oauth2Connect(username, password) {
var token = null;
try {
token = JSON.parse(fs.readFileSync('gitlab_token.json', 'utf-8'));
// If the token is not valid anymore, this request should fail and throw.
await request
.get(this.url + '/api/v4/projects')
.set('Authorization', 'Bearer ' + token.access_token);
} catch (e) {
logger.info('getting/renewing OAuth2 token for user %s', username);
const res = await request
.post(this.url + '/oauth/token')
.send({
grant_type: 'password',
username: username,
password: password,
});
token = res.body;
// FIXME: use redis instead?
fs.writeFileSync('gitlab_token.json', JSON.stringify(token));
}
request.use((req) => req.set('Authorization', 'Bearer ' + token.access_token));
return token;
}
// https://docs.gitlab.com/ce/api/system_hooks.html#list-system-hooks
async getSystemHooks() {
const res = await request.get(this.url + '/api/v4/hooks');
return res.body;
}
// https://docs.gitlab.com/ce/api/system_hooks.html#add-new-system-hook
async addSystemHook(url, token, enableSSLVerification, {pushEvents, tagPushEvents}) {
try {
const res = await request.post(this.url + '/api/v4/hooks')
.send({
url: url,
token: token,
push_events: !!pushEvents,
tag_push_events: !!tagPushEvents,
enable_ssl_verification: enableSSLVerification,
});
return res.body;
} catch (e) {
logger.error(e);
}
}
// https://docs.gitlab.com/ce/api/projects.html#list-all-projects
async getProjects() {
const res = await request.get(this.url + '/api/v4/projects');
return res.body;
}
// https://docs.gitlab.com/ce/api/projects.html#list-all-projects
async getProjects() {
const res = await request.get(this.url + '/api/v4/projects');
return res.body;
}
// https://docs.gitlab.com/ce/api/projects.html#list-project-hooks
async getProjectHooks(projectId) {
try {
const res = await request.get(this.url + '/api/v4/projects/' + projectId + '/hooks');
return res.body;
} catch (e) {
logger.error(e);
}
}
// https://docs.gitlab.com/ce/api/projects.html#get-project-hook
async getProjectHook(projectId, hookId) {
const res = await request.get(this.url + '/api/v4/projects/' + projectId + '/hooks/' + hookId);
return res.body;
}
// https://docs.gitlab.com/ce/api/projects.html#add-project-hook
async addProjectHook(
projectId, url, token, enableSSLVerification,
{ pushEvents, issuesEvents, mergeRequestsEvents, tagPushEvents, noteEvents, jobEvents, pipelineEvents, wikiEvents}
) {
const res = await request.post(this.url + '/api/v4/projects/' + projectId + '/hooks')
.send({
id: projectId,
url: url,
push_events: !!pushEvents,
issues_events: !!issuesEvents,
merge_requests_events: !!mergeRequestsEvents,
tag_push_events: !!tagPushEvents,
note_events: !!noteEvents,
job_events: !!jobEvents,
pipeline_events: !!pipelineEvents,
wiki_events: !!wikiEvents,
enable_ssl_verification: enableSSLVerification,
token: token,
});
return res.body;
}
// https://docs.gitlab.com/ce/api/merge_requests.html#get-single-mr-changes
async getMergeRequestChanges(projectId, mergeRequestIid) {
const res = await request.get(this.url + '/api/v4/projects/' + projectId + '/merge_requests/' + mergeRequestIid + '/changes');
return res.body;
}
// https://docs.gitlab.com/ce/api/merge_requests.html#get-single-mr-commits
async getMergeRequestCommits(projectId, mergeRequestIid) {
const res = await request.get(this.url + '/api/v4/projects/' + projectId + '/merge_requests/' + mergeRequestIid + '/commits');
return res.body;
}
// https://docs.gitlab.com/ce/api/commits.html#get-the-diff-of-a-commit
async getCommitDiff(projectId, commitSha) {
const res = await request.get(this.url + '/api/v4/projects/' + projectId + '/repository/commits/' + commitSha + '/diff');
return res.body;
}
// https://docs.gitlab.com/ce/api/repositories.html#compare-branches-tags-or-commits
async compare(projectId, from, to) {
const res = await request.get(this.url + '/api/v4/projects/' + projectId + '/repository/compare')
.query({
from: from,
to: to,
});
return res.body;
}
// https://docs.gitlab.com/ce/api/notes.html#create-new-merge-request-note
async addMergeRequestNote(projectId, mergeRequestIid, body) {
const res = await request
.post(this.url + '/api/v4/projects/' + projectId + '/merge_requests/' + mergeRequestIid + '/notes')
.send({
body: body,
});
return res.body;
}
}
import config from '/etc/amenda/amenda.json';
import logger from './logger';
export default class GitLabWebhook {
constructor(route, options, eventHeader = false, eventNames = null) {
this.route = route;
this.url = config.webhook.url + route;
this.options = options;
this.eventHeader = eventHeader;
this.eventNames = eventNames;
}
middleware(req, res, next) {
if (req.headers['x-gitlab-token'] != config.webhook.token) {
res.sendStatus(400);
return;
}
const validEventHeader = !!this.eventHeader && req.headers['x-gitlab-event'] == this.eventHeader;
const hasEventName = 'event_name' in req.body && !!this.eventNames;
const validEventName = hasEventName && this.eventNames.indexOf(req.body.event_name) >= 0;
if (validEventHeader && (!hasEventName || validEventName)) {
next();
} else {
res.sendStatus(444);
}
}
callback(req, res) {
throw "GitLabWebhook.callback() must be overriden.";
}
setup(app) {
app.post(this.route, this.middleware.bind(this), this.callback.bind(this));
logger.info('webhook setup on %s', this.route, {webhook: this});
}
static async addMissingProjectHooks(gitlab, token) {
const requiredProjectHooks = require('./webhook/project').required;
const projects = await gitlab.getProjects();
for (var project of projects) {
const projectHooks = await gitlab.getProjectHooks(project.id);
const missingProjectHooks = requiredProjectHooks.filter((rh) => projectHooks.filter(hook => hook.url == rh.url).length == 0);
for (var missingHook of missingProjectHooks) {
logger.info("adding missing project webhook", {project: project.id, webhook: missingHook});
await gitlab.addProjectHook(project.id, missingHook.url, token, false, missingHook.options);
}
}
}
static async addMissingSystemHooks(gitlab, token) {
const requiredSystemHooks = require('./webhook/system').required;
const hooks = await gitlab.getSystemHooks();
const missingSystemHooks = requiredSystemHooks.filter((rh) => hooks.filter(hook => hook.url == rh.url).length == 0);
for (var missingHook of missingSystemHooks) {
logger.info("adding missing system webhook", {webhook: missingHook});
await gitlab.addSystemHook(missingHook.url, token, false, missingHook.options);
}
}
static setupAllWebhooks(app) {
const requiredSystemHooks = require('./webhook/system').required;
const requiredProjectHooks = require('./webhook/project').required;
for (const hook of requiredSystemHooks) {
hook.setup(app);
}
for (const hook of requiredProjectHooks) {
hook.setup(app);
}
}
}
import 'babel-polyfill';
try {
require('source-map-support').install();
} catch (e) {
}
import config from '/etc/amenda/amenda.json';
import express from 'express';
import bodyParser from 'body-parser';
import expressWinston from 'express-winston';
import logger from './logger';
import GitLabAPI from './GitLabAPI';
import GitLabWebhook from './GitLabWebhook';
(async () => {
const app = express();
const gitlab = new GitLabAPI(config.gitlab.url);
app.use(bodyParser.urlencoded({ extended: true }));
app.use(bodyParser.json());
app.use(expressWinston.logger({
winstonInstance: logger,
level: 'debug',
bodyWhitelist: ['event_name', 'object_kind', 'object_attributes'],
}));
await gitlab.oauth2Connect(config.gitlab.adminEmail, config.gitlab.adminPassword);
await GitLabWebhook.addMissingProjectHooks(gitlab, config.webhook.token);
await GitLabWebhook.addMissingSystemHooks(gitlab, config.webhook.token);
GitLabWebhook.setupAllWebhooks(app);
app.get('/ping', (req, res) => res.json({ping: 'pong'}));
const listener = app.listen(3002, () => {
logger.info('webhook server listening', {port: listener.address().port});
});
})();
import { createLogger, format, transports } from 'winston';
const label = 'gitlab-bot';
const MESSAGE = Symbol.for('message');
const customFormat = format((info, opt) => {
info[MESSAGE] = info.timestamp + ' ' + info.level + ': ' + info.message + '\n' + JSON.stringify(
Object.assign({}, info.meta || info, {
level: undefined,
message: undefined,
splat: undefined,
timestamp: undefined,
label: undefined,
}),
false,
2
);
return info;
});
const logger = createLogger({
level: process.env.NODE_ENV == 'production' ? 'info' : 'debug',
exitOnError: true,
format: format.combine(
format.timestamp(),
format.label({ label: label }),
format.json(),
),
transports: [
new transports.File({ filename: 'error.log', level: 'error' }),
new transports.File({ filename: 'combined.log' }),
],
});
if (process.env.NODE_ENV !== 'production') {
logger.add(
new transports.Console({
handleExceptions: true,
format: format.combine(
format.timestamp(),
format.label({ label: label }),
format.colorize(),
format.splat(),
customFormat(),
),
})
);
}
export default logger;
import config from '/etc/amenda/amenda.json';
import { execSync } from 'child_process';
import logger from '../../logger';
import GitLabAPI from '../../GitLabAPI';
import GitLabWebhook from '../../GitLabWebhook';
// https://docs.gitlab.com/ce/user/project/integrations/webhooks.html#merge-request-events
export default class MergeRequest extends GitLabWebhook {
constructor() {
super('/project/merge-request', {mergeRequestsEvents: true}, 'Merge Request Hook');
}
async callback(req, res) {
const webhook = req.body;
logger.info(
'received merge request hook',
{
iid: webhook.object_attributes.iid,
target_project_id: webhook.object_attributes.target_project_id,
source_project_id: webhook.object_attributes.source_project_id,
}
);
const gitlab = new GitLabAPI(config.gitlab.url);
await gitlab.oauth2Connect(config.gitlab.adminEmail, config.gitlab.adminPassword);
const commits = await gitlab.getMergeRequestCommits(
webhook.object_attributes.target_project_id,
webhook.object_attributes.iid
);
var diffs = null;
if (commits.length == 1) {
// If we have a single commit, the diff for this merge request is the diff of that commit
const diff = await gitlab.getCommitDiff(webhook.object_attributes.target_project_id, commits[0].id);
diffs = diff.map(diff => diff.diff);
} else {
// Otherwise, we compare the first and the last commit.
const changes = await gitlab.compare(
webhook.object_attributes.source_project_id,
commits[0].id,
commits[commits.length - 1].id,
);
diffs = changes.diffs.map(diff => diff.diff);
}
logger.debug(
'retrieved diffs for merge request %s',
webhook.object_attributes.iid,
{
iid: webhook.object_attributes.iid,
diffs: diffs,
}
);
const texts = [];
for (var diff of diffs) {
const duralex = execSync(
'/usr/local/bin/duralex',
{input: 'diff\n' + diff, encoding: 'utf-8'}
);
logger.debug(duralex);
const sedlex = execSync(
'/usr/local/bin/sedlex --amendments --commit-message',
{input: duralex, encoding: 'utf-8'}
);
logger.debug(sedlex);
this.findCommitMessages(JSON.parse(sedlex), texts);
}
const note = await gitlab.addMergeRequestNote(
webhook.object_attributes.target_project_id,
webhook.object_attributes.iid,
texts.join('\n\n')
);
res.status(200).send();
}
findCommitMessages(obj, res = []) {
if ('commitMessage' in obj) {
res.push(obj['commitMessage']);
}
for (var propName in obj) {
if (Array.isArray(obj[propName])) {
for (var item of obj[propName]) {
if (typeof item === 'object') {
this.findCommitMessages(item, res);
}
}
}
}
return res;
}
}
import MergeRequest from './MergeRequest'
const required = [
new MergeRequest(),
];
export { required };
import config from '/etc/amenda/amenda.json';
import logger from '../../logger';
import GitLabAPI from '../../GitLabAPI';
import GitLabWebhook from '../../GitLabWebhook';
import {required as requiredProjectHooks} from '../project';
export default class ProjectCreated extends GitLabWebhook {
constructor() {
super('/system/project-create', {}, 'System Hook', ['project_create']);
}
async callback(req, res) {
const webhook = req.body;
const gitlab = new GitLabAPI(config.gitlab.url);
await gitlab.oauth2Connect(config.gitlab.adminEmail, config.gitlab.adminPassword);
const projectHooks = await gitlab.getProjectHooks(webhook.project_id);
for (var projectHook of requiredProjectHooks) {
logger.info(
"adding '%s' webhook to project %s",
projectHook.eventHeader,
webhook.project_id,
{project: webhook.project_id, webhook: projectHook}
);
await gitlab.addProjectHook(
webhook.project_id, projectHook.url, config.webhook.token, false, projectHook.options
);
}
res.status(200).send();
}
}
import ProjectCreated from './ProjectCreated'
const required = [
new ProjectCreated(),
];
export { required };
......@@ -11,6 +11,8 @@ nodejs_version: "6.x"
gitlab_admin_email: "admin@{{ hostname }}"
gitlab_admin_password: "adminadmin"
gitlab_bot_webhook_token: "a153b0d18961a9947a63888de405eb40"
pm2_service_enabled: yes
pm2_service_state: started
pm2_startup: systemd
......@@ -27,5 +29,9 @@ pm2_apps:
cmd: start --name app
args: -- start
path: /vagrant/app
- run: npm
cmd: start --name gitlab-bot
args: -- start
path: /vagrant/gitlab-bot
pm2_post_cmds:
- run: save
......@@ -3,12 +3,11 @@
- hosts:
- all
roles:
- { role: docker, tags: ["api", "doc", "gitlab"] }
- { role: config, tags: ["config", "api", "gitlab-bot"] }
- { role: gitlab, tags: ["api", "gitlab"] }
- { role: nodejs, tags: ["api", "app", "doc"] }
- { role: ssl-certs, tags: ["api", "doc", "gitlab"] }
- { role: reverse-proxy, tags: ["api", "doc", "gitlab"] }
- { role: api, tags: ["api"] }
- { role: app, tags: ["app"] }
- { role: gitlab-bot, tags: ["gitlab-bot", "gitlab"] }
- { role: doc, tags: ["doc"] }
- { role: pm2, tags: ["api", "app"] }
- { role: pm2, tags: ["pm2", "api", "app", "gitlab-bot"] }
---
- src: angstwad.docker_ubuntu
version: v3.2.0
- src: geerlingguy.docker
version: 2.1.0
name: docker
- src: geerlingguy.nodejs
......@@ -19,3 +19,6 @@
- src: weareinteractive.pm2
version: 2.4.0
name: weareinteractive.pm2
- src: telusdigital.python
name: python
---
dependencies:
- { role: nodejs }
---
dependencies:
- { role: nodejs }
---
- name: Create /etc/amenda
file:
path: /etc/amenda
state: directory
- name: Install /etc/amenda/amenda.json
template:
src: amenda.json.j2
dest: /etc/amenda/amenda.json
{
"gitlab": {
"url": "http://127.0.0.1:8000/admin",
"adminEmail": "{{ gitlab_admin_email }}",
"adminPassword": "{{ gitlab_admin_password }}"
},
"webhook": {
"url": "http://172.17.0.1:3002",
"token": "{{ gitlab_bot_webhook_token }}"
}
}
---
- name: Check if /srv/duralex/bin/duralex exists
stat:
path: /srv/duralex/bin/duralex
register: duralex_sedlex_duralex_bin
- name: Install DuraLex
pip:
name: git+git://github.com/Legilibre/DuraLex.git@0.3
virtualenv_python: python3
virtualenv: /opt/duralex-sedlex
when: not duralex_sedlex_duralex_bin.stat.exists
- name: Install duralex dependencies
pip:
executable: pip3
requirements: /srv/duralex/requirements.txt
when: duralex_sedlex_duralex_bin.stat.exists
- name: Link the duralex binary
file:
src: "{{ duralex_sedlex_duralex_bin.stat.exists | ternary('/srv/duralex/bin/duralex', '/opt/duralex-sedlex/bin/duralex') }}"
dest: /usr/local/bin/duralex
state: link
- name: Check if /srv/sedlex/bin/sedlex exists
stat:
path: /srv/sedlex/bin/sedlex
......@@ -38,8 +14,9 @@
- name: Install sedlex dependencies
pip:
executable: pip3
requirements: /srv/sedlex/requirements.txt
virtualenv_python: python3
virtualenv: /opt/sedlex
when: duralex_sedlex_sedlex_bin.stat.exists
- name: Link the sedlex binary
......@@ -47,3 +24,29 @@
src: "{{ duralex_sedlex_sedlex_bin.stat.exists | ternary('/srv/sedlex/bin/sedlex', '/opt/duralex-sedlex/bin/sedlex') }}"
dest: /usr/local/bin/sedlex
state: link
- name: Check if /srv/duralex/bin/duralex exists
stat:
path: /srv/duralex/bin/duralex
register: duralex_sedlex_duralex_bin
- name: Install DuraLex
pip:
name: git+git://github.com/Legilibre/DuraLex.git@0.3
virtualenv_python: python3
virtualenv: /opt/duralex-sedlex
when: not duralex_sedlex_duralex_bin.stat.exists
- name: Install duralex dependencies
pip:
requirements: /srv/duralex/requirements.txt
virtualenv_python: python3
virtualenv: /opt/duralex
when: duralex_sedlex_duralex_bin.stat.exists
- name: Link the duralex binary
file:
src: "{{ duralex_sedlex_duralex_bin.stat.exists | ternary('/srv/duralex/bin/duralex', '/opt/duralex-sedlex/bin/duralex') }}"
dest: /usr/local/bin/duralex
state: link
force: yes
---
dependencies:
- { role: duralex-sedlex }
---
- name: Install dependencies
npm:
path: /vagrant/gitlab-bot
---
gitlab_version: 10.1.0-ce.0
---
dependencies:
- { role: docker }
......@@ -3,7 +3,7 @@
- name: Install Gitlab (via Docker)
docker_container:
name: gitlab
image: gitlab/gitlab-ce:10.0.1-ce.0
image: "gitlab/gitlab-ce:{{ gitlab_version }}"
state: started
restart_policy: always
hostname: "{{ hostname }}"
......
---
dependencies:
- { role: ssl-certs }
......@@ -46,6 +46,10 @@
ProxyPass http://127.0.0.1:8000/admin/ keepalive=off retry=0
ProxyPassReverse http://127.0.0.1:8000/admin/
</Location>
<Location "/admin/api/*">
Order deny,allow
Deny from all
</Location>
<Location "/api">
ProxyPass http://127.0.0.1:3001/api keepalive=off retry=0
......