Merge branch 'ch_30065364' into 'develop'

changes for https://www.topcoder.com/challenges/30065364/?type=develop

See merge request luettich/receiver!9

Author: jmgasper

README.md

@@ -1,108 +1,104 @@
-## Requirements
-
-- Nodejs 8 is required
-- [Apache Kafka](https://kafka.apache.org/)
- - [Local installation guide](https://devops.profitbricks.com/tutorials/install-and-configure-apache-kafka-on-ubuntu-1604-1/)
-## Install dependencies
-
-```shell
-npm install
-```
-
-## Source code lint
-
-eslint is used to lint the javascript source code:
-
-```shell
-npm run lint
-```
-
-## Endpoints
-
-- POST /webhooks/github - The webhook handler for github
-- POST /webhooks/gitlab - The webhook handler for gitlab
-
-
-## Configuration
-
-The following config parameters are supported, they are defined in `config/default.js` and can be configured in system environment:
-
-
-| Name                           | Description                                | Default                          |
-| :----------------------------- | :----------------------------------------: | :------------------------------: |
-| PORT                           | the port the application will listen on    |  3000                            |
-| LOG_LEVEL                      | the log level                              |  info                            |
-| TOPIC                          | the kafka subscribe topic name             |  tc-x-events                    |
-| WEBHOOK_SECRET_TOKEN            | the webhook security token for githost, it must be same as `WEBHOOK_SECRET_TOKEN` configured for Topcoder-X-backend| `ka75hsrq65cFEr61Hd4x`|
-|KAFKA_OPTIONS                   | the connection option for kafka            |  see below about KAFKA options                   |
-| MONGODB_URL  | the MongoDB URL which must be same as Ragnar tool | mongodb://127.0.0.1:27017/ragnar|
-
-KAFKA_OPTIONS should be object as described in https://github.com/SOHU-Co/kafka-node#kafkaclient
-For using with SSL, the options should be as
-```
- {
-    kafkaHost: '<server>',
-    sslOptions: {
-      cert: '<certificate>', 
-      key:  '<key>'
-    }
- }
-```
-
-`config/local.js` will not tracked by git.
-
-## Local Setup
-
-```shell
-npm start
-```
-
-Server should be started at port 3002.
-
-use `ngrok` to make your local deploy accessible by internet:
-```shell
-ngrok http 3002
-```
-
-Copy the forwarding URL to set in `HOOK_BASE_URL` of topcoder-x-ui in config.json
-
-## Setup for verification
-Before verifying the tool, 4 service needs be configured and run them
-- processor
-- receiver
-- Ragnar Tool
-- Topcoder X (both backend and UI)
-
-First login in Ragnar tool with admin and Add owner for which requires topcoder handle, git host's username and type of git host.
-
-Go to Topcoder X UI login with above used topcoder username and
-- go to settings and make sure git hosts are correctly setup, if not click setup and authorize to setup.
-
-- Go to Topcoder X UI and go to project management and add a project from git account and click save, and edit the same project and click 'Add Webhooks' button (you need to add personnel access token), verify that webhooks are set up correctly on git host's project.
-
-Now, receiver service can receive the webhooks from git host's project. Now you can verify this service by following the verfication steps below
-
-## GitHub Verification
-
-- create an issue in the repo, you can see the logs in `receiver` and `processor`, the `issue.created` event is generated.
-- update an issue in the repo, you can see the logs in `receiver` and `processor`, the `issue.updated` event is generated.
-- create a comment on an issue, you can see the logs in `receiver` and `processor`, the `comment.created` event is generated.
-- update a comment on an issue, you can see the logs in `receiver` and `processor`, the `comment.updated` event is generated.
-- assigned a user to an issue, you can see the logs in `receiver` and `processor`, the `issue.assigned` event is generated.
-- un-assigned a user to an issue, you can see the logs in `receiver` and `processor`, the `issue.unassigned` event is generated.
-- add/remove a label to an issue, you can see the logs in `receiver` and `processor`, the `issue.labelUpdated` event is generated.
-- create a pull request, you can see the logs in `receiver` and `processor`, the `pull_request.created` event is generated.
-- close a pull request without merge, you can see the logs in `receiver` and `processor`, the `pull_request.closed` event is generated and the `merged` property is `false`.
-- merge a pull request, you can see the logs in `receiver` and `processor`, the `pull_request.closed` event is generated and the `merged` property is `true`.
-
-## Gitlab Verification
-
-- create an issue in the repo, you can see the logs in `receiver` and `processor`, the `issue.created` event is generated.
-- update an issue in the repo, you can see the logs in `receiver` and `processor`, the `issue.updated` event is generated.
-- create a comment on an issue, you can see the logs in `receiver` and `processor`, the `comment.created` event is generated.
-- assigned a user to an issue, you can see the logs in `receiver` and `processor`, the `issue.assigned` event is generated.
-- un-assigned a user to an issue, you can see the logs in `receiver` and `processor`, the `issue.unassigned` event is generated.
-- add/remove a label to an issue, you can see the logs in `receiver` and `processor`, the `issue.labelUpdated` event is generated.
-- create a pull request, you can see the logs in `receiver` and `processor`, the `pull_request.created` event is generated.
-- close a pull request without merge, you can see the logs in `receiver` and `processor`, the `pull_request.closed` event is generated and the `merged` property is `false`.
+## Requirements
+
+- Nodejs 8 is required
+- [Apache Kafka](https://kafka.apache.org/)
+ - [Local installation guide](https://devops.profitbricks.com/tutorials/install-and-configure-apache-kafka-on-ubuntu-1604-1/)
+## Install dependencies
+
+```shell
+npm install
+```
+
+## Source code lint
+
+eslint is used to lint the javascript source code:
+
+```shell
+npm run lint
+```
+
+## Endpoints
+
+- POST /webhooks/github - The webhook handler for github
+- POST /webhooks/gitlab - The webhook handler for gitlab
+
+
+## Configuration
+
+The following config parameters are supported, they are defined in `config/default.js` and can be configured in system environment:
+
+
+| Name                           | Description                                | Default                          |
+| :----------------------------- | :----------------------------------------: | :------------------------------: |
+| PORT                           | the port the application will listen on    |  3000                            |
+| LOG_LEVEL                      | the log level                              |  info                            |
+| TOPIC                          | the kafka subscribe topic name             |  tc-x-events                    |
+| KAFKA_OPTIONS                   | the connection option for kafka            |  see below about KAFKA options                   |
+| MONGODB_URL  | the MongoDB URL which must be same as Topcoder x tool | mongodb://127.0.0.1:27017/topcoderx|
+
+KAFKA_OPTIONS should be object as described in https://github.com/SOHU-Co/kafka-node#kafkaclient
+For using with SSL, the options should be as
+```
+ {
+    kafkaHost: '<server>',
+    sslOptions: {
+      cert: '<certificate>', 
+      key:  '<key>'
+    }
+ }
+```
+
+`config/local.js` will not tracked by git.
+
+## Local Setup
+
+```shell
+npm start
+```
+
+Server should be started at port 3002.
+
+use `ngrok` to make your local deploy accessible by internet:
+```shell
+ngrok http 3002
+```
+
+Copy the forwarding URL to set in `HOOK_BASE_URL` of topcoder-x in config.json
+
+## Setup for verification
+Before verifying the tool, 3 service needs be configured and run them
+- processor
+- receiver
+- Topcoder X
+
+Go to Topcoder X UI login with above used topcoder username and
+- go to settings and make sure git hosts are correctly setup, if not click setup and authorize to setup.
+
+- Go to Topcoder X UI and go to project management and add a project from git account and click save, and edit the same project and click 'Add Webhooks' button, verify that webhooks are set up correctly on git host's project.
+
+Now, receiver service can receive the webhooks from git host's project. Now you can verify this service by following the verfication steps below
+
+## GitHub Verification
+
+- create an issue in the repo, you can see the logs in `receiver` and `processor`, the `issue.created` event is generated.
+- update an issue in the repo, you can see the logs in `receiver` and `processor`, the `issue.updated` event is generated.
+- create a comment on an issue, you can see the logs in `receiver` and `processor`, the `comment.created` event is generated.
+- update a comment on an issue, you can see the logs in `receiver` and `processor`, the `comment.updated` event is generated.
+- assigned a user to an issue, you can see the logs in `receiver` and `processor`, the `issue.assigned` event is generated.
+- un-assigned a user to an issue, you can see the logs in `receiver` and `processor`, the `issue.unassigned` event is generated.
+- add/remove a label to an issue, you can see the logs in `receiver` and `processor`, the `issue.labelUpdated` event is generated.
+- create a pull request, you can see the logs in `receiver` and `processor`, the `pull_request.created` event is generated.
+- close a pull request without merge, you can see the logs in `receiver` and `processor`, the `pull_request.closed` event is generated and the `merged` property is `false`.
+- merge a pull request, you can see the logs in `receiver` and `processor`, the `pull_request.closed` event is generated and the `merged` property is `true`.
+
+## Gitlab Verification
+
+- create an issue in the repo, you can see the logs in `receiver` and `processor`, the `issue.created` event is generated.
+- update an issue in the repo, you can see the logs in `receiver` and `processor`, the `issue.updated` event is generated.
+- create a comment on an issue, you can see the logs in `receiver` and `processor`, the `comment.created` event is generated.
+- assigned a user to an issue, you can see the logs in `receiver` and `processor`, the `issue.assigned` event is generated.
+- un-assigned a user to an issue, you can see the logs in `receiver` and `processor`, the `issue.unassigned` event is generated.
+- add/remove a label to an issue, you can see the logs in `receiver` and `processor`, the `issue.labelUpdated` event is generated.
+- create a pull request, you can see the logs in `receiver` and `processor`, the `pull_request.created` event is generated.
+- close a pull request without merge, you can see the logs in `receiver` and `processor`, the `pull_request.closed` event is generated and the `merged` property is `false`.
 - merge a pull request, you can see the logs in `receiver` and `processor`, the `pull_request.closed` event is generated and the `merged` property is `true`.

config/default.js

@@ -17,13 +17,12 @@ module.exports = {
   PORT: process.env.PORT || 3002, // eslint-disable-line no-magic-numbers
   LOG_LEVEL: process.env.LOG_LEVEL || 'info',
   TOPIC: process.env.TOPIC || 'tc-x-events',
-  WEBHOOK_SECRET_TOKEN: process.env.WEBHOOK_SECRET_TOKEN || 'ka75hsrq65cFEr61Hd4x',
   KAFKA_OPTIONS: {
     kafkaHost: process.env.KAFKA_HOST || 'localhost:9092',
     sslOptions: {
       cert: process.env.KAFKA_CLIENT_CERT || fs.readFileSync('./kafka_client.cer'), // eslint-disable-line no-sync
       key: process.env.KAFKA_CLIENT_CERT_KEY || fs.readFileSync('./kafka_client.key') // eslint-disable-line no-sync
     }
   },
-  MONGODB_URL: process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/ragnar'
+  MONGODB_URL: process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/topcoderx'
 };

models/Challenge.js renamed to models/Project.js

@@ -1,15 +1,13 @@
 /*
  * Copyright (c) 2018 TopCoder, Inc. All rights reserved.
  */
+'use strict';
 
 /**
- * This module contains the schema of the Challenge.
- *
+ * Schema for project and repository mapping.
  * @author TCSCODER
  * @version 1.0
  */
-'use strict';
-
 const mongoose = require('mongoose');
 
 const schema = new mongoose.Schema({
@@ -18,7 +16,13 @@ const schema = new mongoose.Schema({
   repoUrl: {type: String, required: true},
   rocketChatWebhook: {type: String, required: true},
   rocketChatChannelName: {type: String, required: true},
-  archieved: {type: String, required: true},
-  username: {type: String, required: true}
+  archived: {type: String, required: true},
+  username: {type: String, required: true},
+  secretWebhookKey: {type: String, required: true}
 });
+
+// project id, provider, repositoryId must be unique
+schema.index({tcDirectId: 1}, {unique: true});
+
+
 module.exports = schema;

models/index.js

@@ -24,7 +24,7 @@ const UserUnassignedEvent = require('./UserUnassignedEvent');
 const PullRequestCreatedEvent = require('./PullRequestCreatedEvent');
 const PullRequestClosedEvent = require('./PullRequestClosedEvent');
 const LabelUpdatedEvent = require('./LabelUpdatedEvent');
-const Challenge = require('./Challenge');
+const Project = require('./Project');
 
 module.exports = {
   IssueCreatedEvent,
@@ -36,5 +36,5 @@ module.exports = {
   PullRequestCreatedEvent,
   PullRequestClosedEvent,
   LabelUpdatedEvent,
-  Challenge: connection.model('Challenge', Challenge)
+  Project: connection.model('Project', Project)
 };

routes/middlewares/RepositoryFilter.js

@@ -11,20 +11,20 @@
 const _ = require('lodash');
 
 const logger = require('../../utils/logger');
-const Challenge = require('../../models').Challenge;
+const Project = require('../../models').Project;
 
 module.exports = (provider) => async (req, res, next) => {
   let repoNames = [];
   if (provider === 'github') {
     const repo = req.body.repository || {};
     repoNames = [repo.svn_url, repo.git_url, repo.ssh_url, repo.clone_url];
   } else if (provider === 'gitlab') {
-    const repo = req.body.repository || {};
-    repoNames = [repo.homepage, repo.http_url, repo.url, repo.ssh_url];
+    const repo = req.body.project || {};
+    repoNames = [repo.homepage, repo.http_url, repo.url, repo.ssh_url, repo.web_url];
   }
   let found = false;
-  const challenges = await Challenge.find({archieved: false});
-  found = _.some(challenges, (challenge) => _.includes(repoNames, challenge.repoUrl));
+  const projects = await Project.find({archived: false});
+  found = _.some(projects, (project) => _.includes(repoNames, project.repoUrl));
   if (found) {
     return next();
   }

routes/middlewares/SecurityChecker.js

@@ -9,16 +9,19 @@
  */
 'use strict';
 const crypto = require('crypto');
-const config = require('config');
 const logger = require('../../utils/logger');
+const Project = require('../../models').Project;
 
-module.exports = (provider) => (req, res, next) => {
+module.exports = (provider) => async (req, res, next) => {
   let isValid = false;
+  const params = req.body;
   if (provider === 'github') {
-    const hash = crypto.createHmac('sha1', config.WEBHOOK_SECRET_TOKEN).update(req.rawBody).digest('hex');
+    const projectDetail = await Project.findOne({repoUrl: params.repository.html_url});
+    const hash = crypto.createHmac('sha1', projectDetail.secretWebhookKey).update(req.rawBody).digest('hex');
     isValid = `sha1=${hash}` === req.header('X-Hub-Signature');
   } else if (provider === 'gitlab') {
-    isValid = config.WEBHOOK_SECRET_TOKEN === req.header('X-Gitlab-Token');
+    const projectDetail = await Project.findOne({repoUrl: params.project.web_url});
+    isValid = projectDetail.secretWebhookKey === req.header('X-Gitlab-Token');
   } else {
     // unknown provider
     return next();