Merge remote-tracking branch 'github/develop'

Author: jmgasper

.dockerignore

@@ -0,0 +1,10 @@
+node_modules
+npm-debug.log
+Dockerfile*
+docker-compose*
+.dockerignore
+.git
+.gitignore
+README.md
+LICENSE
+.vscode

Dockerfile

@@ -0,0 +1,8 @@
+FROM node:8.9-alpine
+WORKDIR /usr/src/app
+COPY ["package.json", "package-lock.json*", "npm-shrinkwrap.json*", "./"]
+RUN echo "installing packages"
+RUN npm install --silent
+COPY . .
+EXPOSE 80
+CMD npm start

README.md

@@ -25,28 +25,7 @@ npm run lint
 
 ## Configuration
 
-The following config parameters are supported, they are defined in `config/default.js` and can be configured in system environment:
-
-
-| Name                           | Description                                | Default                          |
-| :----------------------------- | :----------------------------------------: | :------------------------------: |
-| PORT                           | the port the application will listen on    |  3000                            |
-| LOG_LEVEL                      | the log level                              |  info                            |
-| TOPIC                          | the kafka subscribe topic name             |  tc-x-events                    |
-| KAFKA_OPTIONS                   | the connection option for kafka            |  see below about KAFKA options                   |
-| MONGODB_URL  | the MongoDB URL which must be same as Topcoder x tool | mongodb://127.0.0.1:27017/topcoderx|
-
-KAFKA_OPTIONS should be object as described in https://github.com/SOHU-Co/kafka-node#kafkaclient
-For using with SSL, the options should be as
-```
- {
-    kafkaHost: '<server>',
-    sslOptions: {
-      cert: '<certificate>', 
-      key:  '<key>'
-    }
- }
-```
+See "configuration.md"
 
 `config/local.js` will not tracked by git.
 
@@ -63,8 +42,6 @@ use `ngrok` to make your local deploy accessible by internet:
 ngrok http 3002
 ```
 
-Copy the forwarding URL to set in `HOOK_BASE_URL` of topcoder-x in config.json
-
 ## Setup for verification
 Before verifying the tool, 3 service needs be configured and run them
 - processor
@@ -90,6 +67,7 @@ Now, receiver service can receive the webhooks from git host's project. Now you
 - create a pull request, you can see the logs in `receiver` and `processor`, the `pull_request.created` event is generated.
 - close a pull request without merge, you can see the logs in `receiver` and `processor`, the `pull_request.closed` event is generated and the `merged` property is `false`.
 - merge a pull request, you can see the logs in `receiver` and `processor`, the `pull_request.closed` event is generated and the `merged` property is `true`.
+- close an issue in the repo, you can see the logs in `receiver` and `processor`, the `issue.closed` event is generated
 
 ## Gitlab Verification
 
@@ -101,4 +79,5 @@ Now, receiver service can receive the webhooks from git host's project. Now you
 - add/remove a label to an issue, you can see the logs in `receiver` and `processor`, the `issue.labelUpdated` event is generated.
 - create a pull request, you can see the logs in `receiver` and `processor`, the `pull_request.created` event is generated.
 - close a pull request without merge, you can see the logs in `receiver` and `processor`, the `pull_request.closed` event is generated and the `merged` property is `false`.
-- merge a pull request, you can see the logs in `receiver` and `processor`, the `pull_request.closed` event is generated and the `merged` property is `true`.
+- merge a pull request, you can see the logs in `receiver` and `processor`, the `pull_request.closed` event is generated and the `merged` property is `true`.
+- close an issue in the repo, you can see the logs in `receiver` and `processor`, the `issue.closed` event is generated

build.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -eo pipefail
+
+# Builds Docker image of Topcoder-X Receiver application.
+# This script expects a single argument: NODE_ENV, which must be either
+# "development" or "production".
+
+NODE_ENV=$1
+
+ENV=$1
+
+source buildvar.conf
+SECRET_FILE_NAME="${APPNAME}-buildsecvar.conf"
+cp ./../buildscript/$APPNAME/$SECRET_FILE_NAME.enc .
+#ccdecrypt -f $SECRET_FILE_NAME.cpt -K $SECPASSWD
+openssl enc -aes-256-cbc -d -in $SECRET_FILE_NAME.enc -out $SECRET_FILE_NAME -k $SECPASSWD
+source $SECRET_FILE_NAME
+
+AWS_REGION=$(eval "echo \$${ENV}_AWS_REGION")
+AWS_ACCESS_KEY_ID=$(eval "echo \$${ENV}_AWS_ACCESS_KEY_ID")
+AWS_SECRET_ACCESS_KEY=$(eval "echo \$${ENV}_AWS_SECRET_ACCESS_KEY")
+AWS_ACCOUNT_ID=$(eval "echo \$${ENV}_AWS_ACCOUNT_ID")
+AWS_REPOSITORY=$(eval "echo \$${ENV}_AWS_REPOSITORY")
+
+#App variables
+KAFKA_CLIENT_CERT=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT")
+KAFKA_CLIENT_CERT_KEY=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT_KEY")
+KAFKA_HOST=$(eval "echo \$${ENV}_KAFKA_URL")
+ZOO_KEEPER=$(eval "echo \$${ENV}_ZOO_KEEPER")
+TOPIC=$(eval "echo \$${ENV}_TOPIC")
+MONGODB_URI=$(eval "echo \$${ENV}_MONGODB_URI")
+
+LOG_LEVEL=$(eval "echo \$${ENV}_LOG_LEVEL")
+NODE_ENV=$(eval "echo \$${ENV}_NODE_ENV")
+NODE_PORT=$(eval "echo \$${ENV}_NODE_PORT")
+JWKSURI=$(eval "echo \$${ENV}_JWKSURI")
+TEMPLATE_MAP=$(eval "echo \$${ENV}_TEMPLATE_MAP")
+
+TAG=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/ragnar:$CIRCLE_SHA1
+
+docker build -t $TAG .
+
+# Copies "node_modules" from the created image, if necessary for caching.
+docker create --name app $TAG
+
+if [ -d node_modules ]
+then
+  # If "node_modules" directory already exists, we should compare
+  # "package-lock.json" from the code and from the container to decide,
+  # whether we need to re-cache, and thus to copy "node_modules" from
+  # the Docker container.
+  mv package-lock.json old-package-lock.json
+  docker cp app:/usr/src/app/package-lock.json package-lock.json
+ # docker cp .env app:/usr/src/app/
+  set +eo pipefail
+  UPDATE_CACHE=$(cmp package-lock.json old-package-lock.json)
+  set -eo pipefail
+else
+  # If "node_modules" does not exist, then cache must be created.
+  UPDATE_CACHE=1
+fi
+
+if [ "$UPDATE_CACHE" == 1 ]
+then
+  docker cp app:/usr/src/app/node_modules .
+fi

config/default.js

@@ -18,8 +18,8 @@ module.exports = {
   LOG_LEVEL: process.env.LOG_LEVEL || 'info',
   TOPIC: process.env.TOPIC || 'tc-x-events',
   KAFKA_OPTIONS: {
-    kafkaHost: process.env.KAFKA_HOST || 'localhost:9092',
-    sslOptions: {
+    connectionString: process.env.KAFKA_HOST || 'localhost:9092',
+    ssl: {
       cert: process.env.KAFKA_CLIENT_CERT || fs.readFileSync('./kafka_client.cer'), // eslint-disable-line no-sync
       key: process.env.KAFKA_CLIENT_CERT_KEY || fs.readFileSync('./kafka_client.key') // eslint-disable-line no-sync
     }

configuration.md

@@ -0,0 +1,55 @@
+# Topcoder X Receiver Configuration
+The following config parameters are supported, they are defined in `config/default.js` and can be configured in env variables:
+
+
+| Name                                   | Description                                | Default                          |
+| :------------------------------------- | :----------------------------------------: | :------------------------------: |
+| PORT                                   | The port the application will listen on    |  3002                            |
+| LOG_LEVEL                              | The log level                              |  info                            |
+| MONGODB_URI                            | The MongoDB URI.  This needs to be the same MongoDB used by topcoder-x-receiver, topcoder-x-processor, and topcoder-x-site                           | mongodb://127.0.0.1:27017/topcoderx |
+|TOPIC  | The Kafka topic where events are published.  This must be the same as the configured value for topcoder-x-processor| |
+|KAFKA_OPTIONS | Kafka connection options| |
+|KAFKA_HOST | The Kafka host to connect to| localhost:9092 |
+|KAFKA_CLIENT_CERT | The Kafka SSL certificate to use when connecting| Read from kafka_client.cer file, but this can be set as a string like it is on Heroku |
+|KAFKA_CLIENT_CERT_KEY | The Kafka SSL certificate key to use when connecting| Read from kafka_client.key file, but this can be set as a string like it is on Heroku|
+
+KAFKA_OPTIONS should be object as described in https://github.com/oleksiyk/kafka#ssl
+For using with SSL, the options should be as
+```
+ {
+    connectionString: '<server>',
+    ssl: {
+      cert: '<certificate>', 
+      key:  '<key>'
+    }
+ }
+```
+
+## Endpoints
+
+- POST /webhooks/github - The webhook handler for github
+- POST /webhooks/gitlab - The webhook handler for gitlab
+
+## Github Verification
+
+#### Webhook configuration
+
+Configure a Github project with a webhook with a format like this: https://<receiver URL>:<receiver port>/webhooks/github
+
+#### Smoke test
+- Create an issue in the repo, you can see the logs in `receiver`, the `issue.created` event is generated.
+
+You can test other events, but just validating that an issue.created event is generated in Kafka is enough to smoke test the receiver is set up properly.  
+
+## Gitlab Verification
+
+#### Webhook configuration
+
+Configure a Gitlab project with a webhook with a format like this: https://<receiver URL>:<receiver port>/webhooks/gitlab
+
+#### Smoke test
+
+See above - the steps are the same for Github and Gitlab
+
+## Debugging
+You can re-run and debug the responses to webhook requests on Github and Gitlab, in the configuration for the webhook.  This can be useful if things aren't coming through properly in the receiver.

models/IssueClosedEvent.js

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2018 TopCoder, Inc. All rights reserved.
+ */
+/**
+ * This module contains the schema of the issue closed schema.
+ *
+ * @author veshu
+ * @version 1.0
+ */
+'use strict';
+const Joi = require('joi');
+const {issueSchema, repositorySchema} = require('./common');
+
+const IssueClosedEvent = {
+  name: 'issue.closed',
+
+  schema: Joi.object().keys({
+    issue: issueSchema.required(),
+    repository: repositorySchema.required(),
+    assignee: Joi.object().keys({
+      id: Joi.number().allow(null)
+    }).required()
+  })
+};
+
+
+module.exports = IssueClosedEvent;
+

models/Project.js

@@ -14,15 +14,14 @@ const schema = new mongoose.Schema({
   title: {type: String, required: true},
   tcDirectId: {type: Number, required: true},
   repoUrl: {type: String, required: true},
-  rocketChatWebhook: {type: String, required: true},
-  rocketChatChannelName: {type: String, required: true},
+  rocketChatWebhook: {type: String, required: false},
+  rocketChatChannelName: {type: String, required: false},
   archived: {type: String, required: true},
   username: {type: String, required: true},
   secretWebhookKey: {type: String, required: true}
 });
 
-// project id, provider, repositoryId must be unique
-schema.index({tcDirectId: 1}, {unique: true});
+schema.index({tcDirectId: 1});
 
 
 module.exports = schema;

models/index.js

@@ -17,6 +17,7 @@ mongoose.Promise = global.Promise;
 const connection = mongoose.createConnection(config.MONGODB_URL);
 const IssueCreatedEvent = require('./IssueCreatedEvent');
 const IssueUpdatedEvent = require('./IssueUpdatedEvent');
+const IssueClosedEvent = require('./IssueClosedEvent');
 const CommentCreatedEvent = require('./CommentCreatedEvent');
 const CommentUpdatedEvent = require('./CommentUpdatedEvent');
 const UserAssignedEvent = require('./UserAssignedEvent');
@@ -36,5 +37,6 @@ module.exports = {
   PullRequestCreatedEvent,
   PullRequestClosedEvent,
   LabelUpdatedEvent,
-  Project: connection.model('Project', Project)
+  Project: connection.model('Project', Project),
+  IssueClosedEvent
 };