Merge remote-tracking branch 'origin/develop'

Author: jmgasper

.circleci/config.yml

@@ -0,0 +1,75 @@
+version: 2
+defaults: &defaults
+  docker:
+    - image: circleci/python:2.7-stretch-browsers
+install_dependency: &install_dependency
+          name: Installation of build and deployment dependencies.
+          command: |
+            sudo apt install jq
+            sudo pip install awscli --upgrade
+
+install_deploysuite: &install_deploysuite
+          name: Installation of install_deploysuite.
+          command: |
+            git clone --branch v1.4.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
+            cp ./../buildscript/master_deploy.sh .
+            cp ./../buildscript/buildenv.sh .
+            cp ./../buildscript/awsconfiguration.sh .
+restore_cache_settings_for_build: &restore_cache_settings_for_build
+  key: docker-node-modules-la-{{ checksum "package-lock.json" }}
+
+save_cache_settings: &save_cache_settings
+  key: docker-node-modules-la-{{ checksum "package-lock.json" }}
+  paths:
+    - node_modules
+
+builddeploy_steps: &builddeploy_steps
+      - checkout
+      - setup_remote_docker
+      - run: *install_dependency
+      - run: *install_deploysuite
+      - restore_cache: *restore_cache_settings_for_build
+      - run: ./builddocker.sh ${APPNAME}
+      - save_cache: *save_cache_settings
+      - deploy:
+          name: Running MasterScript.
+          command: |
+            ./awsconfiguration.sh $DEPLOY_ENV
+            source awsenvconf
+            ./buildenv.sh -e $DEPLOY_ENV -b ${DEPLOY_ENV}-${APPNAME}-deployvar
+            source buildenvvar
+            ./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${DEPLOY_ENV}-global-appvar,${DEPLOY_ENV}-${APPNAME}-appvar -i ${APPNAME}
+jobs:
+  # Build & Deploy against development backend
+  "build-dev":
+    <<: *defaults
+    environment:
+      DEPLOY_ENV: "dev"
+      APPNAME: "topcoder-x-receiver"
+    steps: *builddeploy_steps
+
+  # Build & Deploy against production backend
+  "build-prod":
+    <<: *defaults
+    environment:
+      DEPLOY_ENV: "prod"
+      APPNAME: "topcoder-x-receiver"
+    steps: *builddeploy_steps
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      # Development builds are executed on "develop" branch only.
+      - "build-dev":
+          context : org-global
+          filters:
+            branches:
+              only:
+                - develop
+      - "build-prod":
+          context : org-global
+          filters:
+            branches:
+              only:
+                - master

.gitignore

@@ -31,3 +31,5 @@ npm-debug.log
 
 /config/local.js
 .DS_Store
+
+package-lock.json

app.js

@@ -9,15 +9,27 @@
  */
 'use strict';
 const express = require('express');
-const logger = require('morgan');
 const cookieParser = require('cookie-parser');
 const bodyParser = require('body-parser');
+const healthcheck = require('topcoder-healthcheck-dropin');
+const logger = require('./utils/logger');
+
+/**
+ * Method to check the service status
+ * @returns {Object} The returned status
+ */
+function check() {
+  // No checks to run. The output of this itself is an indication that the app is actively running
+  return {
+    checksRun: 1
+  };
+}
 
 const webhooks = require('./routes/webhooks');
 
 const app = express();
 
-app.use(logger('dev'));
+// app.use(logger('dev'));
 app.use(bodyParser.json({
   verify: (req, res, buf) => {
     req.rawBody = buf.toString();
@@ -26,6 +38,8 @@ app.use(bodyParser.json({
 app.use(bodyParser.urlencoded({extended: false}));
 app.use(cookieParser());
 
+app.use(healthcheck.middleware([check]));
+
 app.use('/webhooks', webhooks);
 
 // catch 404 and forward to error handler
@@ -35,7 +49,7 @@ app.use((req, res, next) => {
   next(err);
 });
 
-// error handler
+// // error handler
 app.use((err, req, res, next) => { // eslint-disable-line no-unused-vars
   console.log(err);
   res.status(err.status || 500); // eslint-disable-line no-magic-numbers
@@ -45,4 +59,17 @@ app.use((err, req, res, next) => { // eslint-disable-line no-unused-vars
   });
 });
 
+process.on('uncaughtException', (err) => {
+  // Check if error related to Dynamodb conn
+  if (err.code === 'NetworkingError' && err.region) {
+    logger.error('DynamoDB connection failed.');
+  }
+  logger.logFullError(err, 'system');
+});
+
+// handle and log unhanled rejection
+process.on('unhandledRejection', (err) => {
+  logger.logFullError(err, 'system');
+});
+
 module.exports = app;

build.sh

@@ -25,7 +25,7 @@ AWS_REPOSITORY=$(eval "echo \$${ENV}_AWS_REPOSITORY")
 #App variables
 KAFKA_CLIENT_CERT=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT")
 KAFKA_CLIENT_CERT_KEY=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT_KEY")
-KAFKA_HOST=$(eval "echo \$${ENV}_KAFKA_URL")
+KAFKA_URL=$(eval "echo \$${ENV}_KAFKA_URL")
 ZOO_KEEPER=$(eval "echo \$${ENV}_ZOO_KEEPER")
 TOPIC=$(eval "echo \$${ENV}_TOPIC")
 MONGODB_URI=$(eval "echo \$${ENV}_MONGODB_URI")

builddocker.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -eo pipefail
+APP_NAME=$1
+UPDATE_CACHE=""
+#echo "" > docker/api.env
+#docker-compose -f docker/docker-compose.yml build $APP_NAME
+docker build -t $APP_NAME:latest .
+docker create --name app $APP_NAME:latest
+
+if [ -d node_modules ]
+then
+  mv package-lock.json old-package-lock.json
+  docker cp app:/usr/src/app/package-lock.json package-lock.json
+  set +eo pipefail
+  UPDATE_CACHE=$(cmp package-lock.json old-package-lock.json)
+  set -eo pipefail
+else
+  UPDATE_CACHE=1
+fi
+
+if [ "$UPDATE_CACHE" == 1 ]
+then
+  docker cp app:/usr/src/app/node_modules .
+fi

config/default.js

@@ -14,15 +14,20 @@ const fs = require('fs');
  */
 
 module.exports = {
-  PORT: process.env.PORT || 3002, // eslint-disable-line no-magic-numbers
+  PORT: process.env.PORT || 3000, // eslint-disable-line no-magic-numbers
   LOG_LEVEL: process.env.LOG_LEVEL || 'info',
   TOPIC: process.env.TOPIC || 'tc-x-events',
   KAFKA_OPTIONS: {
-    connectionString: process.env.KAFKA_HOST || 'localhost:9092',
+    connectionString: process.env.KAFKA_URL || 'localhost:9092',
     ssl: {
       cert: process.env.KAFKA_CLIENT_CERT || fs.readFileSync('./kafka_client.cer'), // eslint-disable-line no-sync
       key: process.env.KAFKA_CLIENT_CERT_KEY || fs.readFileSync('./kafka_client.key') // eslint-disable-line no-sync
     }
   },
-  MONGODB_URL: process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/topcoderx'
+  DYNAMODB: {
+    AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID || '',
+    AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY || '',
+    AWS_REGION: process.env.AWS_REGION || '',
+    IS_LOCAL: process.env.IS_LOCAL || 'false'
+  }
 };

configuration.md

@@ -6,20 +6,23 @@ The following config parameters are supported, they are defined in `config/defau
 | :------------------------------------- | :----------------------------------------: | :------------------------------: |
 | PORT                                   | The port the application will listen on    |  3002                            |
 | LOG_LEVEL                              | The log level                              |  info                            |
-| MONGODB_URI                            | The MongoDB URI.  This needs to be the same MongoDB used by topcoder-x-receiver, topcoder-x-processor, and topcoder-x-site                           | mongodb://127.0.0.1:27017/topcoderx |
 |TOPIC  | The Kafka topic where events are published.  This must be the same as the configured value for topcoder-x-processor| |
 |KAFKA_OPTIONS | Kafka connection options| |
-|KAFKA_HOST | The Kafka host to connect to| localhost:9092 |
+|KAFKA_URL | The Kafka host to connect to| localhost:9092 |
 |KAFKA_CLIENT_CERT | The Kafka SSL certificate to use when connecting| Read from kafka_client.cer file, but this can be set as a string like it is on Heroku |
 |KAFKA_CLIENT_CERT_KEY | The Kafka SSL certificate key to use when connecting| Read from kafka_client.key file, but this can be set as a string like it is on Heroku|
+|AWS_ACCESS_KEY_ID | The Amazon certificate key to use when connecting. Use local dynamodb you can set fake value|FAKE_ACCESS_KEY_ID |
+|AWS_SECRET_ACCESS_KEY | The Amazon certificate access key to use when connecting. Use local dynamodb you can set fake value|FAKE_SECRET_ACCESS_KEY |
+|AWS_REGION | The Amazon certificate region to use when connecting. Use local dynamodb you can set fake value|FAKE_REGION |
+|IS_LOCAL | Use Amazon DynamoDB Local or server | 'false' |
 
 KAFKA_OPTIONS should be object as described in https://github.com/oleksiyk/kafka#ssl
 For using with SSL, the options should be as
 ```
  {
     connectionString: '<server>',
     ssl: {
-      cert: '<certificate>', 
+      cert: '<certificate>',
       key:  '<key>'
     }
  }
@@ -39,7 +42,7 @@ Configure a Github project with a webhook with a format like this: https://<rece
 #### Smoke test
 - Create an issue in the repo, you can see the logs in `receiver`, the `issue.created` event is generated.
 
-You can test other events, but just validating that an issue.created event is generated in Kafka is enough to smoke test the receiver is set up properly.  
+You can test other events, but just validating that an issue.created event is generated in Kafka is enough to smoke test the receiver is set up properly.
 
 ## Gitlab Verification
 

models/Project.js

@@ -8,20 +8,33 @@
  * @author TCSCODER
  * @version 1.0
  */
-const mongoose = require('mongoose');
+const dynamoose = require('dynamoose');
 
-const schema = new mongoose.Schema({
+const Schema = dynamoose.Schema;
+
+const schema = new Schema({
+  id: {
+    type: String,
+    hashKey: true,
+    required: true
+  },
   title: {type: String, required: true},
-  tcDirectId: {type: Number, required: true},
+  tcDirectId: {
+    type: Number,
+    required: true,
+    index: {
+      global: true,
+      rangeKey: 'id',
+      project: true,
+      name: 'TcDirectIdIndex'
+    }
+  },
   repoUrl: {type: String, required: true},
   rocketChatWebhook: {type: String, required: false},
   rocketChatChannelName: {type: String, required: false},
   archived: {type: String, required: true},
-  username: {type: String, required: true},
+  owner: {type: String, required: true},
   secretWebhookKey: {type: String, required: true}
 });
 
-schema.index({tcDirectId: 1});
-
-
 module.exports = schema;

models/index.js

@@ -11,10 +11,23 @@
 'use strict';
 
 const config = require('config');
-const mongoose = require('mongoose');
+const dynamoose = require('dynamoose');
+
+dynamoose.AWS.config.update({
+  accessKeyId: config.DYNAMODB.AWS_ACCESS_KEY_ID,
+  secretAccessKey: config.DYNAMODB.AWS_SECRET_ACCESS_KEY,
+  region: config.DYNAMODB.AWS_REGION
+});
+
+if (config.DYNAMODB.IS_LOCAL === 'true') {
+  dynamoose.local();
+}
+
+dynamoose.setDefaults({
+  create: false,
+  update: false
+});
 
-mongoose.Promise = global.Promise;
-const connection = mongoose.createConnection(config.MONGODB_URL);
 const IssueCreatedEvent = require('./IssueCreatedEvent');
 const IssueUpdatedEvent = require('./IssueUpdatedEvent');
 const IssueClosedEvent = require('./IssueClosedEvent');
@@ -37,6 +50,6 @@ module.exports = {
   PullRequestCreatedEvent,
   PullRequestClosedEvent,
   LabelUpdatedEvent,
-  Project: connection.model('Project', Project),
+  Project: dynamoose.model('Topcoder_X.Project', Project),
   IssueClosedEvent
 };