DynamoDB updates

Author: jmgasper

config/default.js

@@ -24,5 +24,10 @@ module.exports = {
       key: process.env.KAFKA_CLIENT_CERT_KEY || fs.readFileSync('./kafka_client.key') // eslint-disable-line no-sync
     }
   },
-  MONGODB_URL: process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/topcoderx'
+  DYNAMODB: {
+    AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID || '',
+    AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY || '',
+    AWS_REGION: process.env.AWS_REGION || '',
+    IS_LOCAL: process.env.IS_LOCAL || ''
+  }
 };

configuration.md

@@ -6,20 +6,23 @@ The following config parameters are supported, they are defined in `config/defau
 | :------------------------------------- | :----------------------------------------: | :------------------------------: |
 | PORT                                   | The port the application will listen on    |  3002                            |
 | LOG_LEVEL                              | The log level                              |  info                            |
-| MONGODB_URI                            | The MongoDB URI.  This needs to be the same MongoDB used by topcoder-x-receiver, topcoder-x-processor, and topcoder-x-site                           | mongodb://127.0.0.1:27017/topcoderx |
 |TOPIC  | The Kafka topic where events are published.  This must be the same as the configured value for topcoder-x-processor| |
 |KAFKA_OPTIONS | Kafka connection options| |
 |KAFKA_HOST | The Kafka host to connect to| localhost:9092 |
 |KAFKA_CLIENT_CERT | The Kafka SSL certificate to use when connecting| Read from kafka_client.cer file, but this can be set as a string like it is on Heroku |
 |KAFKA_CLIENT_CERT_KEY | The Kafka SSL certificate key to use when connecting| Read from kafka_client.key file, but this can be set as a string like it is on Heroku|
+|AWS_ACCESS_KEY_ID | The Amazon certificate key to use when connecting. Use local dynamodb you can set fake value|FAKE_ACCESS_KEY_ID |
+|AWS_SECRET_ACCESS_KEY | The Amazon certificate access key to use when connecting. Use local dynamodb you can set fake value|FAKE_SECRET_ACCESS_KEY |
+|AWS_REGION | The Amazon certificate region to use when connecting. Use local dynamodb you can set fake value|FAKE_REGION |
+|IS_LOCAL | Use Amazon DynamoDB Local or server |true|
 
 KAFKA_OPTIONS should be object as described in https://github.com/oleksiyk/kafka#ssl
 For using with SSL, the options should be as
 ```
  {
     connectionString: '<server>',
     ssl: {
-      cert: '<certificate>', 
+      cert: '<certificate>',
       key:  '<key>'
     }
  }
@@ -39,7 +42,7 @@ Configure a Github project with a webhook with a format like this: https://<rece
 #### Smoke test
 - Create an issue in the repo, you can see the logs in `receiver`, the `issue.created` event is generated.
 
-You can test other events, but just validating that an issue.created event is generated in Kafka is enough to smoke test the receiver is set up properly.  
+You can test other events, but just validating that an issue.created event is generated in Kafka is enough to smoke test the receiver is set up properly.
 
 ## Gitlab Verification
 

models/Project.js

@@ -8,20 +8,33 @@
  * @author TCSCODER
  * @version 1.0
  */
-const mongoose = require('mongoose');
+const dynamoose = require('dynamoose');
 
-const schema = new mongoose.Schema({
+const Schema = dynamoose.Schema;
+
+const schema = new Schema({
+  id: {
+    type: String,
+    hashKey: true,
+    required: true
+  },
   title: {type: String, required: true},
-  tcDirectId: {type: Number, required: true},
+  tcDirectId: {
+    type: Number,
+    required: true,
+    index: {
+      global: true,
+      rangeKey: 'id',
+      project: true,
+      name: 'TcDirectIdIndex'
+    }
+  },
   repoUrl: {type: String, required: true},
   rocketChatWebhook: {type: String, required: false},
   rocketChatChannelName: {type: String, required: false},
   archived: {type: String, required: true},
-  username: {type: String, required: true},
+  owner: {type: String, required: true},
   secretWebhookKey: {type: String, required: true}
 });
 
-schema.index({tcDirectId: 1});
-
-
 module.exports = schema;

models/index.js

@@ -11,10 +11,23 @@
 'use strict';
 
 const config = require('config');
-const mongoose = require('mongoose');
+const dynamoose = require('dynamoose');
+
+dynamoose.AWS.config.update({
+  accessKeyId: config.DYNAMODB.AWS_ACCESS_KEY_ID,
+  secretAccessKey: config.DYNAMODB.AWS_SECRET_ACCESS_KEY,
+  region: config.DYNAMODB.AWS_REGION
+});
+
+if (config.DYNAMODB.IS_LOCAL) {
+  dynamoose.local();
+}
+
+dynamoose.setDefaults({
+  create: false,
+  update: false
+});
 
-mongoose.Promise = global.Promise;
-const connection = mongoose.createConnection(config.MONGODB_URL);
 const IssueCreatedEvent = require('./IssueCreatedEvent');
 const IssueUpdatedEvent = require('./IssueUpdatedEvent');
 const IssueClosedEvent = require('./IssueClosedEvent');
@@ -37,6 +50,6 @@ module.exports = {
   PullRequestCreatedEvent,
   PullRequestClosedEvent,
   LabelUpdatedEvent,
-  Project: connection.model('Project', Project),
+  Project: dynamoose.model('Project', Project),
   IssueClosedEvent
 };

package.json

@@ -14,9 +14,9 @@
     "config": "^1.30.0",
     "cookie-parser": "~1.4.3",
     "debug": "^2.6.9",
+    "dynamoose": "^1.1.0",
     "express": "^4.15.5",
     "joi": "^11.4.0",
-    "mongoose": "^4.13.14",
     "morgan": "~1.8.1",
     "no-kafka": "^3.2.10",
     "winston": "^2.4.2"

routes/middlewares/RepositoryFilter.js

@@ -12,6 +12,7 @@ const _ = require('lodash');
 
 const logger = require('../../utils/logger');
 const Project = require('../../models').Project;
+const dbHelper = require('../../utils/db-helper');
 
 module.exports = (provider) => async (req, res, next) => {
   let repoNames = [];
@@ -22,9 +23,10 @@ module.exports = (provider) => async (req, res, next) => {
     const repo = req.body.project || {};
     repoNames = [repo.homepage, repo.http_url, repo.url, repo.ssh_url, repo.web_url];
   }
-  let found = false;
-  const projects = await Project.find({archived: false});
-  found = _.some(projects, (project) => _.includes(repoNames, project.repoUrl));
+  const projects = await dbHelper.scan(Project, {
+    archived: 'false'
+  });
+  const found = _.some(projects, (project) => _.includes(repoNames, project.repoUrl));
   if (found) {
     return next();
   }

routes/middlewares/SecurityChecker.js

@@ -11,16 +11,22 @@
 const crypto = require('crypto');
 const logger = require('../../utils/logger');
 const Project = require('../../models').Project;
+const dbHelper = require('../../utils/db-helper');
 
 module.exports = (provider) => async (req, res, next) => {
   let isValid = false;
   const params = req.body;
   if (provider === 'github') {
-    const projectDetail = await Project.findOne({repoUrl: params.repository.html_url});
+    const projectDetail = await dbHelper.scanOne(Project, {
+      repoUrl: params.repository.html_url
+    });
+
     const hash = crypto.createHmac('sha1', projectDetail.secretWebhookKey).update(req.rawBody).digest('hex');
     isValid = `sha1=${hash}` === req.header('X-Hub-Signature');
   } else if (provider === 'gitlab') {
-    const projectDetail = await Project.findOne({repoUrl: params.project.web_url});
+    const projectDetail = await dbHelper.scanOne(Project, {
+      repoUrl: params.project.web_url
+    });
     isValid = projectDetail.secretWebhookKey === req.header('X-Gitlab-Token');
   } else {
     // unknown provider

utils/db-helper.js

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2018 TopCoder, Inc. All rights reserved.
+ */
+'use strict';
+
+/**
+ * This module contains the database helper methods.
+ *
+ * @version 1.0
+ */
+
+/**
+ * Get data collection by scan parameters
+ * @param {Object} model The dynamoose model to scan
+ * @param {Object} scanParams The scan parameters object
+ * @returns {Promise<void>}
+ */
+async function scan(model, scanParams) {
+  return await new Promise((resolve, reject) => {
+    model.scan(scanParams).exec((err, result) => {
+      if (err) {
+        reject(err);
+      }
+      return resolve(result);
+    });
+  });
+}
+
+/**
+ * Get single data by scan parameters
+ * @param {Object} model The dynamoose model to scan
+ * @param {Object} scanParams The scan parameters object
+ * @returns {Promise<void>}
+ */
+async function scanOne(model, scanParams) {
+  return await new Promise((resolve, reject) => {
+    model.scan(scanParams).exec((err, result) => {
+      if (err) {
+        reject(err);
+      }
+
+      return resolve(result[0]);
+    });
+  });
+}
+
+module.exports = {
+  scan,
+  scanOne
+};