Cleanup

Author: jmgasper

scripts/mongodb-migration/README.MD

@@ -0,0 +1,37 @@
+# Migrate mongodb to dynamodb
+The following config parameters are supported, they are defined in `config/default.js` and can be configured in env variables:
+
+
+| Name                                   | Description                                | Default                          |
+| :------------------------------------- | :----------------------------------------: | :------------------------------: |
+| LOG_LEVEL                              | The log level                              |  info                            |
+| MONGODB_URL                            | The MongoDB URL.  This needs to be the same MongoDB used by topcoder-x-receiver, topcoder-x-processor, and topcoder-x-site                           | mongodb://127.0.0.1:27017/topcoderx |
+|COLLECTION_COUNTS | The counts of testing data for each model| |
+|AWS_ACCESS_KEY_ID | The Amazon certificate key to use when connecting. Use local dynamodb you can set fake value |FAKE_ACCESS_KEY_ID |
+|AWS_SECRET_ACCESS_KEY | The Amazon certificate access key to use when connecting. Use local dynamodb you can set fake value |FAKE_SECRET_ACCESS_KEY |
+|AWS_REGION | The Amazon certificate region to use when connecting. Use local dynamodb you can set fake value |FAKE_REGION |
+|IS_LOCAL | Use Amazon DynamoDB Local or server | 'true' |
+| MIGRATION_DELAY_TIME | The delay used to avoid throughput errors from DynamoDB | 5000 |
+| MIGRATION_DELAY_INTERVAL | The number of requests made before we delay the next call to DynamoDB | 50 |
+
+## Scripts
+
+migrate mongodb data to dynamodb
+```
+npm run migrate-data
+```
+
+create empty tables to dynamodb
+```
+npm run create-empty-tables
+```
+
+## Test Data
+
+**NOTE** This is a *destructive* call.  It will overwrite existing data in mongodb.  Do not do this against production
+
+Create testing data to mongodb, this command will generate bulk fake data to testing migration.
+You also can create regular data by opcoder-x-receiver/topcoder-x-processor/topcoder-x-site
+```
+npm run create-test-data
+```

scripts/mongodb-migration/config/default.js

@@ -0,0 +1,25 @@
+'use strict';
+
+/*
+ * Copyright (c) 2018 TopCoder, Inc. All rights reserved.
+ */
+
+/**
+ * This module contains the configurations of the app.
+ * Changes in 1.1:
+ * @author TCSCODER
+ * @version 1.1
+ */
+
+module.exports = {
+  MONGODB_URL: process.env.MONGODB_URL || ' ',
+  COLLECTION_COUNTS: process.env.COLLECTION_COUNTS || 100,
+  DYNAMODB: {
+    AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID || ' ',
+    AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY || ' ',
+    AWS_REGION: process.env.AWS_REGION || 'us-east-1',
+    IS_LOCAL: process.env.IS_LOCAL || 'false',
+  },
+  MIGRATION_DELAY_TIME: 5 * 1000, // 5 sec
+  MIGRATION_DELAY_INTERVAL: 50
+};

scripts/mongodb-migration/constants.js

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2017 TopCoder, Inc. All rights reserved.
+ */
+
+/**
+ * Define constants.
+ *
+ * @author TCSCODER
+ * @version 1.0
+ */
+
+
+// The user roles
+const USER_ROLES = {
+  OWNER: 'owner',
+};
+
+// The user types
+const USER_TYPES = {
+  GITHUB: 'github',
+  GITLAB: 'gitlab',
+};
+
+
+module.exports = {
+  USER_ROLES,
+  USER_TYPES,
+};

scripts/mongodb-migration/create-empty-tables.js

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2018 TopCoder, Inc. All rights reserved.
+ */
+
+/**
+ * Create empty tables for DynamoDB
+ *
+ * @author TCSCODER
+ * @version 1.0
+ */
+
+const Path = require('path');
+const fs = require('fs');
+const dynamoose = require('dynamoose');
+const config = require('config');
+const logger = require('./utils/logger');
+const schemaPath = './dynamodb/models';
+
+dynamoose.AWS.config.update({
+  accessKeyId: config.DYNAMODB.AWS_ACCESS_KEY_ID,
+  secretAccessKey: config.DYNAMODB.AWS_SECRET_ACCESS_KEY,
+  region: config.DYNAMODB.AWS_REGION,
+});
+
+if (config.DYNAMODB.IS_LOCAL === 'true') {
+  dynamoose.local();
+}
+
+dynamoose.setDefaults({
+  create: true,
+  update: true,
+});
+
+const Table = dynamoose.Table;
+
+fs.readdirSync(schemaPath).forEach((file) => { // eslint-disable-line no-sync
+  if (file !== 'index.js') {
+    const filename = file.split('.')[0];
+    const fileFullPath = Path.join(__dirname + schemaPath.replace('.', ''), filename);
+    const schema = require(fileFullPath); // eslint-disable-line global-require
+    const table_name = 'Topcoder_X.' + filename
+    const table = new Table(table_name, schema, null, dynamoose);
+    table.create((err) => {
+      if(!err) {
+        logger.info(`*** Table ${table_name} has been created ***`);
+      } else {
+        // if table exists, delete and re-create with empty
+        table.delete((err) => {
+          if (!err) {
+            table.create((err) => {
+              if (!err) {
+                logger.info(`*** Table ${table_name} has been created ***`);
+              } else
+              {
+                logger.info(`*** Table ${table_name} created failed -- ${err.message} ***`);
+              }
+            });
+          } else {
+            logger.info(`*** Delete exist Table ${table_name} failed -- ${err.message} ***`);
+          }
+        })
+      }
+    });
+  }
+});

scripts/mongodb-migration/create-test-data.js

@@ -0,0 +1,194 @@
+/*
+ * Copyright (c) 2018 TopCoder, Inc. All rights reserved.
+ */
+
+/**
+ * This create test data
+ *
+ * @author TCSCODER
+ * @version 1.0
+ */
+
+
+const config = require('config');
+const models = require('./mongodb/models');
+const logger = require('./utils/logger');
+const helper = require('./utils/helper');
+
+const projectIds = [];
+
+/**
+ * Create user data
+ */
+async function createBulkUserData() {
+  logger.info('*** create user testing data ***');
+  for (let i = 0; i < config.COLLECTION_COUNTS; i += 1) {
+    const data = {
+      role: 'owner',
+      type: i % 2 === 0 ? 'github' : 'gitlab',
+      userProviderId:i + 1,
+      accessToken: helper.generateIdentifier(),
+      accessTokenExpiration: new Date(),
+      refreshToken: helper.generateIdentifier(),
+      username: `username#${i+1}`,
+    };
+
+    const user = new models.User(data);
+    await user.save();
+  }
+
+}
+
+/**
+ * Create usermapping data
+ */
+async function createBulkUserMappingData() {
+  logger.info('*** create usermapping testing data ***');
+  for (let i = 0; i < config.COLLECTION_COUNTS; i += 1) {
+    const data = {
+      topcoderUsername: `topcoderUsername${i+1}`,
+      githubUsername: `githubUsername${i+1}`,
+      gitlabUsername: `gitlabUsername${i+1}`,
+      githubUserId: i + 1,
+      gitlabUserId: i + 1,
+    };
+
+    const usermapping = new models.UserMapping(data);
+    await usermapping.save();
+  }
+}
+
+/**
+ * Create project data
+ */
+async function createBulkProjectData() {
+  logger.info('*** create project testing data ***');
+  for (let i = 0; i < config.COLLECTION_COUNTS; i += 1) {
+    const data = {
+      title: `title#${i}`,
+      tcDirectId: i + 1,
+      repoUrl: `http://repoUrl${i+1}.com`,
+      archived: i % 2 === 0 ?'false' : 'true',
+      owner: `username#${i+1}`,
+      secretWebhookKey: helper.generateIdentifier(),
+      copilot: `username#${i+1}`,
+    };
+
+    const project = new models.Project(data);
+    await project.save();
+    projectIds.push(project.id);
+  }
+}
+
+/**
+ * Create issue data
+ */
+async function createBulkIssueData() {
+  logger.info('*** create issue testing data ***');
+  for (let i = 0; i < config.COLLECTION_COUNTS; i += 1) {
+    const data = {
+      number: i + 1,
+      title: `issue title #${i}`,
+      body: `This #${i} issue body`,
+      prizes: [i+1],
+      repositoryId: i + 1,
+      labels: ['tcx_Assigned', 'tcx_ReadyForReview'],
+      assignee: `assignee#${i}`,
+      challengeId: i + 1,
+      projectId: projectIds[i % 2 + 1],
+      assignedAt: new Date(),
+      provider: i % 2 === 0 ? 'github' : 'gitlab',
+    };
+
+    const issue = new models.Issue(data);
+    await issue.save();
+  }
+}
+
+/**
+ * Create copilotpayment data
+ */
+async function createBulkCopilotPaymentData() {
+  logger.info('*** create copilot payment testing data ***');
+  for (let i = 0; i < config.COLLECTION_COUNTS; i += 1) {
+    const data = {
+      project: projectIds[i % 2 + 1],
+      amount: (i + 1) * 20,
+      description: `payment description #${i}`,
+      challengeId: i + 1,
+      closed: i % 2 === 0 ? 'false' : 'true',
+      username: `username#${i+1}`,
+      status: 'challenge_create_successful',
+    };
+
+    const payment = new models.CopilotPayment(data);
+    await payment.save();
+  }
+}
+
+/**
+ * Create owner user team data
+ */
+async function createBulkOwnerUserTeamData() {
+  logger.info('*** create OwnerUserTeam testing data ***');
+  for (let i = 0; i < config.COLLECTION_COUNTS; i += 1) {
+    const data = {
+      ownerUsername: `owner username #${i+1}`,
+      type: i % 2 === 0 ? 'gitlab' : 'github',
+      teamId: `teamId#${i+1}`,
+      ownerToken: helper.generateIdentifier(),
+      identifier: helper.generateIdentifier(),
+    };
+
+    const team = new models.OwnerUserTeam(data);
+    await team.save();
+  }
+}
+
+/**
+ * Create owner user group data
+ */
+async function createBulkOwnerUserGroupData() {
+  logger.info('*** create OwnerUserGroup testing data ***');
+  for (let i = 0; i < config.COLLECTION_COUNTS; i += 1) {
+    const data = {
+      ownerUsername: `owner username #${i+1}`,
+      type: i % 2 === 0 ? 'gitlab' : 'github',
+      groupId: `groupId#${i+1}`,
+      identifier: helper.generateIdentifier(),
+    };
+
+    const group = new models.OwnerUserGroup(data);
+    await group.save();
+  }
+}
+
+/**
+ * Clean mongoDB database
+ */
+async function cleanup() {
+  await models.OwnerUserGroup.remove({});
+  await models.OwnerUserTeam.remove({});
+  await models.CopilotPayment.remove({});
+  await models.Issue.remove({});
+  await models.Project.remove({});
+  await models.UserMapping.remove({});
+  await models.User.remove({});
+}
+
+async function start() {
+  await cleanup();
+  await createBulkUserData();
+  await createBulkUserMappingData();
+  await createBulkProjectData();
+  await createBulkIssueData();
+  await createBulkCopilotPaymentData();
+  await createBulkOwnerUserTeamData();
+  await createBulkOwnerUserGroupData();
+
+  logger.info('create testing data finish!');
+  process.exit(0);
+}
+
+
+start();