Issue 179 (#184)

#179 Update data type for challengeId in the index, by creating a new index

Author: callmekatootie

README.md

@@ -116,6 +116,10 @@ This script will load the data from `scripts/data` directory into ES
 npm run start
 ```
 
+#### Duplicating the ES Index
+
+To duplicate the existing ES Index (from the `ES_INDEX` to `ES_INDEX_V2` based on the configs in `config/default.js`) run `npm run create-new-index`
+
 #### Linting JS files
 
 ```

config/default.js

@@ -28,9 +28,10 @@ module.exports = {
   AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
   AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
   esConfig: {
-    HOST: process.env.ES_HOST,
+    HOST: process.env.ES_HOST || 'localhost:9200',
     API_VERSION: process.env.ES_API_VERSION || '6.3',
     ES_INDEX: process.env.ES_INDEX || 'submission',
+    ES_INDEX_V2: process.env.ES_INDEX_V2 || 'new_submission',
     ES_TYPE: process.env.ES_TYPE || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it
   },
   PAGE_SIZE: process.env.PAGE_SIZE || 20,

package.json

@@ -11,6 +11,7 @@
     "create-tables": "node scripts/createTables.js",
     "init-db": "node scripts/importData.js",
     "create-index": "node scripts/createIndex.js",
+    "create-new-index": "node scripts/createNewIndex.js",
     "delete-index": "node scripts/deleteIndex.js",
     "init-es": "node scripts/loadES.js",
     "db-to-es": "node scripts/migrateFromDBToES.js",

scripts/ESloadHelper.js

@@ -20,7 +20,7 @@ const esClient = helper.getEsClient()
 function deleteDatafromES () {
   logger.info('Clear data from ES if any')
   const filter = {
-    index: config.get('esConfig.ES_INDEX'),
+    index: config.get('esConfig.ES_INDEX_V2'),
     type: config.get('esConfig.ES_TYPE'),
     q: '*'
   }
@@ -34,7 +34,7 @@ function * loadReviewTypes () {
   const promises = []
   reviewTypes.forEach((reviewType) => {
     const record = {
-      index: config.get('esConfig.ES_INDEX'),
+      index: config.get('esConfig.ES_INDEX_V2'),
       type: config.get('esConfig.ES_TYPE'),
       id: reviewType.id,
       body: _.extend({ resource: 'reviewType' }, reviewType)
@@ -51,7 +51,7 @@ function * loadSubmissions () {
   const promises = []
   submissions.forEach((submission) => {
     const record = {
-      index: config.get('esConfig.ES_INDEX'),
+      index: config.get('esConfig.ES_INDEX_V2'),
       type: config.get('esConfig.ES_TYPE'),
       id: submission.id,
       body: _.extend({ resource: 'submission' }, submission)
@@ -68,7 +68,7 @@ function * loadReviews () {
   const promises = []
   reviews.forEach((review) => {
     const record = {
-      index: config.get('esConfig.ES_INDEX'),
+      index: config.get('esConfig.ES_INDEX_V2'),
       type: config.get('esConfig.ES_TYPE'),
       id: review.id,
       body: _.extend({ resource: 'review' }, review)
@@ -85,7 +85,7 @@ function * loadReviewSummations () {
   const promises = []
   reviewSummations.forEach((reviewSummation) => {
     const record = {
-      index: config.get('esConfig.ES_INDEX'),
+      index: config.get('esConfig.ES_INDEX_V2'),
       type: config.get('esConfig.ES_TYPE'),
       id: reviewSummation.id,
       body: _.extend({ resource: 'reviewSummation' }, reviewSummation)

scripts/createIndex.js

@@ -44,7 +44,7 @@ co(function * createIndex () {
     }
   }
   yield esClient.indices.create({
-    index: config.get('esConfig.ES_INDEX'),
+    index: config.get('esConfig.ES_INDEX_V2'),
     body
   })
   logger.info('ES Index creation succeeded!')

scripts/createNewIndex.js

@@ -0,0 +1,35 @@
+/**
+ * Copies the existing index to a new index and updates the type of challengeId
+ * to be keyword (It is long in the source index)
+ */
+
+const co = require('co')
+const config = require('config')
+const logger = require('../src/common/logger')
+const helper = require('../src/common/helper')
+
+co(function * createIndex () {
+  logger.info('ES Index creation started!')
+  const esClient = helper.getEsClient()
+  const indices = yield esClient.indices.get({
+    index: config.get('esConfig.ES_INDEX')
+  })
+  const existingIndex = indices[config.get('esConfig.ES_INDEX')]
+  const existingMappings = existingIndex.mappings[config.get('esConfig.ES_TYPE')]
+  const body = { mappings: {} }
+  body.mappings[config.get('esConfig.ES_TYPE')] = {
+    properties: {
+      ...existingMappings.properties,
+      challengeId: { type: 'keyword' }
+    }
+  }
+  yield esClient.indices.create({
+    index: config.get('esConfig.ES_INDEX_V2'),
+    body
+  })
+  logger.info('ES Index creation succeeded!')
+  process.exit(0)
+}).catch((err) => {
+  logger.logFullError(err)
+  process.exit(1)
+})

scripts/data/Submissions.json

@@ -3,7 +3,7 @@
     "id": "a12a4180-65aa-42ec-a945-5fd21dec0501",
     "type": "ContestSubmission",
     "url": "https://software.topcoder.com/review/actions/DownloadContestSubmission?uid=123456",
-    "memberId": 40493050,
+    "memberId": "b24d4180-65aa-42ec-a945-5fd21dec0502",
     "challengeId": 30055732,
     "review": [
       {

scripts/deleteIndex.js

@@ -13,6 +13,9 @@ co(function * deleteIndex () {
   yield esClient.indices.delete({
     index: config.get('esConfig.ES_INDEX')
   })
+  yield esClient.indices.delete({
+    index: config.get('esConfig.ES_INDEX_V2')
+  })
   logger.info('ES Index deletion succeeded!')
   process.exit(0)
 }).catch((err) => {

scripts/migrateFromDBToES.js

@@ -30,7 +30,7 @@ function * migrateRecords (tableName) {
     logger.debug(`Number of ${tableName}s fetched from DB - ` + totalRecords)
     for (let i = 0; i < totalRecords; i++) {
       const record = {
-        index: config.get('esConfig.ES_INDEX'),
+        index: config.get('esConfig.ES_INDEX_V2'),
         type: config.get('esConfig.ES_TYPE'),
         id: records.Items[i].id,
         body: {