Merge branch 'update-mappings' into develop

Author: James Cori

config/default.js

@@ -40,7 +40,8 @@ module.exports = {
     API_VERSION: process.env.ES_API_VERSION || '6.8',
     ES_INDEX: process.env.ES_INDEX || 'challenge',
     ES_TYPE: process.env.ES_TYPE || '_doc', // ES 6.x accepts only 1 Type per index and it's mandatory to define it
-    ES_REFRESH: process.env.ES_REFRESH || 'true'
+    ES_REFRESH: process.env.ES_REFRESH || 'true',
+    TEMP_REINDEXING: process.env.TEMP_REINDEXING || true // if true, it won't delete the existing index when reindexing data
   },
 
   // in bytes

src/init-es.js

@@ -32,7 +32,15 @@ const initES = async () => {
     body.mappings[config.get('ES.ES_TYPE')] = {
       properties: {
         id: { type: 'keyword' }
-      }
+      },
+      dynamic_templates: [{
+        metadata: {
+          path_match: 'metadata.*',
+          mapping: {
+            type: 'text'
+          }
+        }
+      }]
     }
 
     await client.indices.create({

src/scripts/update-es-mappings.js

@@ -0,0 +1,116 @@
+const config = require('config')
+const logger = require('../common/logger')
+const helper = require('../common/helper')
+
+const esClient = helper.getESClient()
+
+function sleep (ms) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms)
+  })
+}
+
+function createIndex (indexName) {
+  const body = { mappings: {} }
+  body.mappings[config.get('ES.ES_TYPE')] = {
+    properties: {
+      id: { type: 'keyword' },
+      name: {
+        type: 'keyword',
+        normalizer: 'custom_sort_normalizer'
+      }
+    },
+    dynamic_templates: [{
+      metadata: {
+        path_match: 'metadata.*',
+        mapping: {
+          type: 'text'
+        }
+      }
+    }]
+  }
+  body.settings = {
+    analysis: {
+      normalizer: {
+        custom_sort_normalizer: {
+          type: 'custom',
+          char_filter: [],
+          filter: ['lowercase', 'asciifolding']
+        }
+      }
+    }
+  }
+
+  return esClient.indices.create({
+    index: indexName,
+    body
+  })
+}
+
+async function updateMappings () {
+  const tempReindexing = config.get('ES.TEMP_REINDEXING')
+  let indexName = config.get('ES.ES_INDEX')
+  let newIndexName = `${indexName}_tmp_dont_use_for_querying`
+
+  if (tempReindexing) {
+    try {
+      logger.info(`Attemp to remove temporary index ${newIndexName}`)
+      await esClient.indices.delete({
+        index: newIndexName
+      })
+      await sleep(500)
+    } catch (e) {
+      logger.info(`Index ${newIndexName} does not exist`)
+    }
+  }
+
+  await createIndex(newIndexName)
+  await sleep(500)
+  logger.info(`Reindexing from ${indexName} to ${newIndexName}`)
+  await esClient.reindex({
+    body: {
+      source: { index: indexName },
+      dest: { index: newIndexName }
+    },
+    waitForCompletion: true
+  })
+
+  if (tempReindexing) {
+    return
+  }
+
+  logger.warn(`Deleting ${indexName}. If script crashes after this point data may be lost and a recreation of index will be required.`)
+
+  await esClient.indices.delete({
+    index: indexName
+  })
+
+  logger.info(`Copying data back into ${indexName}`)
+
+  // This should be replaced with cloneIndex after migration to 7.4+
+  await createIndex(indexName)
+  await sleep(500)
+  await esClient.reindex({
+    body: {
+      source: { index: newIndexName },
+      dest: { index: indexName }
+    },
+    waitForCompletion: true
+  })
+
+  logger.info(`Removing ${newIndexName} index`)
+
+  await esClient.indices.delete({
+    index: newIndexName
+  })
+}
+
+updateMappings()
+  .then(() => {
+    logger.info('Done')
+    process.exit()
+  })
+  .catch((err) => {
+    logger.logFullError(err)
+    process.exit(1)
+  })