diff --git a/package.json b/package.json index 3defd2e..4fa45ef 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "start": "node app.js", "lint": "standard \"**/*.js\"", "insert-data": "node scripts/db/genData.js", - "delete-data": "node scripts/db/dropAll.js" + "delete-data": "node scripts/db/dropAll.js", + "migrate-db-to-es": "node scripts/db/dumpDbToEs.js" }, "repository": { "type": "git", diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js new file mode 100644 index 0000000..e8218ee --- /dev/null +++ b/scripts/db/dumpDbToEs.js @@ -0,0 +1,157 @@ +const _ = require('lodash') +const models = require('../../src/models') +const logger = require('../../src/common/logger') +const { getESClient } = require('../../src/common/es-client') +const { + topResources, + userResources, + organizationResources, + modelToESIndexMapping +} = require('../constants') + +async function cleanupES () { + const client = getESClient() + client.indices.delete({ + index: '_all' + }, function (err, res) { + if (err) { + console.error(err.message) + } else { + console.log('Existing indices have been deleted!') + } + }) +} + +async function insertIntoES (modelName, body) { + const esResourceName = modelToESIndexMapping[modelName] + + if (!esResourceName) { + logger.error(`Cannot insert data into model ${modelName}. No equivalent elasticsearch index found`) + + return + } + + const client = getESClient() + + if (_.includes(_.keys(topResources), esResourceName)) { + await client.create({ + index: topResources[esResourceName].index, + type: topResources[esResourceName].type, + id: body.id, + body, + refresh: 'true' + }) + } else if (_.includes(_.keys(userResources), esResourceName)) { + const userResource = userResources[esResourceName] + + const user = await client.getSource({ + index: topResources.user.index, + type: topResources.user.type, + id: body.userId + }) + + if (userResource.nested === true && userResource.mappingCreated !== true) { + await client.indices.putMapping({ + index: topResources.user.index, + type: topResources.user.type, + include_type_name: true, + body: { + properties: { + [userResource.propertyName]: { + type: 'nested' + } + } + } + }) + userResource.mappingCreated = true + } + + const relateId = body[userResource.relateKey] + + if (!user[userResource.propertyName]) { + user[userResource.propertyName] = [] + } + + if (_.some(user[userResource.propertyName], [userResource.relateKey, relateId])) { + logger.error(`Can't create existing ${esResourceName} with the ${userResource.relateKey}: ${relateId}, userId: ${body.userId}`) + } else { + user[userResource.propertyName].push(body) + await client.update({ + index: topResources.user.index, + type: topResources.user.type, + id: body.userId, + body: { doc: user }, + refresh: 'true' + }) + } + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + const orgResource = organizationResources[esResourceName] + + const organization = await client.getSource({ + index: topResources.organization.index, + type: topResources.organization.type, + id: body.organizationId + }) + + const relateId = body[orgResource.relateKey] + + if (!organization[orgResource.propertyName]) { + organization[orgResource.propertyName] = [] + } + + if (_.some(organization[orgResource.propertyName], [orgResource.relateKey, relateId])) { + logger.error(`Can't create existing ${esResourceName} with the ${orgResource.relateKey}: ${relateId}, organizationId: ${body.organizationId}`) + } else { + organization[orgResource.propertyName].push(body) + await client.update({ + index: topResources.organization.index, + type: topResources.organization.type, + id: body.organizationId, + body: { doc: organization }, + refresh: 'true' + }) + } + } +} + +/** + * import test data + * @return {Promise<void>} + */ +async function main () { + let keys = Object.keys(models) + keys = _.orderBy(keys, k => { + const esResourceName = modelToESIndexMapping[k] + + // Create parent data first + if (_.includes(_.keys(topResources), esResourceName)) { + return -1 + } + + return 1 + }) + + await cleanupES() + + for (let i = 0; i < keys.length; i++) { + const key = keys[i] + if (models[key].tableName) { + try { + const data = await models.DBHelper.find(models[key], []) + for (let i = 0; i < data.length; i++) { + await insertIntoES(key, data[i]) + } + logger.info('import data for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('import data for ' + key + ' failed') + } + } + } + logger.info('all done') + process.exit(0) +} + +(async () => { + main().catch(err => console.error(err)) +})()