From c8a5248f3687ac1c37116fc6158d0beff0d3f2b7 Mon Sep 17 00:00:00 2001 From: xxcxy Date: Wed, 7 Oct 2020 00:25:22 +0800 Subject: [PATCH 01/15] Issue 50 - Fix issues with api (documentation) --- docs/swagger.yaml | 740 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 739 insertions(+), 1 deletion(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index d8f7ba7..cd1de3d 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -3,7 +3,12 @@ swagger: "2.0" info: description: "API for an employee management system to determine employees that\ \ are no longer working on active projects and to understand their qualifications\ - \ and expertise for suitability in other projects" + \ and expertise for suitability in other projects + \n\n**Pagination**\n\n + Requests that return multiple items will be paginated to 20 items by default. You can specify + further pages with the `page` parameter. You can also set a custom page + size up to 100 with the `perPage` parameter. + \n\nPagination response data is included in http headers. By Default, the response header contains links with `next`, `last`, `first`, `prev` resource links.\n\n**Point to note** - the page attributes will have no effect if the data is fetched from db and not from es." version: "1.0.0" title: "UBahn API" host: "ubahn-api-dev.herokuapp.com" @@ -226,6 +231,28 @@ paths: type: "array" items: $ref: "#/definitions/User" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -411,6 +438,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -622,6 +671,28 @@ paths: type: "array" items: $ref: "#/definitions/UserSkill" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -655,6 +726,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -889,6 +982,28 @@ paths: type: "array" items: $ref: "#/definitions/Skill" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -922,6 +1037,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1118,6 +1255,28 @@ paths: type: "array" items: $ref: "#/definitions/SkillsProvider" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1145,6 +1304,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1340,6 +1521,28 @@ paths: type: "array" items: $ref: "#/definitions/Role" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1367,6 +1570,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1563,6 +1788,28 @@ paths: type: "array" items: $ref: "#/definitions/UserRole" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1593,6 +1840,28 @@ paths: responses: "200": description: "OK - the request was successful" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1790,6 +2059,28 @@ paths: type: "array" items: $ref: "#/definitions/ExternalProfile" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1836,6 +2127,28 @@ paths: responses: "200": description: "OK - the request was successful" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2071,6 +2384,28 @@ paths: type: "array" items: $ref: "#/definitions/Achievement" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2106,6 +2441,28 @@ paths: responses: "200": description: "OK - the request was successful" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2338,6 +2695,28 @@ paths: type: "array" items: $ref: "#/definitions/AchievementsProvider" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2365,6 +2744,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2560,6 +2961,28 @@ paths: type: "array" items: $ref: "#/definitions/Organization" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2587,6 +3010,28 @@ paths: responses: "200": description: "success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2783,6 +3228,28 @@ paths: type: "array" items: $ref: "#/definitions/OrganizationSkillsProvider" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2813,6 +3280,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3012,6 +3501,28 @@ paths: type: "array" items: $ref: "#/definitions/UserAttribute" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3056,6 +3567,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3287,6 +3820,28 @@ paths: type: "array" items: $ref: "#/definitions/Attribute" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3309,6 +3864,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3512,6 +4089,28 @@ paths: type: "array" items: $ref: "#/definitions/AttributeGroup" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3545,6 +4144,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3747,6 +4368,28 @@ paths: type: "array" items: $ref: "#/definitions/EnhancedUser" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3795,6 +4438,37 @@ paths: security: - Bearer: [] x-swagger-router-controller: "Search" + head: + tags: + - "Skill Search" + description: "Retrieve header information for a search operation on user attributes in the application. Multiple filters are supported.\n\n**Security** - Note\ + \ that for non-admin users, this endpoint will only return entities that\n\ + the user has created.\n" + operationId: "searchUserAttributesHEAD" + parameters: + - name: "attributeId" + in: "query" + description: "The attribute id" + type: "string" + format: "UUID" + - name: "attributeValue" + in: "query" + description: "The attribute value" + type: "string" + responses: + "200": + description: "OK - the request was successful" + "400": + $ref: "#/definitions/BadRequest" + "401": + $ref: "#/definitions/Unauthorized" + "403": + $ref: "#/definitions/Forbidden" + "500": + $ref: "#/definitions/ServerError" + security: + - Bearer: [] + x-swagger-router-controller: "Search" /skill-search/userAchievements: get: tags: @@ -3833,6 +4507,38 @@ paths: security: - Bearer: [] x-swagger-router-controller: "Search" + head: + tags: + - "Skill Search" + description: "Retrieve header information for a search operation on user achievements in the application. Multiple filters are\nsupported.\n\n**Security** - Note\ + \ that for non-admin users, this endpoint will only return entities that\n\ + the user has created.\n" + operationId: "searchUserAchievementsHEAD" + parameters: + - name: "organizationId" + in: "query" + description: "The organization id" + type: "string" + format: "UUID" + required: true + - name: "keyword" + in: "query" + description: "The query keyword" + type: "string" + responses: + "200": + description: "OK - the request was successful" + "400": + $ref: "#/definitions/BadRequest" + "401": + $ref: "#/definitions/Unauthorized" + "403": + $ref: "#/definitions/Forbidden" + "500": + $ref: "#/definitions/ServerError" + security: + - Bearer: [] + x-swagger-router-controller: "Search" /skill-search/skills: get: tags: @@ -3871,6 +4577,38 @@ paths: security: - Bearer: [] x-swagger-router-controller: "Search" + head: + tags: + - "Skill Search" + description: "Retrieve header information for a search operation on skills associated with an org in the application. Multiple filters are\nsupported.\n\n**Security** - Note\ + \ that for non-admin users, this endpoint will only return entities that\n\ + the user has created.\n" + operationId: "searchSkillsHEAD" + parameters: + - name: "organizationId" + in: "query" + description: "The organization id" + type: "string" + format: "UUID" + required: true + - name: "keyword" + in: "query" + description: "The query keyword" + type: "string" + responses: + "200": + description: "OK - the request was successful" + "400": + $ref: "#/definitions/BadRequest" + "401": + $ref: "#/definitions/Unauthorized" + "403": + $ref: "#/definitions/Forbidden" + "500": + $ref: "#/definitions/ServerError" + security: + - Bearer: [] + x-swagger-router-controller: "Search" securityDefinitions: Bearer: type: "apiKey" From 7c965ca0edafe02df74a05473698973eb089abba Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Tue, 13 Oct 2020 11:38:56 +0530 Subject: [PATCH 02/15] Support enrichment for users endpoints --- src/modules/user/service.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/modules/user/service.js b/src/modules/user/service.js index c7160a8..9b1015d 100644 --- a/src/modules/user/service.js +++ b/src/modules/user/service.js @@ -17,7 +17,11 @@ const methods = helper.getServiceMethods( firstName: joi.string(), lastName: joi.string() }, - { handle: joi.string(), roleId: joi.string() }, + { + handle: joi.string(), + roleId: joi.string(), + enrich: joi.boolean() + }, async query => { let prefix = 'select * from DUser' const dbQueries = [] From 365423a780a8952c1e34c68e802f92ed3d133d3a Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Tue, 13 Oct 2020 20:26:51 +0530 Subject: [PATCH 03/15] Support filtering by external id on users --- src/modules/user/service.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/user/service.js b/src/modules/user/service.js index 9b1015d..8f2cafe 100644 --- a/src/modules/user/service.js +++ b/src/modules/user/service.js @@ -20,7 +20,8 @@ const methods = helper.getServiceMethods( { handle: joi.string(), roleId: joi.string(), - enrich: joi.boolean() + enrich: joi.boolean(), + 'externalProfile.externalId': joi.string() }, async query => { let prefix = 'select * from DUser' From 95eb66b74b35f02b2e24c5e76bccde33cfac4900 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Tue, 13 Oct 2020 20:36:41 +0530 Subject: [PATCH 04/15] Support filtering by org id on users --- src/modules/user/service.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/user/service.js b/src/modules/user/service.js index 8f2cafe..7593d50 100644 --- a/src/modules/user/service.js +++ b/src/modules/user/service.js @@ -21,7 +21,8 @@ const methods = helper.getServiceMethods( handle: joi.string(), roleId: joi.string(), enrich: joi.boolean(), - 'externalProfile.externalId': joi.string() + 'externalProfile.externalId': joi.string(), + 'externalProfile.organizationId': joi.string() }, async query => { let prefix = 'select * from DUser' From 211f395cb98aa283e8e7eed98976ed14438328a4 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Tue, 20 Oct 2020 12:25:34 +0530 Subject: [PATCH 05/15] Merge winning submission from enrich contest --- README.md | 3 +- config/default.js | 4 +- package-lock.json | 317 +++++++++++++++++++++++---------------- package.json | 4 +- scripts/constants.js | 5 + scripts/db/dumpDbToEs.js | 13 +- scripts/db/genData.js | 15 +- src/common/es-client.js | 13 +- src/common/es-helper.js | 95 ++---------- 9 files changed, 236 insertions(+), 233 deletions(-) diff --git a/README.md b/README.md index ce89626..d4bb982 100755 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ - node 12.x - npm 6.x - docker -- elasticsearch 6.x +- elasticsearch 7.7 ## Configuration @@ -36,7 +36,6 @@ Configuration for the application is at config/default.js and config/production. - UBAHN_DELETE_TOPIC: Kafka topic for delete message - UBAHN_AGGREGATE_TOPIC: Kafka topic that is used to combine all create, update and delete message(s) - ES.HOST: Elasticsearch host -- ES.API_VERSION: Elasticsearch API version - ES.DOCUMENTS: Elasticsearch index, type and id mapping for resources. For `ES.DOCUMENTS` configuration, you will find multiple other configurations below it. Each has default values that you can override using the environment variables diff --git a/config/default.js b/config/default.js index b46f98b..e824552 100755 --- a/config/default.js +++ b/config/default.js @@ -50,8 +50,8 @@ module.exports = { // ElasticSearch ES: { - HOST: process.env.ES_HOST || 'localhost:9200', - API_VERSION: process.env.ES_API_VERSION || '7.4', + HOST: process.env.ES_HOST || 'http://localhost:9200', + ENRICH_USER_PIPELINE_NAME: process.env.ENRICH_USER_PIPELINE_NAME || 'enrich_user', // es mapping: _index, _type, _id DOCUMENTS: { achievementprovider: { diff --git a/package-lock.json b/package-lock.json index 5a98387..2a51a95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,33 @@ "js-tokens": "^4.0.0" } }, + "@elastic/elasticsearch": { + "version": "7.9.1", + "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.9.1.tgz", + "integrity": "sha512-NfPADbm9tRK/4ohpm9+aBtJ8WPKQqQaReyBKT225pi2oKQO1IzRlfM+OPplAvbhoH1efrSj1NKk27L+4BCrzXQ==", + "requires": { + "debug": "^4.1.1", + "decompress-response": "^4.2.0", + "ms": "^2.1.1", + "pump": "^3.0.0", + "secure-json-parse": "^2.1.0" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "@hapi/address": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", @@ -78,6 +105,11 @@ "@hapi/hoek": "^8.3.0" } }, + "@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" + }, "@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", @@ -102,9 +134,9 @@ } }, "@types/express": { - "version": "4.17.7", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.7.tgz", - "integrity": "sha512-dCOT5lcmV/uC2J9k0rPafATeeyz+99xTt54ReX11/LObZgfzJqZNcW27zGhYyX+9iSEGXGt5qLPwRSvBZcLvtQ==", + "version": "4.17.8", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.8.tgz", + "integrity": "sha512-wLhcKh3PMlyA2cNAB9sjM1BntnhPMiM0JOBwPBqttjHev2428MLEB4AYVN+d8s2iyCVZac+o41Pflm/ZH5vLXQ==", "requires": { "@types/body-parser": "*", "@types/express-serve-static-core": "*", @@ -122,9 +154,9 @@ } }, "@types/express-serve-static-core": { - "version": "4.17.8", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.8.tgz", - "integrity": "sha512-1SJZ+R3Q/7mLkOD9ewCBDYD2k0WyZQtWYqF/2VvoNN2/uhI49J9CDN4OAm+wGMA0DbArA4ef27xl4+JwMtGggw==", + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.13.tgz", + "integrity": "sha512-RgDi5a4nuzam073lRGKTUIaL3eF2+H7LJvJ8eUnCI0wA6SNjXc44DCmWNiTLs/AZ7QlsFWZiw/gTG3nSQGL0fA==", "requires": { "@types/node": "*", "@types/qs": "*", @@ -140,9 +172,9 @@ } }, "@types/mime": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.2.tgz", - "integrity": "sha512-4kPlzbljFcsttWEq6aBW0OZe6BDajAmyvr2xknBG92tejQnvdGtT9+kXSZ580DqpxY9qG2xeQVF9Dq0ymUTo5Q==" + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.3.tgz", + "integrity": "sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q==" }, "@types/node": { "version": "12.0.2", @@ -150,9 +182,9 @@ "integrity": "sha512-5tabW/i+9mhrfEOUcLDu2xBPsHJ+X5Orqy9FKpale3SjDA17j5AEpYq5vfy3oAeAHGcvANRCO3NV3d2D6q3NiA==" }, "@types/qs": { - "version": "6.9.3", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.3.tgz", - "integrity": "sha512-7s9EQWupR1fTc2pSMtXRQ9w9gLOcrJn+h7HOXw4evxyvVqMi4f+q7d2tnFe3ng3SNHjtK+0EzGMGFUQX4/AQRA==" + "version": "6.9.5", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.5.tgz", + "integrity": "sha512-/JHkVHtx/REVG0VVToGRGH2+23hsYLHdyG+GrvoUGlGAd0ErauXDyvHtRI/7H7mzLm+tBCKA7pfcpkQ1lf58iQ==" }, "@types/range-parser": { "version": "1.2.3", @@ -160,9 +192,9 @@ "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==" }, "@types/serve-static": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.4.tgz", - "integrity": "sha512-jTDt0o/YbpNwZbQmE/+2e+lfjJEJJR0I3OFaKQKPWkASkCoW3i6fsUnqudSMcNAfbtmADGu8f4MV4q+GqULmug==", + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.5.tgz", + "integrity": "sha512-6M64P58N+OXjU432WoLLBQxbA0LRGBCRm7aAGQJ+SMC1IMl0dgRVi9EFfoDcS2a7Xogygk/eGN94CfwU9UF7UQ==", "requires": { "@types/express-serve-static-core": "*", "@types/mime": "*" @@ -189,18 +221,34 @@ "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", "dev": true }, - "agentkeepalive": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", - "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", + "agent-base": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", + "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", "requires": { - "humanize-ms": "^1.2.1" + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } } }, "ajv": { "version": "6.12.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz", "integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==", + "dev": true, "requires": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -306,6 +354,14 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, + "aws-elasticsearch-connector": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/aws-elasticsearch-connector/-/aws-elasticsearch-connector-9.0.0.tgz", + "integrity": "sha512-O+A9HEa14gOiKTAp6U6Ha1RRJvQjc046wIn9CJ69wc+c1c5CfPE4xl4Av6Zyv6dgzs+RVGxdetjm8RpSlTUmhQ==", + "requires": { + "aws4": "^1.10.0" + } + }, "aws-sdk": { "version": "2.668.0", "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.668.0.tgz", @@ -335,9 +391,9 @@ "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.0.tgz", - "integrity": "sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA==" + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.1.tgz", + "integrity": "sha512-zg7Hz2k5lI8kb7U32998pRRFin7zJlkfezGJjUc2heaD4Pw2wObakCDVzkKztTm/Ln7eiVvYsjqak0Ed4LkMDA==" }, "axios": { "version": "0.19.2", @@ -665,6 +721,14 @@ "integrity": "sha1-IwdjLUwEOCuN+KMvcLiVBG1SdF8=", "dev": true }, + "decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "requires": { + "mimic-response": "^2.0.0" + } + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", @@ -781,53 +845,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "elasticsearch": { - "version": "16.7.1", - "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.1.tgz", - "integrity": "sha512-PL/BxB03VGbbghJwISYvVcrR9KbSSkuQ7OM//jHJg/End/uC2fvXg4QI7RXLvCGbhBuNQ8dPue7DOOPra73PCw==", - "requires": { - "agentkeepalive": "^3.4.1", - "chalk": "^1.0.0", - "lodash": "^4.17.10" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - }, - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" - }, - "chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" - } - } - }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -847,6 +864,14 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } + }, "env-variable": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.6.tgz", @@ -907,7 +932,8 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true }, "eslint": { "version": "6.8.0", @@ -1516,12 +1542,25 @@ "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", "requires": { - "ajv": "^6.5.5", + "ajv": "^6.12.3", "har-schema": "^2.0.0" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + } } }, "has": { @@ -1533,21 +1572,6 @@ "function-bind": "^1.1.1" } }, - "has-ansi": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", - "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", - "requires": { - "ansi-regex": "^2.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - } - } - }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -1571,11 +1595,6 @@ "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==", "dev": true }, - "http-aws-es": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/http-aws-es/-/http-aws-es-6.0.0.tgz", - "integrity": "sha512-g+qp7J110/m4aHrR3iit4akAlnW0UljZ6oTq/rCcbsI8KP9x+95vqUtx49M2XQ2JMpwJio3B6gDYx+E8WDxqiA==" - }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -1588,6 +1607,31 @@ "toidentifier": "1.0.0" } }, + "http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "requires": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -1598,12 +1642,28 @@ "sshpk": "^1.7.0" } }, - "humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=", + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", "requires": { - "ms": "^2.0.0" + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } } }, "iconv-lite": { @@ -1992,13 +2052,15 @@ } }, "jwks-rsa": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-1.8.1.tgz", - "integrity": "sha512-CcE8ypsATHwGmzELwzeFjLzPBXTXTrMmDYbn92LTQwYsZdOedp+ZIuYTofUdrWreu8CKRuXmhk17+6/li2sR6g==", + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-1.10.1.tgz", + "integrity": "sha512-UmjOsATVu7eQr17wbBCS+BSoz5LFtl57PtNXHbHFeT1WKomHykCHtn7c8inWVI7tpnsy6CZ1KOMJTgipFwXPig==", "requires": { "@types/express-jwt": "0.0.42", "axios": "^0.19.2", "debug": "^4.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", "jsonwebtoken": "^8.5.1", "limiter": "^1.1.5", "lru-memoizer": "^2.1.2", @@ -2006,11 +2068,11 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -2209,6 +2271,11 @@ "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, + "mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==" + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -2258,9 +2325,9 @@ } }, "nan": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", - "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==", "optional": true }, "natural-compare": { @@ -2683,6 +2750,15 @@ "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -2915,6 +2991,11 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, + "secure-json-parse": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.1.0.tgz", + "integrity": "sha512-GckO+MS/wT4UogDyoI/H/S1L0MCcKS1XX/vp48wfmU7Nw4woBmb8mIpu4zPBQjKlRT88/bt9xdoV4111jPpNJA==" + }, "semaphore-async-await": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/semaphore-async-await/-/semaphore-async-await-1.5.1.tgz", @@ -3337,22 +3418,6 @@ "lodash": "^4.17.15", "superagent": "^3.8.3", "tc-core-library-js": "github:appirio-tech/tc-core-library-js#v2.6.4" - }, - "dependencies": { - "tc-core-library-js": { - "version": "github:appirio-tech/tc-core-library-js#df0b36c51cf80918194cbff777214b3c0cf5a151", - "from": "github:appirio-tech/tc-core-library-js#v2.6.4", - "requires": { - "axios": "^0.19.0", - "bunyan": "^1.8.12", - "jsonwebtoken": "^8.5.1", - "jwks-rsa": "^1.6.0", - "lodash": "^4.17.15", - "millisecond": "^0.1.2", - "r7insight_node": "^1.8.4", - "request": "^2.88.0" - } - } } }, "tc-core-library-js": { diff --git a/package.json b/package.json index 4fa45ef..ce8a809 100644 --- a/package.json +++ b/package.json @@ -15,18 +15,18 @@ "url": "" }, "dependencies": { + "@elastic/elasticsearch": "^7.9.1", "@hapi/joi": "^16.1.8", "@hapi/joi-date": "^2.0.1", "amazon-qldb-driver-nodejs": "^0.1.1-preview.2", + "aws-elasticsearch-connector": "^9.0.0", "aws-sdk": "^2.627.0", "axios": "^0.19.2", "body-parser": "^1.19.0", "config": "^3.2.4", "cors": "^2.8.5", - "elasticsearch": "^16.7.1", "express": "^4.17.1", "get-parameter-names": "^0.3.0", - "http-aws-es": "^6.0.0", "ion-js": "^3.1.2", "js-yaml": "^3.13.1", "lodash": "^4.17.19", diff --git a/scripts/constants.js b/scripts/constants.js index 755b650..42acd44 100644 --- a/scripts/constants.js +++ b/scripts/constants.js @@ -8,10 +8,12 @@ const config = require('config') const topResources = { achievementprovider: { index: config.get('ES.DOCUMENTS.achievementprovider.index'), + enrichPolicy: 'achievementprovider-policy', type: config.get('ES.DOCUMENTS.achievementprovider.type') }, attribute: { index: config.get('ES.DOCUMENTS.attribute.index'), + enrichPolicy: 'attribute-policy', type: config.get('ES.DOCUMENTS.attribute.type') }, attributegroup: { @@ -20,14 +22,17 @@ const topResources = { }, organization: { index: config.get('ES.DOCUMENTS.organization.index'), + enrichPolicy: 'organization-policy', type: config.get('ES.DOCUMENTS.organization.type') }, role: { index: config.get('ES.DOCUMENTS.role.index'), + enrichPolicy: 'role-policy', type: config.get('ES.DOCUMENTS.role.type') }, skill: { index: config.get('ES.DOCUMENTS.skill.index'), + enrichPolicy: 'skill-policy', type: config.get('ES.DOCUMENTS.skill.type') }, skillprovider: { diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index 17c51cf..0d12e73 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -1,4 +1,5 @@ const _ = require('lodash') +const config = require('config') const models = require('../../src/models') const logger = require('../../src/common/logger') const { getESClient } = require('../../src/common/es-client') @@ -38,10 +39,13 @@ async function insertIntoES (modelName, body) { body, refresh: 'true' }) + if (topResources[esResourceName].enrichPolicy) { + await client.enrich.executePolicy({ name: topResources[esResourceName].enrichPolicy }) + } } else if (_.includes(_.keys(userResources), esResourceName)) { const userResource = userResources[esResourceName] - const user = await client.getSource({ + const { body: user } = await client.getSource({ index: topResources.user.index, type: topResources.user.type, id: body.userId @@ -73,18 +77,19 @@ async function insertIntoES (modelName, body) { logger.error(`Can't create existing ${esResourceName} with the ${userResource.relateKey}: ${relateId}, userId: ${body.userId}`) } else { user[userResource.propertyName].push(body) - await client.update({ + await client.index({ index: topResources.user.index, type: topResources.user.type, id: body.userId, - body: { doc: user }, + body: user, + pipeline: config.get('ES.ENRICH_USER_PIPELINE_NAME'), refresh: 'true' }) } } else if (_.includes(_.keys(organizationResources), esResourceName)) { const orgResource = organizationResources[esResourceName] - const organization = await client.getSource({ + const { body: organization } = await client.getSource({ index: topResources.organization.index, type: topResources.organization.type, id: body.organizationId diff --git a/scripts/db/genData.js b/scripts/db/genData.js index 2f501a1..e11dca9 100644 --- a/scripts/db/genData.js +++ b/scripts/db/genData.js @@ -1,4 +1,5 @@ const _ = require('lodash') +const config = require('config') const models = require('../../src/models') const logger = require('../../src/common/logger') const { getESClient } = require('../../src/common/es-client') @@ -28,10 +29,13 @@ async function insertIntoES (modelName, body) { body, refresh: 'true' }) + if (topResources[esResourceName].enrichPolicy) { + await client.enrich.executePolicy({ name: topResources[esResourceName].enrichPolicy }) + } } else if (_.includes(_.keys(userResources), esResourceName)) { const userResource = userResources[esResourceName] - const user = await client.getSource({ + const { body: user } = await client.getSource({ index: topResources.user.index, type: topResources.user.type, id: body.userId @@ -63,18 +67,19 @@ async function insertIntoES (modelName, body) { logger.error(`Can't create existing ${esResourceName} with the ${userResource.relateKey}: ${relateId}, userId: ${body.userId}`) } else { user[userResource.propertyName].push(body) - await client.update({ + await client.index({ index: topResources.user.index, type: topResources.user.type, id: body.userId, - body: { doc: user }, - refresh: 'true' + body: user, + pipeline: config.get('ES.ENRICH_USER_PIPELINE_NAME'), + refresh: 'wait_for' }) } } else if (_.includes(_.keys(organizationResources), esResourceName)) { const orgResource = organizationResources[esResourceName] - const organization = await client.getSource({ + const { body: organization } = await client.getSource({ index: topResources.organization.index, type: topResources.organization.type, id: body.organizationId diff --git a/src/common/es-client.js b/src/common/es-client.js index 2ffc3f1..20c1bb7 100644 --- a/src/common/es-client.js +++ b/src/common/es-client.js @@ -1,6 +1,7 @@ const config = require('config') const AWS = require('aws-sdk') -const elasticsearch = require('elasticsearch') +const elasticsearch = require('@elastic/elasticsearch') +const createAwsElasticsearchConnector = require('aws-elasticsearch-connector') AWS.config.region = config.AWS_REGION @@ -16,20 +17,16 @@ function getESClient () { return esClient } const host = config.ES.HOST - const apiVersion = config.ES.API_VERSION - if (!esClient) { // AWS ES configuration is different from other providers if (/.*amazonaws.*/.test(host)) { esClient = new elasticsearch.Client({ - apiVersion, - host, - connectionClass: require('http-aws-es') // eslint-disable-line global-require + ...createAwsElasticsearchConnector(AWS.config), + node: host }) } else { esClient = new elasticsearch.Client({ - apiVersion, - host + node: host }) } } diff --git a/src/common/es-helper.js b/src/common/es-helper.js index 0b395c5..581ce3f 100644 --- a/src/common/es-helper.js +++ b/src/common/es-helper.js @@ -376,68 +376,6 @@ function parseEnrichFilter (params) { return filters } -/** - * Enrich a resource recursively by following enrich path. - * @param resource the resource to enrich - * @param enrichIdProp the id property of child resource in parent object - * @param item the parent object - * @returns {Promise} the promise of enriched parent object - */ -async function enrichResource (resource, enrichIdProp, item) { - const subDoc = DOCUMENTS[resource] - const filterChain = FILTER_CHAIN[resource] - const subResult = await esClient.getSource({ - index: subDoc.index, - type: subDoc.type, - id: item[enrichIdProp] - }) - - if (filterChain.enrichNext) { - const enrichIdProp = filterChain.idField - // return error if any id is missing in enrich path - if (!subResult[enrichIdProp]) { - throw new Error(`The parent ${resource} is missing id value of child resource ${filterChain.enrichNext}`) - } - // enrich next child resource recursively - await enrichResource(filterChain.enrichNext, enrichIdProp, subResult) - } - item[resource] = subResult -} - -/** - * Enrich a user. - * @param user the user object to enrich - * @returns {Promise<*>} the promise of enriched user - */ -async function enrichUser (user) { - for (const subProp of Object.keys(SUB_USER_DOCUMENTS)) { - const subDoc = SUB_USER_DOCUMENTS[subProp] - const subData = user[subDoc.userField] - const filterChain = FILTER_CHAIN[subProp] - if (subData && subData.length > 0) { - // enrich next level sub resources - for (const subItem of subData) { - await enrichResource(filterChain.enrichNext, filterChain.idField, subItem) - } - } - } - return user -} - -/** - * Enrich users. - * @param users list of users from ES search - * @returns {Promise<*>} the enriched users - */ -async function enrichUsers (users) { - const enrichedUsers = [] - for (const user of users) { - const enriched = await enrichUser(user) - enrichedUsers.push(enriched) - } - return enrichedUsers -} - /** * Get a resource by Id from ES. * @param resource the resource to get @@ -494,12 +432,9 @@ async function getFromElasticSearch (resource, ...args) { logger.debug(`ES query for get ${resource}: ${JSON.stringify(esQuery, null, 2)}`) // query ES - const result = await esClient.getSource(esQuery) + const { body: result } = await esClient.getSource(esQuery) - if (params.enrich && resource === 'user') { - const user = await enrichUser(result) - return user - } else if (subUserDoc) { + if (subUserDoc) { // find top sub doc by sub.id const found = result[subUserDoc.userField].find(sub => sub[filterChain.idField] === params[filterChain.idField]) if (found) { @@ -713,7 +648,7 @@ async function searchSkills (keyword, skillProviderIds) { } logger.debug(`ES query for searching skills: ${JSON.stringify(esQuery, null, 2)}`) - const results = await esClient.search(esQuery) + const { body: results } = await esClient.search(esQuery) return results.hits.hits.map(hit => hit._source) } @@ -1059,7 +994,7 @@ async function resolveResFilter (filter, initialRes) { // query ES with filter const esQuery = buildEsQueryFromFilter(filter) - const result = await esClient.search(esQuery) + const { body: result } = await esClient.search(esQuery) const numHits = getTotalCount(result.hits.total) @@ -1288,7 +1223,7 @@ async function searchElasticSearch (resource, ...args) { } logger.debug(`ES query for search ${resource}: ${JSON.stringify(esQuery, null, 2)}`) - const docs = await esClient.search(esQuery) + const { body: docs } = await esClient.search(esQuery) if (docs.hits && getTotalCount(docs.hits.total) === 0) { return { total: docs.hits.total, @@ -1299,10 +1234,7 @@ async function searchElasticSearch (resource, ...args) { } let result = [] - if (resource === 'user' && params.enrich) { - const users = docs.hits.hits.map(hit => hit._source) - result = await enrichUsers(users) - } else if (topUserSubDoc) { + if (topUserSubDoc) { result = docs.hits.hits[0]._source[topUserSubDoc.userField] // for sub-resource query, it returns all sub-resource items in one user, // so needs filtering and also page size @@ -1420,14 +1352,9 @@ async function searchUsers (authUser, filter, params) { logger.debug(`ES query for searching users: ${JSON.stringify(esQuery, null, 2)}`) console.time('mainesquery') - const docs = await esClient.search(esQuery) + const { body: docs } = await esClient.search(esQuery) console.timeEnd('mainesquery') - const users = docs.hits.hits.map(hit => hit._source) - - logger.debug('Enrich users') - console.time('enrichUsers') - const result = await enrichUsers(users) - console.timeEnd('enrichUsers') + const result = docs.hits.hits.map(hit => hit._source) return { total: getTotalCount(docs.hits.total), @@ -1448,7 +1375,7 @@ async function searchSkillsInOrganization ({ organizationId, keyword }) { const esQueryToGetSkillProviders = buildEsQueryToGetSkillProviderIds(organizationId) logger.debug(`ES query to get skill provider ids: ${JSON.stringify(esQueryToGetSkillProviders, null, 2)}`) - const esResultOfQueryToGetSkillProviders = await esClient.search(esQueryToGetSkillProviders) + const { body: esResultOfQueryToGetSkillProviders } = await esClient.search(esQueryToGetSkillProviders) logger.debug(`ES result: ${JSON.stringify(esResultOfQueryToGetSkillProviders, null, 2)}`) const skillProviderIds = _.flatten(esResultOfQueryToGetSkillProviders.hits.hits.map(hit => hit._source.skillProviders == null ? [] : hit._source.skillProviders.map(sp => sp.skillProviderId))) @@ -1474,7 +1401,7 @@ async function searchAttributeValues ({ attributeId, attributeValue }) { const esQuery = buildEsQueryToGetAttributeValues(attributeId, querystring.unescape(attributeValue), 5) logger.debug(`ES query for searching attribute values: ${JSON.stringify(esQuery, null, 2)}`) - const esResult = await esClient.search(esQuery) + const { body: esResult } = await esClient.search(esQuery) logger.debug(`ES Result: ${JSON.stringify(esResult, null, 2)}`) const result = [] const attributes = esResult.aggregations.attributes.ids.buckets @@ -1502,7 +1429,7 @@ async function searchAchievementValues ({ organizationId, keyword }) { const esQuery = buildEsQueryToGetAchievements(organizationId, querystring.unescape(keyword), 5) logger.debug(`ES query for searching achievement values; ${JSON.stringify(esQuery, null, 2)}`) - const esResult = await esClient.search(esQuery) + const { body: esResult } = await esClient.search(esQuery) logger.debug(`ES response ${JSON.stringify(esResult, null, 2)}`) const result = esResult.aggregations.achievements.buckets.map(a => { const achievementName = a.key From 25ea241145d65b8b72c8d53f9cc30b25598f7205 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Wed, 21 Oct 2020 13:49:20 +0530 Subject: [PATCH 06/15] Working data insertion script --- scripts/constants.js | 149 ++++++++++++++++++++++++++++------ scripts/db/genData.js | 183 +++++++++++++++++++++++++++++++++++------- 2 files changed, 277 insertions(+), 55 deletions(-) diff --git a/scripts/constants.js b/scripts/constants.js index 42acd44..1b0a5b3 100644 --- a/scripts/constants.js +++ b/scripts/constants.js @@ -6,42 +6,136 @@ const config = require('config') const topResources = { + skillprovider: { + index: config.get('ES.DOCUMENTS.skillprovider.index'), + type: config.get('ES.DOCUMENTS.skillprovider.type'), + enrich: { + policyName: 'skillprovider-policy', + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] + }, + pipeline: { + id: 'skillprovider-pipeline', + field: 'skillProviderId', + targetField: 'skillprovider', + maxMatches: '1' + } + }, + + role: { + index: config.get('ES.DOCUMENTS.role.index'), + type: config.get('ES.DOCUMENTS.role.type'), + enrich: { + policyName: 'role-policy', + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] + } + }, + achievementprovider: { index: config.get('ES.DOCUMENTS.achievementprovider.index'), - enrichPolicy: 'achievementprovider-policy', - type: config.get('ES.DOCUMENTS.achievementprovider.type') - }, - attribute: { - index: config.get('ES.DOCUMENTS.attribute.index'), - enrichPolicy: 'attribute-policy', - type: config.get('ES.DOCUMENTS.attribute.type') + type: config.get('ES.DOCUMENTS.achievementprovider.type'), + enrich: { + policyName: 'achievementprovider-policy', + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] + } }, + attributegroup: { index: config.get('ES.DOCUMENTS.attributegroup.index'), - type: config.get('ES.DOCUMENTS.attributegroup.type') - }, - organization: { - index: config.get('ES.DOCUMENTS.organization.index'), - enrichPolicy: 'organization-policy', - type: config.get('ES.DOCUMENTS.organization.type') - }, - role: { - index: config.get('ES.DOCUMENTS.role.index'), - enrichPolicy: 'role-policy', - type: config.get('ES.DOCUMENTS.role.type') + type: config.get('ES.DOCUMENTS.attributegroup.type'), + enrich: { + policyName: 'attributegroup-policy', + matchField: 'id', + enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy'] + }, + pipeline: { + id: 'attributegroup-pipeline', + field: 'attributeGroupId', + targetField: 'attributegroup', + maxMatches: '1' + } }, + skill: { index: config.get('ES.DOCUMENTS.skill.index'), - enrichPolicy: 'skill-policy', - type: config.get('ES.DOCUMENTS.skill.type') + type: config.get('ES.DOCUMENTS.skill.type'), + enrich: { + policyName: 'skill-policy', + matchField: 'id', + enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider'] + }, + ingest: { + pipeline: { + id: 'skillprovider-pipeline' + } + } }, - skillprovider: { - index: config.get('ES.DOCUMENTS.skillprovider.index'), - type: config.get('ES.DOCUMENTS.skillprovider.type') + + attribute: { + index: config.get('ES.DOCUMENTS.attribute.index'), + type: config.get('ES.DOCUMENTS.attribute.type'), + enrich: { + policyName: 'attribute-policy', + matchField: 'id', + enrichFields: ['id', 'name', 'attributeGroupId', 'created', 'updated', 'createdBy', 'updatedBy', 'attributegroup'] + }, + ingest: { + pipeline: { + id: 'attributegroup-pipeline' + } + } }, + + organization: { + index: config.get('ES.DOCUMENTS.organization.index'), + type: config.get('ES.DOCUMENTS.organization.type'), + }, + user: { index: config.get('ES.DOCUMENTS.user.index'), - type: config.get('ES.DOCUMENTS.user.type') + type: config.get('ES.DOCUMENTS.user.type'), + pipeline: { + id: 'user-pipeline', + processors: [ + { + referenceField: config.get('ES.DOCUMENTS.achievement.userField'), + enrichPolicyName: 'achievementprovider-policy', + field: '_ingest._value.achievementsProviderId', + targetField: '_ingest._value.achievementprovider', + maxMatches: '1' + }, + { + referenceField: config.get('ES.DOCUMENTS.externalprofile.userField'), + enrichPolicyName: 'organization-policy', + field: '_ingest._value.organizationId', + targetField: '_ingest._value.organization', + maxMatches: '1' + }, + { + referenceField: config.get('ES.DOCUMENTS.userattribute.userField'), + enrichPolicyName: 'attribute-policy', + field: '_ingest._value.attributeId', + targetField: '_ingest._value.attribute', + maxMatches: '1' + }, + { + referenceField: config.get('ES.DOCUMENTS.userrole.userField'), + enrichPolicyName: 'role-policy', + field: '_ingest._value.roleId', + targetField: '_ingest._value.role', + maxMatches: '1' + }, + { + referenceField: config.get('ES.DOCUMENTS.userskill.userField'), + enrichPolicyName: 'skill-policy', + field: '_ingest._value.skillId', + targetField: '_ingest._value.skill', + maxMatches: '1' + } + ] + } } } @@ -72,7 +166,12 @@ const userResources = { const organizationResources = { organizationskillprovider: { propertyName: config.get('ES.DOCUMENTS.organizationskillprovider.orgField'), - relateKey: 'skillProviderId' + relateKey: 'skillProviderId', + enrich: { + policyName: 'organization-policy', + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy', 'skillProviders'] + } } } diff --git a/scripts/db/genData.js b/scripts/db/genData.js index e11dca9..b1cfa0e 100644 --- a/scripts/db/genData.js +++ b/scripts/db/genData.js @@ -10,6 +10,26 @@ const { modelToESIndexMapping } = require('../constants') +// Declares the ordering of the resource data insertion, to ensure that enrichment happens correctly +const RESOURCES_IN_ORDER = [ + 'skillprovider', + 'role', + 'achievementprovider', + 'attributegroup', + 'skill', + 'attribute', + 'organization', + 'organizationskillprovider', + 'user', + 'userskill', + 'achievement', + 'userrole', + 'externalprofile', + 'userattribute' +] + +const client = getESClient() + async function insertIntoES (modelName, body) { const esResourceName = modelToESIndexMapping[modelName] @@ -19,19 +39,15 @@ async function insertIntoES (modelName, body) { return } - const client = getESClient() - if (_.includes(_.keys(topResources), esResourceName)) { - await client.create({ + await client.index({ index: topResources[esResourceName].index, type: topResources[esResourceName].type, id: body.id, body, - refresh: 'true' + pipeline: topResources[esResourceName].ingest ? topResources[esResourceName].ingest.pipeline.id : undefined, + refresh: 'wait_for' }) - if (topResources[esResourceName].enrichPolicy) { - await client.enrich.executePolicy({ name: topResources[esResourceName].enrichPolicy }) - } } else if (_.includes(_.keys(userResources), esResourceName)) { const userResource = userResources[esResourceName] @@ -72,7 +88,7 @@ async function insertIntoES (modelName, body) { type: topResources.user.type, id: body.userId, body: user, - pipeline: config.get('ES.ENRICH_USER_PIPELINE_NAME'), + pipeline: topResources.user.pipeline.id, refresh: 'wait_for' }) } @@ -95,12 +111,102 @@ async function insertIntoES (modelName, body) { logger.error(`Can't create existing ${esResourceName} with the ${orgResource.relateKey}: ${relateId}, organizationId: ${body.organizationId}`) } else { organization[orgResource.propertyName].push(body) - await client.update({ + await client.index({ index: topResources.organization.index, type: topResources.organization.type, id: body.organizationId, - body: { doc: organization }, - refresh: 'true' + body: organization, + refresh: 'wait_for' + }) + } + } +} + +/** + * Creates and executes the enrich policy for the provided model + * @param {String} modelName The model name + */ +async function createAndExecuteEnrichPolicy (modelName) { + const esResourceName = modelToESIndexMapping[modelName] + + if (_.includes(_.keys(topResources), esResourceName) && topResources[esResourceName].enrich) { + await client.enrich.putPolicy({ + name: topResources[esResourceName].enrich.policyName, + body: { + match: { + indices: topResources[esResourceName].index, + match_field: topResources[esResourceName].enrich.matchField, + enrich_fields: topResources[esResourceName].enrich.enrichFields + } + } + }) + await client.enrich.executePolicy({ name: topResources[esResourceName].enrich.policyName }) + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + // For organization, execute enrich policy AFTER the sub documents on the org (namely orgskillprovider) is in + // This is because external profile on user is enriched with org, and it needs to have the orgskillprovider details in it + await client.enrich.putPolicy({ + name: organizationResources[esResourceName].enrich.policyName, + body: { + match: { + indices: topResources.organization.index, + match_field: organizationResources[esResourceName].enrich.matchField, + enrich_fields: organizationResources[esResourceName].enrich.enrichFields + } + } + }) + await client.enrich.executePolicy({ name: organizationResources[esResourceName].enrich.policyName }) + } +} + +/** + * Creates the ingest pipeline using the enrich policy + * @param {String} modelName The model name + */ +async function createEnrichProcessor (modelName) { + const esResourceName = modelToESIndexMapping[modelName] + + if (_.includes(_.keys(topResources), esResourceName) && topResources[esResourceName].pipeline) { + if (topResources[esResourceName].pipeline.processors) { + const processors = [] + + for (let i = 0; i < topResources[esResourceName].pipeline.processors.length; i++) { + const ep = topResources[esResourceName].pipeline.processors[i] + processors.push({ + foreach: { + field: ep.referenceField, + ignore_missing: true, + processor: { + enrich: { + policy_name: ep.enrichPolicyName, + ignore_missing: true, + field: ep.field, + target_field: ep.targetField, + max_matches: ep.maxMatches + } + } + } + }) + } + + await client.ingest.putPipeline({ + id: topResources[esResourceName].pipeline.id, + body: { + processors + } + }) + } else { + await client.ingest.putPipeline({ + id: topResources[esResourceName].pipeline.id, + body: { + processors: [{ + enrich: { + policy_name: topResources[esResourceName].enrich.policyName, + field: topResources[esResourceName].pipeline.field, + target_field: topResources[esResourceName].pipeline.targetField, + max_matches: topResources[esResourceName].pipeline.maxMatches + } + }] + } }) } } @@ -114,32 +220,49 @@ async function main () { await models.init() let keys = Object.keys(models) - keys = _.orderBy(keys, k => { - const esResourceName = modelToESIndexMapping[k] - // Create parent data first - if (_.includes(_.keys(topResources), esResourceName)) { - return -1 + // Sort the models in the order of insertion (for correct enrichment) + const temp = Array(keys.length).fill(null) + keys.forEach(k => { + if (models[k].tableName) { + const esResourceName = modelToESIndexMapping[k] + const index = RESOURCES_IN_ORDER.indexOf(esResourceName) + temp[index] = k } - - return 1 }) + keys = _.compact(temp) for (let i = 0; i < keys.length; i++) { const key = keys[i] - if (models[key].tableName) { - try { - const data = require(`./data/${key}.json`) - await models.DBHelper.clear(models[key]) - for (let i = 0; i < data.length; i++) { - await models.DBHelper.save(models[key], new models[key]().from(data[i]), true) - await insertIntoES(key, data[i]) - } - logger.info('import data for ' + key + ' done') - } catch (e) { - logger.error(e) - logger.warn('import data for ' + key + ' failed') + try { + const data = require(`./data/${key}.json`) + await models.DBHelper.clear(models[key]) + for (let i = 0; i < data.length; i++) { + logger.info(`Inserting data ${i + 1} of ${data.length}`) + await models.DBHelper.save(models[key], new models[key]().from(data[i]), true) + await insertIntoES(key, data[i]) } + logger.info('import data for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('import data for ' + key + ' failed') + continue + } + + try { + await createAndExecuteEnrichPolicy(key) + logger.info('create and execute enrich policy for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('create and execute enrich policy for ' + key + ' failed') + } + + try { + await createEnrichProcessor(key) + logger.info('create enrich processor (pipeline) for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('create enrich processor (pipeline) for ' + key + ' failed') } } logger.info('all done') From 7f7a6cb007221349fcd110de57e9f0fc35481240 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Wed, 21 Oct 2020 19:06:15 +0530 Subject: [PATCH 07/15] Working db dump script and misc --- README.md | 3 + config/default.js | 10 +- scripts/constants.js | 12 +- scripts/db/dropAll.js | 41 ++++++- scripts/db/dumpDbToEs.js | 225 +++++++++++++++++++++++++++++------ scripts/db/genData.js | 1 - src/common/es-helper.js | 8 -- src/common/service-helper.js | 4 - 8 files changed, 243 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index d4bb982..861c35c 100755 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ Configuration for the application is at config/default.js and config/production. - UBAHN_AGGREGATE_TOPIC: Kafka topic that is used to combine all create, update and delete message(s) - ES.HOST: Elasticsearch host - ES.DOCUMENTS: Elasticsearch index, type and id mapping for resources. +- ATTRIBUTE_GROUP_PIPELINE_ID: The pipeline id for enrichment with attribute group. Default is `attributegroup-pipeline` +- SKILL_PROVIDER_PIPELINE_ID: The pipeline id for enrichment with skill provider. Default is `skillprovider-pipeline` +- USER_PIPELINE_ID: The pipeline id for enrichment of user details. Default is `user-pipeline` For `ES.DOCUMENTS` configuration, you will find multiple other configurations below it. Each has default values that you can override using the environment variables diff --git a/config/default.js b/config/default.js index e824552..c5e9f3a 100755 --- a/config/default.js +++ b/config/default.js @@ -51,7 +51,6 @@ module.exports = { // ElasticSearch ES: { HOST: process.env.ES_HOST || 'http://localhost:9200', - ENRICH_USER_PIPELINE_NAME: process.env.ENRICH_USER_PIPELINE_NAME || 'enrich_user', // es mapping: _index, _type, _id DOCUMENTS: { achievementprovider: { @@ -64,7 +63,8 @@ module.exports = { }, attributegroup: { index: process.env.ATTRIBUTE_GROUP_INDEX || 'attribute_group', - type: '_doc' + type: '_doc', + pipelineId: process.env.ATTRIBUTE_GROUP_PIPELINE_ID || 'attributegroup-pipeline' }, organization: { index: process.env.ORGANIZATION_INDEX || 'organization', @@ -80,11 +80,13 @@ module.exports = { }, skillprovider: { index: process.env.SKILL_PROVIDER_INDEX || 'skill_provider', - type: '_doc' + type: '_doc', + pipelineId: process.env.SKILL_PROVIDER_PIPELINE_ID || 'skillprovider-pipeline' }, user: { index: process.env.USER_INDEX || 'user', - type: '_doc' + type: '_doc', + pipelineId: process.env.USER_PIPELINE_ID || 'user-pipeline' }, // sub resources under user achievement: { diff --git a/scripts/constants.js b/scripts/constants.js index 1b0a5b3..de5d6af 100644 --- a/scripts/constants.js +++ b/scripts/constants.js @@ -15,7 +15,7 @@ const topResources = { enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] }, pipeline: { - id: 'skillprovider-pipeline', + id: config.get('ES.DOCUMENTS.skillprovider.pipelineId'), field: 'skillProviderId', targetField: 'skillprovider', maxMatches: '1' @@ -51,7 +51,7 @@ const topResources = { enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy'] }, pipeline: { - id: 'attributegroup-pipeline', + id: config.get('ES.DOCUMENTS.attributegroup.pipelineId'), field: 'attributeGroupId', targetField: 'attributegroup', maxMatches: '1' @@ -68,7 +68,7 @@ const topResources = { }, ingest: { pipeline: { - id: 'skillprovider-pipeline' + id: config.get('ES.DOCUMENTS.skillprovider.pipelineId') } } }, @@ -83,21 +83,21 @@ const topResources = { }, ingest: { pipeline: { - id: 'attributegroup-pipeline' + id: config.get('ES.DOCUMENTS.attributegroup.pipelineId') } } }, organization: { index: config.get('ES.DOCUMENTS.organization.index'), - type: config.get('ES.DOCUMENTS.organization.type'), + type: config.get('ES.DOCUMENTS.organization.type') }, user: { index: config.get('ES.DOCUMENTS.user.index'), type: config.get('ES.DOCUMENTS.user.type'), pipeline: { - id: 'user-pipeline', + id: config.get('ES.DOCUMENTS.user.pipelineId'), processors: [ { referenceField: config.get('ES.DOCUMENTS.achievement.userField'), diff --git a/scripts/db/dropAll.js b/scripts/db/dropAll.js index cef1924..a30a2cf 100644 --- a/scripts/db/dropAll.js +++ b/scripts/db/dropAll.js @@ -4,11 +4,33 @@ const _ = require('lodash') const models = require('../../src/models') const logger = require('../../src/common/logger') -const { topResources, modelToESIndexMapping } = require('../constants') +const { + topResources, + organizationResources, + modelToESIndexMapping +} = require('../constants') const { getESClient } = require('../../src/common/es-client') async function main () { const client = getESClient() + + try { + logger.info('Deleting all pipelines...') + await client.ingest.deletePipeline({ + id: topResources.user.pipeline.id + }) + await client.ingest.deletePipeline({ + id: topResources.skillprovider.pipeline.id + }) + await client.ingest.deletePipeline({ + id: topResources.attributegroup.pipeline.id + }) + logger.info('Successfully deleted') + } catch (e) { + console.error(e) + logger.warn('Delete all ingest pipelines failed') + } + const keys = Object.keys(models) for (let i = 0; i < keys.length; i++) { const key = keys[i] @@ -16,12 +38,29 @@ async function main () { const esResourceName = modelToESIndexMapping[key] try { if (_.includes(_.keys(topResources), esResourceName)) { + if (topResources[esResourceName].enrich) { + logger.info(`Deleting enrich policy for ${esResourceName}`) + await client.enrich.deletePolicy({ + name: topResources[esResourceName].enrich.policyName + }) + logger.info(`Successfully deleted enrich policy for ${esResourceName}`) + } + logger.info(`Deleting index for ${esResourceName}`) await client.indices.delete({ index: topResources[esResourceName].index }) + logger.info(`Successfully deleted enrich policy for ${esResourceName}`) + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + logger.info('Deleting enrich policy for organization') + await client.enrich.deletePolicy({ + name: organizationResources[esResourceName].enrich.policyName + }) + logger.info('Successfully deleted enrich policy for organization') } + logger.info(`Deleting data in QLDB for ${esResourceName}`) await models.DBHelper.clear(models[key]) + logger.info(`Successfully deleted data in QLDB for ${esResourceName}`) } catch (e) { console.error(e) logger.warn(`drop table ${key} failed`) diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index 0d12e73..c5a23fd 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -1,5 +1,4 @@ const _ = require('lodash') -const config = require('config') const models = require('../../src/models') const logger = require('../../src/common/logger') const { getESClient } = require('../../src/common/es-client') @@ -10,14 +9,62 @@ const { modelToESIndexMapping } = require('../constants') -async function cleanupES () { - const client = getESClient() +// Declares the ordering of the resource data insertion, to ensure that enrichment happens correctly +const RESOURCES_IN_ORDER = [ + 'skillprovider', + 'role', + 'achievementprovider', + 'attributegroup', + 'skill', + 'attribute', + 'organization', + 'organizationskillprovider', + 'user', + 'userskill', + 'achievement', + 'userrole', + 'externalprofile', + 'userattribute' +] - await client.indices.delete({ - index: '_all' - }) +const client = getESClient() - console.log('Existing indices have been deleted!') +/** + * Cleans up the data in elasticsearch + * @param {Array} keys Array of models + */ +async function cleanupES (keys) { + const client = getESClient() + await client.ingest.deletePipeline({ + id: topResources.user.pipeline.id + }) + await client.ingest.deletePipeline({ + id: topResources.skillprovider.pipeline.id + }) + await client.ingest.deletePipeline({ + id: topResources.attributegroup.pipeline.id + }) + for (let i = 0; i < keys.length; i++) { + const key = keys[i] + if (models[key].tableName) { + const esResourceName = modelToESIndexMapping[key] + if (_.includes(_.keys(topResources), esResourceName)) { + if (topResources[esResourceName].enrich) { + await client.enrich.deletePolicy({ + name: topResources[esResourceName].enrich.policyName + }) + } + await client.indices.delete({ + index: topResources[esResourceName].index + }) + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + await client.enrich.deletePolicy({ + name: organizationResources[esResourceName].enrich.policyName + }) + } + } + } + console.log('Existing data in elasticsearch has been deleted!') } async function insertIntoES (modelName, body) { @@ -29,19 +76,15 @@ async function insertIntoES (modelName, body) { return } - const client = getESClient() - if (_.includes(_.keys(topResources), esResourceName)) { - await client.create({ + await client.index({ index: topResources[esResourceName].index, type: topResources[esResourceName].type, id: body.id, body, - refresh: 'true' + pipeline: topResources[esResourceName].ingest ? topResources[esResourceName].ingest.pipeline.id : undefined, + refresh: 'wait_for' }) - if (topResources[esResourceName].enrichPolicy) { - await client.enrich.executePolicy({ name: topResources[esResourceName].enrichPolicy }) - } } else if (_.includes(_.keys(userResources), esResourceName)) { const userResource = userResources[esResourceName] @@ -82,8 +125,8 @@ async function insertIntoES (modelName, body) { type: topResources.user.type, id: body.userId, body: user, - pipeline: config.get('ES.ENRICH_USER_PIPELINE_NAME'), - refresh: 'true' + pipeline: topResources.user.pipeline.id, + refresh: 'wait_for' }) } } else if (_.includes(_.keys(organizationResources), esResourceName)) { @@ -105,12 +148,102 @@ async function insertIntoES (modelName, body) { logger.error(`Can't create existing ${esResourceName} with the ${orgResource.relateKey}: ${relateId}, organizationId: ${body.organizationId}`) } else { organization[orgResource.propertyName].push(body) - await client.update({ + await client.index({ index: topResources.organization.index, type: topResources.organization.type, id: body.organizationId, - body: { doc: organization }, - refresh: 'true' + body: organization, + refresh: 'wait_for' + }) + } + } +} + +/** + * Creates and executes the enrich policy for the provided model + * @param {String} modelName The model name + */ +async function createAndExecuteEnrichPolicy (modelName) { + const esResourceName = modelToESIndexMapping[modelName] + + if (_.includes(_.keys(topResources), esResourceName) && topResources[esResourceName].enrich) { + await client.enrich.putPolicy({ + name: topResources[esResourceName].enrich.policyName, + body: { + match: { + indices: topResources[esResourceName].index, + match_field: topResources[esResourceName].enrich.matchField, + enrich_fields: topResources[esResourceName].enrich.enrichFields + } + } + }) + await client.enrich.executePolicy({ name: topResources[esResourceName].enrich.policyName }) + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + // For organization, execute enrich policy AFTER the sub documents on the org (namely orgskillprovider) is in + // This is because external profile on user is enriched with org, and it needs to have the orgskillprovider details in it + await client.enrich.putPolicy({ + name: organizationResources[esResourceName].enrich.policyName, + body: { + match: { + indices: topResources.organization.index, + match_field: organizationResources[esResourceName].enrich.matchField, + enrich_fields: organizationResources[esResourceName].enrich.enrichFields + } + } + }) + await client.enrich.executePolicy({ name: organizationResources[esResourceName].enrich.policyName }) + } +} + +/** + * Creates the ingest pipeline using the enrich policy + * @param {String} modelName The model name + */ +async function createEnrichProcessor (modelName) { + const esResourceName = modelToESIndexMapping[modelName] + + if (_.includes(_.keys(topResources), esResourceName) && topResources[esResourceName].pipeline) { + if (topResources[esResourceName].pipeline.processors) { + const processors = [] + + for (let i = 0; i < topResources[esResourceName].pipeline.processors.length; i++) { + const ep = topResources[esResourceName].pipeline.processors[i] + processors.push({ + foreach: { + field: ep.referenceField, + ignore_missing: true, + processor: { + enrich: { + policy_name: ep.enrichPolicyName, + ignore_missing: true, + field: ep.field, + target_field: ep.targetField, + max_matches: ep.maxMatches + } + } + } + }) + } + + await client.ingest.putPipeline({ + id: topResources[esResourceName].pipeline.id, + body: { + processors + } + }) + } else { + await client.ingest.putPipeline({ + id: topResources[esResourceName].pipeline.id, + body: { + processors: [{ + enrich: { + policy_name: topResources[esResourceName].enrich.policyName, + field: topResources[esResourceName].pipeline.field, + target_field: topResources[esResourceName].pipeline.targetField, + max_matches: topResources[esResourceName].pipeline.maxMatches + } + }] + } }) } } @@ -122,32 +255,50 @@ async function insertIntoES (modelName, body) { */ async function main () { let keys = Object.keys(models) - keys = _.orderBy(keys, k => { - const esResourceName = modelToESIndexMapping[k] - // Create parent data first - if (_.includes(_.keys(topResources), esResourceName)) { - return -1 + // Sort the models in the order of insertion (for correct enrichment) + const temp = Array(keys.length).fill(null) + keys.forEach(k => { + if (models[k].tableName) { + const esResourceName = modelToESIndexMapping[k] + const index = RESOURCES_IN_ORDER.indexOf(esResourceName) + temp[index] = k } - - return 1 }) + keys = _.compact(temp) - await cleanupES() + await cleanupES(keys) for (let i = 0; i < keys.length; i++) { const key = keys[i] - if (models[key].tableName) { - try { - const data = await models.DBHelper.find(models[key], []) - for (let i = 0; i < data.length; i++) { - await insertIntoES(key, data[i]) - } - logger.info('import data for ' + key + ' done') - } catch (e) { - logger.error(e) - logger.warn('import data for ' + key + ' failed') + try { + const data = await models.DBHelper.find(models[key], []) + + for (let i = 0; i < data.length; i++) { + logger.info(`Inserting data ${i + 1} of ${data.length}`) + await insertIntoES(key, data[i]) } + logger.info('import data for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('import data for ' + key + ' failed') + continue + } + + try { + await createAndExecuteEnrichPolicy(key) + logger.info('create and execute enrich policy for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('create and execute enrich policy for ' + key + ' failed') + } + + try { + await createEnrichProcessor(key) + logger.info('create enrich processor (pipeline) for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('create enrich processor (pipeline) for ' + key + ' failed') } } logger.info('all done') diff --git a/scripts/db/genData.js b/scripts/db/genData.js index b1cfa0e..97681ac 100644 --- a/scripts/db/genData.js +++ b/scripts/db/genData.js @@ -1,5 +1,4 @@ const _ = require('lodash') -const config = require('config') const models = require('../../src/models') const logger = require('../../src/common/logger') const { getESClient } = require('../../src/common/es-client') diff --git a/src/common/es-helper.js b/src/common/es-helper.js index 581ce3f..3cdf833 100644 --- a/src/common/es-helper.js +++ b/src/common/es-helper.js @@ -215,13 +215,11 @@ const FILTER_CHAIN = { skill: { filterNext: 'userskill', queryField: 'skillId', - enrichNext: 'skillprovider', idField: 'skillProviderId' }, attribute: { filterNext: 'userattribute', queryField: 'attributeId', - enrichNext: 'attributegroup', idField: 'attributeGroupId' }, attributegroup: { @@ -247,29 +245,23 @@ const FILTER_CHAIN = { // sub resource userskill: { queryField: 'skillId', - enrichNext: 'skill', idField: 'skillId' }, userrole: { queryField: 'roleId', - enrichNext: 'role', idField: 'roleId' }, externalprofile: { - enrichNext: 'organization', idField: 'organizationId' }, achievement: { - enrichNext: 'achievementprovider', idField: 'achievementsProviderId' }, userattribute: { - enrichNext: 'attribute', idField: 'attributeId' }, organizationskillprovider: { queryField: 'skillProviderId', - enrichNext: 'skillprovider', idField: 'skillProviderId' } } diff --git a/src/common/service-helper.js b/src/common/service-helper.js index 072a0b8..56870e5 100644 --- a/src/common/service-helper.js +++ b/src/common/service-helper.js @@ -259,10 +259,6 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil try { return await esHelper.searchElasticSearch(resource, query, auth) } catch (err) { - // return error if enrich fails - if (resource === 'user' && query.enrich) { - throw errors.elasticSearchEnrichError(err.message) - } logger.logFullError(err) } From 8ac769bc4604e866977dfc8295c881dbc5c70fca Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Wed, 21 Oct 2020 19:12:41 +0530 Subject: [PATCH 08/15] Restore deleted code --- src/common/service-helper.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common/service-helper.js b/src/common/service-helper.js index 56870e5..072a0b8 100644 --- a/src/common/service-helper.js +++ b/src/common/service-helper.js @@ -259,6 +259,10 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil try { return await esHelper.searchElasticSearch(resource, query, auth) } catch (err) { + // return error if enrich fails + if (resource === 'user' && query.enrich) { + throw errors.elasticSearchEnrichError(err.message) + } logger.logFullError(err) } From 310adc9fdb564212799ddc5849c7a1396aeee569 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Thu, 22 Oct 2020 14:41:43 +0530 Subject: [PATCH 09/15] 1. Set the enrich policy name as a config 2. Fix issue where migrate data from db to es would not save boolean fields 3. Fix issue where updating skill or attribute would result in referenced fields to also be passed in bus api --- README.md | 6 +++ config/default.js | 21 +++++--- scripts/constants.js | 14 +++--- scripts/db/dumpDbToEs.js | 98 +++++++++++++++++++++++++++--------- src/common/helper.js | 3 ++ src/common/service-helper.js | 29 ++++++----- 6 files changed, 119 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 861c35c..a0a2dc4 100755 --- a/README.md +++ b/README.md @@ -40,6 +40,12 @@ Configuration for the application is at config/default.js and config/production. - ATTRIBUTE_GROUP_PIPELINE_ID: The pipeline id for enrichment with attribute group. Default is `attributegroup-pipeline` - SKILL_PROVIDER_PIPELINE_ID: The pipeline id for enrichment with skill provider. Default is `skillprovider-pipeline` - USER_PIPELINE_ID: The pipeline id for enrichment of user details. Default is `user-pipeline` +- ATTRIBUTE_GROUP_ENRICH_POLICYNAME: The enrich policy for attribute group. Default is `attributegroup-policy` +- SKILL_PROVIDER_ENRICH_POLICYNAME: The enrich policy for skill provider. Default is `skillprovider-policy` +- ROLE_ENRICH_POLICYNAME: The enrich policy for role. Default is `role-policy` +- ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME: The enrich policy for achievement provider. Default is `achievementprovider-policy` +- SKILL_ENRICH_POLICYNAME: The enrich policy for skill. Default is `skill-policy` +- ATTRIBUTE_ENRICH_POLICYNAME: The enrich policy for skill. Default is `attribute-policy` For `ES.DOCUMENTS` configuration, you will find multiple other configurations below it. Each has default values that you can override using the environment variables diff --git a/config/default.js b/config/default.js index c5e9f3a..66ed133 100755 --- a/config/default.js +++ b/config/default.js @@ -55,33 +55,40 @@ module.exports = { DOCUMENTS: { achievementprovider: { index: process.env.ACHIEVEMENT_PROVIDER_INDEX || 'achievement_provider', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME || 'achievementprovider-policy' }, attribute: { index: process.env.ATTRIBUTE_INDEX || 'attribute', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.ATTRIBUTE_ENRICH_POLICYNAME || 'attribute-policy' }, attributegroup: { index: process.env.ATTRIBUTE_GROUP_INDEX || 'attribute_group', type: '_doc', - pipelineId: process.env.ATTRIBUTE_GROUP_PIPELINE_ID || 'attributegroup-pipeline' + pipelineId: process.env.ATTRIBUTE_GROUP_PIPELINE_ID || 'attributegroup-pipeline', + enrichPolicyName: process.env.ATTRIBUTE_GROUP_ENRICH_POLICYNAME || 'attributegroup-policy' }, organization: { index: process.env.ORGANIZATION_INDEX || 'organization', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.ORGANIZATION_ENRICH_POLICYNAME || 'organization-policy' }, role: { index: process.env.ROLE_INDEX || 'role', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.ROLE_ENRICH_POLICYNAME || 'role-policy' }, skill: { index: process.env.SKILL_INDEX || 'skill', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.SKILL_ENRICH_POLICYNAME || 'skill-policy' }, skillprovider: { index: process.env.SKILL_PROVIDER_INDEX || 'skill_provider', type: '_doc', - pipelineId: process.env.SKILL_PROVIDER_PIPELINE_ID || 'skillprovider-pipeline' + pipelineId: process.env.SKILL_PROVIDER_PIPELINE_ID || 'skillprovider-pipeline', + enrichPolicyName: process.env.SKILL_PROVIDER_ENRICH_POLICYNAME || 'skillprovider-policy' }, user: { index: process.env.USER_INDEX || 'user', diff --git a/scripts/constants.js b/scripts/constants.js index de5d6af..c5d976c 100644 --- a/scripts/constants.js +++ b/scripts/constants.js @@ -10,7 +10,7 @@ const topResources = { index: config.get('ES.DOCUMENTS.skillprovider.index'), type: config.get('ES.DOCUMENTS.skillprovider.type'), enrich: { - policyName: 'skillprovider-policy', + policyName: config.get('ES.DOCUMENTS.skillprovider.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] }, @@ -26,7 +26,7 @@ const topResources = { index: config.get('ES.DOCUMENTS.role.index'), type: config.get('ES.DOCUMENTS.role.type'), enrich: { - policyName: 'role-policy', + policyName: config.get('ES.DOCUMENTS.role.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] } @@ -36,7 +36,7 @@ const topResources = { index: config.get('ES.DOCUMENTS.achievementprovider.index'), type: config.get('ES.DOCUMENTS.achievementprovider.type'), enrich: { - policyName: 'achievementprovider-policy', + policyName: config.get('ES.DOCUMENTS.achievementprovider.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] } @@ -46,7 +46,7 @@ const topResources = { index: config.get('ES.DOCUMENTS.attributegroup.index'), type: config.get('ES.DOCUMENTS.attributegroup.type'), enrich: { - policyName: 'attributegroup-policy', + policyName: config.get('ES.DOCUMENTS.attributegroup.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy'] }, @@ -62,7 +62,7 @@ const topResources = { index: config.get('ES.DOCUMENTS.skill.index'), type: config.get('ES.DOCUMENTS.skill.type'), enrich: { - policyName: 'skill-policy', + policyName: config.get('ES.DOCUMENTS.skill.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider'] }, @@ -77,7 +77,7 @@ const topResources = { index: config.get('ES.DOCUMENTS.attribute.index'), type: config.get('ES.DOCUMENTS.attribute.type'), enrich: { - policyName: 'attribute-policy', + policyName: config.get('ES.DOCUMENTS.attribute.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'name', 'attributeGroupId', 'created', 'updated', 'createdBy', 'updatedBy', 'attributegroup'] }, @@ -168,7 +168,7 @@ const organizationResources = { propertyName: config.get('ES.DOCUMENTS.organizationskillprovider.orgField'), relateKey: 'skillProviderId', enrich: { - policyName: 'organization-policy', + policyName: config.get('ES.DOCUMENTS.organization.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy', 'skillProviders'] } diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index c5a23fd..006f0bb 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -29,40 +29,88 @@ const RESOURCES_IN_ORDER = [ const client = getESClient() +const RESOURCE_NOT_FOUND = 'resource_not_found_exception' +const INDEX_NOT_FOUND = 'index_not_found_exception' + /** * Cleans up the data in elasticsearch * @param {Array} keys Array of models */ async function cleanupES (keys) { const client = getESClient() - await client.ingest.deletePipeline({ - id: topResources.user.pipeline.id - }) - await client.ingest.deletePipeline({ - id: topResources.skillprovider.pipeline.id - }) - await client.ingest.deletePipeline({ - id: topResources.attributegroup.pipeline.id - }) - for (let i = 0; i < keys.length; i++) { - const key = keys[i] - if (models[key].tableName) { - const esResourceName = modelToESIndexMapping[key] - if (_.includes(_.keys(topResources), esResourceName)) { - if (topResources[esResourceName].enrich) { - await client.enrich.deletePolicy({ - name: topResources[esResourceName].enrich.policyName - }) + try { + await client.ingest.deletePipeline({ + id: topResources.user.pipeline.id + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } + + try { + await client.ingest.deletePipeline({ + id: topResources.skillprovider.pipeline.id + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } + + try { + await client.ingest.deletePipeline({ + id: topResources.attributegroup.pipeline.id + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } + + try { + for (let i = 0; i < keys.length; i++) { + const key = keys[i] + if (models[key].tableName) { + const esResourceName = modelToESIndexMapping[key] + if (_.includes(_.keys(topResources), esResourceName)) { + if (topResources[esResourceName].enrich) { + try { + await client.enrich.deletePolicy({ + name: topResources[esResourceName].enrich.policyName + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } + } + + try { + await client.indices.delete({ + index: topResources[esResourceName].index + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== INDEX_NOT_FOUND) { + throw e + } + } + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + try { + await client.enrich.deletePolicy({ + name: organizationResources[esResourceName].enrich.policyName + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } } - await client.indices.delete({ - index: topResources[esResourceName].index - }) - } else if (_.includes(_.keys(organizationResources), esResourceName)) { - await client.enrich.deletePolicy({ - name: organizationResources[esResourceName].enrich.policyName - }) } } + } catch (e) { + console.log(JSON.stringify(e)) + throw e } console.log('Existing data in elasticsearch has been deleted!') } diff --git a/src/common/helper.js b/src/common/helper.js index 604d0fc..56e269e 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -109,6 +109,9 @@ function readerToJson (reader) { toRealValue(r, setValue(name, [])) r.stepOut() break + case IonTypes.BOOL: + setValue(name, r.booleanValue()) + break } nextT = reader.next() } diff --git a/src/common/service-helper.js b/src/common/service-helper.js index 072a0b8..10eb361 100644 --- a/src/common/service-helper.js +++ b/src/common/service-helper.js @@ -190,7 +190,7 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil async function patch (id, entity, auth, params) { await makeSureRefExist(entity) - const dbEntity = await get(id, auth, params) + const dbEntity = await get(id, auth, params, {}, true) const newEntity = new Model() _.extend(newEntity, dbEntity, entity) newEntity.updated = new Date() @@ -218,23 +218,26 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil * @param auth the auth obj * @param params the path parameters * @param query the query parameters + * @param fromDb Should we bypass Elasticsearch for the record and fetch from db instead? * @return {Promise} the db device */ - async function get (id, auth, params, query = {}) { + async function get (id, auth, params, query = {}, fromDb = false) { let recordObj // Merge path and query params const trueParams = _.assign(params, query) - try { - const result = await esHelper.getFromElasticSearch(resource, id, auth, trueParams) - // check permission - permissionCheck(auth, result) - return result - } catch (err) { - // return error if enrich fails or permission fails - if ((resource === 'user' && trueParams.enrich) || (err.status && err.status === 403)) { - throw errors.elasticSearchEnrichError(err.message) + if (!fromDb) { + try { + const result = await esHelper.getFromElasticSearch(resource, id, auth, trueParams) + // check permission + permissionCheck(auth, result) + return result + } catch (err) { + // return error if enrich fails or permission fails + if ((resource === 'user' && trueParams.enrich) || (err.status && err.status === 403)) { + throw errors.elasticSearchEnrichError(err.message) + } + logger.logFullError(err) } - logger.logFullError(err) } if (_.isNil(trueParams) || _.isEmpty(trueParams)) { recordObj = await models.DBHelper.get(Model, id) @@ -302,7 +305,7 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil */ async function remove (id, auth, params) { let payload - await get(id, auth, params) // check exist + await get(id, auth, params, {}, true) // check exist await models.DBHelper.delete(Model, id, buildQueryByParams(params)) if (SUB_USER_DOCUMENTS[resource] || SUB_ORG_DOCUMENTS[resource]) { payload = _.assign({}, params) From 2f5269eee381249b5b3097b32a7a014f12da56c0 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Thu, 22 Oct 2020 23:07:37 +0530 Subject: [PATCH 10/15] Update elasticsearch host to support elastic cloud since we are using enrich feature, which is unsupported by AWS ES --- README.md | 5 ++++- config/default.js | 7 +++++++ package-lock.json | 8 -------- package.json | 1 - src/common/es-client.js | 15 ++++++++++----- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a0a2dc4..9e110f1 100755 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Configuration for the application is at config/default.js and config/production. - UBAHN_UPDATE_TOPIC: Kafka topic for update message - UBAHN_DELETE_TOPIC: Kafka topic for delete message - UBAHN_AGGREGATE_TOPIC: Kafka topic that is used to combine all create, update and delete message(s) -- ES.HOST: Elasticsearch host +- ES_HOST: Elasticsearch host - ES.DOCUMENTS: Elasticsearch index, type and id mapping for resources. - ATTRIBUTE_GROUP_PIPELINE_ID: The pipeline id for enrichment with attribute group. Default is `attributegroup-pipeline` - SKILL_PROVIDER_PIPELINE_ID: The pipeline id for enrichment with skill provider. Default is `skillprovider-pipeline` @@ -46,6 +46,9 @@ Configuration for the application is at config/default.js and config/production. - ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME: The enrich policy for achievement provider. Default is `achievementprovider-policy` - SKILL_ENRICH_POLICYNAME: The enrich policy for skill. Default is `skill-policy` - ATTRIBUTE_ENRICH_POLICYNAME: The enrich policy for skill. Default is `attribute-policy` +- ELASTICCLOUD_ID: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this +- ELASTICCLOUD_USERNAME: The elastic cloud username for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud +- ELASTICCLOUD_PASSWORD: The elastic cloud password for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud For `ES.DOCUMENTS` configuration, you will find multiple other configurations below it. Each has default values that you can override using the environment variables diff --git a/config/default.js b/config/default.js index 66ed133..2d48677 100755 --- a/config/default.js +++ b/config/default.js @@ -51,6 +51,13 @@ module.exports = { // ElasticSearch ES: { HOST: process.env.ES_HOST || 'http://localhost:9200', + + ELASTICCLOUD: { + id: process.env.ELASTICCLOUD_ID, + username: process.env.ELASTICCLOUD_USERNAME, + password: process.env.ELASTICCLOUD_PASSWORD + }, + // es mapping: _index, _type, _id DOCUMENTS: { achievementprovider: { diff --git a/package-lock.json b/package-lock.json index 2a51a95..a2b1144 100644 --- a/package-lock.json +++ b/package-lock.json @@ -354,14 +354,6 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, - "aws-elasticsearch-connector": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/aws-elasticsearch-connector/-/aws-elasticsearch-connector-9.0.0.tgz", - "integrity": "sha512-O+A9HEa14gOiKTAp6U6Ha1RRJvQjc046wIn9CJ69wc+c1c5CfPE4xl4Av6Zyv6dgzs+RVGxdetjm8RpSlTUmhQ==", - "requires": { - "aws4": "^1.10.0" - } - }, "aws-sdk": { "version": "2.668.0", "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.668.0.tgz", diff --git a/package.json b/package.json index ce8a809..b069f0d 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,6 @@ "@hapi/joi": "^16.1.8", "@hapi/joi-date": "^2.0.1", "amazon-qldb-driver-nodejs": "^0.1.1-preview.2", - "aws-elasticsearch-connector": "^9.0.0", "aws-sdk": "^2.627.0", "axios": "^0.19.2", "body-parser": "^1.19.0", diff --git a/src/common/es-client.js b/src/common/es-client.js index 20c1bb7..c5a52d7 100644 --- a/src/common/es-client.js +++ b/src/common/es-client.js @@ -1,7 +1,6 @@ const config = require('config') const AWS = require('aws-sdk') const elasticsearch = require('@elastic/elasticsearch') -const createAwsElasticsearchConnector = require('aws-elasticsearch-connector') AWS.config.region = config.AWS_REGION @@ -17,12 +16,18 @@ function getESClient () { return esClient } const host = config.ES.HOST + const cloudId = config.ES.ELASTICCLOUD.id if (!esClient) { - // AWS ES configuration is different from other providers - if (/.*amazonaws.*/.test(host)) { + if (cloudId) { + // Elastic Cloud configuration esClient = new elasticsearch.Client({ - ...createAwsElasticsearchConnector(AWS.config), - node: host + cloud: { + id: cloudId + }, + auth: { + username: config.ES.ELASTICCLOUD.username, + password: config.ES.ELASTICCLOUD.password + } }) } else { esClient = new elasticsearch.Client({ From 30062520c55d0406f1d971e1801bd5554585d380 Mon Sep 17 00:00:00 2001 From: CWD Date: Thu, 22 Oct 2020 15:10:32 -0400 Subject: [PATCH 11/15] touch --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9e110f1..5c3af8d 100755 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ Configuration for the application is at config/default.js and config/production. For `ES.DOCUMENTS` configuration, you will find multiple other configurations below it. Each has default values that you can override using the environment variables + ## Local deployment Setup your Elasticsearch instance and ensure that it is up and running. From e76171cb348b8f3889abb2f8eee7ebc2fd174a0e Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Fri, 23 Oct 2020 11:39:06 +0530 Subject: [PATCH 12/15] Debug why db to es migration script failed --- scripts/db/dumpDbToEs.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index 006f0bb..650fc51 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -319,15 +319,18 @@ async function main () { for (let i = 0; i < keys.length; i++) { const key = keys[i] + let temp try { const data = await models.DBHelper.find(models[key], []) for (let i = 0; i < data.length; i++) { + temp = data[i] logger.info(`Inserting data ${i + 1} of ${data.length}`) await insertIntoES(key, data[i]) } logger.info('import data for ' + key + ' done') } catch (e) { + logger.error(JSON.stringify(temp)) logger.error(e) logger.warn('import data for ' + key + ' failed') continue From 49aa2fbadefd3af561a503dbc3824c7a13334859 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Fri, 23 Oct 2020 13:10:10 +0530 Subject: [PATCH 13/15] more debugging --- .gitignore | 1 + scripts/db/dumpDbToEs.js | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 1fd8050..d31a0ba 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ scripts/generate .nyc_output .env coverage +.vscode diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index 650fc51..1101814 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -324,13 +324,12 @@ async function main () { const data = await models.DBHelper.find(models[key], []) for (let i = 0; i < data.length; i++) { - temp = data[i] + logger.info(JSON.stringify(data[i])) logger.info(`Inserting data ${i + 1} of ${data.length}`) await insertIntoES(key, data[i]) } logger.info('import data for ' + key + ' done') } catch (e) { - logger.error(JSON.stringify(temp)) logger.error(e) logger.warn('import data for ' + key + ' failed') continue From 354fedf7ef2d570fbda10044f29e24d2e87dc623 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Fri, 23 Oct 2020 18:13:07 +0530 Subject: [PATCH 14/15] Fix issue where data migration from db to es fails --- scripts/db/dumpDbToEs.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index 1101814..7164236 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -319,13 +319,18 @@ async function main () { for (let i = 0; i < keys.length; i++) { const key = keys[i] - let temp try { const data = await models.DBHelper.find(models[key], []) for (let i = 0; i < data.length; i++) { - logger.info(JSON.stringify(data[i])) logger.info(`Inserting data ${i + 1} of ${data.length}`) + logger.info(JSON.stringify(data[i])) + if (!_.isString(data[i].created)) { + data[i].created = new Date() + } + if (!_.isString(data[i].createdBy)) { + data[i].created = 'TonyJ' + } await insertIntoES(key, data[i]) } logger.info('import data for ' + key + ' done') From 6235c7f1c68af3562c52d68206c11769c001cec3 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Fri, 23 Oct 2020 20:40:28 +0530 Subject: [PATCH 15/15] Fix issue where data migration from db to es fails --- scripts/db/dumpDbToEs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index 7164236..9204c19 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -329,7 +329,7 @@ async function main () { data[i].created = new Date() } if (!_.isString(data[i].createdBy)) { - data[i].created = 'TonyJ' + data[i].createdBy = 'TonyJ' } await insertIntoES(key, data[i]) }