diff --git a/.gitignore b/.gitignore index 1fd8050..d31a0ba 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ scripts/generate .nyc_output .env coverage +.vscode diff --git a/README.md b/README.md index ce89626..5c3af8d 100755 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ - node 12.x - npm 6.x - docker -- elasticsearch 6.x +- elasticsearch 7.7 ## Configuration @@ -35,12 +35,24 @@ Configuration for the application is at config/default.js and config/production. - UBAHN_UPDATE_TOPIC: Kafka topic for update message - UBAHN_DELETE_TOPIC: Kafka topic for delete message - UBAHN_AGGREGATE_TOPIC: Kafka topic that is used to combine all create, update and delete message(s) -- ES.HOST: Elasticsearch host -- ES.API_VERSION: Elasticsearch API version +- ES_HOST: Elasticsearch host - ES.DOCUMENTS: Elasticsearch index, type and id mapping for resources. +- ATTRIBUTE_GROUP_PIPELINE_ID: The pipeline id for enrichment with attribute group. Default is `attributegroup-pipeline` +- SKILL_PROVIDER_PIPELINE_ID: The pipeline id for enrichment with skill provider. Default is `skillprovider-pipeline` +- USER_PIPELINE_ID: The pipeline id for enrichment of user details. Default is `user-pipeline` +- ATTRIBUTE_GROUP_ENRICH_POLICYNAME: The enrich policy for attribute group. Default is `attributegroup-policy` +- SKILL_PROVIDER_ENRICH_POLICYNAME: The enrich policy for skill provider. Default is `skillprovider-policy` +- ROLE_ENRICH_POLICYNAME: The enrich policy for role. Default is `role-policy` +- ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME: The enrich policy for achievement provider. Default is `achievementprovider-policy` +- SKILL_ENRICH_POLICYNAME: The enrich policy for skill. Default is `skill-policy` +- ATTRIBUTE_ENRICH_POLICYNAME: The enrich policy for skill. Default is `attribute-policy` +- ELASTICCLOUD_ID: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this +- ELASTICCLOUD_USERNAME: The elastic cloud username for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud +- ELASTICCLOUD_PASSWORD: The elastic cloud password for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud For `ES.DOCUMENTS` configuration, you will find multiple other configurations below it. Each has default values that you can override using the environment variables + ## Local deployment Setup your Elasticsearch instance and ensure that it is up and running. diff --git a/config/default.js b/config/default.js index b46f98b..2d48677 100755 --- a/config/default.js +++ b/config/default.js @@ -50,41 +50,57 @@ module.exports = { // ElasticSearch ES: { - HOST: process.env.ES_HOST || 'localhost:9200', - API_VERSION: process.env.ES_API_VERSION || '7.4', + HOST: process.env.ES_HOST || 'http://localhost:9200', + + ELASTICCLOUD: { + id: process.env.ELASTICCLOUD_ID, + username: process.env.ELASTICCLOUD_USERNAME, + password: process.env.ELASTICCLOUD_PASSWORD + }, + // es mapping: _index, _type, _id DOCUMENTS: { achievementprovider: { index: process.env.ACHIEVEMENT_PROVIDER_INDEX || 'achievement_provider', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME || 'achievementprovider-policy' }, attribute: { index: process.env.ATTRIBUTE_INDEX || 'attribute', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.ATTRIBUTE_ENRICH_POLICYNAME || 'attribute-policy' }, attributegroup: { index: process.env.ATTRIBUTE_GROUP_INDEX || 'attribute_group', - type: '_doc' + type: '_doc', + pipelineId: process.env.ATTRIBUTE_GROUP_PIPELINE_ID || 'attributegroup-pipeline', + enrichPolicyName: process.env.ATTRIBUTE_GROUP_ENRICH_POLICYNAME || 'attributegroup-policy' }, organization: { index: process.env.ORGANIZATION_INDEX || 'organization', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.ORGANIZATION_ENRICH_POLICYNAME || 'organization-policy' }, role: { index: process.env.ROLE_INDEX || 'role', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.ROLE_ENRICH_POLICYNAME || 'role-policy' }, skill: { index: process.env.SKILL_INDEX || 'skill', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.SKILL_ENRICH_POLICYNAME || 'skill-policy' }, skillprovider: { index: process.env.SKILL_PROVIDER_INDEX || 'skill_provider', - type: '_doc' + type: '_doc', + pipelineId: process.env.SKILL_PROVIDER_PIPELINE_ID || 'skillprovider-pipeline', + enrichPolicyName: process.env.SKILL_PROVIDER_ENRICH_POLICYNAME || 'skillprovider-policy' }, user: { index: process.env.USER_INDEX || 'user', - type: '_doc' + type: '_doc', + pipelineId: process.env.USER_PIPELINE_ID || 'user-pipeline' }, // sub resources under user achievement: { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index d8f7ba7..cd1de3d 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -3,7 +3,12 @@ swagger: "2.0" info: description: "API for an employee management system to determine employees that\ \ are no longer working on active projects and to understand their qualifications\ - \ and expertise for suitability in other projects" + \ and expertise for suitability in other projects + \n\n**Pagination**\n\n + Requests that return multiple items will be paginated to 20 items by default. You can specify + further pages with the `page` parameter. You can also set a custom page + size up to 100 with the `perPage` parameter. + \n\nPagination response data is included in http headers. By Default, the response header contains links with `next`, `last`, `first`, `prev` resource links.\n\n**Point to note** - the page attributes will have no effect if the data is fetched from db and not from es." version: "1.0.0" title: "UBahn API" host: "ubahn-api-dev.herokuapp.com" @@ -226,6 +231,28 @@ paths: type: "array" items: $ref: "#/definitions/User" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -411,6 +438,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -622,6 +671,28 @@ paths: type: "array" items: $ref: "#/definitions/UserSkill" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -655,6 +726,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -889,6 +982,28 @@ paths: type: "array" items: $ref: "#/definitions/Skill" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -922,6 +1037,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1118,6 +1255,28 @@ paths: type: "array" items: $ref: "#/definitions/SkillsProvider" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1145,6 +1304,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1340,6 +1521,28 @@ paths: type: "array" items: $ref: "#/definitions/Role" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1367,6 +1570,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1563,6 +1788,28 @@ paths: type: "array" items: $ref: "#/definitions/UserRole" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1593,6 +1840,28 @@ paths: responses: "200": description: "OK - the request was successful" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1790,6 +2059,28 @@ paths: type: "array" items: $ref: "#/definitions/ExternalProfile" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -1836,6 +2127,28 @@ paths: responses: "200": description: "OK - the request was successful" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2071,6 +2384,28 @@ paths: type: "array" items: $ref: "#/definitions/Achievement" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2106,6 +2441,28 @@ paths: responses: "200": description: "OK - the request was successful" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2338,6 +2695,28 @@ paths: type: "array" items: $ref: "#/definitions/AchievementsProvider" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2365,6 +2744,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2560,6 +2961,28 @@ paths: type: "array" items: $ref: "#/definitions/Organization" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2587,6 +3010,28 @@ paths: responses: "200": description: "success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2783,6 +3228,28 @@ paths: type: "array" items: $ref: "#/definitions/OrganizationSkillsProvider" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -2813,6 +3280,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3012,6 +3501,28 @@ paths: type: "array" items: $ref: "#/definitions/UserAttribute" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3056,6 +3567,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3287,6 +3820,28 @@ paths: type: "array" items: $ref: "#/definitions/Attribute" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3309,6 +3864,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3512,6 +4089,28 @@ paths: type: "array" items: $ref: "#/definitions/AttributeGroup" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3545,6 +4144,28 @@ paths: responses: "200": description: "Success response" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3747,6 +4368,28 @@ paths: type: "array" items: $ref: "#/definitions/EnhancedUser" + headers: + X-Next-Page: + type: integer + description: The index of the next page + X-Page: + type: integer + description: The index of the current page (starting at 1) + X-Per-Page: + type: integer + description: The number of items to list per page + X-Prev-Page: + type: integer + description: The index of the previous page + X-Total: + type: integer + description: The total number of items + X-Total-Pages: + type: integer + description: The total number of pages + Link: + type: string + description: Pagination link header. "400": $ref: "#/definitions/BadRequest" "401": @@ -3795,6 +4438,37 @@ paths: security: - Bearer: [] x-swagger-router-controller: "Search" + head: + tags: + - "Skill Search" + description: "Retrieve header information for a search operation on user attributes in the application. Multiple filters are supported.\n\n**Security** - Note\ + \ that for non-admin users, this endpoint will only return entities that\n\ + the user has created.\n" + operationId: "searchUserAttributesHEAD" + parameters: + - name: "attributeId" + in: "query" + description: "The attribute id" + type: "string" + format: "UUID" + - name: "attributeValue" + in: "query" + description: "The attribute value" + type: "string" + responses: + "200": + description: "OK - the request was successful" + "400": + $ref: "#/definitions/BadRequest" + "401": + $ref: "#/definitions/Unauthorized" + "403": + $ref: "#/definitions/Forbidden" + "500": + $ref: "#/definitions/ServerError" + security: + - Bearer: [] + x-swagger-router-controller: "Search" /skill-search/userAchievements: get: tags: @@ -3833,6 +4507,38 @@ paths: security: - Bearer: [] x-swagger-router-controller: "Search" + head: + tags: + - "Skill Search" + description: "Retrieve header information for a search operation on user achievements in the application. Multiple filters are\nsupported.\n\n**Security** - Note\ + \ that for non-admin users, this endpoint will only return entities that\n\ + the user has created.\n" + operationId: "searchUserAchievementsHEAD" + parameters: + - name: "organizationId" + in: "query" + description: "The organization id" + type: "string" + format: "UUID" + required: true + - name: "keyword" + in: "query" + description: "The query keyword" + type: "string" + responses: + "200": + description: "OK - the request was successful" + "400": + $ref: "#/definitions/BadRequest" + "401": + $ref: "#/definitions/Unauthorized" + "403": + $ref: "#/definitions/Forbidden" + "500": + $ref: "#/definitions/ServerError" + security: + - Bearer: [] + x-swagger-router-controller: "Search" /skill-search/skills: get: tags: @@ -3871,6 +4577,38 @@ paths: security: - Bearer: [] x-swagger-router-controller: "Search" + head: + tags: + - "Skill Search" + description: "Retrieve header information for a search operation on skills associated with an org in the application. Multiple filters are\nsupported.\n\n**Security** - Note\ + \ that for non-admin users, this endpoint will only return entities that\n\ + the user has created.\n" + operationId: "searchSkillsHEAD" + parameters: + - name: "organizationId" + in: "query" + description: "The organization id" + type: "string" + format: "UUID" + required: true + - name: "keyword" + in: "query" + description: "The query keyword" + type: "string" + responses: + "200": + description: "OK - the request was successful" + "400": + $ref: "#/definitions/BadRequest" + "401": + $ref: "#/definitions/Unauthorized" + "403": + $ref: "#/definitions/Forbidden" + "500": + $ref: "#/definitions/ServerError" + security: + - Bearer: [] + x-swagger-router-controller: "Search" securityDefinitions: Bearer: type: "apiKey" diff --git a/package-lock.json b/package-lock.json index 5a98387..a2b1144 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,33 @@ "js-tokens": "^4.0.0" } }, + "@elastic/elasticsearch": { + "version": "7.9.1", + "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.9.1.tgz", + "integrity": "sha512-NfPADbm9tRK/4ohpm9+aBtJ8WPKQqQaReyBKT225pi2oKQO1IzRlfM+OPplAvbhoH1efrSj1NKk27L+4BCrzXQ==", + "requires": { + "debug": "^4.1.1", + "decompress-response": "^4.2.0", + "ms": "^2.1.1", + "pump": "^3.0.0", + "secure-json-parse": "^2.1.0" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "@hapi/address": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", @@ -78,6 +105,11 @@ "@hapi/hoek": "^8.3.0" } }, + "@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" + }, "@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", @@ -102,9 +134,9 @@ } }, "@types/express": { - "version": "4.17.7", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.7.tgz", - "integrity": "sha512-dCOT5lcmV/uC2J9k0rPafATeeyz+99xTt54ReX11/LObZgfzJqZNcW27zGhYyX+9iSEGXGt5qLPwRSvBZcLvtQ==", + "version": "4.17.8", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.8.tgz", + "integrity": "sha512-wLhcKh3PMlyA2cNAB9sjM1BntnhPMiM0JOBwPBqttjHev2428MLEB4AYVN+d8s2iyCVZac+o41Pflm/ZH5vLXQ==", "requires": { "@types/body-parser": "*", "@types/express-serve-static-core": "*", @@ -122,9 +154,9 @@ } }, "@types/express-serve-static-core": { - "version": "4.17.8", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.8.tgz", - "integrity": "sha512-1SJZ+R3Q/7mLkOD9ewCBDYD2k0WyZQtWYqF/2VvoNN2/uhI49J9CDN4OAm+wGMA0DbArA4ef27xl4+JwMtGggw==", + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.13.tgz", + "integrity": "sha512-RgDi5a4nuzam073lRGKTUIaL3eF2+H7LJvJ8eUnCI0wA6SNjXc44DCmWNiTLs/AZ7QlsFWZiw/gTG3nSQGL0fA==", "requires": { "@types/node": "*", "@types/qs": "*", @@ -140,9 +172,9 @@ } }, "@types/mime": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.2.tgz", - "integrity": "sha512-4kPlzbljFcsttWEq6aBW0OZe6BDajAmyvr2xknBG92tejQnvdGtT9+kXSZ580DqpxY9qG2xeQVF9Dq0ymUTo5Q==" + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.3.tgz", + "integrity": "sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q==" }, "@types/node": { "version": "12.0.2", @@ -150,9 +182,9 @@ "integrity": "sha512-5tabW/i+9mhrfEOUcLDu2xBPsHJ+X5Orqy9FKpale3SjDA17j5AEpYq5vfy3oAeAHGcvANRCO3NV3d2D6q3NiA==" }, "@types/qs": { - "version": "6.9.3", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.3.tgz", - "integrity": "sha512-7s9EQWupR1fTc2pSMtXRQ9w9gLOcrJn+h7HOXw4evxyvVqMi4f+q7d2tnFe3ng3SNHjtK+0EzGMGFUQX4/AQRA==" + "version": "6.9.5", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.5.tgz", + "integrity": "sha512-/JHkVHtx/REVG0VVToGRGH2+23hsYLHdyG+GrvoUGlGAd0ErauXDyvHtRI/7H7mzLm+tBCKA7pfcpkQ1lf58iQ==" }, "@types/range-parser": { "version": "1.2.3", @@ -160,9 +192,9 @@ "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==" }, "@types/serve-static": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.4.tgz", - "integrity": "sha512-jTDt0o/YbpNwZbQmE/+2e+lfjJEJJR0I3OFaKQKPWkASkCoW3i6fsUnqudSMcNAfbtmADGu8f4MV4q+GqULmug==", + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.5.tgz", + "integrity": "sha512-6M64P58N+OXjU432WoLLBQxbA0LRGBCRm7aAGQJ+SMC1IMl0dgRVi9EFfoDcS2a7Xogygk/eGN94CfwU9UF7UQ==", "requires": { "@types/express-serve-static-core": "*", "@types/mime": "*" @@ -189,18 +221,34 @@ "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", "dev": true }, - "agentkeepalive": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", - "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", + "agent-base": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", + "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", "requires": { - "humanize-ms": "^1.2.1" + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } } }, "ajv": { "version": "6.12.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz", "integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==", + "dev": true, "requires": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -335,9 +383,9 @@ "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.0.tgz", - "integrity": "sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA==" + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.1.tgz", + "integrity": "sha512-zg7Hz2k5lI8kb7U32998pRRFin7zJlkfezGJjUc2heaD4Pw2wObakCDVzkKztTm/Ln7eiVvYsjqak0Ed4LkMDA==" }, "axios": { "version": "0.19.2", @@ -665,6 +713,14 @@ "integrity": "sha1-IwdjLUwEOCuN+KMvcLiVBG1SdF8=", "dev": true }, + "decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "requires": { + "mimic-response": "^2.0.0" + } + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", @@ -781,53 +837,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "elasticsearch": { - "version": "16.7.1", - "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.1.tgz", - "integrity": "sha512-PL/BxB03VGbbghJwISYvVcrR9KbSSkuQ7OM//jHJg/End/uC2fvXg4QI7RXLvCGbhBuNQ8dPue7DOOPra73PCw==", - "requires": { - "agentkeepalive": "^3.4.1", - "chalk": "^1.0.0", - "lodash": "^4.17.10" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - }, - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" - }, - "chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" - } - } - }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -847,6 +856,14 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } + }, "env-variable": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.6.tgz", @@ -907,7 +924,8 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true }, "eslint": { "version": "6.8.0", @@ -1516,12 +1534,25 @@ "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", "requires": { - "ajv": "^6.5.5", + "ajv": "^6.12.3", "har-schema": "^2.0.0" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + } } }, "has": { @@ -1533,21 +1564,6 @@ "function-bind": "^1.1.1" } }, - "has-ansi": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", - "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", - "requires": { - "ansi-regex": "^2.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - } - } - }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -1571,11 +1587,6 @@ "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==", "dev": true }, - "http-aws-es": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/http-aws-es/-/http-aws-es-6.0.0.tgz", - "integrity": "sha512-g+qp7J110/m4aHrR3iit4akAlnW0UljZ6oTq/rCcbsI8KP9x+95vqUtx49M2XQ2JMpwJio3B6gDYx+E8WDxqiA==" - }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -1588,6 +1599,31 @@ "toidentifier": "1.0.0" } }, + "http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "requires": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -1598,12 +1634,28 @@ "sshpk": "^1.7.0" } }, - "humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=", + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", "requires": { - "ms": "^2.0.0" + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } } }, "iconv-lite": { @@ -1992,13 +2044,15 @@ } }, "jwks-rsa": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-1.8.1.tgz", - "integrity": "sha512-CcE8ypsATHwGmzELwzeFjLzPBXTXTrMmDYbn92LTQwYsZdOedp+ZIuYTofUdrWreu8CKRuXmhk17+6/li2sR6g==", + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-1.10.1.tgz", + "integrity": "sha512-UmjOsATVu7eQr17wbBCS+BSoz5LFtl57PtNXHbHFeT1WKomHykCHtn7c8inWVI7tpnsy6CZ1KOMJTgipFwXPig==", "requires": { "@types/express-jwt": "0.0.42", "axios": "^0.19.2", "debug": "^4.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", "jsonwebtoken": "^8.5.1", "limiter": "^1.1.5", "lru-memoizer": "^2.1.2", @@ -2006,11 +2060,11 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -2209,6 +2263,11 @@ "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, + "mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==" + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -2258,9 +2317,9 @@ } }, "nan": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", - "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==", "optional": true }, "natural-compare": { @@ -2683,6 +2742,15 @@ "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -2915,6 +2983,11 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, + "secure-json-parse": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.1.0.tgz", + "integrity": "sha512-GckO+MS/wT4UogDyoI/H/S1L0MCcKS1XX/vp48wfmU7Nw4woBmb8mIpu4zPBQjKlRT88/bt9xdoV4111jPpNJA==" + }, "semaphore-async-await": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/semaphore-async-await/-/semaphore-async-await-1.5.1.tgz", @@ -3337,22 +3410,6 @@ "lodash": "^4.17.15", "superagent": "^3.8.3", "tc-core-library-js": "github:appirio-tech/tc-core-library-js#v2.6.4" - }, - "dependencies": { - "tc-core-library-js": { - "version": "github:appirio-tech/tc-core-library-js#df0b36c51cf80918194cbff777214b3c0cf5a151", - "from": "github:appirio-tech/tc-core-library-js#v2.6.4", - "requires": { - "axios": "^0.19.0", - "bunyan": "^1.8.12", - "jsonwebtoken": "^8.5.1", - "jwks-rsa": "^1.6.0", - "lodash": "^4.17.15", - "millisecond": "^0.1.2", - "r7insight_node": "^1.8.4", - "request": "^2.88.0" - } - } } }, "tc-core-library-js": { diff --git a/package.json b/package.json index 4fa45ef..b069f0d 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "url": "" }, "dependencies": { + "@elastic/elasticsearch": "^7.9.1", "@hapi/joi": "^16.1.8", "@hapi/joi-date": "^2.0.1", "amazon-qldb-driver-nodejs": "^0.1.1-preview.2", @@ -23,10 +24,8 @@ "body-parser": "^1.19.0", "config": "^3.2.4", "cors": "^2.8.5", - "elasticsearch": "^16.7.1", "express": "^4.17.1", "get-parameter-names": "^0.3.0", - "http-aws-es": "^6.0.0", "ion-js": "^3.1.2", "js-yaml": "^3.13.1", "lodash": "^4.17.19", diff --git a/scripts/constants.js b/scripts/constants.js index 755b650..c5d976c 100644 --- a/scripts/constants.js +++ b/scripts/constants.js @@ -6,37 +6,136 @@ const config = require('config') const topResources = { + skillprovider: { + index: config.get('ES.DOCUMENTS.skillprovider.index'), + type: config.get('ES.DOCUMENTS.skillprovider.type'), + enrich: { + policyName: config.get('ES.DOCUMENTS.skillprovider.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] + }, + pipeline: { + id: config.get('ES.DOCUMENTS.skillprovider.pipelineId'), + field: 'skillProviderId', + targetField: 'skillprovider', + maxMatches: '1' + } + }, + + role: { + index: config.get('ES.DOCUMENTS.role.index'), + type: config.get('ES.DOCUMENTS.role.type'), + enrich: { + policyName: config.get('ES.DOCUMENTS.role.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] + } + }, + achievementprovider: { index: config.get('ES.DOCUMENTS.achievementprovider.index'), - type: config.get('ES.DOCUMENTS.achievementprovider.type') - }, - attribute: { - index: config.get('ES.DOCUMENTS.attribute.index'), - type: config.get('ES.DOCUMENTS.attribute.type') + type: config.get('ES.DOCUMENTS.achievementprovider.type'), + enrich: { + policyName: config.get('ES.DOCUMENTS.achievementprovider.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] + } }, + attributegroup: { index: config.get('ES.DOCUMENTS.attributegroup.index'), - type: config.get('ES.DOCUMENTS.attributegroup.type') - }, - organization: { - index: config.get('ES.DOCUMENTS.organization.index'), - type: config.get('ES.DOCUMENTS.organization.type') - }, - role: { - index: config.get('ES.DOCUMENTS.role.index'), - type: config.get('ES.DOCUMENTS.role.type') + type: config.get('ES.DOCUMENTS.attributegroup.type'), + enrich: { + policyName: config.get('ES.DOCUMENTS.attributegroup.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy'] + }, + pipeline: { + id: config.get('ES.DOCUMENTS.attributegroup.pipelineId'), + field: 'attributeGroupId', + targetField: 'attributegroup', + maxMatches: '1' + } }, + skill: { index: config.get('ES.DOCUMENTS.skill.index'), - type: config.get('ES.DOCUMENTS.skill.type') + type: config.get('ES.DOCUMENTS.skill.type'), + enrich: { + policyName: config.get('ES.DOCUMENTS.skill.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider'] + }, + ingest: { + pipeline: { + id: config.get('ES.DOCUMENTS.skillprovider.pipelineId') + } + } }, - skillprovider: { - index: config.get('ES.DOCUMENTS.skillprovider.index'), - type: config.get('ES.DOCUMENTS.skillprovider.type') + + attribute: { + index: config.get('ES.DOCUMENTS.attribute.index'), + type: config.get('ES.DOCUMENTS.attribute.type'), + enrich: { + policyName: config.get('ES.DOCUMENTS.attribute.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'name', 'attributeGroupId', 'created', 'updated', 'createdBy', 'updatedBy', 'attributegroup'] + }, + ingest: { + pipeline: { + id: config.get('ES.DOCUMENTS.attributegroup.pipelineId') + } + } }, + + organization: { + index: config.get('ES.DOCUMENTS.organization.index'), + type: config.get('ES.DOCUMENTS.organization.type') + }, + user: { index: config.get('ES.DOCUMENTS.user.index'), - type: config.get('ES.DOCUMENTS.user.type') + type: config.get('ES.DOCUMENTS.user.type'), + pipeline: { + id: config.get('ES.DOCUMENTS.user.pipelineId'), + processors: [ + { + referenceField: config.get('ES.DOCUMENTS.achievement.userField'), + enrichPolicyName: 'achievementprovider-policy', + field: '_ingest._value.achievementsProviderId', + targetField: '_ingest._value.achievementprovider', + maxMatches: '1' + }, + { + referenceField: config.get('ES.DOCUMENTS.externalprofile.userField'), + enrichPolicyName: 'organization-policy', + field: '_ingest._value.organizationId', + targetField: '_ingest._value.organization', + maxMatches: '1' + }, + { + referenceField: config.get('ES.DOCUMENTS.userattribute.userField'), + enrichPolicyName: 'attribute-policy', + field: '_ingest._value.attributeId', + targetField: '_ingest._value.attribute', + maxMatches: '1' + }, + { + referenceField: config.get('ES.DOCUMENTS.userrole.userField'), + enrichPolicyName: 'role-policy', + field: '_ingest._value.roleId', + targetField: '_ingest._value.role', + maxMatches: '1' + }, + { + referenceField: config.get('ES.DOCUMENTS.userskill.userField'), + enrichPolicyName: 'skill-policy', + field: '_ingest._value.skillId', + targetField: '_ingest._value.skill', + maxMatches: '1' + } + ] + } } } @@ -67,7 +166,12 @@ const userResources = { const organizationResources = { organizationskillprovider: { propertyName: config.get('ES.DOCUMENTS.organizationskillprovider.orgField'), - relateKey: 'skillProviderId' + relateKey: 'skillProviderId', + enrich: { + policyName: config.get('ES.DOCUMENTS.organization.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy', 'skillProviders'] + } } } diff --git a/scripts/db/dropAll.js b/scripts/db/dropAll.js index cef1924..a30a2cf 100644 --- a/scripts/db/dropAll.js +++ b/scripts/db/dropAll.js @@ -4,11 +4,33 @@ const _ = require('lodash') const models = require('../../src/models') const logger = require('../../src/common/logger') -const { topResources, modelToESIndexMapping } = require('../constants') +const { + topResources, + organizationResources, + modelToESIndexMapping +} = require('../constants') const { getESClient } = require('../../src/common/es-client') async function main () { const client = getESClient() + + try { + logger.info('Deleting all pipelines...') + await client.ingest.deletePipeline({ + id: topResources.user.pipeline.id + }) + await client.ingest.deletePipeline({ + id: topResources.skillprovider.pipeline.id + }) + await client.ingest.deletePipeline({ + id: topResources.attributegroup.pipeline.id + }) + logger.info('Successfully deleted') + } catch (e) { + console.error(e) + logger.warn('Delete all ingest pipelines failed') + } + const keys = Object.keys(models) for (let i = 0; i < keys.length; i++) { const key = keys[i] @@ -16,12 +38,29 @@ async function main () { const esResourceName = modelToESIndexMapping[key] try { if (_.includes(_.keys(topResources), esResourceName)) { + if (topResources[esResourceName].enrich) { + logger.info(`Deleting enrich policy for ${esResourceName}`) + await client.enrich.deletePolicy({ + name: topResources[esResourceName].enrich.policyName + }) + logger.info(`Successfully deleted enrich policy for ${esResourceName}`) + } + logger.info(`Deleting index for ${esResourceName}`) await client.indices.delete({ index: topResources[esResourceName].index }) + logger.info(`Successfully deleted enrich policy for ${esResourceName}`) + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + logger.info('Deleting enrich policy for organization') + await client.enrich.deletePolicy({ + name: organizationResources[esResourceName].enrich.policyName + }) + logger.info('Successfully deleted enrich policy for organization') } + logger.info(`Deleting data in QLDB for ${esResourceName}`) await models.DBHelper.clear(models[key]) + logger.info(`Successfully deleted data in QLDB for ${esResourceName}`) } catch (e) { console.error(e) logger.warn(`drop table ${key} failed`) diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index 17c51cf..9204c19 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -9,14 +9,110 @@ const { modelToESIndexMapping } = require('../constants') -async function cleanupES () { +// Declares the ordering of the resource data insertion, to ensure that enrichment happens correctly +const RESOURCES_IN_ORDER = [ + 'skillprovider', + 'role', + 'achievementprovider', + 'attributegroup', + 'skill', + 'attribute', + 'organization', + 'organizationskillprovider', + 'user', + 'userskill', + 'achievement', + 'userrole', + 'externalprofile', + 'userattribute' +] + +const client = getESClient() + +const RESOURCE_NOT_FOUND = 'resource_not_found_exception' +const INDEX_NOT_FOUND = 'index_not_found_exception' + +/** + * Cleans up the data in elasticsearch + * @param {Array} keys Array of models + */ +async function cleanupES (keys) { const client = getESClient() + try { + await client.ingest.deletePipeline({ + id: topResources.user.pipeline.id + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } - await client.indices.delete({ - index: '_all' - }) + try { + await client.ingest.deletePipeline({ + id: topResources.skillprovider.pipeline.id + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } + + try { + await client.ingest.deletePipeline({ + id: topResources.attributegroup.pipeline.id + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } + + try { + for (let i = 0; i < keys.length; i++) { + const key = keys[i] + if (models[key].tableName) { + const esResourceName = modelToESIndexMapping[key] + if (_.includes(_.keys(topResources), esResourceName)) { + if (topResources[esResourceName].enrich) { + try { + await client.enrich.deletePolicy({ + name: topResources[esResourceName].enrich.policyName + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } + } - console.log('Existing indices have been deleted!') + try { + await client.indices.delete({ + index: topResources[esResourceName].index + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== INDEX_NOT_FOUND) { + throw e + } + } + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + try { + await client.enrich.deletePolicy({ + name: organizationResources[esResourceName].enrich.policyName + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } + } + } + } + } catch (e) { + console.log(JSON.stringify(e)) + throw e + } + console.log('Existing data in elasticsearch has been deleted!') } async function insertIntoES (modelName, body) { @@ -28,20 +124,19 @@ async function insertIntoES (modelName, body) { return } - const client = getESClient() - if (_.includes(_.keys(topResources), esResourceName)) { - await client.create({ + await client.index({ index: topResources[esResourceName].index, type: topResources[esResourceName].type, id: body.id, body, - refresh: 'true' + pipeline: topResources[esResourceName].ingest ? topResources[esResourceName].ingest.pipeline.id : undefined, + refresh: 'wait_for' }) } else if (_.includes(_.keys(userResources), esResourceName)) { const userResource = userResources[esResourceName] - const user = await client.getSource({ + const { body: user } = await client.getSource({ index: topResources.user.index, type: topResources.user.type, id: body.userId @@ -73,18 +168,19 @@ async function insertIntoES (modelName, body) { logger.error(`Can't create existing ${esResourceName} with the ${userResource.relateKey}: ${relateId}, userId: ${body.userId}`) } else { user[userResource.propertyName].push(body) - await client.update({ + await client.index({ index: topResources.user.index, type: topResources.user.type, id: body.userId, - body: { doc: user }, - refresh: 'true' + body: user, + pipeline: topResources.user.pipeline.id, + refresh: 'wait_for' }) } } else if (_.includes(_.keys(organizationResources), esResourceName)) { const orgResource = organizationResources[esResourceName] - const organization = await client.getSource({ + const { body: organization } = await client.getSource({ index: topResources.organization.index, type: topResources.organization.type, id: body.organizationId @@ -100,12 +196,102 @@ async function insertIntoES (modelName, body) { logger.error(`Can't create existing ${esResourceName} with the ${orgResource.relateKey}: ${relateId}, organizationId: ${body.organizationId}`) } else { organization[orgResource.propertyName].push(body) - await client.update({ + await client.index({ index: topResources.organization.index, type: topResources.organization.type, id: body.organizationId, - body: { doc: organization }, - refresh: 'true' + body: organization, + refresh: 'wait_for' + }) + } + } +} + +/** + * Creates and executes the enrich policy for the provided model + * @param {String} modelName The model name + */ +async function createAndExecuteEnrichPolicy (modelName) { + const esResourceName = modelToESIndexMapping[modelName] + + if (_.includes(_.keys(topResources), esResourceName) && topResources[esResourceName].enrich) { + await client.enrich.putPolicy({ + name: topResources[esResourceName].enrich.policyName, + body: { + match: { + indices: topResources[esResourceName].index, + match_field: topResources[esResourceName].enrich.matchField, + enrich_fields: topResources[esResourceName].enrich.enrichFields + } + } + }) + await client.enrich.executePolicy({ name: topResources[esResourceName].enrich.policyName }) + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + // For organization, execute enrich policy AFTER the sub documents on the org (namely orgskillprovider) is in + // This is because external profile on user is enriched with org, and it needs to have the orgskillprovider details in it + await client.enrich.putPolicy({ + name: organizationResources[esResourceName].enrich.policyName, + body: { + match: { + indices: topResources.organization.index, + match_field: organizationResources[esResourceName].enrich.matchField, + enrich_fields: organizationResources[esResourceName].enrich.enrichFields + } + } + }) + await client.enrich.executePolicy({ name: organizationResources[esResourceName].enrich.policyName }) + } +} + +/** + * Creates the ingest pipeline using the enrich policy + * @param {String} modelName The model name + */ +async function createEnrichProcessor (modelName) { + const esResourceName = modelToESIndexMapping[modelName] + + if (_.includes(_.keys(topResources), esResourceName) && topResources[esResourceName].pipeline) { + if (topResources[esResourceName].pipeline.processors) { + const processors = [] + + for (let i = 0; i < topResources[esResourceName].pipeline.processors.length; i++) { + const ep = topResources[esResourceName].pipeline.processors[i] + processors.push({ + foreach: { + field: ep.referenceField, + ignore_missing: true, + processor: { + enrich: { + policy_name: ep.enrichPolicyName, + ignore_missing: true, + field: ep.field, + target_field: ep.targetField, + max_matches: ep.maxMatches + } + } + } + }) + } + + await client.ingest.putPipeline({ + id: topResources[esResourceName].pipeline.id, + body: { + processors + } + }) + } else { + await client.ingest.putPipeline({ + id: topResources[esResourceName].pipeline.id, + body: { + processors: [{ + enrich: { + policy_name: topResources[esResourceName].enrich.policyName, + field: topResources[esResourceName].pipeline.field, + target_field: topResources[esResourceName].pipeline.targetField, + max_matches: topResources[esResourceName].pipeline.maxMatches + } + }] + } }) } } @@ -117,32 +303,57 @@ async function insertIntoES (modelName, body) { */ async function main () { let keys = Object.keys(models) - keys = _.orderBy(keys, k => { - const esResourceName = modelToESIndexMapping[k] - // Create parent data first - if (_.includes(_.keys(topResources), esResourceName)) { - return -1 + // Sort the models in the order of insertion (for correct enrichment) + const temp = Array(keys.length).fill(null) + keys.forEach(k => { + if (models[k].tableName) { + const esResourceName = modelToESIndexMapping[k] + const index = RESOURCES_IN_ORDER.indexOf(esResourceName) + temp[index] = k } - - return 1 }) + keys = _.compact(temp) - await cleanupES() + await cleanupES(keys) for (let i = 0; i < keys.length; i++) { const key = keys[i] - if (models[key].tableName) { - try { - const data = await models.DBHelper.find(models[key], []) - for (let i = 0; i < data.length; i++) { - await insertIntoES(key, data[i]) + try { + const data = await models.DBHelper.find(models[key], []) + + for (let i = 0; i < data.length; i++) { + logger.info(`Inserting data ${i + 1} of ${data.length}`) + logger.info(JSON.stringify(data[i])) + if (!_.isString(data[i].created)) { + data[i].created = new Date() + } + if (!_.isString(data[i].createdBy)) { + data[i].createdBy = 'TonyJ' } - logger.info('import data for ' + key + ' done') - } catch (e) { - logger.error(e) - logger.warn('import data for ' + key + ' failed') + await insertIntoES(key, data[i]) } + logger.info('import data for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('import data for ' + key + ' failed') + continue + } + + try { + await createAndExecuteEnrichPolicy(key) + logger.info('create and execute enrich policy for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('create and execute enrich policy for ' + key + ' failed') + } + + try { + await createEnrichProcessor(key) + logger.info('create enrich processor (pipeline) for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('create enrich processor (pipeline) for ' + key + ' failed') } } logger.info('all done') diff --git a/scripts/db/genData.js b/scripts/db/genData.js index 2f501a1..97681ac 100644 --- a/scripts/db/genData.js +++ b/scripts/db/genData.js @@ -9,6 +9,26 @@ const { modelToESIndexMapping } = require('../constants') +// Declares the ordering of the resource data insertion, to ensure that enrichment happens correctly +const RESOURCES_IN_ORDER = [ + 'skillprovider', + 'role', + 'achievementprovider', + 'attributegroup', + 'skill', + 'attribute', + 'organization', + 'organizationskillprovider', + 'user', + 'userskill', + 'achievement', + 'userrole', + 'externalprofile', + 'userattribute' +] + +const client = getESClient() + async function insertIntoES (modelName, body) { const esResourceName = modelToESIndexMapping[modelName] @@ -18,20 +38,19 @@ async function insertIntoES (modelName, body) { return } - const client = getESClient() - if (_.includes(_.keys(topResources), esResourceName)) { - await client.create({ + await client.index({ index: topResources[esResourceName].index, type: topResources[esResourceName].type, id: body.id, body, - refresh: 'true' + pipeline: topResources[esResourceName].ingest ? topResources[esResourceName].ingest.pipeline.id : undefined, + refresh: 'wait_for' }) } else if (_.includes(_.keys(userResources), esResourceName)) { const userResource = userResources[esResourceName] - const user = await client.getSource({ + const { body: user } = await client.getSource({ index: topResources.user.index, type: topResources.user.type, id: body.userId @@ -63,18 +82,19 @@ async function insertIntoES (modelName, body) { logger.error(`Can't create existing ${esResourceName} with the ${userResource.relateKey}: ${relateId}, userId: ${body.userId}`) } else { user[userResource.propertyName].push(body) - await client.update({ + await client.index({ index: topResources.user.index, type: topResources.user.type, id: body.userId, - body: { doc: user }, - refresh: 'true' + body: user, + pipeline: topResources.user.pipeline.id, + refresh: 'wait_for' }) } } else if (_.includes(_.keys(organizationResources), esResourceName)) { const orgResource = organizationResources[esResourceName] - const organization = await client.getSource({ + const { body: organization } = await client.getSource({ index: topResources.organization.index, type: topResources.organization.type, id: body.organizationId @@ -90,12 +110,102 @@ async function insertIntoES (modelName, body) { logger.error(`Can't create existing ${esResourceName} with the ${orgResource.relateKey}: ${relateId}, organizationId: ${body.organizationId}`) } else { organization[orgResource.propertyName].push(body) - await client.update({ + await client.index({ index: topResources.organization.index, type: topResources.organization.type, id: body.organizationId, - body: { doc: organization }, - refresh: 'true' + body: organization, + refresh: 'wait_for' + }) + } + } +} + +/** + * Creates and executes the enrich policy for the provided model + * @param {String} modelName The model name + */ +async function createAndExecuteEnrichPolicy (modelName) { + const esResourceName = modelToESIndexMapping[modelName] + + if (_.includes(_.keys(topResources), esResourceName) && topResources[esResourceName].enrich) { + await client.enrich.putPolicy({ + name: topResources[esResourceName].enrich.policyName, + body: { + match: { + indices: topResources[esResourceName].index, + match_field: topResources[esResourceName].enrich.matchField, + enrich_fields: topResources[esResourceName].enrich.enrichFields + } + } + }) + await client.enrich.executePolicy({ name: topResources[esResourceName].enrich.policyName }) + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + // For organization, execute enrich policy AFTER the sub documents on the org (namely orgskillprovider) is in + // This is because external profile on user is enriched with org, and it needs to have the orgskillprovider details in it + await client.enrich.putPolicy({ + name: organizationResources[esResourceName].enrich.policyName, + body: { + match: { + indices: topResources.organization.index, + match_field: organizationResources[esResourceName].enrich.matchField, + enrich_fields: organizationResources[esResourceName].enrich.enrichFields + } + } + }) + await client.enrich.executePolicy({ name: organizationResources[esResourceName].enrich.policyName }) + } +} + +/** + * Creates the ingest pipeline using the enrich policy + * @param {String} modelName The model name + */ +async function createEnrichProcessor (modelName) { + const esResourceName = modelToESIndexMapping[modelName] + + if (_.includes(_.keys(topResources), esResourceName) && topResources[esResourceName].pipeline) { + if (topResources[esResourceName].pipeline.processors) { + const processors = [] + + for (let i = 0; i < topResources[esResourceName].pipeline.processors.length; i++) { + const ep = topResources[esResourceName].pipeline.processors[i] + processors.push({ + foreach: { + field: ep.referenceField, + ignore_missing: true, + processor: { + enrich: { + policy_name: ep.enrichPolicyName, + ignore_missing: true, + field: ep.field, + target_field: ep.targetField, + max_matches: ep.maxMatches + } + } + } + }) + } + + await client.ingest.putPipeline({ + id: topResources[esResourceName].pipeline.id, + body: { + processors + } + }) + } else { + await client.ingest.putPipeline({ + id: topResources[esResourceName].pipeline.id, + body: { + processors: [{ + enrich: { + policy_name: topResources[esResourceName].enrich.policyName, + field: topResources[esResourceName].pipeline.field, + target_field: topResources[esResourceName].pipeline.targetField, + max_matches: topResources[esResourceName].pipeline.maxMatches + } + }] + } }) } } @@ -109,32 +219,49 @@ async function main () { await models.init() let keys = Object.keys(models) - keys = _.orderBy(keys, k => { - const esResourceName = modelToESIndexMapping[k] - // Create parent data first - if (_.includes(_.keys(topResources), esResourceName)) { - return -1 + // Sort the models in the order of insertion (for correct enrichment) + const temp = Array(keys.length).fill(null) + keys.forEach(k => { + if (models[k].tableName) { + const esResourceName = modelToESIndexMapping[k] + const index = RESOURCES_IN_ORDER.indexOf(esResourceName) + temp[index] = k } - - return 1 }) + keys = _.compact(temp) for (let i = 0; i < keys.length; i++) { const key = keys[i] - if (models[key].tableName) { - try { - const data = require(`./data/${key}.json`) - await models.DBHelper.clear(models[key]) - for (let i = 0; i < data.length; i++) { - await models.DBHelper.save(models[key], new models[key]().from(data[i]), true) - await insertIntoES(key, data[i]) - } - logger.info('import data for ' + key + ' done') - } catch (e) { - logger.error(e) - logger.warn('import data for ' + key + ' failed') + try { + const data = require(`./data/${key}.json`) + await models.DBHelper.clear(models[key]) + for (let i = 0; i < data.length; i++) { + logger.info(`Inserting data ${i + 1} of ${data.length}`) + await models.DBHelper.save(models[key], new models[key]().from(data[i]), true) + await insertIntoES(key, data[i]) } + logger.info('import data for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('import data for ' + key + ' failed') + continue + } + + try { + await createAndExecuteEnrichPolicy(key) + logger.info('create and execute enrich policy for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('create and execute enrich policy for ' + key + ' failed') + } + + try { + await createEnrichProcessor(key) + logger.info('create enrich processor (pipeline) for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('create enrich processor (pipeline) for ' + key + ' failed') } } logger.info('all done') diff --git a/src/common/es-client.js b/src/common/es-client.js index 2ffc3f1..c5a52d7 100644 --- a/src/common/es-client.js +++ b/src/common/es-client.js @@ -1,6 +1,6 @@ const config = require('config') const AWS = require('aws-sdk') -const elasticsearch = require('elasticsearch') +const elasticsearch = require('@elastic/elasticsearch') AWS.config.region = config.AWS_REGION @@ -16,20 +16,22 @@ function getESClient () { return esClient } const host = config.ES.HOST - const apiVersion = config.ES.API_VERSION - + const cloudId = config.ES.ELASTICCLOUD.id if (!esClient) { - // AWS ES configuration is different from other providers - if (/.*amazonaws.*/.test(host)) { + if (cloudId) { + // Elastic Cloud configuration esClient = new elasticsearch.Client({ - apiVersion, - host, - connectionClass: require('http-aws-es') // eslint-disable-line global-require + cloud: { + id: cloudId + }, + auth: { + username: config.ES.ELASTICCLOUD.username, + password: config.ES.ELASTICCLOUD.password + } }) } else { esClient = new elasticsearch.Client({ - apiVersion, - host + node: host }) } } diff --git a/src/common/es-helper.js b/src/common/es-helper.js index 0b395c5..3cdf833 100644 --- a/src/common/es-helper.js +++ b/src/common/es-helper.js @@ -215,13 +215,11 @@ const FILTER_CHAIN = { skill: { filterNext: 'userskill', queryField: 'skillId', - enrichNext: 'skillprovider', idField: 'skillProviderId' }, attribute: { filterNext: 'userattribute', queryField: 'attributeId', - enrichNext: 'attributegroup', idField: 'attributeGroupId' }, attributegroup: { @@ -247,29 +245,23 @@ const FILTER_CHAIN = { // sub resource userskill: { queryField: 'skillId', - enrichNext: 'skill', idField: 'skillId' }, userrole: { queryField: 'roleId', - enrichNext: 'role', idField: 'roleId' }, externalprofile: { - enrichNext: 'organization', idField: 'organizationId' }, achievement: { - enrichNext: 'achievementprovider', idField: 'achievementsProviderId' }, userattribute: { - enrichNext: 'attribute', idField: 'attributeId' }, organizationskillprovider: { queryField: 'skillProviderId', - enrichNext: 'skillprovider', idField: 'skillProviderId' } } @@ -376,68 +368,6 @@ function parseEnrichFilter (params) { return filters } -/** - * Enrich a resource recursively by following enrich path. - * @param resource the resource to enrich - * @param enrichIdProp the id property of child resource in parent object - * @param item the parent object - * @returns {Promise} the promise of enriched parent object - */ -async function enrichResource (resource, enrichIdProp, item) { - const subDoc = DOCUMENTS[resource] - const filterChain = FILTER_CHAIN[resource] - const subResult = await esClient.getSource({ - index: subDoc.index, - type: subDoc.type, - id: item[enrichIdProp] - }) - - if (filterChain.enrichNext) { - const enrichIdProp = filterChain.idField - // return error if any id is missing in enrich path - if (!subResult[enrichIdProp]) { - throw new Error(`The parent ${resource} is missing id value of child resource ${filterChain.enrichNext}`) - } - // enrich next child resource recursively - await enrichResource(filterChain.enrichNext, enrichIdProp, subResult) - } - item[resource] = subResult -} - -/** - * Enrich a user. - * @param user the user object to enrich - * @returns {Promise<*>} the promise of enriched user - */ -async function enrichUser (user) { - for (const subProp of Object.keys(SUB_USER_DOCUMENTS)) { - const subDoc = SUB_USER_DOCUMENTS[subProp] - const subData = user[subDoc.userField] - const filterChain = FILTER_CHAIN[subProp] - if (subData && subData.length > 0) { - // enrich next level sub resources - for (const subItem of subData) { - await enrichResource(filterChain.enrichNext, filterChain.idField, subItem) - } - } - } - return user -} - -/** - * Enrich users. - * @param users list of users from ES search - * @returns {Promise<*>} the enriched users - */ -async function enrichUsers (users) { - const enrichedUsers = [] - for (const user of users) { - const enriched = await enrichUser(user) - enrichedUsers.push(enriched) - } - return enrichedUsers -} - /** * Get a resource by Id from ES. * @param resource the resource to get @@ -494,12 +424,9 @@ async function getFromElasticSearch (resource, ...args) { logger.debug(`ES query for get ${resource}: ${JSON.stringify(esQuery, null, 2)}`) // query ES - const result = await esClient.getSource(esQuery) + const { body: result } = await esClient.getSource(esQuery) - if (params.enrich && resource === 'user') { - const user = await enrichUser(result) - return user - } else if (subUserDoc) { + if (subUserDoc) { // find top sub doc by sub.id const found = result[subUserDoc.userField].find(sub => sub[filterChain.idField] === params[filterChain.idField]) if (found) { @@ -713,7 +640,7 @@ async function searchSkills (keyword, skillProviderIds) { } logger.debug(`ES query for searching skills: ${JSON.stringify(esQuery, null, 2)}`) - const results = await esClient.search(esQuery) + const { body: results } = await esClient.search(esQuery) return results.hits.hits.map(hit => hit._source) } @@ -1059,7 +986,7 @@ async function resolveResFilter (filter, initialRes) { // query ES with filter const esQuery = buildEsQueryFromFilter(filter) - const result = await esClient.search(esQuery) + const { body: result } = await esClient.search(esQuery) const numHits = getTotalCount(result.hits.total) @@ -1288,7 +1215,7 @@ async function searchElasticSearch (resource, ...args) { } logger.debug(`ES query for search ${resource}: ${JSON.stringify(esQuery, null, 2)}`) - const docs = await esClient.search(esQuery) + const { body: docs } = await esClient.search(esQuery) if (docs.hits && getTotalCount(docs.hits.total) === 0) { return { total: docs.hits.total, @@ -1299,10 +1226,7 @@ async function searchElasticSearch (resource, ...args) { } let result = [] - if (resource === 'user' && params.enrich) { - const users = docs.hits.hits.map(hit => hit._source) - result = await enrichUsers(users) - } else if (topUserSubDoc) { + if (topUserSubDoc) { result = docs.hits.hits[0]._source[topUserSubDoc.userField] // for sub-resource query, it returns all sub-resource items in one user, // so needs filtering and also page size @@ -1420,14 +1344,9 @@ async function searchUsers (authUser, filter, params) { logger.debug(`ES query for searching users: ${JSON.stringify(esQuery, null, 2)}`) console.time('mainesquery') - const docs = await esClient.search(esQuery) + const { body: docs } = await esClient.search(esQuery) console.timeEnd('mainesquery') - const users = docs.hits.hits.map(hit => hit._source) - - logger.debug('Enrich users') - console.time('enrichUsers') - const result = await enrichUsers(users) - console.timeEnd('enrichUsers') + const result = docs.hits.hits.map(hit => hit._source) return { total: getTotalCount(docs.hits.total), @@ -1448,7 +1367,7 @@ async function searchSkillsInOrganization ({ organizationId, keyword }) { const esQueryToGetSkillProviders = buildEsQueryToGetSkillProviderIds(organizationId) logger.debug(`ES query to get skill provider ids: ${JSON.stringify(esQueryToGetSkillProviders, null, 2)}`) - const esResultOfQueryToGetSkillProviders = await esClient.search(esQueryToGetSkillProviders) + const { body: esResultOfQueryToGetSkillProviders } = await esClient.search(esQueryToGetSkillProviders) logger.debug(`ES result: ${JSON.stringify(esResultOfQueryToGetSkillProviders, null, 2)}`) const skillProviderIds = _.flatten(esResultOfQueryToGetSkillProviders.hits.hits.map(hit => hit._source.skillProviders == null ? [] : hit._source.skillProviders.map(sp => sp.skillProviderId))) @@ -1474,7 +1393,7 @@ async function searchAttributeValues ({ attributeId, attributeValue }) { const esQuery = buildEsQueryToGetAttributeValues(attributeId, querystring.unescape(attributeValue), 5) logger.debug(`ES query for searching attribute values: ${JSON.stringify(esQuery, null, 2)}`) - const esResult = await esClient.search(esQuery) + const { body: esResult } = await esClient.search(esQuery) logger.debug(`ES Result: ${JSON.stringify(esResult, null, 2)}`) const result = [] const attributes = esResult.aggregations.attributes.ids.buckets @@ -1502,7 +1421,7 @@ async function searchAchievementValues ({ organizationId, keyword }) { const esQuery = buildEsQueryToGetAchievements(organizationId, querystring.unescape(keyword), 5) logger.debug(`ES query for searching achievement values; ${JSON.stringify(esQuery, null, 2)}`) - const esResult = await esClient.search(esQuery) + const { body: esResult } = await esClient.search(esQuery) logger.debug(`ES response ${JSON.stringify(esResult, null, 2)}`) const result = esResult.aggregations.achievements.buckets.map(a => { const achievementName = a.key diff --git a/src/common/helper.js b/src/common/helper.js index 604d0fc..56e269e 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -109,6 +109,9 @@ function readerToJson (reader) { toRealValue(r, setValue(name, [])) r.stepOut() break + case IonTypes.BOOL: + setValue(name, r.booleanValue()) + break } nextT = reader.next() } diff --git a/src/common/service-helper.js b/src/common/service-helper.js index 072a0b8..10eb361 100644 --- a/src/common/service-helper.js +++ b/src/common/service-helper.js @@ -190,7 +190,7 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil async function patch (id, entity, auth, params) { await makeSureRefExist(entity) - const dbEntity = await get(id, auth, params) + const dbEntity = await get(id, auth, params, {}, true) const newEntity = new Model() _.extend(newEntity, dbEntity, entity) newEntity.updated = new Date() @@ -218,23 +218,26 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil * @param auth the auth obj * @param params the path parameters * @param query the query parameters + * @param fromDb Should we bypass Elasticsearch for the record and fetch from db instead? * @return {Promise} the db device */ - async function get (id, auth, params, query = {}) { + async function get (id, auth, params, query = {}, fromDb = false) { let recordObj // Merge path and query params const trueParams = _.assign(params, query) - try { - const result = await esHelper.getFromElasticSearch(resource, id, auth, trueParams) - // check permission - permissionCheck(auth, result) - return result - } catch (err) { - // return error if enrich fails or permission fails - if ((resource === 'user' && trueParams.enrich) || (err.status && err.status === 403)) { - throw errors.elasticSearchEnrichError(err.message) + if (!fromDb) { + try { + const result = await esHelper.getFromElasticSearch(resource, id, auth, trueParams) + // check permission + permissionCheck(auth, result) + return result + } catch (err) { + // return error if enrich fails or permission fails + if ((resource === 'user' && trueParams.enrich) || (err.status && err.status === 403)) { + throw errors.elasticSearchEnrichError(err.message) + } + logger.logFullError(err) } - logger.logFullError(err) } if (_.isNil(trueParams) || _.isEmpty(trueParams)) { recordObj = await models.DBHelper.get(Model, id) @@ -302,7 +305,7 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil */ async function remove (id, auth, params) { let payload - await get(id, auth, params) // check exist + await get(id, auth, params, {}, true) // check exist await models.DBHelper.delete(Model, id, buildQueryByParams(params)) if (SUB_USER_DOCUMENTS[resource] || SUB_ORG_DOCUMENTS[resource]) { payload = _.assign({}, params) diff --git a/src/modules/user/service.js b/src/modules/user/service.js index c7160a8..7593d50 100644 --- a/src/modules/user/service.js +++ b/src/modules/user/service.js @@ -17,7 +17,13 @@ const methods = helper.getServiceMethods( firstName: joi.string(), lastName: joi.string() }, - { handle: joi.string(), roleId: joi.string() }, + { + handle: joi.string(), + roleId: joi.string(), + enrich: joi.boolean(), + 'externalProfile.externalId': joi.string(), + 'externalProfile.organizationId': joi.string() + }, async query => { let prefix = 'select * from DUser' const dbQueries = []