Initial search by EMSI skills

Author: jmgasper

.nvmrc

@@ -1 +1 @@
-v8.9.2
+v12.22.12

app-constants.js

@@ -28,6 +28,10 @@ const MAMBO_GET_REWARDS_ALLOWED_FIELDS = [
   'awardedOn', 'expiryOn', 'isExpired', 'id'
 ]
 
+const BOOLEAN_OPERATOR = {
+  AND: 'AND',
+  OR: 'OR'
+}
 module.exports = {
   ADMIN_ROLES,
   SEARCH_BY_EMAIL_ROLES,
@@ -36,5 +40,6 @@ module.exports = {
   EVENT_MIME_TYPE,
   TOPICS,
   ES_SEARCH_MAX_SIZE,
-  MAMBO_GET_REWARDS_ALLOWED_FIELDS
+  MAMBO_GET_REWARDS_ALLOWED_FIELDS,
+  BOOLEAN_OPERATOR
 }

src/common/eshelper.js

@@ -3,6 +3,7 @@
  */
 const _ = require('lodash')
 const config = require('config')
+const { BOOLEAN_OPERATOR } = require('../../app-constants')
 
 /**
  * Fetch members profile form ES
@@ -58,6 +59,35 @@ async function getMembers (query, esClient, currentUser) {
   return docsMembers
 }
 
+/**
+ * Search members by skills
+ * @param {Object} query the HTTP request query
+ * @returns {Object} members skills
+ */
+async function searchBySkills (query, esClient) {
+  // construct ES query for skills
+  const esQuerySkills = {
+    index: config.get('ES.MEMBER_SKILLS_ES_INDEX'),
+    type: config.get('ES.MEMBER_SKILLS_ES_TYPE'),
+    body: {
+      sort: [{ userHandle: { order: query.sort } }]
+    }
+  }
+  const boolQuerySkills = []
+
+  if (query.handlesLower) {
+    boolQuerySkills.push({ query: { terms: { handleLower: query.handlesLower } } })
+  }
+  esQuerySkills.body.query = {
+    bool: {
+      filter: boolQuerySkills
+    }
+  }
+  // search with constructed query
+  const docsSkills = await esClient.search(esQuerySkills)
+  return docsSkills
+}
+
 /**
  * Fetch members skills form ES
  * @param {Object} query the HTTP request query
@@ -146,6 +176,66 @@ async function getSuggestion (query, esClient, currentUser) {
   return docsSuggestionMembers
 }
 
+/**
+ * Gets the members skills documents matching the provided criteria from Elasticsearch
+ * @param skillIds
+ * @param skillsBooleanOperator
+ * @param page
+ * @param perPage
+ * @param esClient
+ * @returns {Promise<*>}
+ */
+async function searchMembersSkills (skillIds, skillsBooleanOperator, page, perPage, esClient) {
+  // construct ES query for members skills
+  const esQuerySkills = {
+    index: config.get('ES.MEMBER_PROFILE_ES_INDEX'),
+    type: config.get('ES.MEMBER_PROFILE_ES_TYPE'),
+    from: 0,
+    size: 100,
+    body: {
+      sort: [{ createdAt: { order: 'desc' } }],
+      query: {
+        bool: {
+          filter: { bool: {} }
+        }
+      }
+    }
+  }
+
+  const mustMatchQuery = [] // will contain the filters with AND operator
+  const shouldFilter = [] // will contain the filters with OR operator
+
+  if (skillsBooleanOperator === BOOLEAN_OPERATOR.AND) {
+    for (const skillId of skillIds) {
+      const matchPhrase = {}
+      matchPhrase[`emsiSkills.emsiId`] = `${skillId}`
+      mustMatchQuery.push({
+        match_phrase: matchPhrase
+      })
+    }
+  } else {
+    for (const skillId of skillIds) {
+      const matchPhrase = {}
+      matchPhrase[`emsiSkills.emsiId`] = `${skillId}`
+      shouldFilter.push({
+        match_phrase: matchPhrase// eslint-disable-line
+      })
+    }
+  }
+
+  if (mustMatchQuery.length > 0) {
+    esQuerySkills.body.query.bool.filter.bool.must = mustMatchQuery
+  }
+
+  if (shouldFilter.length > 0) {
+    esQuerySkills.body.query.bool.filter.bool.should = shouldFilter
+  }
+  // search with constructed query
+  return esClient.search(esQuerySkills)
+}
+
+
+
 /**
  * Get total items
  * @param {Object} docs the HTTP request query
@@ -164,5 +254,6 @@ module.exports = {
   getMembersSkills,
   getMembersStats,
   getSuggestion,
-  getTotal
+  getTotal,
+  searchMembersSkills,
 }

src/common/helper.js

@@ -779,6 +779,26 @@ const getM2MToken = () => {
   )
 }
 
+/**
+ * Gets the list of parameters from the query as an array
+ *
+ * @param query
+ * @param parameterName
+ * @returns {*[]}
+ */
+const getParamsFromQueryAsArray = async (query, parameterName) => {
+  const paramsArray = []
+  if (!_.isEmpty(query[parameterName])) {
+    if (!_.isArray(query[parameterName])) {
+      paramsArray.push(query[parameterName])
+    } else {
+      paramsArray.push(...query[parameterName])
+    }
+  }
+  return paramsArray
+}
+
+
 module.exports = {
   wrapExpress,
   autoWrapExpress,
@@ -812,5 +832,6 @@ module.exports = {
   getGroupId,
   getAllowedGroupIds,
   getMemberGroups,
-  getM2MToken
+  getM2MToken,
+  getParamsFromQueryAsArray
 }

src/controllers/SearchController.js

@@ -26,7 +26,18 @@ async function autocomplete (req, res) {
   res.send(result.result)
 }
 
+/**
+ * Search members with additional parameters, like skills
+ * @param {Object} req the request
+ * @param {Object} res the response
+ */
+async function searchMembersBySkills (req, res) {
+  const result = await service.searchMembersBySkills(req.authUser, req.query)
+  helper.setResHeaders(req, res, result)
+  res.send(result.result)
+}
 module.exports = {
   searchMembers,
+  searchMembersBySkills,
   autocomplete
 }

src/routes.js

@@ -23,10 +23,10 @@ module.exports = {
       scopes: [MEMBERS.READ, MEMBERS.ALL]
     }
   },
-  '/search/members/autocomplete': {
+  '/members/searchBySkills': {
     get: {
       controller: 'SearchController',
-      method: 'autocomplete',
+      method: 'searchMembersBySkills',
       auth: 'jwt',
       scopes: [MEMBERS.READ, MEMBERS.ALL]
     }
@@ -160,5 +160,5 @@ module.exports = {
       allowNoToken: true,
       scopes: [MEMBERS.READ, MEMBERS.ALL]
     }
-  },
+  }
 }

src/scripts/view-es-data.js

@@ -12,13 +12,14 @@ if (process.argv.length <= 2) {
   process.exit(1)
 }
 const indexName = process.argv[2]
+const indexType = process.argv[3]
 
 const esClient = helper.getESClient()
 
 async function showESData () {
   const result = await esClient.search({
     index: indexName,
-    type: config.get('ES.MEMBER_PROFILE_ES_TYPE') // type name is same for all indices
+    type: indexType // type name is same for all indices
   })
   return result.hits.hits || []
 }

src/services/SearchService.js

@@ -9,12 +9,17 @@ const helper = require('../common/helper')
 const eshelper = require('../common/eshelper')
 const logger = require('../common/logger')
 const errors = require('../common/errors')
+const { BOOLEAN_OPERATOR } = require('../../app-constants')
 
 const MEMBER_FIELDS = ['userId', 'handle', 'handleLower', 'firstName', 'lastName',
   'status', 'addresses', 'photoURL', 'homeCountryCode', 'competitionCountryCode',
   'description', 'email', 'tracks', 'maxRating', 'wins', 'createdAt', 'createdBy',
   'updatedAt', 'updatedBy', 'skills', 'stats', 'emsiSkills']
 
+const MEMBER_SORT_BY_FIELDS = ['userId', 'country', 'handle', 'firstName', 'lastName', 
+  'accountAge', 'numberOfChallengesWon', 
+  'numberOfChallengesPlaced']
+
 const MEMBER_AUTOCOMPLETE_FIELDS = ['userId', 'handle', 'handleLower',
   'status', 'email', 'createdAt', 'updatedAt']
 
@@ -39,19 +44,42 @@ async function searchMembers (currentUser, query) {
   }
 
   if (query.email != null && query.email.length > 0) {
-  if (currentUser == null) {
-    throw new errors.UnauthorizedError("Authentication token is required to query users by email");
-  }
-  if (!helper.hasSearchByEmailRole(currentUser)) {
-    throw new errors.BadRequestError("Admin role is required to query users by email");
-  }
+    if (currentUser == null) {
+      throw new errors.UnauthorizedError('Authentication token is required to query users by email')
+    }
+    if (!helper.hasSearchByEmailRole(currentUser)) {
+      throw new errors.BadRequestError('Admin role is required to query users by email')
+    }
   }
 
   // search for the members based on query
   const docsMembers = await eshelper.getMembers(query, esClient, currentUser)
 
+  return fillMembers(docsMembers, query, fields)
+}
+
+searchMembers.schema = {
+  currentUser: Joi.any(),
+  query: Joi.object().keys({
+    handleLower: Joi.string(),
+    handlesLower: Joi.array(),
+    handle: Joi.string(),
+    handles: Joi.array(),
+    email: Joi.string(),
+    userId: Joi.number(),
+    userIds: Joi.array(),
+    term: Joi.string(),
+    fields: Joi.string(),
+    page: Joi.page(),
+    perPage: Joi.perPage(),
+    sort: Joi.sort()
+  })
+}
+
+async function fillMembers(docsMembers, query, fields) {
   // get the total
   const total = eshelper.getTotal(docsMembers)
+
   let results = []
   if (total > 0) {
     // extract member profiles from hits
@@ -108,24 +136,68 @@ async function searchMembers (currentUser, query) {
   return { total: total, page: query.page, perPage: query.perPage, result: results }
 }
 
-searchMembers.schema = {
+// TODO - use some caching approach to replace these in-memory objects
+/**
+ * Search members by the given search query
+ *
+ * @param query The search query by which to search members
+ *
+ * @returns {Promise<[]>} The array of members matching the given query
+ */
+const searchMembersBySkills = async (currentUser, query) => {
+  const esClient = await helper.getESClient()
+  let skillIds = await helper.getParamsFromQueryAsArray(query, 'skillId')
+  const result = searchMembersBySkillsWithOptions(currentUser, query, skillIds, BOOLEAN_OPERATOR.AND, query.page, query.perPage, query.sortBy, query.sortOrder, esClient)
+  return result
+}
+
+searchMembersBySkills.schema = {
   currentUser: Joi.any(),
   query: Joi.object().keys({
-    handleLower: Joi.string(),
-    handlesLower: Joi.array(),
-    handle: Joi.string(),
-    handles: Joi.array(),
-    email: Joi.string(),
-    userId: Joi.number(),
-    userIds: Joi.array(),
-    term: Joi.string(),
-    fields: Joi.string(),
+    skillId: Joi.alternatives().try(Joi.string(), Joi.array().items(Joi.string())),
     page: Joi.page(),
     perPage: Joi.perPage(),
-    sort: Joi.sort()
+    sortBy: Joi.string().valid(MEMBER_SORT_BY_FIELDS).default('numberOfChallengesWon'),
+    sortOrder: Joi.string().valid('asc', 'desc').default('desc')
   })
 }
 
+/**
+ * Search members matching the given skills
+ *
+ * @param currentUser
+ * @param skillsFilter
+ * @param skillsBooleanOperator
+ * @param page
+ * @param perPage
+ * @param sortBy
+ * @param sortOrder
+ * @param esClient
+ * @returns {Promise<*[]|{total, perPage, numberOfPages: number, data: *[], page}>}
+ */
+const searchMembersBySkillsWithOptions = async (currentUser, query, skillsFilter, skillsBooleanOperator, page, perPage, sortBy, sortOrder, esClient) => {
+  let fields = helper.parseCommaSeparatedString(query.fields, MEMBER_FIELDS) || MEMBER_FIELDS
+  // if current user is not admin and not M2M, then exclude the admin/M2M only fields
+  if (!currentUser || (!currentUser.isMachine && !helper.hasAdminRole(currentUser))) {
+    fields = _.without(fields, ...config.SEARCH_SECURE_FIELDS)
+    MEMBER_STATS_FIELDS = _.without(MEMBER_STATS_FIELDS, ...config.STATISTICS_SECURE_FIELDS)
+  }
+
+  const emptyResult = {
+    total: 0,
+    page,
+    perPage,
+    numberOfPages: 0,
+    data: []
+  }
+  if (_.isEmpty(skillsFilter)) {
+    return emptyResult
+  }
+
+  const membersSkillsDocs = await eshelper.searchMembersSkills(skillsFilter, skillsBooleanOperator, page, perPage, esClient)
+  
+  return fillMembers(membersSkillsDocs, query, fields)
+}
 /**
  * members autocomplete.
  * @param {Object} currentUser the user who performs operation
@@ -148,7 +220,7 @@ async function autocomplete (currentUser, query) {
     // custom filter & sort
     let regex = new RegExp(`^${query.term}`, `i`)
     // sometimes .payload is not defined. so use _source instead
-    results = results.map(x => ({...x, payload: x.payload || x._source}))
+    results = results.map(x => ({ ...x, payload: x.payload || x._source }))
     results = results
       .filter(x => regex.test(x.payload.handle))
       .sort((a, b) => a.payload.handle.localeCompare(b.payload.handle))
@@ -175,6 +247,7 @@ autocomplete.schema = {
 
 module.exports = {
   searchMembers,
+  searchMembersBySkills,
   autocomplete
 }