diff --git a/src/services/challengeService.js b/src/services/challengeService.js index 1aa2478..b96f822 100644 --- a/src/services/challengeService.js +++ b/src/services/challengeService.js @@ -343,7 +343,7 @@ async function getChallengeIDsFromV4 (filter, perPage, page = 1) { * @param {Number} page * @returns {Object} { total, ids } */ -async function getChallengeIDsFromV5 (filter, perPage, page = 1) { +async function getChallengeIDsFromV5 (filter, perPage, lastDate) { // logger.warn(`getChallengeIDsFromV5 ${JSON.stringify(filter)} perPage ${perPage} page ${page}`) const boolQuery = [] const mustQuery = [] @@ -373,7 +373,6 @@ async function getChallengeIDsFromV5 (filter, perPage, page = 1) { type: config.get('ES.CHALLENGE_ES_TYPE'), // refresh: config.get('ES.ES_REFRESH'), size: perPage, - from: perPage * (page - 1), body: { _source: ['legacyId', 'id'], version: 'true', @@ -390,6 +389,9 @@ async function getChallengeIDsFromV5 (filter, perPage, page = 1) { ] } } + if (lastDate) { + esQuery.body.search_after = [Number(lastDate)] + } // Search with constructed query let docs // logger.warn(`V5 Challenge IDs Query ${JSON.stringify(esQuery)}`) @@ -407,14 +409,30 @@ async function getChallengeIDsFromV5 (filter, perPage, page = 1) { } // logger.warn(JSON.stringify(docs)) // Extract data from hits - if (docs.hits.total > 0) { + let result = _.map(docs.hits.hits, item => item._source) + logger.info(`ES Search Hits Total -> ${docs.hits.total}`) + let newLastDate = null + if (result.length > 0) { + logger.info(`ES Search Result Length -> ${result.length}`) + const endSortDate = docs.hits.hits[result.length - 1].sort + if (endSortDate && endSortDate.length) { + logger.info(`LastSortDate: ${JSON.stringify(endSortDate)}`) + newLastDate = endSortDate[0] + } + logger.info(`newLastDate Timestamp: ${newLastDate}`) return { total: docs.hits.total, ids: _.map(docs.hits.hits, hit => _.toNumber(hit._source.legacyId)), - v5Ids: _.map(docs.hits.hits, hit => hit._source.id) + v5Ids: _.map(docs.hits.hits, hit => hit._source.id), + lastDate: newLastDate } } - return false + return { + total: docs.hits.total, + ids: [], + v5Ids: [], + lastDate: newLastDate + } } async function getChallengeListingFromV4ES (legacyId) { diff --git a/src/services/challengeSyncStatusService.js b/src/services/challengeSyncStatusService.js index 40ff221..1c9dcf5 100644 --- a/src/services/challengeSyncStatusService.js +++ b/src/services/challengeSyncStatusService.js @@ -73,6 +73,7 @@ async function getSyncProgress (filter, perPage = 100, page = 1) { docs = await getESClient().search(esQuery) } catch (e) { // Catch error when the ES is fresh and has no data + logger.error(`Sync Queue Challenge IDs try/catch ${JSON.stringify(e)}`) docs = { hits: { total: 0, diff --git a/src/services/syncService.js b/src/services/syncService.js index 8d94454..6be57ca 100644 --- a/src/services/syncService.js +++ b/src/services/syncService.js @@ -213,7 +213,7 @@ async function getV4ChallengeIds (filter) { } async function getV5LegacyChallengeIds (filter) { - let page = 1 + let lastKey = null let running = true let v5Ids = [] const perPage = 1000 @@ -221,12 +221,13 @@ async function getV5LegacyChallengeIds (filter) { while (running) { // logger.debug(`V5 Challenge IDs - Getting ${page}`) - const { total, ids } = await challengeService.getChallengeIDsFromV5(filter, perPage, page) + const { total, ids, lastDate } = await challengeService.getChallengeIDsFromV5(filter, perPage, lastKey) + // Record the last call's end timestamp + lastKey = lastDate; if (ids && ids.length > 0) { // logger.warn(`IDs ${JSON.stringify(ids)}`) combinedTotal = total v5Ids = _.concat(v5Ids, ids) - page += 1 } else { running = false }