Skip to content

Commit 6d153c0

Browse files
callmekatootiecallmekatootie
committed
#179 - Support v5 challenge ids but ensure that we store legacy in db
1 parent 18f47d0 commit 6d153c0

11 files changed

+46
-78
lines changed

‎README.md

-4
Original file line numberDiff line numberDiff line change
@@ -116,10 +116,6 @@ This script will load the data from `scripts/data` directory into ES
116116
npm run start
117117
```
118118

119-
#### Duplicating the ES Index
120-
121-
To duplicate the existing ES Index (from the `ES_INDEX` to `ES_INDEX_V2` based on the configs in `config/default.js`) run `npm run create-new-index`
122-
123119
#### Linting JS files
124120

125121
```

‎config/default.js

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ module.exports = {
3232
HOST: process.env.ES_HOST || 'localhost:9200',
3333
API_VERSION: process.env.ES_API_VERSION || '6.3',
3434
ES_INDEX: process.env.ES_INDEX || 'submission',
35-
ES_INDEX_V2: process.env.ES_INDEX_V2 || 'new_submission',
3635
ES_TYPE: process.env.ES_TYPE || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it
3736
},
3837
PAGE_SIZE: process.env.PAGE_SIZE || 20,

‎package.json

-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
"create-tables": "node scripts/createTables.js",
1212
"init-db": "node scripts/importData.js",
1313
"create-index": "node scripts/createIndex.js",
14-
"create-new-index": "node scripts/createNewIndex.js",
1514
"delete-index": "node scripts/deleteIndex.js",
1615
"init-es": "node scripts/loadES.js",
1716
"db-to-es": "node scripts/migrateFromDBToES.js",

‎scripts/ESloadHelper.js

+5-5
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ const esClient = helper.getEsClient()
2020
function deleteDatafromES () {
2121
logger.info('Clear data from ES if any')
2222
const filter = {
23-
index: config.get('esConfig.ES_INDEX_V2'),
23+
index: config.get('esConfig.ES_INDEX'),
2424
type: config.get('esConfig.ES_TYPE'),
2525
q: '*'
2626
}
@@ -34,7 +34,7 @@ function * loadReviewTypes () {
3434
const promises = []
3535
reviewTypes.forEach((reviewType) => {
3636
const record = {
37-
index: config.get('esConfig.ES_INDEX_V2'),
37+
index: config.get('esConfig.ES_INDEX'),
3838
type: config.get('esConfig.ES_TYPE'),
3939
id: reviewType.id,
4040
body: _.extend({ resource: 'reviewType' }, reviewType)
@@ -51,7 +51,7 @@ function * loadSubmissions () {
5151
const promises = []
5252
submissions.forEach((submission) => {
5353
const record = {
54-
index: config.get('esConfig.ES_INDEX_V2'),
54+
index: config.get('esConfig.ES_INDEX'),
5555
type: config.get('esConfig.ES_TYPE'),
5656
id: submission.id,
5757
body: _.extend({ resource: 'submission' }, submission)
@@ -68,7 +68,7 @@ function * loadReviews () {
6868
const promises = []
6969
reviews.forEach((review) => {
7070
const record = {
71-
index: config.get('esConfig.ES_INDEX_V2'),
71+
index: config.get('esConfig.ES_INDEX'),
7272
type: config.get('esConfig.ES_TYPE'),
7373
id: review.id,
7474
body: _.extend({ resource: 'review' }, review)
@@ -85,7 +85,7 @@ function * loadReviewSummations () {
8585
const promises = []
8686
reviewSummations.forEach((reviewSummation) => {
8787
const record = {
88-
index: config.get('esConfig.ES_INDEX_V2'),
88+
index: config.get('esConfig.ES_INDEX'),
8989
type: config.get('esConfig.ES_TYPE'),
9090
id: reviewSummation.id,
9191
body: _.extend({ resource: 'reviewSummation' }, reviewSummation)

‎scripts/createIndex.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ co(function * createIndex () {
1717
// fields not specified below will be 'text' by default
1818
properties: {
1919
resource: { type: 'keyword' },
20-
challengeId: { type: 'keyword' },
20+
challengeId: { type: 'long' },
2121
memberId: { type: 'keyword' },
2222
type: { type: 'keyword' },
2323
isFileSubmission: { type: 'boolean' },
@@ -44,7 +44,7 @@ co(function * createIndex () {
4444
}
4545
}
4646
yield esClient.indices.create({
47-
index: config.get('esConfig.ES_INDEX_V2'),
47+
index: config.get('esConfig.ES_INDEX'),
4848
body
4949
})
5050
logger.info('ES Index creation succeeded!')

‎scripts/createNewIndex.js

-36
This file was deleted.

‎scripts/deleteIndex.js

+1-4
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,8 @@ const helper = require('../src/common/helper')
1010
co(function * deleteIndex () {
1111
logger.info('ES Index deletion started!')
1212
const esClient = helper.getEsClient()
13-
// yield esClient.indices.delete({
14-
// index: config.get('esConfig.ES_INDEX')
15-
// })
1613
yield esClient.indices.delete({
17-
index: config.get('esConfig.ES_INDEX_V2')
14+
index: config.get('esConfig.ES_INDEX')
1815
})
1916
logger.info('ES Index deletion succeeded!')
2017
process.exit(0)

‎scripts/migrateFromDBToES.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ function * migrateRecords (tableName) {
3030
logger.debug(`Number of ${tableName}s fetched from DB - ` + totalRecords)
3131
for (let i = 0; i < totalRecords; i++) {
3232
const record = {
33-
index: config.get('esConfig.ES_INDEX_V2'),
33+
index: config.get('esConfig.ES_INDEX'),
3434
type: config.get('esConfig.ES_TYPE'),
3535
id: records.Items[i].id,
3636
body: {

‎src/common/helper.js

+18-21
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ function prepESFilter (query, actResource) {
178178
}
179179

180180
const searchCriteria = {
181-
index: config.get('esConfig.ES_INDEX_V2'),
181+
index: config.get('esConfig.ES_INDEX'),
182182
type: config.get('esConfig.ES_TYPE'),
183183
size: pageSize,
184184
from: (page - 1) * pageSize, // Es Index starts from 0
@@ -345,14 +345,18 @@ function * getM2Mtoken (parentSpan) {
345345
* @param {Object} parentSpan the parent Span object
346346
* @returns {String} Legacy Challenge ID of the given challengeId
347347
*/
348-
function * getlegacyChallengeId (challengeId, token, parentSpan) {
348+
function * getLegacyChallengeId (challengeId, parentSpan) {
349349
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(challengeId)) {
350+
logger.debug(`${challengeId} detected as uuid. Fetching legacy challenge id`)
350351
const getlegacyChallengeIdSpan = tracer.startChildSpans('helper.getlegacyChallengeId', parentSpan)
352+
const token = yield getM2Mtoken(getlegacyChallengeIdSpan)
351353
try {
352354
const response = yield request.get(`${config.CHALLENGEAPI_V5_URL}/${challengeId}`)
353355
.set('Authorization', `Bearer ${token}`)
354356
.set('Content-Type', 'application/json')
355-
return response.body.legacyId
357+
const legacyId = parseInt(response.body.legacyId, 10)
358+
logger.debug(`Legacy challenge id is ${legacyId} for v5 challenge id ${challengeId}`)
359+
return legacyId
356360
} catch (err) {
357361
getlegacyChallengeIdSpan.setTag('error', true)
358362
logger.error(`Error while accessing ${config.CHALLENGEAPI_V5_URL}/${challengeId}`)
@@ -381,19 +385,17 @@ function * getSubmissionPhaseId (challengeId, parentSpan) {
381385
try {
382386
let phaseId = null
383387
let response
384-
let legacyChallengeId
385388

386389
const token = yield getM2Mtoken(getSubmissionPhaseIdSpan)
387390

388391
const getChallengePhasesSpan = tracer.startChildSpans('getChallengePhases', getSubmissionPhaseIdSpan)
389392
getChallengePhasesSpan.setTag('challengeId', challengeId)
390393
try {
391-
legacyChallengeId = yield getlegacyChallengeId(challengeId, token, getChallengePhasesSpan)
392-
response = yield request.get(`${config.CHALLENGEAPI_URL}/${legacyChallengeId}/phases`)
394+
response = yield request.get(`${config.CHALLENGEAPI_URL}/${challengeId}/phases`)
393395
.set('Authorization', `Bearer ${token}`)
394396
.set('Content-Type', 'application/json')
395397
} catch (ex) {
396-
logger.error(`Error while accessing ${config.CHALLENGEAPI_URL}/${legacyChallengeId}/phases`)
398+
logger.error(`Error while accessing ${config.CHALLENGEAPI_URL}/${challengeId}/phases`)
397399
logger.debug('Setting submissionPhaseId to Null')
398400
response = null
399401
// log error
@@ -437,7 +439,6 @@ function * checkCreateAccess (authUser, subEntity, parentSpan) {
437439

438440
try {
439441
let response
440-
let legacyChallengeId
441442

442443
// User can only create submission for themselves
443444
if (authUser.userId !== subEntity.memberId) {
@@ -449,12 +450,11 @@ function * checkCreateAccess (authUser, subEntity, parentSpan) {
449450
const getChallengeDetailSpan = tracer.startChildSpans('getChallengeDetail', checkCreateAccessSpan)
450451
getChallengeDetailSpan.setTag('challengeId', subEntity.challengeId)
451452
try {
452-
legacyChallengeId = yield getlegacyChallengeId(subEntity.challengeId, token, getChallengeDetailSpan)
453-
response = yield request.get(`${config.CHALLENGEAPI_URL}?filter=id=${legacyChallengeId}`)
453+
response = yield request.get(`${config.CHALLENGEAPI_URL}?filter=id=${subEntity.challengeId}`)
454454
.set('Authorization', `Bearer ${token}`)
455455
.set('Content-Type', 'application/json')
456456
} catch (ex) {
457-
logger.error(`Error while accessing ${config.CHALLENGEAPI_URL}?filter=id=${legacyChallengeId}`)
457+
logger.error(`Error while accessing ${config.CHALLENGEAPI_URL}?filter=id=${subEntity.challengeId}`)
458458
logger.error(ex)
459459
// log error
460460
getChallengeDetailSpan.log({
@@ -506,7 +506,6 @@ function * checkGetAccess (authUser, submission, parentSpan) {
506506
try {
507507
let resources
508508
let challengeDetails
509-
let legacyChallengeId
510509
// Allow downloading Own submission
511510
if (submission.memberId === authUser.userId) {
512511
return true
@@ -517,12 +516,11 @@ function * checkGetAccess (authUser, submission, parentSpan) {
517516
const getChallengeResourcesSpan = tracer.startChildSpans('getChallengeResources', checkGetAccessSpan)
518517
getChallengeResourcesSpan.setTag('challengeId', submission.challengeId)
519518
try {
520-
legacyChallengeId = yield getlegacyChallengeId(submission.challengeId, token, getChallengeResourcesSpan)
521-
resources = yield request.get(`${config.CHALLENGEAPI_URL}/${legacyChallengeId}/resources`)
519+
resources = yield request.get(`${config.CHALLENGEAPI_URL}/${submission.challengeId}/resources`)
522520
.set('Authorization', `Bearer ${token}`)
523521
.set('Content-Type', 'application/json')
524522
} catch (ex) {
525-
logger.error(`Error while accessing ${config.CHALLENGEAPI_URL}/${legacyChallengeId}/resources`)
523+
logger.error(`Error while accessing ${config.CHALLENGEAPI_URL}/${submission.challengeId}/resources`)
526524
logger.error(ex)
527525
// log error
528526
getChallengeResourcesSpan.log({
@@ -538,11 +536,11 @@ function * checkGetAccess (authUser, submission, parentSpan) {
538536
const getChallengeDetailSpan = tracer.startChildSpans('getChallengeDetail', checkGetAccessSpan)
539537
getChallengeDetailSpan.setTag('challengeId', submission.challengeId)
540538
try {
541-
challengeDetails = yield request.get(`${config.CHALLENGEAPI_URL}?filter=id=${legacyChallengeId}`)
539+
challengeDetails = yield request.get(`${config.CHALLENGEAPI_URL}?filter=id=${submission.challengeId}`)
542540
.set('Authorization', `Bearer ${token}`)
543541
.set('Content-Type', 'application/json')
544542
} catch (ex) {
545-
logger.error(`Error while accessing ${config.CHALLENGEAPI_URL}?filter=id=${legacyChallengeId}`)
543+
logger.error(`Error while accessing ${config.CHALLENGEAPI_URL}?filter=id=${submission.challengeId}`)
546544
logger.error(ex)
547545
// log error
548546
getChallengeDetailSpan.log({
@@ -636,18 +634,16 @@ function * checkReviewGetAccess (authUser, submission, parentSpan) {
636634

637635
try {
638636
let challengeDetails
639-
let legacyChallengeId
640637
const token = yield getM2Mtoken(checkReviewGetAccessSpan)
641638

642639
const getChallengeDetailSpan = tracer.startChildSpans('getChallengeDetail', checkReviewGetAccessSpan)
643640
getChallengeDetailSpan.setTag('challengeId', submission.challengeId)
644641
try {
645-
legacyChallengeId = yield getlegacyChallengeId(submission.challengeId, token, getChallengeDetailSpan)
646-
challengeDetails = yield request.get(`${config.CHALLENGEAPI_URL}?filter=id=${legacyChallengeId}`)
642+
challengeDetails = yield request.get(`${config.CHALLENGEAPI_URL}?filter=id=${submission.challengeId}`)
647643
.set('Authorization', `Bearer ${token}`)
648644
.set('Content-Type', 'application/json')
649645
} catch (ex) {
650-
logger.error(`Error while accessing ${config.CHALLENGEAPI_URL}?filter=id=${legacyChallengeId}`)
646+
logger.error(`Error while accessing ${config.CHALLENGEAPI_URL}?filter=id=${submission.challengeId}`)
651647

652648
// log error
653649
getChallengeDetailSpan.log({
@@ -800,6 +796,7 @@ module.exports = {
800796
fetchFromES,
801797
camelize,
802798
setPaginationHeaders,
799+
getLegacyChallengeId,
803800
getSubmissionPhaseId,
804801
checkCreateAccess,
805802
checkGetAccess,

‎src/services/SubmissionService.js

+18-2
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,12 @@ function * listSubmissions (authUser, query, span) {
250250
const listSubmissionsSpan = tracer.startChildSpans('SubmissionService.listSubmissions', span)
251251
let data = []
252252

253+
if (query.challengeId) {
254+
// Submission api only works with legacy challenge id
255+
// If it is a v5 challenge id, get the associated legacy challenge id
256+
query.challengeId = yield helper.getLegacyChallengeId(query.challengeId, listSubmissionsSpan)
257+
}
258+
253259
try {
254260
data = yield helper.fetchFromES(query, helper.camelize(table), listSubmissionsSpan)
255261
logger.info(`listSubmissions: returning ${data.rows.length} submissions for query: ${JSON.stringify(query)}`)
@@ -341,14 +347,18 @@ function * createSubmission (authUser, files, entity, span) {
341347
throw new errors.HttpStatusError(400, 'The file should be uploaded under the "submission" attribute')
342348
}
343349

350+
// Submission api only works with legacy challenge id
351+
// If it is a v5 challenge id, get the associated legacy challenge id
352+
const challengeId = yield helper.getLegacyChallengeId(entity.challengeId, createSubmissionSpan)
353+
344354
const currDate = (new Date()).toISOString()
345355

346356
const item = {
347357
id: submissionId,
348358
type: entity.type,
349359
url: url,
350360
memberId: entity.memberId,
351-
challengeId: entity.challengeId,
361+
challengeId: challengeId,
352362
created: currDate,
353363
updated: currDate,
354364
createdBy: authUser.handle || authUser.sub,
@@ -366,7 +376,7 @@ function * createSubmission (authUser, files, entity, span) {
366376
if (entity.submissionPhaseId) {
367377
item.submissionPhaseId = entity.submissionPhaseId
368378
} else {
369-
item.submissionPhaseId = yield helper.getSubmissionPhaseId(entity.challengeId, createSubmissionSpan)
379+
item.submissionPhaseId = yield helper.getSubmissionPhaseId(challengeId, createSubmissionSpan)
370380
}
371381

372382
if (entity.fileType) {
@@ -457,6 +467,12 @@ function * _updateSubmission (authUser, submissionId, entity, parentSpan) {
457467
throw new errors.HttpStatusError(404, `Submission with ID = ${submissionId} is not found`)
458468
}
459469

470+
if (entity.challengeId) {
471+
// Submission api only works with legacy challenge id
472+
// If it is a v5 challenge id, get the associated legacy challenge id
473+
entity.challengeId = yield helper.getLegacyChallengeId(entity.challengeId, updateSubmissionSpan)
474+
}
475+
460476
const currDate = (new Date()).toISOString()
461477
// Record used for updating in Database
462478
const record = {

‎test/unit/SubmissionService.test.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,7 @@ describe('Submission Service tests', () => {
304304
res.should.have.status(200)
305305
res.body.should.have.keys(Object.keys(_.extend({ fileType: 'zip' }, testSubmission.Item)))
306306
res.body.id.should.not.be.eql(null)
307-
res.body.challengeId.should.be.eql(testChallengeV5APIResponse.id)
307+
res.body.challengeId.should.be.eql(parseInt(testChallengeV5APIResponse.legacyId, 10))
308308
res.body.type.should.be.eql(testSubmission.Item.type)
309309
res.body.url.should.be.eql(testSubmission.Item.url)
310310
res.body.fileType.should.be.eql('zip')

0 commit comments

Comments
 (0)