#21 - Dont listen to submission update events any more

Author: callmekatootie

README.md

@@ -21,7 +21,6 @@ The following parameters can be set in config files or in env variables:
 - KAFKA_GROUP_ID: consumer group id; default value: 'scorecard-processor'
 - REVIEW_TOPIC : Review topic, default value is 'submission.notification.score'
 - CREATE_SUBMISSION_TOPIC : create submission topic, default value is 'submission.notification.create'
-- UPDATE_SUBMISSION_TOPIC : update submission topic, default value is 'submission.notification.update'
 - SUBMISSION_API_URL: submission api url, default is 'https://api.topcoder-dev.com/v5'
 - CHALLENGE_API_URL: challenge API URL, default is 'https://api.topcoder-dev.com/v4/challenges'
 - SCORECARD_API_URL: scorecard API URL, default is 'http://localhost:4000/scorecards'
@@ -55,7 +54,6 @@ Configuration for the tests is at `config/test.js`, only add such new configurat
 - use another terminal, go to same directory, create the needed topics:
   `bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic submission.notification.score`
   `bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic submission.notification.create`
-  `bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic submission.notification.update`
   `bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic avscan.action.scan`
   `bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic or.action.review`
 - verify that the topics are created:

Verification.md

@@ -97,56 +97,6 @@ debug: Successfully processed message
 {"topic":"or.action.review","originator":"tc-scorecard-processor","timestamp":"2019-04-07T21:36:04.215Z","mime-type":"application/json","payload":{"resource":"submission","id":"104366f8-f46b-45db-a971-11bc69e6c8ff","type":"Contest Submission","url":"https://s3.amazonaws.com/topcoder-dev-submissions-dmz/30054740-8547899-SUBMISSION_ZIP-1554188341581.zip","memberId":8547899,"challengeId":30049360,"created":"2019-04-02T06:59:29.785Z","updated":"2019-04-02T06:59:29.785Z","createdBy":"TonyJ","updatedBy":"TonyJ","submissionPhaseId":764644,"fileType":"zip","isFileSubmission":false,"eventType":"CREATE"}}
 ```
 
-- start kafka-console-producer to write messages to `submission.notification.update` topic:
-  `bin/kafka-console-producer.sh --broker-list localhost:9092 --topic submission.notification.update`
-- write message:
-  `{ "topic": "submission.notification.update", "originator": "or-app", "timestamp": "2019-02-25T00:00:00", "mime-type": "application/json", "payload": { "resource": "submission", "id": "104366f8-f46b-45db-a971-11bc69e6c8ff", "type": "Contest Submission", "url": "https://s3.amazonaws.com/topcoder-dev-submissions-dmz/30054740-8547899-SUBMISSION_ZIP-1554188341581.zip", "memberId": 8547899, "challengeId": 30049360, "created": "2019-04-02T06:59:29.785Z", "updated": "2019-04-02T06:59:29.785Z", "createdBy": "TonyJ", "updatedBy": "TonyJ", "submissionPhaseId": 764644, "fileType": "zip", "isFileSubmission": false } }`
-- watch the app console, it should show logging of processing the message:
-```
-debug: Get M2M token
-debug: Get challenge details
-debug: Scorecard id: 30001610
-debug: Current phase: Registration
-debug: Get scorecard details
-debug: Post Kafka message for score system AV Scan: {
-    "topic": "avscan.action.scan",
-    "originator": "tc-scorecard-processor",
-    "timestamp": "2019-04-07T21:41:52.713Z",
-    "mime-type": "application/json",
-    "payload": {
-        "status": "unscanned",
-        "submissionId": "104366f8-f46b-45db-a971-11bc69e6c8ff",
-        "url": "https://s3.amazonaws.com/topcoder-dev-submissions-dmz/30054740-8547899-SUBMISSION_ZIP-1554188341581.zip",
-        "fileName": "30054740-8547899-SUBMISSION_ZIP-1554188341581.zip"
-    }
-}
-debug: Post Kafka message for score system OR: {
-    "topic": "or.action.review",
-    "originator": "tc-scorecard-processor",
-    "timestamp": "2019-04-07T21:41:52.729Z",
-    "mime-type": "application/json",
-    "payload": {
-        "resource": "submission",
-        "id": "104366f8-f46b-45db-a971-11bc69e6c8ff",
-        "type": "Contest Submission",
-        "url": "https://s3.amazonaws.com/topcoder-dev-submissions-dmz/30054740-8547899-SUBMISSION_ZIP-1554188341581.zip",
-        "memberId": 8547899,
-        "challengeId": 30049360,
-        "created": "2019-04-02T06:59:29.785Z",
-        "updated": "2019-04-02T06:59:29.785Z",
-        "createdBy": "TonyJ",
-        "updatedBy": "TonyJ",
-        "submissionPhaseId": 764644,
-        "fileType": "zip",
-        "isFileSubmission": false,
-        "eventType": "UPDATE"
-    }
-}
-debug: EXIT processSubmission
-debug: output arguments
-debug: Successfully processed message
-```
-
 - the kafka-console-consumer listening to topic `avscan.action.scan` should show:
 ```
 {"topic":"avscan.action.scan","originator":"tc-scorecard-processor","timestamp":"2019-04-07T21:41:52.713Z","mime-type":"application/json","payload":{"status":"unscanned","submissionId":"104366f8-f46b-45db-a971-11bc69e6c8ff","url":"https://s3.amazonaws.com/topcoder-dev-submissions-dmz/30054740-8547899-SUBMISSION_ZIP-1554188341581.zip","fileName":"30054740-8547899-SUBMISSION_ZIP-1554188341581.zip"}}
@@ -157,19 +107,6 @@ debug: Successfully processed message
 {"topic":"or.action.review","originator":"tc-scorecard-processor","timestamp":"2019-04-07T21:41:52.729Z","mime-type":"application/json","payload":{"resource":"submission","id":"104366f8-f46b-45db-a971-11bc69e6c8ff","type":"Contest Submission","url":"https://s3.amazonaws.com/topcoder-dev-submissions-dmz/30054740-8547899-SUBMISSION_ZIP-1554188341581.zip","memberId":8547899,"challengeId":30049360,"created":"2019-04-02T06:59:29.785Z","updated":"2019-04-02T06:59:29.785Z","createdBy":"TonyJ","updatedBy":"TonyJ","submissionPhaseId":764644,"fileType":"zip","isFileSubmission":false,"eventType":"UPDATE"}}
 ```
 
-- you may write invalid messages like below:
-  `{ "topic": "submission.notification.update", "originator": "or-app", "timestamp": "invalid", "mime-type": "application/json", "payload": { "resource": "submission", "id": "104366f8-f46b-45db-a971-11bc69e6c8ff", "type": "Contest Submission", "url": "https://s3.amazonaws.com/topcoder-dev-submissions-dmz/30054740-8547899-SUBMISSION_ZIP-1554188341581.zip", "memberId": 8547899, "challengeId": 30049360, "created": "2019-04-02T06:59:29.785Z", "updated": "2019-04-02T06:59:29.785Z", "createdBy": "TonyJ", "updatedBy": "TonyJ", "submissionPhaseId": 764644, "fileType": "zip", "isFileSubmission": false } }`
-
-  `{ "topic": "submission.notification.update", "originator": "or-app", "timestamp": "2019-02-25T00:00:00", "mime-type": "application/json", "payload": { "resource": "other", "id": "104366f8-f46b-45db-a971-11bc69e6c8ff", "type": "Contest Submission", "url": "https://s3.amazonaws.com/topcoder-dev-submissions-dmz/30054740-8547899-SUBMISSION_ZIP-1554188341581.zip", "memberId": 8547899, "challengeId": 30049360, "created": "2019-04-02T06:59:29.785Z", "updated": "2019-04-02T06:59:29.785Z", "createdBy": "TonyJ", "updatedBy": "TonyJ", "submissionPhaseId": 764644, "fileType": "zip", "isFileSubmission": false } }`
-
-  `{ "topic": "submission.notification.update", "originator": "or-app", "timestamp": "2019-02-25T00:00:00", "mime-type": "application/json", "payload": { "resource": "submission", "memberId": 8547899, "challengeId": 30049360, "created": "2019-04-02T06:59:29.785Z", "updated": "2019-04-02T06:59:29.785Z", "createdBy": "TonyJ", "updatedBy": "TonyJ", "submissionPhaseId": 764644, "fileType": "zip", "isFileSubmission": false } }`
-
-  `[ { , abc`
-
-- the app console will show proper error messages
-
-
-
 ## Unit test Coverage
 
   129 passing (1s)

config/default.js

@@ -18,8 +18,6 @@ module.exports = {
   REVIEW_TOPIC: process.env.REVIEW_TOPIC || 'submission.notification.score',
   // Kafka topic related to create submission
   CREATE_SUBMISSION_TOPIC: process.env.CREATE_SUBMISSION_TOPIC || 'submission.notification.create',
-  // Kafka topic related to update submission
-  UPDATE_SUBMISSION_TOPIC: process.env.UPDATE_SUBMISSION_TOPIC || 'submission.notification.update',
 
   SUBMISSION_API_URL: process.env.SUBMISSION_API_URL || 'https://api.topcoder-dev.com/v5',
 

src/app.js

@@ -46,7 +46,7 @@ const dataHandler = (messageSet, topic, partition) => Promise.each(messageSet, a
   return (async () => {
     if (topic === config.REVIEW_TOPIC) {
       await ReviewProcessorService.processReview(messageJSON)
-    } else if (topic === config.CREATE_SUBMISSION_TOPIC || topic === config.UPDATE_SUBMISSION_TOPIC) {
+    } else if (topic === config.CREATE_SUBMISSION_TOPIC) {
       await SubmissionProcessorService.processSubmission(messageJSON)
     } else {
       throw new Error(`Invalid topic: ${topic}`)
@@ -76,7 +76,7 @@ function check () {
 }
 
 if (consumer) {
-  const topics = [config.REVIEW_TOPIC, config.CREATE_SUBMISSION_TOPIC, config.UPDATE_SUBMISSION_TOPIC]
+  const topics = [config.REVIEW_TOPIC, config.CREATE_SUBMISSION_TOPIC]
 
   consumer
     .init([{

test/common/testData.js

@@ -85,38 +85,6 @@ const testTopics = {
         isFileSubmission: false
       }
     }
-  },
-  'submission.notification.update': {
-    operation: 'update',
-    requiredFields: ['topic', 'originator', 'timestamp', 'mime-type',
-      'payload.resource', 'payload.id', 'payload.url',
-      'payload.legacyChallengeId'],
-    integerFields: ['payload.legacyChallengeId'],
-    stringFields: ['originator', 'mime-type', 'payload.resource', 'payload.id',
-      'payload.url'],
-    dateFields: ['timestamp'],
-    booleanFields: [],
-    testMessage: {
-      topic: 'submission.notification.update',
-      originator: 'or-app',
-      timestamp: '2019-02-25T00:00:00',
-      'mime-type': 'application/json',
-      payload: {
-        resource: 'submission',
-        id: '104366f8-f46b-45db-a971-11bc69e6c8ff',
-        type: 'Contest Submission',
-        url: 'https://s3.amazonaws.com/topcoder-dev-submissions-dmz/30054740-8547899-SUBMISSION_ZIP-1554188341581.zip#testing',
-        memberId: 121212,
-        legacyChallengeId: 30049360,
-        created: '2019-04-02T06:59:29.785Z',
-        updated: '2019-04-03T06:59:29.785Z',
-        createdBy: 'user3',
-        updatedBy: 'user4',
-        submissionPhaseId: 345345,
-        fileType: 'zip',
-        isFileSubmission: true
-      }
-    }
   }
 }
 

test/common/testHelper.js

@@ -203,7 +203,7 @@ async function consumeMessages () {
   // consume and commit all not processed messages
   const consumer = new Kafka.GroupConsumer(helper.getKafkaOptions())
   await consumer.init([{
-    subscriptions: [config.REVIEW_TOPIC, config.CREATE_SUBMISSION_TOPIC, config.UPDATE_SUBMISSION_TOPIC,
+    subscriptions: [config.REVIEW_TOPIC, config.CREATE_SUBMISSION_TOPIC,
       testData.avScanTopic, testData.reviewActionTopic],
     handler: (messageSet, topic, partition) => Promise.each(messageSet,
       (m) => consumer.commitOffset({ topic, partition, offset: m.offset }))

test/e2e/submission.processor.test.js

@@ -26,7 +26,7 @@ describe('Topcoder - Scorecard Submission Processor E2E Test', () => {
     testHelper.clearInterceptedLogging()
   })
 
-  for (const topic of [config.CREATE_SUBMISSION_TOPIC, config.UPDATE_SUBMISSION_TOPIC]) {
+  for (const topic of [config.CREATE_SUBMISSION_TOPIC]) {
     const { operation, requiredFields, integerFields, stringFields, dateFields,
       booleanFields, testMessage } = testTopics[topic]
 

test/unit/submission.processor.test.js

@@ -24,7 +24,7 @@ describe('Topcoder - Scorecard Submission Processor Unit Test', () => {
     testHelper.clearInterceptedLogging()
   })
 
-  for (const topic of [config.CREATE_SUBMISSION_TOPIC, config.UPDATE_SUBMISSION_TOPIC]) {
+  for (const topic of [config.CREATE_SUBMISSION_TOPIC]) {
     const { operation, requiredFields, integerFields, stringFields, dateFields,
       booleanFields, testMessage } = testTopics[topic]