Submission from Marathon Match Legacy challenge

Author: coderReview

Dockerfile

@@ -16,23 +16,41 @@ You can update the configuration file or set values to the corresponding environ
 - `KAFKA_URL` The comma delimited list of initial brokers list
 - `KAFKA_CLIENT_CERT` The client cert, can be (1) the path to the cert file, or (2) the cert content
 - `KAFKA_CLIENT_CERT_KEY` The client cert key, can be (1) the path to the cert key file, or (2) the cert key content
-- `KAFKA_NEW_SUBMISSION_TOPIC` The topic from which the app consumes events
-- `KAFKA_NEW_SUBMISSION_ORIGINATOR` The event originator
+- `KAFKA_NEW_SUBMISSION_TOPIC` The new submission topic from which the app consumes events
+- `KAFKA_NEW_SUBMISSION_ORIGINATOR` The new submission event originator
+- `KAFKA_UPDATE_SUBMISSION_TOPIC` The update submission topic from which the app consumes events
 - `SUBMISSION_API_URL` The Submission API URL
 - `SUBMISSION_TIMEOUT` The Submission API timeout
 - `DB_NAME` legacy database name 'dbname@db_server_name'
 - `DB_USERNAME` database username
 - `DB_PASSWORD` database password
 - `ID_SEQ_UPLOAD` upload database sequence
 - `ID_SEQ_SUBMISSION` submission database sequence
-
+- `AUTH0_URL` auth0 url
+- `AUTH0_AUDIENCE` auth0 audience
+- `TOKEN_CACHE_TIME` auth0 token cache time
+- `AUTH0_CLIENT_ID` auth0 client id
+- `AUTH0_CLIENT_SECRET` auth0 client secret
+- `CHALLENGE_INFO_API` The challenge info api template with {cid} gets replaced with challenge id
+- `CHALLENGE_SUBTRACK` The sub track of marathon match challenge 
+
+ `./config/production.js`, `./config/staging.js`, `./config/test.js` will use same configuration variables as `./config/default.js` except `./config/test.js` will have new configurations for test only.
+- `MOCK_SUBMISSION_API_PORT` The mock submission api port
+- `MOCK_SERVER_PORT` The mock server port for challenge api
+ 
+ `./config/mock.js` will use same configuration variables as `./config/default.js` except
+- `MOCK_SERVER_PORT` The mock server port for challenge api
+ 
+ `./test/test_files/sqlParams.json` will load necessary sql params used in test, it will only work if you run `./test/sql/test.sql`.
+ 
 > NOTE: ALL COMMANDS BELOW EXECUTE UNDER ```<legacy-sub-procecssor>``` directory
 
 To build the application you must set the `DB_SERVER_NAME` environment variable. This variable holds the database hostname.
 
 ## Build Application Docker Image
 We only need to do this once
 ```bash
+export DB_SERVER_NAME=informix
 docker-compose build lsp-app
 ```
 
@@ -58,19 +76,20 @@ docker-compose up lsp-app-install
 
 ## Deployment
 ```bash
+export DB_SERVER_NAME=informix
 docker-compose up lsp-app
 ```
 
 ## Running Test
 - Make sure you're running a clean database (you can take down tc-informix container and then up it again)
 - Stop `legacy-sub-processor` application if it was running
-- Install test data
+- Install test data and you may start service `tc-informix` with command `docker-compose up tc-informix` if not started 
 ```bash
 docker cp test/sql/test.sql iif_innovator_c:/
 docker exec -ti iif_innovator_c bash -c "source /home/informix/ifx_informixoltp_tcp.env && dbaccess - /test.sql"
 ```
 - Run kafka container (and create topic if you haven't do this before)
-- Run
+- Run (you can use  `docker-compose run lsp-app-test` too and it is more suitable for test command)
 ```bash
 docker-compose up lsp-app-test
 ```

Procfile

@@ -1,102 +1,91 @@
-# Topcoder - Submission Legacy Processor Application - Verification
-------------
-
-## Setup Kafka
-
-- Download Kafka v1.1.0: https://kafka.apache.org/downloads
-- Extract the downloaded file to a directory
-- To enable SSL, copy the following lines to the end of `<your-kafka-directory>/config/server.properties`
-
-    ```
-    listeners=plaintext://localhost:9092,ssl://localhost:9093
-    ssl.truststore.location=<absolute-path-to-server.truststore.jks>
-    ssl.truststore.password=test1234
-    ssl.keystore.location=<absolute-path-to-server.keystore.jks>
-    ssl.keystore.password=test1234
-    ssl.key.password=test1234
-    ```
-
-    Replace `<absolute-path-to-server.truststore.jks>` to the absolute path to the file `tc-submission-legacy-processor/test/kafka-ssl/server.truststore.jks`
-    Replace `<absolute-path-to-server.truststore.jks>` to the absolute path to the file `tc-submission-legacy-processor/test/kafka-ssl/server.keystore.jks`
-
-- Example on my Windows PC:
-
-    ```
-    listeners=plaintext://localhost:9092,ssl://localhost:9093
-    ssl.truststore.location=D:/Others/tc-submission-legacy-processor/tc-submission-legacy-processor/test/kafka-ssl/server.truststore.jks
-    ssl.truststore.password=test1234
-    ssl.keystore.location=D:/Others/tc-submission-legacy-processor/tc-submission-legacy-processor/test/kafka-ssl/server.keystore.jks
-    ssl.keystore.password=test1234
-    ssl.key.password=test1234
-    ```
-
-## Start ZooKeeper
-
-- `cd <your-kafka-directory>`
-- For Linux: `bin/zookeeper-server-start.sh config/zookeeper.properties`
-- For Windows: `bin\windows\zookeeper-server-start.bat config\zookeeper.properties`
-
-## Start Kafka
-
-- `cd <your-kafka-directory>`
-- For Linux: `bin/kafka-server-start.sh config/server.properties`
-- For Windows: `bin\windows\kafka-server-start.bat config/server.properties`
-
-## Create Test Topics in Kafka
-
-- `cd <your-kafka-directory>`
-- For Linux:
-    ```
-    bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic different-topic
-    bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic new-submission-topic
-    ```
-- For Windows:
-    ```
-    bin\windows\kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic different-topic
-    bin\windows\kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic new-submission-topic
-    ```
-
-## Run Unit Tests
-
-- `cd tc-submission-legacy-processor`
-- `npm i`
-- `npm test`
-
-The detailed test coverage report (html) is generated to `./coverage` directory
-
-## Manual Verification
-
-### Start the Mock Submission API
-
-- `cd tc-submission-legacy-processor`
-- `npm i`
-- `npm run mock-submission-api`
-
-### Start the Application in Test Environment
-
-- `cd tc-submission-legacy-processor`
-- For Linux: 
-    ```
-    export NODE_ENV=test
-    npm start
-    ```
-- For Windows: 
-    ```
-    set NODE_ENV=test
-    npm start
-    ```
-
-### Send Test Events and Verify
-
-- `cd tc-submission-legacy-processor`
-- For Linux: `export NODE_ENV=test`
-- For Windows: `set NODE_ENV=test`
-- Run `npm run produce-test-event 0` and verify that the app doesn't consume this message (no log)
-- Run `npm run produce-test-event 1` and verify that the app skips this message (log: `Skipped null or empty event`)
-- Run `npm run produce-test-event 2` and verify that the app skips this message (log: `Skipped null or empty event`)
-- Run `npm run produce-test-event 3` and verify that the app skips this message (log: `Skipped non well-formed JSON message: ...`)
-- Run `npm run produce-test-event 4` and verify that the app skips this message (log: ` Skipped invalid event, reasons: "topic" is required ...`)
-- Run `npm run produce-test-event 5` and verify that the app skips this message (log: `Skipped invalid event, reasons: "timestamp" must be...`)
-- Run `npm run produce-test-event 6` and verify that the app skips this message (log: `Skipped event from topic wrong-topic`)
-- Run `npm run produce-test-event 7` and verify that the app skips this message (log: `Skipped event from topic wrong-originator`)
-- Run `npm run produce-test-event 8` and verify that the app makes call to the Submission API successfully (log: `Updated to the Submission API: id 111, ...`) and the Mock Submission API receives `PUT /submissions/111` request with request body like `{"id":111,"legacySubmissionId":1531219317896}`. `legacySubmissionId` is current epoch in the current implementation.
+# Topcoder - Submission Legacy Processor Application - Verification
+------------
+
+Please check [docs/Validation.md](/docs/Validation.md) and  [docs/Verification_with_DB.md](/docs/Verification_with_DB.md).
+
+Please note currently you have to verify application with database or please check [docs/Verification_with_DB.md](/docs/Verification_with_DB.md) only.
+
+I would recommend you to verify and test with docker otherwise you need to check related Dockerfile to have better understanding about how to setup environment properly(not recommend).
+
+
+## Topcoder - Marathon Match - Legacy Processor
+Please verify under linux or osx and I test under Ubuntu 18.04 and OSX 12 and windows may have issues to verify with docker.
+
+Please check README.md and ensure you could run tests in docker environment successfully and you can check coverage folder to ensure 
+`src/services/MarathonSubmissionService.js`,`src/services/NonMarathonSubmissionService.js`,`src/services/LegacySubmissionIdService.js` are fully tested.
+
+## Setup data in direct
+Only necessary MM challenge related test data updated in `./test/sql/test.sql` and you can also setup complete test data using direct application.
+
+You can follow [docs/Verification_with_DB.md](/docs/Verification_with_DB.md) to setup MM challenge in direct app, choose Marathon Match under Data menu during creating challenge and create new project with billing account if error to save mm challenge, in last step you have to create as draft challenge,add user as Submitter and get match  submission phase id. 
+
+Currently tc-direct have [issue to save Match Round ID](https://github.com/appirio-tech/direct-app/issues/341) or you may see such logs from command `docker-compose logs tc-direct` if you want to save in page and refresh page.
+```bash 
+         | 07:48:37,097 ERROR [ExceptionMappingInterceptor] Invalid action class configuration that references an unknown class named [saveDraftContestAction]
+tc-direct_1            | java.lang.RuntimeException: Invalid action class configuration that references an unknown class named [saveDraftContestAction]
+```
+
+So you have to run such sql and replace <mm challenge id> with your new created mm challenge id.
+```bash
+database tcs_catalog;
+INSERT INTO informix.project_info
+(project_id, project_info_type_id, value, create_user, create_date, modify_user, modify_date)
+VALUES(<mm challenge id>, 56, 2001, '132456', current, '132456', current);
+```
+
+Even you run sql direct app will still fail to show details page with such error
+```bash 
+ java.lang.NullPointerException
+tc-direct_1            | 	at com.topcoder.direct.services.view.action.analytics.longcontest.MarathonMatchHelper.getMarathonMatchDetails(MarathonMatchHelper.java:115)
+tc-direct_1            | 	at com.topcoder.direct.services.view.action.contest.launch.GetContestAction.executeAction(GetContestAction.java:476)
+tc-direct_1            | 	at com.topcoder.direct.services.view.action.BaseDirectStrutsAction.execute(BaseDirectStrutsAction.java:305)
+tc-direct_1            | 	at sun.reflect.GeneratedMethodAccessor553.invoke(Unknown Source)
+
+```
+
+Please check https://github.com/appirio-tech/direct-app/blob/dev/src/java/main/com/topcoder/direct/services/view/action/contest/launch/GetContestAction.java#L502
+
+Even latest direct codes will comment out related codes to solve this issue so just leave direct page there.
+
+## Run Legacy Submission Proc. app
+
+Make sure related services started and test data prepared and start app with `NODE_ENV=mock` to mock challenge api otherwise new created mm challenge will still consider as non mm challenge
+```bash
+export NODE_ENV=mock
+export DB_SERVER_NAME=informix
+docker-compose up lsp-app
+```
+
+## Send Test data
+From previous data setup I got:
+- challengeId = 40005570
+- memberId = 132458 (user)
+- submissionPhaseId = 100024
+
+Let's send mm submission with example = 0 event to kafka:
+
+```bash
+docker exec -ti lsp-app bash -c "npm run produce-test-event mm 40005570 132458 100024 0"
+```
+
+Let's send mm submission with example = 1 event to kafka:
+
+```bash
+docker exec -ti lsp-app bash -c "npm run produce-test-event mm 40005570 132458 100024 1"
+```
+
+or you can run sample mm submission message directly(valid if run `test/sql/test.sql`)
+```bash
+docker exec -ti lsp-app bash -c "npm run produce-test-event 9"
+```
+
+
+Please note currently processor will call challenge api to check whether challenge is MM challenge and default server api.topcoder-dev.com may not exist mm challenge created in local direct application.
+So when we start app with NODE_ENV=mock it will use mock challenge api configurations and start mock challenge api server.
+
+## Verify Database
+Open your database explorer (**DBeaver** application, for instance). Connect to database informixoltp
+Check table: `long_component_state`, `long_submission` or run below sql
+```bash
+select lcs.status_id,lcs.points, lcs.example_submission_number,lcs.submission_number,ls.*  from informixoltp:long_submission ls, informixoltp:long_component_state lcs where ls.long_component_state_id=lcs.long_component_state_id
+```

README.md

@@ -28,6 +28,7 @@ echo "================================"
 echo "tc-informix has created"
 echo "Executing kafka topics"
 echo "================================"
+sleep 15
 docker exec -ti kafka bash -c "kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic submission.notification.create"
 docker exec -ti kafka bash -c "kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic submission.notification.update"
 echo "================================"

Validation.md

@@ -55,5 +55,9 @@ module.exports = {
 
   AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
 
-  AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET
+  AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
+
+  CHALLENGE_INFO_API: process.env.CHALLENGE_INFO_API || 'https://api.topcoder-dev.com/v4/challenges?filter=id={cid}', // {cid} gets replaced with challenge id
+
+  CHALLENGE_SUBTRACK: process.env.CHALLENGE_SUBTRACK || 'MARATHON_MATCH'
 }

build.sh

@@ -0,0 +1,9 @@
+/**
+ * The mock configuration.
+ */
+const MOCK_SERVER_PORT = 3003
+module.exports = {
+  MOCK_SERVER_PORT,
+  LOG_LEVEL: 'debug',
+  CHALLENGE_INFO_API: `http://localhost:${MOCK_SERVER_PORT}/challenges?filter=id={cid}`
+}

config/default.js

@@ -55,5 +55,7 @@ module.exports = {
 
   AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
 
-  AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET
+  AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
+
+  CHALLENGE_INFO_API: process.env.CHALLENGE_INFO_API
 }

config/mock.js

@@ -45,5 +45,7 @@ module.exports = {
   ID_SEQ_UPLOAD: process.env.ID_SEQ_UPLOAD || 'upload_id_seq',
 
   // The Informix Submission Table Sequence Name
-  ID_SEQ_SUBMISSION: process.env.ID_SEQ_SUBMISSION || 'submission_id_seq'
+  ID_SEQ_SUBMISSION: process.env.ID_SEQ_SUBMISSION || 'submission_id_seq',
+
+  CHALLENGE_INFO_API: process.env.CHALLENGE_INFO_API
 }

config/production.js

@@ -1,7 +1,11 @@
 /**
  * The test configuration.
  */
+const MOCK_SUBMISSION_API_PORT = 3000
+const MOCK_SERVER_PORT = 3001
 module.exports = {
+  MOCK_SUBMISSION_API_PORT,
+  MOCK_SERVER_PORT,
   LOG_LEVEL: 'debug',
 
   // The client group ID for committing and fetching offsets.
@@ -27,7 +31,7 @@ module.exports = {
   KAFKA_NEW_SUBMISSION_ORIGINATOR: 'new-submission-originator',
 
   // The Submission API URL
-  SUBMISSION_API_URL: 'http://localhost:3000',
+  SUBMISSION_API_URL: `http://localhost:${MOCK_SUBMISSION_API_PORT}`,
 
   // The Submission API timeout
   SUBMISSION_TIMEOUT: 2000,
@@ -45,5 +49,9 @@ module.exports = {
   ID_SEQ_UPLOAD: 'upload_id_seq',
 
   // The Informix Submission Table Sequence Name
-  ID_SEQ_SUBMISSION: 'submission_id_seq'
+  ID_SEQ_SUBMISSION: 'submission_id_seq',
+
+  CHALLENGE_INFO_API: `http://localhost:${MOCK_SERVER_PORT}/challenges?filter=id={cid}`, // {cid} gets replaced with challenge id
+
+  CHALLENGE_SUBTRACK: 'MARATHON_MATCH'
 }

config/staging.js

@@ -77,7 +77,6 @@ services:
     container_name: mock-submission-api
     volumes:
       - ".:/app"
-      - "PORT=3000"
     command: run mock-submission-api
 
   lsp-app-install:
@@ -98,7 +97,7 @@ services:
     image: lsp-app:latest
     container_name: lsp-app
     environment:
-      - "NODE_ENV=development"
+      - "NODE_ENV=${NODE_ENV:-development}"
       - "PORT=3300"
     volumes:
       - ".:/app"
@@ -150,6 +149,7 @@ services:
       - SUBMISSION_API_URL=${SUBMISSION_API_URL}
       - DB_USERNAME=${DB_USERNAME}
       - DB_PASSWORD=${DB_PASSWORD}
+      - CHALLENGE_INFO_API=${CHALLENGE_INFO_API}
     volumes:
       - ".:/app"
     command: run start:docker
@@ -175,6 +175,7 @@ services:
       - SUBMISSION_API_URL=${SUBMISSION_API_URL}
       - DB_USERNAME=${DB_USERNAME}
       - DB_PASSWORD=${DB_PASSWORD}
+      - CHALLENGE_INFO_API=${CHALLENGE_INFO_API}
     volumes:
       - ".:/app"
     command: run start:docker