Skip to content
This repository was archived by the owner on Mar 13, 2025. It is now read-only.

Commit 7d764b5

Browse files
jmgasperjmgasper
committed
Merge remote-tracking branch 'origin/develop'
# Conflicts: # config/default.js # services/CopilotPaymentService.js # services/IssueService.js # utils/git-helper.js # utils/topcoder-api-helper.js
2 parents f23cc56 + db3a02c commit 7d764b5

36 files changed

+3595
-1316
lines changed

‎.circleci/config.yml

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
version: 2
2+
defaults: &defaults
3+
docker:
4+
- image: circleci/python:2.7-stretch-browsers
5+
install_dependency: &install_dependency
6+
name: Installation of build and deployment dependencies.
7+
command: |
8+
sudo apt install jq
9+
sudo pip install awscli --upgrade
10+
sudo pip install docker-compose
11+
install_deploysuite: &install_deploysuite
12+
name: Installation of install_deploysuite.
13+
command: |
14+
git clone --branch v1.4.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
15+
cp ./../buildscript/master_deploy.sh .
16+
cp ./../buildscript/buildenv.sh .
17+
cp ./../buildscript/awsconfiguration.sh .
18+
restore_cache_settings_for_build: &restore_cache_settings_for_build
19+
key: docker-node-modules-{{ checksum "package-lock.json" }}
20+
21+
save_cache_settings: &save_cache_settings
22+
key: docker-node-modules-{{ checksum "package-lock.json" }}
23+
paths:
24+
- node_modules
25+
26+
builddeploy_steps: &builddeploy_steps
27+
- checkout
28+
- setup_remote_docker
29+
- run: *install_dependency
30+
- run: *install_deploysuite
31+
- restore_cache: *restore_cache_settings_for_build
32+
- run: ./build.sh
33+
- save_cache: *save_cache_settings
34+
- deploy:
35+
name: Running MasterScript.
36+
command: |
37+
./awsconfiguration.sh $DEPLOY_ENV
38+
source awsenvconf
39+
./buildenv.sh -e $DEPLOY_ENV -b ${DEPLOY_ENV}-${APPNAME}-deployvar
40+
source buildenvvar
41+
./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${DEPLOY_ENV}-global-appvar,${DEPLOY_ENV}-${APPNAME}-appvar -i ${APPNAME}
42+
43+
44+
jobs:
45+
# Build & Deploy against development backend
46+
"build-dev":
47+
<<: *defaults
48+
environment:
49+
DEPLOY_ENV: "dev"
50+
APPNAME: "topcoder-x-processor"
51+
steps: *builddeploy_steps
52+
53+
"build-prod":
54+
<<: *defaults
55+
environment:
56+
DEPLOY_ENV: "prod"
57+
APPNAME: "topcoder-x-processor"
58+
steps: *builddeploy_steps
59+
60+
workflows:
61+
version: 2
62+
build:
63+
jobs:
64+
# Development builds are executed on "develop" branch only.
65+
- "build-dev":
66+
context : org-global
67+
filters:
68+
branches:
69+
only:
70+
- develop
71+
72+
# Production builds are exectuted only on tagged commits to the
73+
# master branch.
74+
- "build-prod":
75+
context : org-global
76+
filters:
77+
branches:
78+
only: master

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@
55

66
*.cer
77
*.key
8+
.idea

‎README.md

Lines changed: 147 additions & 121 deletions
Large diffs are not rendered by default.

‎build.sh

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
set -eo pipefail
3+
APP_NAME="topcoder-x-processor"
4+
UPDATE_CACHE=""
5+
docker-compose -f docker/docker-compose.yml build $APP_NAME
6+
docker create --name app $APP_NAME:latest
7+
8+
if [ -d node_modules ]
9+
then
10+
mv package-lock.json old-package-lock.json
11+
docker cp app:/$APP_NAME/package-lock.json package-lock.json
12+
set +eo pipefail
13+
UPDATE_CACHE=$(cmp package-lock.json old-package-lock.json)
14+
set -eo pipefail
15+
else
16+
UPDATE_CACHE=1
17+
fi
18+
19+
if [ "$UPDATE_CACHE" == 1 ]
20+
then
21+
docker cp app:/$APP_NAME/node_modules .
22+
fi

‎config/default.js

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,28 +17,14 @@ module.exports = {
1717
PARTITION: process.env.PARTITION || 0,
1818
TOPIC: process.env.TOPIC || 'tc-x-events',
1919
KAFKA_OPTIONS: {
20-
connectionString: process.env.KAFKA_HOST || 'localhost:9092',
20+
connectionString: process.env.KAFKA_URL || 'localhost:9092',
2121
ssl: {
2222
cert: process.env.KAFKA_CLIENT_CERT || fs.readFileSync('./kafka_client.cer'), // eslint-disable-line no-sync
2323
key: process.env.KAFKA_CLIENT_CERT_KEY || fs.readFileSync('./kafka_client.key'), // eslint-disable-line no-sync
2424
passphrase: 'secret', // NOTE:* This configuration specifies the private key passphrase used while creating it.
2525
}
2626
},
27-
MONGODB_URL: process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/topcoderx',
2827
TC_DEV_ENV: process.env.NODE_ENV === 'production' ? false : true,
29-
TC_AUTHN_URL: process.env.TC_AUTHN_URL || 'https://topcoder-dev.auth0.com/oauth/ro',
30-
TC_AUTHN_REQUEST_BODY: {
31-
username: process.env.TC_USERNAME || 'mess',
32-
password: process.env.TC_PASSWORD || 'appirio123',
33-
client_id: process.env.TC_CLIENT_ID || 'JFDo7HMkf0q2CkVFHojy3zHWafziprhT',
34-
sso: false,
35-
scope: 'openid profile offline_access',
36-
response_type: 'token',
37-
connection: process.env.CLIENT_V2CONNECTION || 'TC-User-Database',
38-
grant_type: 'password',
39-
device: 'Browser'
40-
},
41-
TC_AUTHZ_URL: process.env.TC_AUTHZ_URL || 'https://api.topcoder-dev.com/v3/authorizations',
4228
NEW_CHALLENGE_TEMPLATE: process.env.NEW_CHALLENGE_TEMPLATE || {
4329
milestoneId: 1,
4430
subTrack: 'FIRST_2_FINISH',
@@ -86,8 +72,23 @@ module.exports = {
8672
READY_FOR_REVIEW_ISSUE_LABEL: process.env.READY_FOR_REVIEW_ISSUE_LABEL || 'tcx_ReadyForReview',
8773
ASSIGNED_ISSUE_LABEL: process.env.READY_FOR_REVIEW_ISSUE_LABEL || 'tcx_Assigned',
8874
OPEN_FOR_PICKUP_ISSUE_LABEL: process.env.READY_FOR_REVIEW_ISSUE_LABEL || 'tcx_OpenForPickup',
75+
NOT_READY_ISSUE_LABEL: process.env.NOT_READY_ISSUE_LABEL || 'tcx_NotReady',
76+
CANCELED_ISSUE_LABEL: process.env.CANCELED_ISSUE_LABEL || 'tcx_Canceled',
8977
TC_OR_DETAIL_LINK: process.env.TC_OR_DETAIL_LINK || 'https://software.topcoder-dev.com/review/actions/ViewProjectDetails?pid=',
90-
RETRY_COUNT: process.env.RETRY_COUNT || 3,
78+
RETRY_COUNT: process.env.RETRY_COUNT || 2,
9179
RETRY_INTERVAL: process.env.RETRY_INTERVAL || 120000, // 2 minutes
92-
CANCEL_CHALLENGE_INTERVAL: process.env.CANCEL_CHALLENGE_INTERVAL || 24*60*60*1000,
80+
CANCEL_CHALLENGE_INTERVAL: process.env.CANCEL_CHALLENGE_INTERVAL || 24 * 60 * 60 * 1000, // 24 Hours
81+
DYNAMODB: {
82+
AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID,
83+
AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY,
84+
AWS_REGION: process.env.AWS_REGION,
85+
IS_LOCAL: process.env.IS_LOCAL
86+
},
87+
// Configuration for m2m token generation
88+
AUTH0_URL: process.env.AUTH0_URL, // Auth0 credentials for Submission Service
89+
AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE,
90+
TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME || 43200,
91+
AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
92+
AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
93+
AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL
9394
};

‎config/test.js

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,23 @@ const defaultConfig = require('./default');
1313
const testConfig = {
1414
TC_URL: process.env.TC_URL || 'https://www.topcoder-dev.com',
1515
TC_DEV_API_URL: process.env.TC_DEV_API_URL || 'https://api.topcoder-dev.com/v3',
16-
MAX_RETRY_COUNT: process.env.MAX_RETRY_COUNT || 25,
17-
WAIT_TIME: process.env.WAIT_TIME || 30000,
16+
MAX_RETRY_COUNT: process.env.MAX_RETRY_COUNT || 17,
17+
WAIT_TIME: process.env.WAIT_TIME || 60000,
1818
TC_DIRECT_ID: process.env.TC_DIRECT_ID || 7377,
1919
TOPCODER_USER_NAME: process.env.TOPCODER_USER_NAME || 'mess',
2020
HOOK_BASE_URL: process.env.HOOK_BASE_URL || '',
2121
GITHUB_ACCESS_TOKEN: process.env.GITHUB_ACCESS_TOKEN || '',
2222
GITHUB_REPOSITORY_NAME: process.env.GITHUB_REPOSITORY_NAME || '',
2323
GITLAB_USERNAME: process.env.GITLAB_USERNAME || '',
2424
GITLAB_PASSWORD: process.env.GITLAB_PASSWORD || '',
25-
GITLAB_REPOSITORY_NAME: process.env.GITLAB_REPOSITORY_NAME || '',
25+
GITLAB_REPOSITORY_NAME: process.env.GITLAB_REPOSITORY_NAME || 'test-unit',
26+
GITLAB_REPO_URL: process.env.GITLAB_REPO_URL || 'https://gitlab.com/nauhil/test-unit',
27+
LABELS: process.env.LABELS || [{ name: 'tcx_OpenForPickup', color: '428BCA' }, { name: 'tcx_Assigned', color: '004E00' }, { name: 'tcx_ReadyForReview', color: 'D1D100' }, { name: 'tcx_Paid', color: '7F8C8D' }, { name: 'tcx_Feedback', color: 'FF0000' }, { name: 'tcx_FixAccepted', color: '69D100' },
28+
{name:'tcx_NotReady', color: '000000'}],
29+
READY_FOR_REVIEW_ISSUE_LABEL: process.env.READY_FOR_REVIEW_ISSUE_LABEL || 'tcx_ReadyForReview',
30+
ASSIGNED_ISSUE_LABEL: process.env.READY_FOR_REVIEW_ISSUE_LABEL || 'tcx_Assigned',
31+
OPEN_FOR_PICKUP_ISSUE_LABEL: process.env.READY_FOR_REVIEW_ISSUE_LABEL || 'tcx_OpenForPickup',
32+
FIX_ACCEPTED_ISSUE_LABEL: process.env.FIX_ACCEPTED_ISSUE_LABEL || 'tcx_FixAccepted',
2633
};
2734

2835
module.exports = Object.assign(defaultConfig, testConfig);

‎configuration.md

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,12 @@ The following config parameters are supported, they are defined in `config/defau
66
| :----------------------------- | :----------------------------------------: | :------------------------------: |
77
| LOG_LEVEL | the log level | debug |
88
| PARTITION | The Kafka partition | 0|
9-
| MONGODB_URI | The MongoDB URI. This needs to be the same MongoDB used by topcoder-x-receiver, topcoder-x-processor, and topcoder-x-site | mongodb://127.0.0.1:27017/topcoderx |
109
|TOPIC | The Kafka topic where events are published. This must be the same as the configured value for topcoder-x-processor| |
1110
|KAFKA_OPTIONS | Kafka connection options| |
12-
|KAFKA_HOST | The Kafka host to connect to| localhost:9092 |
11+
|KAFKA_URL | The Kafka host to connect to| localhost:9092 |
1312
|KAFKA_CLIENT_CERT | The Kafka SSL certificate to use when connecting| Read from kafka_client.cer file, but this can be set as a string like it is on Heroku |
1413
|KAFKA_CLIENT_CERT_KEY | The Kafka SSL certificate key to use when connecting| Read from kafka_client.key file, but this can be set as a string like it is on Heroku|
1514
|TC_DEV_ENV| the flag whether to use topcoder development api or production| false|
16-
| TC_AUTHN_URL | the Topcoder authentication url | https://topcoder-dev.auth0.com/oauth/ro |
17-
| TC_AUTHN_REQUEST_BODY | the Topcoder authentication request body. This makes use of some environment variables: `TC_USERNAME`, `TC_PASSWORD`, `TC_CLIENT_ID`, `CLIENT_V2CONNECTION` | see `default.js` |
18-
| TC_AUTHZ_URL | the Topcoder authorization url | https://api.topcoder-dev.com/v3/authorizations |
1915
| NEW_CHALLENGE_TEMPLATE | the body template for new challenge request. You can change the subTrack, reviewTypes, technologies, .. here | see `default.js` |
2016
| NEW_CHALLENGE_DURATION_IN_DAYS | the duration of new challenge | 5 |
2117
| NODE_MAILER_OPTIONS| the node mailer smtp options, see [here](https://nodemailer.com/smtp/ for more detail)| see `default.js` |
@@ -31,7 +27,18 @@ The following config parameters are supported, they are defined in `config/defau
3127
|RETRY_COUNT| the number of times an event should be retried to process| 3|
3228
|RETRY_INTERVAL| the interval at which the event should be retried to process in milliseconds | 120000|
3329
|READY_FOR_REVIEW_ISSUE_LABEL| the label name for ready for review, should be one of the label configured in topcoder x ui|'tcx_ReadyForReview'|
30+
|NOT_READY_ISSUE_LABEL| the label name for not ready, should be one of the label configured in topcoder x ui|'tcx_NotReady'|
31+
|CANCELED_ISSUE_LABEL| the label name for canceled, should be one of the label configured in topcoder x ui|'tcx_Canceled'|
3432
|CANCEL_CHALLENGE_INTERVAL| the time in millisecond after which the challenge will be closed| '24*60*60*1000'|
33+
|AWS_ACCESS_KEY_ID | The Amazon certificate key to use when connecting. Use local dynamodb you can set fake value|FAKE_ACCESS_KEY_ID |
34+
|AWS_SECRET_ACCESS_KEY | The Amazon certificate access key to use when connecting. Use local dynamodb you can set fake value|FAKE_SECRET_ACCESS_KEY |
35+
|AWS_REGION | The Amazon certificate region to use when connecting. Use local dynamodb you can set fake value|FAKE_REGION |
36+
|IS_LOCAL | Use Amazon DynamoDB Local or server. |'false' |
37+
|AUTH0_URL| The Auth0 URL for generating Machine-to-machine token |https://topcoder-dev.auth0.com/oauth/token|
38+
|AUTH0_AUDIENCE| The audience of Auth0 to generate M2M Token |https://m2m.topcoder-dev.com/|
39+
|TOKEN_CACHE_TIME| The machine-to-machine token cache validation time |43200|
40+
|AUTH0_CLIENT_ID| The Auth0 ClientID for generating Machine-to-machine token ||
41+
|AUTH0_CLIENT_SECRET| The Auth0 Client Secret for generating Machine-to-machine token ||
3542

3643
KAFKA_OPTIONS should be object as described in https://github.com/oleksiyk/kafka#ssl
3744
For using with SSL, the options should be as
@@ -47,20 +54,21 @@ For using with SSL, the options should be as
4754

4855
The following config paramaters are supported in the test environment defined in `config/test.js` and can be configured in the system environment. Note that the test config inherits all config options present in the default config and adds/overrides some config options.
4956

50-
| Name | Description | Default |
51-
| :----------------------------- | :----------------------------------------: | :------------------------------: | |
52-
| TC_URL | the topcoder development url | https://www.topcoder-dev.com |
53-
| TC_DEV_API_URL | the topcoder development api url | https://api.topcoder-dev.com/v3|
54-
| MAX_RETRY_COUNT | the maximum number of times to re-test before concluding that test failed | https://api.topcoder-dev.com/v3|
55-
| WAIT_TIME | the amount of time in milliseconds to wait before running a re-test | 30000 |
56-
| TC_DIRECT_ID | the topcoder direct id of the repository which is set up with a valid billing account | 7377 |
57-
| TOPCODER_USER_NAME | a valid username for topcoder dev platform | mess |
58-
| HOOK_BASE_URL | the webhook url of topcoder-x-receiver | |
59-
| GITHUB_ACCESS_TOKEN | github personal access token | |
60-
| GITHUB_REPOSITORY_NAME | the name of the repository to create for testing (should not already exist) | |
61-
| GITLAB_USERNAME | gitlab username | |
62-
| GITLAB_PASSWORD | gitlab password | |
63-
| GITLAB_REPOSITORY_NAME | the name of the repository to create for testing (should not already exist) | |
57+
| Name | Description | Default |
58+
|:--|:--|:--|
59+
| TC_URL | the topcoder development url |https://www.topcoder-dev.com |
60+
| TC_DEV_API_URL | the topcoder development api url |https://api.topcoder-dev.com/v3|
61+
| MAX_RETRY_COUNT | the maximum number of times to re-test before concluding that test failed | 17 |
62+
| WAIT_TIME | the amount of time in milliseconds to wait before running a re-test | 60000 |
63+
| TC_DIRECT_ID | the topcoder direct id of the repository which is set up with a valid billing account | 7377 |
64+
| TOPCODER_USER_NAME | a valid username for topcoder dev platform | mess |
65+
| HOOK_BASE_URL | the webhook url of topcoder-x-receiver | |
66+
| GITHUB_ACCESS_TOKEN | github personal access token | |
67+
| GITHUB_REPOSITORY_NAME | the name of the repository to create for testing (should not already exist) | |
68+
| GITLAB_USERNAME | gitlab username | |
69+
| GITLAB_PASSWORD | gitlab password | |
70+
| GITLAB_REPOSITORY_NAME | the name of the repository to create for testing (should already exist) | |
71+
| GITLAB_REPO_URL | the URL of the repository to create for testing (should already exist) | |
6472

6573
## Github Verification
6674

@@ -71,7 +79,7 @@ Configure a Github project with a webhook with a format like this: https://<rece
7179
#### Smoke test
7280
- Create an issue in the repo, you can see the logs in `receiver`, the `issue.created` event is generated. You should then see the processor receive the event and process it accordingly. It's important to validate that the issue.created event is seen by the receiver *and* the processor. This ensures that the Kafka communication between the two services is working properly.
7381

74-
You can test other events, but just validating that an issue.created event is generated in Kafka is enough to smoke test the receiver is set up properly.
82+
You can test other events, but just validating that an issue.created event is generated in Kafka is enough to smoke test the receiver is set up properly.
7583

7684
## Github Verification
7785

‎docker/Dockerfile

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Use the base image with Node.js
2+
FROM node:8.12
3+
4+
# Copy the current directory into the Docker image
5+
COPY . /topcoder-x-processor
6+
7+
# Set working directory for future use
8+
WORKDIR /topcoder-x-processor
9+
10+
# Install the dependencies from package.json
11+
RUN npm install
12+
#RUN npm run build
13+
#RUN npm run test
14+
15+
CMD npm start

‎docker/docker-compose.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
version: '3'
2+
services:
3+
topcoder-x-processor:
4+
image: topcoder-x-processor:latest
5+
build:
6+
context: ../
7+
dockerfile: docker/Dockerfile

‎index.js

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,20 @@
44
'use strict';
55

66
const kafka = require('./utils/kafka');
7+
const logger = require('./utils/logger');
8+
9+
process.on('uncaughtException', (err) => {
10+
// Check if error related to Dynamodb conn
11+
if (err.code === 'NetworkingError' && err.region) {
12+
logger.error('DynamoDB connection failed.');
13+
}
14+
logger.logFullError(err, 'system');
15+
});
16+
17+
// handle and log unhanled rejection
18+
process.on('unhandledRejection', (err) => {
19+
logger.logFullError(err, 'system');
20+
});
721

822
// run the server
923
kafka.run();

‎models/CopilotPayment.js

Lines changed: 58 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,21 +10,67 @@
1010

1111
'use strict';
1212

13-
const mongoose = require('mongoose');
13+
const dynamoose = require('dynamoose');
1414

15-
const schema = new mongoose.Schema({
16-
project: {type: mongoose.Schema.Types.ObjectId, ref: 'Project'},
15+
const Schema = dynamoose.Schema;
16+
17+
const schema = new Schema({
18+
id: {
19+
type: String,
20+
hashKey: true,
21+
required: true
22+
},
23+
project: {
24+
type: String,
25+
index: {
26+
global: true,
27+
rangeKey: 'id',
28+
project: true,
29+
name: 'ProjectIndex'
30+
}
31+
},
1732
amount: {type: Number, required: true},
1833
description: {type: String, required: true},
19-
challengeId: {type: Number, required: false},
20-
closed: {type: Boolean, required: true, default: false},
21-
username: {type: String, required: true},
22-
status: {type: String}
34+
challengeId: {
35+
type: Number,
36+
required: false,
37+
index: {
38+
global: true,
39+
rangeKey: 'id',
40+
project: true,
41+
name: 'ChallengeIdIndex'
42+
}
43+
},
44+
closed: {
45+
type: String,
46+
required: true,
47+
default: 'false',
48+
index: {
49+
global: true,
50+
rangeKey: 'id',
51+
project: true,
52+
name: 'ClosedIndex'
53+
}
54+
},
55+
username: {
56+
type: String,
57+
required: true,
58+
index: {
59+
global: true,
60+
rangeKey: 'id',
61+
project: true,
62+
name: 'UsernameIndex'
63+
}
64+
},
65+
status: {
66+
type: String,
67+
index: {
68+
global: true,
69+
rangeKey: 'id',
70+
project: true,
71+
name: 'StatusIndex'
72+
}
73+
}
2374
});
2475

25-
schema.index({project: 1});
26-
schema.index({username: 1});
27-
schema.index({challengeId: 1});
28-
schema.index({closed: 1});
29-
3076
module.exports = schema;

0 commit comments

Comments
 (0)