Skip to content
This repository was archived by the owner on Mar 13, 2025. It is now read-only.

Commit eff0717

Browse files
jmgasperjmgasper
committed
DynamoDB changes
1 parent b1af73c commit eff0717

16 files changed

+518
-161
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@
55

66
*.cer
77
*.key
8+
.idea

‎README.md

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
- Nodejs 8 is required
44
- [Apache Kafka](https://kafka.apache.org/)
5-
- MongoDB 3.4
5+
- DynamoDB
66

77
## Install dependencies
88

@@ -29,7 +29,6 @@ The following config parameters are supported, they are defined in `config/defau
2929
| TOPIC | the kafka subscribe topic name | tc-x-events |
3030
| PARTITION | the kafka partition | 0|
3131
| KAFKA_OPTIONS | the connection option for kafka | see below about KAFKA options |
32-
| MONGODB_URL | the MongoDB URL which must be same as Topcoder x tool | mongodb://127.0.0.1:27017/topcoderx |
3332
|TC_DEV_ENV| the flag whether to use topcoder development api or production| false|
3433
| TC_AUTHN_URL | the Topcoder authentication url | https://topcoder-dev.auth0.com/oauth/ro |
3534
| TC_AUTHN_REQUEST_BODY | the Topcoder authentication request body. This makes use of some environment variables: `TC_USERNAME`, `TC_PASSWORD`, `TC_CLIENT_ID`, `CLIENT_V2CONNECTION` | see `default.js` |
@@ -49,6 +48,10 @@ The following config parameters are supported, they are defined in `config/defau
4948
|RETRY_COUNT| the number of times an event should be retried to process| 3|
5049
|RETRY_INTERVAL| the interval at which the event should be retried to process in milliseconds | 120000|
5150
|READY_FOR_REVIEW_ISSUE_LABEL| the label name for ready for review, should be one of the label configured in topcoder x ui|'tcx_ReadyForReview'|
51+
|AWS_ACCESS_KEY_ID | The Amazon certificate key to use when connecting. Use local dynamodb you can set fake value|FAKE_ACCESS_KEY_ID |
52+
|AWS_SECRET_ACCESS_KEY | The Amazon certificate access key to use when connecting. Use local dynamodb you can set fake value|FAKE_SECRET_ACCESS_KEY |
53+
|AWS_REGION | The Amazon certificate region to use when connecting. Use local dynamodb you can set fake value|FAKE_REGION |
54+
|IS_LOCAL | Use Amazon DynamoDB Local or server. |true |
5255

5356
KAFKA_OPTIONS should be object as described in https://github.com/oleksiyk/kafka#ssl
5457
For using with SSL, the options should be as

‎config/default.js

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ module.exports = {
2424
passphrase: 'secret', // NOTE:* This configuration specifies the private key passphrase used while creating it.
2525
}
2626
},
27-
MONGODB_URL: process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/topcoderx',
2827
TC_DEV_ENV: process.env.NODE_ENV === 'production' ? false : true,
2928
TC_AUTHN_URL: process.env.TC_AUTHN_URL || 'https://topcoder-dev.auth0.com/oauth/ro',
3029
TC_AUTHN_REQUEST_BODY: {
@@ -91,4 +90,10 @@ module.exports = {
9190
RETRY_COUNT: process.env.RETRY_COUNT || 2,
9291
RETRY_INTERVAL: process.env.RETRY_INTERVAL || 120000, // 2 minutes
9392
CANCEL_CHALLENGE_INTERVAL: process.env.CANCEL_CHALLENGE_INTERVAL || 24 * 60 * 60 * 1000, // 24 Hours
93+
DYNAMODB: {
94+
AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID || '',
95+
AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY || '',
96+
AWS_REGION: process.env.AWS_REGION || '',
97+
IS_LOCAL: process.env.IS_LOCAL || '',
98+
},
9499
};

‎configuration.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ The following config parameters are supported, they are defined in `config/defau
66
| :----------------------------- | :----------------------------------------: | :------------------------------: |
77
| LOG_LEVEL | the log level | debug |
88
| PARTITION | The Kafka partition | 0|
9-
| MONGODB_URI | The MongoDB URI. This needs to be the same MongoDB used by topcoder-x-receiver, topcoder-x-processor, and topcoder-x-site | mongodb://127.0.0.1:27017/topcoderx |
109
|TOPIC | The Kafka topic where events are published. This must be the same as the configured value for topcoder-x-processor| |
1110
|KAFKA_OPTIONS | Kafka connection options| |
1211
|KAFKA_HOST | The Kafka host to connect to| localhost:9092 |
@@ -34,6 +33,10 @@ The following config parameters are supported, they are defined in `config/defau
3433
|NOT_READY_ISSUE_LABEL| the label name for not ready, should be one of the label configured in topcoder x ui|
3534
'Not Ready'|
3635
|CANCEL_CHALLENGE_INTERVAL| the time in millisecond after which the challenge will be closed| '24*60*60*1000'|
36+
|AWS_ACCESS_KEY_ID | The Amazon certificate key to use when connecting. Use local dynamodb you can set fake value|FAKE_ACCESS_KEY_ID |
37+
|AWS_SECRET_ACCESS_KEY | The Amazon certificate access key to use when connecting. Use local dynamodb you can set fake value|FAKE_SECRET_ACCESS_KEY |
38+
|AWS_REGION | The Amazon certificate region to use when connecting. Use local dynamodb you can set fake value|FAKE_REGION |
39+
|IS_LOCAL | Use Amazon DynamoDB Local or server. |true |
3740

3841
KAFKA_OPTIONS should be object as described in https://github.com/oleksiyk/kafka#ssl
3942
For using with SSL, the options should be as

‎models/CopilotPayment.js

Lines changed: 58 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,21 +10,67 @@
1010

1111
'use strict';
1212

13-
const mongoose = require('mongoose');
13+
const dynamoose = require('dynamoose');
1414

15-
const schema = new mongoose.Schema({
16-
project: {type: mongoose.Schema.Types.ObjectId, ref: 'Project'},
15+
const Schema = dynamoose.Schema;
16+
17+
const schema = new Schema({
18+
id: {
19+
type: String,
20+
hashKey: true,
21+
required: true
22+
},
23+
project: {
24+
type: String,
25+
index: {
26+
global: true,
27+
rangeKey: 'id',
28+
project: true,
29+
name: 'ProjectIndex'
30+
}
31+
},
1732
amount: {type: Number, required: true},
1833
description: {type: String, required: true},
19-
challengeId: {type: Number, required: false},
20-
closed: {type: Boolean, required: true, default: false},
21-
username: {type: String, required: true},
22-
status: {type: String}
34+
challengeId: {
35+
type: Number,
36+
required: false,
37+
index: {
38+
global: true,
39+
rangeKey: 'id',
40+
project: true,
41+
name: 'ChallengeIdIndex'
42+
}
43+
},
44+
closed: {
45+
type: String,
46+
required: true,
47+
default: 'false',
48+
index: {
49+
global: true,
50+
rangeKey: 'id',
51+
project: true,
52+
name: 'ClosedIndex'
53+
}
54+
},
55+
username: {
56+
type: String,
57+
required: true,
58+
index: {
59+
global: true,
60+
rangeKey: 'id',
61+
project: true,
62+
name: 'UsernameIndex'
63+
}
64+
},
65+
status: {
66+
type: String,
67+
index: {
68+
global: true,
69+
rangeKey: 'id',
70+
project: true,
71+
name: 'StatusIndex'
72+
}
73+
}
2374
});
2475

25-
schema.index({project: 1});
26-
schema.index({username: 1});
27-
schema.index({challengeId: 1});
28-
schema.index({closed: 1});
29-
3076
module.exports = schema;

‎models/Issue.js

Lines changed: 47 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -8,36 +8,60 @@
88
* @author TCSCODER
99
* @version 1.0
1010
*/
11-
const mongoose = require('mongoose');
11+
const dynamoose = require('dynamoose');
1212

13-
const schema = new mongoose.Schema({
13+
const Schema = dynamoose.Schema;
14+
15+
const schema = new Schema({
16+
id: {type: String, hashKey: true, required: true},
1417
// From the receiver service
15-
number: { type: Number, required: true },
16-
title: { type: String, required: true },
17-
body: String,
18-
prizes: [{ type: Number, required: true }], // extracted from title
19-
provider: { type: String, required: true }, // github or gitlab
20-
repositoryId: { type: Number, required: true },
21-
labels: [{ type: String, required: true }],
22-
assignee: { type: String, required: false },
18+
number: {
19+
type: Number,
20+
required: true,
21+
index: {
22+
global: true,
23+
rangeKey: 'id',
24+
project: true,
25+
name: 'NumberIndex'
26+
}
27+
},
28+
title: {type: String, required: true},
29+
body: {type: String},
30+
prizes: {type: [Number], required: true}, // extracted from title
31+
provider: {
32+
type: String,
33+
required: true,
34+
index: {
35+
global: true,
36+
rangeKey: 'id',
37+
project: true,
38+
name: 'ProviderIndex'
39+
}
40+
}, // github or gitlab
41+
repositoryId: {
42+
type: Number,
43+
required: true,
44+
index: {
45+
global: true,
46+
rangeKey: 'id',
47+
project: true,
48+
name: 'RepositoryIdIndex'
49+
}
50+
},
51+
labels: {
52+
type: [String],
53+
required: false
54+
},
55+
assignee: {type: String, required: false},
2356
updatedAt: {
2457
type: Date,
2558
default: Date.now
2659
},
2760
// From topcoder api
28-
challengeId: { type: Number, required: false, unique: true, sparse: true },
29-
projectId: { type: mongoose.Schema.Types.ObjectId, ref: 'Project' },
30-
status: { type: String },
31-
assignedAt: { type: Date, required: false }
61+
challengeId: {type: Number, required: false},
62+
projectId: {type: String},
63+
status: {type: String},
64+
assignedAt: {type: Date, required: false}
3265
});
3366

34-
// Issue number, provider, repositoryId must be unique
35-
schema.index({ number: 1, provider: 1, repositoryId: 1 }, { unique: true });
36-
schema.index({ labels: 1 });
37-
schema.index({ projectId: 1 });
38-
39-
schema.pre('save', function preSave(next) {
40-
this.updatedAt = Date.now(); // eslint-disable-line
41-
return next();
42-
});
4367
module.exports = schema;

‎models/Project.js

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,27 @@
88
* @author TCSCODER
99
* @version 1.0
1010
*/
11-
const mongoose = require('mongoose');
11+
const dynamoose = require('dynamoose');
1212

13-
const schema = new mongoose.Schema({
13+
const Schema = dynamoose.Schema;
14+
15+
const schema = new Schema({
16+
id: {
17+
type: String,
18+
hashKey: true,
19+
required: true
20+
},
1421
title: {type: String, required: true},
15-
tcDirectId: {type: Number, required: true},
22+
tcDirectId: {
23+
type: Number,
24+
required: true,
25+
index: {
26+
global: true,
27+
rangeKey: 'id',
28+
project: true,
29+
name: 'TcDirectIdIndex'
30+
}
31+
},
1632
repoUrl: {type: String, required: true},
1733
rocketChatWebhook: {type: String, required: false},
1834
rocketChatChannelName: {type: String, required: false},
@@ -22,6 +38,4 @@ const schema = new mongoose.Schema({
2238
copilot: {type: String, required: true}
2339
});
2440

25-
schema.index({tcDirectId: 1});
26-
2741
module.exports = schema;

‎models/User.js

Lines changed: 52 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,25 +7,63 @@
77
'use strict';
88

99
const _ = require('lodash');
10-
const mongoose = require('mongoose');
10+
const dynamoose = require('dynamoose');
1111
const constants = require('../constants');
1212

13-
const Schema = mongoose.Schema;
13+
const Schema = dynamoose.Schema;
1414

1515
const schema = new Schema({
16-
userProviderId: {type: Number, required: true},
17-
username: {type: String, required: true},
18-
role: {type: String, required: true, enum: _.values(constants.USER_ROLES)},
19-
type: {type: String, required: true, enum: _.values(constants.USER_TYPES)},
16+
id: {
17+
type: String,
18+
hashKey: true,
19+
required: true
20+
},
21+
userProviderId: {
22+
type: Number,
23+
required: true,
24+
index: {
25+
global: true,
26+
rangeKey: 'id',
27+
project: true,
28+
name: 'UsesProviderIdIndex'
29+
}
30+
},
31+
username: {
32+
type: String,
33+
required: true,
34+
index: {
35+
global: true,
36+
rangeKey: 'id',
37+
project: true,
38+
name: 'UsernameIndex'
39+
}
40+
},
41+
role: {
42+
type: String,
43+
required: true,
44+
enum: _.values(constants.USER_ROLES),
45+
index: {
46+
global: true,
47+
project: true,
48+
name: 'RoleIndex',
49+
rangeKey: 'id'
50+
}
51+
},
52+
type: {
53+
type: String,
54+
required: true,
55+
enum: _.values(constants.USER_TYPES),
56+
index: {
57+
global: true,
58+
rangeKey: 'id',
59+
name: 'TypeIndex',
60+
project: true
61+
}
62+
},
2063
// gitlab token data
21-
accessToken: String,
22-
accessTokenExpiration: Date,
23-
refreshToken: String
64+
accessToken: {type: String, required: false},
65+
accessTokenExpiration: {type: Date, required: false},
66+
refreshToken: {type: String, required: false}
2467
});
2568

26-
schema.index({userProviderId: 1});
27-
schema.index({username: 1});
28-
schema.index({role: 1});
29-
schema.index({type: 1});
30-
3169
module.exports = schema;

‎models/UserMapping.js

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,30 @@
33
*/
44
'use strict';
55

6-
const mongoose = require('mongoose');
6+
const dynamoose = require('dynamoose');
77

8-
const Schema = mongoose.Schema;
8+
const Schema = dynamoose.Schema;
99

1010
const schema = new Schema({
11-
topcoderUsername: {type: String, required: true, unique: true},
11+
id: {
12+
type: String,
13+
required: true,
14+
hashKey: true
15+
},
16+
topcoderUsername: {
17+
type: String,
18+
required: true,
19+
index: {
20+
global: true,
21+
project: true,
22+
rangKey: 'id',
23+
name: 'TopcoderUsernameIndex'
24+
}
25+
},
1226
githubUsername: String,
1327
gitlabUsername: String,
1428
githubUserId: Number,
1529
gitlabUserId: Number
1630
});
1731

18-
schema.index({topcoderUsername: 1}, {unique: true});
19-
2032
module.exports = schema;

0 commit comments

Comments
 (0)