Skip to content

Dev #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 33 commits into from
Feb 11, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
e7cc26d
for identity changes
Dec 17, 2019
69ea871
Update package.json
nkumar-topcoder Dec 17, 2019
c92df70
producer dynamodb file
Dec 17, 2019
0326ce2
Update config.yml
nkumar-topcoder Dec 17, 2019
810d0c9
Update config.yml
nkumar-topcoder Dec 17, 2019
c3d5062
Update config.yml
nkumar-topcoder Dec 17, 2019
cda1e5f
fix dynomodb ecsrole
Dec 18, 2019
40ab9ed
Update config.yml
nkumar-topcoder Dec 18, 2019
8855c20
[skip ci]
nkumar-topcoder Dec 18, 2019
95d4add
[skip ci]
nkumar-topcoder Dec 18, 2019
7f5cbc8
[skip ci]
nkumar-topcoder Dec 18, 2019
4369241
[skip ci]
nkumar-topcoder Dec 18, 2019
f862ef0
Update config.yml
nkumar-topcoder Dec 18, 2019
3dbfde7
log update
Jan 2, 2020
a045399
Merge branch 'dev-test-pg' of https://github.com/topcoder-platform/po…
Jan 2, 2020
5865d47
refactoring code
Jan 3, 2020
02f5826
cleaning...
Jan 3, 2020
ca055bd
cleaning....
Jan 3, 2020
2994830
cleaning...
Jan 3, 2020
3802ff4
cleaning...
Jan 3, 2020
acc2e3f
rearranging audit code..
Jan 3, 2020
d3c05b8
Merge pull request #3 from topcoder-platform/dev-test-pg-rf
nkumar-topcoder Jan 3, 2020
2a96987
[skip ci]
nkumar-topcoder Jan 4, 2020
96b4bec
slack integration
Jan 7, 2020
cdbdfbd
reconsiler check-in
Jan 11, 2020
32a4ded
dynamodb payload, config changes
Jan 11, 2020
3e81f9b
[skip ci]
nkumar-topcoder Jan 11, 2020
acc099d
[skip ci]
nkumar-topcoder Jan 11, 2020
2db7b7a
Merge pull request #4 from topcoder-platform/dev-test-pg
nkumar-topcoder Jan 14, 2020
d5901d9
Update default.js
nkumar-topcoder Jan 14, 2020
e13bee4
[skip ci]
nkumar-topcoder Jan 20, 2020
ef7c139
[skip ci]
nkumar-topcoder Jan 20, 2020
f4356e1
[skip ci]
nkumar-topcoder Jan 20, 2020
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 26 additions & 5 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,16 +41,22 @@ build_steps: &build_steps
command: |
./awsconfiguration.sh ${DEPLOY_ENV}
source awsenvconf
./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar

./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar
source buildenvvar
./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer

echo "Running Masterscript - deploy postgres-ifx-processer producer"
if [ -e ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json; fi
./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar
source buildenvvar
./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer


echo "Running Masterscript - deploy postgres-ifx-processer producer_dd"
if [ -e ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json; fi
./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer_dd-deployvar
source buildenvvar
./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
jobs:
# Build & Deploy against development backend #
"build-dev":
Expand All @@ -61,7 +67,16 @@ jobs:
GLOBAL_ENV: "dev"
APP_NAME: "postgres-ifx-processer"
steps: *build_steps
# Build & Deploy against production backend
# Build & Deploy against development backend #
"build-test":
<<: *defaults
environment:
DEPLOY_ENV: "DEV"
LOGICAL_ENV: "TEST"
GLOBAL_ENV: "dev"
APP_NAME: "postgres-ifx-processer"
steps: *build_steps
# Build & Deploy against production backend
"build-prod":
<<: *defaults
environment:
Expand All @@ -82,10 +97,16 @@ workflows:
only:
- dev
- dev-retryfeature
- "build-test":
context : org-global
filters:
branches:
only:
- dev-test-pg
- dev-test-pg-rf
- "build-prod":
context : org-global
filters:
branches:
only:
- master

18 changes: 16 additions & 2 deletions config/default.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ module.exports = {
database: process.env.PG_DATABASE || 'postgres', // database must exist before running the tool
password: process.env.PG_PASSWORD || 'password',
port: parseInt(process.env.PG_PORT, 10) || 5432,
triggerFunctions: process.env.TRIGGER_FUNCTIONS || ['db_notifications'], // List of trigger functions to listen to
triggerTopics: process.env.TRIGGER_TOPICS || ['db.postgres.sync'], // Names of the topic in the trigger payload
triggerFunctions: process.env.TRIGGER_FUNCTIONS || ['dev_db_notifications'], // List of trigger functions to listen to
triggerTopics: process.env.TRIGGER_TOPICS || ['dev.db.postgres.sync'], // Names of the topic in the trigger payload
triggerOriginators: process.env.TRIGGER_ORIGINATORS || ['tc-postgres-delta-processor'] // Names of the originator in the trigger payload
},
KAFKA: { // Kafka connection options
Expand All @@ -38,6 +38,20 @@ module.exports = {
errorTopic: process.env.ERROR_TOPIC || 'db.scorecardtable.error',
recipients: ['[email protected]'] // Kafka partitions to use
},
SLACK: {
URL: process.env.SLACKURL || 'us-east-1',
SLACKCHANNEL: process.env.SLACKCHANNEL || 'ifxpg-migrator',
SLACKNOTIFY: process.env.SLACKNOTIFY || 'false'
},
RECONSILER:{
RECONSILER_START: process.env.RECONSILER_START || 10,
RECONSILER_END: process.env.RECONSILER_END || 5,
RECONSILER_DURATION_TYPE: process.env.RECONSILER_DURATION_TYPE || 'm'
},
DYNAMODB:
{
DYNAMODB_TABLE: process.env.DYNAMODB_TABLE || 'dev_pg_ifx_payload_sync'
},

AUTH0_URL: process.env.AUTH0_URL ,
AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE ,
Expand Down
6 changes: 3 additions & 3 deletions config/test.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ module.exports = {
host: process.env.INFORMIX_HOST || 'localhost',
port: parseInt(process.env.INFORMIX_PORT, 10) || 2021,
user: process.env.INFORMIX_USER || 'informix',
password: process.env.INFORMIX_PASSWORD || '1nf0rm1x',
database: process.env.INFORMIX_DATABASE || 'tcs_catalog',
server: process.env.INFORMIX_SERVER || 'informixoltp_tcp',
password: process.env.INFORMIX_PASSWORD || 'password',
database: process.env.INFORMIX_DATABASE || 'db',
server: process.env.INFORMIX_SERVER || 'informixp',
minpool: parseInt(process.env.MINPOOL, 10) || 1,
maxpool: parseInt(process.env.MAXPOOL, 10) || 60,
maxsize: parseInt(process.env.MAXSIZE, 10) || 0,
Expand Down
4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@
"lint:fix": "standard --env mocha --fix",
"producer": "node ./src/producer.js",
"consumer": "node ./src/consumer.js",
"start": "npm run producer & npm run consumer"
"producer_dd": "node ./src/producer.js failover",
"start": "npm run producer & npm run producer_dd & npm run consumer"
},
"author": "Topcoder",
"license": "ISC",
"dependencies": {
"aws-sdk": "*",
"config": "^3.2.2",
"informix-wrapper": "git+https://github.com/appirio-tech/informix-wrapper.git",
"no-kafka": "^3.4.3",
Expand Down
Loading