Skip to content

Commit df124a0

Browse files
nkumar-topcodernkumar-topcoder
authored
Merge pull request #13 from topcoder-platform/dev
merge to master from dev
2 parents a7fee80 + be514a7 commit df124a0

12 files changed

+1323
-178
lines changed

‎.circleci/config.yml

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,11 @@ defaults: &defaults
66
install_dependency: &install_dependency
77
name: Installation of build and deployment dependencies.
88
command: |
9+
sudo apt-get update
910
sudo apt install jq
1011
sudo pip install awscli --upgrade
1112
sudo pip install docker-compose
12-
sudo apt-get install default-jdk
13+
sudo apt-get install default-jdk --fix-missing
1314
install_deploysuite: &install_deploysuite
1415
name: Installation of install_deploysuite.
1516
#Git Clone -change back to v1.3 or latest once counter var is generalized.
@@ -39,31 +40,51 @@ build_steps: &build_steps
3940
- deploy:
4041
name: Running MasterScript.
4142
command: |
43+
4244
./awsconfiguration.sh ${DEPLOY_ENV}
4345
source awsenvconf
46+
#scorecard test consumer remove later
47+
#./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-consumer_scorecard-deployvar
48+
#source buildenvvar
49+
#./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
50+
51+
#scorecard test producer remove later
52+
#echo "Running Masterscript - deploy postgres-ifx-processer producer"
53+
#if [ -e ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json; fi
54+
#./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer_scorecard-deployvar
55+
#source buildenvvar
56+
#./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
4457
4558
# ./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar
4659
#source buildenvvar
4760
#./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
4861
62+
4963
echo "Running Masterscript - deploy postgres-ifx-processer producer"
50-
#if [ -e ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json; fi
64+
if [ -e ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json; fi
5165
./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar
5266
source buildenvvar
5367
./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
5468
55-
#echo "Running Masterscript - deploy postgres-ifx-processer producer_dd"
56-
#if [ -e ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json; fi
57-
#./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer_dd-deployvar
58-
#source buildenvvar
59-
#./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
69+
echo "Running Masterscript - deploy postgres-ifx-processer producer_dd"
70+
if [ -e ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json; fi
71+
./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer_dd-deployvar
72+
source buildenvvar
73+
./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
74+
6075
6176
#echo "Running Masterscript - deploy postgres-ifx-processer reconsiler1"
6277
#if [ -e ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar.json; fi
6378
#./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar
6479
#source buildenvvar
6580
#./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
66-
81+
82+
#echo "Running Masterscript - deploy postgres-ifx-processer reconsiler2"
83+
#if [ -e ${LOGICAL_ENV}-${APP_NAME}-reconsiler2-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar.json; fi
84+
#./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-reconsiler2-deployvar
85+
#source buildenvvar
86+
#./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
87+
6788
jobs:
6889
# Build & Deploy against development backend #
6990
"build-dev":
@@ -103,13 +124,12 @@ workflows:
103124
branches:
104125
only:
105126
- dev
106-
- dev-retryfeature
107127
- "build-test":
108128
context : org-global
109129
filters:
110130
branches:
111-
only:
112-
- dev-test-pg
131+
only:
132+
- dev-test-pg
113133
- dev-test-pg-rf
114134
- "build-prod":
115135
context : org-global

‎config/default.js

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ module.exports = {
2222
database: process.env.PG_DATABASE || 'postgres', // database must exist before running the tool
2323
password: process.env.PG_PASSWORD || 'password',
2424
port: parseInt(process.env.PG_PORT, 10) || 5432,
25-
triggerFunctions: process.env.TRIGGER_FUNCTIONS || ['prod_db_notifications'], // List of trigger functions to listen to
26-
triggerTopics: process.env.TRIGGER_TOPICS || ['prod.db.postgres.sync'], // Names of the topic in the trigger payload
25+
26+
triggerFunctions: process.env.TRIGGER_FUNCTIONS || 'dev_db_notifications', // List of trigger functions to listen to
27+
triggerTopics: process.env.TRIGGER_TOPICS || ['dev.db.postgres.sync'], // Names of the topic in the trigger payload
2728
triggerOriginators: process.env.TRIGGER_ORIGINATORS || ['tc-postgres-delta-processor'] // Names of the originator in the trigger payload
2829
},
2930
KAFKA: { // Kafka connection options
@@ -34,32 +35,36 @@ module.exports = {
3435
},
3536
topic: process.env.KAFKA_TOPIC || 'db.topic.sync', // Kafka topic to push and receive messages
3637
partition: process.env.partition || [0], // Kafka partitions to use
37-
maxRetry: process.env.MAX_RETRY || 3,
38+
maxRetry: process.env.MAX_RETRY || 10,
3839
errorTopic: process.env.ERROR_TOPIC || 'db.scorecardtable.error',
39-
recipients: ['[email protected]'] // Kafka partitions to use
40+
recipients: ['[email protected]'], // Kafka partitions to use,
41+
KAFKA_URL: process.env.KAFKA_URL,
42+
KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID || 'dev-postgres-ifx-consumer',
43+
KAFKA_CLIENT_CERT: process.env.KAFKA_CLIENT_CERT ? process.env.KAFKA_CLIENT_CERT.replace('\\n', '\n') : null,
44+
KAFKA_CLIENT_CERT_KEY: process.env.KAFKA_CLIENT_CERT_KEY ? process.env.KAFKA_CLIENT_CERT_KEY.replace('\\n', '\n') : null,
4045
},
4146
SLACK: {
4247
URL: process.env.SLACKURL || 'us-east-1',
4348
SLACKCHANNEL: process.env.SLACKCHANNEL || 'ifxpg-migrator',
44-
SLACKNOTIFY: process.env.SLACKNOTIFY || 'false'
49+
SLACKNOTIFY: process.env.SLACKNOTIFY || 'false'
4550
},
46-
RECONSILER:{
51+
RECONSILER: {
4752
RECONSILER_START: process.env.RECONSILER_START || 5,
4853
RECONSILER_END: process.env.RECONSILER_END || 1,
4954
RECONSILER_DURATION_TYPE: process.env.RECONSILER_DURATION_TYPE || 'm'
5055
},
5156
DYNAMODB:
52-
{
53-
DYNAMODB_TABLE: process.env.DYNAMODB_TABLE || 'dev_pg_ifx_payload_sync',
54-
DD_ElapsedTime: process.env.DD_ElapsedTime || 600000
55-
},
57+
{
58+
DYNAMODB_TABLE: process.env.DYNAMODB_TABLE || 'dev_pg_ifx_payload_sync',
59+
DD_ElapsedTime: process.env.DD_ElapsedTime || 600000
60+
},
5661

57-
AUTH0_URL: process.env.AUTH0_URL ,
58-
AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE ,
59-
TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME ,
60-
AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID ,
61-
AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET ,
62-
BUSAPI_URL : process.env.BUSAPI_URL ,
63-
KAFKA_ERROR_TOPIC : process.env.KAFKA_ERROR_TOPIC ,
64-
AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL
62+
AUTH0_URL: process.env.AUTH0_URL,
63+
AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE,
64+
TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
65+
AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
66+
AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
67+
BUSAPI_URL: process.env.BUSAPI_URL,
68+
KAFKA_ERROR_TOPIC: process.env.KAFKA_ERROR_TOPIC,
69+
AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL
6570
}

0 commit comments

Comments
 (0)