merge to master from dev

.circleci/config.yml

@@ -6,10 +6,11 @@ defaults: &defaults
 install_dependency: &install_dependency
           name: Installation of build and deployment dependencies.
           command: |
+            sudo apt-get update
             sudo apt install jq
             sudo pip install awscli --upgrade
             sudo pip install docker-compose
-            sudo apt-get install default-jdk
+            sudo apt-get install default-jdk --fix-missing
 install_deploysuite: &install_deploysuite
           name: Installation of install_deploysuite.
           #Git Clone -change back to v1.3 or latest once counter var is generalized.
@@ -39,31 +40,51 @@ build_steps: &build_steps
       - deploy:
           name: Running MasterScript.
           command: |
+
             ./awsconfiguration.sh ${DEPLOY_ENV}
             source awsenvconf
+            #scorecard test consumer remove later
+            #./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-consumer_scorecard-deployvar
+            #source buildenvvar
+            #./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
+
+            #scorecard test producer remove later
+            #echo "Running Masterscript - deploy postgres-ifx-processer producer"
+            #if [ -e ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json ]; then  sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json; fi
+            #./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer_scorecard-deployvar
+            #source buildenvvar
+            #./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
 
             # ./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar
             #source buildenvvar
             #./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
 
+
             echo "Running Masterscript - deploy postgres-ifx-processer producer"
-            #if [ -e ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json ]; then  sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json; fi
+            if [ -e ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json ]; then  sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json; fi
             ./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar
             source buildenvvar
             ./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
 
-            #echo "Running Masterscript - deploy postgres-ifx-processer producer_dd"
-            #if [ -e ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json ]; then  sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json; fi
-            #./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer_dd-deployvar
-            #source buildenvvar
-            #./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
+            echo "Running Masterscript - deploy postgres-ifx-processer producer_dd"
+            if [ -e ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json ]; then  sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json; fi
+            ./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer_dd-deployvar
+            source buildenvvar
+            ./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
+
 
             #echo "Running Masterscript - deploy postgres-ifx-processer reconsiler1"
             #if [ -e ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar.json ]; then  sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar.json; fi
             #./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar
             #source buildenvvar
             #./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
-
+
+            #echo "Running Masterscript - deploy postgres-ifx-processer reconsiler2"
+            #if [ -e ${LOGICAL_ENV}-${APP_NAME}-reconsiler2-deployvar.json ]; then  sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar.json; fi
+            #./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-reconsiler2-deployvar
+            #source buildenvvar
+            #./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
+
 jobs:
   # Build & Deploy against development backend #
   "build-dev":
@@ -103,13 +124,12 @@ workflows:
             branches:
               only:
                 - dev
-                - dev-retryfeature
       - "build-test":
           context : org-global
           filters:
             branches:
-              only:
-                - dev-test-pg             
+              only:  
+                - dev-test-pg
                 - dev-test-pg-rf             
       - "build-prod":
           context : org-global

config/default.js

@@ -22,8 +22,9 @@ module.exports = {
     database: process.env.PG_DATABASE || 'postgres', // database must exist before running the tool
     password: process.env.PG_PASSWORD || 'password',
     port: parseInt(process.env.PG_PORT, 10) || 5432,
-    triggerFunctions: process.env.TRIGGER_FUNCTIONS || ['prod_db_notifications'], // List of trigger functions to listen to
-    triggerTopics: process.env.TRIGGER_TOPICS || ['prod.db.postgres.sync'], // Names of the topic in the trigger payload
+
+    triggerFunctions: process.env.TRIGGER_FUNCTIONS || 'dev_db_notifications', // List of trigger functions to listen to
+    triggerTopics: process.env.TRIGGER_TOPICS || ['dev.db.postgres.sync'], // Names of the topic in the trigger payload
     triggerOriginators: process.env.TRIGGER_ORIGINATORS || ['tc-postgres-delta-processor'] // Names of the originator in the trigger payload
   },
   KAFKA: { // Kafka connection options
@@ -34,32 +35,36 @@ module.exports = {
     },
     topic: process.env.KAFKA_TOPIC || 'db.topic.sync', // Kafka topic to push and receive messages
     partition: process.env.partition || [0], // Kafka partitions to use
-    maxRetry: process.env.MAX_RETRY || 3,
+    maxRetry: process.env.MAX_RETRY || 10,
     errorTopic: process.env.ERROR_TOPIC || 'db.scorecardtable.error',
-    recipients: ['[email protected]'] // Kafka partitions to use
+    recipients: ['[email protected]'], // Kafka partitions to use,
+    KAFKA_URL: process.env.KAFKA_URL,
+    KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID || 'dev-postgres-ifx-consumer',
+    KAFKA_CLIENT_CERT: process.env.KAFKA_CLIENT_CERT ? process.env.KAFKA_CLIENT_CERT.replace('\\n', '\n') : null,
+    KAFKA_CLIENT_CERT_KEY: process.env.KAFKA_CLIENT_CERT_KEY ? process.env.KAFKA_CLIENT_CERT_KEY.replace('\\n', '\n') : null,
   },
   SLACK: {
     URL: process.env.SLACKURL || 'us-east-1',
     SLACKCHANNEL: process.env.SLACKCHANNEL || 'ifxpg-migrator',
-    SLACKNOTIFY:  process.env.SLACKNOTIFY || 'false'
+    SLACKNOTIFY: process.env.SLACKNOTIFY || 'false'
   },
-   RECONSILER:{
+  RECONSILER: {
     RECONSILER_START: process.env.RECONSILER_START || 5,
     RECONSILER_END: process.env.RECONSILER_END || 1,
     RECONSILER_DURATION_TYPE: process.env.RECONSILER_DURATION_TYPE || 'm'
   },
   DYNAMODB:
-	{
-	DYNAMODB_TABLE: process.env.DYNAMODB_TABLE || 'dev_pg_ifx_payload_sync',
-  	DD_ElapsedTime: process.env.DD_ElapsedTime || 600000
-	},
+  {
+    DYNAMODB_TABLE: process.env.DYNAMODB_TABLE || 'dev_pg_ifx_payload_sync',
+    DD_ElapsedTime: process.env.DD_ElapsedTime || 600000
+  },
 
-	AUTH0_URL: process.env.AUTH0_URL ,
-	AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE ,
-	TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME ,
-	AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID ,
-	AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET  ,
-	BUSAPI_URL : process.env.BUSAPI_URL ,
-	KAFKA_ERROR_TOPIC : process.env.KAFKA_ERROR_TOPIC ,
-	AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL 
+  AUTH0_URL: process.env.AUTH0_URL,
+  AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE,
+  TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
+  AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
+  AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
+  BUSAPI_URL: process.env.BUSAPI_URL,
+  KAFKA_ERROR_TOPIC: process.env.KAFKA_ERROR_TOPIC,
+  AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL
 }