Skip to content

Commit a6f011e

Browse files
nkumar-topcodernkumar-topcoder
authored
Merge pull request #12 from topcoder-platform/dev-test-pg
Dev test pg
2 parents 4f801ed + b8600d2 commit a6f011e

11 files changed

+1082
-175
lines changed

‎.circleci/config.yml

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,11 @@ defaults: &defaults
66
install_dependency: &install_dependency
77
name: Installation of build and deployment dependencies.
88
command: |
9+
sudo apt-get update
910
sudo apt install jq
1011
sudo pip install awscli --upgrade
1112
sudo pip install docker-compose
12-
sudo apt-get install default-jdk
13+
sudo apt-get install default-jdk --fix-missing
1314
install_deploysuite: &install_deploysuite
1415
name: Installation of install_deploysuite.
1516
#Git Clone -change back to v1.3 or latest once counter var is generalized.
@@ -39,30 +40,42 @@ build_steps: &build_steps
3940
- deploy:
4041
name: Running MasterScript.
4142
command: |
43+
4244
./awsconfiguration.sh ${DEPLOY_ENV}
4345
source awsenvconf
44-
45-
# ./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar
46+
#scorecard test consumer remove later
47+
#./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-consumer_scorecard-deployvar
4648
#source buildenvvar
4749
#./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
4850
49-
echo "Running Masterscript - deploy postgres-ifx-processer producer"
50-
if [ -e ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json; fi
51-
./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar
51+
#scorecard test producer remove later
52+
#echo "Running Masterscript - deploy postgres-ifx-processer producer"
53+
#if [ -e ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json; fi
54+
#./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer_scorecard-deployvar
55+
#source buildenvvar
56+
#./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
57+
58+
./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar
5259
source buildenvvar
5360
./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
61+
62+
#echo "Running Masterscript - deploy postgres-ifx-processer producer"
63+
#if [ -e ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-consumer-deployvar.json; fi
64+
#./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar
65+
#source buildenvvar
66+
#./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
5467
55-
echo "Running Masterscript - deploy postgres-ifx-processer producer_dd"
56-
if [ -e ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json; fi
57-
./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer_dd-deployvar
58-
source buildenvvar
59-
./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
60-
61-
#echo "Running Masterscript - deploy postgres-ifx-processer reconsiler1"
62-
#if [ -e ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar.json; fi
63-
#./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar
68+
#echo "Running Masterscript - deploy postgres-ifx-processer producer_dd"
69+
#if [ -e ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-producer-deployvar.json; fi
70+
#./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-producer_dd-deployvar
6471
#source buildenvvar
6572
#./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
73+
74+
echo "Running Masterscript - deploy postgres-ifx-processer reconsiler1"
75+
if [ -e ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar.json ]; then sudo rm -vf ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar.json; fi
76+
./buildenv.sh -e ${DEPLOY_ENV} -b ${LOGICAL_ENV}-${APP_NAME}-reconsiler1-deployvar
77+
source buildenvvar
78+
./master_deploy.sh -d ECS -e ${DEPLOY_ENV} -t latest -s ${GLOBAL_ENV}-global-appvar,${LOGICAL_ENV}-${APP_NAME}-appvar -i postgres-ifx-processer
6679
6780
jobs:
6881
# Build & Deploy against development backend #
@@ -103,13 +116,12 @@ workflows:
103116
branches:
104117
only:
105118
- dev
106-
- dev-retryfeature
107119
- "build-test":
108120
context : org-global
109121
filters:
110122
branches:
111-
only:
112-
- dev-test-pg
123+
only:
124+
- dev-test-pg
113125
- dev-test-pg-rf
114126
- "build-prod":
115127
context : org-global

‎config/default.js

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ module.exports = {
2222
database: process.env.PG_DATABASE || 'postgres', // database must exist before running the tool
2323
password: process.env.PG_PASSWORD || 'password',
2424
port: parseInt(process.env.PG_PORT, 10) || 5432,
25-
triggerFunctions: process.env.TRIGGER_FUNCTIONS || ['dev_db_notifications'], // List of trigger functions to listen to
26-
triggerTopics: process.env.TRIGGER_TOPICS || ['dev.db.postgres.sync'], // Names of the topic in the trigger payload
25+
triggerFunctions: process.env.TRIGGER_FUNCTIONS || ['test_db_notifications'], // List of trigger functions to listen to
26+
triggerTopics: process.env.TRIGGER_TOPICS || ['test.db.postgres.sync'], // Names of the topic in the trigger payload
2727
triggerOriginators: process.env.TRIGGER_ORIGINATORS || ['tc-postgres-delta-processor'] // Names of the originator in the trigger payload
2828
},
2929
KAFKA: { // Kafka connection options
@@ -34,32 +34,36 @@ module.exports = {
3434
},
3535
topic: process.env.KAFKA_TOPIC || 'db.topic.sync', // Kafka topic to push and receive messages
3636
partition: process.env.partition || [0], // Kafka partitions to use
37-
maxRetry: process.env.MAX_RETRY || 3,
37+
maxRetry: process.env.MAX_RETRY || 10,
3838
errorTopic: process.env.ERROR_TOPIC || 'db.scorecardtable.error',
39-
recipients: ['[email protected]'] // Kafka partitions to use
39+
recipients: ['[email protected]'], // Kafka partitions to use,
40+
KAFKA_URL: process.env.KAFKA_URL,
41+
KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID || 'test-postgres-ifx-consumer',
42+
KAFKA_CLIENT_CERT: process.env.KAFKA_CLIENT_CERT ? process.env.KAFKA_CLIENT_CERT.replace('\\n', '\n') : null,
43+
KAFKA_CLIENT_CERT_KEY: process.env.KAFKA_CLIENT_CERT_KEY ? process.env.KAFKA_CLIENT_CERT_KEY.replace('\\n', '\n') : null,
4044
},
4145
SLACK: {
4246
URL: process.env.SLACKURL || 'us-east-1',
4347
SLACKCHANNEL: process.env.SLACKCHANNEL || 'ifxpg-migrator',
44-
SLACKNOTIFY: process.env.SLACKNOTIFY || 'false'
48+
SLACKNOTIFY: process.env.SLACKNOTIFY || 'false'
4549
},
46-
RECONSILER:{
50+
RECONSILER: {
4751
RECONSILER_START: process.env.RECONSILER_START || 5,
4852
RECONSILER_END: process.env.RECONSILER_END || 1,
4953
RECONSILER_DURATION_TYPE: process.env.RECONSILER_DURATION_TYPE || 'm'
5054
},
5155
DYNAMODB:
52-
{
53-
DYNAMODB_TABLE: process.env.DYNAMODB_TABLE || 'dev_pg_ifx_payload_sync',
54-
DD_ElapsedTime: process.env.DD_ElapsedTime || 600000
55-
},
56+
{
57+
DYNAMODB_TABLE: process.env.DYNAMODB_TABLE || 'test_pg_ifx_payload_sync',
58+
DD_ElapsedTime: process.env.DD_ElapsedTime || 600000
59+
},
5660

57-
AUTH0_URL: process.env.AUTH0_URL ,
58-
AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE ,
59-
TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME ,
60-
AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID ,
61-
AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET ,
62-
BUSAPI_URL : process.env.BUSAPI_URL ,
63-
KAFKA_ERROR_TOPIC : process.env.KAFKA_ERROR_TOPIC ,
64-
AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL
61+
AUTH0_URL: process.env.AUTH0_URL,
62+
AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE,
63+
TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
64+
AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
65+
AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
66+
BUSAPI_URL: process.env.BUSAPI_URL,
67+
KAFKA_ERROR_TOPIC: process.env.KAFKA_ERROR_TOPIC,
68+
AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL
6569
}

‎informix-identity-trigger-proc.sql

Lines changed: 121 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
database common_oltp
2+
23
DROP PROCEDURE proc_user_update(varchar,decimal);
3-
DROP PROCEDURE proc_user_update;
44
CREATE PROCEDURE informix.proc_user_update(
55
new_handle varchar(50),
66
user_id decimal(10,0))
77
if (USER != 'ifxsyncuser') then
88
UPDATE user SET handle_lower = lower(new_handle), modify_date = current WHERE user.user_id = user_id;
99
End if;
10-
end procedure;
10+
end procedure;
11+
12+
DROP PROCEDURE proc_user_update;
1113
create procedure "informix".proc_user_update(
1214
user_id DECIMAL(10,0),
1315
old_first_name VARCHAR(64),
@@ -25,7 +27,7 @@ new_middle_name VARCHAR(64),
2527
old_timezone_id decimal(5,0),
2628
new_timezone_id decimal(5,0)
2729
)
28-
30+
if (USER != 'ifxsyncuser') then
2931
if ((old_first_name != new_first_name) or (old_last_name != new_last_name ) or (old_middle_name != new_middle_name )) then
3032
insert into audit_user (column_name, old_value, new_value,
3133
user_id)
@@ -57,7 +59,7 @@ user_id)
5759
user_id)
5860
values ('TIMEZONE_ID', old_timezone_id, new_timezone_id, user_id);
5961
End If;
60-
if (USER != 'ifxsyncuser') then
62+
6163
UPDATE user SET handle_lower = lower(new_handle), modify_date = current WHERE user.user_id = user_id;
6264
End if;
6365
end procedure;
@@ -75,7 +77,7 @@ new_primary_ind DECIMAL(1,0),
7577
old_status_id DECIMAL(3,0),
7678
new_status_id DECIMAL(3,0)
7779
)
78-
80+
if (USER != 'ifxsyncuser') then
7981
if (old_email_type_id != new_email_type_id) then
8082
insert into audit_user (column_name, old_value, new_value, user_id)
8183
values ('EMAIL_TYPE', old_email_type_id, new_email_type_id, user_id);
@@ -95,12 +97,122 @@ new_status_id DECIMAL(3,0)
9597
insert into audit_user (column_name, old_value, new_value, user_id)
9698
values ('EMAIL_PRIMARY_IND', old_primary_ind, new_primary_ind, user_id);
9799
End If;
98-
if (USER != 'ifxsyncuser') then
100+
99101
update email set modify_date = current where email.email_id = email_id;
100102
End if;
101103
end procedure;
102104

105+
DROP PROCEDURE informix.proc_user_last_login;
106+
CREATE PROCEDURE informix.proc_user_last_login (user_id DECIMAL(10,0), o_last_login DATETIME YEAR TO FRACTION,
107+
n_last_login DATETIME YEAR TO FRACTION)
108+
109+
if (o_last_login != n_last_login) then
110+
if (USER != 'ifxsyncuser') then
111+
insert into corona_event (corona_event_type_id,user_id, corona_event_timestamp) values (1, user_id, n_last_login);
112+
end if;
113+
End if;
114+
end procedure;
115+
116+
DROP PROCEDURE informix.proc_phone_update;
117+
CREATE PROCEDURE informix.proc_phone_update(
118+
phone_id decimal(10,0),
119+
user_id DECIMAL(10,0),
120+
old_phone_type_id DECIMAL(5,0),
121+
new_phone_type_id DECIMAL(5,0),
122+
old_number VARCHAR(64),
123+
new_number VARCHAR(64),
124+
old_primary_ind DECIMAL(1,0),
125+
new_primary_ind DECIMAL(1,0)
126+
)
127+
if (USER != 'ifxsyncuser') then
128+
if (old_phone_type_id != new_phone_type_id) then
129+
insert into audit_user (column_name, old_value, new_value, user_id)
130+
values ('PHONE_TYPE', old_phone_type_id, new_phone_type_id, user_id);
131+
End If;
132+
133+
if (old_number != new_number) then
134+
insert into audit_user (column_name, old_value, new_value, user_id)
135+
values ('PHONE_NUMBER', old_number, new_number, user_id);
136+
End If;
137+
138+
if (old_primary_ind != new_primary_ind) then
139+
insert into audit_user (column_name, old_value, new_value, user_id)
140+
values ('PHONE_PRIMARY_IND', old_primary_ind, new_primary_ind, user_id);
141+
End If;
142+
update phone set modify_date = current where phone.phone_id = phone_id;
143+
End if;
144+
end procedure;
145+
146+
DROP PROCEDURE informix.proc_address_update;
147+
CREATE PROCEDURE informix.proc_address_update(
148+
address_id DECIMAL(10,0),
149+
old_address_type_id DECIMAL(5,0),
150+
new_address_type_id DECIMAL(5,0),
151+
old_address1 VARCHAR(254),
152+
new_address1 VARCHAR(254),
153+
old_address2 VARCHAR(254),
154+
new_address2 VARCHAR(254),
155+
old_address3 VARCHAR(254),
156+
new_address3 VARCHAR(254),
157+
old_city VARCHAR(64),
158+
new_city VARCHAR(64),
159+
old_state_code CHAR(2),
160+
new_state_code CHAR(2),
161+
old_province VARCHAR(64),
162+
new_province VARCHAR(64),
163+
old_zip VARCHAR(15),
164+
new_zip VARCHAR(15),
165+
old_country_code CHAR(3),
166+
new_country_code CHAR(3)
167+
)
168+
define user_id DECIMAL(10,0);
169+
let user_id = NVL((select min(x.user_id) from user_address_xref x where x.address_id = address_id), -1);
170+
if (USER != 'ifxsyncuser') then
171+
if (user_id > 0 and old_address1 != new_address1) then
172+
insert into audit_user (column_name, old_value, new_value,
173+
user_id)
174+
values ('ADDRESS1', old_address1, new_address1, user_id);
175+
End If;
176+
if (user_id > 0 and old_address2 != new_address2) then
177+
insert into audit_user (column_name, old_value, new_value,
178+
user_id)
179+
values ('ADDRESS2', old_address2, new_address2, user_id);
180+
End If;
181+
if (user_id > 0 and old_address3 != new_address3) then
182+
insert into audit_user (column_name, old_value, new_value,
183+
user_id)
184+
values ('ADDRESS3', old_address3, new_address3, user_id);
185+
End If;
186+
if (user_id > 0 and old_city != new_city) then
187+
insert into audit_user (column_name, old_value, new_value,
188+
user_id)
189+
values ('ADDRESS_CITY', old_city, new_city, user_id);
190+
End If;
191+
if (user_id > 0 and old_state_code != new_state_code) then
192+
insert into audit_user (column_name, old_value, new_value,
193+
user_id)
194+
values ('ADDRESS_STATE', old_state_code, new_state_code, user_id);
195+
End If;
196+
if (user_id > 0 and old_province != new_province) then
197+
insert into audit_user (column_name, old_value, new_value,
198+
user_id)
199+
values ('ADDRESS_PROVINCE', old_province, new_province, user_id);
200+
End If;
201+
if (user_id > 0 and old_zip != new_zip) then
202+
insert into audit_user (column_name, old_value, new_value,
203+
user_id)
204+
values ('ADDRESS_ZIP', old_zip, new_zip, user_id);
205+
End If;
206+
if (user_id > 0 and old_country_code != new_country_code) then
207+
insert into audit_user (column_name, old_value, new_value,
208+
user_id)
209+
values ('ADDRESS_COUNTRY', old_country_code, new_country_code, user_id);
210+
End If;
103211

212+
update address set modify_date = current where address.address_id = address_id;
213+
End if;
214+
end procedure;
215+
104216
database informixoltp
105217
DROP PROCEDURE informix.proc_coder_update;
106218
CREATE PROCEDURE informix.proc_coder_update(
@@ -110,7 +222,8 @@ v_oldlanguage_id decimal(3,0), v_newlanguage_id decimal(3,0),
110222
v_oldcoder_type_id decimal(3,0), v_newcoder_type_id decimal(3,0),
111223
v_oldcomp_country_code varchar(3), v_newcomp_country_code varchar(3)
112224
)
113-
225+
if (USER != 'ifxsyncuser') then
226+
114227
if (v_oldquote != v_newquote) then
115228
insert into audit_coder (column_name, old_value, new_value, user_id)
116229
values ('QUOTE', v_oldquote , v_newquote, v_oldcoder_id);
@@ -131,10 +244,8 @@ v_oldcomp_country_code varchar(3), v_newcomp_country_code varchar(3)
131244
values ('COMP_COUNTRY', v_oldcomp_country_code , v_newcomp_country_code, v_oldcoder_id);
132245
End if;
133246

134-
if (USER != 'ifxsyncuser') then
135247
update coder set modify_date = current where coder_id = v_oldcoder_id;
136248
End if;
137-
138249
end procedure;
139250

140251
database tcs_catalog;
@@ -155,7 +266,7 @@ new_rating decimal(5,4)
155266
update user_reliability set modify_date = current where user_id = p_user_id and phase_id = p_phase_id;
156267
End if;
157268
end procedure;
158-
269+
159270
DROP PROCEDURE proc_rating_update;
160271
CREATE PROCEDURE informix.proc_rating_update(
161272
p_user_id DECIMAL(10,0),

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
"dependencies": {
1818
"aws-sdk": "*",
1919
"config": "^3.2.2",
20-
"informix-wrapper": "git+https://github.com/appirio-tech/informix-wrapper.git",
20+
"informix-wrapper": "git+https://github.com/appirio-tech/informix-wrapper.git#prepare_stmt_fix",
2121
"no-kafka": "^3.4.3",
2222
"pg": "^7.12.1",
2323
"sleep": "^6.1.0",

0 commit comments

Comments
 (0)