Skip to content

Commit 4f4f849

Browse files
AranyajitAranyajit
authored
Merge pull request #718 from topcoder-platform/feature/tsjr-313-generate-tcUserId-for-existing-job_candidates
Enhance scripts for data processing
2 parents db19927 + 5f4d15d commit 4f4f849

File tree

2 files changed

+87
-36
lines changed

2 files changed

+87
-36
lines changed

‎scripts/alter-bookings-db-tcUserId.sql

-4
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,6 @@ ALTER COLUMN member_id TYPE TEXT,
2828
ALTER COLUMN created_by TYPE TEXT,
2929
ALTER COLUMN updated_by TYPE TEXT;
3030

31-
ALTER TABLE bookings.roles
32-
ALTER COLUMN created_by TYPE TEXT,
33-
ALTER COLUMN updated_by TYPE TEXT;
34-
3531
ALTER TABLE bookings.user_meeting_settings
3632
ALTER COLUMN created_by TYPE TEXT,
3733
ALTER COLUMN updated_by TYPE TEXT;

‎scripts/process-remaining-ubahn-uuids.js

+87-32
Original file line numberDiff line numberDiff line change
@@ -3,66 +3,121 @@ const tcUserId = require('./common/tcUserId')
33
const Sequelize = require('sequelize')
44
const _ = require('lodash')
55
const helper = require('../src/common/helper')
6+
const request = require('superagent')
7+
const config = require('config')
68

7-
// read cli arguments, pass the table name
8-
const tableName = process.argv[2]
9-
// read cli arguments, pass the column names to update
10-
const columnNames = process.argv[3]
9+
/**
10+
*
11+
* @param {string} handle - The member handle
12+
* @returns {Promise<object>} The member details from the member API
13+
*/
14+
async function getMemberDetailsByHandle (handle) {
15+
const token = await helper.getM2MToken()
16+
let res
17+
try {
18+
res = await request
19+
.get(`${config.TOPCODER_MEMBERS_API}/${handle}`)
20+
.query({
21+
fields: 'userId,handle,handleLower'
22+
})
23+
.set('Authorization', `Bearer ${token}`)
24+
.set('Accept', 'application/json')
25+
return res.body
26+
} catch (error) {
27+
console.log(`Unable to find member with handle ${handle}`)
28+
return {}
29+
}
30+
}
1131

1232
const processRemainingUUIDs = async (tableName, columnNames) => {
1333
const dbUrl = process.env.UBAHN_DB_URL
1434
const MODE = process.env.MODE || 'test'
35+
const handleToIDMap = {}
1536

1637
if (_.isUndefined(dbUrl) || _.isNull(dbUrl)) {
1738
console.log('Ubahn DB URL not set, exiting!')
1839
process.exit(0)
1940
}
2041

42+
// only for readability
43+
console.log()
44+
console.log('---------------------------------------------------------------------------------------------------------------')
45+
console.log()
46+
2147
for (const columnName of _.split(columnNames, ',')) {
22-
const query = `SELECT DISTINCT ${columnName} FROM bookings.${tableName} WHERE LENGTH(${columnName}) > 9 AND ${columnName} <> '00000000-0000-0000-0000-000000000000';`
48+
const transaction = await models.sequelize.transaction()
2349

24-
console.log(`Executing query in table ${tableName} against column ${columnName}`)
25-
console.log(`SQL query: ${query}`)
50+
try {
51+
// check each column to find distinct existing uuids which have not yet been converted to TC legacy user id
52+
const query = `SELECT DISTINCT ${columnName} FROM bookings.${tableName} WHERE LENGTH(${columnName}) > 9 AND ${columnName} <> '00000000-0000-0000-0000-000000000000';`
53+
console.log(`Executing query in table ${tableName} against column ${columnName}`)
54+
console.log(`SQL query: ${query}`)
55+
let results = await models.sequelize.query(query, { type: Sequelize.QueryTypes.SELECT })
2656

27-
let results = await models.sequelize.query(query, { type: Sequelize.QueryTypes.SELECT })
57+
if (results.length > 0) {
58+
results = _.uniq(_.map(_.filter(results, val => toString(val[`${columnName}`]).length > 9), val => val[`${columnName}`]))
59+
console.log(`SQL query result: ${JSON.stringify(results)}`)
2860

29-
if (results.length > 0) {
30-
results = _.uniq(_.map(_.filter(results, val => toString(val[`${columnName}`]).length > 9), val => val[`${columnName}`]))
31-
console.log(`SQL query result: ${JSON.stringify(results)}`)
61+
// get the ubahn uuid to handle map
62+
const ubahnConn = await tcUserId.getUbahnDatabaseConnection(dbUrl)
63+
const uuidToHandleMap = await tcUserId.getUserUbahnUUIDToHandleMap(ubahnConn, results)
3264

33-
const ubahnConn = await tcUserId.getUbahnDatabaseConnection(dbUrl)
34-
const uuidToHandleMap = await tcUserId.getUserUbahnUUIDToHandleMap(ubahnConn, results)
65+
// get the handle to legacy topcoder id map
66+
for (const handle of Object.values(uuidToHandleMap)) {
67+
console.log(`handle to search for ${handle}`)
68+
if (_.isUndefined(handleToIDMap[handle])) {
69+
const member = await getMemberDetailsByHandle(handle)
70+
handleToIDMap[member.handleLower] = member.userId
71+
}
72+
}
3573

36-
const handleToIDMap = {}
37-
const batches = _.chunk(Object.values(uuidToHandleMap), 30)
38-
for (const batch of batches) {
39-
console.log(`Batch of handles: ${JSON.stringify(batch)}`)
40-
const memberAPIRes = await helper.getMemberDetailsByHandles(batch)
41-
_.forEach(memberAPIRes, member => {
42-
handleToIDMap[member.handleLower] = member.userId
43-
})
44-
}
74+
// build the update queries
75+
let sql = ''
76+
for (const [key, value] of Object.entries(uuidToHandleMap)) {
77+
if (!_.isUndefined(handleToIDMap[value.toLowerCase()])) {
78+
sql += `UPDATE bookings.${tableName} SET ${columnName} = '${handleToIDMap[value.toLowerCase()]}' WHERE ${columnName} = '${key}';`
79+
}
80+
}
4581

46-
let sql = ''
47-
for (const [key, value] of Object.entries(uuidToHandleMap)) {
48-
if (!_.isUndefined(handleToIDMap[value.toLowerCase()])) {
49-
sql += `UPDATE bookings.${tableName} SET ${columnName} = '${handleToIDMap[value.toLowerCase()]}' WHERE ${columnName} = '${key}';`
82+
// execute update queries if and only if it's not in test mode
83+
if (sql !== '') {
84+
console.log(`SQL UPDATE statements: ${sql}`)
85+
if (MODE !== 'test') {
86+
console.log('Executing UPDATE statements')
87+
await models.sequelize.query(sql, { type: Sequelize.QueryTypes.UPDATE, transaction: transaction })
88+
}
89+
} else {
90+
console.log(`No UPDATE statements to execute against column ${columnName}`)
5091
}
92+
} else {
93+
console.log(`No data eligible to be updated for table: ${tableName} against column ${columnName}`)
5194
}
52-
console.log(`SQL UPDATE statements: ${sql}`)
53-
if (MODE !== test && sql.length > 0) {
54-
await models.sequelize.query(sql, { type: Sequelize.QueryTypes.UPDATE })
55-
}
56-
} else {
57-
console.log(`No data eligible to be updated for table: ${tableName} against column ${columnName}`)
95+
96+
// only for readability
97+
console.log('---------------------------------------------------------------------------------------------------------------')
98+
console.log()
99+
100+
await transaction.commit()
101+
} catch (error) {
102+
console.log('Error encountered')
103+
console.error(JSON.stringify(error))
104+
await transaction.rollback()
58105
}
59106
}
107+
108+
console.log(`DONE processing table ${tableName}`)
60109
}
61110

111+
// read cli arguments, pass the table name
112+
const tableName = process.argv[2]
113+
// read cli arguments, pass the column names to update
114+
const columnNames = process.argv[3]
115+
62116
processRemainingUUIDs(tableName, columnNames).then(res => {
63117
console.log(`Processed remaining records for model '${tableName}' against columns: ${columnNames}`)
64118
process.exit(0)
65119
}).catch(err => {
120+
console.log('Error encountered!')
66121
console.error(`${JSON.stringify(err)}`)
67122
process.exit(1)
68123
})

0 commit comments

Comments
 (0)