Skip to content

Commit f6abe98

Browse files
maxceemmaxceem
authored
Merge pull request #174 from imcaizheng/recruit-crm-job-sync
Create script for migrating `isApplicationPageActive` from CSV file
2 parents 151c026 + a131404 commit f6abe98

File tree

13 files changed

+464
-69
lines changed

13 files changed

+464
-69
lines changed

‎scripts/common/helper.js

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* Provide some commonly used functions for scripts.
3+
*/
4+
const csv = require('csv-parser')
5+
const fs = require('fs')
6+
const request = require('superagent')
7+
8+
/**
9+
* Load CSV data from file.
10+
*
11+
* @param {String} pathname the pathname for the file
12+
* @param {Object} fieldNameMap mapping values of headers
13+
* @returns {Array} the result jobs data
14+
*/
15+
async function loadCSVFromFile (pathname, fieldNameMap = {}) {
16+
let lnum = 1
17+
const result = []
18+
return new Promise((resolve, reject) => {
19+
fs.createReadStream(pathname)
20+
.pipe(csv({
21+
mapHeaders: ({ header }) => fieldNameMap[header] || header
22+
}))
23+
.on('data', (data) => {
24+
result.push({ ...data, _lnum: lnum })
25+
lnum += 1
26+
})
27+
.on('error', err => reject(err))
28+
.on('end', () => resolve(result))
29+
})
30+
}
31+
32+
/**
33+
* Get pathname from command line arguments.
34+
*
35+
* @returns {String} the pathname
36+
*/
37+
function getPathnameFromCommandline () {
38+
if (process.argv.length < 3) {
39+
throw new Error('pathname for the csv file is required')
40+
}
41+
const pathname = process.argv[2]
42+
if (!fs.existsSync(pathname)) {
43+
throw new Error(`pathname: ${pathname} path not exist`)
44+
}
45+
if (!fs.lstatSync(pathname).isFile()) {
46+
throw new Error(`pathname: ${pathname} path is not a regular file`)
47+
}
48+
return pathname
49+
}
50+
51+
/**
52+
* Sleep for a given number of milliseconds.
53+
*
54+
* @param {Number} milliseconds the sleep time
55+
* @returns {undefined}
56+
*/
57+
async function sleep (milliseconds) {
58+
return new Promise((resolve) => setTimeout(resolve, milliseconds))
59+
}
60+
61+
/**
62+
* Find taas job by external id.
63+
*
64+
* @param {String} token the auth token
65+
* @param {String} taasApiUrl url for TaaS API
66+
* @param {String} externalId the external id
67+
* @returns {Object} the result
68+
*/
69+
async function getJobByExternalId (token, taasApiUrl, externalId) {
70+
const { body: jobs } = await request.get(`${taasApiUrl}/jobs`)
71+
.query({ externalId })
72+
.set('Authorization', `Bearer ${token}`)
73+
if (!jobs.length) {
74+
throw new Error(`externalId: ${externalId} job not found`)
75+
}
76+
return jobs[0]
77+
}
78+
79+
module.exports = {
80+
loadCSVFromFile,
81+
getPathnameFromCommandline,
82+
sleep,
83+
getJobByExternalId
84+
}

‎scripts/common/logger.js

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
/*
2+
* Logger for scripts.
3+
*/
4+
5+
module.exports = {
6+
info: (message) => console.log(`INFO: ${message}`),
7+
debug: (message) => console.log(`DEBUG: ${message}`),
8+
warn: (message) => console.log(`WARN: ${message}`),
9+
error: (message) => console.log(`ERROR: ${message}`)
10+
}

‎scripts/recruit-crm-job-import/helper.js

Lines changed: 22 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,27 @@
22
* Provide some commonly used functions for the RCRM import script.
33
*/
44
const config = require('./config')
5+
const _ = require('lodash')
56
const request = require('superagent')
6-
const { getM2MToken } = require('../../src/common/helper')
7+
const commonHelper = require('../common/helper')
78

8-
/**
9-
* Sleep for a given number of milliseconds.
10-
*
11-
* @param {Number} milliseconds the sleep time
12-
* @returns {undefined}
9+
/*
10+
* Function to get M2M token
11+
* @returns {Promise}
1312
*/
14-
async function sleep (milliseconds) {
15-
return new Promise((resolve) => setTimeout(resolve, milliseconds))
16-
}
13+
const getM2MToken = (() => {
14+
const m2mAuth = require('tc-core-library-js').auth.m2m
15+
const m2m = m2mAuth(_.pick(config, [
16+
'AUTH0_URL',
17+
'AUTH0_AUDIENCE',
18+
'AUTH0_CLIENT_ID',
19+
'AUTH0_CLIENT_SECRET',
20+
'AUTH0_PROXY_SERVER_URL'
21+
]))
22+
return async () => {
23+
return await m2m.getMachineToken(config.AUTH0_CLIENT_ID, config.AUTH0_CLIENT_SECRET)
24+
}
25+
})()
1726

1827
/**
1928
* Create a new job via taas api.
@@ -38,13 +47,7 @@ async function createJob (data) {
3847
*/
3948
async function getJobByExternalId (externalId) {
4049
const token = await getM2MToken()
41-
const { body: jobs } = await request.get(`${config.TAAS_API_URL}/jobs`)
42-
.query({ externalId })
43-
.set('Authorization', `Bearer ${token}`)
44-
if (!jobs.length) {
45-
throw new Error(`externalId: ${externalId} job not found`)
46-
}
47-
return jobs[0]
50+
return commonHelper.getJobByExternalId(token, config.TAAS_API_URL, externalId)
4851
}
4952

5053
/**
@@ -131,7 +134,9 @@ async function getProjectByDirectProjectId (directProjectId) {
131134
}
132135

133136
module.exports = {
134-
sleep,
137+
sleep: commonHelper.sleep,
138+
loadCSVFromFile: commonHelper.loadCSVFromFile,
139+
getPathnameFromCommandline: commonHelper.getPathnameFromCommandline,
135140
createJob,
136141
getJobByExternalId,
137142
updateResourceBookingStatus,

‎scripts/recruit-crm-job-import/index.js

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22
* Script to import Jobs data from Recruit CRM to Taas API.
33
*/
44

5-
const csv = require('csv-parser')
6-
const fs = require('fs')
75
const Joi = require('joi')
86
.extend(require('@joi/date'))
97
const _ = require('lodash')
@@ -38,48 +36,6 @@ function validateJob (job) {
3836
return jobSchema.validate(job)
3937
}
4038

41-
/**
42-
* Load Recruit CRM jobs data from file.
43-
*
44-
* @param {String} pathname the pathname for the file
45-
* @returns {Array} the result jobs data
46-
*/
47-
async function loadRcrmJobsFromFile (pathname) {
48-
let lnum = 1
49-
const result = []
50-
return new Promise((resolve, reject) => {
51-
fs.createReadStream(pathname)
52-
.pipe(csv({
53-
mapHeaders: ({ header }) => constants.fieldNameMap[header] || header
54-
}))
55-
.on('data', (data) => {
56-
result.push({ ...data, _lnum: lnum })
57-
lnum += 1
58-
})
59-
.on('error', err => reject(err))
60-
.on('end', () => resolve(result))
61-
})
62-
}
63-
64-
/**
65-
* Get pathname for a csv file from command line arguments.
66-
*
67-
* @returns {undefined}
68-
*/
69-
function getPathname () {
70-
if (process.argv.length < 3) {
71-
throw new Error('pathname for the csv file is required')
72-
}
73-
const pathname = process.argv[2]
74-
if (!fs.existsSync(pathname)) {
75-
throw new Error(`pathname: ${pathname} path not exist`)
76-
}
77-
if (!fs.lstatSync(pathname).isFile()) {
78-
throw new Error(`pathname: ${pathname} path is not a regular file`)
79-
}
80-
return pathname
81-
}
82-
8339
/**
8440
* Process single job data. The processing consists of:
8541
* - Validate the data.
@@ -146,8 +102,8 @@ async function processJob (job, info = []) {
146102
* @returns {undefined}
147103
*/
148104
async function main () {
149-
const pathname = getPathname()
150-
const jobs = await loadRcrmJobsFromFile(pathname)
105+
const pathname = helper.getPathnameFromCommandline()
106+
const jobs = await helper.loadCSVFromFile(pathname, constants.fieldNameMap)
151107
const report = new Report()
152108
for (const job of jobs) {
153109
logger.debug(`processing line #${job._lnum} - ${JSON.stringify(job)}`)

‎scripts/recruit-crm-job-import/logger.js

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
11
/*
22
* Logger for the RCRM import script.
33
*/
4+
const logger = require('../common/logger')
45

5-
module.exports = {
6-
info: (message) => console.log(`INFO: ${message}`),
7-
debug: (message) => console.log(`DEBUG: ${message}`),
8-
warn: (message) => console.log(`WARN: ${message}`),
9-
error: (message) => console.log(`ERROR: ${message}`)
10-
}
6+
module.exports = logger

‎scripts/recruit-crm-job-sync/README.md

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
Recruit CRM Job Data Sync Script
2+
===
3+
4+
# Configuration
5+
Configuration file is at `./scripts/recruit-crm-job-sync/config.js`.
6+
7+
8+
# Usage
9+
``` bash
10+
node scripts/recruit-crm-job-sync <pathname-to-a-csv-file>
11+
```
12+
13+
By default the script updates jobs via `TC_API`.
14+
15+
# Example
16+
17+
1. Follow the README for `taas-apis` to deploy Taas API locally
18+
2. Create two jobs via `Jobs > create job with booking manager` in Postman, with external ids `51913016` and `51892637` for each of the jobs.
19+
20+
**NOTE**: The external ids `51913016` and `51902826` could be found at `scripts/recruit-crm-job-sync/example_data.csv` under the Slug column.
21+
22+
3. Configure env variable `RCRM_SYNC_TAAS_API_URL` so that the script could make use of the local API:
23+
24+
``` bash
25+
export RCRM_SYNC_TAAS_API_URL=http://localhost:3000/api/v5
26+
```
27+
28+
4. Run the script against the sample CSV file and pipe the output from the script to a temporary file:
29+
30+
``` bash
31+
node scripts/recruit-crm-job-sync scripts/recruit-crm-job-sync/example_data.csv | tee /tmp/report.txt
32+
```
33+
34+
The output should be like this:
35+
36+
``` bash
37+
DEBUG: processing line #1 - {"ID":"1","Name":"Data job Engineer","Description":"","Qualification":"","Specialization":"","Minimum Experience In Years":"1","Maximum Experience In Years":"3","Minimum Annual Salary":"10","Maximum Annual Salary":"20","Number Of Openings":"2","Job Status":"Closed","Company":"company 1","Contact":" ","Currency":"$","allowApply":"Yes","Collaborator":"","Locality":"","City":"","Job Code":"J123456","Createdby":"abc","Created On":"02-Jun-20","Updated By":"abc","Updated On":"17-Feb-21","Owner":"abc","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"51892637","_lnum":1}
38+
ERROR: #1 - [EXTERNAL_ID_NOT_FOUND] externalId: 51892637 job not found
39+
DEBUG: processed line #1
40+
DEBUG: processing line #2 - {"ID":"2","Name":"JAVA coffee engineer","Description":"","Qualification":"","Specialization":"","Minimum Experience In Years":"2","Maximum Experience In Years":"5","Minimum Annual Salary":"10","Maximum Annual Salary":"20","Number Of Openings":"10","Job Status":"Closed","Company":"company 2","Contact":"abc","Currency":"$","allowApply":"Yes","Collaborator":"","Locality":"","City":"","Job Code":"J123457","Createdby":"abc","Created On":"02-Jun-20","Updated By":"abc","Updated On":"12-Nov-20","Owner":"abc","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"51913016","_lnum":2}
41+
DEBUG: jobId: 34cee9aa-e45f-47ed-9555-ffd3f7196fec isApplicationPageActive(current): false - isApplicationPageActive(to be synced): true
42+
INFO: #2 - id: 34cee9aa-e45f-47ed-9555-ffd3f7196fec isApplicationPageActive: true "job" updated
43+
DEBUG: processed line #2
44+
DEBUG: processing line #3 - {"ID":"3","Name":"QA Seleinium","Description":"","Qualification":"","Specialization":"","Minimum Experience In Years":"3","Maximum Experience In Years":"7","Minimum Annual Salary":"10","Maximum Annual Salary":"20","Number Of Openings":"4","Job Status":"Canceled","Company":"company 3","Contact":" ","Currency":"$","allowApply":"No","Collaborator":"","Locality":"","City":"","Job Code":"J123458","Createdby":"abc","Created On":"04-Jun-20","Updated By":"abc","Updated On":"12-Nov-20","Owner":"abc","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"51902826","_lnum":3}
45+
DEBUG: jobId: 4acde317-c364-4b79-aa77-295b98143c8b isApplicationPageActive(current): false - isApplicationPageActive(to be synced): false
46+
WARN: #3 - isApplicationPageActive is already set
47+
DEBUG: processed line #3
48+
DEBUG: processing line #4 - {"ID":"5","Name":"Data Engineers and Data Architects","Description":"","Qualification":"","Specialization":"","Minimum Experience In Years":"4","Maximum Experience In Years":"9","Minimum Annual Salary":"10","Maximum Annual Salary":"20","Number Of Openings":"8","Job Status":"Closed","Company":"company 4","Contact":" ","Currency":"$","allowApply":"Yes","Collaborator":"","Locality":"","City":"","Job Code":"J123459","Createdby":"abc","Created On":"09-Jun-20","Updated By":"abc","Updated On":"12-Nov-20","Owner":"abc","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"51811161","_lnum":4}
49+
ERROR: #4 - [EXTERNAL_ID_NOT_FOUND] externalId: 51811161 job not found
50+
DEBUG: processed line #4
51+
DEBUG: processing line #5 - {"ID":"6","Name":"Docker Engineer","Description":"Java & J2EE or Python, Docker, Kubernetes, AWS or GCP","Qualification":"","Specialization":"","Minimum Experience In Years":"5","Maximum Experience In Years":"10","Minimum Annual Salary":"10","Maximum Annual Salary":"20","Number Of Openings":"5","Job Status":"Closed","Company":"company 5","Contact":" ","Currency":"$","allowApply":"No","Collaborator":"","Locality":"","City":"","Job Code":"J123460","Createdby":"abc","Created On":"12-Jun-20","Updated By":"abc","Updated On":"12-Nov-20","Owner":"abc","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"51821342","_lnum":5}
52+
ERROR: #5 - [EXTERNAL_ID_NOT_FOUND] externalId: 51821342 job not found
53+
DEBUG: processed line #5
54+
DEBUG: processing line #6 - {"ID":"7","Name":"lambda Developers","Description":"","Qualification":"","Specialization":"","Minimum Experience In Years":"0","Maximum Experience In Years":"0","Minimum Annual Salary":"10","Maximum Annual Salary":"20","Number Of Openings":"2","Job Status":"Closed","Company":"company 6","Contact":"abc","Currency":"$","allowApply":"Yes","Collaborator":"","Locality":"","City":"","Job Code":"J123461","Createdby":"abc","Created On":"12-Jun-20","Updated By":"abc","Updated On":"12-Nov-20","Owner":"abc","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"51831524","_lnum":6}
55+
ERROR: #6 - [EXTERNAL_ID_NOT_FOUND] externalId: 51831524 job not found
56+
DEBUG: processed line #6
57+
DEBUG: processing line #7 - {"ID":"","Name":"","Description":"","Qualification":"","Specialization":"","Minimum Experience In Years":"","Maximum Experience In Years":"","Minimum Annual Salary":"","Maximum Annual Salary":"","Number Of Openings":"","Job Status":"","Company":"","Contact":"","Currency":"","allowApply":"","Collaborator":"","Locality":"","City":"","Job Code":"","Createdby":"","Created On":"","Updated By":"","Updated On":"","Owner":"","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"","_lnum":7}
58+
ERROR: #7 - "allowApply" must be one of [Yes, No]
59+
DEBUG: processed line #7
60+
INFO: === summary ===
61+
INFO: No. of records read = 7
62+
INFO: No. of records updated for field isApplicationPageActive = true = 1
63+
INFO: No. of records updated for field isApplicationPageActive = false = 0
64+
INFO: No. of records : externalId not found = 4
65+
INFO: No. of records failed(all) = 5
66+
INFO: No. of records failed(excluding "externalId not found") = 1
67+
INFO: No. of records skipped = 1
68+
INFO: done!
69+
```
70+
71+
The following command could be used to extract the summary from the output:
72+
73+
``` bash
74+
cat /tmp/report.txt | grep 'No. of records' | cut -d' ' -f2-
75+
```
76+
77+
To list all skipped lines:
78+
79+
``` bash
80+
cat /tmp/report.txt | grep 'WARN' -B 3

‎scripts/recruit-crm-job-sync/config.js

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Configuration for the RCRM sync script.
3+
* Namespace is created to allow to configure the env variables for this script independently.
4+
*/
5+
6+
const config = require('config')
7+
8+
const namespace = process.env.RCRM_SYNC_CONFIG_NAMESAPCE || 'RCRM_SYNC_'
9+
10+
module.exports = {
11+
SLEEP_TIME: process.env[`${namespace}SLEEP_TIME`] || 500,
12+
TAAS_API_URL: process.env[`${namespace}TAAS_API_URL`] || config.TC_API,
13+
14+
AUTH0_URL: process.env[`${namespace}AUTH0_URL`] || config.AUTH0_URL,
15+
AUTH0_AUDIENCE: process.env[`${namespace}AUTH0_AUDIENCE`] || config.AUTH0_AUDIENCE,
16+
TOKEN_CACHE_TIME: process.env[`${namespace}TOKEN_CACHE_TIME`] || config.TOKEN_CACHE_TIME,
17+
AUTH0_CLIENT_ID: process.env[`${namespace}AUTH0_CLIENT_ID`] || config.AUTH0_CLIENT_ID,
18+
AUTH0_CLIENT_SECRET: process.env[`${namespace}AUTH0_CLIENT_SECRET`] || config.AUTH0_CLIENT_SECRET,
19+
AUTH0_PROXY_SERVER_URL: process.env[`${namespace}AUTH0_PROXY_SERVER_URL`] || config.AUTH0_PROXY_SERVER_URL
20+
}

‎scripts/recruit-crm-job-sync/constants.js

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
/*
2+
* Constants for the RCRM sync script.
3+
*/
4+
5+
module.exports = {
6+
ProcessingStatus: {
7+
Successful: 'successful',
8+
Failed: 'failed',
9+
Skipped: 'skipped'
10+
},
11+
fieldNameMap: {
12+
'Allow Apply': 'allowApply',
13+
Slug: 'externalId'
14+
}
15+
}

‎scripts/recruit-crm-job-sync/example_data.csv

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
ID,Name,Description,Qualification,Specialization,Minimum Experience In Years,Maximum Experience In Years,Minimum Annual Salary,Maximum Annual Salary,Number Of Openings,Job Status,Company,Contact,Currency,Allow Apply,Collaborator,Locality,City,Job Code,Createdby,Created On,Updated By,Updated On,Owner,Custom Column 1,Custom Column 2,Custom Column 3,Custom Column 4,Custom Column 5,Custom Column 6,Custom Column 7,Custom Column 8,Custom Column 9,Custom Column 10,Custom Column 11,Custom Column 12,Custom Column 13,Custom Column 14,Custom Column 15,Slug
2+
1,Data job Engineer,,,,1,3,10,20,2,Closed,company 1, ,$,Yes,,,,J123456,abc,02-Jun-20,abc,17-Feb-21,abc,,,,,,,,,,,,,,,,51892637
3+
2,JAVA coffee engineer,,,,2,5,10,20,10,Closed,company 2,abc,$,Yes,,,,J123457,abc,02-Jun-20,abc,12-Nov-20,abc,,,,,,,,,,,,,,,,51913016
4+
3,QA Seleinium,,,,3,7,10,20,4,Canceled,company 3, ,$,No,,,,J123458,abc,04-Jun-20,abc,12-Nov-20,abc,,,,,,,,,,,,,,,,51902826
5+
5,Data Engineers and Data Architects,,,,4,9,10,20,8,Closed,company 4, ,$,Yes,,,,J123459,abc,09-Jun-20,abc,12-Nov-20,abc,,,,,,,,,,,,,,,,51811161
6+
6,Docker Engineer,"Java & J2EE or Python, Docker, Kubernetes, AWS or GCP",,,5,10,10,20,5,Closed,company 5, ,$,No,,,,J123460,abc,12-Jun-20,abc,12-Nov-20,abc,,,,,,,,,,,,,,,,51821342
7+
7,lambda Developers,,,,0,0,10,20,2,Closed,company 6,abc,$,Yes,,,,J123461,abc,12-Jun-20,abc,12-Nov-20,abc,,,,,,,,,,,,,,,,51831524
8+
,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

0 commit comments

Comments
 (0)