diff --git a/config/default.js b/config/default.js index 2256a15..e17d3f4 100644 --- a/config/default.js +++ b/config/default.js @@ -16,6 +16,7 @@ module.exports = { LOG_LEVEL: process.env.LOG_LEVEL || 'debug', PARTITION: process.env.PARTITION || 0, TOPIC: process.env.TOPIC || 'tc-x-events', + TOPIC_NOTIFICATION: process.env.TOPIC_NOTIFICATION || 'notifications.action.create', KAFKA_OPTIONS: { connectionString: process.env.KAFKA_URL || 'localhost:9092', groupId: process.env.KAFKA_GROUP_ID || 'topcoder-x-processor', @@ -25,6 +26,11 @@ module.exports = { passphrase: 'secret', // NOTE:* This configuration specifies the private key passphrase used while creating it. } }, + MAIL_NOTICIATION: { + type: 'tcx.mail_notification', + sendgridTemplateId: 'xxxxxx', + subject: 'Topcoder X Alert' + }, NEW_CHALLENGE_TEMPLATE: process.env.NEW_CHALLENGE_TEMPLATE || { status: 'Draft' }, @@ -68,5 +74,10 @@ module.exports = { ROLE_ID_SUBMITTER: process.env.ROLE_ID_SUBMITTER || '732339e7-8e30-49d7-9198-cccf9451e221', TYPE_ID_TASK: process.env.TYPE_ID_TASK || 'ecd58c69-238f-43a4-a4bb-d172719b9f31', DEFAULT_TIMELINE_TEMPLATE_ID: process.env.DEFAULT_TIMELINE_TEMPLATE_ID || '53a307ce-b4b3-4d6f-b9a1-3741a58f77e6', - DEFAULT_TRACK_ID: process.env.DEFAULT_TRACK_ID || '9b6fc876-f4d9-4ccb-9dfd-419247628825' + DEFAULT_TRACK_ID: process.env.DEFAULT_TRACK_ID || '9b6fc876-f4d9-4ccb-9dfd-419247628825', + GITLAB_ACCESS_TOKEN_DEFAULT_EXPIRATION: 3600 * 2, + GITLAB_REFRESH_TOKEN_BEFORE_EXPIRATION: 300, + GITLAB_CLIENT_ID: process.env.GITLAB_CLIENT_ID, + GITLAB_CLIENT_SECRET: process.env.GITLAB_CLIENT_SECRET, + GITLAB_OWNER_USER_CALLBACK_URL: process.env.GITLAB_OWNER_USER_CALLBACK_URL }; diff --git a/configuration.md b/configuration.md index 5ca0e4f..a8f5ddb 100644 --- a/configuration.md +++ b/configuration.md @@ -16,6 +16,9 @@ The following config parameters are supported, they are defined in `config/defau | NEW_CHALLENGE_DURATION_IN_DAYS | the duration of new challenge | 5 | |TC_URL| the base URL of topcoder to get the challenge URL| defaults to `https://www.topcoder-dev.com`| |GITLAB_API_BASE_URL| the URL for gitlab host| defaults to `https://gitlab.com`| +| GITLAB_CLIENT_ID | the GitLab client id | No default - needs to be set up with same value found in topcoder-x-ui | +| GITLAB_CLIENT_SECRET | the GitLab client secret | No default - needs to be set up with same value found in topcoder-x-ui | +| GITLAB_OWNER_USER_CALLBACK_URL | the GitLab callback redirect uri for refreshing copilot token | No default - needs to be set up with same owner user callback value in topcoder-x-ui | |PAID_ISSUE_LABEL|the label name for paid, should be one of the label configured in topcoder x ui|'tcx_Paid'| |FIX_ACCEPTED_ISSUE_LABEL|the label name for fix accepted, should be one of the label configured in topcoder x ui|'tcx_FixAccepted'| |ASSIGNED_ISSUE_LABEL| the label name for assigned, should be one of the label configured in topcoder x ui| 'tcx_Assigned'| diff --git a/index.js b/index.js index 3eed2a5..ef15b99 100644 --- a/index.js +++ b/index.js @@ -5,7 +5,7 @@ const config = require('config'); const _ = require('lodash'); -const kafka = require('./utils/kafka'); +const kafkaConsumer = require('./utils/kafka-consumer'); const logger = require('./utils/logger'); process.on('uncaughtException', (err) => { @@ -22,7 +22,9 @@ process.on('unhandledRejection', (err) => { }); // dump the configuration to logger -const ignoreConfigLog = ['cert', 'key', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AUTH0_CLIENT_ID', 'AUTH0_CLIENT_SECRET']; +const ignoreConfigLog = ['cert', 'key', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AUTH0_CLIENT_ID', 'AUTH0_CLIENT_SECRET', + 'GITLAB_CLIENT_ID', 'GITLAB_CLIENT_SECRET']; + /** * Print configs to logger * @param {Object} params the config params @@ -55,4 +57,4 @@ dumpConfigs(config, 0); logger.debug('--- End of List of Configurations ---'); // run the server -kafka.run(); +kafkaConsumer.run(); diff --git a/models/Project.js b/models/Project.js index 08a7837..df53b8d 100755 --- a/models/Project.js +++ b/models/Project.js @@ -23,6 +23,11 @@ const schema = new Schema({ type: Number, required: true }, + tags: { + type: String, + required: true, + default: '' + }, rocketChatWebhook: {type: String, required: false}, rocketChatChannelName: {type: String, required: false}, archived: {type: String, required: true}, diff --git a/services/ChallengeService.js b/services/ChallengeService.js new file mode 100644 index 0000000..ae4dd22 --- /dev/null +++ b/services/ChallengeService.js @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2022 TopCoder, Inc. All rights reserved. + */ +'use strict'; + +/** + * This service processes incoming pure challenge events. + * + * @author TCSCODER + * @version 1.0 + */ +const _ = require('lodash'); +const Joi = require('joi'); +const logger = require('../utils/logger'); +const topcoderApiHelper = require('../utils/topcoder-api-helper'); +const dbHelper = require('../utils/db-helper'); + +/** + * Update challenge tags + * @param {Object} event the event + */ +async function handleChallengeTagsUpdate(event) { + const tags = event.data.tags.split(','); + await Promise.all( + event.data.challengeUUIDsList.map(async (challengeUUIDs) => { + if (_.isString(challengeUUIDs)) { // repoUrl + challengeUUIDs = await dbHelper.queryChallengeUUIDsByRepoUrl(challengeUUIDs); + } + return challengeUUIDs.map(async (challengeUUID) => await topcoderApiHelper.updateChallenge(challengeUUID, {tags})); + }).reduce((a, b) => _.concat(a, b), []) + ).then((resps) => { + logger.debug(`handleChallengeTagsUpdate updated ${_.size(resps)} challenges successfully.`); + }).catch((err) => { + logger.error(`handleChallengeTagsUpdate failed. Internal Error: ${err}`); + throw new Error(`handleChallengeTagsUpdate failed. Internal Error: ${err}`); + }); +} + +/** + * Process pure challenge event. + * @param {Object} event the event + */ +async function process(event) { + Joi.attempt(event, process.schema); + + if (event.event === 'challengeTags.update') { + await handleChallengeTagsUpdate(event); + } +} + +process.schema = Joi.object().keys({ + event: Joi.string().valid('challengeTags.update').required(), + data: Joi.object().keys({ + challengeUUIDsList: Joi.array().items( + Joi.alternatives().try(Joi.string(), Joi.array().items(Joi.string())) + ).required(), + tags: Joi.string().required() + }).required(), + retryCount: Joi.number().integer().default(0).optional() +}); + + +module.exports = { + process +}; + +logger.buildService(module.exports); diff --git a/services/CopilotPaymentService.js b/services/CopilotPaymentService.js index edfe5a4..f828244 100644 --- a/services/CopilotPaymentService.js +++ b/services/CopilotPaymentService.js @@ -194,6 +194,7 @@ async function handlePaymentAdd(event, payment) { const newChallenge = { name: challengeTitle, projectId: project.tcDirectId, + tags: !!project.tags ? project.tags.split(',') : [], detailedRequirements: challengeRequirements, prizes: [payment.amount], reviewType: 'INTERNAL' diff --git a/services/GithubService.js b/services/GithubService.js index a53bc36..6f2eafe 100644 --- a/services/GithubService.js +++ b/services/GithubService.js @@ -53,7 +53,7 @@ async function _authenticate(accessToken) { }); return octokit.rest; } catch (err) { - throw errors.convertGitHubError(err, 'Failed to authenticate to Github using access token of copilot.'); + throw errors.handleGitHubError(err, 'Failed to authenticate to Github using access token of copilot.'); } } @@ -75,7 +75,7 @@ async function _removeAssignees(github, owner, repo, number, assignees) { assignees }); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during remove assignees from issue.'); + throw errors.handleGitHubError(err, 'Error occurred during remove assignees from issue.'); } } @@ -93,6 +93,17 @@ async function _getUsernameById(id) { return user ? user.login : null; } +/** + * Get github issue url + * @param {String} repoPath the repo path + * @param {Number} number the issue number + * @returns {String} the url + * @private + */ +function _getIssueUrl(repoPath, number) { + return `https://github.com/${repoPath}/issues/${number}`; +} + /** * updates the title of github issue * @param {Object} copilot the copilot @@ -107,7 +118,7 @@ async function updateIssue(copilot, repoFullName, number, title) { try { await github.issues.update({owner, repo, issue_number: number, title}); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during updating issue.'); + throw errors.handleGitHubError(err, 'Error occurred during updating issue.', copilot.topcoderUsername, _getIssueUrl(repoFullName, number)); } logger.debug(`Github issue title is updated for issue number ${number}`); } @@ -139,7 +150,7 @@ async function assignUser(copilot, repoFullName, number, user) { } await github.issues.addAssignees({owner, repo, issue_number: number, assignees: [user]}); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during assigning issue user.'); + throw errors.handleGitHubError(err, 'Error occurred during assigning issue user.', copilot.topcoderUsername, _getIssueUrl(repoFullName, number)); } logger.debug(`Github issue with number ${number} is assigned to ${user}`); } @@ -184,7 +195,7 @@ async function createComment(copilot, repoFullName, number, body) { body = helper.prepareAutomatedComment(body, copilot); await github.issues.createComment({owner, repo, issue_number: number, body}); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during creating comment on issue.'); + throw errors.handleGitHubError(err, 'Error occurred during creating comment on issue.', copilot.topcoderUsername, _getIssueUrl(repoFullName, number)); } logger.debug(`Github comment is added on issue with message: "${body}"`); } @@ -262,7 +273,7 @@ async function markIssueAsPaid(copilot, repoFullName, number, challengeUUID, exi const body = helper.prepareAutomatedComment(commentMessage, copilot); await github.issues.createComment({owner, repo, issue_number: number, body}); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during updating issue as paid.'); + throw errors.handleGitHubError(err, 'Error occurred during updating issue as paid.', copilot.topcoderUsername, _getIssueUrl(repoFullName, number)); } logger.debug(`Github issue title is updated for as paid and fix accepted for ${number}`); } @@ -291,7 +302,7 @@ async function changeState(copilot, repoFullName, number, state) { try { await github.issues.update({owner, repo, issue_number: number, state}); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during updating status of issue.'); + throw errors.handleGitHubError(err, 'Error occurred during updating status of issue.', copilot.topcoderUsername, _getIssueUrl(repoFullName, number)); } logger.debug(`Github issue state is updated to '${state}' for issue number ${number}`); } @@ -317,7 +328,7 @@ async function addLabels(copilot, repoFullName, number, labels) { try { await github.issues.update({owner, repo, issue_number: number, labels}); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during adding label in issue.'); + throw errors.handleGitHubError(err, 'Error occurred during adding label in issue.', copilot.topcoderUsername, _getIssueUrl(repoFullName, number)); } logger.debug(`Github issue is updated with new labels for ${number}`); } diff --git a/services/GitlabService.js b/services/GitlabService.js index 24fa38a..9e100f3 100644 --- a/services/GitlabService.js +++ b/services/GitlabService.js @@ -15,10 +15,20 @@ const Joi = require('joi'); const GitlabAPI = require('node-gitlab-api'); const logger = require('../utils/logger'); const errors = require('../utils/errors'); +const models = require('../models'); const helper = require('../utils/helper'); +const dbHelper = require('../utils/db-helper'); +const superagent = require('superagent'); +const superagentPromise = require('superagent-promise'); + +const request = superagentPromise(superagent, Promise); +// milliseconds per second +const MS_PER_SECOND = 1000; const copilotUserSchema = Joi.object().keys({ accessToken: Joi.string().required(), + accessTokenExpiration: Joi.date().required(), + refreshToken: Joi.string().required(), userProviderId: Joi.number().required(), topcoderUsername: Joi.string() }).required(); @@ -37,7 +47,7 @@ async function _authenticate(accessToken) { }); return gitlab; } catch (err) { - throw errors.convertGitLabError(err, 'Failed to during authenticate to Github using access token of copilot.'); + throw errors.handleGitLabError(err, 'Failed to during authenticate to Github using access token of copilot.'); } } @@ -55,25 +65,38 @@ async function _removeAssignees(gitlab, projectId, issueId, assignees) { const oldAssignees = _.difference(issue.assignee_ids, assignees); await gitlab.projects.issues.edit(projectId, issueId, {assignee_ids: oldAssignees}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during remove assignees from issue.'); + throw errors.handleGitLabError(err, 'Error occurred during remove assignees from issue.'); } } +/** + * Get gitlab issue url + * @param {String} repoPath the repo path + * @param {Number} issueId the issue number + * @returns {String} the url + * @private + */ +function _getIssueUrl(repoPath, issueId) { + return `https://gitlab.com/${repoPath}/issues/${issueId}`; +} + /** * creates the comments on gitlab issue * @param {Object} copilot the copilot - * @param {Number} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue number * @param {string} body the comment body text */ -async function createComment(copilot, projectId, issueId, body) { +async function createComment(copilot, project, issueId, body) { + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, body}, createComment.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); try { body = helper.prepareAutomatedComment(body, copilot); await gitlab.projects.issues.notes.create(projectId, issueId, {body}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during creating comment on issue.'); + throw errors.handleGitLabError(err, 'Error occurred during creating comment on issue.', copilot.topcoderUsername, _getIssueUrl(project.full_name, issueId)); } logger.debug(`Gitlab comment is added on issue with message: "${body}"`); } @@ -88,17 +111,19 @@ createComment.schema = { /** * updates the title of gitlab issue * @param {Object} copilot the copilot - * @param {Number} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue number * @param {string} title new title */ -async function updateIssue(copilot, projectId, issueId, title) { +async function updateIssue(copilot, project, issueId, title) { + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, title}, updateIssue.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); try { await gitlab.projects.issues.edit(projectId, issueId, {title}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during updating issue.'); + throw errors.handleGitLabError(err, 'Error occurred during updating issue.', copilot.topcoderUsername, _getIssueUrl(project.full_name, issueId)); } logger.debug(`Gitlab issue title is updated for issue number ${issueId}`); } @@ -113,13 +138,15 @@ updateIssue.schema = { /** * Assigns the issue to user login * @param {Object} copilot the copilot - * @param {Number} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue number * @param {Number} userId the user id of assignee */ -async function assignUser(copilot, projectId, issueId, userId) { +async function assignUser(copilot, project, issueId, userId) { + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, userId}, assignUser.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); try { const issue = await gitlab.projects.issues.show(projectId, issueId); const oldAssignees = _.without(issue.assignee_ids, userId); @@ -128,7 +155,7 @@ async function assignUser(copilot, projectId, issueId, userId) { } await gitlab.projects.issues.edit(projectId, issueId, {assignee_ids: [userId]}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during assigning issue user.'); + throw errors.handleGitLabError(err, 'Error occurred during assigning issue user.', copilot.topcoderUsername, _getIssueUrl(project.full_name, issueId)); } logger.debug(`Gitlab issue with number ${issueId} is assigned to ${issueId}`); } @@ -143,13 +170,15 @@ assignUser.schema = { /** * Removes an assignee from the issue * @param {Object} copilot the copilot - * @param {Number} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue number * @param {Number} userId the user id of assignee to remove */ -async function removeAssign(copilot, projectId, issueId, userId) { +async function removeAssign(copilot, project, issueId, userId) { + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, userId}, removeAssign.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); await _removeAssignees(gitlab, projectId, issueId, [userId]); logger.debug(`Gitlab user ${userId} is unassigned from issue number ${issueId}`); } @@ -164,7 +193,8 @@ removeAssign.schema = assignUser.schema; */ async function getUsernameById(copilot, userId) { Joi.attempt({copilot, userId}, getUsernameById.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); const user = await gitlab.users.show(userId); return user ? user.username : null; } @@ -182,7 +212,8 @@ getUsernameById.schema = { */ async function getUserIdByLogin(copilot, login) { Joi.attempt({copilot, login}, getUserIdByLogin.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); const user = await gitlab.users.all({username: login}); return user.length ? user[0].id : null; } @@ -195,16 +226,18 @@ getUserIdByLogin.schema = { /** * updates the gitlab issue as paid and fix accepted * @param {Object} copilot the copilot - * @param {Number} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue number * @param {String} challengeUUID the challenge uuid * @param {Array} existLabels the issue labels * @param {String} winner the winner topcoder handle * @param {Boolean} createCopilotPayments the option to create copilot payments or not */ -async function markIssueAsPaid(copilot, projectId, issueId, challengeUUID, existLabels, winner, createCopilotPayments) { // eslint-disable-line max-params +async function markIssueAsPaid(copilot, project, issueId, challengeUUID, existLabels, winner, createCopilotPayments) { // eslint-disable-line max-params + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, challengeUUID, existLabels, winner, createCopilotPayments}, markIssueAsPaid.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); const labels = _(existLabels).filter((i) => i !== config.FIX_ACCEPTED_ISSUE_LABEL) .push(config.FIX_ACCEPTED_ISSUE_LABEL, config.PAID_ISSUE_LABEL).value(); try { @@ -222,7 +255,7 @@ async function markIssueAsPaid(copilot, projectId, issueId, challengeUUID, exist const body = helper.prepareAutomatedComment(commentMessage, copilot); await gitlab.projects.issues.notes.create(projectId, issueId, {body}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during updating issue as paid.'); + throw errors.handleGitLabError(err, 'Error occurred during updating issue as paid.', copilot.topcoderUsername, _getIssueUrl(project.full_name, issueId)); } logger.debug(`Gitlab issue is updated for as paid and fix accepted for ${issueId}`); } @@ -240,17 +273,19 @@ markIssueAsPaid.schema = { /** * change the state of gitlab issue * @param {Object} copilot the copilot - * @param {string} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue issue id * @param {string} state new state */ -async function changeState(copilot, projectId, issueId, state) { +async function changeState(copilot, project, issueId, state) { + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, state}, changeState.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); try { await gitlab.projects.issues.edit(projectId, issueId, {state_event: state}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during updating status of issue.'); + throw errors.handleGitLabError(err, 'Error occurred during updating status of issue.', copilot.topcoderUsername, _getIssueUrl(project.full_name, issueId)); } logger.debug(`Gitlab issue state is updated to '${state}' for issue number ${issueId}`); } @@ -265,17 +300,19 @@ changeState.schema = { /** * updates the gitlab issue with new labels * @param {Object} copilot the copilot - * @param {string} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue issue id * @param {Number} labels the labels */ -async function addLabels(copilot, projectId, issueId, labels) { +async function addLabels(copilot, project, issueId, labels) { + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, labels}, addLabels.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); try { await gitlab.projects.issues.edit(projectId, issueId, {labels: _.join(labels, ',')}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during adding label in issue.'); + throw errors.handleGitLabError(err, 'Error occurred during adding label in issue.', copilot.topcoderUsername, _getIssueUrl(project.full_name, issueId)); } logger.debug(`Gitlab issue is updated with new labels for ${issueId}`); } @@ -287,6 +324,35 @@ addLabels.schema = { labels: Joi.array().items(Joi.string()).required() }; +/** + * Refresh the copilot access token if token is needed + * @param {Object} copilot the copilot + * @returns {Promise} the promise result of copilot with refreshed token + */ +async function _refreshGitlabUserAccessToken(copilot) { + if (copilot.accessTokenExpiration && new Date().getTime() > copilot.accessTokenExpiration.getTime() - + (config.GITLAB_REFRESH_TOKEN_BEFORE_EXPIRATION * MS_PER_SECOND)) { + const refreshTokenResult = await request + .post(`${config.GITLAB_API_BASE_URL}/oauth/token`) + .query({ + client_id: config.GITLAB_CLIENT_ID, + client_secret: config.GITLAB_CLIENT_SECRET, + refresh_token: copilot.refreshToken, + grant_type: 'refresh_token', + redirect_uri: config.GITLAB_OWNER_USER_CALLBACK_URL + }) + .end(); + // save user token data + const expiresIn = refreshTokenResult.body.expires_in || config.GITLAB_ACCESS_TOKEN_DEFAULT_EXPIRATION; + return await dbHelper.update(models.User, copilot.id, { + accessToken: refreshTokenResult.body.access_token, + accessTokenExpiration: new Date(new Date().getTime() + expiresIn * MS_PER_SECOND), + refreshToken: refreshTokenResult.body.refresh_token + }); + } + return copilot; +} + module.exports = { createComment, diff --git a/services/IssueService.js b/services/IssueService.js index e1d1cb7..0d24826 100755 --- a/services/IssueService.js +++ b/services/IssueService.js @@ -636,6 +636,7 @@ async function handleIssueCreate(event, issue, forceAssign = false) { issue.challengeUUID = await topcoderApiHelper.createChallenge({ name: issue.title, projectId, + tags: !!project.tags ? project.tags.split(',') : [], detailedRequirements: issue.body, prizes: issue.prizes }); diff --git a/services/NotificationService.js b/services/NotificationService.js new file mode 100644 index 0000000..76d2c9d --- /dev/null +++ b/services/NotificationService.js @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2022 TopCoder, Inc. All rights reserved. + */ +'use strict'; + +/** + * This service processes incoming pure challenge events. + * + * @author TCSCODER + * @version 1.0 + */ +const Joi = require('joi'); +const logger = require('../utils/logger'); +const notification = require('../utils/notification'); + +/** + * Send token expired notification + * @param {Object} event the event + */ +async function handleTokenExpired(event) { + try { + const {copilotHandle, provider} = event.data; + await notification.sendTokenExpiredAlert(copilotHandle, '', provider); + logger.debug('Send token expired notification success'); + } catch (err) { + logger.debug(`Send token expired notification failed. Internal Error: ${err}`); + } +} + +/** + * Process notification event. + * @param {Object} event the event + */ +async function process(event) { + Joi.attempt(event, process.schema); + + if (event.event === 'notification.tokenExpired') { + await handleTokenExpired(event); + } +} + +process.schema = Joi.object().keys({ + event: Joi.string().valid('notification.tokenExpired').required(), + data: Joi.object().keys({ + copilotHandle: Joi.string().required(), + provider: Joi.string().required() + }).required(), + retryCount: Joi.number().integer().default(0).optional() +}); + + +module.exports = { + process +}; + +logger.buildService(module.exports); diff --git a/services/UserService.js b/services/UserService.js index 3b1b342..71379e7 100755 --- a/services/UserService.js +++ b/services/UserService.js @@ -97,6 +97,8 @@ async function getRepositoryCopilotOrOwner(provider, repoFullName) { return { accessToken: user.accessToken, + accessTokenExpiration: user.accessTokenExpiration, + refreshToken: user.refreshToken, userProviderId: user.userProviderId, topcoderUsername: userMapping.topcoderUsername }; diff --git a/utils/db-helper.js b/utils/db-helper.js index 38425b5..29cc2e4 100644 --- a/utils/db-helper.js +++ b/utils/db-helper.js @@ -83,8 +83,10 @@ async function queryOneIssue(model, repositoryId, number, provider) { async function queryOneActiveProject(model, repoUrl) { return await new Promise((resolve, reject) => { queryOneActiveRepository(models.Repository, repoUrl).then((repo) => { - if (!repo) resolve(null); - else model.queryOne('id').eq(repo.projectId).consistent() + if (!repo) { + resolve(null); + } else { + model.queryOne('id').eq(repo.projectId).consistent() .exec((err, result) => { if (err) { logger.debug(`queryOneActiveProject. Error. ${err}`); @@ -92,6 +94,7 @@ async function queryOneActiveProject(model, repoUrl) { } return resolve(result); }); + } }); }); } @@ -342,6 +345,24 @@ async function queryOneActiveRepository(model, url) { }); } +/** + * Get Issue's challengeUUID by repoUrl + * @param {String} repoUrl The repo url + * @returns {Promise<Object>} + */ +async function queryChallengeUUIDsByRepoUrl(repoUrl) { + return await new Promise((resolve, reject) => { + models.Issue.scan('repoUrl').eq(repoUrl) + .attributes(['challengeUUID']) + .exec((err, results) => { + if (err) { + return reject(err); + } + return resolve(results.map(({challengeUUID}) => challengeUUID)); + }); + }); +} + module.exports = { getById, scan, @@ -357,6 +378,7 @@ module.exports = { queryOneUserMappingByGithubUsername, queryOneUserMappingByGitlabUsername, queryOneUserMappingByTCUsername, + queryChallengeUUIDsByRepoUrl, removeCopilotPayment, removeIssue }; diff --git a/utils/errors.js b/utils/errors.js index 29592de..81f6750 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -12,6 +12,7 @@ const _ = require('lodash'); const constants = require('../constants'); +const notification = require('./notification'); // the error class wrapper class ProcessorError extends Error { @@ -27,12 +28,17 @@ class ProcessorError extends Error { const errors = {}; /** -* Convert github api error. +* Handle github api error. Return converted error. * @param {Error} err the github api error * @param {String} message the error message +* @param {String} copilotHandle the handle name of the copilot +* @param {String} repoPath the link to related github page * @returns {Error} converted error */ -errors.convertGitHubError = function convertGitHubError(err, message) { +errors.handleGitHubError = function handleGitHubError(err, message, copilotHandle, repoPath) { + if (err.statusCode === 401 && copilotHandle && repoPath) { // eslint-disable-line no-magic-numbers + notification.sendTokenExpiredAlert(copilotHandle, repoPath, 'Github'); + } let resMsg = `${message}. ${err.message}.`; const detail = _.get(err, 'response.body.message'); if (detail) { @@ -47,12 +53,17 @@ errors.convertGitHubError = function convertGitHubError(err, message) { }; /** - * Convert gitlab api error. + * Handle gitlab api error. Return converted error. * @param {Error} err the gitlab api error * @param {String} message the error message +* @param {String} copilotHandle the handle name of the copilot +* @param {String} repoPath the link to related gitlab page * @returns {Error} converted error */ -errors.convertGitLabError = function convertGitLabError(err, message) { +errors.handleGitLabError = function handleGitLabError(err, message, copilotHandle, repoPath) { + if (err.statusCode === 401 && copilotHandle && repoPath) { // eslint-disable-line no-magic-numbers + notification.sendTokenExpiredAlert(copilotHandle, repoPath, 'Gitlab'); + } let resMsg = `${message}. ${err.message}.`; const detail = _.get(err, 'response.body.message'); if (detail) { diff --git a/utils/git-helper.js b/utils/git-helper.js index 28791d0..cc53626 100644 --- a/utils/git-helper.js +++ b/utils/git-helper.js @@ -27,7 +27,7 @@ class GitHelper { if (event.provider === 'github') { await gitHubService.createComment(event.copilot, event.data.repository.full_name, issueNumber, comment); } else if (event.provider === 'gitlab') { - await gitlabService.createComment(event.copilot, event.data.repository.id, issueNumber, comment); + await gitlabService.createComment(event.copilot, event.data.repository, issueNumber, comment); } } @@ -41,7 +41,7 @@ class GitHelper { if (event.provider === 'github') { await gitHubService.addLabels(event.copilot, event.data.repository.full_name, issueNumber, labels); } else if (event.provider === 'gitlab') { - await gitlabService.addLabels(event.copilot, event.data.repository.id, issueNumber, labels); + await gitlabService.addLabels(event.copilot, event.data.repository, issueNumber, labels); } } @@ -54,7 +54,7 @@ class GitHelper { if (event.provider === 'github') { await gitHubService.changeState(event.copilot, event.data.repository.full_name, issue.number, 'open'); } else if (event.provider === 'gitlab') { - await gitlabService.changeState(event.copilot, event.data.repository.id, issue.number, 'reopen'); + await gitlabService.changeState(event.copilot, event.data.repository, issue.number, 'reopen'); } } @@ -84,7 +84,7 @@ class GitHelper { if (event.provider === 'github') { await gitHubService.removeAssign(event.copilot, event.data.repository.full_name, issueNumber, assigneeUsername); } else if (event.provider === 'gitlab') { - await gitlabService.removeAssign(event.copilot, event.data.repository.id, issueNumber, assigneeUserId); + await gitlabService.removeAssign(event.copilot, event.data.repository, issueNumber, assigneeUserId); } } @@ -98,7 +98,7 @@ class GitHelper { if (event.provider === 'github') { await gitHubService.updateIssue(event.copilot, event.data.repository.full_name, issueNumber, newTitle); } else if (event.provider === 'gitlab') { - await gitlabService.updateIssue(event.copilot, event.data.repository.id, issueNumber, newTitle); + await gitlabService.updateIssue(event.copilot, event.data.repository, issueNumber, newTitle); } } @@ -113,7 +113,7 @@ class GitHelper { await gitHubService.assignUser(event.copilot, event.data.repository.full_name, issueNumber, assignedUser); } else if (event.provider === 'gitlab') { const userId = await gitlabService.getUserIdByLogin(event.copilot, assignedUser); - await gitlabService.assignUser(event.copilot, event.data.repository.id, issueNumber, userId); + await gitlabService.assignUser(event.copilot, event.data.repository, issueNumber, userId); } } @@ -131,7 +131,7 @@ class GitHelper { await gitHubService.markIssueAsPaid(event.copilot, event.data.repository.full_name, issueNumber, challengeUUID, existLabels, winner, createCopilotPayments); } else if (event.provider === 'gitlab') { - await gitlabService.markIssueAsPaid(event.copilot, event.data.repository.id, issueNumber, challengeUUID, existLabels, winner, + await gitlabService.markIssueAsPaid(event.copilot, event.data.repository, issueNumber, challengeUUID, existLabels, winner, createCopilotPayments); } else if (event.provider === 'azure') { await azureService.markIssueAsPaid(event.copilot, event.data.repository.full_name, issueNumber, challengeUUID, existLabels); diff --git a/utils/kafka-consumer.js b/utils/kafka-consumer.js new file mode 100644 index 0000000..30655a7 --- /dev/null +++ b/utils/kafka-consumer.js @@ -0,0 +1,99 @@ +/** + * Module wrapper for consume kafka topic. + * + * @author TCSCODER + * @version 1.0 + */ +'use strict'; + +const config = require('config'); +const _ = require('lodash'); + +const healthcheck = require('topcoder-healthcheck-dropin'); +const IssueService = require('../services/IssueService'); +const CopilotPaymentService = require('../services/CopilotPaymentService'); +const ChallengeService = require('../services/ChallengeService'); +const NotificationService = require('../services/NotificationService'); +const logger = require('./logger'); +const kafka = require('./kafka'); + +/** + * Handle the message from kafka + * @param {Object} messageSet object to handle + */ +function messageHandler(messageSet) { + logger.debug(` topics ======= ${JSON.stringify(messageSet)}`); + messageSet.forEach((item) => { + // The event should be a JSON object + let event; + try { + const message = JSON.parse(item.message.value.toString('utf8')); + event = JSON.parse(message.payload.value); + message.payload.value = event; + logger.debug(`received message from kafka: ${JSON.stringify(_.omit(message, 'payload.value.data.issue.body'))}`); + } catch (err) { + logger.error(`"message" is not a valid JSON-formatted string: ${err.message}`); + return; + } + + if (event && _.includes(['issue.created', 'issue.updated', 'issue.closed', 'issue.recreated', + 'comment.created', 'comment.updated', 'issue.assigned', 'issue.labelUpdated', 'issue.unassigned'] + , event.event)) { + IssueService + .process(event) + .catch(logger.error); + } + if (event && _.includes(['copilotPayment.add', 'copilotPayment.update', 'copilotPayment.delete', 'copilotPayment.checkUpdates'] + , event.event)) { + CopilotPaymentService + .process(event) + .catch(logger.error); + } + if (event && _.includes(['challengeTags.update'] + , event.event)) { + ChallengeService + .process(event) + .catch(logger.error); + } + if (event && _.includes(['notification.tokenExpired'] + , event.event)) { + NotificationService + .process(event) + .catch(logger.error); + } + }); +} + +/** + * check if there is kafka connection alive + * @returns {Boolean} true + */ +function check() { + // if (!this.consumer.client.initialBrokers && !this.consumer.client.initialBrokers.length) { + // logger.info(`Brokers Exist Check Failed ${this.consumer.client.initialBrokers} ${this.consumer.client.initialBrokers.length}`) + // return false; + // } + // let connected = true; + // this.consumer.client.initialBrokers.forEach((conn) => { + // logger.info(`Brokers Check Failed ${conn.connected}`) + // connected = conn.connected && connected; + // }); + + // return connected; + return true; +} + +/** + * run the consumer + */ +function run() { + kafka.consumer.init().then(() => { + logger.info('kafka consumer is ready'); + healthcheck.init([check]); + kafka.consumer.subscribe(config.TOPIC, {}, messageHandler); + }).catch((err) => { + logger.error(`kafka consumer is not connected. ${err.stack}`); + }); +} + +module.exports = {run}; diff --git a/utils/kafka-sender.js b/utils/kafka-sender.js new file mode 100644 index 0000000..a053456 --- /dev/null +++ b/utils/kafka-sender.js @@ -0,0 +1,61 @@ +/** + * Module wrapper for sending messages to kafka. + * + * @author TCSCODER + * @version 1.0 + */ +'use strict'; + +const config = require('config'); +const kafka = require('./kafka'); + +/** + * Send message to general topic in kafka. + * @param {String} message the message to send + * @returns {Object} Result from kafka + */ +function send(message) { + const data = JSON.stringify({ + topic: config.TOPIC, + originator: 'topcoder-x-processor', + timestamp: (new Date()).toISOString(), + 'mime-type': 'application/json', + payload: { + value: message + } + }); + return kafka.producer.send({ + topic: config.TOPIC, + message: { + value: data + } + }); +} + +/** + * Send message to notification topic in kafka. + * @param {String} notification the message to send + * @returns {Object} Result from kafka + */ +function sendNotification(notification) { + const data = JSON.stringify({ + topic: config.TOPIC_NOTIFICATION, + originator: 'topcoder-x-processor', + timestamp: (new Date()).toISOString(), + 'mime-type': 'application/json', + payload: { + notifications: [notification] + } + }); + return kafka.producer.send({ + topic: config.TOPIC_NOTIFICATION, + message: { + value: data + } + }); +} + +module.exports = { + send, + sendNotification +}; diff --git a/utils/kafka.js b/utils/kafka.js index b731664..3056fba 100644 --- a/utils/kafka.js +++ b/utils/kafka.js @@ -12,11 +12,7 @@ 'use strict'; const config = require('config'); -const _ = require('lodash'); const kafka = require('no-kafka'); -const healthcheck = require('topcoder-healthcheck-dropin'); -const IssueService = require('../services/IssueService'); -const CopilotPaymentService = require('../services/CopilotPaymentService'); const logger = require('./logger'); class Kafka { @@ -29,82 +25,7 @@ class Kafka { }).catch((err) => { logger.error(`kafka producer is not connected. ${err.stack}`); }); - this.check = this.check.bind(this); - } - - messageHandler(messageSet) { - logger.debug(` topics ======= ${JSON.stringify(messageSet)}`); - messageSet.forEach((item) => { - // The event should be a JSON object - let event; - try { - const message = JSON.parse(item.message.value.toString('utf8')); - event = JSON.parse(message.payload.value); - message.payload.value = event; - logger.debug(`received message from kafka: ${JSON.stringify(_.omit(message, 'payload.value.data.issue.body'))}`); - } catch (err) { - logger.error(`"message" is not a valid JSON-formatted string: ${err.message}`); - return; - } - - if (event && _.includes(['issue.created', 'issue.updated', 'issue.closed', 'issue.recreated', - 'comment.created', 'comment.updated', 'issue.assigned', 'issue.labelUpdated', 'issue.unassigned'] - , event.event)) { - IssueService - .process(event) - .catch(logger.error); - } - if (event && _.includes(['copilotPayment.add', 'copilotPayment.update', 'copilotPayment.delete', 'copilotPayment.checkUpdates'] - , event.event)) { - CopilotPaymentService - .process(event) - .catch(logger.error); - } - }); - } - - // check if there is kafka connection alive - check() { - // if (!this.consumer.client.initialBrokers && !this.consumer.client.initialBrokers.length) { - // logger.info(`Brokers Exist Check Failed ${this.consumer.client.initialBrokers} ${this.consumer.client.initialBrokers.length}`) - // return false; - // } - // let connected = true; - // this.consumer.client.initialBrokers.forEach((conn) => { - // logger.info(`Brokers Check Failed ${conn.connected}`) - // connected = conn.connected && connected; - // }); - - // return connected; - return true; - } - - run() { - this.consumer.init().then(() => { - logger.info('kafka consumer is ready'); - healthcheck.init([this.check]); - this.consumer.subscribe(config.TOPIC, {}, this.messageHandler); - }).catch((err) => { - logger.error(`kafka consumer is not connected. ${err.stack}`); - }); - } - - send(message) { - const data = JSON.stringify({ - topic: config.TOPIC, - originator: 'topcoder-x-processor', - timestamp: (new Date()).toISOString(), - 'mime-type': 'application/json', - payload: { - value: message - } - }); - return this.producer.send({ - topic: config.TOPIC, - message: { - value: data - } - }); + // this.check = this.check.bind(this); } } diff --git a/utils/notification.js b/utils/notification.js new file mode 100644 index 0000000..f294275 --- /dev/null +++ b/utils/notification.js @@ -0,0 +1,62 @@ +/** + * This module contains the helper methods + * for sending notification action to kafka service. + * + * @author TCSCODER + * @version 1.0 + */ +'use strict'; + +const config = require('config'); + +const kafkaSender = require('./kafka-sender'); +const topcoderApiHelper = require('./topcoder-api-helper'); +const logger = require('./logger'); + +const notification = {}; + +/** + * get content template to send + * @param {String} repoPath the repo path + * @returns {String} + */ +function getContent(repoPath) { + return repoPath + ? + `Hi {handle}, + You made an update to ticket {link}, but Topcoder-X couldn't process it properly because your {provider} token has expired. To fix this, please login to x.topcoder.com, click your handle in the upper right and then "Settings" to refresh your token. You will need to redo the action that failed in {provider}.` // eslint-disable-line max-len + : + `Hi {handle}, + You made an operation on {provider}, but Topcoder-X couldn't process it properly because your {provider} token has expired. To fix this, please login to x.topcoder.com, click your handle in the upper right and then "Settings" to refresh your token. You will need to redo the action that failed in {provider}.`; // eslint-disable-line max-len +} + +notification.sendTokenExpiredAlert = async function sendTokenExpiredAlert(copilotHandle, repoPath, provider) { + const copilotId = await topcoderApiHelper.getTopcoderMemberId(copilotHandle); + const notificationConfigs = config.MAIL_NOTICIATION; + const content = getContent(repoPath); + logger.debug(`Sending mail notification to copilot ${copilotHandle} Repo: ${repoPath} Provider: ${provider}`); + await kafkaSender.sendNotification({ + serviceId: 'email', + type: notificationConfigs.type, + details: { + from: 'noreply@topcoder.com', + recipients: [ + { + userId: copilotId + } + ], + cc: [], + data: { + subject: notificationConfigs.subject, + body: content + .replace(/{handle}/g, copilotHandle) + .replace(/{link}/g, repoPath) + .replace(/{provider}/g, provider) + }, + sendgridTemplateId: notificationConfigs.sendgridTemplateId, + version: 'v3' + } + }); +}; + +module.exports = notification; diff --git a/utils/topcoder-api-helper.js b/utils/topcoder-api-helper.js index 94d18b1..ade67d0 100644 --- a/utils/topcoder-api-helper.js +++ b/utils/topcoder-api-helper.js @@ -75,6 +75,7 @@ async function createChallenge(challenge) { }], timelineTemplateId: config.DEFAULT_TIMELINE_TEMPLATE_ID, projectId: challenge.projectId, + tags: challenge.tags, trackId: config.DEFAULT_TRACK_ID, legacy: { pureV5Task: true