From 4b160ab4d1f19116b4db0fd4845a845493378366 Mon Sep 17 00:00:00 2001 From: eisbilir Date: Wed, 27 Sep 2023 13:55:07 +0300 Subject: [PATCH 1/7] chore: add request id log --- app-routes.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app-routes.js b/app-routes.js index 3632e13e..50c9fe94 100644 --- a/app-routes.js +++ b/app-routes.js @@ -5,8 +5,10 @@ const _ = require("lodash"); const config = require("config"); const HttpStatus = require("http-status-codes"); +const uuid = require("uuid/v4"); const helper = require("./src/common/helper"); const errors = require("./src/common/errors"); +const logger = require("./src/common/logger"); const routes = require("./src/routes"); const authenticator = require("tc-core-library-js").middleware.jwtAuthenticator; @@ -26,7 +28,9 @@ module.exports = (app) => { const actions = []; actions.push((req, res, next) => { - req.signature = `${def.controller}#${def.method}`; + req._id = uuid(); + req.signature = `${req._id}-${def.controller}#${def.method}`; + logger.info(`Started request handling, ${req.signature}`); next(); }); From 8b07c46f559224e23992a1f4f7d79ef3442fb14d Mon Sep 17 00:00:00 2001 From: eisbilir Date: Wed, 27 Sep 2023 16:03:43 +0300 Subject: [PATCH 2/7] feat: update logging --- app-routes.js | 5 ++++- src/common/logger.js | 14 ++++++++++---- src/services/ChallengeService.js | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/app-routes.js b/app-routes.js index 50c9fe94..96f13bac 100644 --- a/app-routes.js +++ b/app-routes.js @@ -108,7 +108,10 @@ module.exports = (app) => { } }); } - + actions.push((req, res, next) => { + logger.info(`Done request handling, ${req.signature}`); + next(); + }); actions.push(method); app[verb](`/${config.API_VERSION}${path}`, helper.autoWrapExpress(actions)); }); diff --git a/src/common/logger.js b/src/common/logger.js index 82ae38ed..9952ecb7 100644 --- a/src/common/logger.js +++ b/src/common/logger.js @@ -30,10 +30,16 @@ logger.logFullError = (err, signature) => { if (signature) { logger.error(`Error happened in ${signature}`); } - logger.error(util.inspect(err)); - if (!err.logged) { - logger.error(err.stack); - err.logged = true; + if (err.isJoi) { + logger.error( + `${e.name} details: ${JSON.stringify(e.details)} input:${JSON.stringify(e._object)}` + ); + } else if (err.isAxiosError) { + logger.error(`${err.message} - ${err.response.data}`); + } else if (err.httpStatus) { + logger.error(err.message); + } else { + logger.error(util.inspect(err)); } }; diff --git a/src/services/ChallengeService.js b/src/services/ChallengeService.js index 336156d6..3d3a5b60 100644 --- a/src/services/ChallengeService.js +++ b/src/services/ChallengeService.js @@ -1420,7 +1420,7 @@ async function updateChallenge(currentUser, challengeId, data) { // Remove fields from data that are not allowed to be updated and that match the existing challenge data = sanitizeData(sanitizeChallenge(data), challenge); - console.debug("Sanitized Data:", data); + logger.debug("Sanitized Data:", JSON.stringify(data)); await validateChallengeUpdateRequest(currentUser, challenge, data); From 9e6563ebdb0fc10e4fb972f8aa4b2dd8d87dfbbe Mon Sep 17 00:00:00 2001 From: eisbilir Date: Wed, 27 Sep 2023 17:23:47 +0300 Subject: [PATCH 3/7] fix: error log --- src/common/logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/logger.js b/src/common/logger.js index 9952ecb7..84814e20 100644 --- a/src/common/logger.js +++ b/src/common/logger.js @@ -32,7 +32,7 @@ logger.logFullError = (err, signature) => { } if (err.isJoi) { logger.error( - `${e.name} details: ${JSON.stringify(e.details)} input:${JSON.stringify(e._object)}` + `${err.name} details: ${JSON.stringify(err.details)} input:${JSON.stringify(err._object)}` ); } else if (err.isAxiosError) { logger.error(`${err.message} - ${err.response.data}`); From 9396cf31d58a17ec19cd1436d81a3d7a26d523a2 Mon Sep 17 00:00:00 2001 From: eisbilir Date: Wed, 27 Sep 2023 17:47:06 +0300 Subject: [PATCH 4/7] feat: use count api --- app-routes.js | 2 +- src/common/helper.js | 31 +++++++++++++++++++++++++++ src/phase-management/PhaseAdvancer.js | 11 ++++++---- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/app-routes.js b/app-routes.js index 96f13bac..bf064e95 100644 --- a/app-routes.js +++ b/app-routes.js @@ -108,11 +108,11 @@ module.exports = (app) => { } }); } + actions.push(method); actions.push((req, res, next) => { logger.info(`Done request handling, ${req.signature}`); next(); }); - actions.push(method); app[verb](`/${config.API_VERSION}${path}`, helper.autoWrapExpress(actions)); }); }); diff --git a/src/common/helper.js b/src/common/helper.js index 67a37435..f39800f0 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -253,6 +253,22 @@ async function getChallengeResources(challengeId, roleId = null) { return result; } +/** + * Get challenge resources count + * @param {String} challengeId the challenge id + * @returns {Promise} the challenge resources count + */ +async function getChallengeResourcesCount(challengeId, roleId = null) { + const token = await m2mHelper.getM2MToken(); + const url = `${config.RESOURCES_API_URL}/count?challengeId=${challengeId}${ + roleId ? `&roleId=${roleId}` : "" + }`; + const res = await axios.get(url, { + headers: { Authorization: `Bearer ${token}` }, + }); + return res.data; +} + /** * Create challenge resources * @param {String} challengeId the challenge id @@ -1053,6 +1069,19 @@ async function getChallengeSubmissions(challengeId) { return allSubmissions; } +/** + * Get challenge submissions count + * @param {String} challengeId the challenge id + * @returns {Promise} the submission counts + */ +async function getChallengeSubmissionsCount(challengeId) { + const token = await m2mHelper.getM2MToken(); + const res = await axios.get(`${config.SUBMISSIONS_API_URL}/${challengeId}/count`, { + headers: { Authorization: `Bearer ${token}` }, + }); + return res.data; +} + /** * Get member by ID * @param {String} userId the user ID @@ -1183,6 +1212,7 @@ module.exports = { downloadFromS3, deleteFromS3, getChallengeResources, + getChallengeResourcesCount, createResource, getUserGroups, ensureNoDuplicateOrNullElements, @@ -1206,6 +1236,7 @@ module.exports = { getProjectIdByRoundId, getGroupById, getChallengeSubmissions, + getChallengeSubmissionsCount, getMemberById, createSelfServiceProject, activateProject, diff --git a/src/phase-management/PhaseAdvancer.js b/src/phase-management/PhaseAdvancer.js index f6304e3c..08f34e14 100644 --- a/src/phase-management/PhaseAdvancer.js +++ b/src/phase-management/PhaseAdvancer.js @@ -274,15 +274,18 @@ class PhaseAdvancer { async #getRegistrantCount(challengeId) { console.log(`Getting registrant count for challenge ${challengeId}`); // TODO: getChallengeResources loops through all pages, which is not necessary here, we can just get the first page and total count from header[X-Total] - const submitters = await helper.getChallengeResources(challengeId, config.SUBMITTER_ROLE_ID); - return submitters.length; + const submitters = await helper.getChallengeResourcesCount( + challengeId, + config.SUBMITTER_ROLE_ID + ); + return submitters[config.SUBMITTER_ROLE_ID] || 0; } async #getSubmissionCount(challengeId) { console.log(`Getting submission count for challenge ${challengeId}`); // TODO: getChallengeSubmissions loops through all pages, which is not necessary here, we can just get the first page and total count from header[X-Total] - const submissions = await helper.getChallengeSubmissions(challengeId); - return submissions.length; + const submissions = await helper.getChallengeSubmissionsCount(challengeId); + return submissions["Contest Submission"] || 0; } async #areAllSubmissionsReviewed(challengeId) { From 227775397645e53b353136a6cb925c8753629ca6 Mon Sep 17 00:00:00 2001 From: eisbilir Date: Wed, 27 Sep 2023 18:25:04 +0300 Subject: [PATCH 5/7] feat: ignore logs for checkHealth --- app-routes.js | 12 +++++------- src/services/ChallengeService.js | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/app-routes.js b/app-routes.js index bf064e95..4a943233 100644 --- a/app-routes.js +++ b/app-routes.js @@ -28,9 +28,11 @@ module.exports = (app) => { const actions = []; actions.push((req, res, next) => { - req._id = uuid(); - req.signature = `${req._id}-${def.controller}#${def.method}`; - logger.info(`Started request handling, ${req.signature}`); + if (def.method !== "checkHealth") { + req._id = uuid(); + req.signature = `${req._id}-${def.controller}#${def.method}`; + logger.info(`Started request handling, ${req.signature}`); + } next(); }); @@ -109,10 +111,6 @@ module.exports = (app) => { }); } actions.push(method); - actions.push((req, res, next) => { - logger.info(`Done request handling, ${req.signature}`); - next(); - }); app[verb](`/${config.API_VERSION}${path}`, helper.autoWrapExpress(actions)); }); }); diff --git a/src/services/ChallengeService.js b/src/services/ChallengeService.js index 3d3a5b60..01f2e4b6 100644 --- a/src/services/ChallengeService.js +++ b/src/services/ChallengeService.js @@ -1420,7 +1420,7 @@ async function updateChallenge(currentUser, challengeId, data) { // Remove fields from data that are not allowed to be updated and that match the existing challenge data = sanitizeData(sanitizeChallenge(data), challenge); - logger.debug("Sanitized Data:", JSON.stringify(data)); + logger.debug(`Sanitized Data: ${JSON.stringify(data)}`); await validateChallengeUpdateRequest(currentUser, challenge, data); From aafc65fca40f34a4ba21fa21d42bf92822bf0453 Mon Sep 17 00:00:00 2001 From: eisbilir Date: Fri, 29 Sep 2023 15:47:03 +0300 Subject: [PATCH 6/7] feat: add retry policy --- package.json | 4 ++-- src/services/ChallengeService.js | 21 ++++++++++++++++++++- yarn.lock | 30 +++++++++++++++--------------- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index b992d707..6a8db892 100644 --- a/package.json +++ b/package.json @@ -42,8 +42,8 @@ "dependencies": { "@grpc/grpc-js": "^1.8.12", "@opensearch-project/opensearch": "^2.2.0", - "@topcoder-framework/domain-challenge": "^0.22.0", - "@topcoder-framework/lib-common": "^0.22.0", + "@topcoder-framework/domain-challenge": "^0.23.0", + "@topcoder-framework/lib-common": "^0.23.0", "aws-sdk": "^2.1145.0", "axios": "^0.19.0", "axios-retry": "^3.4.0", diff --git a/src/services/ChallengeService.js b/src/services/ChallengeService.js index 01f2e4b6..c22ae80c 100644 --- a/src/services/ChallengeService.js +++ b/src/services/ChallengeService.js @@ -46,7 +46,26 @@ const { const deepEqual = require("deep-equal"); const { getM2MToken } = require("../common/m2m-helper"); -const challengeDomain = new ChallengeDomain(GRPC_CHALLENGE_SERVER_HOST, GRPC_CHALLENGE_SERVER_PORT); +const challengeDomain = new ChallengeDomain( + GRPC_CHALLENGE_SERVER_HOST, + GRPC_CHALLENGE_SERVER_PORT, + { + "grpc.service_config": JSON.stringify({ + methodConfig: [ + { + name: [{}], + retryPolicy: { + maxAttempts: 5, + initialBackoff: "0.5s", + maxBackoff: "30s", + backoffMultiplier: 2, + retryableStatusCodes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "INTERNAL"], + }, + }, + ], + }), + } +); const phaseAdvancer = new PhaseAdvancer(challengeDomain); /** diff --git a/yarn.lock b/yarn.lock index 35cef421..a52ead2b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -255,31 +255,31 @@ resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf" integrity sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A== -"@topcoder-framework/client-relational@^0.22.0": - version "0.22.0" - resolved "https://topcoder-409275337247.d.codeartifact.us-east-1.amazonaws.com/npm/topcoder-framework/@topcoder-framework/client-relational/-/client-relational-0.22.0.tgz#0e096758ffd8c9d0eb986b2f9328ed247930abfe" - integrity sha512-We0sb8pdxOZfzX8WzKxczhXl16jmZ6cN/eBgDv5jR8qpVoXhLTa2iaTLqiRYUWi9ZvHCN6vmNQ607w0IU/iRFQ== +"@topcoder-framework/client-relational@^0.23.0": + version "0.23.0" + resolved "https://topcoder-409275337247.d.codeartifact.us-east-1.amazonaws.com/npm/topcoder-framework/@topcoder-framework/client-relational/-/client-relational-0.23.0.tgz#c15d102c47044e23df0d1823b5eec898d8a8bb15" + integrity sha512-/0K4TRTHCe3bRUM5H1wYlPbKZazg/PJNfHkb07i1tyi77YiH4YT8zgW9A23SAoW4/k3xsveSkI8U+YbmIqnALw== dependencies: "@grpc/grpc-js" "^1.8.0" - "@topcoder-framework/lib-common" "^0.22.0" + "@topcoder-framework/lib-common" "^0.23.0" topcoder-interface "github:topcoder-platform/plat-interface-definition#v0.0.58-beta-1" tslib "^2.4.1" -"@topcoder-framework/domain-challenge@^0.22.0": - version "0.22.0" - resolved "https://topcoder-409275337247.d.codeartifact.us-east-1.amazonaws.com/npm/topcoder-framework/@topcoder-framework/domain-challenge/-/domain-challenge-0.22.0.tgz#bcb7f7a602e424d9932fd0693935aa5f1f2439a4" - integrity sha512-PT2Zts56QKtntSJQxjH8slRjrYISuUGCZdYmyQcy+ak0nQL0COhQ0puqJ6mfIA9Ml3Ggi8Vmk/G9Ti12h1YNDg== +"@topcoder-framework/domain-challenge@^0.23.0": + version "0.23.0" + resolved "https://topcoder-409275337247.d.codeartifact.us-east-1.amazonaws.com/npm/topcoder-framework/@topcoder-framework/domain-challenge/-/domain-challenge-0.23.0.tgz#31964a09864afb0f4969ea22addeb89ba47d9689" + integrity sha512-IRLM18PCekatQsea1pzfZWGiiO3+8vxQrM3BmWGGpls2BAOIrS+/y1MLJDM23DasXEmMOz+zFQe939VQyrZ7Qw== dependencies: "@grpc/grpc-js" "^1.8.0" - "@topcoder-framework/client-relational" "^0.22.0" - "@topcoder-framework/lib-common" "^0.22.0" + "@topcoder-framework/client-relational" "^0.23.0" + "@topcoder-framework/lib-common" "^0.23.0" topcoder-interface "github:topcoder-platform/plat-interface-definition#v0.0.58-beta-1" tslib "^2.4.1" -"@topcoder-framework/lib-common@^0.22.0": - version "0.22.0" - resolved "https://topcoder-409275337247.d.codeartifact.us-east-1.amazonaws.com/npm/topcoder-framework/@topcoder-framework/lib-common/-/lib-common-0.22.0.tgz#bd3428b0199410a5151326d1d9731c404c255fb5" - integrity sha512-sHdOAyCGcNaDT9esc9Q3sNaqvVAwHPv6NCTlTAt5O9dcSpdz2AyEur8mS5WccFclKhF5ZB9BM1bbWxO8i9WXGQ== +"@topcoder-framework/lib-common@^0.23.0": + version "0.23.0" + resolved "https://topcoder-409275337247.d.codeartifact.us-east-1.amazonaws.com/npm/topcoder-framework/@topcoder-framework/lib-common/-/lib-common-0.23.0.tgz#01fb1e9c2a32760387f3444ce991b9164c75f577" + integrity sha512-burWJaxo/rt1KtqxTlXRenG4qUK1QioSq8S7ouq6pUlhC1vyJSiBCum7QWR6E9H/sWit8wRl+gZBMEVVJVHbHQ== dependencies: "@grpc/grpc-js" "^1.8.0" rimraf "^3.0.2" From f039b1f4f0d0d3a0634d758f1d855796aeaeaf1c Mon Sep 17 00:00:00 2001 From: eisbilir Date: Fri, 29 Sep 2023 16:01:12 +0300 Subject: [PATCH 7/7] fix: service name --- src/services/ChallengeService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/ChallengeService.js b/src/services/ChallengeService.js index c22ae80c..78b8e240 100644 --- a/src/services/ChallengeService.js +++ b/src/services/ChallengeService.js @@ -53,7 +53,7 @@ const challengeDomain = new ChallengeDomain( "grpc.service_config": JSON.stringify({ methodConfig: [ { - name: [{}], + name: [{ service: "" }], retryPolicy: { maxAttempts: 5, initialBackoff: "0.5s",