Stream downloads instead of saving to a buffer (#23)

Support streaming downloads and fixing vulnerabilities

Author: dhruvit-r

bin/topcoder-cli.js

@@ -41,8 +41,8 @@ program
         `  "username": "<Topcoder username>",\n` +
         `  "password": "<Topcoder password>",\n` +
         `  "m2m": {\n` +
-        `    client_id: "<Client ID for M2M authentication>",\n` +
-        `    client_secret: "<Client Secret for M2M authentication>"\n` +
+        `    "client_id": "<Client ID for M2M authentication>",\n` +
+        `    "client_secret": "<Client Secret for M2M authentication>"\n` +
         `  }\n` +
         `}\n` +
         `and execute command \`topcoder submit\` to submit the contents of ` +
@@ -72,8 +72,8 @@ program
     'Challenge ID for submissions to be fetched'
   )
   .option(
-    '-u, --userId <id>',
-    'Fetch only the submission of for a particular user id'
+    '-m, --memberId <id>',
+    'Fetch only the submission of for a particular member id'
   )
   .option(
     '-s, --submissionId <id>',
@@ -83,25 +83,25 @@ program
   .option('--dev', 'Points to Topcoder development environment')
   .on('--help', () => {
     console.log(
-      `\nUse CLI parameters or create a file .topcoderrc in JSON format with below details` +
+      `\nUse CLI parameters or create a file .topcoderrc in JSON format with below details\n` +
         `{\n` +
-        `  "userId": "<Topcoder memberId",\n` +
+        `  "memberId": "<Topcoder memberId",\n` +
         `  "challengeId": "<Topcoder challengeId",\n` +
         `  "submissionId": "<Topcoder submissionId",\n` +
         `  "latest": true,\n` +
         `  "username": "<Topcoder username>",\n` +
         `  "password": "<Topcoder password>",\n` +
         `  "m2m": {\n` +
-        `    client_id: "<Client ID for M2M authentication>",\n` +
-        `    client_secret: "<Client Secret for M2M authentication>"\n` +
+        `    "client_id": "<Client ID for M2M authentication>",\n` +
+        `    "client_secret": "<Client Secret for M2M authentication>"\n` +
         `  }\n` +
         `}\n` +
         `and execute command \`topcoder fetch-submissions\` to fetch submissions ` +
         `for a challenge and save them.\n` +
         `You may specify the m2m config or the username and password config, ` +
         `but not both.\n` +
         `If the submissionId parameter is provided, you must not provide the ` +
-        `userId or the latest parameters.\n` +
+        `memberId or the latest parameters.\n` +
         `The challengeId parameter is always required.`
     )
   })
@@ -140,8 +140,8 @@ program
         `  "username": "<Topcoder username>",\n` +
         `  "password": "<Topcoder password>",\n` +
         `  "m2m": {\n` +
-        `    client_id: "<Client ID for M2M authentication>",\n` +
-        `    client_secret: "<Client Secret for M2M authentication>"\n` +
+        `    "client_id": "<Client ID for M2M authentication>",\n` +
+        `    "client_secret": "<Client Secret for M2M authentication>"\n` +
         `  }\n` +
         `}\n` +
         `and execute command \`topcoder fetch-artifacts\` to fetch submissions for` +

package.json

@@ -12,7 +12,7 @@
   },
   "dependencies": {
     "@hapi/joi": "^16.1.8",
-    "@topcoder-platform/topcoder-submission-api-wrapper": "^1.1.0",
+    "@topcoder-platform/topcoder-submission-api-wrapper": "^1.2.0",
     "adm-zip": "^0.4.11",
     "commander": "^3.0.0",
     "content-disposition": "^0.5.3",
@@ -22,10 +22,8 @@
     "ini": "^1.3.5",
     "lodash": "^4.17.15",
     "moment": "^2.24.0",
-    "progress": "^2.0.3",
     "prompts": "^2.2.1",
     "superagent": "^5.0.5",
-    "tc-core-library-js": "https://github.com/appirio-tech/tc-core-library-js/archive/v2.6.3.tar.gz",
     "winston": "^3.2.1"
   },
   "bin": {

src/services/fetchArtifactsService.js

@@ -38,17 +38,32 @@ async function downloadArtifacts (submissionId, artifacts, savePath) {
     )
     try {
       // Download the artifact
-      const req = await submissionApiClient.downloadArtifact(
+      let req = await submissionApiClient.downloadArtifact(
         submissionId,
-        artifactId
+        artifactId,
+        null,
+        true
       )
+      // Get the temporary path
+      const temporaryFilePath = path.join(savePath, `artifact-${artifactId}.tcdownload`)
+      // Save the file
+      const fStream = fs.createWriteStream(temporaryFilePath)
+      const writeStream = req.pipe(fStream)
+      // Wait for write to complete
+      await new Promise((resolve, reject) => {
+        req.on('response', (_req) => {
+          req = _req
+        })
+        writeStream.on('finish', resolve)
+        writeStream.on('error', reject)
+      })
       // Get file name from headers
       const disposition = _.get(req, 'headers.content-disposition')
       const fileName = contentDisposition.parse(disposition).parameters.filename
-      // Get file path
-      const filePath = path.join(savePath, `${fileName}`)
+      // Get the final file path
+      const filePath = path.join(savePath, fileName)
       // Save the file
-      await fs.writeFile(filePath, req.body)
+      await fs.move(temporaryFilePath, filePath)
       // Log the result
       logger.info(
         `[${idx + 1}/${artifacts.length}] ` +

src/services/fetchSubmissionService.js

@@ -14,17 +14,17 @@ let submissionApiClient
 const schemaForRC = helper.defaultAuthSchema
   .keys({
     challengeId: Joi.string().required(),
-    userId: Joi.number().integer(),
+    memberId: Joi.number().integer(),
     submissionId: Joi.string(),
     latest: Joi.boolean()
   })
-  .without('submissionId', ['userId', 'latest'])
-  .without('userId', 'submissionId')
+  .without('submissionId', ['memberId', 'latest'])
+  .without('memberId', 'submissionId')
   .without('latest', 'submissionId')
   .unknown()
 
 // Acceptable CLI params
-const validCLIParams = ['challengeId', 'userId', 'submissionId', 'latest']
+const validCLIParams = ['challengeId', 'memberId', 'submissionId', 'latest']
 
 /**
  * Download the submissions to the savePath directory sequentially.s
@@ -43,14 +43,27 @@ async function downloadSubmissions (submissions, savePath) {
     )
     try {
       // Download the submission
-      const req = await submissionApiClient.downloadSubmission(submission.id)
+      let req = await submissionApiClient.downloadSubmission(submission.id, null, true)
+      // Get the temporary path
+      const temporaryFilePath = path.join(savePath, `submission-${submission.id}.tcdownload`)
+      // Save the file
+      const fStream = fs.createWriteStream(temporaryFilePath)
+      const writeStream = req.pipe(fStream)
+      // Wait for write to complete
+      await new Promise((resolve, reject) => {
+        req.on('response', (_req) => {
+          req = _req
+        })
+        writeStream.on('finish', resolve)
+        writeStream.on('error', reject)
+      })
       // Get file name from headers
       const disposition = _.get(req, 'headers.content-disposition')
       const fileName = contentDisposition.parse(disposition).parameters.filename
-      // Get file path
-      const filePath = path.join(savePath, `${fileName}`)
+      // Get the final file path
+      const filePath = path.join(savePath, fileName)
       // Save the file
-      await fs.writeFile(filePath, req.body)
+      await fs.move(temporaryFilePath, filePath)
       // Log the result
       logger.info(
         `[${idx + 1}/${submissions.length}] ` +
@@ -79,7 +92,7 @@ async function fetchSubmissions (currDir, cliParams) {
     password,
     m2m,
     challengeId,
-    userId,
+    memberId,
     submissionId,
     latest
   } = await helper.readFromRCFile(rcPath, params, schemaForRC, validCLIParams)
@@ -116,9 +129,9 @@ async function fetchSubmissions (currDir, cliParams) {
       perPage: 100,
       page
     }
-    // If there's a userId specified, filter by userId.
-    if (userId) {
-      query.memberId = userId
+    // If there's a memberId specified, filter by memberId.
+    if (memberId) {
+      query.memberId = memberId
     }
     // Get a list of submissions
     const nextSubmissions = await submissionApiClient.searchSubmissions(query)