Upload scan details as artifacts instead of directly uploading to S3

Author: callmekatootie

README.md

@@ -40,26 +40,8 @@ The following parameters can be set in config files or in env variables:
 - SONARQUBE_TOKEN: SonarQube access token. Optional parameter, must be set when ACL is configured for the project.
 More details about access tokens - https://docs.sonarqube.org/latest/user-guide/user-token/
 
-- AWS_S3_BUCKET: AWS S3 Bucket name. Bucket must be already exist
-- AWS_S3_PREFIX: AWS S3 Key prefix. Optional parameter. Must end with slash
-
 - All variables starting with prefix `AUTH0` corresponds to Auth0 related credentials
 
-### AWS Credentials
-* Please refer to following [documentation](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/getting-your-credentials.html) in order to get AWS credentials.
-* Depending on your Operating System, create AWS credentials file in the path listed below
-```
-Linux, Unix, and macOS users: ~/.aws/credentials
-Windows users: C:\Users\USER_NAME\.aws\credentials
-```
-Credentials file should look like below
-```
-[default]
-aws_access_key_id = SOME_ACCESS_KEY_ID
-aws_secret_access_key = SOME_SECRET_ACCESS_KEY
-```
-* Credentials can be set in environment variables. [Documentation](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html)
-
 ## Local deployment
 
 1. From the project root directory, run the following command to install the dependencies
@@ -112,7 +94,7 @@ To run the Submission quality processor using docker, follow the below steps
 
 2. Rename the file `sample.api.env` to `api.env`
 
-3. Set the required Auth0, Submission API URL, SonarQube config and AWS S3 configuration in the file `api.env`
+3. Set the required Auth0, Submission API URL and SonarQube config in the file `api.env`
 
 4. Once that is done, run the following command
 

config/default.js

@@ -11,9 +11,6 @@ module.exports = {
   SONARQUBE_HOST: process.env.SONARQUBE_HOST || 'http://localhost:9000',
   SONARQUBE_TOKEN: process.env.SONARQUBE_TOKEN,
 
-  AWS_S3_BUCKET: process.env.AWS_S3_BUCKET,
-  AWS_S3_PREFIX: process.env.AWS_S3_PREFIX || '',
-
   AUTH0_URL: process.env.AUTH0_URL,
   AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE,
   TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,

config/test.js

@@ -4,5 +4,6 @@
 
 module.exports = {
   AUTH0_URL: 'http://test', // Since we use Nock, just dummy value stored
-  AWS_S3_BUCKET: 'dummy-bucket'
+  SONARQUBE_HOST: 'http://localhost',
+  SONARQUBE_TOKEN: 'access-token'
 }

docker/sample.api.env

@@ -3,5 +3,4 @@ AUTH0_AUDIENCE=<Auth0 Audience>
 AUTH0_CLIENT_ID=<Auth0 Client ID>
 AUTH0_CLIENT_SECRET=<Auth0 Client Secret>
 SUBMISSION_API_URL=<Submission API Endpoint>
-AWS_S3_BUCKET=<S3 bucket>
-SONARQUBE_HOST=<SonarQube Host>
+SONARQUBE_HOST=<SonarQube Host>

package-lock.json

@@ -27,7 +27,6 @@
     "standard": "^12.0.1"
   },
   "dependencies": {
-    "aws-sdk": "^2.503.0",
     "bluebird": "^3.5.5",
     "body-parser": "^1.19.0",
     "config": "^2.0.2",

package.json

@@ -10,7 +10,6 @@ const cors = require('cors')
 const bodyParser = require('body-parser')
 const httpStatus = require('http-status-codes')
 const _ = require('lodash')
-const sonarService = require('./services/SonarService')
 const helper = require('./common/helper')
 const logger = require('./common/logger')
 const routes = require('./routes')
@@ -32,10 +31,6 @@ app.use(cors())
 app.use(bodyParser.json())
 app.use(bodyParser.urlencoded({ extended: true }))
 
-// configure sonarqube service
-sonarService.configure(config.get('SONARQUBE_HOST'),
-  config.has('SONARQUBE_TOKEN') ? config.get('SONARQUBE_TOKEN') : undefined)
-
 const apiRouter = express.Router()
 
 app.use(healthcheck.middleware([check]))

src/app.js

@@ -5,16 +5,10 @@
 const config = require('config')
 const _ = require('lodash')
 const request = require('superagent')
-const AWS = require('aws-sdk')
 
 const m2mAuth = require('tc-core-library-js').auth.m2m
 const m2m = m2mAuth(_.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'AUTH0_PROXY_SERVER_URL']))
 
-AWS.config.region = config.REGION
-const s3 = new AWS.S3()
-const AWS_S3_BUCKET = config.get('AWS_S3_BUCKET')
-const AWS_S3_PREFIX = config.get('AWS_S3_PREFIX')
-
 /**
  * Wrap async function to standard express function
  * @param {Function} fn the async function
@@ -116,29 +110,9 @@ const reqToV5APIWithFile = async (path, formData, fileFieldName) => {
     .attach(fileFieldName, formData[fileFieldName].data, formData[fileFieldName].name)
 }
 
-/**
- * Upload specified content on s3.
- * @param {String} key s3 file key
- * @param {Object} content Content to upload
- * @param {String} contentType Content type
- *
- * @returns {Promise}
- */
-const s3upload = async (key, content, contentType) => {
-  return new Promise((resolve, reject) => {
-    s3.upload({
-      Bucket: AWS_S3_BUCKET,
-      Key: `${AWS_S3_PREFIX}${key}`,
-      ContentType: contentType,
-      Body: content
-    }, err => err ? reject(err) : resolve())
-  })
-}
-
 module.exports = {
   wrapExpress,
   autoWrapExpress,
   reqToSubmissionAPI,
-  reqToV5APIWithFile,
-  s3upload
+  reqToV5APIWithFile
 }

src/common/helper.js

@@ -5,6 +5,7 @@ const request = require('superagent')
 const _ = require('lodash')
 const joi = require('joi')
 const logger = require('../common/logger')
+const config = require('config')
 
 // List of issue types that metrics and details must be collected for
 const ISSUE_TYPES = ['code_smells', 'bugs', 'security_hotspots', 'vulnerabilities']
@@ -34,8 +35,8 @@ const ISSUES_SCHEMA = joi.object().keys({
 }).unknown(true).required()
 
 // Configuration
-let SONARQUBE_HOST
-let SONARQUBE_TOKEN
+const SONARQUBE_HOST = config.get('SONARQUBE_HOST')
+const SONARQUBE_TOKEN = config.has('SONARQUBE_TOKEN') ? config.get('SONARQUBE_TOKEN') : undefined
 
 /**
  * Send GET request to SonarQube's endpoint
@@ -185,24 +186,8 @@ getScanResults.schema = joi.object().keys({
   analysedAt: joi.string().required()
 }).required()
 
-/**
- * Configure SonarQube service
- * @param {String} sqHost SonarQube host
- * @param {String} sqToken SonarQube token
- */
-function configure (sqHost, sqToken) {
-  SONARQUBE_HOST = sqHost
-  SONARQUBE_TOKEN = sqToken
-}
-
-configure.schema = joi.object().keys({
-  sqHost: joi.string().required(),
-  sqToken: joi.string()
-}).required()
-
 module.exports = {
-  getScanResults,
-  configure
+  getScanResults
 }
 
 logger.buildService(module.exports)

src/services/SonarService.js

@@ -7,7 +7,6 @@ const config = require('config')
 const logger = require('../common/logger')
 const helper = require('../common/helper')
 
-const RESULT_FILE_NAME = 'SonarQubeResults'
 const TYPE_ID = 'c56a4180-65aa-42ec-a945-5fd21dec0501'
 
 /**
@@ -33,26 +32,31 @@ updateSubmissionStatus.schema = joi.object().keys({
 })
 /**
  * Upload submission artifacts
+ * @param {String} submissionId The submission id to upload the artifact to
+ * @param {String} filename Artifact filename
+ * @param {String} typeId Associated artifact type id
  * @param {Object} body Artifacts content
- *
  * @returns {Promise}
  */
-async function uploadArtifacts (body) {
+async function uploadArtifacts (submissionId, filename, typeId, body) {
   const zip = new JSZip()
-  zip.file(`${RESULT_FILE_NAME}.json`, JSON.stringify(body, null, 2))
+  zip.file(`${filename}.json`, JSON.stringify(body, null, 2))
   const content = await zip.generateAsync({ type: 'nodebuffer' })
   const artifactPayload = {
     artifact: {
-      name: `${RESULT_FILE_NAME}.zip`,
+      name: `${filename}.zip`,
       data: content
     },
-    typeId: TYPE_ID
+    typeId: typeId
   }
 
-  return helper.reqToV5APIWithFile(`${config.SUBMISSION_API_URL}/submissions/${body.project.key}/artifacts`, artifactPayload, 'artifact')
+  return helper.reqToV5APIWithFile(`${config.SUBMISSION_API_URL}/submissions/${submissionId}/artifacts`, artifactPayload, 'artifact')
 }
 
 uploadArtifacts.schema = joi.object().keys({
+  submissionId: joi.string().required(),
+  filename: joi.string().required(),
+  typeId: joi.string().required(),
   body: joi.object().required()
 })
 

src/services/SubmissionService.js

@@ -5,7 +5,6 @@
 const joi = require('joi')
 const logger = require('../common/logger')
 const uuid = require('uuid/v4')
-const { s3upload } = require('../common/helper')
 
 const { updateSubmissionStatus, uploadArtifacts } = require('./SubmissionService')
 const { getScanResults } = require('./SonarService')
@@ -27,12 +26,12 @@ async function processScanResults (body) {
         scoreCardId: randomId
       })
 
-      await uploadArtifacts(body)
+      await uploadArtifacts(body.project.key, 'SonarQubeSummary', 'c56a4180-65aa-42ec-a945-5fd21dec0501', body) // temporary type id
     })(),
 
     (async () => {
       const scanResults = await getScanResults(body.project.key, body.analysedAt)
-      await s3upload(`${body.project.key}.json`, JSON.stringify(scanResults), 'application/json')
+      await uploadArtifacts(body.project.key, 'SonarQubeDetails', '50b917df-5b81-4081-8e20-7fc8a6aabe54', scanResults) // temporary type id
     })()
   ])
 }