#638 - Provide new route to get all uploads and support capturing failed records data in uploads

Author: callmekatootie

docs/swagger.yaml

@@ -90,6 +90,65 @@ paths:
       - api_key: []
       x-swagger-router-controller: "Templates"
   /uploads:
+    get:
+      summary: "Get all uploads"
+      description: |
+        "Return all uploads. You can specify the date from when you would like the records.""
+      tags:
+        - "uploads"
+      operationId: "getUploads"
+      parameters:
+      - name: "from"
+        in: "query"
+        description: "The date from when you would like the uploads (inclusive)"
+        required: false
+        type: "string"
+        default: "today's value"
+      responses:
+        "200":
+          description: "Success"
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/Upload'
+        "400":
+          $ref: "#/definitions/BadRequest"
+        "401":
+          $ref: "#/definitions/Unauthorized"
+        "403":
+          $ref: "#/definitions/Forbidden"
+        "404":
+          $ref: "#/definitions/NotFound"
+        "500":
+          $ref: "#/definitions/ServerError"
+
+    head:
+      summary: "Get only status code for fetching all uploads"
+      description: "Equivalent to GET /uploads, but returns only status codes"
+      tags:
+        - "uploads"
+      operationId: "headUploads"
+      parameters:
+      - name: "from"
+        in: "query"
+        description: "The date from when you would like the uploads (inclusive)"
+        required: false
+        type: "string"
+        default: "today's value"
+      responses:
+        "200":
+          description: "Success"
+        "400":
+          $ref: "#/definitions/BadRequest"
+        "401":
+          $ref: "#/definitions/Unauthorized"
+        "403":
+          $ref: "#/definitions/Forbidden"
+        "404":
+          $ref: "#/definitions/NotFound"
+        "500":
+          $ref: "#/definitions/ServerError"
+
     post:
       tags:
       - "uploads"
@@ -176,7 +235,7 @@ paths:
         description: "The request body"
         required: true
         schema:
-          $ref: "#/definitions/body"
+          $ref: "#/definitions/PartiallyUpdateUpload"
       responses:
         "200":
           description: "Success"
@@ -258,6 +317,10 @@ definitions:
           type: "string"
           format: "uri"
           description: "S3 signed url"
+        failedRecordsUrl:
+          type: "string"
+          format: "uri"
+          description: "S3 signed url"
         status:
           type: "string"
           enum:
@@ -368,7 +431,7 @@ definitions:
         example: "Creating a resource with a name already exists."
         description: "The conflict error message."
     description: "The conflict error entity."
-  body:
+  PartiallyUpdateUpload:
     type: "object"
     required:
     - "status"
@@ -377,3 +440,5 @@ definitions:
         type: "string"
       info:
         type: "string"
+      failedRecordsObjectKey:
+        type: "string"

src/common/helper.js

@@ -129,6 +129,23 @@ async function getById (modelName, id) {
   })
 }
 
+/**
+ * Get all Data by model
+ * @param {String} modelName The dynamoose model name
+ * @returns found record(s)
+ */
+async function getAll (modelName, query) {
+  return new Promise((resolve, reject) => {
+    models[modelName].scan(query).exec((err, result) => {
+      if (err) {
+        reject(err)
+      } else {
+        resolve(result)
+      }
+    })
+  })
+}
+
 /**
  * Create item in database
  * @param {Object} modelName The dynamoose model name
@@ -201,11 +218,16 @@ async function uploadToS3 (bucket, file, fileName) {
  * @returns {string} signed url
  */
 function generateS3Url (objectKey) {
-  const url = s3.getSignedUrl('getObject', {
-    Bucket: config.UPLOAD_S3_BUCKET,
-    Key: objectKey,
-    Expires: parseInt(config.S3_OBJECT_URL_EXPIRY_TIME)
-  })
+  let url
+
+  if (objectKey) {
+    url = s3.getSignedUrl('getObject', {
+      Bucket: config.UPLOAD_S3_BUCKET,
+      Key: objectKey,
+      Expires: parseInt(config.S3_OBJECT_URL_EXPIRY_TIME)
+    })
+  }
+
   return url
 }
 
@@ -291,6 +313,7 @@ module.exports = {
   createTable,
   deleteTable,
   getById,
+  getAll,
   create,
   update,
   uploadToS3,

src/controllers/UploadController.js

@@ -13,6 +13,16 @@ async function uploadEntity (req, res) {
   res.send(result)
 }
 
+/**
+ * Get upload
+ * @param {Object} req the request
+ * @param {Object} res the response
+ */
+async function getEntities (req, res) {
+  const result = await service.getEntities(req.query)
+  res.send(result)
+}
+
 /**
  * Get upload
  * @param {Object} req the request
@@ -34,6 +44,7 @@ async function partiallyUpdate (req, res) {
 }
 
 module.exports = {
+  getEntities,
   getEntity,
   uploadEntity,
   partiallyUpdate

src/models/Upload.js

@@ -43,6 +43,10 @@ const schema = new Schema({
   updatedBy: {
     type: String,
     required: true
+  },
+  failedRecordsObjectKey: {
+    type: String,
+    required: false
   }
 },
 {

src/routes.js

@@ -29,6 +29,13 @@ module.exports = {
       auth: 'jwt',
       access: constants.AllAuthenticatedUsers,
       scopes: [constants.Scopes.CreateUpload, constants.Scopes.AllUpload]
+    },
+    get: {
+      controller: 'UploadController',
+      method: 'getEntities',
+      auth: 'jwt',
+      access: constants.AllAuthenticatedUsers,
+      scopes: [constants.Scopes.GetUpload, constants.Scopes.AllUpload]
     }
   },
   '/uploads/:id': {

src/services/UploadService.js

@@ -32,6 +32,39 @@ async function ensureFileTypeIsValid (upload) {
   }
 }
 
+/**
+ * Returns all uploads (filtered by created date)
+ * @param {Object} query the filter query
+ * @returns {Object} the upload of given id
+ */
+async function getEntities (query) {
+  if (!query.from) {
+    query.from = (new Date((new Date()).setDate((new Date()).getDate() - 1)).toISOString()) // 24 hours ago
+  }
+  const uploads = await helper.getAll(config.AMAZON.DYNAMODB_UPLOAD_TABLE, { created: { ge: query.from } })
+  const res = _.map(uploads, (upload) => {
+    upload = _.extend(
+      _.omit(
+        _.pickBy(upload, _.isString),
+        ['objectKey', 'failedRecordsObjectKey']
+      ),
+      {
+        url: helper.generateS3Url(upload.objectKey),
+        failedRecordsUrl: helper.generateS3Url(upload.failedRecordsObjectKey)
+      }
+    )
+
+    return upload
+  })
+  return res
+}
+
+getEntity.schema = {
+  query: Joi.object().keys({
+    from: Joi.date().iso()
+  })
+}
+
 /**
  * Get upload entity by id.
  * @param {String} id the upload id
@@ -40,7 +73,15 @@ async function ensureFileTypeIsValid (upload) {
 async function getEntity (id) {
   // get from DB
   const upload = await helper.getById(config.AMAZON.DYNAMODB_UPLOAD_TABLE, id)
-  const res = _.extend(_.omit(_.pickBy(upload, _.isString), 'objectKey'), { url: helper.generateS3Url(upload.objectKey) })
+  const res = _.extend(
+    _.omit(
+      _.pickBy(upload, _.isString),
+      ['objectKey', 'failedRecordsObjectKey']),
+    {
+      url: helper.generateS3Url(upload.objectKey),
+      failedRecordsUrl: helper.generateS3Url(upload.failedRecordsObjectKey)
+    }
+  )
   return res
 }
 
@@ -109,7 +150,15 @@ async function partiallyUpdate (authUser, id, data) {
   }
   // then update data in DB
   await helper.update(upload, data)
-  const res = _.extend(_.omit(_.pickBy(upload, _.isString), 'objectKey'), { url: helper.generateS3Url(upload.objectKey) })
+  const res = _.extend(
+    _.omit(
+      _.pickBy(upload, _.isString),
+      ['objectKey', 'failedRecordsObjectKey']),
+    {
+      url: helper.generateS3Url(upload.objectKey),
+      failedRecordsUrl: helper.generateS3Url(upload.failedRecordsObjectKey)
+    }
+  )
   return res
 }
 
@@ -118,11 +167,13 @@ partiallyUpdate.schema = {
   authUser: Joi.object().required(),
   data: Joi.object().keys({
     status: Joi.string().valid('pending', 'completed', 'failed').required(),
-    info: Joi.string()
+    info: Joi.string(),
+    failedRecordsObjectKey: Joi.string()
   }).required()
 }
 
 module.exports = {
+  getEntities,
   getEntity,
   create,
   partiallyUpdate