topcoder-platform · ThomasKranitsas · Sep 14, 2021
diff --git a/.gitignore b/.gitignore
@@ -17,6 +17,9 @@ lib-cov
 # Coverage directory used by tools like istanbul
 coverage
 
+# Newman tests
+newman
+
 # nyc test coverage
 .nyc_output
 
@@ -44,6 +47,7 @@ typings/
 
 # Optional eslint cache
 .eslintcache
+.eslintrc.y*ml
 
 # Optional REPL history
 .node_repl_history

diff --git a/README.md b/README.md
@@ -100,6 +100,9 @@ It starts Elasticsearch, DynamoDB and S3 compatible server.
 5. Seed/Insert data to ES and DB: `npm run seed-data`
 6. View DB table data: `npm run view-db-data <ModelName>`, ModelName can be `Member`, `MemberTrait`, `MemberDistributionStats`, `MemberHistoryStats`, `MemberStats`, `MemberSkill` or `MemberFinancial`
 7. View ES data: `npm run view-es-data <IndexName>`, IndexName can be `member`, `member_trait`
+8. Start mock api: `npm run mock-api`
+9. Start newman tests: `npm run test:newman`
+10. Clear data generated during newman tests: `npm run test:newman:clear`
 
 ## Local Deployment
 
@@ -131,10 +134,10 @@ docker-compose up
 
 5. When you are running the application for the first time, It will take some time initially to download the image and install the dependencies
 
-
 ## Running tests
 
 ### Prepare
+
 - Various config parameters should be properly set.
 - Start Local services.
 - Create DynamoDB tables.
@@ -144,6 +147,7 @@ docker-compose up
 
 
 ### Running unit tests
+
 To run unit tests alone
 
 ```bash
@@ -155,21 +159,78 @@ To run unit tests with coverage report
 ```bash
 npm run test:cov
 ```
-
-### Running integration tests
-To run integration tests alone
-
-```bash
-npm run e2e
-```
-
-To run integration tests with coverage report
-
-```bash
-npm run e2e:cov
-```
+### Running E2E Automated Postman Tests
+
+1. In the `member-api` root directory create `.env` file with the next environment variables.
+
+    ```bash
+    # Auth0 config
+    AUTH_SECRET=
+    AUTH0_URL=
+    AUTH0_AUDIENCE=
+    AUTH0_CLIENT_ID=
+    AUTH0_CLIENT_SECRET=
+    AUTH_V2_URL=
+    AUTH_V2_CLIENT_ID=
+    AUTH_V3_URL=
+    # Following configs are used to generate tokens.
+    ADMIN_CREDENTIALS_USERNAME=
+    ADMIN_CREDENTIALS_PASSWORD=
+    MANAGER_CREDENTIALS_USERNAME=
+    MANAGER_CREDENTIALS_PASSWORD=
+    COPILOT_CREDENTIALS_USERNAME=
+    COPILOT_CREDENTIALS_PASSWORD=
+    USER_CREDENTIALS_USERNAME=
+    USER_CREDENTIALS_PASSWORD=
+    # Locally deployed services
+    IS_LOCAL_DB=true
+    IS_LOCAL_S3=true
+    PHOTO_S3_BUCKET=photos
+    AWS_ACCESS_KEY_ID=FAKE_ACCESS_KEY
+    AWS_SECRET_ACCESS_KEY=FAKE_SECRET_ACCESS_KEY
+    MOCK_API_PORT=4000
+    AUTOMATED_TESTING_NAME_PREFIX=POSTMANE2E-
+    S3_ENDPOINT=http://localhost:9000
+    API_BASE_URL=http://localhost:3000
+    BUSAPI_URL=http://localhost:4000/v5
+    TAGS_BASE_URL=http://localhost:4000
+    TAGS_API_VERSION=/v3
+    TAGS_FILTER=/tags
+    ```
+1. Install npm dependencies
+   ```bash
+   npm install
+   ```
+1. Start Elasticsearch, DynamoDB and S3
+   ```bash
+   cd local
+   docker-compose up -d
+   ```
+1. Initialize Database and Elasticsearch
+   ```bash
+   npm run create-tables
+   npm run init-db
+   npm run init-es
+   npm run seed-data
+   ```
+1. login minio at http://localhost:9000 and create bucket with name `photos`
+1. Start mock api and member api in TEST mode
+   ```bash
+   npm run mock-api
+   npm run start:test
+   ```
+1. Run tests
+   ```bash
+   npm run test:newman
+   ```
+1. To run the tests again, clear data and seed again
+   ```bash
+   npm run test:newman:clear
+   npm run seed-data
+   ```
 
 ## Verification
+
 Refer to the verification document `Verification.md`
 
 ## Notes
@@ -189,4 +250,3 @@ Refer to the verification document `Verification.md`
 - the tests use mock S3 service to upload member photo, so you may use the provided mock AWS credential for tests,
   but Postman tests require using real AWS S3, you need to follow README.md to create S3 bucket and provide your own AWS credential
   so that the upload photo API works
-
diff --git a/Verification.md b/Verification.md
@@ -6,6 +6,12 @@
 - import Postman collection and environment in the docs folder to Postman
 - run the Postman tests
 
+## Clear Test Data Service Verification
+- After running newman tests, clear data with `npm run test:newman:clear`
+- To check if data cleared properly, run following commands
+- `npm run view-db-data <ModelName>`
+- `npm run view-es-data <IndexName>`
+
 ## DynamoDB Verification
 Run command `npm run view-db-data <ModelName>` to view table data, ModelName can be `Member`, `MemberTrait`, `MemberDistributionStats`, `MemberHistoryStats`, `MemberStats`, `MemberSkill` or `MemberFinancial`
 

diff --git a/app-bootstrap.js b/app-bootstrap.js
@@ -7,4 +7,4 @@ const Joi = require('joi')
 Joi.page = () => Joi.number().integer().min(1).default(1)
 Joi.perPage = () => Joi.number().integer().min(1).max(100).default(50)
 Joi.size = () => Joi.number().integer().min(1).max(1000).default(500)
-Joi.sort = () => Joi.string().default("asc")
+Joi.sort = () => Joi.string().default('asc')
diff --git a/app-routes.js b/app-routes.js
@@ -41,7 +41,7 @@ module.exports = (app) => {
         // add Authenticator/Authorization check if route has auth
         actions.push((req, res, next) => {
           // When authorization token is not provided and allow no token is enabled then bypass
-          if(!_.get(req, 'headers.authorization') && def.allowNoToken) {
+          if (!_.get(req, 'headers.authorization') && def.allowNoToken) {
             next()
           } else {
             authenticator(_.pick(config, ['AUTH_SECRET', 'VALID_ISSUERS']))(req, res, next)
@@ -50,7 +50,7 @@ module.exports = (app) => {
 
         actions.push((req, res, next) => {
           // When authorization token is not provided and allow no token is enabled then bypass
-          if(!_.get(req, 'headers.authorization') && def.allowNoToken) {
+          if (!_.get(req, 'headers.authorization') && def.allowNoToken) {
             next()
           } else {
             if (req.authUser.isMachine) {

diff --git a/config/default.js b/config/default.js
@@ -1,7 +1,7 @@
 /**
  * The configuration file.
  */
-
+require('dotenv').config()
 module.exports = {
   LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
   PORT: process.env.PORT || 3000,
@@ -25,15 +25,16 @@ module.exports = {
   TAGS: {
     TAGS_BASE_URL: process.env.TAGS_BASE_URL || 'https://api.topcoder-dev.com',
     TAGS_API_VERSION: process.env.TAGS_API_VERSION || '/v3',
-    TAGS_FILTER: process.env.TAGS_FILTER || '/tags/?filter=domain%3DSKILLS%26status%3DAPPROVED&limit=1000',
+    TAGS_FILTER: process.env.TAGS_FILTER || '/tags/?filter=domain%3DSKILLS%26status%3DAPPROVED&limit=1000'
   },
-  
+
   // aws config params
   AMAZON: {
     AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID,
     AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY,
     AWS_REGION: process.env.AWS_REGION || 'us-east-1',
     IS_LOCAL_DB: process.env.IS_LOCAL_DB ? process.env.IS_LOCAL_DB === 'true' : false,
+    IS_LOCAL_S3: process.env.IS_LOCAL_S3 ? process.env.IS_LOCAL_S3 === 'true' : false,
     DYNAMODB_URL: process.env.DYNAMODB_URL || 'http://localhost:7777',
     PHOTO_S3_BUCKET: process.env.PHOTO_S3_BUCKET || 'topcoder-dev-media/member/profile',
     S3_API_VERSION: process.env.S3_API_VERSION || '2006-03-01'
@@ -48,7 +49,7 @@ module.exports = {
     MEMBER_PROFILE_ES_INDEX: process.env.MEMBER_PROFILE_ES_INDEX || 'members-2020-01',
     // member type, ES 6.x accepts only 1 Type per index and it's mandatory to define it
     MEMBER_PROFILE_ES_TYPE: process.env.MEMBER_PROFILE_ES_TYPE || 'profiles',
-    MEMBER_TRAIT_ES_INDEX: process.env.MEMBER_TRAIT_ES_INDEX || 'members-2020-01',
+    MEMBER_TRAIT_ES_INDEX: process.env.MEMBER_TRAIT_ES_INDEX || 'membertraits-2020-01',
     MEMBER_TRAIT_ES_TYPE: process.env.MEMBER_TRAIT_ES_TYPE || 'profiletraits',
     MEMBER_STATS_ES_INDEX: process.env.MEMBER_STATS_ES_INDEX || 'memberstats-2020-01',
     MEMBER_STATS_ES_TYPE: process.env.MEMBER_STATS_ES_TYPE || 'stats',
@@ -62,12 +63,12 @@ module.exports = {
   // file upload max size in bytes
   FILE_UPLOAD_SIZE_LIMIT: process.env.FILE_UPLOAD_SIZE_LIMIT
     ? Number(process.env.FILE_UPLOAD_SIZE_LIMIT) : 10 * 1024 * 1024, // 10M
-  
+
   // photo URL template, its <key> will be replaced with S3 object key,
   // the URL is specific to AWS region and bucket, you may go to AWS console S3 service to
   // see bucket object URL to get the URL structure
   PHOTO_URL_TEMPLATE: process.env.PHOTO_URL_TEMPLATE || 'https://topcoder-dev-media.s3.us-east-1.amazonaws.com/member/profile/<key>',
-  
+
   // verify token expiration in minutes
   VERIFY_TOKEN_EXPIRATION: process.env.VERIFY_TOKEN_EXPIRATION || 60,
 
@@ -84,33 +85,33 @@ module.exports = {
       READ: process.env.SCOPE_MEMBERS_READ || 'read:user_profiles',
       UPDATE: process.env.SCOPE_MEMBERS_UPDATE || 'update:user_profiles',
       DELETE: process.env.SCOPE_MEMBERS_DELETE || 'delete:user_profiles',
-      ALL: process.env.SCOPE_MEMBERS_ALL || 'all:user_profiles',
+      ALL: process.env.SCOPE_MEMBERS_ALL || 'all:user_profiles'
     }
   },
-  
+
   // Member identifiable info fields, only admin, M2M, or member himself can get these fields
-  MEMBER_SECURE_FIELDS: process.env.MEMBER_SECURE_FIELDS 
-    ? process.env.MEMBER_SECURE_FIELDS.split(',') 
+  MEMBER_SECURE_FIELDS: process.env.MEMBER_SECURE_FIELDS
+    ? process.env.MEMBER_SECURE_FIELDS.split(',')
     : ['firstName', 'lastName', 'email', 'addresses', 'createdBy', 'updatedBy'],
 
   // Member traits identifiable info fields, only admin, M2M, or member himself can fetch these fields
-  MEMBER_TRAIT_SECURE_FIELDS: process.env.MEMBER_TRAIT_SECURE_FIELDS 
-    ? process.env.MEMBER_TRAIT_SECURE_FIELDS.split(',') 
+  MEMBER_TRAIT_SECURE_FIELDS: process.env.MEMBER_TRAIT_SECURE_FIELDS
+    ? process.env.MEMBER_TRAIT_SECURE_FIELDS.split(',')
     : ['createdBy', 'updatedBy'],
 
   // Misc identifiable info fields, only admin, M2M, or member himself can fetch these fields
-  MISC_SECURE_FIELDS: process.env.MISC_SECURE_FIELDS 
-    ? process.env.MISC_SECURE_FIELDS.split(',') 
+  MISC_SECURE_FIELDS: process.env.MISC_SECURE_FIELDS
+    ? process.env.MISC_SECURE_FIELDS.split(',')
     : ['createdBy', 'updatedBy'],
-  
+
   // Member Search identifiable info fields, only admin, M2M, or member himself can fetch these fields
-  SEARCH_SECURE_FIELDS: process.env.SEARCH_SECURE_FIELDS 
-  ? process.env.SEARCH_SECURE_FIELDS.split(',') 
-  : ['firstName', 'lastName', 'email', 'addresses', 'createdBy', 'updatedBy'],
-  
+  SEARCH_SECURE_FIELDS: process.env.SEARCH_SECURE_FIELDS
+    ? process.env.SEARCH_SECURE_FIELDS.split(',')
+    : ['firstName', 'lastName', 'email', 'addresses', 'createdBy', 'updatedBy'],
+
   // Member Statistics identifiable info fields, only admin, M2M, or member himself can fetch these fields
   STATISTICS_SECURE_FIELDS: process.env.STATISTICS_SECURE_FIELDS
     ? process.env.STATISTICS_SECURE_FIELDS.split(',')
     : ['createdBy', 'updatedBy']
-  
+
 }
diff --git a/config/test.js b/config/test.js
@@ -1,14 +1,33 @@
 /**
  * Configuration file to be used while running tests
  */
-
+require('dotenv').config()
 module.exports = {
-  ADMIN_TOKEN: process.env.ADMIN_TOKEN || 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJyb2xlcyI6WyJUb3Bjb2RlciBVc2VyIiwiQ29ubmVjdCBTdXBwb3J0IiwiYWRtaW5pc3RyYXRvciIsInRlc3RSb2xlIiwiYWFhIiwidG9ueV90ZXN0XzEiLCJDb25uZWN0IE1hbmFnZXIiLCJDb25uZWN0IEFkbWluIiwiY29waWxvdCIsIkNvbm5lY3QgQ29waWxvdCBNYW5hZ2VyIl0sImlzcyI6Imh0dHBzOi8vYXBpLnRvcGNvZGVyLWRldi5jb20iLCJoYW5kbGUiOiJUb255SiIsImV4cCI6MTY5Mjc5NTIxMSwidXNlcklkIjoiODU0Nzg5OSIsImlhdCI6MTU0OTc5MTYxMSwiZW1haWwiOiJ0amVmdHMrZml4QHRvcGNvZGVyLmNvbSIsImp0aSI6ImY5NGQxZTI2LTNkMGUtNDZjYS04MTE1LTg3NTQ1NDRhMDhmMSJ9.sAku5sLBpfTkq4OANJA-eiZCiOxx4u6U6OgpTlk_OU4',
-  USER_TOKEN: process.env.USER_TOKEN || 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJyb2xlcyI6WyJUb3Bjb2RlciBVc2VyIl0sImlzcyI6Imh0dHBzOi8vYXBpLnRvcGNvZGVyLWRldi5jb20iLCJoYW5kbGUiOiJkZW5pcyIsImV4cCI6MTY4MjgwMDE2OSwidXNlcklkIjoiMjUxMjgwIiwiaWF0IjoxNTQ5Nzk5NTY5LCJlbWFpbCI6ImVtYWlsQGRvbWFpbi5jb20ueiIsImp0aSI6IjljNDUxMWM1LWMxNjUtNGExYi04OTllLWI2NWFkMGUwMmI1NSJ9.BCF6xW3aQfHDDFbgGvvOKzvwEXVLWGf-TgF5JrtM9Tg',
-  EXPIRED_TOKEN: process.env.EXPIRED_TOKEN || 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJyb2xlcyI6WyJjb3BpbG90IiwiQ29ubmVjdCBTdXBwb3J0Il0sImlzcyI6Imh0dHBzOi8vYXBpLnRvcGNvZGVyLWRldi5jb20iLCJoYW5kbGUiOiJHaG9zdGFyIiwiZXhwIjoxNTQ5ODAwMDc3LCJ1c2VySWQiOiIxNTE3NDMiLCJpYXQiOjE1NDk3OTk0NzcsImVtYWlsIjoiZW1haWxAZG9tYWluLmNvbS56IiwianRpIjoiMTJjMWMxMGItOTNlZi00NTMxLTgzMDUtYmE2NjVmYzRlMWI0In0.2n8k9pb16sE7LOLF_7mjAvEVKgggzS-wS3_8n2-R4RU',
-  INVALID_TOKEN: process.env.INVALID_TOKEN || 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJyb2xlcyI6WyJBZG1pbmlzdHJhdG9yIl0sImlzcyI6Imh0dHBzOi8vYXBpLnRvcGNvZGVyLmNvbSIsImhhbmRsZSI6IlRvbnlKIiwiZXhwIjo1NTUzMDE5OTI1OSwidXNlcklkIjoiNDA0MzMyODgiLCJpYXQiOjE1MzAxOTg2NTksImVtYWlsIjoiYWRtaW5AdG9wY29kZXIuY29tIiwianRpIjoiYzNhYzYwOGEtNTZiZS00NWQwLThmNmEtMzFmZTk0Yjk1NjFjIn0.ePREgnJrBixP4URf1dd8FHISN2_6eRM5gjCReS0ZMK4',
-  M2M_FULL_ACCESS_TOKEN: process.env.M2M_FULL_ACCESS_TOKEN || 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJodHRwczovL3RvcGNvZGVyLWRldi5hdXRoMC5jb20vIiwic3ViIjoiZW5qdzE4MTBlRHozWFR3U08yUm4yWTljUVRyc3BuM0JAY2xpZW50cyIsImF1ZCI6Imh0dHBzOi8vbTJtLnRvcGNvZGVyLWRldi5jb20vIiwiaWF0IjoxNTUwOTA2Mzg4LCJleHAiOjE2ODA5OTI3ODgsImF6cCI6ImVuancxODEwZUR6M1hUd1NPMlJuMlk5Y1FUcnNwbjNCIiwic2NvcGUiOiJhbGw6bWVtYmVycyIsImd0eSI6ImNsaWVudC1jcmVkZW50aWFscyJ9.Eo_cyyPBQfpWp_8-NSFuJI5MvkEV3UJZ3ONLcFZedoA',
-  M2M_READ_ACCESS_TOKEN: process.env.M2M_READ_ACCESS_TOKEN || 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJodHRwczovL3RvcGNvZGVyLWRldi5hdXRoMC5jb20vIiwic3ViIjoiZW5qdzE4MTBlRHozWFR3U08yUm4yWTljUVRyc3BuM0JAY2xpZW50cyIsImF1ZCI6Imh0dHBzOi8vbTJtLnRvcGNvZGVyLWRldi5jb20vIiwiaWF0IjoxNTUwOTA2Mzg4LCJleHAiOjE2ODA5OTI3ODgsImF6cCI6ImVuancxODEwZUR6M1hUd1NPMlJuMlk5Y1FUcnNwbjNCIiwic2NvcGUiOiJyZWFkOm1lbWJlcnMiLCJndHkiOiJjbGllbnQtY3JlZGVudGlhbHMifQ.F-dEZXJC7Ue7dHCi3XQdEvxhtr69hU4MwTcr-APHnK4',
-  M2M_UPDATE_ACCESS_TOKEN: process.env.M2M_UPDATE_ACCESS_TOKEN || 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJodHRwczovL3RvcGNvZGVyLWRldi5hdXRoMC5jb20vIiwic3ViIjoiZW5qdzE4MTBlRHozWFR3U08yUm4yWTljUVRyc3BuM0JAY2xpZW50cyIsImF1ZCI6Imh0dHBzOi8vbTJtLnRvcGNvZGVyLWRldi5jb20vIiwiaWF0IjoxNTUwOTA2Mzg4LCJleHAiOjE2ODA5OTI3ODgsImF6cCI6ImVuancxODEwZUR6M1hUd1NPMlJuMlk5Y1FUcnNwbjNCIiwic2NvcGUiOiJ1cGRhdGU6bWVtYmVycyIsImd0eSI6ImNsaWVudC1jcmVkZW50aWFscyJ9.wImcvhkF9QPOCSEfZ01U-YxYM8NZi1yqgRmw3eiNn1Q',
-  S3_ENDPOINT: process.env.S3_ENDPOINT || 'localhost:9000'
+  LOG_LEVEL: process.env.LOG_LEVEL || 'info',
+  AUTH_V2_URL: process.env.AUTH_V2_URL,
+  AUTH_V2_CLIENT_ID: process.env.AUTH_V2_CLIENT_ID,
+  AUTH_V3_URL: process.env.AUTH_V3_URL,
+  MOCK_API_PORT: process.env.MOCK_API_PORT || 4000,
+  ADMIN_CREDENTIALS_USERNAME: process.env.ADMIN_CREDENTIALS_USERNAME,
+  ADMIN_CREDENTIALS_PASSWORD: process.env.ADMIN_CREDENTIALS_PASSWORD,
+  MANAGER_CREDENTIALS_USERNAME: process.env.MANAGER_CREDENTIALS_USERNAME,
+  MANAGER_CREDENTIALS_PASSWORD: process.env.MANAGER_CREDENTIALS_PASSWORD,
+  COPILOT_CREDENTIALS_USERNAME: process.env.COPILOT_CREDENTIALS_USERNAME,
+  COPILOT_CREDENTIALS_PASSWORD: process.env.COPILOT_CREDENTIALS_PASSWORD,
+  USER_CREDENTIALS_USERNAME: process.env.USER_CREDENTIALS_USERNAME,
+  USER_CREDENTIALS_PASSWORD: process.env.USER_CREDENTIALS_PASSWORD,
+  WAIT_TIME: 6000,
+  AUTOMATED_TESTING_REPORTERS_FORMAT: process.env.AUTOMATED_TESTING_REPORTERS_FORMAT
+    ? process.env.AUTOMATED_TESTING_REPORTERS_FORMAT.split(',')
+    : ['cli', 'html'],
+  AUTOMATED_TESTING_NAME_PREFIX: process.env.AUTOMATED_TESTING_NAME_PREFIX || 'POSTMANE2E-',
+  S3_ENDPOINT: process.env.S3_ENDPOINT || 'http://localhost:9000',
+  API_BASE_URL: process.env.API_BASE_URL || 'http://localhost:3000',
+  BUSAPI_URL: process.env.BUSAPI_URL || 'http://localhost:4000/v5',
+  TAGS: {
+    TAGS_BASE_URL: process.env.TAGS_BASE_URL || 'http://localhost:4000',
+    TAGS_API_VERSION: process.env.TAGS_API_VERSION || '/v3',
+    TAGS_FILTER: process.env.TAGS_FILTER || '/tags'
+  }
+
 }
diff --git a/local/docker-compose.yml b/local/docker-compose.yml
@@ -15,7 +15,10 @@ services:
     image: minio/minio
     ports:
       - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - ~/minio/data:/data
     environment:
       MINIO_ACCESS_KEY: "FAKE_ACCESS_KEY"
       MINIO_SECRET_KEY: "FAKE_SECRET_ACCESS_KEY"
-    command: "server /data"
+    command: "server --console-address :9001 /data"