Merge pull request #1 from topcoder-platform/develop

First prod push

Author: vikasrohit

.circleci/config.yml

@@ -0,0 +1,78 @@
+version: 2
+defaults: &defaults
+    docker:
+      - image: circleci/python:2.7.18-stretch-browsers
+install_dependency: &install_dependency
+  name: Installation of build and deployment dependencies.
+  command: |
+    sudo apt install jq
+    sudo pip install awscli --upgrade
+    sudo pip install docker-compose
+install_deploysuite: &install_deploysuite
+  name: Installation of install_deploysuite.
+  command: |
+    git clone --branch master_hostfix_v1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
+    cp ./../buildscript/master_deploy.sh .
+    cp ./../buildscript/buildenv.sh .
+    cp ./../buildscript/awsconfiguration.sh .
+restore_cache_settings_for_build: &restore_cache_settings_for_build
+  key: docker-node-modules-{{ checksum "package-lock.json" }}
+
+save_cache_settings: &save_cache_settings
+  key: docker-node-modules-{{ checksum "package-lock.json" }}
+  paths:
+    - node_modules
+
+builddeploy_steps: &builddeploy_steps
+      - checkout
+      - setup_remote_docker
+      - run: *install_dependency
+      - run: *install_deploysuite
+      - restore_cache: *restore_cache_settings_for_build
+      - run: ./build.sh ${APPNAME}
+      - save_cache: *save_cache_settings
+      - deploy:
+          name: Running MasterScript.
+          command: |
+            ./awsconfiguration.sh $DEPLOY_ENV
+            source awsenvconf
+            ./buildenv.sh -e $DEPLOY_ENV -b ${LOGICAL_ENV}-${APPNAME}-deployvar
+            source buildenvvar
+            ./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${LOGICAL_ENV}-global-appvar,${LOGICAL_ENV}-${APPNAME}-appvar -i ${APPNAME}
+jobs:
+  # Build & Deploy against development backend
+  "build-dev":
+    <<: *defaults
+    environment:
+      DEPLOY_ENV: "DEV"
+      LOGICAL_ENV: "dev"
+      APPNAME: "skills-api"
+    steps: *builddeploy_steps
+
+  "build-prod":
+    <<: *defaults
+    environment:
+      DEPLOY_ENV: "PROD"
+      LOGICAL_ENV: "prod"
+      APPNAME: "skills-api"
+    steps: *builddeploy_steps
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      # Development builds are executed on "develop" branch only.
+      - "build-dev":
+          context : org-global
+          filters:
+            branches:
+              only:
+                - develop
+
+      # Production builds are exectuted only on tagged commits to the
+      # master branch.
+      - "build-prod":
+          context : org-global
+          filters:
+            branches:
+              only: master

README.md

@@ -1,2 +1,157 @@
-# skills-api
-V5 Skills API
+# Skills API
+
+* [Prerequisites](#prerequisites)
+* [Configuration](#configuration)
+* [Local deployment](#local-deployment)
+* [Migrations](#migrations)
+* [Local Deployment with Docker](#local-deployment-with-docker)
+* [NPM Commands](#npm-commands)
+* [JWT Authentication](#jwt-authentication)
+* [Documentation](#documentation)
+
+## Prerequisites
+
+- node 12.x+
+- npm 6.x+
+- docker
+- elasticsearch 7.7+
+- PostgreSQL
+
+## Configuration
+
+Configuration for the application is at `config/default.js` and `config/production.js`. The following parameters can be set in config files or in env variables:
+
+- LOG_LEVEL: the log level
+- PORT: the server port
+- AUTH_SECRET: TC Authentication secret
+- VALID_ISSUERS: valid issuers for TC authentication
+- PAGE_SIZE: the default pagination limit
+- MAX_PAGE_SIZE: the maximum pagination size
+- API_VERSION: the API version
+- DB_NAME: the database name
+- DB_USERNAME: the database username
+- DB_PASSWORD: the database password
+- DB_HOST: the database host
+- DB_PORT: the database port
+- ES_HOST: Elasticsearch host
+- ES_REFRESH: Should elastic search refresh. Default is 'true'. Values can be 'true', 'wait_for', 'false'
+- ELASTICCLOUD_ID: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this
+- ELASTICCLOUD_USERNAME: The elastic cloud username for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud
+- ELASTICCLOUD_PASSWORD: The elastic cloud password for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud
+- ES.DOCUMENTS: Elasticsearch index, type and id mapping for resources.
+- SKILL_INDEX: The Elastic search index for skill. Default is `skill`
+- SKILL_ENRICH_POLICYNAME: The enrich policy for skill. Default is `skill-policy`
+- TAXONOMY_INDEX: The Elastic search index for taxonomy. Default is `taxonomy`
+- TAXONOMY_PIPELINE_ID: The pipeline id for enrichment with taxonomy. Default is `taxonomy-pipeline`
+- TAXONOMY_ENRICH_POLICYNAME: The enrich policy for taxonomy. Default is `taxonomy-policy`
+- MAX_BATCH_SIZE: Restrict number of records in memory during bulk insert (Used by the db to es migration script)
+- MAX_BULK_SIZE: The Bulk Indexing Maximum Limits. Default is `100` (Used by the db to es migration script)
+
+
+## Local deployment
+
+Setup your Postgresql DB and Elasticsearch instance and ensure that they are up and running.
+
+- Follow *Configuration* section to update config values, like database, ES host etc ..
+- Goto *skills-api*, run `npm i`
+- Create database using `npm run create-db`.
+- Run the migrations - `npm run migrations up`. This will create the tables.
+- Then run `npm run insert-data` and insert mock data into the database.
+- Run `npm run migrate-db-to-es` to sync data with ES.
+- Startup server `npm run start`
+
+## Migrations
+
+Migrations are located under the `./scripts/db/` folder. Run `npm run migrations up` and `npm run migrations down` to execute the migrations or remove the earlier ones
+
+## Local Deployment with Docker
+
+- Navigate to the directory `docker-pgsql-es` folder. Rename `sample.env` to `.env` and change any values if required.
+- Run `docker-compose up -d` to have docker instances of pgsql and elasticsearch to use with the api
+
+- Create database using `npm run create-db`.
+- Run the migrations - `npm run migrations up`. This will create the tables.
+- Then run `npm run insert-data` and insert mock data into the database.
+- Run `npm run migrate-db-to-es` to sync data with ES.
+
+- Navigate to the directory `docker`
+
+- Rename the file `sample.env` to `.env`
+
+- Set the required DB configurations and ElasticSearch host in the file `.env`
+
+- Once that is done, run the following command
+
+    ```bash
+    docker-compose up
+    ```
+
+- When you are running the application for the first time, It will take some time initially to download the image and install the dependencies
+
+## NPM Commands
+
+| Command                    | Description |
+|--------------------|--|
+| `npm run start`  | Start app |
+| `npm run start:dev`  | Start app on any changes (useful during development). |
+| `npm run lint`     | Check for for lint errors. |
+| `npm run lint:fix` | Check for for lint errors and fix error automatically when possible. |
+| `npm run create-db`    | Create the database |
+| `npm run insert-data`    | Insert data into the database |
+| `npm run migrate-db-to-es`    | Migrate data into elastic search from database |
+| `npm run delete-data`  | Delete the data from the database |
+| `npm run migrations up`  | Run up migration |
+| `npm run migrations down`  | Run down migration |
+| `npm run generate:doc:permissions` | Generate [permissions.html](docs/permissions.html) |
+| `npm run generate:doc:permissions:dev` | Generate [permissions.html](docs/permissions.html) on any changes (useful during development). |
+
+## JWT Authentication
+Authentication is handled via Authorization (Bearer) token header field. Token is a JWT token.
+
+Here is a sample user token that is valid for a very long time for a user with administrator role.
+
+```
+<provide_in_forums>
+
+# here is the payload data decoded from the token
+{
+  "roles": [
+    "Topcoder User",
+    "administrator"
+  ],
+  "iss": "https://api.topcoder.com",
+  "handle": "tc-Admin",
+  "exp": 1685571460,
+  "userId": "23166768",
+  "iat": 1585570860,
+  "email": "[email protected]",
+  "jti": "0f1ef1d3-2b33-4900-bb43-48f2285f9630"
+}
+```
+
+and this is a sample M2M token with scopes `all:connect_project`, `all:projects` and `write:projects`.
+
+```
+<provided_in_forums>
+
+# here is the payload data decoded from the token
+{
+  "iss": "https://topcoder-dev.auth0.com/",
+  "sub": "enjw1810eDz3XTwSO2Rn2Y9cQTrspn3B@clients",
+  "aud": "https://m2m.topcoder-dev.com/",
+  "iat": 1550906388,
+  "exp": 2147483648,
+  "azp": "enjw1810eDz3XTwSO2Rn2Y9cQTrspn3B",
+  "scope": "all:connect_project all:projects write:projects",
+  "gty": "client-credentials"
+}
+```
+
+These tokens have been signed with the secret `CLIENT_SECRET`. This secret should match the `AUTH_SECRET` entry in `config/default.js`. You can modify the payload of these tokens to generate tokens with different roles or different scopes using https://jwt.io
+
+**Note** Please check with `src/constants.js` for all available user roles and M2M scopes.
+
+## Documentation
+
+- [permissions.html](docs/permissions.html) - the list of all permissions in Skills API.
+- [swagger.yaml](docs/swagger.yaml) - the Swagger API Definition.

app.js

@@ -0,0 +1,88 @@
+/**
+ * The application entry point
+ */
+
+require('./src/bootstrap')
+const config = require('config')
+const express = require('express')
+const cross = require('cors')
+const bodyParser = require('body-parser')
+const _ = require('lodash')
+const http = require('http')
+const swaggerUi = require('swagger-ui-express')
+const jsyaml = require('js-yaml')
+const fs = require('fs')
+const path = require('path')
+const logger = require('./src/common/logger')
+const errorMiddleware = require('./src/common/error.middleware')
+const routes = require('./src/route')
+const { permissions, jwtAuthenticator } = require('tc-core-library-js').middleware
+const app = express()
+const httpServer = http.Server(app)
+const models = require('./src/models')
+const initPermissions = require('./src/permissions')
+
+app.set('port', config.PORT)
+app.use(bodyParser.json())
+app.use(bodyParser.urlencoded({ extended: true }))
+app.use(cross())
+const apiRouter = express.Router({})
+
+// load all routes
+_.each(routes, (verbs, url) => {
+  _.each(verbs, (def, verb) => {
+    if (!def.method) {
+      throw new Error(`${verb.toUpperCase()} ${url} method is undefined`)
+    }
+    if (def.auth && def.auth !== 'jwt') {
+      throw new Error(`auth type "${def.auth}" is not supported`)
+    }
+
+    const actions = []
+    // Authentication
+    if (def.auth) {
+      actions.push((req, res, next) => {
+        jwtAuthenticator(_.pick(config, ['AUTH_SECRET', 'VALID_ISSUERS']))(req, res, next)
+      })
+    }
+    // Authorization
+    if (def.permission) {
+      actions.push(permissions(def.permission))
+    }
+    // main middleware
+    actions.push(async (req, res, next) => {
+      try {
+        await def.method(req, res, next)
+      } catch (e) {
+        next(e)
+      }
+    })
+
+    logger.info(`Endpoint discovered : ${verb.toLocaleUpperCase()} /${config.API_VERSION}${url}`)
+    apiRouter[verb](`/${config.API_VERSION}${url}`, actions)
+  })
+})
+app.use('/', apiRouter)
+const spec = fs.readFileSync(path.join(__dirname, 'docs/swagger.yaml'), 'utf8')
+const swaggerDoc = jsyaml.safeLoad(spec)
+
+app.use('/docs', swaggerUi.serve, swaggerUi.setup(swaggerDoc))
+
+app.use(errorMiddleware())
+app.use('*', (req, res) => {
+  const pathKey = req.baseUrl.substring(config.API_VERSION.length + 1)
+  const route = routes[pathKey]
+  if (route) {
+    res.status(405).json({ message: 'The requested method is not supported.' })
+  } else {
+    res.status(404).json({ message: 'The requested resource cannot found.' })
+  }
+});
+
+(async () => {
+  await models.init()
+  initPermissions() // initialize permission policies
+  httpServer.listen(app.get('port'), () => {
+    logger.info(`Express server listening on port ${app.get('port')}`)
+  })
+})()

build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -eo pipefail
+APP_NAME=$1
+UPDATE_CACHE=""
+echo "" > docker/api.env
+docker build -f docker/Dockerfile -t $APP_NAME:latest .
+docker create --name app $APP_NAME:latest
+
+if [ -d node_modules ]
+then
+  mv package-lock.json old-package-lock.json
+  docker cp app:/usr/src/app/package-lock.json package-lock.json
+  set +eo pipefail
+  UPDATE_CACHE=$(cmp package-lock.json old-package-lock.json)
+  set -eo pipefail
+else
+  UPDATE_CACHE=1
+fi
+
+if [ "$UPDATE_CACHE" == 1 ]
+then
+  docker cp app:/usr/src/app/node_modules .
+fi