From 115c341b72fd48d0976ef8f53920016b170637cd Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Fri, 13 Nov 2020 19:58:37 -0500 Subject: [PATCH 1/6] Add new Jenkinsfile and supporting shell scripts --- Jenkinsfile | 100 ++++++++++++++--------------------- jenkins/build-and-package.sh | 32 +++++++++++ jenkins/deploy-production.sh | 18 +++++++ jenkins/deploy-staging.sh | 18 +++++++ 4 files changed, 109 insertions(+), 59 deletions(-) create mode 100755 jenkins/build-and-package.sh create mode 100755 jenkins/deploy-production.sh create mode 100755 jenkins/deploy-staging.sh diff --git a/Jenkinsfile b/Jenkinsfile index 60eb44730..d54dbbe46 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,90 +1,72 @@ #!groovy -// import shared library: https://github.com/cmu-delphi/jenkins-shared-library +// Import shared lib. @Library('jenkins-shared-library') _ -pipeline { +/* + Declare variables. + - indicator_list should contain all the indicators to handle in the pipeline. + - Keep in sync with '.github/workflows/python-ci.yml'. + - TODO: Get this list automatically from python-ci.yml at runtime? + */ +def indicator_list = ["cdc_covidnet", "claims_hosp", "combo_cases_and_deaths", "google_symptoms", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph", "safegraph_patterns", "usafacts"] +def build_package = [:] +def deploy_staging = [:] +def deploy_production = [:] +pipeline { agent any - stages { - - stage ("Environment") { + stage('Build and Package') { when { - anyOf { - branch "deploy-*"; - changeRequest target: "deploy-*", comparator: "GLOB" - } + branch "main"; } steps { script { - // Get the indicator name from the pipeline env. - if ( env.CHANGE_TARGET ) { - INDICATOR = env.CHANGE_TARGET.replaceAll("deploy-", "") + indicator_list.each { indicator -> + build_package[indicator] = { + sh "jenkins/build-and-package.sh ${indicator}" + } } - else if ( env.BRANCH_NAME ) { - INDICATOR = env.BRANCH_NAME.replaceAll("deploy-", "") - } - else { - INDICATOR = "" - } - } - } - } - - stage('Build') { - when { - changeRequest target: "deploy-*", comparator: "GLOB" - } - steps { - sh "jenkins/${INDICATOR}-jenkins-build.sh" - } - } - - stage('Test') { - when { - changeRequest target: "deploy-*", comparator: "GLOB" - } - steps { - sh "jenkins/${INDICATOR}-jenkins-test.sh" - } - } - - stage('Package') { - when { - changeRequest target: "deploy-*", comparator: "GLOB" - } - steps { - sh "jenkins/${INDICATOR}-jenkins-package.sh" + parallel build_package + } } } - - stage('Deploy to staging env') { + stage('Deploy staging') { when { - changeRequest target: "deploy-*", comparator: "GLOB" + branch "main"; } steps { - sh "jenkins/jenkins-deploy-staging.sh ${INDICATOR}" + script { + indicator_list.each { indicator -> + deploy_staging[indicator] = { + sh "jenkins/deploy-staging.sh ${indicator}" + } + } + parallel deploy_staging + } } } - - stage('Deploy') { + stage('Deploy production') { when { - branch "deploy-*" + branch "prod"; } steps { - sh "jenkins/${INDICATOR}-jenkins-deploy.sh" + script { + indicator_list.each { indicator -> + deploy_production[indicator] = { + sh "jenkins/deploy-production.sh ${indicator}" + } + } + parallel deploy_production + } } } } - post { always { script { - /* - Use slackNotifier.groovy from shared library and provide current - build result as parameter. - */ + // Use slackNotifier.groovy from shared lib. slackNotifier(currentBuild.currentResult) } } diff --git a/jenkins/build-and-package.sh b/jenkins/build-and-package.sh new file mode 100755 index 000000000..10150bff8 --- /dev/null +++ b/jenkins/build-and-package.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Jenkins build-and-package +# + +set -eo pipefail +source ~/.bash_profile + +# Vars +local_indicator=$1 + +# +# Build +# + +cd "${WORKSPACE}/${local_indicator}" || exit + +# Set up venv +python -m venv env +source env/bin/activate +pip install ../_delphi_utils_python/. +pip install . + +# +# Package +# + +cd "${WORKSPACE}" || exit + +# Create .tar.gz for deployment +# TODO: Switcheroo to `artifacts` after this goes live! +tar -czvf "${JENKINS_HOME}/test-artifacts/${local_indicator}.tar.gz" "${local_indicator}" diff --git a/jenkins/deploy-production.sh b/jenkins/deploy-production.sh new file mode 100755 index 000000000..9479ff460 --- /dev/null +++ b/jenkins/deploy-production.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# Jenkins deploy +# + +set -eo pipefail +source ~/.bash_profile + +# +# Deploy +# + +local_indicator=$1 + +cd "${WORKSPACE}/ansible" || exit + +# Ansible! +ansible-playbook ansible-deploy.yaml --extra-vars "indicator=${local_indicator}" -i inventory diff --git a/jenkins/deploy-staging.sh b/jenkins/deploy-staging.sh new file mode 100755 index 000000000..c1e8ebffa --- /dev/null +++ b/jenkins/deploy-staging.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# Jenkins deploy +# + +set -eo pipefail +source ~/.bash_profile + +# +# Deploy +# + +local_indicator=$1 + +cd "${WORKSPACE}/ansible" || exit + +# Ansible! +ansible-playbook ansible-deploy-staging.yaml --extra-vars "indicator=${local_indicator}" -i inventory From 144f16e266e4a90cf7fec0b8c85bfef5fbffd36b Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Fri, 13 Nov 2020 20:01:06 -0500 Subject: [PATCH 2/6] Use test directory until we cut over --- ansible/vars.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/vars.yaml b/ansible/vars.yaml index 374990c92..32ec88db6 100644 --- a/ansible/vars.yaml +++ b/ansible/vars.yaml @@ -1,7 +1,8 @@ --- runtime_user: "indicators" jenkins_user: "jenkins" -jenkins_artifact_dir: "/var/lib/jenkins/artifacts" +#jenkins_artifact_dir: "/var/lib/jenkins/artifacts" +jenkins_artifact_dir: "/var/lib/jenkins/test-artifacts" indicators_runtime_dir: "/home/{{ runtime_user }}/runtime" package: "{{ indicator }}.tar.gz" # {{ indicator }} is passed in from the Jenkins shell script wrapper. python_version: "3.8.2" From c995549c7a899218e8ca2a183459a38f729cc608 Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Fri, 13 Nov 2020 20:11:42 -0500 Subject: [PATCH 3/6] Extra comment just in case --- ansible/vars.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yaml b/ansible/vars.yaml index 32ec88db6..b1abd0e31 100644 --- a/ansible/vars.yaml +++ b/ansible/vars.yaml @@ -1,7 +1,7 @@ --- runtime_user: "indicators" jenkins_user: "jenkins" -#jenkins_artifact_dir: "/var/lib/jenkins/artifacts" +#jenkins_artifact_dir: "/var/lib/jenkins/artifacts" # TODO: Switcheroo to `artifacts` after this goes live! jenkins_artifact_dir: "/var/lib/jenkins/test-artifacts" indicators_runtime_dir: "/home/{{ runtime_user }}/runtime" package: "{{ indicator }}.tar.gz" # {{ indicator }} is passed in from the Jenkins shell script wrapper. From fd64033f039ece7075e23d9b3a904177546006ff Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Mon, 16 Nov 2020 11:54:57 -0500 Subject: [PATCH 4/6] File issue for TODO --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d54dbbe46..6d0681ee8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,7 +7,7 @@ Declare variables. - indicator_list should contain all the indicators to handle in the pipeline. - Keep in sync with '.github/workflows/python-ci.yml'. - - TODO: Get this list automatically from python-ci.yml at runtime? + - TODO: #527 Get this list automatically from python-ci.yml at runtime. */ def indicator_list = ["cdc_covidnet", "claims_hosp", "combo_cases_and_deaths", "google_symptoms", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph", "safegraph_patterns", "usafacts"] def build_package = [:] From f95334a17b50be2a819e65f920564829f1aba8da Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Mon, 16 Nov 2020 12:56:04 -0500 Subject: [PATCH 5/6] Switch to production directory --- jenkins/build-and-package.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jenkins/build-and-package.sh b/jenkins/build-and-package.sh index 10150bff8..400e1db1a 100755 --- a/jenkins/build-and-package.sh +++ b/jenkins/build-and-package.sh @@ -28,5 +28,4 @@ pip install . cd "${WORKSPACE}" || exit # Create .tar.gz for deployment -# TODO: Switcheroo to `artifacts` after this goes live! -tar -czvf "${JENKINS_HOME}/test-artifacts/${local_indicator}.tar.gz" "${local_indicator}" +tar -czvf "${JENKINS_HOME}/artifacts/${local_indicator}.tar.gz" "${local_indicator}" From d007201f1dc3801b28686cad8c4462ea8c106661 Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Mon, 16 Nov 2020 12:56:38 -0500 Subject: [PATCH 6/6] Switch to production directory --- ansible/vars.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/vars.yaml b/ansible/vars.yaml index b1abd0e31..374990c92 100644 --- a/ansible/vars.yaml +++ b/ansible/vars.yaml @@ -1,8 +1,7 @@ --- runtime_user: "indicators" jenkins_user: "jenkins" -#jenkins_artifact_dir: "/var/lib/jenkins/artifacts" # TODO: Switcheroo to `artifacts` after this goes live! -jenkins_artifact_dir: "/var/lib/jenkins/test-artifacts" +jenkins_artifact_dir: "/var/lib/jenkins/artifacts" indicators_runtime_dir: "/home/{{ runtime_user }}/runtime" package: "{{ indicator }}.tar.gz" # {{ indicator }} is passed in from the Jenkins shell script wrapper. python_version: "3.8.2"