Merge pull request #2005 from cmu-delphi/refactor-jenkins-pipeline

Refactor Jenkins pipeline

Author: korlaxxalrok

Jenkinsfile

@@ -10,7 +10,7 @@
    - TODO: #527 Get this list automatically from python-ci.yml at runtime.
  */
 
-def indicator_list = ["backfill_corrections", "changehc", "claims_hosp", "google_symptoms", "hhs_hosp", "nchs_mortality", "quidel_covidtest", "sir_complainsalot", "doctor_visits", "nwss_wastewater", "nssp"]
+def indicator_list = ['backfill_corrections', 'changehc', 'claims_hosp', 'google_symptoms', 'hhs_hosp', 'nchs_mortality', 'quidel_covidtest', 'sir_complainsalot', 'doctor_visits', 'nwss_wastewater', 'nssp']
 def build_package_main = [:]
 def build_package_prod = [:]
 def deploy_staging = [:]
@@ -19,39 +19,62 @@ def deploy_production = [:]
 pipeline {
     agent any
     stages {
-        stage('Build and Package main') {
+        stage('Build dev/feature branch') {
+            when  {
+                not {
+                    anyOf {
+                        branch 'main'
+                        branch 'prod'
+                    }
+                }
+            }
+            steps {
+                script {
+                    indicator_list.each { indicator ->
+                        stage("Build ${indicator}") {
+                            sh "jenkins/build-indicator.sh ${indicator}"
+                        }
+                    }
+                }
+            }
+        }
+        stage('Build and Package main branch') {
             when {
-                branch "main";
+                branch 'main'
             }
             steps {
                 script {
                     indicator_list.each { indicator ->
-                        build_package_main[indicator] = {
-                            sh "jenkins/build-and-package.sh ${indicator} main"
+                        stage("Build ${indicator}") {
+                            sh "jenkins/build-indicator.sh ${indicator}"
+                        }
+                        stage("Package ${indicator}") {
+                            sh "jenkins/package-indicator.sh ${indicator} main"
                         }
                     }
-                    parallel build_package_main
                 }
             }
         }
-        stage('Build and Package prod') {
+        stage('Build and Package prod branch') {
             when {
-                branch "prod";
+                branch 'prod'
             }
             steps {
                 script {
                     indicator_list.each { indicator ->
-                        build_package_prod[indicator] = {
-                            sh "jenkins/build-and-package.sh ${indicator} prod"
+                        stage("Build ${indicator}") {
+                            sh "jenkins/build-indicator.sh ${indicator}"
+                        }
+                        stage("Package ${indicator}") {
+                            sh "jenkins/package-indicator.sh ${indicator} prod"
                         }
                     }
-                    parallel build_package_prod
                 }
             }
         }
-        stage('Deploy staging') {
+        stage('Deploy main branch to staging env') {
             when {
-                branch "main";
+                branch 'main'
             }
             steps {
                 script {
@@ -64,9 +87,9 @@ pipeline {
                 }
             }
         }
-        stage('Deploy production') {
+        stage('Deploy prod branch to production env') {
             when {
-                branch "prod";
+                branch 'prod'
             }
             steps {
                 script {

jenkins/build-and-package.sh renamed to jenkins/build-indicator.sh

@@ -1,18 +1,13 @@
 #!/usr/bin/env bash
 #
-# Jenkins build-and-package
+# Jenkins build 
 #
 
 set -eo pipefail
 source ~/.bash_profile
 
 # Vars
 local_indicator=$1
-branch=$2
-
-#
-# Build
-#
 
 cd "${WORKSPACE}/${local_indicator}" || exit
 
@@ -23,12 +18,3 @@ pip install pip==23.0.1 --retries 10 --timeout 20
 pip install numpy --retries 10 --timeout 20
 pip install ../_delphi_utils_python/. --retries 10 --timeout 20
 [ ! -f setup.py ] || pip install . --retries 10 --timeout 20
-
-#
-# Package
-#
-
-cd "${WORKSPACE}" || exit
-
-# Create .tar.gz for deployment
-tar -czvf "${JENKINS_HOME}/artifacts/${branch}_${local_indicator}.tar.gz" "${local_indicator}"

jenkins/package-indicator.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Jenkins package
+#
+
+set -eo pipefail
+source ~/.bash_profile
+
+# Vars
+local_indicator=$1
+branch=$2
+
+cd "${WORKSPACE}" || exit
+
+# Create .tar.gz for deployment
+tar -czvf "${JENKINS_HOME}/artifacts/${branch}_${local_indicator}.tar.gz" "${local_indicator}"