Merge branch 'main' of github.com:cmu-delphi/covidcast-indicators into archive

Author: sgsmob

.github/ISSUE_TEMPLATE/feature_release.md

@@ -15,7 +15,7 @@ assignees: 'benjaminysmith'
 
 <!-- relevant for most work -->
 
-- [ ] API [documentation](https://github.com/cmu-delphi/delphi-epidata/tree/main/docs/api) and/or [changelog](https://github.com/cmu-delphi/delphi-epidata/blob/main/docs/api/covidcast_changelog.md)
+- [ ] API [documentation](https://github.com/cmu-delphi/delphi-epidata/blob/main/docs/api/covidcast-signals/_source-template.md) and/or [changelog](https://github.com/cmu-delphi/delphi-epidata/blob/main/docs/api/covidcast_changelog.md)
 - [ ] API mailing list notification
 
 <!-- relevant for new signals -->

.github/pull_request_template.md

@@ -2,6 +2,7 @@
 Type of change (bug fix, new feature, etc), brief description, and motivation for these changes.
 
 ### Changelog
+Itemize code/test/documentation changes and files added/removed.
 - change1
 - change2
 

.github/workflows/python-ci.yml

@@ -5,18 +5,18 @@ name: Python package
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, prod ]
   pull_request:
     types: [ opened, synchronize, reopened, ready_for_review ]
-    branches: [ main ]
+    branches: [ main, prod ]
 
 jobs:
   build:
     runs-on: ubuntu-20.04
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        packages: [_delphi_utils_python, cdc_covidnet, claims_hosp, combo_cases_and_deaths, google_symptoms, jhu, nchs_mortality, quidel, quidel_covidtest, safegraph, safegraph_patterns, usafacts]
+        packages: [_delphi_utils_python, cdc_covidnet, changehc, claims_hosp, combo_cases_and_deaths, google_symptoms, jhu, nchs_mortality, quidel, quidel_covidtest, safegraph, safegraph_patterns, usafacts]
     defaults:
       run:
         working-directory: ${{ matrix.packages }}
@@ -34,7 +34,7 @@ jobs:
       run: |
         make install
     - name: Lint
-      if: ${{ matrix.packages != 'claims_hosp' && matrix.packages != 'quidel'}}
+      if: ${{ matrix.packages != 'claims_hosp' }}
       run: |
         make lint
     - name: Test

Jenkinsfile

@@ -1,90 +1,72 @@
 #!groovy
 
-// import shared library: https://github.com/cmu-delphi/jenkins-shared-library
+// Import shared lib.
 @Library('jenkins-shared-library') _
 
-pipeline {
+/*
+   Declare variables.
+   - indicator_list should contain all the indicators to handle in the pipeline.
+   - Keep in sync with '.github/workflows/python-ci.yml'.
+   - TODO: #527 Get this list automatically from python-ci.yml at runtime.
+ */
+def indicator_list = ["cdc_covidnet", "changehc", "claims_hosp", "combo_cases_and_deaths", "google_symptoms", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph", "safegraph_patterns", "usafacts"]
+def build_package = [:]
+def deploy_staging = [:]
+def deploy_production = [:]
 
+pipeline {
     agent any
-
     stages {
-
-        stage ("Environment") {            
+        stage('Build and Package') {
             when {
-                anyOf {
-                    branch "deploy-*";
-                    changeRequest target: "deploy-*", comparator: "GLOB"
-                }
+                branch "main";
             }
             steps {
                 script {
-                    // Get the indicator name from the pipeline env.
-                    if ( env.CHANGE_TARGET ) {
-                        INDICATOR = env.CHANGE_TARGET.replaceAll("deploy-", "")
+                    indicator_list.each { indicator ->
+                        build_package[indicator] = {
+                            sh "jenkins/build-and-package.sh ${indicator}"
+                        }
                     }
-                    else if ( env.BRANCH_NAME ) {
-                        INDICATOR = env.BRANCH_NAME.replaceAll("deploy-", "")
-                    }
-                    else {
-                        INDICATOR = ""
-                    }
-                } 
-            }
-        }
-
-        stage('Build') {
-            when {
-                changeRequest target: "deploy-*", comparator: "GLOB"
-            }
-            steps {
-                sh "jenkins/${INDICATOR}-jenkins-build.sh"
-            }
-        }
-
-        stage('Test') {
-            when {
-                changeRequest target: "deploy-*", comparator: "GLOB"
-            }
-            steps {
-                sh "jenkins/${INDICATOR}-jenkins-test.sh"
-            }
-        }
-        
-        stage('Package') {
-            when {
-                changeRequest target: "deploy-*", comparator: "GLOB"
-            }
-            steps {
-                sh "jenkins/${INDICATOR}-jenkins-package.sh"
+                    parallel build_package
+                }
             }
         }
-
-        stage('Deploy to staging env') {
+        stage('Deploy staging') {
             when {
-                changeRequest target: "deploy-*", comparator: "GLOB"
+                branch "main";
             }
             steps {
-                sh "jenkins/jenkins-deploy-staging.sh ${INDICATOR}"
+                script {
+                    indicator_list.each { indicator ->
+                        deploy_staging[indicator] = {
+                            sh "jenkins/deploy-staging.sh ${indicator}"
+                        }
+                    }
+                    parallel deploy_staging
+                }
             }
         }
-
-        stage('Deploy') {
+        stage('Deploy production') {
             when {
-                branch "deploy-*"
+                branch "prod";
             }
             steps {
-                sh "jenkins/${INDICATOR}-jenkins-deploy.sh"
+                script {
+                    indicator_list.each { indicator ->
+                        deploy_production[indicator] = {
+                            sh "jenkins/deploy-production.sh ${indicator}"
+                        }
+                    }
+                    parallel deploy_production
+                }
             }
         }
     }
-
     post {
         always {
             script {
-                /*
-                Use slackNotifier.groovy from shared library and provide current
-                build result as parameter.
-                */   
+                // Use slackNotifier.groovy from shared lib.
                 slackNotifier(currentBuild.currentResult)
             }
         }

_delphi_utils_python/delphi_utils/export.py

@@ -14,7 +14,8 @@ def create_export_csv(
     sensor: str,
     metric: Optional[str] = None,
     start_date: Optional[datetime] = None,
-    end_date: Optional[datetime] = None
+    end_date: Optional[datetime] = None,
+    remove_null_samples: Optional[bool] = False
 ):
     """Export data in the format expected by the Delphi API.
 
@@ -34,6 +35,8 @@ def create_export_csv(
         Earliest date to export or None if no minimum date restrictions should be applied.
     end_date: Optional[datetime]
         Latest date to export or None if no maximum date restrictions should be applied.
+    remove_null_samples: Optional[bool]
+        Whether to remove entries whose sample sizes are null.
     """
     df = df.copy()
 
@@ -54,6 +57,7 @@ def create_export_csv(
         else:
             export_filename = f"{date.strftime('%Y%m%d')}_{geo_res}_{metric}_{sensor}.csv"
         export_file = join(export_dir, export_filename)
-        df[df["timestamp"] == date][["geo_id", "val", "se", "sample_size",]].to_csv(
-            export_file, index=False, na_rep="NA"
-        )
+        export_df = df[df["timestamp"] == date][["geo_id", "val", "se", "sample_size",]]
+        if remove_null_samples:
+            export_df = export_df[export_df["sample_size"].notnull()]
+        export_df.to_csv(export_file, index=False, na_rep="NA")