Merge pull request #447 from cmu-delphi/deploy-changehc

Merge deploy-changehc to main

Author: krivard

Jenkinsfile

@@ -59,6 +59,15 @@ pipeline {
             }
         }
 
+        stage('Deploy to staging env') {
+            when {
+                changeRequest target: "deploy-*", comparator: "GLOB"
+            }
+            steps {
+                sh "jenkins/jenkins-deploy-staging.sh ${INDICATOR}"
+            }
+        }
+
         stage('Deploy') {
             when {
                 branch "deploy-*"

ansible/ansible-deploy-staging.yaml

@@ -0,0 +1,44 @@
+---
+- hosts: runtime_host_staging
+  vars_files:
+    - vars.yaml
+    - vault.yaml
+  tasks:
+    - name: Copy and unarchive the package into the indicators runtime host directory.
+      unarchive:
+        src: "{{ jenkins_artifact_dir }}/{{ package }}"
+        dest: "{{ indicators_runtime_dir }}"
+        owner: "{{ runtime_user }}"
+        group: "{{ runtime_user }}"
+
+    - name: Mutate Python bin path used in venv.
+      file:
+        src: "{{ pyenv_python_path }}"
+        dest: "{{ indicators_runtime_dir }}/{{ indicator }}/env/bin/python"
+        owner: "{{ runtime_user }}"
+        group: "{{ runtime_user }}"
+        state: link
+
+    - name: Check to see if we have a params file to send.
+      local_action: stat path="files/{{ indicator }}-params-prod.json"
+      register: file
+
+    - name: Check to see if we have a params template to send.
+      local_action: stat path="templates/{{ indicator }}-params-prod.json.j2"
+      register: template
+
+    - name: Set production params file.
+      copy:
+        src: files/{{ indicator }}-params-prod.json
+        dest: "{{ indicators_runtime_dir }}/{{ indicator }}/params.json"
+        owner: "{{ runtime_user }}"
+        group: "{{ runtime_user }}"
+      when: file.stat.exists
+
+    - name: Set production params template.
+      template:
+        src: templates/{{ indicator }}-params-prod.json.j2
+        dest: "{{ indicators_runtime_dir }}/{{ indicator }}/params.json"
+        owner: "{{ runtime_user }}"
+        group: "{{ runtime_user }}"
+      when: template.stat.exists

ansible/inventory

@@ -1,2 +1,5 @@
 [runtime_host]
-delphi-master-prod-01.delphi.cmu.edu
+delphi-master-prod-01.delphi.cmu.edu
+
+[runtime_host_staging]
+app-mono-dev-01.delphi.cmu.edu

ansible/templates/changehc-params-prod.json.j2

@@ -0,0 +1,28 @@
+{
+  "static_file_dir": "./static",
+  "export_dir": "/common/covidcast/receiving/changehc",
+  "cache_dir": "./cache",
+  "input_denom_file": "./tests/test_data/20200601_All_Outpatients_By_County.dat",
+  "input_covid_file": "./tests/test_data/20200601_Covid_Outpatients_By_County.dat",
+  "start_date": "2020-02-01",
+  "end_date": null,
+  "drop_date": null,
+  "n_backfill_days": 60,
+  "n_waiting_days": 3,
+  "se": false,
+  "parallel": false,
+  "geos": ["state", "msa", "hrr", "county"],
+  "weekday": [true, false],
+  "wip_signal": "",
+  "aws_credentials": {
+    "aws_access_key_id": "",
+    "aws_secret_access_key": ""
+  },
+  "bucket_name": "",
+  "ftp_conn": {
+    "host": "{{ changehc_sftp_host }}",
+    "user": "{{ changehc_sftp_user }}",
+    "pass": "{{ changehc_sftp_password }}",
+    "port": "{{ changehc_sftp_port }}"
+  }
+}

ansible/vars.yaml

@@ -13,3 +13,11 @@ delphi_aws_access_key_id: "{{ vault_delphi_aws_access_key_id }}"
 delphi_aws_secret_access_key: "{{ vault_delphi_aws_secret_access_key }}"
 safegraph_aws_access_key_id: "{{ vault_safegraph_aws_access_key_id }}"
 safegraph_aws_secret_access_key: "{{ vault_safegraph_aws_secret_access_key }}"
+quidel_datadrop_mail_server: "{{ vault_quidel_datadrop_mail_server }}"
+quidel_datadrop_account: "{{ vault_quidel_datadrop_account }}"
+quidel_datadrop_password: "{{ vault_quidel_datadrop_password }}"
+quidel_datadrop_sender: "{{ vault_quidel_datadrop_sender }}"
+changehc_sftp_host: "{{ vault_changehc_sftp_host }}"
+changehc_sftp_port: "{{ vault_changehc_sftp_port }}"
+changehc_sftp_user: "{{ vault_changehc_sftp_user }}"
+changehc_sftp_password: "{{ vault_changehc_sftp_password }}"

ansible/vault.yaml

@@ -1,22 +1,42 @@
 $ANSIBLE_VAULT;1.1;AES256
-39633436633363346633333638666438316131313337333132396634393538383432623239316463
-3435333238376331383439366161656639353039326163370a376363633535623363383233646533
-63363865646561323132663032383331346332373364333465643330616638623466333039623831
-6530663236313234360a336264656239383166663934303335386238386139386132626165386138
-32663164326237323534636263663263666634383339613362633939323565356437663666653436
-31353362316334313561333430626361616337643133346664636434313664373333653839323630
-65346331383135656135386263643564333063626563336365333865663333353337393866666139
-64613735663363323938633161666662653161633835383832656164343836383339376661396332
-66353131373265373931366130383632633466363036373562363232663162333966316563373535
-65343336363732303132366335616335333334373063313562336330336661353239646533356461
-62313365633336613037626261353639323937363066363062356234653631346233373965636461
-63326237663537363338346566326232353632663463386135393535343436373335393430393865
-33393631623762636230656263363462346561323064653561393666373735313836666238323238
-66366564666266343636663666386566336637373036633966643961346636373066356632326464
-63336565656666336436383938346461646431353265353133633736363761623634346262616436
-61653633326333356330626638386665313865343233393637623662383634346534326537623662
-34326633623431343835346339656335386330333664373166313766366339663736376261343965
-63616461666230616131326537373130313239663931313330356538356161333537666237376362
-64613232333834303737323438616437303666643166383439393030316533343530363863613034
-39653761626439356133393164363561316535633230633438316137623333376633663665393634
-63333161376263613766353030616336386531303565346263366239653232333764
+66636361333161346433333963373239653963623663663037323339313139303634646361373037
+3031383130623131653561656465333831613164396563310a613735366261303931663033663031
+30643562663566393861616263633338333965383361623236616339663237653733333936366539
+3465666462363634390a356463663863333536313765613065353336626633623264633132356634
+38623763306165306333663139646465313038303134333461666364656231383965393364666266
+39613864386337323032323033623166623265333033396130336633366431333337646263363963
+65666461346164643164313838373766316165643635663238326638626136333462373035306161
+65383662653061633939383039316562646431343436303838656237323465633936373463336335
+39663938303130383665336335303534613135656562326134643730666661356439393838383832
+65336562363865626665616438633238353065383735363136363931353161653036643231623939
+39303635373861343036663265343636333066616336663338376438663130373438636364626335
+32313638633262616664636337323063323166383636356132326339643631613138376266343838
+64653636383663626535653334393730393534616165326436396431636237323535393262646233
+33313266643962626665353738356262393435313065623863633235373561393231313533383662
+65316435386562633763386435643666643930336233383266633033323530633232656337623635
+66616435366532663166303766393939346635666136336635373834613834366532313638616464
+32313765353362346266323938393561333039663063306538316133343765623632313231613733
+31353364316662323332383964326338386164363062353636336636316233636466313838626263
+66323435346434663864396262623636316434626631643463653337653063623733323233623731
+66636564633438633731346162653561326532616165313866646531376436616266303764666237
+31636430623564396432346230666565326137613364363064363963356332346234303865303937
+37343863653939323139333533633230323465653962343332306464376138396137626566373063
+38626231643961383466376561666338663236613638333831663633376464656432333063363164
+34346566636232313734646137303531643536613537373664643663623263653831656136323239
+34353034633038316263386636376234393563393466643133383866656134646463666433336234
+32323665323634323635386430316238656366376561653365613934303262343938303833323464
+32353062363132376131326466393266313662376330663831323136313262333066656261353234
+35353966333733333639646130383235333738643833376166396564313666633135613938323236
+35643261376537663338323834363734343735366131313133623433323138313631653639323438
+61626631656665643939666437303932336437656234626639333138353466613463343032326639
+63313432666364306137343131656630326263616135343034303836396435383330626463373733
+37303434363735616339353661316461333535656131323631633731353539316264643735633862
+31363638353435303963333864393733396232643931613866636463653638646236356563333938
+61323134326131353264313264663661333261323533333537323361653636326636653137306330
+36643164643438346436613538353835656132366163383865333237383038353839626439346632
+66663132313836353766383031383732383838666261653230613336323631346533613361636331
+61326535646438643732373130646366653738343236396561623666396231316330383266623839
+64666138373032353136666234383434343162323761623061326233363532396132623237306362
+35636535313336636566393532303337333630393165616264653466376566323533323737343534
+64343831653131306138383263663933396338373736663565373739373330336432326666386230
+31616338336265333832613331623339656231643838353261373037636435396531

changehc/README.md

@@ -74,4 +74,3 @@ should not include critical sub-routines.
 - config.py: Config: constants for reading data and transformations, Constants: constants for sanity checks
 - constants.py: constants for signal names
 - weekday.py: Weekday: Adjusts for weekday effect
-

changehc/delphi_changehc/download_ftp_files.py

@@ -57,7 +57,7 @@ def download(out_path, ftp_conn):
         client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
         client.connect(ftp_conn["host"], username=ftp_conn["user"],
-                       password=ftp_conn["pass"][1:] + ftp_conn["pass"][0],
+                       password=ftp_conn["pass"],
                        port=ftp_conn["port"],
                        allow_agent=False, look_for_keys=False)
         sftp = client.open_sftp()

changehc/params.json.template

@@ -2,16 +2,16 @@
   "static_file_dir": "./static",
   "export_dir": "./receiving",
   "cache_dir": "./cache",
-  "input_emr_file": "./tests/test_data/SYNICUE_CMB_INPATIENT_11062020.csv.gz",
-  "input_claims_file": "./tests/test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz",
+  "input_denom_file": "./tests/test_data/20200601_All_Outpatients_By_County.dat",
+  "input_covid_file": "./tests/test_data/20200601_Covid_Outpatients_By_County.dat",
   "start_date": "2020-02-01",
   "end_date": null,
   "drop_date": null,
   "n_backfill_days": 60,
   "n_waiting_days": 3,
   "se": false,
   "parallel": false,
-  "geos": ["state", "msa", "hrr", "fips"],
+  "geos": ["state", "msa", "hrr", "county"],
   "weekday": [true, false],
   "wip_signal": "",
   "aws_credentials": {

jenkins/changehc-jenkins-build.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Change HC: Jenkins build
+#
+
+set -eo pipefail
+source ~/.bash_profile
+
+#
+# Build
+#
+
+local_indicator="changehc"
+
+cd "${WORKSPACE}/${local_indicator}" || exit
+
+# Set up venv
+python -m venv env
+source env/bin/activate
+pip install ../_delphi_utils_python/.
+pip install .

jenkins/changehc-jenkins-deploy.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Change HC: Jenkins deploy
+#
+
+set -eo pipefail
+source ~/.bash_profile
+
+#
+# Deploy
+#
+
+local_indicator="changehc"
+
+cd "${WORKSPACE}/ansible" || exit
+
+# Ansible!
+ansible-playbook ansible-deploy.yaml --extra-vars "indicator=${local_indicator}" -i inventory

jenkins/changehc-jenkins-package.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Change HC: Jenkins package
+#
+
+set -eo pipefail
+source ~/.bash_profile
+
+#
+# Package
+#
+
+local_indicator="changehc"
+
+cd "${WORKSPACE}" || exit
+
+# Create .tar.gz for deployment
+tar -czvf "${JENKINS_HOME}/artifacts/${local_indicator}.tar.gz" "${local_indicator}"

jenkins/changehc-jenkins-test.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# Change HC: Jenkins test
+#
+
+set -eo pipefail
+source ~/.bash_profile
+
+#
+# Test
+#
+
+local_indicator="changehc"
+
+cd "${WORKSPACE}/${local_indicator}" || exit
+
+# Linter
+env/bin/pylint delphi_"${local_indicator}"
+
+# Unit tests and code coverage
+cd tests || exit && \
+  ../env/bin/pytest --cov=delphi_"${local_indicator}" --cov-report=term-missing

jenkins/jenkins-deploy-staging.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Jenkins deploy
+#
+
+set -eo pipefail
+source ~/.bash_profile
+
+#
+# Deploy
+#
+
+local_indicator=$1
+
+cd "${WORKSPACE}/ansible" || exit
+
+# Ansible!
+ansible-playbook ansible-deploy-staging.yaml --extra-vars "indicator=${local_indicator}" -i inventory