Merge pull request #425 from cmu-delphi/bgc-fix-deploy-changehc

Update changehc deployment

Author: rumackaaron

Jenkinsfile

@@ -59,6 +59,15 @@ pipeline {
             }
         }
 
+        stage('Deploy to staging env') {
+            when {
+                changeRequest target: "deploy-*", comparator: "GLOB"
+            }
+            steps {
+                sh "jenkins/jenkins-deploy-staging.sh ${INDICATOR}"
+            }
+        }
+
         stage('Deploy') {
             when {
                 branch "deploy-*"

ansible/ansible-deploy-staging.yaml

@@ -0,0 +1,44 @@
+---
+- hosts: runtime_host_staging
+  vars_files:
+    - vars.yaml
+    - vault.yaml
+  tasks:
+    - name: Copy and unarchive the package into the indicators runtime host directory.
+      unarchive:
+        src: "{{ jenkins_artifact_dir }}/{{ package }}"
+        dest: "{{ indicators_runtime_dir }}"
+        owner: "{{ runtime_user }}"
+        group: "{{ runtime_user }}"
+
+    - name: Mutate Python bin path used in venv.
+      file:
+        src: "{{ pyenv_python_path }}"
+        dest: "{{ indicators_runtime_dir }}/{{ indicator }}/env/bin/python"
+        owner: "{{ runtime_user }}"
+        group: "{{ runtime_user }}"
+        state: link
+
+    - name: Check to see if we have a params file to send.
+      local_action: stat path="files/{{ indicator }}-params-prod.json"
+      register: file
+
+    - name: Check to see if we have a params template to send.
+      local_action: stat path="templates/{{ indicator }}-params-prod.json.j2"
+      register: template
+
+    - name: Set production params file.
+      copy:
+        src: files/{{ indicator }}-params-prod.json
+        dest: "{{ indicators_runtime_dir }}/{{ indicator }}/params.json"
+        owner: "{{ runtime_user }}"
+        group: "{{ runtime_user }}"
+      when: file.stat.exists
+
+    - name: Set production params template.
+      template:
+        src: templates/{{ indicator }}-params-prod.json.j2
+        dest: "{{ indicators_runtime_dir }}/{{ indicator }}/params.json"
+        owner: "{{ runtime_user }}"
+        group: "{{ runtime_user }}"
+      when: template.stat.exists

ansible/inventory

@@ -1,2 +1,5 @@
 [runtime_host]
-delphi-master-prod-01.delphi.cmu.edu
+delphi-master-prod-01.delphi.cmu.edu
+
+[runtime_host_staging]
+app-mono-dev-01.delphi.cmu.edu

ansible/templates/changehc-params-prod.json.j2

@@ -1,17 +1,17 @@
 {
   "static_file_dir": "./static",
-  "export_dir": "./receiving",
+  "export_dir": "/common/covidcast/receiving/changehc",
   "cache_dir": "./cache",
-  "input_emr_file": "./tests/test_data/SYNICUE_CMB_INPATIENT_11062020.csv.gz",
-  "input_claims_file": "./tests/test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz",
+  "input_denom_file": "./tests/test_data/20200601_All_Outpatients_By_County.dat",
+  "input_covid_file": "./tests/test_data/20200601_Covid_Outpatients_By_County.dat",
   "start_date": "2020-02-01",
   "end_date": null,
   "drop_date": null,
   "n_backfill_days": 60,
   "n_waiting_days": 3,
   "se": false,
   "parallel": false,
-  "geos": ["state", "msa", "hrr", "fips"],
+  "geos": ["state", "msa", "hrr", "county"],
   "weekday": [true, false],
   "wip_signal": "",
   "aws_credentials": {

changehc/delphi_changehc/download_ftp_files.py

@@ -57,7 +57,7 @@ def download(out_path, ftp_conn):
         client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
         client.connect(ftp_conn["host"], username=ftp_conn["user"],
-                       password=ftp_conn["pass"][1:] + ftp_conn["pass"][0],
+                       password=ftp_conn["pass"],
                        port=ftp_conn["port"],
                        allow_agent=False, look_for_keys=False)
         sftp = client.open_sftp()

changehc/params.json.template

@@ -2,16 +2,16 @@
   "static_file_dir": "./static",
   "export_dir": "./receiving",
   "cache_dir": "./cache",
-  "input_emr_file": "./tests/test_data/SYNICUE_CMB_INPATIENT_11062020.csv.gz",
-  "input_claims_file": "./tests/test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz",
+  "input_denom_file": "./tests/test_data/20200601_All_Outpatients_By_County.dat",
+  "input_covid_file": "./tests/test_data/20200601_Covid_Outpatients_By_County.dat",
   "start_date": "2020-02-01",
   "end_date": null,
   "drop_date": null,
   "n_backfill_days": 60,
   "n_waiting_days": 3,
   "se": false,
   "parallel": false,
-  "geos": ["state", "msa", "hrr", "fips"],
+  "geos": ["state", "msa", "hrr", "county"],
   "weekday": [true, false],
   "wip_signal": "",
   "aws_credentials": {

jenkins/jenkins-deploy-staging.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Jenkins deploy
+#
+
+set -eo pipefail
+source ~/.bash_profile
+
+#
+# Deploy
+#
+
+local_indicator=$1
+
+cd "${WORKSPACE}/ansible" || exit
+
+# Ansible!
+ansible-playbook ansible-deploy-staging.yaml --extra-vars "indicator=${local_indicator}" -i inventory