Finalize production config for google_health deployment (#165)

* Switch to midas export dir

* Properly rename Jenkins pipeline stage scripts

* Handle google_health's testing needs
- Add: Ansible playbook for securely handling placing the
  `params.json` template we need for testing. This will happen during
  the Jenkins build stage when we are setting up the venv in the
  workspace on the Jenkins server.
- Add: Test `params.json` template.
- Add: Jenkins user variable.
- Fix: Was incorrectly trying to use a file instead of a template in
  `ansible-deploy.yaml`.
- Add: Call the small Ansible playbook from the Jenkins build wrapper.

* Change to the Ansible root dir before trying to do Ansible things

* Delegate to localhost

* Tell Ansible we want to connect locally for this playbook

* Remove set -x from Bash scripts

* used set instead of list to compare dict.keys() (#186)

Co-authored-by: Vishakha Srivastava <[email protected]>

Author: 2 people

ansible/ansible-deploy.yaml

@@ -36,7 +36,7 @@
       when: file.stat.exists
 
     - name: Set production params template.
-      copy:
+      template:
         src: templates/{{ indicator }}-params-prod.json.j2
         dest: "{{ indicators_runtime_dir }}/{{ indicator }}/params.json"
         owner: "{{ runtime_user }}"

ansible/google_health-build.yaml

@@ -0,0 +1,15 @@
+---
+- hosts: localhost
+  connection: local
+  vars_files:
+    - vars.yaml
+    - vault.yaml
+  tasks:
+    - name: Set test params template.
+      template:
+        src: templates/{{ indicator }}-params-test.json.j2
+        dest: "{{ workspace }}/{{ indicator }}/tests/params.json"
+        owner: "{{ jenkins_user }}"
+        group: "{{ jenkins_user }}"
+        mode: "0644"
+      delegate_to: localhost

ansible/templates/google_health-params-prod.json.j2

@@ -1,6 +1,6 @@
 {
   "static_file_dir": "./static",
-  "export_dir": "./receiving",
+  "export_dir": "/common/covidcast/receiving/ght",
   "cache_dir": "./cache",
   "start_date": "2020-02-01",
   "end_date": "",

ansible/templates/google_health-params-test.json.j2

@@ -0,0 +1,8 @@
+{
+  "static_file_dir": "../static",
+  "export_dir": "./receiving",
+  "cache_dir": "../cache",
+  "start_date": "2020-02-11",
+  "end_date": "2020-04-30",
+  "ght_key": "{{ google_health_api_key }}"
+}

ansible/vars.yaml

@@ -1,8 +1,9 @@
 ---
 runtime_user: "indicators"
+jenkins_user: "jenkins"
 jenkins_artifact_dir: "/var/lib/jenkins/artifacts"
 indicators_runtime_dir: "/home/{{ runtime_user }}/runtime"
-package: "{{ indicator }}.tar.gz" # This is passed in the Ansible invocation.
+package: "{{ indicator }}.tar.gz" # {{ indicator }} is passed in from the Jenkins shell script wrapper.
 python_version: "3.8.2"
 pyenv_python_path: "/home/{{ runtime_user }}/.pyenv/versions/{{ python_version }}/bin/python"
 

google_health/tests/test_pull_api.py

@@ -36,7 +36,7 @@ def test_class_state(self):
         ]
 
         assert len(res["lines"][0]["points"]) == 2
-        assert [x for x in res["lines"][0]["points"][0].keys()] == ["date", "value"]
+        assert set([x for x in res["lines"][0]["points"][0].keys()]) == ["date", "value"]
         assert res["lines"][0]["points"][0]["date"] == "May 05 2020"
         assert res["lines"][0]["points"][1]["date"] == "May 06 2020"
 
@@ -55,7 +55,7 @@ def test_class_dma(self):
         ]
 
         assert len(res["lines"][0]["points"]) == 2
-        assert [x for x in res["lines"][0]["points"][0].keys()] == ["date", "value"]
+        assert set([x for x in res["lines"][0]["points"][0].keys()]) == ["date", "value"]
         assert res["lines"][0]["points"][0]["date"] == "May 05 2020"
         assert res["lines"][0]["points"][1]["date"] == "May 06 2020"
 

jenkins/google_health-jenkins-build copy.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+# JHU: Jenkins build
+#
+
+set -eo pipefail
+source ~/.bash_profile
+
+#
+# Build
+#
+
+local_indicator="google_health"
+
+cd "${WORKSPACE}/${local_indicator}" || exit
+
+# Set up venv
+python -m venv env
+source env/bin/activate
+pip install ../_delphi_utils_python/.
+pip install .
+
+# Ansible!
+# We need to call some extra Ansible here to handle placing a special params.json
+# template so that our tests can complete. The below calls a small playbook that
+# runs locally on the build (Jenkins) server to place the file.
+
+cd "${WORKSPACE}/ansible" || exit
+
+ansible-playbook google_health-build.yaml \
+  --extra-vars "indicator=${local_indicator} workspace=${WORKSPACE}" -i localhost,

jenkins/google_health-jenkins-build.sh

@@ -3,7 +3,7 @@
 # Jenkins deploy
 #
 
-set -exo pipefail
+set -eo pipefail
 source ~/.bash_profile
 
 #

jenkins/google_health-jenkins-deploy copy.sh renamed to jenkins/google_health-jenkins-deploy.sh

@@ -3,7 +3,7 @@
 # Jenkins package
 #
 
-set -exo pipefail
+set -eo pipefail
 source ~/.bash_profile
 
 #

jenkins/google_health-jenkins-package copy.sh renamed to jenkins/google_health-jenkins-package.sh

@@ -3,7 +3,7 @@
 # JHU: Jenkins test
 #
 
-set -exo pipefail
+set -eo pipefail
 source ~/.bash_profile
 
 #