diff --git a/ansible/ansible-deploy.yaml b/ansible/ansible-deploy.yaml index be5f361db..c5bacf75d 100644 --- a/ansible/ansible-deploy.yaml +++ b/ansible/ansible-deploy.yaml @@ -36,7 +36,7 @@ when: file.stat.exists - name: Set production params template. - copy: + template: src: templates/{{ indicator }}-params-prod.json.j2 dest: "{{ indicators_runtime_dir }}/{{ indicator }}/params.json" owner: "{{ runtime_user }}" diff --git a/ansible/google_health-build.yaml b/ansible/google_health-build.yaml new file mode 100644 index 000000000..426d541d9 --- /dev/null +++ b/ansible/google_health-build.yaml @@ -0,0 +1,15 @@ +--- +- hosts: localhost + connection: local + vars_files: + - vars.yaml + - vault.yaml + tasks: + - name: Set test params template. + template: + src: templates/{{ indicator }}-params-test.json.j2 + dest: "{{ workspace }}/{{ indicator }}/tests/params.json" + owner: "{{ jenkins_user }}" + group: "{{ jenkins_user }}" + mode: "0644" + delegate_to: localhost diff --git a/ansible/templates/google_health-params-prod.json.j2 b/ansible/templates/google_health-params-prod.json.j2 index b1308fd52..5990adee9 100644 --- a/ansible/templates/google_health-params-prod.json.j2 +++ b/ansible/templates/google_health-params-prod.json.j2 @@ -1,6 +1,6 @@ { "static_file_dir": "./static", - "export_dir": "./receiving", + "export_dir": "/common/covidcast/receiving/ght", "cache_dir": "./cache", "start_date": "2020-02-01", "end_date": "", diff --git a/ansible/templates/google_health-params-test.json.j2 b/ansible/templates/google_health-params-test.json.j2 new file mode 100644 index 000000000..a25a6912a --- /dev/null +++ b/ansible/templates/google_health-params-test.json.j2 @@ -0,0 +1,8 @@ +{ + "static_file_dir": "../static", + "export_dir": "./receiving", + "cache_dir": "../cache", + "start_date": "2020-02-11", + "end_date": "2020-04-30", + "ght_key": "{{ google_health_api_key }}" +} diff --git a/ansible/vars.yaml b/ansible/vars.yaml index 000a394fa..bff5e0739 100644 --- a/ansible/vars.yaml +++ b/ansible/vars.yaml @@ -1,8 +1,9 @@ --- runtime_user: "indicators" +jenkins_user: "jenkins" jenkins_artifact_dir: "/var/lib/jenkins/artifacts" indicators_runtime_dir: "/home/{{ runtime_user }}/runtime" -package: "{{ indicator }}.tar.gz" # This is passed in the Ansible invocation. +package: "{{ indicator }}.tar.gz" # {{ indicator }} is passed in from the Jenkins shell script wrapper. python_version: "3.8.2" pyenv_python_path: "/home/{{ runtime_user }}/.pyenv/versions/{{ python_version }}/bin/python" diff --git a/google_health/tests/test_pull_api.py b/google_health/tests/test_pull_api.py index 638e1bd4d..ec6632d19 100644 --- a/google_health/tests/test_pull_api.py +++ b/google_health/tests/test_pull_api.py @@ -36,7 +36,7 @@ def test_class_state(self): ] assert len(res["lines"][0]["points"]) == 2 - assert [x for x in res["lines"][0]["points"][0].keys()] == ["date", "value"] + assert set([x for x in res["lines"][0]["points"][0].keys()]) == ["date", "value"] assert res["lines"][0]["points"][0]["date"] == "May 05 2020" assert res["lines"][0]["points"][1]["date"] == "May 06 2020" @@ -55,7 +55,7 @@ def test_class_dma(self): ] assert len(res["lines"][0]["points"]) == 2 - assert [x for x in res["lines"][0]["points"][0].keys()] == ["date", "value"] + assert set([x for x in res["lines"][0]["points"][0].keys()]) == ["date", "value"] assert res["lines"][0]["points"][0]["date"] == "May 05 2020" assert res["lines"][0]["points"][1]["date"] == "May 06 2020" diff --git a/jenkins/google_health-jenkins-build copy.sh b/jenkins/google_health-jenkins-build copy.sh deleted file mode 100755 index d58ac46b2..000000000 --- a/jenkins/google_health-jenkins-build copy.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# -# JHU: Jenkins build -# - -set -exo pipefail -source ~/.bash_profile - -# -# Build -# - -local_indicator="google_health" - -cd "${WORKSPACE}/${local_indicator}" || exit - -# Set up venv -python -m venv env -source env/bin/activate -pip install ../_delphi_utils_python/. -pip install . diff --git a/jenkins/google_health-jenkins-build.sh b/jenkins/google_health-jenkins-build.sh new file mode 100755 index 000000000..ddc4e52c0 --- /dev/null +++ b/jenkins/google_health-jenkins-build.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# JHU: Jenkins build +# + +set -eo pipefail +source ~/.bash_profile + +# +# Build +# + +local_indicator="google_health" + +cd "${WORKSPACE}/${local_indicator}" || exit + +# Set up venv +python -m venv env +source env/bin/activate +pip install ../_delphi_utils_python/. +pip install . + +# Ansible! +# We need to call some extra Ansible here to handle placing a special params.json +# template so that our tests can complete. The below calls a small playbook that +# runs locally on the build (Jenkins) server to place the file. + +cd "${WORKSPACE}/ansible" || exit + +ansible-playbook google_health-build.yaml \ + --extra-vars "indicator=${local_indicator} workspace=${WORKSPACE}" -i localhost, \ No newline at end of file diff --git a/jenkins/google_health-jenkins-deploy copy.sh b/jenkins/google_health-jenkins-deploy.sh similarity index 93% rename from jenkins/google_health-jenkins-deploy copy.sh rename to jenkins/google_health-jenkins-deploy.sh index 86f9bc516..1354d1908 100755 --- a/jenkins/google_health-jenkins-deploy copy.sh +++ b/jenkins/google_health-jenkins-deploy.sh @@ -3,7 +3,7 @@ # Jenkins deploy # -set -exo pipefail +set -eo pipefail source ~/.bash_profile # diff --git a/jenkins/google_health-jenkins-package copy.sh b/jenkins/google_health-jenkins-package.sh similarity index 93% rename from jenkins/google_health-jenkins-package copy.sh rename to jenkins/google_health-jenkins-package.sh index 74fe9d861..63352a223 100755 --- a/jenkins/google_health-jenkins-package copy.sh +++ b/jenkins/google_health-jenkins-package.sh @@ -3,7 +3,7 @@ # Jenkins package # -set -exo pipefail +set -eo pipefail source ~/.bash_profile # diff --git a/jenkins/google_health-jenkins-test copy.sh b/jenkins/google_health-jenkins-test.sh similarity index 95% rename from jenkins/google_health-jenkins-test copy.sh rename to jenkins/google_health-jenkins-test.sh index dd558faef..27abf60c9 100755 --- a/jenkins/google_health-jenkins-test copy.sh +++ b/jenkins/google_health-jenkins-test.sh @@ -3,7 +3,7 @@ # JHU: Jenkins test # -set -exo pipefail +set -eo pipefail source ~/.bash_profile #