diff --git a/ansible/templates/safegraph-params-prod.json.j2 b/ansible/templates/safegraph-params-prod.json.j2 new file mode 100644 index 000000000..86cdaa82e --- /dev/null +++ b/ansible/templates/safegraph-params-prod.json.j2 @@ -0,0 +1,12 @@ +{ + "static_file_dir": "./static", + "raw_data_dir": "/common/safegraph", + "export_dir": "./receiving", + "cache_dir": "./cache", + "n_core": "12", + "aws_access_key_id": "{{ safegraph_aws_access_key_id }}", + "aws_secret_access_key": "{{ safegraph_aws_secret_access_key }}", + "aws_default_region": "us-east-1", + "aws_endpoint": "https://s3.wasabisys.com", + "wip_signal": "" +} diff --git a/ansible/vars.yaml b/ansible/vars.yaml index 55af65fcc..66a032a2d 100644 --- a/ansible/vars.yaml +++ b/ansible/vars.yaml @@ -11,4 +11,5 @@ pyenv_python_path: "/home/{{ runtime_user }}/.pyenv/versions/{{ python_version } google_health_api_key: "{{ vault_google_health_api_key }}" delphi_aws_access_key_id: "{{ vault_delphi_aws_access_key_id }}" delphi_aws_secret_access_key: "{{ vault_delphi_aws_secret_access_key }}" - +safegraph_aws_access_key_id: "{{ vault_safegraph_aws_access_key_id }}" +safegraph_aws_secret_access_key: "{{ vault_safegraph_aws_secret_access_key }}" diff --git a/ansible/vault.yaml b/ansible/vault.yaml index 2d07321a9..0a540b86b 100644 --- a/ansible/vault.yaml +++ b/ansible/vault.yaml @@ -1,16 +1,22 @@ $ANSIBLE_VAULT;1.1;AES256 -66386163643862646634343162646465663762643034303563333833633661333932646164656462 -6166646131623132393238336263623562373065643633310a663232373237396361623462613333 -62373663383565623263306539636431623230633065626363666531366662363065343066363031 -3738616663336665340a326138333634306137363837396366303861663064326333613662656630 -62306331646637326637363766366237663037306665343761643263646663316535343561623137 -63313365653535393639626465343232396261643239303430383138633135346466323834336665 -33633064353034613836313265613466623961373565363835343430373138376336363966316365 -35663664396436313432376264316663326130306134326231303234393561643436623039613136 -63366638396262383762383336643930343661636461646162653734336334306239383132643435 -39333665643738643966356431333830646561353263353063326330643731616130396466343339 -39346437653063303336626663623835613938633834396430353634383366386237353862643766 -37393738353231666565303031393839306463373461393761653866653330646534393832303264 -30323038646166366465396235623731343539313633326539663966333437623733626131653437 -62326632656462383835656235373664366566343866383938343639613737623631616231616135 -633863383761366461363532353137323936 +39633436633363346633333638666438316131313337333132396634393538383432623239316463 +3435333238376331383439366161656639353039326163370a376363633535623363383233646533 +63363865646561323132663032383331346332373364333465643330616638623466333039623831 +6530663236313234360a336264656239383166663934303335386238386139386132626165386138 +32663164326237323534636263663263666634383339613362633939323565356437663666653436 +31353362316334313561333430626361616337643133346664636434313664373333653839323630 +65346331383135656135386263643564333063626563336365333865663333353337393866666139 +64613735663363323938633161666662653161633835383832656164343836383339376661396332 +66353131373265373931366130383632633466363036373562363232663162333966316563373535 +65343336363732303132366335616335333334373063313562336330336661353239646533356461 +62313365633336613037626261353639323937363066363062356234653631346233373965636461 +63326237663537363338346566326232353632663463386135393535343436373335393430393865 +33393631623762636230656263363462346561323064653561393666373735313836666238323238 +66366564666266343636663666386566336637373036633966643961346636373066356632326464 +63336565656666336436383938346461646431353265353133633736363761623634346262616436 +61653633326333356330626638386665313865343233393637623662383634346534326537623662 +34326633623431343835346339656335386330333664373166313766366339663736376261343965 +63616461666230616131326537373130313239663931313330356538356161333537666237376362 +64613232333834303737323438616437303666643166383439393030316533343530363863613034 +39653761626439356133393164363561316535633230633438316137623333376633663665393634 +63333161376263613766353030616336386531303565346263366239653232333764 diff --git a/jenkins/safegraph-jenkins-build.sh b/jenkins/safegraph-jenkins-build.sh new file mode 100755 index 000000000..c724daf8c --- /dev/null +++ b/jenkins/safegraph-jenkins-build.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# JHU: Jenkins build +# + +set -eo pipefail +source ~/.bash_profile + +# +# Build +# + +local_indicator="safegraph" + +cd "${WORKSPACE}/${local_indicator}" || exit + +# Set up venv +python -m venv env +source env/bin/activate +pip install ../_delphi_utils_python/. +pip install . diff --git a/jenkins/safegraph-jenkins-deploy.sh b/jenkins/safegraph-jenkins-deploy.sh new file mode 100755 index 000000000..c5e3fc0e8 --- /dev/null +++ b/jenkins/safegraph-jenkins-deploy.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# Jenkins deploy +# + +set -eo pipefail +source ~/.bash_profile + +# +# Deploy +# + +local_indicator="safegraph" + +cd "${WORKSPACE}/ansible" || exit + +# Ansible! +ansible-playbook ansible-deploy.yaml --extra-vars "indicator=${local_indicator}" -i inventory diff --git a/jenkins/safegraph-jenkins-package.sh b/jenkins/safegraph-jenkins-package.sh new file mode 100755 index 000000000..b8a669347 --- /dev/null +++ b/jenkins/safegraph-jenkins-package.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# Jenkins package +# + +set -eo pipefail +source ~/.bash_profile + +# +# Package +# + +local_indicator="safegraph" + +cd "${WORKSPACE}" || exit + +# Create .tar.gz for deployment +tar -czvf "${JENKINS_HOME}/artifacts/${local_indicator}.tar.gz" "${local_indicator}" diff --git a/jenkins/safegraph-jenkins-test.sh b/jenkins/safegraph-jenkins-test.sh new file mode 100755 index 000000000..eb609941b --- /dev/null +++ b/jenkins/safegraph-jenkins-test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# JHU: Jenkins test +# + +set -eo pipefail +source ~/.bash_profile + +# +# Test +# + +local_indicator="safegraph" + +cd "${WORKSPACE}/${local_indicator}" || exit + +# Linter +env/bin/pylint --disable=C --disable=W --disable=R delphi_"${local_indicator}" + +# Unit tests and code coverage +cd tests || exit && \ + ../env/bin/pytest --cov=delphi_"${local_indicator}" --cov-report=term-missing diff --git a/safegraph/run-safegraph.sh b/safegraph/run-safegraph.sh new file mode 100644 index 000000000..01c54f0e6 --- /dev/null +++ b/safegraph/run-safegraph.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Run the Safegraph indicator +# + +#set -eo pipefail + +# Purge the receiving directory. +echo "Purging ./receiving..." +rm -f ./receiving/* + +# Run the indicator code. +echo "Running the indicator..." +env/bin/python -m delphi_safegraph + +# Copy the files to the ingestion directory. +echo "Copying files to the ingestion directory..." +# Hack to make cp care less about missing recent files since we don't always have them. +cp $(date +"receiving/%Y%m*") /common/covidcast/receiving/safegraph 2>/dev/null +cp $(date --date='-1 month' +"receiving/%Y%m*") /common/covidcast/receiving/safegraph