diff --git a/dev/local/epidata-refresh.sh b/dev/local/epidata-refresh.sh new file mode 100755 index 000000000..25feb04e0 --- /dev/null +++ b/dev/local/epidata-refresh.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +if [ -z "$1" ]; then + echo "USAGE:" + echo "$0 [database] [web] [python] [testdir|test.py ...]" + echo "Docker control panel for delphi-epidata development" + echo + echo " Assumes you have installed your environment using" + echo " delphi-epidata/dev/local/install.sh." + echo + echo " Cleans up dangling Docker images before starting." + echo + echo " Checks for the delphi-net bridge, and creates it if it doesn't exist." + echo + echo " Creates all prereq images (delphi_database, delphi_python) only if they don't" + echo " exist. If you need to rebuild a prereq, you're probably doing something" + echo " complicated, and can figure out the rebuild command on your own." + echo + echo " database: Stops currently-running delphi_database_epidata instances, if any." + echo " Rebuilds delphi_database_epidata image." + echo " Runs image in the background and pipes stdout to a log file." + echo " Blocks until database is ready to receive connections." + echo + echo " web: Stops currently-running delphi_web_epidata instances, if any." + echo " Rebuilds delphi_web_epidata image." + echo " Runs image in the background and pipes stdout to a log file." + echo + echo " python: Rebuilds delphi_web_python image. You shouldn't need to do this" + echo " often; only if you are installing a new environment, or have" + echo " made changes to delphi-epidata/dev/docker/python/Dockerfile." + echo + echo " Loops through the remaining arguments and runs them as pytest within the" + echo " delphi_web_python image. This uses bindmounts into the local filesystem," + echo " so any changes you have made to src/acquisition/, tests/, or integrations/" + echo " will automatically take effect without having to rebuild delphi_web_python." + echo " Saves all test output to output.txt for review, in case it excees your" + echo " terminal buffer. Halts on first failure and lists failed tests." + echo + echo " Always run this from the `driver` directory. Never put anything other than" + echo " code in the `driver` directory, since everything under the `driver` directory" + echo " will get folded into the Docker images created. If it takes more than 1m to" + echo " build Docker images, you have probably accidentally stored other data in the" + echo " `driver` directory tree." + echo + echo " Depending on your operating system, you may need to be root or use sudo." + exit 0 +fi + +docker images -f "dangling=true" -q | xargs docker rmi >/dev/null 2>&1 +docker network ls | grep delphi-net || docker network create --driver bridge delphi-net + +LOGS=../driver-logs +NOW=`date "+%Y-%m-%d"` + +if [ "$1" == "database" ]; then + shift + LOGFILE=${LOGS}/delphi_database_epidata/${NOW}.log + docker ps | grep delphi_database_epidata && docker stop delphi_database_epidata + # only build prereqs if we need them + docker images delphi_database | grep delphi || \ + docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile . || exit 1 + docker build -t delphi_database_epidata -f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile . || exit 1 + docker run --rm -p 127.0.0.1:13306:3306 --network delphi-net --name delphi_database_epidata \ + --mount type=bind,source="$(pwd)"/repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ + delphi_database_epidata \ + >${LOGFILE} 2>&1 & + while true; do + sed -n '/Temporary server stopped/,/mysqld: ready for connections/p' ${LOGFILE} | grep "ready for connections" && break + tail -1 ${LOGFILE} + sleep 1 + done + grep ERROR ${LOGFILE} && exit 1 +fi + +if [ "$1" == "web" ]; then + shift + docker ps | grep delphi_web_epidata && docker stop delphi_web_epidata + cd repos/delphi/delphi-epidata && docker build -t delphi_web_epidata -f ./devops/Dockerfile . && cd - || exit 1 + docker run --rm -p 127.0.0.1:10080:80 \ + --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \ + --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" \ + --network delphi-net --name delphi_web_epidata delphi_web_epidata >>${LOGS}/delphi_web_epidata/${NOW}.log 2>&1 & +fi +if [ "$1" == "python" ]; then + shift + # only build prereqs if we need them + docker images delphi_python | grep delphi || \ + docker build -t delphi_python -f repos/delphi/operations/dev/docker/python/Dockerfile . || exit 1 + docker build -t delphi_web_python \ + -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile . || exit 1 +fi +rm -f output.txt +for t in $@; do + set -x + docker run --rm --network delphi-net \ + --mount type=bind,source="$(pwd)"/repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ + --mount type=bind,source="$(pwd)"/repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ + --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" \ + delphi_web_python \ + python -m pytest --import-mode importlib $t | tee -a output.txt + set +x + [[ $? = "0" ]] || exit 1 + grep -e "FAIL" -e "tests did not pass" output.txt && exit 1 +done diff --git a/dev/local/install.sh b/dev/local/install.sh new file mode 100644 index 000000000..be4ac54b5 --- /dev/null +++ b/dev/local/install.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +mkdir -p driver/repos/delphi driver-logs/delphi_database_epidata driver-logs/delphi_web_epidata +cd driver/repos/delphi +git clone https://github.com/cmu-delphi/operations +git clone https://github.com/cmu-delphi/delphi-epidata +git clone https://github.com/cmu-delphi/utils +cd ../../ +ln -s repos/delphi/delphi-epidata/dev/local/epidata-refresh.sh diff --git a/docs/epidata_development.md b/docs/epidata_development.md index 5264d0b7b..66e68fcba 100644 --- a/docs/epidata_development.md +++ b/docs/epidata_development.md @@ -5,6 +5,17 @@ nav_order: 4 # Epidata API Development Guide +## Quickstart + +``` +$ curl "https://raw.githubusercontent.com/cmu-delphi/delphi-epidata/dev/dev/local/install.sh" | bash +$ cd driver +$ [sudo] ./epidata-refresh.sh database web python +``` +(sudo requirement depends on your Docker installation and operating system) + +## Long version + **Prerequisite:** this guide assumes that you have read the [frontend development guide](https://github.com/cmu-delphi/operations/blob/main/docs/frontend_development.md).