diff --git a/dev/local/.dockerignore b/dev/local/.dockerignore new file mode 100644 index 000000000..ebfb131ea --- /dev/null +++ b/dev/local/.dockerignore @@ -0,0 +1,6 @@ +# Ignore everything by default +* +# Don't ignore repos dir +!repos +# Ignore everything to do with git +**/*.git diff --git a/dev/local/Makefile b/dev/local/Makefile new file mode 100644 index 000000000..52c9e98f0 --- /dev/null +++ b/dev/local/Makefile @@ -0,0 +1,140 @@ +# Docker control panel for delphi-epidata development. +# +# Usage: make [pdb=1] [test=] +# +# Assumes you have installed your environment using +# delphi-epidata/dev/local/install.sh. +# +# Checks for the delphi-net bridge and creates if it doesn't exist. +# +# Creates all prereq images (delphi_database, delphi_python) only if they don't +# exist. If you need to rebuild a prereq, you're probably doing something +# complicated, and can figure out the rebuild command on your own. +# +# +# Commands: +# +# web: Stops currently-running delphi_web_epidata instances, if any. +# Rebuilds delphi_web_epidata image. +# Runs image in the background and pipes stdout to a log file. +# +# db: Stops currently-running delphi_database_epidata instances, if any. +# Rebuilds delphi_database_epidata image. +# Runs image in the background and pipes stdout to a log file. +# Blocks until database is ready to receive connections. +# +# python: Rebuilds delphi_web_python image. You shouldn't need to do this +# often; only if you are installing a new environment, or have +# made changes to delphi-epidata/dev/docker/python/Dockerfile. +# +# all: Runs the commands 'web' 'db' and 'python'. +# +# test: Runs test and integrations in delphi-epidata. If test +# optional arg is provided, then only the tests in that subdir +# are run. +# +# clean: Cleans up dangling Docker images. +# +# +# Optional arguments: +# pdb=1 Drops you into debug mode upon test failure, if running tests. +# test= Only runs tests in the directories provided here, e.g. +# repos/delphi/delphi-epidata/tests/acquisition/covidcast + + +# Set optional argument defaults +ifdef pdb + override pdb=--pdb +else + pdb= +endif + +ifndef test + test=repos/delphi/delphi-epidata/tests repos/delphi/delphi-epidata/integrations +endif + +SHELL:=/bin/sh + +# Get the Makefile's absolute path: https://stackoverflow.com/a/324782/4784655 +# (if called from a symlink, the path is the location of the symlink) +CWD:=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +NOW:=$(shell date "+%Y-%m-%d") +LOG_WEB:=delphi_web_epidata_$(NOW).log +LOG_DB:=delphi_database_epidata_$(NOW).log +WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata') +DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata') + + +.PHONY=web +web: + @# Stop container if running + @if [ $(WEB_CONTAINER_ID) ]; then\ + docker stop $(WEB_CONTAINER_ID);\ + fi + + @# Setup virtual network if it doesn't exist + @docker network ls | grep delphi-net || docker network create --driver bridge delphi-net + + @# Build the web_epidata image + @cd repos/delphi/delphi-epidata;\ + docker build -t delphi_web_epidata -f ./devops/Dockerfile .;\ + cd ../../../ + + @# Run the web server + @docker run --rm -p 127.0.0.1:10080:80 \ + --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \ + --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" \ + --network delphi-net --name delphi_web_epidata \ + delphi_web_epidata >$(LOG_WEB) 2>&1 & + +.PHONY=db +db: + @# Stop container if running + @if [ $(DATABASE_CONTAINER_ID) ]; then\ + docker stop $(DATABASE_CONTAINER_ID);\ + fi + + @# Only build prereqs if we need them + @docker images delphi_database | grep delphi || \ + docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile . + + @# Build the database_epidata image + @docker build -t delphi_database_epidata \ + -f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile . + + @# Run the database + @docker run --rm -p 127.0.0.1:13306:3306 \ + --network delphi-net --name delphi_database_epidata \ + delphi_database_epidata >$(LOG_DB) 2>&1 & + + @# Block until DB is ready + @while true; do \ + sed -n '/Temporary server stopped/,/mysqld: ready for connections/p' $(LOG_DB) | grep "ready for connections" && break; \ + tail -1 $(LOG_DB); \ + sleep 1; \ + done + +.PHONY=py +py: + @# Build the python image + @docker build -t delphi_python \ + -f repos/delphi/operations/dev/docker/python/Dockerfile . + + @docker build -t delphi_web_python \ + -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile . + +.PHONY=all +all: web db py + +.PHONY=test +test: + @docker run -i --rm --network delphi-net \ + --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ + --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ + --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \ + --env "FLASK_SECRET=abc" \ + delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log + +.PHONY=clean +clean: + @docker images -f "dangling=true" -q | xargs docker rmi >/dev/null 2>&1 diff --git a/dev/local/epidata-refresh.sh b/dev/local/epidata-refresh.sh deleted file mode 100755 index 25feb04e0..000000000 --- a/dev/local/epidata-refresh.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash - -if [ -z "$1" ]; then - echo "USAGE:" - echo "$0 [database] [web] [python] [testdir|test.py ...]" - echo "Docker control panel for delphi-epidata development" - echo - echo " Assumes you have installed your environment using" - echo " delphi-epidata/dev/local/install.sh." - echo - echo " Cleans up dangling Docker images before starting." - echo - echo " Checks for the delphi-net bridge, and creates it if it doesn't exist." - echo - echo " Creates all prereq images (delphi_database, delphi_python) only if they don't" - echo " exist. If you need to rebuild a prereq, you're probably doing something" - echo " complicated, and can figure out the rebuild command on your own." - echo - echo " database: Stops currently-running delphi_database_epidata instances, if any." - echo " Rebuilds delphi_database_epidata image." - echo " Runs image in the background and pipes stdout to a log file." - echo " Blocks until database is ready to receive connections." - echo - echo " web: Stops currently-running delphi_web_epidata instances, if any." - echo " Rebuilds delphi_web_epidata image." - echo " Runs image in the background and pipes stdout to a log file." - echo - echo " python: Rebuilds delphi_web_python image. You shouldn't need to do this" - echo " often; only if you are installing a new environment, or have" - echo " made changes to delphi-epidata/dev/docker/python/Dockerfile." - echo - echo " Loops through the remaining arguments and runs them as pytest within the" - echo " delphi_web_python image. This uses bindmounts into the local filesystem," - echo " so any changes you have made to src/acquisition/, tests/, or integrations/" - echo " will automatically take effect without having to rebuild delphi_web_python." - echo " Saves all test output to output.txt for review, in case it excees your" - echo " terminal buffer. Halts on first failure and lists failed tests." - echo - echo " Always run this from the `driver` directory. Never put anything other than" - echo " code in the `driver` directory, since everything under the `driver` directory" - echo " will get folded into the Docker images created. If it takes more than 1m to" - echo " build Docker images, you have probably accidentally stored other data in the" - echo " `driver` directory tree." - echo - echo " Depending on your operating system, you may need to be root or use sudo." - exit 0 -fi - -docker images -f "dangling=true" -q | xargs docker rmi >/dev/null 2>&1 -docker network ls | grep delphi-net || docker network create --driver bridge delphi-net - -LOGS=../driver-logs -NOW=`date "+%Y-%m-%d"` - -if [ "$1" == "database" ]; then - shift - LOGFILE=${LOGS}/delphi_database_epidata/${NOW}.log - docker ps | grep delphi_database_epidata && docker stop delphi_database_epidata - # only build prereqs if we need them - docker images delphi_database | grep delphi || \ - docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile . || exit 1 - docker build -t delphi_database_epidata -f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile . || exit 1 - docker run --rm -p 127.0.0.1:13306:3306 --network delphi-net --name delphi_database_epidata \ - --mount type=bind,source="$(pwd)"/repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ - delphi_database_epidata \ - >${LOGFILE} 2>&1 & - while true; do - sed -n '/Temporary server stopped/,/mysqld: ready for connections/p' ${LOGFILE} | grep "ready for connections" && break - tail -1 ${LOGFILE} - sleep 1 - done - grep ERROR ${LOGFILE} && exit 1 -fi - -if [ "$1" == "web" ]; then - shift - docker ps | grep delphi_web_epidata && docker stop delphi_web_epidata - cd repos/delphi/delphi-epidata && docker build -t delphi_web_epidata -f ./devops/Dockerfile . && cd - || exit 1 - docker run --rm -p 127.0.0.1:10080:80 \ - --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \ - --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" \ - --network delphi-net --name delphi_web_epidata delphi_web_epidata >>${LOGS}/delphi_web_epidata/${NOW}.log 2>&1 & -fi -if [ "$1" == "python" ]; then - shift - # only build prereqs if we need them - docker images delphi_python | grep delphi || \ - docker build -t delphi_python -f repos/delphi/operations/dev/docker/python/Dockerfile . || exit 1 - docker build -t delphi_web_python \ - -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile . || exit 1 -fi -rm -f output.txt -for t in $@; do - set -x - docker run --rm --network delphi-net \ - --mount type=bind,source="$(pwd)"/repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ - --mount type=bind,source="$(pwd)"/repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ - --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" \ - delphi_web_python \ - python -m pytest --import-mode importlib $t | tee -a output.txt - set +x - [[ $? = "0" ]] || exit 1 - grep -e "FAIL" -e "tests did not pass" output.txt && exit 1 -done diff --git a/dev/local/install.sh b/dev/local/install.sh index be4ac54b5..bfe097e94 100644 --- a/dev/local/install.sh +++ b/dev/local/install.sh @@ -1,9 +1,45 @@ #!/bin/bash +# Bootstrap delphi-epidata development +# +# Downloads the repos needed for local delphi-epidata development into current dir +# and provides a Makefile with Docker control commands. +# +# Creates the directory structure: +# +# driver/ +# .dockerignore +# Makefile +# repos/ +# delphi/ +# operations/ +# delphi-epidata/ +# utils/ +# flu-contest/ +# nowcast/ +# github-deploy-repo/ +# undefx/ +# py3tester/ +# undef-analysis/ +# +# Leaves you in driver, the main workdir. +# -mkdir -p driver/repos/delphi driver-logs/delphi_database_epidata driver-logs/delphi_web_epidata + +mkdir -p driver/repos/delphi cd driver/repos/delphi git clone https://github.com/cmu-delphi/operations git clone https://github.com/cmu-delphi/delphi-epidata git clone https://github.com/cmu-delphi/utils +git clone https://github.com/cmu-delphi/flu-contest +git clone https://github.com/cmu-delphi/nowcast +git clone https://github.com/cmu-delphi/github-deploy-repo +cd ../../ + +mkdir -p repos/undefx +cd repos/undefx +git clone https://github.com/undefx/py3tester +git clone https://github.com/undefx/undef-analysis cd ../../ -ln -s repos/delphi/delphi-epidata/dev/local/epidata-refresh.sh + +ln -s repos/delphi/delphi-epidata/dev/local/Makefile +ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore diff --git a/docs/epidata_development.md b/docs/epidata_development.md index 66e68fcba..bd02f5e20 100644 --- a/docs/epidata_development.md +++ b/docs/epidata_development.md @@ -7,10 +7,16 @@ nav_order: 4 ## Quickstart +In the directory where you want to work run the following + ``` $ curl "https://raw.githubusercontent.com/cmu-delphi/delphi-epidata/dev/dev/local/install.sh" | bash -$ cd driver -$ [sudo] ./epidata-refresh.sh database web python +$ [sudo] make all +$ [sudo] make test +# To drop into debugger on error +$ [sudo] make test pdb=1 +# To test only a subset of tests +$ [sudo] make test test=repos/delphi/delphi-epidata/integrations/acquisition ``` (sudo requirement depends on your Docker installation and operating system)