Skip to content

Commit 09013fa

Browse files
melange396melange396
authored
Merge branch 'v4-schema-revisions-release-prep' into ds/remove-wip-tests
2 parents 789a404 + a6d53f4 commit 09013fa

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+3205
-3179
lines changed

‎.bumpversion.cfg

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[bumpversion]
2-
current_version = 0.3.12
2+
current_version = 0.3.20
33
commit = False
44
tag = False
55

‎.github/workflows/ci.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ jobs:
6262
- name: Start services
6363
run: |
6464
docker network create --driver bridge delphi-net
65-
docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata delphi_database_epidata
65+
docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
6666
docker run --rm -d -p 10080:80 --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
6767
docker ps
6868

‎dev/docker/database/epidata/Dockerfile

+20-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
1-
# start with the `delphi_database` image
2-
FROM delphi_database
1+
# start with a standard percona mysql image
2+
FROM percona:ps-8
3+
4+
# percona exits with the mysql user but we need root for additional setup
5+
USER root
6+
7+
# use delphi's timezome
8+
RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime
9+
10+
# specify a development-only password for the database user "root"
11+
ENV MYSQL_ROOT_PASSWORD pass
312

413
# create the `epidata` database
514
ENV MYSQL_DATABASE epidata
@@ -8,8 +17,17 @@ ENV MYSQL_DATABASE epidata
817
ENV MYSQL_USER user
918
ENV MYSQL_PASSWORD pass
1019

20+
# provide DDL which will configure dev environment at container startup
21+
COPY repos/delphi/delphi-epidata/dev/docker/database/epidata/_init.sql /docker-entrypoint-initdb.d/
22+
1123
# provide DDL which will create empty tables at container startup
1224
COPY repos/delphi/delphi-epidata/src/ddl/*.sql /docker-entrypoint-initdb.d/
1325

26+
# provide additional configuration needed for percona
27+
COPY repos/delphi/delphi-epidata/dev/docker/database/mysql.d/*.cnf /etc/my.cnf.d/
28+
1429
# grant access to SQL scripts
1530
RUN chmod o+r /docker-entrypoint-initdb.d/*.sql
31+
32+
# restore mysql user for percona
33+
USER mysql

‎dev/docker/database/epidata/_init.sql

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
CREATE DATABASE covid;
2+
GRANT ALL ON covid.* TO 'user';

‎dev/docker/database/mysql.d/my.cnf

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[mysqld]
2+
default_authentication_plugin=mysql_native_password

‎dev/local/.dockerignore

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Ignore everything by default
2+
*
3+
# Don't ignore repos dir
4+
!repos
5+
# Ignore everything to do with git
6+
**/*.git

‎dev/local/Makefile

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# Docker control panel for delphi-epidata development.
2+
#
3+
# Usage: make <command> [pdb=1] [test=<test-subdir>]
4+
#
5+
# Assumes you have installed your environment using
6+
# delphi-epidata/dev/local/install.sh.
7+
#
8+
# Checks for the delphi-net bridge and creates if it doesn't exist.
9+
#
10+
# Creates all prereq images (delphi_database, delphi_python) only if they don't
11+
# exist. If you need to rebuild a prereq, you're probably doing something
12+
# complicated, and can figure out the rebuild command on your own.
13+
#
14+
#
15+
# Commands:
16+
#
17+
# web: Stops currently-running delphi_web_epidata instances, if any.
18+
# Rebuilds delphi_web_epidata image.
19+
# Runs image in the background and pipes stdout to a log file.
20+
#
21+
# db: Stops currently-running delphi_database_epidata instances, if any.
22+
# Rebuilds delphi_database_epidata image.
23+
# Runs image in the background and pipes stdout to a log file.
24+
# Blocks until database is ready to receive connections.
25+
#
26+
# python: Rebuilds delphi_web_python image. You shouldn't need to do this
27+
# often; only if you are installing a new environment, or have
28+
# made changes to delphi-epidata/dev/docker/python/Dockerfile.
29+
#
30+
# all: Runs the commands 'web' 'db' and 'python'.
31+
#
32+
# test: Runs test and integrations in delphi-epidata. If test
33+
# optional arg is provided, then only the tests in that subdir
34+
# are run.
35+
#
36+
# clean: Cleans up dangling Docker images.
37+
#
38+
#
39+
# Optional arguments:
40+
# pdb=1 Drops you into debug mode upon test failure, if running tests.
41+
# test= Only runs tests in the directories provided here, e.g.
42+
# repos/delphi/delphi-epidata/tests/acquisition/covidcast
43+
44+
45+
# Set optional argument defaults
46+
ifdef pdb
47+
override pdb=--pdb
48+
else
49+
pdb=
50+
endif
51+
52+
ifndef test
53+
test=repos/delphi/delphi-epidata/tests repos/delphi/delphi-epidata/integrations
54+
endif
55+
56+
SHELL:=/bin/sh
57+
58+
# Get the Makefile's absolute path: https://stackoverflow.com/a/324782/4784655
59+
# (if called from a symlink, the path is the location of the symlink)
60+
CWD:=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
61+
NOW:=$(shell date "+%Y-%m-%d")
62+
LOG_WEB:=delphi_web_epidata_$(NOW).log
63+
LOG_DB:=delphi_database_epidata_$(NOW).log
64+
WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata')
65+
DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata')
66+
67+
68+
.PHONY=web
69+
web:
70+
@# Stop container if running
71+
@if [ $(WEB_CONTAINER_ID) ]; then\
72+
docker stop $(WEB_CONTAINER_ID);\
73+
fi
74+
75+
@# Setup virtual network if it doesn't exist
76+
@docker network ls | grep delphi-net || docker network create --driver bridge delphi-net
77+
78+
@# Build the web_epidata image
79+
@cd repos/delphi/delphi-epidata;\
80+
docker build -t delphi_web_epidata -f ./devops/Dockerfile .;\
81+
cd ../../../
82+
83+
@# Run the web server
84+
@docker run --rm -p 127.0.0.1:10080:80 \
85+
--env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \
86+
--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" \
87+
--network delphi-net --name delphi_web_epidata \
88+
delphi_web_epidata >$(LOG_WEB) 2>&1 &
89+
90+
.PHONY=db
91+
db:
92+
@# Stop container if running
93+
@if [ $(DATABASE_CONTAINER_ID) ]; then\
94+
docker stop $(DATABASE_CONTAINER_ID);\
95+
fi
96+
97+
@# Only build prereqs if we need them
98+
@docker images delphi_database | grep delphi || \
99+
docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile .
100+
101+
@# Build the database_epidata image
102+
@docker build -t delphi_database_epidata \
103+
-f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
104+
105+
@# Run the database
106+
@docker run --rm -p 127.0.0.1:13306:3306 \
107+
--network delphi-net --name delphi_database_epidata \
108+
--cap-add=sys_nice \
109+
delphi_database_epidata >$(LOG_DB) 2>&1 &
110+
111+
@# Block until DB is ready
112+
@while true; do \
113+
sed -n '/Temporary server stopped/,/mysqld: ready for connections/p' $(LOG_DB) | grep "ready for connections" && break; \
114+
tail -1 $(LOG_DB); \
115+
sleep 1; \
116+
done
117+
118+
.PHONY=py
119+
py:
120+
@# Build the python image
121+
@docker build -t delphi_python \
122+
-f repos/delphi/operations/dev/docker/python/Dockerfile .
123+
124+
@docker build -t delphi_web_python \
125+
-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
126+
127+
.PHONY=all
128+
all: web db py
129+
130+
.PHONY=test
131+
test:
132+
@docker run -i --rm --network delphi-net \
133+
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
134+
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
135+
--env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \
136+
--env "FLASK_SECRET=abc" \
137+
delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log
138+
139+
.PHONY=clean
140+
clean:
141+
@docker images -f "dangling=true" -q | xargs docker rmi >/dev/null 2>&1

‎dev/local/install.sh

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/bin/bash
2+
# Bootstrap delphi-epidata development
3+
#
4+
# Downloads the repos needed for local delphi-epidata development into current dir
5+
# and provides a Makefile with Docker control commands.
6+
#
7+
# Creates the directory structure:
8+
#
9+
# driver/
10+
# .dockerignore
11+
# Makefile
12+
# repos/
13+
# delphi/
14+
# operations/
15+
# delphi-epidata/
16+
# utils/
17+
# flu-contest/
18+
# nowcast/
19+
# github-deploy-repo/
20+
# undefx/
21+
# py3tester/
22+
# undef-analysis/
23+
#
24+
# Leaves you in driver, the main workdir.
25+
#
26+
27+
28+
mkdir -p driver/repos/delphi
29+
cd driver/repos/delphi
30+
git clone https://github.com/cmu-delphi/operations
31+
git clone https://github.com/cmu-delphi/delphi-epidata
32+
git clone https://github.com/cmu-delphi/utils
33+
git clone https://github.com/cmu-delphi/flu-contest
34+
git clone https://github.com/cmu-delphi/nowcast
35+
git clone https://github.com/cmu-delphi/github-deploy-repo
36+
cd ../../
37+
38+
mkdir -p repos/undefx
39+
cd repos/undefx
40+
git clone https://github.com/undefx/py3tester
41+
git clone https://github.com/undefx/undef-analysis
42+
cd ../../
43+
44+
ln -s repos/delphi/delphi-epidata/dev/local/Makefile
45+
ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore

‎docs/Gemfile.lock

+2-2
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ GEM
211211
jekyll-seo-tag (~> 2.1)
212212
minitest (5.14.4)
213213
multipart-post (2.1.1)
214-
nokogiri (1.13.3)
214+
nokogiri (1.13.6)
215215
mini_portile2 (~> 2.8.0)
216216
racc (~> 1.4)
217217
octokit (4.20.0)
@@ -246,7 +246,7 @@ GEM
246246
thread_safe (0.3.6)
247247
typhoeus (1.4.0)
248248
ethon (>= 0.9.0)
249-
tzinfo (1.2.9)
249+
tzinfo (1.2.10)
250250
thread_safe (~> 0.1)
251251
tzinfo-data (1.2021.1)
252252
tzinfo (>= 1.0.0)

‎docs/api/covidcast-signals/dsew-cpr.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ grand_parent: COVIDcast Epidata API
1515
* **Time type:** day (see [date format docs](../covidcast_times.md))
1616
* **License:** [Public Domain US Government](https://www.usa.gov/government-works)
1717

18-
The Community Profile Report (CPR) is published by the Data Strategy and Execution Workgroup (DSEW) of the White House COVID-19 Team. For more information, see the [official description and data dictionary at healthdata.gov](https://healthdata.gov/Health/COVID-19-Community-Profile-Report/gqxm-d9w9) for "COVID-19 Community Profile Report".
18+
The Community Profile Report (CPR) is published by the Data Strategy and Execution Workgroup (DSEW) of the White House COVID-19 Team. For more information, see the [official description at healthdata.gov](https://healthdata.gov/Health/COVID-19-Community-Profile-Report/gqxm-d9w9) for "COVID-19 Community Profile Report". Each issue of the CPR is made available as an attachment on that page. You can view all attachments by scrolling to the bottom of the "About this dataset" panel and clicking "Show more".
1919

2020
This data source provides various COVID-19 related metrics, of which we report hospital admissions and vaccinations.
2121

0 commit comments

Comments
 (0)