Merge branch 'dev' into #848

Author: melange396

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.9
+current_version = 4.1.10
 commit = False
 tag = False
 

.dockerignore

@@ -1,6 +1,7 @@
 /delphi-epidata
-/.mypy_cache
+**/.mypy_cache
 /.github
 /docs
-__pycache__
-/node_modules
+**/__pycache__
+**/.pytest_cache
+**/node_modules

.editorconfig

@@ -0,0 +1,22 @@
+# EditorConfig helps developers define and maintain consistent
+# coding styles between different editors and IDEs
+# editorconfig.org
+
+root = true
+
+[*]
+# We recommend you to keep these unchanged
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+
+[*.py]
+# Change these settings to your own preference
+indent_style = space
+indent_size = 4
+
+
+[*.md]
+trim_trailing_whitespace = false

.env.example

@@ -1,18 +1,6 @@
 FLASK_DEBUG=True
 SQLALCHEMY_DATABASE_URI=sqlite:///test.db
 FLASK_SECRET=abc
-SECRET_TWITTER=abc
-SECRET_GHT=abc
-SECRET_FLUVIEW=abc
-SECRET_CDC=abc
-SECRET_SENSORS=abc
-SECRET_SENSOR_TWTR=abc
-SECRET_SENSOR_GFT=abc
-SECRET_SENSOR_GHT=abc
-SECRET_SENSOR_GHTJ=abc
-SECRET_SENSOR_CDC=abc
-SECRET_SENSOR_QUID=abc
-SECRET_SENSOR_WIKI=abc
-SECRET_QUIDEL=abc
-SECRET_NOROSTAT=abc
-SECRET_AFHSB=abc
+#API_KEY_REQUIRED_STARTING_AT=2021-07-30
+API_KEY_ADMIN_PASSWORD=abc
+API_KEY_REGISTER_WEBHOOK_TOKEN=abc

.git-blame-ignore-revs

@@ -0,0 +1,24 @@
+# style(black): format cdc acquisition
+980b0b7e80c7923b79e14fee620645e680785703
+# style(black): format covidcast_nowcast acquisition
+9e6ff16f599e8feec34a08dd1bddbc5eae347b55
+# style(black): format ecdc acquisition
+d1141d904da4e62992b97c92d5caebd8fadffd42
+# style(black): format flusurv acquisition
+08af0f6b7bff85bbc2b193b63b5abf6a16ba03e4
+# style(black): format fluview acquisition
+0133ef2042c4df8867e91595eb1f64873edb4632
+# style(black): format ght acquisition
+b8900a0bc846888885310911efd6e26459effa99
+# style(black): format kcdc acquisition
+a849384c884934b3b7c3c67b68aa6240277d6b6d
+# style(black): format nidss acquisition
+d04af3c02fda7708a16bec0952b1aa7475acaec7
+# style(black): format paho acquisition
+7f60fbba572c1b6e5153a9ef216895bdc2f7f5b3
+# style(black): format quidel acquisition
+b9ceb400d9248c8271e8342275664ac5524e335d
+# style(black): format twitter acquisition
+07ed83e5768f717ab0f9a62a9209e4e2cffa058d
+# style(black): format wiki acquisition
+923852eafa86b8f8b182d499489249ba8f815843

.github/workflows/ci.yaml

@@ -53,24 +53,31 @@ jobs:
         run: |
           docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
           docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
+          sudo docker build -t delphi_redis -f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .
           cd ./repos/delphi/delphi-epidata
           docker build -t delphi_web_epidata -f ./devops/Dockerfile .
           cd ../../../
 
       # MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
       # see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
-      - name: Start services
+      - name: Start database and Redis services
         run: |
           docker network create --driver bridge delphi-net
           docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
-          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
-          docker ps
+          docker run --rm -d -p 6379:6379 --network delphi-net --env "REDIS_PASSWORD=1234" --name delphi_redis delphi_redis
+          
 
       - run: |
           wget https://raw.githubusercontent.com/eficode/wait-for/master/wait-for
           chmod +x wait-for
           ./wait-for localhost:13306 -- echo 'ready'
           sleep 10s
+
+      - name: Start delphi_web_epidata
+        run: |
+          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --env "TESTING_MODE=True" --network delphi-net --name delphi_web_epidata delphi_web_epidata
+          docker ps
+
       - name: Run Unit Tests
         run: |
           docker run --rm --network delphi-net --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests
@@ -81,7 +88,7 @@ jobs:
 
       - name: Clean Up
         run: |
-          docker stop delphi_database_epidata delphi_web_epidata
+          docker stop delphi_database_epidata delphi_web_epidata delphi_redis
           docker network remove delphi-net
 
   build_js_client:
@@ -94,7 +101,7 @@ jobs:
         uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '14.x'
+          node-version: '16.x'
       - name: Cache Node.js modules
         uses: actions/cache@v2
         with:
@@ -108,7 +115,8 @@ jobs:
   image:
     needs: build
     # only on main and dev branch
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
+    # TODO: #1112 Remove `|| github.ref == 'refs/heads/api-keys'` after transition to production status.
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/api-keys'
 
     runs-on: ubuntu-latest
     steps:

.github/workflows/create-release.yml

@@ -39,7 +39,7 @@ jobs:
           base: main
           title: Release Delphi Epidata ${{ steps.version.outputs.next_tag }}
           labels: chore
-          reviewers: krivard
-          assignees: krivard
+          reviewers: melange396
+          assignees: melange396
           body: |
             Releasing Delphi Epidata ${{ steps.version.outputs.next_tag }}.

.github/workflows/dependabot-assignments.yml

@@ -0,0 +1,17 @@
+name: Dependabot auto-assign reviewer
+on: pull_request
+
+permissions:
+  pull-requests: write
+
+jobs:
+  dependabot:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.CMU_DELPHI_AUTOMATION_MACHINE_DEPENDABOT_PAT }}
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - name: Assign team to PR
+        run: gh pr edit "$PR_URL" --add-reviewer "cmu-delphi/code-reviewers"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}

.github/workflows/performance-tests.yml

@@ -14,7 +14,8 @@ jobs:
   run-perftests:
     # Make sure 1. this is a PR, not an issue 2. it contains "/run performance test" anywhere in the body
     if: github.event.issue.pull_request && contains(github.event.comment.body, '/run performance test')
-    runs-on: ubuntu-latest
+    # Run this on Delphi's self-hosted runner
+    runs-on: self-hosted
     outputs:
       request_count: ${{ steps.output.outputs.request_count }}
       failure_count: ${{ steps.output.outputs.failure_count }}
@@ -60,6 +61,7 @@ jobs:
         run: |
           cd ../driver
           sudo make web sql="${{ secrets.DB_CONN_STRING }}"
+          sudo make redis
       - name: Check out delphi-admin
         uses: actions/checkout@v3
         with:
@@ -71,7 +73,7 @@ jobs:
         run: |
           cd delphi-admin/load-testing/locust
           docker build -t locust .
-          export CSV=v4-requests-as_of.csv
+          export CSV=v4-requests-small.csv
           touch output_stats.csv && chmod 666 output_stats.csv
           touch output_stats_history.csv && chmod 666 output_stats_history.csv
           touch output_failures.csv && chmod 666 output_failures.csv

.github/workflows/release-helper.yml

@@ -95,7 +95,7 @@ jobs:
         uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '14.x'
+          node-version: '16.x'
       - name: Cache Node.js modules
         uses: actions/cache@v2
         with:
@@ -147,7 +147,7 @@ jobs:
           base: dev
           title: "chore: sync main->dev"
           labels: chore
-          reviewers: krivard
-          assignees: krivard
+          reviewers: melange396
+          assignees: melange396
           body: |
             Syncing Main->Dev.

.github/workflows/update_gdocs_data.yml

@@ -31,7 +31,7 @@ jobs:
           commit-message: 'chore: update docs'
           title: Update Google Docs Meta Data
           labels: chore
-          reviewers: krivard
-          assignees: krivard
+          reviewers: melange396
+          assignees: melange396
           body: |
             Updating Google Docs Meta Data

deploy.json

@@ -138,15 +138,6 @@
       "add-header-comment": true
     },
 
-    "// acquisition - norostat",
-    {
-      "type": "move",
-      "src": "src/acquisition/norostat/",
-      "dst": "[[package]]/acquisition/norostat/",
-      "match": "^.*\\.(py)$",
-      "add-header-comment": true
-    },
-
     "// acquisition - paho",
     {
       "type": "move",
@@ -174,15 +165,6 @@
       "add-header-comment": true
     },
 
-    "// acquisition - afhsb",
-    {
-      "type": "move",
-      "src": "src/acquisition/afhsb/",
-      "dst": "[[package]]/acquisition/afhsb/",
-      "match": "^.*\\.(py)$",
-      "add-header-comment": true
-    },
-
     "// acquisition - covidcast",
     {
       "type": "move",
@@ -235,6 +217,16 @@
       "add-header-comment": true
     },
 
+    "// maintenance",
+    {
+      "type": "move",
+      "src": "src/maintenance/",
+      "dst": "[[package]]/maintenance/",
+      "match": "^.*\\.(py)$",
+      "recursive": true,
+      "add-header-comment": true
+    },
+
     "// move flask tests out of the way",
     {
       "type": "move",

dev/docker/python/Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.8-buster
 
+RUN apt-get update && apt-get install -y r-base && Rscript -e "install.packages(c('httr','xml2'))"
+
 WORKDIR /usr/src/app
 
 COPY repos repos

dev/docker/redis/Dockerfile

@@ -0,0 +1,5 @@
+FROM redis
+
+ENV REDIS_PASSWORD=$REDIS_PASSWORD
+
+CMD ["sh", "-c", "exec redis-server --requirepass \"$REDIS_PASSWORD\""]

dev/local/Makefile

@@ -64,8 +64,10 @@ CWD:=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 NOW:=$(shell date "+%Y-%m-%d")
 LOG_WEB:=delphi_web_epidata_$(NOW).log
 LOG_DB:=delphi_database_epidata_$(NOW).log
+LOG_REDIS:=delphi_redis_instance_$(NOW).log
 WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata')
 DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata')
+REDIS_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_redis')
 
 M1=
 ifeq ($(shell uname -smp), Darwin arm64 arm)
@@ -98,6 +100,11 @@ web:
 		--env "MODULE_NAME=delphi.epidata.server.main" \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
+		--env "REDIS_HOST=delphi_redis" \
+		--env "REDIS_PASSWORD=1234" \
+		--env "API_KEY_ADMIN_PASSWORD=test_admin_password" \
+		--env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \
+		--env "TESTING_MODE=True" \
 		--network delphi-net --name delphi_web_epidata \
 		delphi_web_epidata >$(LOG_WEB) 2>&1 &
 
@@ -136,8 +143,25 @@ py:
 		$(M1) \
 		-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
 
+.PHONY=redis
+redis:
+	@# Stop container if running
+	@if [ $(REDIS_CONTAINER_ID) ]; then\
+		docker stop $(REDIS_CONTAINER_ID);\
+	fi
+
+	@docker build -t delphi_redis \
+		$(M1) \
+		-f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .
+
+	@docker run --rm -d -p 127.0.0.1:6379:6379 \
+		$(M1) \
+		--network delphi-net \
+		--env "REDIS_PASSWORD=1234" \
+		--name delphi_redis delphi_redis >$(LOG_REDIS) 2>&1 &
+
 .PHONY=all
-all: db web py
+all: db web py redis
 
 .PHONY=test
 test:
@@ -149,6 +173,17 @@ test:
 		--env "FLASK_SECRET=abc" \
 		delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log
 
+.PHONY=r-test
+r-test:
+	@docker run -i --rm --network delphi-net \
+		$(M1) \
+		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
+		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
+		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
+		--env "FLASK_SECRET=abc" \
+		delphi_web_python Rscript repos/delphi/delphi-epidata/integrations/client/test_delphi_epidata.R | tee r-test_output_$(NOW).log
+
+
 .PHONY=bash
 bash:
 	@docker run -it --rm --network delphi-net \

dev/local/setup.cfg

@@ -1,12 +1,11 @@
 [metadata]
 name = Delphi Development
-version = 0.4.9
+version = 4.1.10
 
 [options]
 packages =
     delphi.epidata
     delphi.epidata.acquisition
-    delphi.epidata.acquisition.afhsb
     delphi.epidata.acquisition.cdcp
     delphi.epidata.acquisition.covid_hosp
     delphi.epidata.acquisition.covid_hosp.common
@@ -21,13 +20,14 @@ packages =
     delphi.epidata.acquisition.ght
     delphi.epidata.acquisition.kcdc
     delphi.epidata.acquisition.nidss
-    delphi.epidata.acquisition.norostat
     delphi.epidata.acquisition.paho
     delphi.epidata.acquisition.quidel
     delphi.epidata.acquisition.twtr
     delphi.epidata.acquisition.wiki
     delphi.epidata.client
     delphi.epidata.server
+    delphi.epidata.server.admin
+    delphi.epidata.server.admin.templates
     delphi.epidata.server.covidcast_issues_migration
     delphi.epidata.server.endpoints
     delphi.epidata.server.endpoints.covidcast_utils