API keys, local PR edition

.dockerignore

@@ -1,6 +1,7 @@
 /delphi-epidata
-/.mypy_cache
+**/.mypy_cache
 /.github
 /docs
-__pycache__
-/node_modules
+**/__pycache__
+**/.pytest_cache
+**/node_modules

.env.example

@@ -1,18 +1,8 @@
 FLASK_DEBUG=True
 SQLALCHEMY_DATABASE_URI=sqlite:///test.db
 FLASK_SECRET=abc
-SECRET_TWITTER=abc
-SECRET_GHT=abc
-SECRET_FLUVIEW=abc
-SECRET_CDC=abc
-SECRET_SENSORS=abc
-SECRET_SENSOR_TWTR=abc
-SECRET_SENSOR_GFT=abc
-SECRET_SENSOR_GHT=abc
-SECRET_SENSOR_GHTJ=abc
-SECRET_SENSOR_CDC=abc
-SECRET_SENSOR_QUID=abc
-SECRET_SENSOR_WIKI=abc
-SECRET_QUIDEL=abc
-SECRET_NOROSTAT=abc
-SECRET_AFHSB=abc
+#API_REQUIRED_STARTING_AT=2021-07-30
+API_KEY_ADMIN_PASSWORD=abc
+API_KEY_REGISTER_WEBHOOK_TOKEN=abc
+RECAPTCHA_SITE_KEY
+RECAPTCHA_SECRET_KEY

.github/workflows/ci.yaml

@@ -53,24 +53,31 @@ jobs:
         run: |
           docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
           docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
+          sudo docker build -t delphi_redis -f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .
           cd ./repos/delphi/delphi-epidata
           docker build -t delphi_web_epidata -f ./devops/Dockerfile .
           cd ../../../
 
       # MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
       # see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
-      - name: Start services
+      - name: Start database and Redis services
         run: |
           docker network create --driver bridge delphi-net
           docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
-          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
-          docker ps
+          docker run --rm -d -p 6379:6379 --network delphi-net --env "REDIS_PASSWORD=1234" --name delphi_redis delphi_redis
+
 
       - run: |
           wget https://raw.githubusercontent.com/eficode/wait-for/master/wait-for
           chmod +x wait-for
           ./wait-for localhost:13306 -- echo 'ready'
           sleep 10s
+
+      - name: Start delphi_web_epidata
+        run: |
+          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --network delphi-net --name delphi_web_epidata delphi_web_epidata
+          docker ps
+
       - name: Run Unit Tests
         run: |
           docker run --rm --network delphi-net --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests
@@ -81,7 +88,7 @@ jobs:
 
       - name: Clean Up
         run: |
-          docker stop delphi_database_epidata delphi_web_epidata
+          docker stop delphi_database_epidata delphi_web_epidata delphi_redis
           docker network remove delphi-net
 
   build_js_client:
@@ -108,7 +115,8 @@ jobs:
   image:
     needs: build
     # only on main and dev branch
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
+    # TODO: #1112 Remove `|| github.ref == 'refs/heads/api-keys'` after transition to production status.
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/api-keys'
 
     runs-on: ubuntu-latest
     steps:

dev/docker/redis/Dockerfile

@@ -0,0 +1,5 @@
+FROM redis
+
+ENV REDIS_PASSWORD=$REDIS_PASSWORD
+
+CMD ["sh", "-c", "exec redis-server --requirepass \"$REDIS_PASSWORD\""]

dev/local/Makefile

@@ -64,8 +64,10 @@ CWD:=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 NOW:=$(shell date "+%Y-%m-%d")
 LOG_WEB:=delphi_web_epidata_$(NOW).log
 LOG_DB:=delphi_database_epidata_$(NOW).log
+LOG_REDIS:=delphi_redis_instance_$(NOW).log
 WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata')
 DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata')
+REDIS_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_redis')
 
 M1=
 ifeq ($(shell uname -smp), Darwin arm64 arm)
@@ -98,6 +100,10 @@ web:
 		--env "MODULE_NAME=delphi.epidata.server.main" \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
+		--env "REDIS_HOST=delphi_redis" \
+		--env "REDIS_PASSWORD=1234" \
+		--env "API_KEY_ADMIN_PASSWORD=test_admin_password" \
+		--env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \
 		--network delphi-net --name delphi_web_epidata \
 		delphi_web_epidata >$(LOG_WEB) 2>&1 &
 
@@ -136,8 +142,25 @@ py:
 		$(M1) \
 		-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
 
+.PHONY=redis
+redis:
+	@# Stop container if running
+	@if [ $(REDIS_CONTAINER_ID) ]; then\
+		docker stop $(REDIS_CONTAINER_ID);\
+	fi
+
+	@docker build -t delphi_redis \
+		$(M1) \
+		-f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .
+
+	@docker run --rm -d -p 127.0.0.1:6379:6379 \
+		$(M1) \
+		--network delphi-net \
+		--env "REDIS_PASSWORD=1234" \
+		--name delphi_redis delphi_redis >$(LOG_REDIS) 2>&1 &
+
 .PHONY=all
-all: db web py
+all: db web py redis
 
 .PHONY=test
 test:

dev/local/setup.cfg

@@ -28,6 +28,8 @@ packages =
     delphi.epidata.acquisition.wiki
     delphi.epidata.client
     delphi.epidata.server
+    delphi.epidata.server.admin
+    delphi.epidata.server.admin.templates
     delphi.epidata.server.covidcast_issues_migration
     delphi.epidata.server.endpoints
     delphi.epidata.server.endpoints.covidcast_utils

integrations/acquisition/covid_hosp/facility/test_scenarios.py

@@ -29,6 +29,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'
@@ -40,6 +41,8 @@ def setUp(self):
         cur.execute('truncate table covid_hosp_facility')
         cur.execute('truncate table covid_hosp_facility_key')
         cur.execute('truncate table covid_hosp_meta')
+        cur.execute('delete from api_user')
+        cur.execute('insert into api_user(api_key, email) values ("key", "emai")')
 
   @freeze_time("2021-03-16")
   def test_acquire_dataset(self):

integrations/acquisition/covid_hosp/state_daily/test_scenarios.py

@@ -33,6 +33,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'
@@ -43,6 +44,8 @@ def setUp(self):
       with db.new_cursor() as cur:
         cur.execute('truncate table covid_hosp_state_timeseries')
         cur.execute('truncate table covid_hosp_meta')
+        cur.execute('delete from api_user')
+        cur.execute('insert into api_user(api_key, email) values("key", "email")')
 
   @freeze_time("2021-03-16")
   def test_acquire_dataset(self):

integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py

@@ -29,6 +29,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'
@@ -39,6 +40,8 @@ def setUp(self):
       with db.new_cursor() as cur:
         cur.execute('truncate table covid_hosp_state_timeseries')
         cur.execute('truncate table covid_hosp_meta')
+        cur.execute('delete from api_user')
+        cur.execute('insert into api_user(api_key, email) values ("key", "email")')
 
   @freeze_time("2021-03-17")
   def test_acquire_dataset(self):

integrations/acquisition/covidcast/test_covidcast_meta_caching.py

@@ -60,12 +60,20 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = BASE_URL
+    Epidata.auth = ('epidata', 'key')
 
   def tearDown(self):
     """Perform per-test teardown."""
     self.cur.close()
     self.cnx.close()
 
+  @staticmethod
+  def _make_request():
+    params = {'endpoint': 'covidcast_meta', 'cached': 'true'}
+    response = requests.get(Epidata.BASE_URL, params=params, auth=Epidata.auth)
+    response.raise_for_status()
+    return response.json()
+
   def test_caching(self):
     """Populate, query, cache, query, and verify the cache."""
 
@@ -147,10 +155,7 @@ def test_caching(self):
     self.cnx.commit()
 
     # fetch the cached version (manually)
-    params = {'endpoint': 'covidcast_meta', 'cached': 'true'}
-    response = requests.get(BASE_URL, params=params)
-    response.raise_for_status()
-    epidata4 = response.json()
+    epidata4 = self._make_request()
 
     # make sure the cache was actually served
     self.assertEqual(epidata4, {
@@ -170,10 +175,7 @@ def test_caching(self):
     self.cnx.commit()
 
     # fetch the cached version (manually)
-    params = {'endpoint': 'covidcast_meta', 'cached': 'true'}
-    response = requests.get(BASE_URL, params=params)
-    response.raise_for_status()
-    epidata5 = response.json()
+    epidata5 = self._make_request()
 
     # make sure the cache was returned anyhow
     self.assertEqual(epidata4, epidata5)

integrations/acquisition/covidcast/test_csv_uploading.py

@@ -57,6 +57,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
   def tearDown(self):
     """Perform per-test teardown."""

integrations/acquisition/covidcast_nowcast/test_csv_uploading.py

@@ -41,6 +41,8 @@ def setUp(self):
         database='epidata')
     cur = cnx.cursor()
     cur.execute('truncate table covidcast_nowcast')
+    cur.execute('delete from api_user')
+    cur.execute('insert into api_user(api_key, email) values ("key", "email")')
     cnx.commit()
     cur.close()
 
@@ -54,6 +56,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
   def tearDown(self):
     """Perform per-test teardown."""

integrations/client/test_delphi_epidata.py

@@ -41,6 +41,7 @@ def localSetUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'

integrations/client/test_nowcast.py

@@ -28,6 +28,8 @@ def setUp(self):
     cur = cnx.cursor()
 
     cur.execute('truncate table covidcast_nowcast')
+    cur.execute('delete from api_user')
+    cur.execute('insert into api_user(api_key, email) values ("key", "email")')
 
     cnx.commit()
     cur.close()
@@ -38,6 +40,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'

integrations/server/test_covid_hosp.py

@@ -17,6 +17,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'
@@ -27,6 +28,8 @@ def setUp(self):
       with db.new_cursor() as cur:
         cur.execute('truncate table covid_hosp_state_timeseries')
         cur.execute('truncate table covid_hosp_meta')
+        cur.execute('delete from api_user')
+        cur.execute('insert into api_user(api_key, email) values ("key", "email")')
 
 
   def insert_issue(self, cur, issue, value, record_type):

integrations/server/test_covidcast.py

@@ -6,15 +6,12 @@
 
 # third party
 import mysql.connector
-import requests
 
 # first party
 from delphi_utils import Nans
 from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow
 from delphi.epidata.client.delphi_epidata import Epidata
 
-# use the local instance of the Epidata API
-BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
 
 class CovidcastTests(CovidcastBase):
   """Tests the `covidcast` endpoint."""
@@ -25,7 +22,9 @@ def localSetUp(self):
 
   def request_based_on_row(self, row: CovidcastTestRow, **kwargs):
     params = self.params_from_row(row, endpoint='covidcast', **kwargs)
-    Epidata.BASE_URL = BASE_URL
+    # use the local instance of the Epidata API
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
     response = Epidata.covidcast(**params) 
 
     return response

integrations/server/test_covidcast_endpoints.py

@@ -15,6 +15,7 @@
 # use the local instance of the Epidata API
 BASE_URL = "http://delphi_web_epidata/epidata/covidcast"
 BASE_URL_OLD = "http://delphi_web_epidata/epidata/api.php"
+AUTH = ('epidata', 'key')
 
 
 class CovidcastEndpointTests(CovidcastBase):
@@ -36,7 +37,7 @@ def _fetch(self, endpoint="/", is_compatibility=False, **params):
                 params.setdefault("data_source", params.get("source"))
         else:
             url = f"{BASE_URL}{endpoint}"
-        response = requests.get(url, params=params)
+        response = requests.get(url, params=params, auth=AUTH)
         response.raise_for_status()
         return response.json()