Merge branch 'api-keys' into api_key_v2

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.6
+current_version = 0.4.7
 commit = False
 tag = False
 

.github/workflows/ci.yaml

@@ -58,11 +58,13 @@ jobs:
           docker build -t delphi_web_epidata -f ./devops/Dockerfile .
           cd ../../../
 
+      # MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
+      # see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
       - name: Start services
         run: |
           docker network create --driver bridge delphi-net
           docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
-          docker run --rm -d -p 10080:80 --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "RATELIMIT_STORAGE_URL=redis://delphi_redis_instance:6379" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --network delphi-net --name delphi_web_epidata delphi_web_epidata
+          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "RATELIMIT_STORAGE_URL=redis://delphi_redis_instance:6379" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --network delphi-net --name delphi_web_epidata delphi_web_epidata
           docker run --rm -p 6379:6379 --network delphi-net --name delphi_redis_instance delphi_redis_instance
           docker ps
 
@@ -108,7 +110,8 @@ jobs:
   image:
     needs: build
     # only on main and dev branch
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
+    # TODO: #1112 Remove `|| github.ref == 'refs/heads/api-keys'` after transition to production status.
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/api-keys'
 
     runs-on: ubuntu-latest
     steps:

.github/workflows/performance-tests.yml

@@ -15,6 +15,14 @@ jobs:
     # Make sure 1. this is a PR, not an issue 2. it contains "/run performance test" anywhere in the body
     if: github.event.issue.pull_request && contains(github.event.comment.body, '/run performance test')
     runs-on: ubuntu-latest
+    outputs:
+      request_count: ${{ steps.output.outputs.request_count }}
+      failure_count: ${{ steps.output.outputs.failure_count }}
+      med_time: ${{ steps.output.outputs.med_time }}
+      avg_time: ${{ steps.output.outputs.avg_time }}
+      min_time: ${{ steps.output.outputs.min_time }}
+      max_time: ${{ steps.output.outputs.max_time }}
+      requests_per_sec: ${{ steps.output.outputs.requests_per_sec }}
     steps:
       - name: Set up WireGuard
         uses: egor-tensin/[email protected]
@@ -52,19 +60,60 @@ jobs:
         run: |
           cd ../driver
           sudo make web sql="${{ secrets.DB_CONN_STRING }}"
+      - name: Check out delphi-admin
+        uses: actions/checkout@v3
+        with:
+          repository: cmu-delphi/delphi-admin
+          token: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
+          path: delphi-admin
       - name: Build & run Locust
+        continue-on-error: true # sometimes ~2-5 queries fail, we shouldn't end the run if that's the case
         run: |
-          cd ../driver/repos/delphi/delphi-epidata/tests/performance
+          cd delphi-admin/load-testing/locust
           docker build -t locust .
-          docker run --net=host -v $PWD:/mnt/locust -e CSV=/mnt/locust/v4-requests-smaller.csv locust -f /mnt/locust/v4.py --host http://127.0.0.1:10080/ --users 10 --spawn-rate 1 --headless -t 15m
+          export CSV=v4-requests-as_of.csv
+          touch output_stats.csv && chmod 666 output_stats.csv
+          touch output_stats_history.csv && chmod 666 output_stats_history.csv
+          touch output_failures.csv && chmod 666 output_failures.csv
+          touch output_exceptions.csv && chmod 666 output_exceptions.csv
+          docker run --net=host -v $PWD:/mnt/locust -e CSV="/mnt/locust/${CSV}" locust -f /mnt/locust/v4.py --host http://127.0.0.1:10080/ --users 10 --spawn-rate 1 --headless -i "$(cat ${CSV} | wc -l)" --csv=/mnt/locust/output
+      - name: Produce output for summary
+        id: output
+        uses: jannekem/run-python-script-action@v1
+        with:
+          script: |
+            import os
+
+            def write_string(name, value):
+              with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+                print(f'{name}={value}', file=fh)
+
+            def write_float(name, value):
+              write_string(name, "{:.2f}".format(float(value)))
 
-  comment-output:
+            with open("delphi-admin/load-testing/locust/output_stats.csv", "r", encoding="utf-8", errors="ignore") as scraped:
+              final_line = scraped.readlines()[-1].split(",")
+              write_string('request_count', final_line[2])
+              write_string('failure_count', final_line[3])
+              write_float('med_time', final_line[4])
+              write_float('avg_time', final_line[5])
+              write_float('min_time', final_line[6])
+              write_float('max_time', final_line[7])
+              write_float('requests_per_sec', final_line[9])
+
+      - name: Archive results as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: locust-output
+          path: |
+            delphi-admin/load-testing/locust/output_*.csv
+
+  comment-success:
     runs-on: ubuntu-latest
-    if: success() || failure() # but not if skipped
+    if: success()
     needs: run-perftests
     steps:
       - name: Comment run results
-        # URL that links to run results
         env:
           GITHUB_WORKFLOW_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
         uses: actions/github-script@v5
@@ -75,6 +124,33 @@ jobs:
               issue_number: context.issue.number,
               owner: context.repo.owner,
               repo: context.repo.repo,
-              body: '✅ Performance tests complete! Click here to view results: ${{ env.GITHUB_WORKFLOW_URL }}'
+              body: `✅ Performance tests complete! Result summary:
+              - Total requests: **${{ needs.run-perftests.outputs.request_count }}**
+              - Total failures: **${{ needs.run-perftests.outputs.failure_count }}**
+              - Min response time: **${{ needs.run-perftests.outputs.min_time }} ms**
+              - Max response time: **${{ needs.run-perftests.outputs.max_time }} ms**
+              - Average response time: **${{ needs.run-perftests.outputs.avg_time }} ms**
+              - Median response time: **${{ needs.run-perftests.outputs.med_time }} ms**
+              - Requests per second: **${{ needs.run-perftests.outputs.requests_per_sec }}**
+              
+              Click here to view full results: ${{ env.GITHUB_WORKFLOW_URL }}.`
             })
   
+  comment-failure:
+    runs-on: ubuntu-latest
+    if: failure()
+    needs: run-perftests
+    steps:
+      - name: Comment run results
+        env:
+          GITHUB_WORKFLOW_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        uses: actions/github-script@v5
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `❌ Performance tests failed! Click here to view full results: ${{ env.GITHUB_WORKFLOW_URL }}.`
+            })

deploy.json

@@ -17,6 +17,15 @@
       "add-header-comment": true
     },
 
+    "// common",
+    {
+      "type": "move",
+      "src": "src/common/",
+      "dst": "[[package]]/common/",
+      "match": "^.*\\.(py)$",
+      "add-header-comment": true
+    },
+
     "// server",
     {
       "type": "move",
@@ -25,6 +34,13 @@
       "match": "^.*\\.(py)$",
       "add-header-comment": true
     },
+    {
+      "type": "move",
+      "src": "src/server/utils",
+      "dst": "[[package]]/server/utils/",
+      "match": "^.*\\.(py)$",
+      "add-header-comment": true
+    },
     {
       "type": "move",
       "src": "src/server/endpoints",

dev/local/Makefile

@@ -91,8 +91,11 @@ web:
 		cd -
 
 	@# Run the web server
+	@# MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
+	@# see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
 	@docker run --rm -p 127.0.0.1:10080:80 \
 		$(M1) \
+		--env "MODULE_NAME=delphi.epidata.server.main" \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
 		--network delphi-net --name delphi_web_epidata \

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 0.4.6
+version = 0.4.7
 
 [options]
 packages =

devops/Dockerfile

@@ -8,14 +8,16 @@ LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epida
 
 COPY ./devops/gunicorn_conf.py /app
 COPY ./devops/start_wrapper.sh /
-COPY ./src/server/ /app/app/
-COPY --from=builder ./src/build/lib/ /app/app/lib/
+RUN mkdir -p /app/delphi/epidata
+COPY ./src/server /app/delphi/epidata/server
+COPY ./src/common /app/delphi/epidata/common
+COPY --from=builder ./src/build/lib/ /app/delphi/epidata/lib/
 
 COPY requirements.api.txt /app/requirements_also.txt
 
 RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime \
-      && rm -rf /app/app/__pycache__ /app/app/*.php \
-      && chmod -R o+r /app/app \
+      && rm -rf /app/delphi/epidata/__pycache__ \
+      && chmod -R o+r /app/delphi/epidata \
       && chmod 755 /start_wrapper.sh \
       && pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt
 # the file /tmp/requirements.txt is created in the parent docker definition.  (see:

docs/symptom-survey/publications.md

@@ -26,10 +26,14 @@ Pandemic"](https://www.pnas.org/topic/548) in *PNAS*:
 
 Research publications using the survey data include:
 
+- Taube JC, Susswein Z, Bansal S (2023). [Spatiotemporal Trends in Self-Reported
+  Mask-Wearing Behavior in the United States: Analysis of a Large
+  Cross-sectional Survey](https://doi.org/10.2196/42128). *JMIR Public Health
+  and Surveillance* 9:e42128.
 - Rebecca L. Weintraub et al (2023). [Identifying COVID-19 Vaccine Deserts and
   Ways to Reduce Them: A Digital Tool to Support Public Health
   Decision-Making](https://doi.org/10.2105/AJPH.2022.307198). *American Journal
-  of Public Health*.
+  of Public Health* 113 (4), 363-367.
 - Anzalone AJ, Sun J, Vinson AJ, Beasley WH, Hillegass WB, Murray K, et al.
   (2023). [Community risks for SARS-CoV-2 infection among fully vaccinated US
   adults by rurality: A retrospective cohort study from the National COVID
@@ -38,7 +42,7 @@ Research publications using the survey data include:
 - Rufino, J., Baquero, C., Frey, D. et al (2023). [Using survey data to estimate
   the impact of the omicron variant on vaccine efficacy against COVID-19
   infection](https://doi.org/10.1038/s41598-023-27951-3). *Scientific Reports*
-  13, 900 (2023).
+  13, 900.
 - Rader, B., Astley, C.M., Sewalk, K. et al (2022). [Spatial modeling of vaccine
   deserts as barriers to controlling
   SARS-CoV-2](https://doi.org/10.1038/s43856-022-00183-8). *Communications

integrations/acquisition/covid_hosp/facility/test_scenarios.py

@@ -39,6 +39,7 @@ def setUp(self):
     with Database.connect() as db:
       with db.new_cursor() as cur:
         cur.execute('truncate table covid_hosp_facility')
+        cur.execute('truncate table covid_hosp_facility_key')
         cur.execute('truncate table covid_hosp_meta')
         cur.execute('delete from api_user')
         cur.execute('insert into api_user(api_key, tracking, registered) values ("key", 1, 1)')

integrations/server/test_covidcast_endpoints.py

@@ -183,32 +183,6 @@ def match_row(trend, row):
             self.assertEqual(trend["max_value"], first.value)
             self.assertEqual(trend["max_trend"], "decreasing")
 
-    def test_correlation(self):
-        """Request a signal from the /correlation endpoint."""
-
-        num_rows = 30
-        reference_rows = [CovidcastTestRow.make_default_row(signal="ref", time_value=20200401 + i, value=i) for i in range(num_rows)]
-        first = reference_rows[0]
-        self._insert_rows(reference_rows)
-        other_rows = [CovidcastTestRow.make_default_row(signal="other", time_value=20200401 + i, value=i) for i in range(num_rows)]
-        other = other_rows[0]
-        self._insert_rows(other_rows)
-        max_lag = 3
-
-        out = self._fetch("/correlation", reference=first.signal_pair(), others=other.signal_pair(), geo=first.geo_pair(), window="20200401-20201212", lag=max_lag)
-        self.assertEqual(out["result"], 1)
-        df = pd.DataFrame(out["epidata"])
-        self.assertEqual(len(df), max_lag * 2 + 1)  # -...0...+
-        self.assertEqual(df["geo_type"].unique().tolist(), [first.geo_type])
-        self.assertEqual(df["geo_value"].unique().tolist(), [first.geo_value])
-        self.assertEqual(df["signal_source"].unique().tolist(), [other.source])
-        self.assertEqual(df["signal_signal"].unique().tolist(), [other.signal])
-
-        self.assertEqual(df["lag"].tolist(), list(range(-max_lag, max_lag + 1)))
-        self.assertEqual(df["r2"].unique().tolist(), [1.0])
-        self.assertEqual(df["slope"].unique().tolist(), [1.0])
-        self.assertEqual(df["intercept"].tolist(), [3.0, 2.0, 1.0, 0.0, -1.0, -2.0, -3.0])
-        self.assertEqual(df["samples"].tolist(), [num_rows - abs(l) for l in range(-max_lag, max_lag + 1)])
 
     def test_csv(self):
         """Request a signal from the /csv endpoint."""