diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 4b95ce86d..99c2373b6 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 4.1.2
+current_version = 4.1.3
commit = False
tag = False
diff --git a/dev/local/setup.cfg b/dev/local/setup.cfg
index 7dedcefe4..443359b25 100644
--- a/dev/local/setup.cfg
+++ b/dev/local/setup.cfg
@@ -1,6 +1,6 @@
[metadata]
name = Delphi Development
-version = 4.1.2
+version = 4.1.3
[options]
packages =
diff --git a/docs/api/api_keys.md b/docs/api/api_keys.md
index 80f74101c..21e875fe4 100644
--- a/docs/api/api_keys.md
+++ b/docs/api/api_keys.md
@@ -40,6 +40,14 @@ If you choose to
[register for an API key](https://api.delphi.cmu.edu/epidata/admin/registration_form),
there are several ways to use your key to authenticate your requests:
+### Using a client
+
+* covidcast
+ * [R client](https://cmu-delphi.github.io/covidcast/covidcastR/reference/covidcast_signal.html#api-keys-1)
+ * [Python client](https://cmu-delphi.github.io/covidcast/covidcast-py/html/signals.html#covidcast.use_api_key)
+* [epidatr](https://github.com/cmu-delphi/epidatr#api-keys)
+* [delphi-epidata](https://cmu-delphi.github.io/delphi-epidata/api/client_libraries.html)
+
### Via request parameter
The request parameter “api_key” can be used to pass the API key to the server.
diff --git a/docs/api/client_libraries.md b/docs/api/client_libraries.md
index 64618bbaf..8bea7f667 100644
--- a/docs/api/client_libraries.md
+++ b/docs/api/client_libraries.md
@@ -4,9 +4,17 @@ parent: Other Endpoints (COVID-19 and Other Diseases)
nav_order: 1
---
-# Epidata API Client Libraries.
+# Epidata API Client Libraries
-Epidata clients are available for
+For anyone looking for COVIDCast data, please visit our [COVIDCast Libraries](covidcast_clients.md).
+
+We are currently working on fully-featured Epidata clients for R and Python. They are not ready
+for release yet, but you can track our development progress and help us test them out at:
+
+* [epidatr](https://github.com/cmu-delphi/epidatr)
+* [epidatpy](https://github.com/cmu-delphi/epidatpy)
+
+In the meantime, minimalist Epidata clients remain available for
[JavaScript](https://github.com/cmu-delphi/delphi-epidata/blob/master/src/client/delphi_epidata.js),
[Python](https://github.com/cmu-delphi/delphi-epidata/blob/master/src/client/delphi_epidata.py),
and
@@ -15,10 +23,10 @@ The following samples show how to import the library and fetch Delphi's COVID-19
Surveillance Streams from Facebook Survey CLI for county 06001 and days
`20200401` and `20200405-20200414` (11 days total).
-For anyone looking for COVIDCast data, please visit our [COVIDCast Libraries](covidcast_clients.md).
-
### JavaScript (in a web browser)
+The minimalist JavaScript client does not currently support API keys. If you need API key support in JavaScript, contact delphi-support+privacy@andrew.cmu.edu.
+
````html
@@ -45,6 +53,8 @@ in the same directory as your Python script.
````python
# Import
from delphi_epidata import Epidata
+# [Optional] configure your API key, if desired
+#Epidata.auth = ('epidata', )
# Fetch data
res = Epidata.covidcast('fb-survey', 'smoothed_cli', 'day', 'county', [20200401, Epidata.range(20200405, 20200414)], '06001')
print(res['result'], res['message'], len(res['epidata']))
@@ -54,6 +64,8 @@ print(res['result'], res['message'], len(res['epidata']))
````R
+# [Optional] configure your API key, if desired
+#option('epidata.auth', )
# Import
source('delphi_epidata.R')
# Fetch data
diff --git a/docs/symptom-survey/publications.md b/docs/symptom-survey/publications.md
index 28d3dfea9..49c705351 100644
--- a/docs/symptom-survey/publications.md
+++ b/docs/symptom-survey/publications.md
@@ -26,15 +26,20 @@ Pandemic"](https://www.pnas.org/topic/548) in *PNAS*:
Research publications using the survey data include:
+- GK Charles, SP Braunstein, JL Barker, et al (2023). [How do psychobehavioural
+ variables shed light on heterogeneity in COVID-19 vaccine acceptance? Evidence
+ from United States general population surveys on a probability panel and
+ social media](https://doi.org/10.1136/bmjopen-2022-066897). *BMJ Open*
+ 13:e066897.
- S. Soorapanth, R. Cheung, X. Zhang, A. H. Mokdad, G. A. Mensah (2023).
[Rural–Urban Differences in Vaccination and Hesitancy Rates and Trust: US
COVID-19 Trends and Impact Survey on a Social Media Platform, May 2021–April
2022](https://doi.org/10.2105/AJPH.2023.307274). *American Journal of Public
- Health*.
+ Health* 113 (6), 680-688.
- See also the associated editorial: T. Callaghan (2023). [Vaccine Uptake and
Hesitancy in Rural America in the Wake of the COVID-19
Pandemic](https://doi.org/10.2105/AJPH.2023.307305). *American Journal of
- Public Health*.
+ Public Health* 113 (6), 615-617.
- M. Rubinstein, A. Haviland, and J. Breslau (2023). [The effect of COVID-19
vaccinations on self-reported depression and anxiety during February
2021](https://doi.org/10.1080/2330443X.2023.2190008). *Statistics and Public
diff --git a/src/client/delphi_epidata.R b/src/client/delphi_epidata.R
index 8f80cda4e..06e9c2209 100644
--- a/src/client/delphi_epidata.R
+++ b/src/client/delphi_epidata.R
@@ -15,7 +15,7 @@ Epidata <- (function() {
# API base url
BASE_URL <- getOption('epidata.url', default = 'https://api.delphi.cmu.edu/epidata/')
- client_version <- '4.1.2'
+ client_version <- '4.1.3'
auth <- getOption("epidata.auth", default = NA)
diff --git a/src/client/delphi_epidata.js b/src/client/delphi_epidata.js
index c23ec9919..cf06ae976 100644
--- a/src/client/delphi_epidata.js
+++ b/src/client/delphi_epidata.js
@@ -22,7 +22,7 @@
}
})(this, function (exports, fetchImpl, jQuery) {
const BASE_URL = "https://api.delphi.cmu.edu/epidata/";
- const client_version = "4.1.2";
+ const client_version = "4.1.3";
// Helper function to cast values and/or ranges to strings
function _listitem(value) {
diff --git a/src/client/packaging/npm/package.json b/src/client/packaging/npm/package.json
index b7c7d5f93..40c3d53a6 100644
--- a/src/client/packaging/npm/package.json
+++ b/src/client/packaging/npm/package.json
@@ -2,7 +2,7 @@
"name": "delphi_epidata",
"description": "Delphi Epidata API Client",
"authors": "Delphi Group",
- "version": "4.1.2",
+ "version": "4.1.3",
"license": "MIT",
"homepage": "https://github.com/cmu-delphi/delphi-epidata",
"bugs": {
diff --git a/src/client/packaging/pypi/delphi_epidata/__init__.py b/src/client/packaging/pypi/delphi_epidata/__init__.py
index 547775f8f..c8f19f67c 100644
--- a/src/client/packaging/pypi/delphi_epidata/__init__.py
+++ b/src/client/packaging/pypi/delphi_epidata/__init__.py
@@ -1,4 +1,4 @@
from .delphi_epidata import Epidata
name = 'delphi_epidata'
-__version__ = '4.1.2'
+__version__ = '4.1.3'
diff --git a/src/client/packaging/pypi/setup.py b/src/client/packaging/pypi/setup.py
index 570910316..e57e565b6 100644
--- a/src/client/packaging/pypi/setup.py
+++ b/src/client/packaging/pypi/setup.py
@@ -5,7 +5,7 @@
setuptools.setup(
name="delphi_epidata",
- version="4.1.2",
+ version="4.1.3",
author="David Farrow",
author_email="dfarrow0@gmail.com",
description="A programmatic interface to Delphi's Epidata API.",
diff --git a/src/server/_common.py b/src/server/_common.py
index 8322f2da6..56d4c38ec 100644
--- a/src/server/_common.py
+++ b/src/server/_common.py
@@ -8,7 +8,7 @@
from werkzeug.local import LocalProxy
from delphi.epidata.common.logger import get_structured_logger
-from ._config import SECRET, REVERSE_PROXIED
+from ._config import SECRET, REVERSE_PROXY_DEPTH
from ._db import engine
from ._exceptions import DatabaseErrorException, EpiDataException
from ._security import current_user, _is_public_route, resolve_auth_token, show_no_api_key_warning, update_key_last_time_used, ERROR_MSG_INVALID_KEY
@@ -31,14 +31,29 @@ def _get_db() -> Connection:
def get_real_ip_addr(req): # `req` should be a Flask.request object
- if REVERSE_PROXIED:
- # NOTE: ONLY trust these headers if reverse proxied!!!
+ if REVERSE_PROXY_DEPTH:
+ # we only expect/trust (up to) "REVERSE_PROXY_DEPTH" number of proxies between this server and the outside world.
+ # a REVERSE_PROXY_DEPTH of 0 means not proxied, i.e. server is globally directly reachable.
+ # a negative proxy depth is a special case to trust the whole chain -- not generally recommended unless the
+ # most-external proxy is configured to disregard "X-Forwarded-For" from outside.
+ # really, ONLY trust the following headers if reverse proxied!!!
if "X-Forwarded-For" in req.headers:
- return req.headers["X-Forwarded-For"].split(",")[
- 0
- ] # take the first (or only) address from the comma-sep list
+ full_proxy_chain = req.headers["X-Forwarded-For"].split(",")
+ # eliminate any extra addresses at the front of this list, as they could be spoofed.
+ if REVERSE_PROXY_DEPTH > 0:
+ depth = REVERSE_PROXY_DEPTH
+ else:
+ # special case for -1/negative: setting `depth` to 0 will not strip any items from the chain
+ depth = 0
+ trusted_proxy_chain = full_proxy_chain[-depth:]
+ # accept the first (or only) address in the remaining trusted part of the chain as the actual remote address
+ return trusted_proxy_chain[0].strip()
+
+ # fall back to "X-Real-Ip" if "X-Forwarded-For" isnt present
if "X-Real-Ip" in req.headers:
return req.headers["X-Real-Ip"]
+
+ # if we are not proxied (or we are proxied but the headers werent present and we fell through to here), just use the remote ip addr as the true client address
return req.remote_addr
diff --git a/src/server/_config.py b/src/server/_config.py
index dadeaa99c..168512a3d 100644
--- a/src/server/_config.py
+++ b/src/server/_config.py
@@ -7,7 +7,7 @@
load_dotenv()
-VERSION = "4.1.2"
+VERSION = "4.1.3"
MAX_RESULTS = int(10e6)
MAX_COMPATIBILITY_RESULTS = int(3650)
@@ -26,14 +26,41 @@
SECRET = os.environ.get("FLASK_SECRET", "secret")
URL_PREFIX = os.environ.get("FLASK_PREFIX", "") # if not empty, this value should begin but not end in a slash ('/')
-# REVERSE_PROXIED is a boolean value that indicates whether or not this server instance
-# is running behind a reverse proxy (like nginx).
-# in dev and testing, it is fine (or even preferable) for this variable to be set to 'TRUE'
-# even if it is not actually the case. in prod, it is very important that this is set accurately --
-# it should _only_ be set to 'TRUE' if it really is behind a reverse proxy, as remote addresses can be "spoofed"
-# which can carry security/identity implications. conversely, if this is set to 'FALSE' when in fact
-# running behind a reverse proxy, it can hinder logging accuracy. it defaults to 'FALSE' for safety.
-REVERSE_PROXIED = os.environ.get("REVERSE_PROXIED", "FALSE").upper() == "TRUE"
+
+"""
+REVERSE_PROXY_DEPTH is an integer value that indicates how many "chained" and trusted reverse proxies (like nginx) this
+ server instance is running behind. "chained" refers to proxies forwarding to other proxies, and then ultimately
+ forwarding to the app server itself. each of these proxies appends the remote address of the request to the
+ "X-Forwarded-For" header. in many situations, the most externally facing proxy (the first in the chain, the one that
+ faces the "open internet") can and should be set to write its own "X-Forwarded-For" header, ignoring and replacing
+ (or creating anew, if it didnt exist) such a header from the client request -- thus preserving the chain of trusted
+ proxies under our control.
+
+however, in our typical production environment, the most externally facing "proxy" is the AWS application load balancer,
+ which seemingly cannot be configured to provide this trust boundary without losing the referring client address
+ (see: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/x-forwarded-headers.html ). accordingly, in
+ our current typical production environment, REVERSE_PROXY_DEPTH should be set to "2": one for the AWS application load
+ balancer, and one for our own nginx/haproxy instance. thus "2" is our default value.
+
+it is important that REVERSE_PROXY_DEPTH is set accurately for two reasons...
+
+setting it too high (or to -1) will respect more of the entries in the "X-Forwarded-For" header than are appropriate.
+ this can allow remote addresses to be "spoofed" when a client fakes this header, carrying security/identity
+ implications. in dev and testing, it is not particularly dangerous for this variable to be set to -1 (special case
+ for an "infinite" depth, where any and all proxy hops will be trusted).
+
+setting it too low can hinder logging accuracy -- that can cause an intermediate proxy IP address to be used as the
+ "real" client IP address, which could cause requests to be rate-limited inappropriately.
+
+setting REVERSE_PROXY_DEPTH to "0" essentially indicates there are no proxies between this server and the outside
+ world. in this case, the "X-Forwarded-For" header is ignored.
+"""
+REVERSE_PROXY_DEPTH = int(os.environ.get("PROXY_DEPTH", 4))
+# TODO: ^ this value should be "4" for the prod CC API server processes, and is currently unclear
+# for prod AWS API server processes (but should be the same or lower)... when thats properly
+# determined, set the default to the minimum of the two environments and special case the
+# other in conf file(s).
+
REGION_TO_STATE = {
"hhs1": ["VT", "CT", "ME", "MA", "NH", "RI"],
diff --git a/src/server/admin/models.py b/src/server/admin/models.py
index 3755a517b..62cbc186d 100644
--- a/src/server/admin/models.py
+++ b/src/server/admin/models.py
@@ -19,6 +19,8 @@
Column("role_id", ForeignKey("user_role.id")),
)
+def _default_date_now():
+ return dtime.strftime(dtime.now(), "%Y-%m-%d")
class User(Base):
__tablename__ = "api_user"
@@ -26,8 +28,8 @@ class User(Base):
roles = relationship("UserRole", secondary=association_table)
api_key = Column(String(50), unique=True, nullable=False)
email = Column(String(320), unique=True, nullable=False)
- created = Column(Date, default=dtime.strftime(dtime.now(), "%Y-%m-%d"))
- last_time_used = Column(Date, default=dtime.strftime(dtime.now(), "%Y-%m-%d"))
+ created = Column(Date, default=_default_date_now)
+ last_time_used = Column(Date, default=_default_date_now)
def __init__(self, api_key: str, email: str = None) -> None:
self.api_key = api_key
diff --git a/src/server/endpoints/admin.py b/src/server/endpoints/admin.py
index d352e473b..17bc9ca9b 100644
--- a/src/server/endpoints/admin.py
+++ b/src/server/endpoints/admin.py
@@ -5,6 +5,7 @@
from werkzeug.exceptions import NotFound, Unauthorized
from werkzeug.utils import redirect
+from .._common import log_info_with_request
from .._config import ADMIN_PASSWORD, API_KEY_REGISTRATION_FORM_LINK, API_KEY_REMOVAL_REQUEST_LINK, REGISTER_WEBHOOK_TOKEN
from .._security import resolve_auth_token
from ..admin.models import User, UserRole
@@ -44,6 +45,24 @@ def user_exists(user_email: str = None, api_key: str = None):
return True if user else False
+# ~~~~ PUBLIC ROUTES ~~~~
+
+
+@bp.route("/registration_form", methods=["GET"])
+def registration_form_redirect():
+ # TODO: replace this with our own hosted registration form instead of external
+ return redirect(API_KEY_REGISTRATION_FORM_LINK, code=302)
+
+
+@bp.route("/removal_request", methods=["GET"])
+def removal_request_redirect():
+ # TODO: replace this with our own hosted form instead of external
+ return redirect(API_KEY_REMOVAL_REQUEST_LINK, code=302)
+
+
+# ~~~~ PRIVLEGED ROUTES ~~~~
+
+
@bp.route("/", methods=["GET", "POST"])
def _index():
token = _require_admin()
@@ -88,21 +107,6 @@ def _detail(user_id: int):
return _render("detail", token, flags, user=user.as_dict)
-def register_new_key(api_key: str, email: str) -> str:
- User.create_user(api_key=api_key, email=email)
- return api_key
-
-
-@bp.route("/registration_form", methods=["GET"])
-def registration_form_redirect():
- # TODO: replace this with our own hosted registration form instead of external
- return redirect(API_KEY_REGISTRATION_FORM_LINK, code=302)
-
-@bp.route("/removal_request", methods=["GET"])
-def removal_request_redirect():
- # TODO: replace this with our own hosted form instead of external
- return redirect(API_KEY_REMOVAL_REQUEST_LINK, code=302)
-
@bp.route("/register", methods=["POST"])
def _register():
body = request.get_json()
@@ -117,5 +121,16 @@ def _register():
"User with email and/or API Key already exists, use different parameters or contact us for help",
409,
)
- api_key = register_new_key(user_api_key, user_email)
- return make_response(f"Successfully registered API key '{api_key}'", 200)
+ User.create_user(api_key=user_api_key, email=user_email)
+ return make_response(f"Successfully registered API key '{user_api_key}'", 200)
+
+
+@bp.route("/diagnostics", methods=["GET", "PUT", "POST", "DELETE"])
+def diags():
+ # allows us to get useful diagnostic information written into server logs,
+ # such as a full current "X-Forwarded-For" path as inserted into headers by intermediate proxies...
+ # (but only when initiated purposefully by us to keep junk out of the logs)
+ _require_admin()
+ log_info_with_request("diagnostics", headers=request.headers)
+ response_text = f"request path: {request.headers.get('X-Forwarded-For', 'idk')}"
+ return make_response(response_text, 200, {'content-type': 'text/plain'})