From 3994eef479eccd6d03c0a66f677b05ad338d635b Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Tue, 6 Sep 2022 16:42:44 -0400 Subject: [PATCH 1/7] Build an image from this branch --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9ed1cfde9..b0c98cc94 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -109,7 +109,7 @@ jobs: needs: build # only on main and dev branch #if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' # TODO: Revert after v4 migration is complete - if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/v4-schema-revisions-release-prep' + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/v4-schema-revisions-release-prep' || github.ref == 'refs/heads/v4-schema-revisions-release-prep-flaskalchemy' runs-on: ubuntu-latest steps: From 443c44b2d3df29b3a724bd5bc985412f22d3a8e0 Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Tue, 6 Sep 2022 16:43:13 -0400 Subject: [PATCH 2/7] Update to latest Flask and SQLAlchemy --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f94c98dc3..fc9641531 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ itsdangerous<2.1 jinja2==3.0.3 werkzeug<2.1 -Flask==1.1.2 -SQLAlchemy==1.3.22 +Flask==2.2.2 +SQLAlchemy==1.4.40 mysqlclient==2.0.2 python-dotenv==0.15.0 orjson==3.4.7 From 831e4b622f027c3cb3934e60c56f703438be6a46 Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Tue, 6 Sep 2022 16:49:44 -0400 Subject: [PATCH 3/7] Update werkzeug --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fc9641531..95f103acc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ itsdangerous<2.1 jinja2==3.0.3 -werkzeug<2.1 +werkzeug==2.2.2 Flask==2.2.2 SQLAlchemy==1.4.40 mysqlclient==2.0.2 From 6c72760c9d769269bc7b809fa0fa33251dae6a62 Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Tue, 6 Sep 2022 17:10:15 -0400 Subject: [PATCH 4/7] Use Row instead of RowProxy --- src/server/_query.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server/_query.py b/src/server/_query.py index bd1b37270..1cce5d2d5 100644 --- a/src/server/_query.py +++ b/src/server/_query.py @@ -14,7 +14,7 @@ ) from sqlalchemy import text -from sqlalchemy.engine import RowProxy +from sqlalchemy.engine import Row from ._common import db, app from ._db import metadata @@ -197,7 +197,7 @@ def filter_pair(pair: TimePair, i) -> str: def parse_row( - row: RowProxy, + row: Row, fields_string: Optional[Sequence[str]] = None, fields_int: Optional[Sequence[str]] = None, fields_float: Optional[Sequence[str]] = None, @@ -245,7 +245,7 @@ def run_query(p: APrinter, query_tuple: Tuple[str, Dict[str, Any]]): return db.execution_options(stream_results=True).execute(full_query, **params) -def _identity_transform(row: Dict[str, Any], _: RowProxy) -> Dict[str, Any]: +def _identity_transform(row: Dict[str, Any], _: Row) -> Dict[str, Any]: """ identity transform """ @@ -257,7 +257,7 @@ def execute_queries( fields_string: Sequence[str], fields_int: Sequence[str], fields_float: Sequence[str], - transform: Callable[[Dict[str, Any], RowProxy], Dict[str, Any]] = _identity_transform, + transform: Callable[[Dict[str, Any], Row], Dict[str, Any]] = _identity_transform, ): """ execute the given queries and return the response to send them @@ -317,7 +317,7 @@ def execute_query( fields_string: Sequence[str], fields_int: Sequence[str], fields_float: Sequence[str], - transform: Callable[[Dict[str, Any], RowProxy], Dict[str, Any]] = _identity_transform, + transform: Callable[[Dict[str, Any], Row], Dict[str, Any]] = _identity_transform, ): """ execute the given query and return the response to send it From 3d55cfa380c3de04345cc71e991330cff26a4bc3 Mon Sep 17 00:00:00 2001 From: george haff Date: Wed, 7 Sep 2022 14:25:04 -0400 Subject: [PATCH 5/7] fixes build for bumped versions of flask & etc. updated Dockerfile to let pip account for all version constraints simultaneously (previous non-simultaneous (ie, sequential) requirements installation resulted in a version conflict on `greenlets` package, see: https://github.com/tiangolo/meinheld-gunicorn-docker/issues/22 ). also bumped python version in that Dockerfile to 3.8 to match python versions elsewhere in the codebase. *note: this fix also requires a change to version pinning in a requirements.txt of the operations branch (see: https://github.com/cmu-delphi/operations/blob/v4-schema-revisions-release-prep-flaskalchemy/dev/docker/python/assets/requirements.txt ) --- devops/Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/devops/Dockerfile b/devops/Dockerfile index 7b56ec7be..602ba59bd 100644 --- a/devops/Dockerfile +++ b/devops/Dockerfile @@ -3,13 +3,16 @@ WORKDIR /src COPY . /src RUN npm ci && npm run build -FROM tiangolo/meinheld-gunicorn:python3.7 +FROM tiangolo/meinheld-gunicorn:python3.8 LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epidata # use delphi's timezome RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime -COPY requirements.txt /app -RUN pip install --no-cache-dir -r requirements.txt +COPY requirements.txt /app/requirements_also.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt +# the file /tmp/requirements.txt is created in the parent docker definition. (see: +# https://github.com/tiangolo/meinheld-gunicorn-docker/blob/master/docker-images/python3.8.dockerfile#L5 ) +# this combined requirements installation ensures all version constrants are accounted for. # disable python stdout buffering ENV PYTHONUNBUFFERED 1 From f06a7f0eb8fe79b3fc44a8adece15df5a85ccf27 Mon Sep 17 00:00:00 2001 From: george haff Date: Fri, 9 Sep 2022 10:55:17 -0400 Subject: [PATCH 6/7] make some sqlalchemy config options explicit (and not dependent on external specification) because they fundamentally change behaviors --- src/server/_config.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/server/_config.py b/src/server/_config.py index f14ed3bb3..8d6f3723b 100644 --- a/src/server/_config.py +++ b/src/server/_config.py @@ -11,7 +11,16 @@ MAX_COMPATIBILITY_RESULTS = int(3650) SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI", "sqlite:///test.db") -SQLALCHEMY_ENGINE_OPTIONS = json.loads(os.environ.get("SQLALCHEMY_ENGINE_OPTIONS", "{}")) + +# defaults +SQLALCHEMY_ENGINE_OPTIONS = { + "pool_pre_ping": True, # enable ping test for validity of recycled pool connections on connect() calls + "pool_recycle": 5 # seconds after which a recycled pool connection is considered invalid +} +# update with overrides of defaults or additions from external configs +SQLALCHEMY_ENGINE_OPTIONS.update( + json.loads(os.environ.get("SQLALCHEMY_ENGINE_OPTIONS", "{}"))) + SECRET = os.environ.get("FLASK_SECRET", "secret") URL_PREFIX = os.environ.get("FLASK_PREFIX", "/") From fd7b815960d2c027431f7650d85cb1861a779d78 Mon Sep 17 00:00:00 2001 From: george haff Date: Fri, 9 Sep 2022 10:58:59 -0400 Subject: [PATCH 7/7] remove reference to this branch from ci.yaml, in anticipation of merge and deletion of this branch --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b0c98cc94..9ed1cfde9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -109,7 +109,7 @@ jobs: needs: build # only on main and dev branch #if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' # TODO: Revert after v4 migration is complete - if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/v4-schema-revisions-release-prep' || github.ref == 'refs/heads/v4-schema-revisions-release-prep-flaskalchemy' + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/v4-schema-revisions-release-prep' runs-on: ubuntu-latest steps: