Add support for src/common directory

Author: rzats

.github/workflows/ci.yaml

@@ -61,7 +61,7 @@ jobs:
         run: |
           docker network create --driver bridge delphi-net
           docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
-          docker run --rm -d -p 10080:80 --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
+          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
           docker ps
 
       - run: |

deploy.json

@@ -17,6 +17,15 @@
       "add-header-comment": true
     },
 
+    "// common",
+    {
+      "type": "move",
+      "src": "src/common/",
+      "dst": "[[package]]/common/",
+      "match": "^.*\\.(py)$",
+      "add-header-comment": true
+    },
+
     "// server",
     {
       "type": "move",

dev/local/Makefile

@@ -93,6 +93,7 @@ web:
 	@# Run the web server
 	@docker run --rm -p 127.0.0.1:10080:80 \
 		$(M1) \
+		--env "MODULE_NAME=delphi.epidata.server.main" \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
 		--network delphi-net --name delphi_web_epidata \

devops/Dockerfile

@@ -8,14 +8,16 @@ LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epida
 
 COPY ./devops/gunicorn_conf.py /app
 COPY ./devops/start_wrapper.sh /
-COPY ./src/server/ /app/app/
-COPY --from=builder ./src/build/lib/ /app/app/lib/
+RUN mkdir -p /app/delphi/epidata
+COPY ./src/server /app/delphi/epidata/server
+COPY ./src/common /app/delphi/epidata/common
+COPY --from=builder ./src/build/lib/ /app/delphi/epidata/lib/
 
 COPY requirements.api.txt /app/requirements_also.txt
 
 RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime \
-      && rm -rf /app/app/__pycache__ /app/app/*.php \
-      && chmod -R o+r /app/app \
+      && rm -rf /app/delphi/epidata/__pycache__ /app/delphi/epidata/*.php \
+      && chmod -R o+r /app/delphi/epidata \
       && chmod 755 /start_wrapper.sh \
       && pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt
 # the file /tmp/requirements.txt is created in the parent docker definition.  (see:

src/acquisition/covid_hosp/common/database.py

@@ -11,7 +11,7 @@
 
 # first party
 import delphi.operations.secrets as secrets
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 Columndef = namedtuple("Columndef", "csv_name sql_name dtype")
 

src/acquisition/covidcast/covidcast_meta_cache_updater.py

@@ -7,7 +7,7 @@
 
 # first party
 from delphi.epidata.acquisition.covidcast.database import Database
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 from delphi.epidata.client.delphi_epidata import Epidata
 
 def get_argument_parser():

src/acquisition/covidcast/csv_importer.py

@@ -16,7 +16,7 @@
 from delphi_utils import Nans
 from delphi.utils.epiweek import delta_epiweeks
 from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 DataFrameRow = NamedTuple('DFRow', [
   ('geo_id', str),

src/acquisition/covidcast/csv_to_database.py

@@ -11,7 +11,7 @@
 from delphi.epidata.acquisition.covidcast.csv_importer import CsvImporter, PathDetails
 from delphi.epidata.acquisition.covidcast.database import Database, DBLoadStateException
 from delphi.epidata.acquisition.covidcast.file_archiver import FileArchiver
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 
 def get_argument_parser():

src/acquisition/covidcast/database.py

@@ -14,7 +14,7 @@
 
 # first party
 import delphi.operations.secrets as secrets
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow
 
 

src/acquisition/covidcast/delete_batch.py

@@ -8,7 +8,7 @@
 
 # first party
 from delphi.epidata.acquisition.covidcast.database import Database
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 
 def get_argument_parser():

src/acquisition/covidcast/file_archiver.py

@@ -6,7 +6,7 @@
 import shutil
 
 # first party
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 class FileArchiver:
   """Archives files by moving and compressing."""

src/acquisition/covidcast/signal_dash_data_generator.py

@@ -15,7 +15,7 @@
 # first party
 import covidcast
 import delphi.operations.secrets as secrets
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 
 LOOKBACK_DAYS_FOR_COVERAGE = 56

src/acquisition/common/logger.py renamed to src/common/logger.py

@@ -1,4 +1,3 @@
-"""Structured logger utility for creating JSON logs in Delphi pipelines."""
 import logging
 import os
 import sys
@@ -56,7 +55,7 @@ def get_structured_logger(name=__name__,
         handlers=handlers
         )
 
-    def add_pid(_logger, _method_name, event_dict):
+    def add_pid(logger, method_name, event_dict):
         """
         Add current PID to the event dict.
         """

src/server/_common.py

@@ -6,7 +6,7 @@
 from sqlalchemy.engine import Connection, Engine
 from werkzeug.local import LocalProxy
 
-from .utils.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 from ._config import SECRET, SQLALCHEMY_DATABASE_URI, SQLALCHEMY_ENGINE_OPTIONS
 from ._exceptions import DatabaseErrorException, EpiDataException
 

src/server/_printer.py

@@ -8,7 +8,7 @@
 
 from ._config import MAX_RESULTS, MAX_COMPATIBILITY_RESULTS
 from ._common import is_compatibility_mode
-from .utils.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 
 def print_non_standard(format: str, data):

src/server/endpoints/covidcast_meta.py

@@ -8,7 +8,7 @@
 from .._params import extract_strings
 from .._printer import create_printer
 from .._query import filter_fields
-from ..utils.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 bp = Blueprint("covidcast_meta", __name__)