Merge pull request #1200 from cmu-delphi/krivard/update-template

Update Python template.

Author: krivard

_template_python/REVIEW.md

@@ -14,8 +14,7 @@ outputs should be given for non-trivial functions
 
 **Structure**
 
-- [ ] code should use 4 spaces for indentation; other style decisions are
-flexible, but be consistent within a module
+- [ ] code should pass lint checks (`make lint`)
 - [ ] any required metadata files are checked into the repository and placed
 within the directory `static`
 - [ ] any intermediate files that are created and stored by the module should
@@ -29,11 +28,11 @@ included in this template file
 
 **Testing**
 
-- [ ] module can be installed in a new virtual environment
-- [ ] pylint with the default `.pylint` settings run over the module produces
-minimal warnings; warnings that do exist have been confirmed as false positives
+- [ ] module can be installed in a new virtual environment (`make install`)
 - [ ] reasonably high level of unit test coverage covering all of the main logic
 of the code (e.g., missing coverage for raised errors that do not currently seem
 possible to reach are okay; missing coverage for options that will be needed are
 not)
-- [ ] all unit tests run without errors
+- [ ] all unit tests run without errors (`make test`)
+- [ ] indicator directory has been added to GitHub CI
+(`covidcast-indicators/.github/workflows/python-ci.yml`)

_template_python/delphi_NAME/constants.py

@@ -1,7 +1,18 @@
 """
-Registry for signal names
+Registry for variations
 """
 
+## example:
+#GEOS = [
+#    "nation",
+#    "hhs",
+#    "state",
+#    "hrr",
+#    "msa",
+#    "county"
+#]
+
+GEOS = []
 
 ## example:
 # FULL_TIME = "full_time_work_prop"
@@ -15,3 +26,11 @@
 # ]
 
 SIGNALS = []
+
+## example:
+#SMOOTHERS = [
+#    (Smoother("identity", impute_method=None), ""),
+#    (Smoother("moving_average", window_length=7), "_7dav"),
+#]
+
+SMOOTHERS = []

_template_python/delphi_NAME/run.py

@@ -14,8 +14,15 @@
             unpublished signals are.  See `delphi_utils.add_prefix()`
         - Any other indicator-specific settings
 """
-from delphi_utils import add_prefix
-from .constants import SIGNALS
+from datetime import timedelta
+import pandas as pd
+import time
+
+from delphi_utils.export import create_export_csv
+from delphi_utils.geomap import GeoMapper
+from delphi_utils import get_structured_logger
+
+from .constants import GEOS, SIGNALS, SMOOTHERS
 
 def run_module(params):
     """
@@ -25,11 +32,46 @@ def run_module(params):
     --------
     params:  Dict[str, Any]
         Nested dictionary of parameters.
-
-    Returns
-    -------
-    prints the updated signal names
     """
-    wip_signal = params["indicator"]["wip_signal"]
-    signal_names = add_prefix(SIGNALS, wip_signal, prefix="wip_")
-    print(signal_names)
+    start_time = time.time()
+    logger = get_structured_logger(
+        __name__, filename=params["common"].get("log_filename"),
+        log_exceptions=params["common"].get("log_exceptions", True))
+    mapper = GeoMapper()
+    run_stats = []
+    ## build the base version of the signal at the most detailed geo level you can get.
+    ## compute stuff here or farm out to another function or file
+    all_data = pd.DataFrame(columns=["timestamp", "val", "zip", "sample_size", "se"])
+    ## aggregate & smooth
+    ## TODO: add num/prop variations if needed
+    for sensor, smoother, geo in product(SIGNALS, SMOOTHERS, GEOS):
+        df = mapper.replace_geocode(
+            all_data, "zip", geo,
+            new_col="geo_id",
+            date_col="timestamp")
+        ## TODO: recompute sample_size, se here if not NA
+        df["val"] = df[["geo_id", "val"]].groupby("geo_id")["val"].transform(
+            smoother[0].smooth
+        )
+        sensor_name = sensor + smoother[1] ## TODO: +num/prop variation if used
+        # don't export first 6 days for smoothed signals since they'll be nan.
+        start_date = min(df.timestamp) + timedelta(6) if smoother[1] else min(df.timestamp)
+        dates = create_export_csv(
+            df,
+            params["common"]["export_dir"],
+            geo,
+            sensor_name,
+            start_date=start_date)
+        if len(dates) > 0:
+            run_stats.append((max(dates), len(dates)))
+    ## log this indicator run
+    elapsed_time_in_seconds = round(time.time() - start_time, 2)
+    min_max_date = run_stats and min(s[0] for s in run_stats)
+    csv_export_count = sum(s[-1] for s in run_stats)
+    max_lag_in_days = min_max_date and (datetime.now() - min_max_date).days
+    formatted_min_max_date = min_max_date and min_max_date.strftime("%Y-%m-%d")
+    logger.info("Completed indicator run",
+                elapsed_time_in_seconds = elapsed_time_in_seconds,
+                csv_export_count = csv_export_count,
+                max_lag_in_days = max_lag_in_days,
+                oldest_final_export_date = formatted_min_max_date)

_template_python/params.json.template

@@ -1,6 +1,22 @@
 {
-  "static_file_dir": "./static",
-  "export_dir": "./receiving",
-  "cache_dir": "./cache",
-  "wip_signal": ""
+  "common": {
+    "export_dir": "./receiving",
+    "log_filename": "NAME.log"
+  },
+  "validation": {
+    "common": {
+      "data_source": "NAME",
+      "span_length": 14,
+      "min_expected_lag": {"all": "1"},
+      "max_expected_lag": {"all": "3"},
+      "dry_run": true,
+      "suppressed_errors": []
+    },
+    "static": {
+      "minimum_sample_size": 0,
+      "missing_se_allowed": true,
+      "missing_sample_size_allowed": true
+    },
+    "dynamic": {}
+  }
 }

_template_python/setup.py

@@ -23,7 +23,7 @@
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",
-        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
     ],
     packages=find_packages(),
 )