Merge pull request #1215 from cmu-delphi/release/indicators_v0.1.10_utils_v0.1.10

Release covidcast-indicators 0.1.10

Author: krivard

.bumpversion.cfg

@@ -0,0 +1,5 @@
+[bumpversion]
+current_version = 0.1.10
+commit = True
+message = chore: bump covidcast-indicators to {new_version}
+tag = False

.github/workflows/create-delphi-utils-release.yml

@@ -1,4 +1,4 @@
-name: Create Delphi Utils Release
+name: (Deprecated) Create Delphi Utils Release
 
 on:
   workflow_dispatch:

.github/workflows/create-release.yml

@@ -0,0 +1,74 @@
+name: Create Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      versionName:
+        description: 'Semantic Version Number (i.e., 5.5.0 or patch, minor, major, prepatch, preminor, premajor, prerelease)'
+        required: true
+        default: patch
+
+jobs:
+  create-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          ref: prod
+          ssh-key: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_SSH }}
+      - name: Reset prod branch
+        run: |
+          git fetch origin main:main
+          git reset --hard main
+          git config --global user.email [email protected]
+          git config --global user.name "Delphi Deploy Bot"
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install bump2version
+        run: python -m pip install bump2version
+      - name: Check for delphi-utils changes
+        uses: dorny/paths-filter@v2
+        id: changes
+        with:
+          base: 'prod'
+          ref: 'main'
+          filters: |
+            utils:
+              - '_delphi_utils_python/**'
+      - name: Bump delphi-utils version
+        id: utils-changed
+        if: steps.changes.outputs.utils == 'true'
+        working-directory: ./_delphi_utils_python
+        run: |
+          echo -n "::set-output name=version::"
+          bump2version --list ${{ github.event.inputs.versionName }} | grep ^new_version | sed -r s,"^.*=",,
+          echo -e "\n::set-output name=msg::(*new*)"
+      - name: Detect delphi-utils version
+        id: utils-unchanged
+        if: steps.changes.outputs.utils == 'false'
+        working-directory: ./_delphi_utils_python
+        run: |
+          echo -n "::set-output name=version::"
+          bump2version --list -n ${{ github.event.inputs.versionName }} | grep ^current_version | sed -r s,"^.*=",,
+          echo -e "\n::set-output name=msg::(same as it was)"
+      - name: Bump covidcast-indicators version
+        id: indicators
+        run: |
+          echo -n "::set-output name=version::"
+          bump2version --list ${{ github.event.inputs.versionName }} | grep ^new_version | sed -r s,"^.*=",,       
+      - name: Create pull request into prod
+        uses: peter-evans/create-pull-request@v3
+        with:
+          branch: release/indicators_v${{ steps.indicators.outputs.version }}_utils_v${{ steps.utils-changed.outputs.version }}${{ steps.utils-unchanged.outputs.version }}
+          base: prod
+          title: Release covidcast-indicators ${{ steps.indicators.outputs.version }}
+          labels: chore
+          reviewers: krivard
+          assignees: krivard
+          body: |
+            Releasing:
+            * covidcast-indicators ${{ steps.indicators.outputs.version }}
+            * delphi-utils ${{ steps.utils-changed.outputs.version }}${{ steps.utils-unchanged.outputs.version }} ${{steps.utils-changed.outputs.msg }}${{ steps.utils-unchanged.outputs.msg }}

.github/workflows/publish-release.yml

@@ -0,0 +1,125 @@
+name: Publish release
+
+on:
+  push:
+    branches: [ prod ]
+  workflow_dispatch:
+
+jobs:
+  correct_repository:
+    runs-on: ubuntu-latest
+    steps:
+      - name: fail on fork
+        if: github.repository_owner != 'cmu-delphi'
+        run: exit 1
+
+  check:
+    runs-on: ubuntu-latest
+    needs: correct_repository
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          ssh-key: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_SSH }}
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install bump2version
+        run: python -m pip install bump2version
+      - name: Check for delphi-utils changes
+        uses: dorny/paths-filter@v2
+        id: changes
+        with:
+          # when base and ref are the same, changes are detected against the most recent commit before the push.
+          # see https://github.com/dorny/paths-filter#usage for details.
+          base: 'prod'
+          ref: 'prod'
+          filters: |
+            utils:
+              - '_delphi_utils_python/**'
+      - name: Extract indicators version
+        id: indicators
+        run: |
+          echo -n "::set-output name=version::"
+          bump2version --dry-run --list patch | grep ^current_version | sed -r s,"^.*=",,
+      - name: Tag indicators
+        run: git tag covidcast-indicators/v${{ steps.indicators.outputs.version }}
+      - name: Extract utils version
+        id: utils
+        working-directory: ./_delphi_utils_python
+        run: |
+          echo -n "::set-output name=version::"
+          bump2version --dry-run --list patch | grep ^current_version | sed -r s,"^.*=",,
+      - name: Tag utils
+        if: ${{ steps.changes.outputs.utils == 'true' }}
+        run: git tag delphi-utils/v${{ steps.utils.outputs.version }}
+      - name: Publish tags
+        run: git push --tags 
+    outputs:
+      indicators-version: ${{ steps.indicators.outputs.version }}
+      utils-version: ${{ steps.utils.outputs.version }}
+      publish-utils: ${{ steps.changes.outputs.utils }}
+
+  upload_pypi:
+    needs: check
+    if: ${{ needs.check.outputs.publish-utils == 'true' }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: _delphi_utils_python
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install testing dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pylint pytest pydocstyle wheel twine
+      - name: Install
+        run: |
+          make install
+      - name: Release
+        run: |
+          make release
+      - uses: actions/upload-artifact@v2
+        with:
+          name: delphi_utils
+          path: _delphi_utils_python/dist/*.tar.gz
+      - name: Publish a Python distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.DELPHI_PYPI_PROD_TOKEN }}
+          packages_dir: _delphi_utils_python/dist/
+          skip_existing: true
+
+  sync_main:
+    needs: correct_repository
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          ref: main
+          ssh-key: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_SSH }}
+      - name: Reset main branch
+        run: |
+          git fetch origin prod:prod
+          git reset --hard prod
+      - name: Create pull request into main
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
+          branch: bot/sync-prod-main
+          commit-message: 'chore: sync prod-main'
+          base: main
+          title: 'chore: sync prod->main'
+          labels: chore
+          reviewers: krivard
+          assignees: krivard
+          body: |
+            Syncing Prod->Main.

.github/workflows/release-delphi-utils.yml

@@ -1,9 +1,9 @@
-name: Release Delphi Utils
+name: (Deprecated) Release Delphi Utils
 
 on:
   push:
     branches:
-      - prod
+      - not_prod
     paths:
       - '_delphi_utils_python/**'
   workflow_dispatch:

_delphi_utils_python/.bumpversion.cfg

@@ -1,8 +1,8 @@
 [bumpversion]
-current_version = 0.1.9
-commit = False
+current_version = 0.1.10
+commit = True
+message = chore: bump delphi_utils to {new_version}
 tag = False
-tag_name = delphi-utils/v{new_version}
 
 [bumpversion:file:setup.py]
 

_delphi_utils_python/delphi_utils/__init__.py

@@ -14,4 +14,4 @@
 from .signal import add_prefix
 from .nancodes import Nans
 
-__version__ = "0.1.9"
+__version__ = "0.1.10"

_delphi_utils_python/setup.py

@@ -24,7 +24,7 @@
 
 setup(
     name="delphi_utils",
-    version="0.1.9",
+    version="0.1.10",
     description="Shared Utility Functions for Indicators",
     long_description=long_description,
     long_description_content_type="text/markdown",

_template_python/REVIEW.md

@@ -14,8 +14,7 @@ outputs should be given for non-trivial functions
 
 **Structure**
 
-- [ ] code should use 4 spaces for indentation; other style decisions are
-flexible, but be consistent within a module
+- [ ] code should pass lint checks (`make lint`)
 - [ ] any required metadata files are checked into the repository and placed
 within the directory `static`
 - [ ] any intermediate files that are created and stored by the module should
@@ -29,11 +28,11 @@ included in this template file
 
 **Testing**
 
-- [ ] module can be installed in a new virtual environment
-- [ ] pylint with the default `.pylint` settings run over the module produces
-minimal warnings; warnings that do exist have been confirmed as false positives
+- [ ] module can be installed in a new virtual environment (`make install`)
 - [ ] reasonably high level of unit test coverage covering all of the main logic
 of the code (e.g., missing coverage for raised errors that do not currently seem
 possible to reach are okay; missing coverage for options that will be needed are
 not)
-- [ ] all unit tests run without errors
+- [ ] all unit tests run without errors (`make test`)
+- [ ] indicator directory has been added to GitHub CI
+(`covidcast-indicators/.github/workflows/python-ci.yml`)

_template_python/delphi_NAME/constants.py

@@ -1,7 +1,18 @@
 """
-Registry for signal names
+Registry for variations
 """
 
+## example:
+#GEOS = [
+#    "nation",
+#    "hhs",
+#    "state",
+#    "hrr",
+#    "msa",
+#    "county"
+#]
+
+GEOS = []
 
 ## example:
 # FULL_TIME = "full_time_work_prop"
@@ -15,3 +26,11 @@
 # ]
 
 SIGNALS = []
+
+## example:
+#SMOOTHERS = [
+#    (Smoother("identity", impute_method=None), ""),
+#    (Smoother("moving_average", window_length=7), "_7dav"),
+#]
+
+SMOOTHERS = []

_template_python/delphi_NAME/run.py

@@ -14,8 +14,15 @@
             unpublished signals are.  See `delphi_utils.add_prefix()`
         - Any other indicator-specific settings
 """
-from delphi_utils import add_prefix
-from .constants import SIGNALS
+from datetime import timedelta
+import pandas as pd
+import time
+
+from delphi_utils.export import create_export_csv
+from delphi_utils.geomap import GeoMapper
+from delphi_utils import get_structured_logger
+
+from .constants import GEOS, SIGNALS, SMOOTHERS
 
 def run_module(params):
     """
@@ -25,11 +32,46 @@ def run_module(params):
     --------
     params:  Dict[str, Any]
         Nested dictionary of parameters.
-
-    Returns
-    -------
-    prints the updated signal names
     """
-    wip_signal = params["indicator"]["wip_signal"]
-    signal_names = add_prefix(SIGNALS, wip_signal, prefix="wip_")
-    print(signal_names)
+    start_time = time.time()
+    logger = get_structured_logger(
+        __name__, filename=params["common"].get("log_filename"),
+        log_exceptions=params["common"].get("log_exceptions", True))
+    mapper = GeoMapper()
+    run_stats = []
+    ## build the base version of the signal at the most detailed geo level you can get.
+    ## compute stuff here or farm out to another function or file
+    all_data = pd.DataFrame(columns=["timestamp", "val", "zip", "sample_size", "se"])
+    ## aggregate & smooth
+    ## TODO: add num/prop variations if needed
+    for sensor, smoother, geo in product(SIGNALS, SMOOTHERS, GEOS):
+        df = mapper.replace_geocode(
+            all_data, "zip", geo,
+            new_col="geo_id",
+            date_col="timestamp")
+        ## TODO: recompute sample_size, se here if not NA
+        df["val"] = df[["geo_id", "val"]].groupby("geo_id")["val"].transform(
+            smoother[0].smooth
+        )
+        sensor_name = sensor + smoother[1] ## TODO: +num/prop variation if used
+        # don't export first 6 days for smoothed signals since they'll be nan.
+        start_date = min(df.timestamp) + timedelta(6) if smoother[1] else min(df.timestamp)
+        dates = create_export_csv(
+            df,
+            params["common"]["export_dir"],
+            geo,
+            sensor_name,
+            start_date=start_date)
+        if len(dates) > 0:
+            run_stats.append((max(dates), len(dates)))
+    ## log this indicator run
+    elapsed_time_in_seconds = round(time.time() - start_time, 2)
+    min_max_date = run_stats and min(s[0] for s in run_stats)
+    csv_export_count = sum(s[-1] for s in run_stats)
+    max_lag_in_days = min_max_date and (datetime.now() - min_max_date).days
+    formatted_min_max_date = min_max_date and min_max_date.strftime("%Y-%m-%d")
+    logger.info("Completed indicator run",
+                elapsed_time_in_seconds = elapsed_time_in_seconds,
+                csv_export_count = csv_export_count,
+                max_lag_in_days = max_lag_in_days,
+                oldest_final_export_date = formatted_min_max_date)