Skip to content

Commit 1035362

Browse files
nmdefriesnmdefries
authored
Merge branch 'main' into produce-microdata-raceeth
2 parents c7f63d4 + 50c1da3 commit 1035362

File tree

128 files changed

+2413
-1126
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

128 files changed

+2413
-1126
lines changed

‎.bumpversion.cfg

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
[bumpversion]
2+
current_version = 0.1.21
3+
commit = True
4+
message = chore: bump covidcast-indicators to {new_version}
5+
tag = False

‎.github/workflows/build-container-images.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ jobs:
99
runs-on: ubuntu-latest
1010
strategy:
1111
matrix:
12-
packages: [ "" ]
12+
packages: [ facebook ]
1313
steps:
1414
- name: Checkout code
1515
uses: actions/checkout@v2

‎.github/workflows/create-delphi-utils-release.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Create Delphi Utils Release
1+
name: (Deprecated) Create Delphi Utils Release
22

33
on:
44
workflow_dispatch:

‎.github/workflows/create-release.yml

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
name: Create Release
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
versionName:
7+
description: 'Semantic Version Number (i.e., 5.5.0 or patch, minor, major, prepatch, preminor, premajor, prerelease)'
8+
required: true
9+
default: patch
10+
11+
jobs:
12+
create-release:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Check out code
16+
uses: actions/checkout@v2
17+
with:
18+
ref: prod
19+
ssh-key: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_SSH }}
20+
- name: Reset prod branch
21+
run: |
22+
git fetch origin main:main
23+
git reset --hard main
24+
git config --global user.email [email protected]
25+
git config --global user.name "Delphi Deploy Bot"
26+
- name: Set up Python 3.8
27+
uses: actions/setup-python@v2
28+
with:
29+
python-version: 3.8
30+
- name: Install bump2version
31+
run: python -m pip install bump2version
32+
- name: Check for delphi-utils changes
33+
uses: dorny/paths-filter@v2
34+
id: changes
35+
with:
36+
base: 'prod'
37+
ref: 'main'
38+
filters: |
39+
utils:
40+
- '_delphi_utils_python/**'
41+
- name: Bump delphi-utils version
42+
id: utils-changed
43+
if: steps.changes.outputs.utils == 'true'
44+
working-directory: ./_delphi_utils_python
45+
run: |
46+
echo -n "::set-output name=version::"
47+
bump2version --list ${{ github.event.inputs.versionName }} | grep ^new_version | sed -r s,"^.*=",,
48+
echo -e "\n::set-output name=msg::(*new*)"
49+
- name: Detect delphi-utils version
50+
id: utils-unchanged
51+
if: steps.changes.outputs.utils == 'false'
52+
working-directory: ./_delphi_utils_python
53+
run: |
54+
echo -n "::set-output name=version::"
55+
bump2version --list -n ${{ github.event.inputs.versionName }} | grep ^current_version | sed -r s,"^.*=",,
56+
echo -e "\n::set-output name=msg::(same as it was)"
57+
- name: Bump covidcast-indicators version
58+
id: indicators
59+
run: |
60+
echo -n "::set-output name=version::"
61+
bump2version --list ${{ github.event.inputs.versionName }} | grep ^new_version | sed -r s,"^.*=",,
62+
- name: Create pull request into prod
63+
uses: peter-evans/create-pull-request@v3
64+
with:
65+
branch: release/indicators_v${{ steps.indicators.outputs.version }}_utils_v${{ steps.utils-changed.outputs.version }}${{ steps.utils-unchanged.outputs.version }}
66+
base: prod
67+
title: Release covidcast-indicators ${{ steps.indicators.outputs.version }}
68+
labels: chore
69+
reviewers: krivard
70+
assignees: krivard
71+
body: |
72+
Releasing:
73+
* covidcast-indicators ${{ steps.indicators.outputs.version }}
74+
* delphi-utils ${{ steps.utils-changed.outputs.version }}${{ steps.utils-unchanged.outputs.version }} ${{steps.utils-changed.outputs.msg }}${{ steps.utils-unchanged.outputs.msg }}

‎.github/workflows/publish-release.yml

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
name: Publish release
2+
3+
on:
4+
push:
5+
branches: [ prod ]
6+
workflow_dispatch:
7+
8+
jobs:
9+
correct_repository:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: fail on fork
13+
if: github.repository_owner != 'cmu-delphi'
14+
run: exit 1
15+
16+
check:
17+
runs-on: ubuntu-latest
18+
needs: correct_repository
19+
steps:
20+
- name: Check out code
21+
uses: actions/checkout@v2
22+
with:
23+
ssh-key: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_SSH }}
24+
- name: Set up Python 3.8
25+
uses: actions/setup-python@v2
26+
with:
27+
python-version: 3.8
28+
- name: Install bump2version
29+
run: python -m pip install bump2version
30+
- name: Check for delphi-utils changes
31+
uses: dorny/paths-filter@v2
32+
id: changes
33+
with:
34+
# when base and ref are the same, changes are detected against the most recent commit before the push.
35+
# see https://github.com/dorny/paths-filter#usage for details.
36+
base: 'prod'
37+
ref: 'prod'
38+
filters: |
39+
utils:
40+
- '_delphi_utils_python/**'
41+
- name: Extract indicators version
42+
id: indicators
43+
run: |
44+
echo -n "::set-output name=version::"
45+
bump2version --dry-run --list patch | grep ^current_version | sed -r s,"^.*=",,
46+
- name: Tag indicators
47+
run: git tag covidcast-indicators/v${{ steps.indicators.outputs.version }}
48+
- name: Extract utils version
49+
id: utils
50+
working-directory: ./_delphi_utils_python
51+
run: |
52+
echo -n "::set-output name=version::"
53+
bump2version --dry-run --list patch | grep ^current_version | sed -r s,"^.*=",,
54+
- name: Tag utils
55+
if: ${{ steps.changes.outputs.utils == 'true' }}
56+
run: git tag delphi-utils/v${{ steps.utils.outputs.version }}
57+
- name: Publish tags
58+
run: git push --tags
59+
outputs:
60+
indicators-version: ${{ steps.indicators.outputs.version }}
61+
utils-version: ${{ steps.utils.outputs.version }}
62+
publish-utils: ${{ steps.changes.outputs.utils }}
63+
64+
upload_pypi:
65+
needs: check
66+
if: ${{ needs.check.outputs.publish-utils == 'true' }}
67+
runs-on: ubuntu-latest
68+
defaults:
69+
run:
70+
working-directory: _delphi_utils_python
71+
steps:
72+
- name: Check out code
73+
uses: actions/checkout@v2
74+
- name: Set up Python 3.8
75+
uses: actions/setup-python@v2
76+
with:
77+
python-version: 3.8
78+
- name: Install testing dependencies
79+
run: |
80+
python -m pip install --upgrade pip
81+
pip install pylint pytest pydocstyle wheel twine
82+
- name: Install
83+
run: |
84+
make install
85+
- name: Release
86+
run: |
87+
make release
88+
- uses: actions/upload-artifact@v2
89+
with:
90+
name: delphi_utils
91+
path: _delphi_utils_python/dist/*.tar.gz
92+
- name: Publish a Python distribution to PyPI
93+
uses: pypa/gh-action-pypi-publish@release/v1
94+
with:
95+
user: __token__
96+
password: ${{ secrets.DELPHI_PYPI_PROD_TOKEN }}
97+
packages_dir: _delphi_utils_python/dist/
98+
skip_existing: true
99+
100+
sync_main:
101+
needs: correct_repository
102+
runs-on: ubuntu-latest
103+
steps:
104+
- name: Check out code
105+
uses: actions/checkout@v2
106+
with:
107+
ref: main
108+
ssh-key: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_SSH }}
109+
- name: Reset main branch
110+
run: |
111+
git fetch origin prod:prod
112+
git reset --hard prod
113+
- name: Create pull request into main
114+
uses: peter-evans/create-pull-request@v3
115+
with:
116+
token: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
117+
branch: bot/sync-prod-main
118+
commit-message: 'chore: sync prod-main'
119+
base: main
120+
title: 'chore: sync prod->main'
121+
labels: chore
122+
reviewers: krivard
123+
assignees: krivard
124+
body: |
125+
Syncing Prod->Main.

‎.github/workflows/r-ci.yml

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,15 @@ jobs:
4848
${{ runner.os }}-r-facebook-survey-
4949
- name: Install R dependencies
5050
run: |
51-
if ( packageVersion("readr") != "1.4.0" ) {
51+
if ( !require("readr") || packageVersion("readr") != "1.4.0" ) {
5252
install.packages("devtools")
5353
devtools::install_version("readr", version = "1.4.0")
5454
}
55-
install.packages("remotes")
56-
remotes::update_packages(c("rcmdcheck", "mockr"), upgrade="always")
55+
56+
if ( !require("remotes") ) {
57+
install.packages("remotes")
58+
}
59+
remotes::update_packages(c("rcmdcheck", "mockr", "remotes"), upgrade="always")
5760
dependency_list <- remotes::dev_package_deps(dependencies=TRUE)
5861
remotes::update_packages(dependency_list$package[dependency_list$package != "readr"], upgrade="always")
5962
shell: Rscript {0}

‎.github/workflows/release-delphi-utils.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
name: Release Delphi Utils
1+
name: (Deprecated) Release Delphi Utils
22

33
on:
44
push:
55
branches:
6-
- prod
6+
- not_prod
77
paths:
88
- '_delphi_utils_python/**'
99
workflow_dispatch:

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
params.json
55

66
# Do not commit output files
7-
receiving/*.csv
7+
**/receiving/*.csv
88

99
# Do not commit hidden macOS files
1010
.DS_Store

‎_delphi_utils_python/.bumpversion.cfg

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
[bumpversion]
2-
current_version = 0.1.9
3-
commit = False
2+
current_version = 0.1.15
3+
commit = True
4+
message = chore: bump delphi_utils to {new_version}
45
tag = False
5-
tag_name = delphi-utils/v{new_version}
66

77
[bumpversion:file:setup.py]
88

‎_delphi_utils_python/delphi_utils/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,4 @@
1414
from .signal import add_prefix
1515
from .nancodes import Nans
1616

17-
__version__ = "0.1.9"
17+
__version__ = "0.1.15"

‎_delphi_utils_python/delphi_utils/archive.py

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,11 @@
4040
from git import Repo
4141
from git.refs.head import Head
4242
import pandas as pd
43+
import numpy as np
4344

4445
from .utils import read_params
4546
from .logger import get_structured_logger
47+
from .nancodes import Nans
4648

4749
Files = List[str]
4850
FileDiffMap = Dict[str, Optional[str]]
@@ -73,8 +75,10 @@ def diff_export_csv(
7375
changed_df is the pd.DataFrame of common rows from after_csv with changed values.
7476
added_df is the pd.DataFrame of added rows from after_csv.
7577
"""
76-
export_csv_dtypes = {"geo_id": str, "val": float,
77-
"se": float, "sample_size": float}
78+
export_csv_dtypes = {
79+
"geo_id": str, "val": float, "se": float, "sample_size": float,
80+
"missing_val": int, "missing_se": int, "missing_sample_size": int
81+
}
7882

7983
before_df = pd.read_csv(before_csv, dtype=export_csv_dtypes)
8084
before_df.set_index("geo_id", inplace=True)
@@ -89,12 +93,23 @@ def diff_export_csv(
8993
before_df_cmn = before_df.reindex(common_idx)
9094
after_df_cmn = after_df.reindex(common_idx)
9195

92-
# Exact comparisons, treating NA == NA as True
93-
same_mask = before_df_cmn == after_df_cmn
94-
same_mask |= pd.isna(before_df_cmn) & pd.isna(after_df_cmn)
96+
# If CSVs have different columns (no missingness), mark all values as new
97+
if ("missing_val" in before_df_cmn.columns) ^ ("missing_val" in after_df_cmn.columns):
98+
same_mask = after_df_cmn.copy()
99+
same_mask.loc[:] = False
100+
else:
101+
# Exact comparisons, treating NA == NA as True
102+
same_mask = before_df_cmn == after_df_cmn
103+
same_mask |= pd.isna(before_df_cmn) & pd.isna(after_df_cmn)
104+
105+
# Code deleted entries as nans with the deleted missing code
106+
deleted_df = before_df.loc[deleted_idx, :].copy()
107+
deleted_df[["val", "se", "sample_size"]] = np.nan
108+
if "missing_val" in after_df_cmn.columns:
109+
deleted_df[["missing_val", "missing_se", "missing_sample_size"]] = Nans.DELETED
95110

96111
return (
97-
before_df.loc[deleted_idx, :],
112+
deleted_df,
98113
after_df_cmn.loc[~(same_mask.all(axis=1)), :],
99114
after_df.loc[added_idx, :])
100115

@@ -227,11 +242,11 @@ def diff_exports(self) -> Tuple[Files, FileDiffMap, Files]:
227242

228243
deleted_df, changed_df, added_df = diff_export_csv(
229244
before_file, after_file)
230-
new_issues_df = pd.concat([changed_df, added_df], axis=0)
245+
new_issues_df = pd.concat([deleted_df, changed_df, added_df], axis=0)
231246

232247
if len(deleted_df) > 0:
233248
print(
234-
f"Warning, diff has deleted indices in {after_file} that will be ignored")
249+
f"Diff has deleted indices in {after_file} that have been coded as nans.")
235250

236251
# Write the diffs to diff_file, if applicable
237252
if len(new_issues_df) > 0:
@@ -414,6 +429,9 @@ def archive_exports(self, # pylint: disable=arguments-differ
414429
archive_success.append(exported_file)
415430
except FileNotFoundError:
416431
archive_fail.append(exported_file)
432+
except shutil.SameFileError:
433+
# no need to copy if the cached file is the same
434+
archive_success.append(exported_file)
417435

418436
self._exports_archived = True
419437

0 commit comments

Comments
 (0)