debug jhu, still msa issue

Author: jsharpna

_delphi_utils_python/delphi_utils/geomap.py

@@ -51,16 +51,17 @@ class GeoMapper:
        + county -> state : unweighted
        + county -> msa : unweighted
        + county -> megacounty
+       + county -> hrr
+       + county -> zip
        - zip -> state
        - zip -> msa
-       - fips -> hrr
 
     Geotypes (listed by default column name):
         zip: zip5, length 5 str of 0-9 with leading 0's
         fips: county, length 5 str of 0-9 with leading 0's
         msa: metro stat area, length 5 str of 0-9 with leading 0's
-        st_code: state code, str of 0-9
-        state_id: state id, str of A-Z
+        st_code: state code, str in 1-99
+        state_id: state id, str in AA-ZZ
         hrr: hrr id, int 1-500
     """
 
@@ -147,11 +148,26 @@ def load_jhu_uid_fips_cross(self):
                                                     'weight': float})
         self.jhu_uid_fips_cross = GeoMapper.convert_int_to_str5(self.jhu_uid_fips_cross, int_col='fips', str_col='fips')
 
+    @staticmethod
+    def convert_int_to_str5(data,
+                            int_col='fips',
+                            str_col='fips'):
+        """convert int to a string of length 5"""
+        data = data.copy()
+        data[str_col] = data[int_col].astype(str).str.zfill(5)
+        return data
 
-    def convert_fips_to_stcode(self,
+    def convert_intfips_to_str(self,
                                data,
-                               fips_col='fips',
-                               stcode_col='st_code'):
+                               intfips_col='fips',
+                               strfips_col='fips'):
+        """convert fips to a string of length 5"""
+        return GeoMapper.convert_int_to_str5(data,int_col=intfips_col,str_col=strfips_col)
+
+    def convert_fips_to_stcode(self,
+                               data: pd.DataFrame,
+                               fips_col: str = 'fips',
+                               stcode_col: str = 'st_code'):
         """create st_code column from fips column
 
         Args:
@@ -168,23 +184,6 @@ def convert_fips_to_stcode(self,
         data[stcode_col] = data[fips_col].str[:2]
         return data
 
-    @staticmethod
-    def convert_int_to_str5(data,
-                            int_col='fips',
-                            str_col='fips'):
-        """convert int to a string of length 5"""
-        data = data.copy()
-        data[str_col] = data[int_col].astype(str).str.zfill(5)
-        return data
-
-    def convert_intfips_to_str(self,
-                               data,
-                               intfips_col='fips',
-                               strfips_col='fips'):
-        """convert fips to a string of length 5"""
-        return GeoMapper.convert_int_to_str5(data,int_col=intfips_col,str_col=strfips_col)
-
-
     def convert_stcode_to_state_id(self,
                                data,
                                stcode_col='st_code',
@@ -365,16 +364,11 @@ def county_to_state(self,
         if count_cols:
             data = data[[fips_col,date_col] + count_cols].copy()
         data = self.convert_fips_to_state_id(data,fips_col=fips_col,state_id_col=state_id_col,full=full)
-        data.dropna(subset=['st_code'], axis=0, inplace=True)
+        data.dropna(subset=[state_id_col], axis=0, inplace=True)
         data.drop([fips_col,'st_code'],axis=1,inplace=True)
-        # assert not data[state_id_col].isnull().values.any(), "nan states, probably invalid fips"
-        if date_col:
-            assert not data[date_col].isnull().values.any(), "nan dates not allowed"
-            data.fillna(0,inplace=True)
-            data = data.groupby([date_col,state_id_col]).sum()
-        else:
-            data.fillna(0,inplace=True)
-            data = data.groupby(state_id_col).sum()
+        assert not data[date_col].isnull().values.any(), "nan dates not allowed"
+        # data.fillna(0,inplace=True)
+        data = data.groupby([date_col,state_id_col]).sum()
         return data.reset_index()
 
     def county_to_msa(self,
@@ -828,4 +822,4 @@ def county_to_hrr(self,
         zip_col = "_zip_col_temp"
         data = self.county_to_zip(data, fips_col=fips_col, zip_col=zip_col, date_col=date_col, count_cols=count_cols)
         data = self.zip_to_hrr(data, zip_col=zip_col, date_col=date_col, count_cols=count_cols, hrr_col=hrr_col)
-        return data.astype(dtype={hrr_col: int})
+        return data.astype(dtype={hrr_col: int})

_delphi_utils_python/tests/test_geomap.py

@@ -1,8 +1,6 @@
-import pytest
-import os
+from delphi_utils.geomap import GeoMapper
 import pandas as pd
 import numpy as np
-from delphi_utils.geomap import GeoMapper
 
 
 class TestGeoMapper:

jhu/delphi_jhu/geo.py

@@ -37,6 +37,7 @@ def geo_map(df: pd.DataFrame, geo_res: str, sensor: str):
     elif geo_res == "msa":
         df = gmpr.county_to_msa(df, fips_col="fips", msa_col="geo_id", date_col="timestamp")
         df['geo_id'] = df['geo_id'].astype(int)
+        print(df[df['population']==0])
     elif geo_res == 'hrr':
         df = gmpr.county_to_hrr(df, fips_col="fips", hrr_col="geo_id", date_col="timestamp")
         df['geo_id'] = df['geo_id'].astype(int)

jhu/delphi_jhu/pull.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
-import numpy as np
-import pandas as pd
-import re
 
 from delphi_utils import GeoMapper
+import pandas as pd
+import numpy as np
+import re
 
 def detect_date_col(col_name: str):
     """determine if column name is a date"""

jhu/tests/compare_receiving.py

@@ -5,33 +5,40 @@
 import pandas as pd
 import os
 
-def load_files():
+rec_pattern = ""
+# rec_pattern = "county_deaths_incidence_num"
+
+def load_files(pattern = "", num = 1000):
     rec_dir = os.listdir('../receiving')
-    rec_stable_dir = os.listdir('../receiving_rf')
-    rec_common = set(rec_dir) & set(rec_stable_dir)
+    suff = "stable"
+    rec_stable_dir = os.listdir(f'../receiving_{suff}')
+    rec_common = list(set(rec_dir) & set(rec_stable_dir))
     print(set(rec_dir).symmetric_difference(rec_stable_dir))
+    num_iter = 0
     for rec in rec_common:
-        df_rec = pd.read_csv(f'../receiving/{rec}').set_index('geo_id')
-        df_stable = pd.read_csv(f'../receiving_rf/{rec}').set_index('geo_id')
-        try:
-            df_join = df_rec.join(df_stable, rsuffix='_stable' )
-        except:
-            print(df_rec.info())
-            print(df_stable.info())
-            assert False, f"failed join on {rec}"
-        yield rec, df_join
+        if num_iter <= num:
+            num_iter += 1
+            df_rec = pd.read_csv(f'../receiving/{rec}').set_index('geo_id')
+            df_stable = pd.read_csv(f'../receiving_{suff}/{rec}').set_index('geo_id')
+            try:
+                df_join = df_rec.join(df_stable, rsuffix='_stable' )
+            except:
+                print(df_rec.info())
+                print(df_stable.info())
+                assert False, f"failed join on {rec}"
+            yield rec, df_join
 
 def main():
-    load_iter = load_files()
+    load_iter = load_files(rec_pattern)
     for rec, df in load_iter:
         if ('msa' in rec) and False:
             msa_ds = (df['val'] - df['val_stable']).sum()
             print(f'{msa_ds} value diff')
-        if (df.eval('abs(val - val_stable)').sum() > 0.01) and ("county" in rec):
+        if (df.eval('abs(val - val_stable)').sum() > 0.01):
             print(f'Printing {rec} difference')
             df_diff = df[df.eval('val != val_stable')]
-            print(df_diff.head())
             print(df_diff.shape)
+            df_diff.to_csv(f'rec_diffs/diff_{rec}.csv')
             # assert "county_confirmed_7dav_incidence_num" not in rec, f"{rec}!!!"
             #input('w')
 

jhu/tests/receiving/.gitignore

@@ -1,120 +0,0 @@
-# You should hard commit a prototype for this file, but we
-# want to avoid accidental adding of API tokens and other
-# private data parameters
-params.json
-
-# Do not commit output files
-receiving/*.csv
-
-# Remove macOS files
-.DS_Store
-
-# virtual environment
-dview/
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-coverage.xml
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-.static_storage/
-.media/
-local_settings.py
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/

jhu/tests/test_smooth.py

@@ -10,16 +10,16 @@
 class TestSmooth:
     def test_output_files_smoothed(self, run_as_module):
 
-        dates = [str(x) for x in range(20200304, 20200311)]
+        dates = [str(x) for x in range(20200701, 20200730)]
 
         smoothed = pd.read_csv(
-            join("receiving",
+            join("../receiving",
                 f"{dates[-1]}_state_confirmed_7dav_cumulative_num.csv")
         )
 
         raw = pd.concat([
             pd.read_csv(
-                join("receiving",
+                join("../receiving",
                     f"{date}_state_confirmed_cumulative_num.csv")
             ) for date in dates
         ])