From 993f7a6cc591f43afb57df13144bb3f779dd2187 Mon Sep 17 00:00:00 2001 From: Amaris Sim Date: Wed, 12 Feb 2025 11:01:38 -0500 Subject: [PATCH 1/8] first implimentation of rsv --- nhsn/delphi_nhsn/constants.py | 13 +- nhsn/delphi_nhsn/run.py | 4 +- nhsn/tests/test_data/20241212.csv.gz | Bin 3467 -> 3958 bytes nhsn/tests/test_data/20241212_prelim.csv.gz | Bin 3410 -> 3970 bytes nhsn/tests/test_data/page.json | 159 ++++++++++++++++++- nhsn/tests/test_data/prelim_page.json | 161 +++++++++++++++++++- nhsn/tests/test_patch.py | 6 +- nhsn/tests/test_pull.py | 21 ++- 8 files changed, 352 insertions(+), 12 deletions(-) diff --git a/nhsn/delphi_nhsn/constants.py b/nhsn/delphi_nhsn/constants.py index d904057fe..da6216322 100644 --- a/nhsn/delphi_nhsn/constants.py +++ b/nhsn/delphi_nhsn/constants.py @@ -8,20 +8,25 @@ # column name from socrata TOTAL_ADMISSION_COVID_COL = "totalconfc19newadm" TOTAL_ADMISSION_FLU_COL = "totalconfflunewadm" +TOTAL_ADMISSION_RSV_COL = "totalconfrsvnewadm" NUM_HOSP_REPORTING_COVID_COL = "totalconfc19newadmhosprep" NUM_HOSP_REPORTING_FLU_COL = "totalconfflunewadmhosprep" - +NUM_HOSP_REPORTING_RSV_COL = "totalconfrsvnewadmhosprep" # signal name TOTAL_ADMISSION_COVID = "confirmed_admissions_covid_ew" TOTAL_ADMISSION_FLU = "confirmed_admissions_flu_ew" +TOTAL_ADMISSION_RSV = "confirmed_admissions_rsv_ew" NUM_HOSP_REPORTING_COVID = "hosprep_confirmed_admissions_covid_ew" NUM_HOSP_REPORTING_FLU = "hosprep_confirmed_admissions_flu_ew" +NUM_HOSP_REPORTING_RSV = "hosprep_confirmed_admissions_rsv_ew" SIGNALS_MAP = { TOTAL_ADMISSION_COVID: TOTAL_ADMISSION_COVID_COL, TOTAL_ADMISSION_FLU: TOTAL_ADMISSION_FLU_COL, + TOTAL_ADMISSION_RSV: TOTAL_ADMISSION_RSV_COL, NUM_HOSP_REPORTING_COVID: NUM_HOSP_REPORTING_COVID_COL, NUM_HOSP_REPORTING_FLU: NUM_HOSP_REPORTING_FLU_COL, + NUM_HOSP_REPORTING_RSV: NUM_HOSP_REPORTING_RSV_COL, } TYPE_DICT = { @@ -29,8 +34,10 @@ "geo_id": str, TOTAL_ADMISSION_COVID: float, TOTAL_ADMISSION_FLU: float, + TOTAL_ADMISSION_RSV: float, NUM_HOSP_REPORTING_COVID: float, NUM_HOSP_REPORTING_FLU: float, + NUM_HOSP_REPORTING_RSV: float, } # signal mapping for secondary, preliminary source @@ -39,8 +46,10 @@ PRELIM_SIGNALS_MAP = { f"{TOTAL_ADMISSION_COVID}_prelim": TOTAL_ADMISSION_COVID_COL, f"{TOTAL_ADMISSION_FLU}_prelim": TOTAL_ADMISSION_FLU_COL, + f"{TOTAL_ADMISSION_RSV}_prelim": TOTAL_ADMISSION_RSV_COL, f"{NUM_HOSP_REPORTING_COVID}_prelim": NUM_HOSP_REPORTING_COVID_COL, f"{NUM_HOSP_REPORTING_FLU}_prelim": NUM_HOSP_REPORTING_FLU_COL, + f"{NUM_HOSP_REPORTING_RSV}_prelim": NUM_HOSP_REPORTING_RSV_COL, } PRELIM_TYPE_DICT = { @@ -48,6 +57,8 @@ "geo_id": str, f"{TOTAL_ADMISSION_COVID}_prelim": float, f"{TOTAL_ADMISSION_FLU}_prelim": float, + f"{TOTAL_ADMISSION_RSV}_prelim": float, f"{NUM_HOSP_REPORTING_COVID}_prelim": float, f"{NUM_HOSP_REPORTING_FLU}_prelim": float, + f"{NUM_HOSP_REPORTING_RSV}_prelim": float, } diff --git a/nhsn/delphi_nhsn/run.py b/nhsn/delphi_nhsn/run.py index 7cf9bbace..26f555311 100644 --- a/nhsn/delphi_nhsn/run.py +++ b/nhsn/delphi_nhsn/run.py @@ -63,10 +63,10 @@ def run_module(params, logger=None): geo_mapper = GeoMapper() signal_df_dict = dict() if not nhsn_df.empty: - signal_df_dict.update({signal: nhsn_df for signal in SIGNALS_MAP}) + signal_df_dict.update({signal: nhsn_df for signal in SIGNALS_MAP if signal in nhsn_df.columns}) # some of the source backups do not include for preliminary data if not preliminary_nhsn_df.empty: - signal_df_dict.update({signal: preliminary_nhsn_df for signal in PRELIM_SIGNALS_MAP}) + signal_df_dict.update({signal: preliminary_nhsn_df for signal in PRELIM_SIGNALS_MAP if signal in preliminary_nhsn_df.columns}) for geo, signals_df in product(GEOS, signal_df_dict.items()): signal, df_pull = signals_df diff --git a/nhsn/tests/test_data/20241212.csv.gz b/nhsn/tests/test_data/20241212.csv.gz index 26f91c200dfd1cae144458dfe91b0a6cf767d596..f2e6ce1822991848e2e9051333e24e7316883caa 100644 GIT binary patch literal 3958 zcmV-+4~g&}iwFqdyR2sd|1vN#H83(UGA?6tb^z>K%aR?raozhX`~jC0$V@!7Q%GCe z3)^c4)ohLE8Fe$rru6$hnWzT}1@Jaw7!EEvc)JRPS3Pj@;2@Fw=i}qAk53;yKK=6H z{mbL+f4_eI_~paLhnJ7fPq(M9fB*RO>E$nvAHKBd_aDCg_R?m2di>B{K0iEkmzNgE z+ZsK5-TSY}qjJ4ISKK<^ulnjo*ALH6e|@0xx92aPKD~cA`t9_wOA_7s%SQyH}AzfsNo`t<9d?&+MTz`K2)R94(Tc6{a(^MOo>!qx*!%JCXV52@)?2fve6&;8lb+~_u zd7*v1dGY93=DnWBWL`Xajd`zUAkT}pk~6H^+{=hwA>WlW$nQ!T0~qvnM$^Rr27O)e zw@-7QAI9+QOSXw?l-pGwL0QX(I6fDgW{Vk^25fB?wq%;1MrRR zyOItglfrSYEmF=U9aJWT<6c|9%-`Zb$MZiPv%>(rqFy=I8?pJQziGX$zY&{{`kU6X z{{Dp>(>s^nG5wcwf3V-;oV)$opKn|eFJ5jOGB4zBRKk+ujYW!HvI|kKDR`|VT&y$~ zy+S89_G#gAW43Be(-N0(qb1Qz^1@`*>Yvqc`=8WKP|@qX+M@Q4nsdZo;58#iQ;RPx zQ;tQG#CCE{wgO*LEoh&oWU^S0Dp8KvPy-WC_(wYt1E|1viDZRAI|#*qZ*~-MKa~IbAN8jXS5x@0m~oy`eVzIf zua8mm4w>?557|s*(8et>1HZ@%*>6S2OfKU@?3>IIo!NzyvbHXK`X#hE&Th@sPir~u z<|cK~Hy62cN;z%AJIHZ`wt~V-ImS5UC6x|Y-{1fL-(NSJulr4r```ch$6E@H0$!@z zVk*JACD1JZnqhH#3yH-CHIu**BfC%nxo2{Vfb7|&Xy31p6SrU*AWvnHaLXkF%$#8W z0W*aF-JXcG%fbn!qSnV~9D#T5aQ~b)EMtITYOw924jJ`n{P1KU^{>m6jOUX?JUD=s zlFkM`bJ9yLKJMR29vYPXCScN}M`1p$MwfPQ)9x15`%(rI{Rc7O-&)C z*1cAsj_^~l54jHGmdEx075^LGo5pbB(;MIatbCsbeD7m+hMAZ`&R&^3P=X`?WJ40K zAB>fqBm_-~ID9p*13~n2ULb#!??tm1R0&A~5(k?@d4yYHijX`C;ZkIg409ouq5hL* zMZ>ydpXGop=e$S@ZpKU#e4kc+O=@>N#Jy2FEWu!2@B|9_pxO(UK9q*>BlnDNqe`&K zGv!B7urud%DhM0r*A)%#GY>d93=RyQRz463lW!e4pXEK9NB`+7jug@T+d5U4*78lMPUq-qKpw#tks_?)xx=jKK z5DIb4=4~3VL6ZD73H+3kz!kxD67U)7*(L$cIVy%g`~WQ8G9)U{CLn}>B`Hv-q6MT1 zU9NKgR(~qrmGPq{*hO>|Gd?m4>;eO5B0DMJXXqJGi*XnNd^!qjs3zwl)Kb_mh~-0U z^-boIb@J)bbqes|G6kq^a(c?eM1M%i{k}Mbd=d84i2lhGfOW`u3P{yafQ3Xi1yrrc zyGhPv|4RF7dM5j|{#f>3+rLuwAEdwM&}($R$m?|fH~Oz6emN*5hoee&c{(&x!$tdZ zcY3!DO;e%G`hB)TBUg`9mM*079I{^#sq7D|Df>(H00~yY%H=t&zMkqCEp<&GY``q$ zJ%#$xsyW!I!rU@B$-(&6-@)jcc$Y@SpR26PV>0TuWv``84mIfK-q%wHKd+N~H3Gb@ z8b{i6(8l{A+YoqqWB#9&`Dx_t@vK6`+&wH1&Zhpso>=yUm~#XM!BHGk`~zZO)K&X4 zR5h#gcbE}W`6rkGL=gDVt3n3>A2UitvLnLW0{Vo$;Z^x3Ofos`LZ&R9THe<_ z*!OAW*s#b|`I~P?Pr>|{F`KW)n_nkqYSccPA)IKxI(sB{&YsoVBb(AWXunk2r@P8u z6f1i=R@(O$<)3EC|Jwc)%71rVZ#l7~L;0tP@{fvzS1A8!?{}5IbW*>$_EYIEub2K! ze|_HXw4aghMWldk{ACUG*{prQFYuObC zCRhIVq9v98!JXuV+@ji2y*$eP9%jH8j*xZR_;CC9rR3J{L$E21eHy=*21{E8ODg+g zO-jaOspx0(j%>6`bpve+gwD^IlI{BLo4yD6+sUXvzHu@x&L`@|r)b2!ucV!g`E$$sX(0B$r}L_@F9-GZ$WcWv z&zk~A<&C*{;N=h9srq@Otx&V#4=@)FBd(8iM-~v*MJw1{PYL44-1C9VlP|9N_p_5q z_H28<;$kKHXkN)uH+h5Y07IHsRipY`V<|>dFKLtdX-wK{Ju?5N-<>?R2re?+)h*TA z2f@@by-o47hC^)bor)MCeq;Kdl<9luR;DKjx+M!^Rh1932`)jLI8y%(squrFyi;UD zuV5KhRE|Ng>oyVQgW`N$k1{#sS>zBgVZD%WW94_%`QUc|*n`+LLF%lBV$UJxM&(2E zFVYgdFY2!GeOj3{p7y!cEh-R?uAy_?uZaKA$s7E!vdP*Ht4-d$_nCr9H%C{~XvQ8lyH)q5T0W{J0 zq{h^5ma$SlZy|6cxjFUMln_XbpCV~l-L7tt>BHLR@-=plt?EPYz3mabofqPK0UfN4l~b)z?i7mj)g zu)bz@H^?QP$0F5MvT#+wYkrMA$&tH=9;y*Nxsm#1aERN!?V8T(?rv#ZIjd`*4)(v( z-fS*?UUT3#dnZK2+Dvx;vn>Si`1G z?qeO%3jJ2-NR9Y8r6yY6f_Q9`K%aZ{>Qs$34(Wl#R<}-W!gms;F{Av}*f6*ei=}{sHaAq@GFjQyiv6hKDUhT273QH`=4*+Qa)|v z^1ZEGrdMv|9vvUCJ>dt~4aE`fR|+rs-&vd*&&vCu>i%M3H|o!^sC`}aYEXl(;XNzQ zOAV_5Ra5bQ37|G+e1@DoX@V*7|h~10KwPW{Vp}9N1qP?VWisVV& z_W!B46ZMxGzVF_`_kULE?;&i|&k*Len-rmGLD^n+7{N6fkrP79WN~It&jB+8WV?h2 zsQ?lp@?Im8u_gs8E9XOvI}75(8~~EdI-O*V1qaH#u7VjWiQuug!0ob!NP^Ohtl<#5 z051o-E0<^3*b=*>qPZ+0SLSyXokqc6?tN4igB4lKf)B+)DfWn;QRu!)i8|spA1AHh z3i}FRsPyCT;Ds8c?&8d;@6srUXW0H;5KmR!q4ZijB!AzpZwt3?2l3qN|DAd2$48Jd zZT|h^y4%v-j{6Ki3#Wm%Cw1q__{7&O-RhS6!4~xS@t2R!Pyb49X7px8Z)SAYjQ$S* Q0RR630GYofhdE0C0M#)cLI3~& literal 3467 zcmV;64RrD!iwFop3y^05|1vN#H83$UF)m|sb^z>}$!_Jy5k>d@3V*;8kr6q2XZQoW zu)Q{_C3Sc8APH3}_wW1O$mIBvQfVOr1_Ve1+vm@Z50{5;zkhysdj8@5?(25{)7`h2FvyPcpn?8aZ!q2I&(AD`}i-=A!&vpJyl>cuc;B!p zc=$`JiigdslF!a_Q3yJmDS13%d^Y3x_8K0K7@z)lzP)zCKFVti4_Y@m z9<*+>d@}R3w5~LLGVfTF*T44Zhp)Fk|8%PGvA{QjM)#&SpNak7aw_+2(?Rgtri0{9 z#eORDQ-L3TWh=VuiTF5z}3`H{!wFVw)X&0T7H;qNp7+wd~nc&ogxYpH- zpTv=Re~G@P!Ny-gz;(N2KXPBQHEP>XhrPwMp0A9@F9(i0D@x)C=RB)tu1IOoQ<2sX zaSNvsS9RuT>9wfyt<$@FvXQXuA@!tx|Ci&5;I`*MvCg{G)V*}e0DfNMgcGl>h*1x6 z!9*FeQBMXx>MCVcOUXiNLEB0_`K%3psK#&B93~axA37(DiHalk1i~h1m#)xYD05+Y ztfZhLW{aZ&ZI7?dMLO=PUEFeOQK2?lhztLs!NrIC!N2$9Um5T(q~h#prW{KNBwLgW ziw5b$>;o@d`M&!Yx#zvZ_2YdLsZq-$V*K;0QUC|07ElA-U?bxtt0b6WN~u;q`2yLB zD|yn)s$`mT8j3evl`_G+W(8_vSa&cfbgscr2l1KQDyF9RB8@!`L##%?q@93vrv*CS zATW7~PPKF{Mq^+z<~VmUDh@UZZqY{feP2sspfyU{7z2IHD+6B{Z#+8L8qabLvJ$G* z_!WS;cl8E9zXhNV*8qIO>^Hm{FKxB#`}vb$^9itv?oR2(ZoiJp&Gbv1-$=RMxG>@& zTsZPjl5q?6j=;;39IF#74>1V?PB>Lxjeo(JA4j4QjxMuLg&x@$Lj$V~l*BGYgiSUj zxd$rt!e;Qryh19mni9>cbnQnP|Ek!1&+U;p^Ziohc!{aknVCv<%ka*8K=Hkxc&(%c z#ZcgN85Dz-#HR}oLO3|zn5DvI;uVVs9iRppQ?Ll$8<`0Ap6(q9n7E@ZVXDN3s2=*n zqcC4B6MzMF)>F9-HxoBOqMEwIg#;@eNn1LJ2y<#7eU}{|5qMP^YBDdCYEEIyWMQa| z(yLyBm{Yh%VsIM|FlX~P6mI1fiOnOywZGsNb`19YGq;fa$8O>E`0sQJR|qaO@;!c` z_uuCizA%4-+SP`Liffm{;cl}*<&|%!hvH^=B5qK%Xjn#}v2WC>LiY`BAr(U!LK>u@ zZxORc#6q@}++$;7Cd`=mj9yJgjC^pEt!s(qby~DrvY+e@aVB&6+SV#ed+_PZICJ03 zv}sX$Y}_75SqxFeSyQ2Tw$JuhVzC<%Vl^oegDp7rWg8s#=-AH| zxMQ$349W|hq31JO_EW=Pxbh71nP+%C{yRKF=MTPHs(JMc^NDAe_$0r>Gd!LBRnO2L zT*1w8ICTyE_q&Fh`3`lfjz_UF6~a|Mjh%*eKYVbENm6o=rH`q>P_&22(m>W|Xm&JG zr!eTs6&Y6JCBXfGj5(x2?R`;z@e)jHFDzCL21xS_{Q|@omut^G`Y&mzXntXh3n+4E zl8o{J!}o;YFjn>qG2jaThS_IuEs8{hsMyIF`?gf*9o>Ce#6aYPhC@h;%Ns7ke*@ao zRR^+>hrzR@@emM_;Kfb!UKkaVh5>R^+}K!P(z&98FuigWMj&P|b=gU|El+1UgNkTE z?4{3@SBC)99!PCrv!=E_lkslbQeoY}&|8?C`zlp32wb2`_taR2b|iT=hzW4~LGc!r)_d zp~VdxFcNtgO68YDdq*j8>E@mP#N}ybtg8B;vHoW zQ(6`z#cYW=pE6Bs0@Yd)FLdzP+eLtH7YHpFmX?rPyucQ{6$4csYyv$uZSV>U$cRsi zZf(=yS62AFGTxZu&kK7U^7?+v6=kL064jD^X-mH{px+y2f7uV@UEPPd^_2y}WyEVv zpyftn*TsrWx>ss zlzyq1RBRbz7*)m-Ox#uYg_f1(+9EwRJgtx!QUinhaDj&`XD7ZNVJ!On{poPK9>TL#{4S{RQDz${N z_u&J9?}@-VK%i%2%SF7DxV&#>4UsKY zlabX1xRx+(lLa&Uy)X`1qS)IcVxh22q+T}5flYWd$2C)&gC)46x)p`BeLmc!NMmndXBZR}Fy$;$|MYm+(rxebrzd zp+u8g2#GH|WNNv%5Ub_>n6F+x9dtR%7KFAiBRvz>+{1$>IL;IYFB?zJrL#prmq$tB z;rQ2*#LY8{_2{d`99~}+G`IZaB=KwW-`Gmucne6$ygs-dA5tXqb|Ionng`KwK9r4s tqSq;swt#V}eSUoSvwX<;L(cz4a{ey>009600|55l2_^kH004`mwp#!I diff --git a/nhsn/tests/test_data/20241212_prelim.csv.gz b/nhsn/tests/test_data/20241212_prelim.csv.gz index 9ef69030170d76b077fa7fe51c0366db2fd5dc37..7e89290fc9db03ab39ca5112626e86d6eff11466 100644 GIT binary patch literal 3970 zcmV-|4}I_-iwFo4!>nfl|1vN#H83(UGA?6tb^z>K$#NvO5xx6cYATS4-8Y3#KG;6k zK08JvC5&|5}$n-hcb$xvu#9a9>|O-Q6{prxB=I2i<+! z<}by)bGbftTqobI+V1<*cTbN$+)@7N>FejuAD{Pmn|kT}{Pg_sm!^L0`ok~Zx>Hp3 zhbgN1wdJL*?_0Gz4W)ZS#mSau%UUg0LD%)Clz^dk9~#2FZfXEuH#LwSf_(_{A;6neY6x|k%2N;V-8|}Ox(4Gz zcsH+SG+hJnA-ad)UOwOm;zM)~fjz|Z5YIy_kBm_?lQBOJZPK-U^yRyQD+_fzzI=cS z(>E;*Mh1=jS!<*mTNfEhoSB?ePH^RF@M(IY>b)y_ScC`@M*SoqHJ86DNos&ncPG^ML&m$Trh4Xo`Okp z3=^kl=D-tk@Uu%3p;x4d$=f-}r7dTh=7ioZ%OFa#nK(o#o0&5XW^&fejD=)1xz>Uu znT=#DCh}2Rw6|Nwt^bK-QYi!+tX|c2PSG2! zBhux`yJgMnj1KWxPloa~r=C4`|GT;*uWY+=+u2>Ck>SLG z+ibGJJ37{rf#^6p1?vQV?4p7@1rZ0bHprz7dLqFt1Rw{6sP!^|$8<8SPGVFp|Nfo) zF_N8hLBPuByzTyfd5g zVa3dcU&P4$Y?fNQ-cn;GGw6#IINK|ynBuA;IyENKX~c??s-HQ|@PZZofB&1$dfRV^ z;D7(~y9qjvJRT}fA!hH)>>&_Ws35mI9kw@KrKe01(!I=C1)45_sFS%eGFCsUKBrsc&S#vvvK}Vonug*U~3Pz z;B5|_g9Vidv20~8Wscg&f|_nM8{I24A6+i??)y_ea? zzi@uMuXy!Dr{0t=j^7&T#4|5k;guqlI;3FPiyf0qB4q=B7RyK6gWz0A0cSCb#NJAg zM0Z2xSnzBk7g|6y?i7S7^d!81&H-b@I8X}-N_wUZyBkFaupt939ZP*;V6m305td3u z%_%8~^e-K#A%>XsMTSuGf*=<+P~>Cf1+RM6y7AC`)rB2$2gBv8)T&KAb8 zA(P_FQL2Js-mpt{hc72MEH%g(jO1Ng77ZLl{#vLzr2t5T!ad^-1pHoXHC#HMD`p7X zGJ}3qf^;6KAy@%Db6P0FuiW5@d(kiybhqyuVCy@vqn5JGsc?ffQaLC$*c^9mxH4X~ zvA=LqMZGy!TSdhTYSKG3mY37c4a{e5c)|Q1^mXxVXHMra_Tn__ZI!OaTm6Gey++9L z2jhB75!B<&6jY){- z$mo(ted`s9pf$J1D{aM_9)blU;$Z?90LNvt0T`Pa%5b9edSjy!XXK@MU*%v zp{b@^$in#(S%{@2(WH>Fx)DAKOSo(V`q{Fh%8eM1#S%5z$s(%h$#6{?vMAOmH>5Ro zBo=~L$|FZY`u81)^Y|MbiJctQ8p^jg5`3EjaB>5hhzyd8VXs(kO5hiU1IWB5VmfSvcOJ@KanuOiCoM7SzSCtf&JjCa+G4 zMuSqcJ1yvZyV_rU0b4l*$1aOEPRbAJWoucjs!BAt4KZ2OpvlqhHAFnFNKRbMsg)dj z(of*%ogUsgJy0*ocrrWmXe<>r~$N75N(J+K+v%;hInx#BA6(}&FaasqnaFK4t8x>RIs1z z*^$tYlAOGTQA5%YlhbfgN>n)ytO9Z1h7x14sfjTyK_ky2Wv-P3S1o0Ac(g>BK!V_xAc#$BlslHMT@zh+NQc8v# zQtkzmBo1KWu`~!e2{lL-L@cR|D!dDOjqizko;?aNqO7?HYsBpotH+6cb%iJZZ77CpB8+<|eKy%Z=L=IfD=md7 zoTOqc8BCp>gqMUT9K?ZvZJ%Il?Wu_5=A7z)6G65(v}OE4?0C!!O)5MAS64tTUV$dG z4XJ465`S9#jV90pkz1Ou8gGhAbUUobSLeBwLXV9&~R$fd#Vr=1`Ioe1J(^RM>#@Kho{w(ef4!YQjz58d@U zchV3qT;z4)#J_WAW@-MwlcpN1DG7*VY#{~KB(^o7kV4RKX)f7n zuc>SpIoeezL4_uCDZ$3od5C@GzVssv%(^YMq&|1H!}N5vYCR=?U<8fR!S<^e0mh|D z2Jt&1ym3ZoAzT=Nte4z!h$Mak#K)TmV@CFEM1l`7M<_361mUo6;2;7!KrJQ7XMs1e zamCYRGNO-BL&O9RBw}IV0r4Eb0|2rE0Z4%)n*bjm`IIWZ0bayh$0ylki4wWCEV?%w z5^_r}Ejs}=rIL6KKI)DP8={y#0lIyEiL6^T4uqhZMCh=|Kxj;f47Gj(7r!C3q!2p? zC$>HprS3KbFD#=$B0VgmkakkIGG0}2x^OBRb7G~qZKKM0aTzokw48Qgh-YGW*$TJ% z4w^5fHdJIxz9|P$jlSkMTwdrE@*{uF{aYa(lZC9+UA}Pdy-vD!{7*czwPZKIV3r+C4`7TE@ zritQYCs#2-N||Rw+qeh~bW} z?4ksyHrGM+^wqnVNMIUrTNZ^(=Ql#ZX@<>%vLr!RJOZ<1Y4%aokLiY&`UWjsp?z>l9ucP?M!iHyc+X7KE-%UId3&4 zY&21N^h z08j}&1#yopyM-DSE?^BYi3KIqHYSZYQSsO%iy^90!%wVE1iX{leW=YQTQ)i!N&^o< z_bzENL+*QtDHV&azmo|vU4lxf+O-6qQMWS1Iw_Gu+3jSldnbptP7b+44p!nsk}L>8 za-y2Y&CsmSlip|Rlcpm$$hIRegM)=3nPmzf4&Bb0g6w{XUWXres9YK-~CP~;SxI|t^Wc4AyF!d1>9;WH=nHB5ft-6{|e)z@PjX&MAp4V5>3)Seh^<TlNwGl5&a~WRh5ua7Wew zrtD(sZeMb%o4uC?{Ghv+^lycTG_a)Wv88d+p-4lH{;+*1LY_78U=9>fU}VkvUqQTe zifEy%6u~g{HTM-@B}274$(2GI$u=bKLn7T|2YcLuN#F=rA7ENQO%XXK8@S=^LVF80 zz`i%S#}!-wWt@a|QjB8plN0|MHm3Iu?h2Mm4n`HM=D@PeV>9XV&H|$I6J%`;O;J#t zXKT+4VRUOAd4;=kAx7OksCzM-&$>lq+ve2!3kVY^w^0}yM;&axRD0ec{`?;nH cYBCXjbge%oWB&sH0RR630ApM;5wl4E0I=wF8vpg&-1tHa&Y^8dEhkp{B3*qGWGqJr=RYn{_^G9^Yf>dW!_CYkAC^`^69rt z`_lN+Z{O#}WjxNeOuL?-cU+9$wc)s@$3H$j{C;_{t`I z;v#y}GoJ41QQ_-!;nTzJO}y?^)9MMV#`Nd68gkjPx$JH0MmxOEUmu@0T3NTY+4n80 z!$wbY*SC78yS}gA{(IQBJmArkM_rilsB3*Ijyvt{9e0ne<#^V)b3E(ZJ^C|_XPrC8 zv(DX9&!rC6t-B|>*XX8aCDlz2@BFxDQSV@~VMlw42ReOz{CYo@@3v>++NgDN_(;m^ zAL{tBV`s0x!3TZcctS&?@G%DGw-Ff)AYf-V^QwU_UXrO_rLsnsPM7CSA$0PrgtBS{oisZ z_jT7s@awLPKGgG}mVZ#Ea_7pO%0Hd^i~T!0=T3k7`$?P@J!d&pSKV3bDF?N}xo{5t zlzi}KchZn~kd##LO}tP1>XQ|s(Vr!pY7*;=oFHO4PHU6keCT(a$SYfV!4@CbTsgD| zXIIuDKb`$gMU>T8!qk#oK_MEZmYSTf>T5k!z4(`L=u*1gib1W#o!r5(m{qHPrN|}M z=^FMeuF7u2^k=P(bC)x`;SB$`|A!j-*FXMx%F#*0<<(QlH3oN%a*7a1JUczbtUjOy zNod0JIZHlyE#l5TK{_r&QuuqSIPcEkq!Of_eLR&~AS*49VyyA(p|c>OoU&?$XfR#@>zF@hc`iq}W&9iG#|_G_7P329>ecacr=+b+-*$#Y)MWrhukl|ErP6%XhDJ2woHd8Dim?#~Cya@TZ#)usJM*@oU7^lU zroxn3Nk~To7e{ScjX$iDbQD(m<5HU0NGV~qrch&JH12pj-w!svD;vvzjUg3h&okv% zN+9*3WUO}(Ps~1$Pf8Z}7+Le)<#I0ZBvPZ6vxu?Lvq=FXEW`jF$cGg)p0Y~9LQOH% z>d(GFxZ+AaYi3h2)j18to3ctdWAV)j49BokFe#L-!7vEru-tU0KSG;jgCn> z9qmpFl)l1a@|2xw>CBAA$F|ht+R3jt_)sv7D|oN_S{fg%Q`(2|(bv52@s;sLqjM6vJ_UDa2kV{qmG$vdLcWgb(ZRd&YEuxkS)@sJ{NwlxM* zl8RtvFKh;1%uiCK)l_PJ)U}f|I#w~Cn2T@v6hkJ*OH92@4Vre#@O%0|=KCRYucd~} zP~g`LnL$h9^93d$9b9kBQn99bKnr09^uWayq=N4wF%cTj7?O{P6?F+yB|b*=P$(XT zxx7p~7Ocab+I4_1Zh}NLeTfSRt9&GjX)h5X)j~GU6#x_PRW9TtVk*__WXvRGsE*>R zUW1sOyaZ-&EAN=IITD2-T?4Z@7X0m33}uI8uOAso9zQme`|&p#$_0>9i_}{TrQYTz zuS`FY?Xb9;@1eWWv@0%%x3iRQJIO3<8AaQ^;j9W>AKWB`L`*_4#H4RQxJL*?=#|{V zZG$P+KC_x$Oz)dYj3-V*9iPlO(L5t>~=V+HMln9&UT%N(nMO zIb1b`o2NzH02zzjkRTt3d|yP$DB=#vwbLSQ^{{PbDnTPe12L6?@D}b;HPTb!@q-z$4JxH04A9VE_tCvUWq6LE{4Qx_X9A5YFRR z;&!Uwyq>#8scjxvxLt5%7H$U%uuBnJq}9YoKDKz(lzzr#jLT!|TM8x34b4v0I71|dCf6t*sC-{k4g+w{*u)D_ znSBQJqDVyCi%QO@+uo#ij1#zG1_CKGSVCOfHE|iX58zW*9k@qY2G5>GOCU;u7l)~9 zV5m)21_)B2u2`_-bE62cf92+mK+YiUTuHg@h-Z38MHoVv(Jb<>LsV<3uPqkWB;8CU zTasIFIiu2B?4A4OmpL)3+rDHOd_BZvOfB64 z5i(i<30Y(;_O_oG#kL;3Ya5aU0kVNnEwSTWYG6#Z$5UGdCt7+3E3zGGSTxzTzr_1I zf7*8oYb@4jakfQ2)h{WV>TM0jioame7Wb0o=M0--+7$hu;=59DE22h4@satKl!g8g zQj9KRNp->y8(#990fOMDF|4w%`FP0CjTCM~gPqld7IUaLq6;k|h@TUb=#(s^g9Hm9 zr2q)BBHV9;J1nsj^N;Y6T*Nh0M$%!a#HP|3C@m>1C6i*dJfTmS3vdGQT23)^DBACf za9{68T6`@nKREkGZT`_9mIoh!o=+Rt!VEd0)dFVw=wK`h7+)E0I4V~P-!6CsFTcT= zcI#-7aW57)<~wBg|A@sJ1n-5d z-*rKyO8;YPb}Ll2aHoU`5Lugn%yW7RzXDr0B7bBbsC+L}4usTD=_Hi4($)YlF{zEU z+qSS{ij<0H!n=eJ?D2cJBc`wcOD9b*l}S5F##Qf`(qQ9LqkX+Pe(nr=Bl(sSEV8a6KH8>WRp*+vEi%p?z7P*O zMTE%k_#Odp!(->6TDZ-1oXtKLi@v4BPTbWbhT1zmW+vfMn~h|yre(Z)VZgMUw(VN- zf_i-23`Ngy8AVWe6}!#Ja*hY4Mxk*RhP7@?&9Si z3#a8WS7RA(1a3G{z#Z9s5Nse?KFxJ5@1SE@r)2tGWS!{v>UhIcI4;cMbw|8%f;kmk z=;(J@n8i!hL7@G{Gu)Q0k+RFA=&k7*l^fGFhbNhpEosEahC6sj3plWnHb%zgC~M0^ zYkSwHFzTVuLd$h`Ci<3&hLl)j3%D6~lb_WFxR%{+lLa#jy)Z^vR@>VoVnMb|q+V8= zkWG>`$0eL-jaM@Om4+wm{eFk+l@}JS-@Lh9s<8?rl(^}MeOwl z*f+%8G=x4Ttb_Q;i-f?}%Mf>fY2+iEpR@OWz)Nte5WOX5dO-<{bl_QH46v;@T?ts) zL8qilKEeT3!()NGnKbVu{H(W^Fw9FzG{J>{Alu&9unb{{)iRPyt}ky$x}0SVN!#?1 zo*!(+a^oS#Y3<-;J0RelC`+!;498+K!Cr)t4k+Im2DwPrP0> oWQp!GDI#@{wOPtFE@iVb`6tiy!~XyP0RR6300sRlsxvtN09C@mlK=n! diff --git a/nhsn/tests/test_data/page.json b/nhsn/tests/test_data/page.json index 5d4eda759..3d72ded40 100644 --- a/nhsn/tests/test_data/page.json +++ b/nhsn/tests/test_data/page.json @@ -1,4 +1,161 @@ -[ +[{ + "weekendingdate": "2021-08-21T00:00:00.000", + "jurisdiction": "MD", + "numinptbeds": "2972.43", + "numinptbedsadult": "2582.14", + "numinptbedsped": "195.86", + "numinptbedsocc": "2339.14", + "numinptbedsoccadult": "2056.86", + "numinptbedsoccped": "154.43", + "numicubeds": "329.86", + "numicubedsadult": "289.43", + "numicubedsped": "0.0", + "numicubedsocc": "247.29", + "numicubedsoccadult": "226.86", + "numicubedsoccped": "0.29", + "numconfc19hosppatsadult": "36.5", + "numconfc19hosppatsped": "1.71", + "totalconfc19hosppats": "38.21", + "numconffluhosppatsadult": "0.0", + "numconffluhosppatsped": "0.0", + "totalconffluhosppats": "0.86", + "numconfrsvhosppatsadult": "0.0", + "numconfrsvhosppatsped": "0.0", + "totalconfrsvhosppats": "0.0", + "numconfc19icupatsadult": "3.43", + "numconfc19icupatsped": "0.71", + "totalconfc19icupats": "4.14", + "numconffluicupatsadult": "0.0", + "numconffluicupatsped": "0.0", + "totalconffluicupats": "0.0", + "numconfrsvicupatsadult": "0.0", + "numconfrsvicupatsped": "0.0", + "totalconfrsvicupats": "0.0", + "numconfc19newadmped0to4": "1.0", + "numconfc19newadmped5to17": "0.0", + "totalconfc19newadmped": "1.0", + "numconfc19newadmadult18to49": "6.0", + "totalconfc19newadmadult": "52.0", + "numconfc19newadmunk": "0.0", + "totalconfc19newadm": "53.0", + "totalconfflunewadmped": "0.0", + "totalconfflunewadmadult": "0.0", + "totalconfflunewadm": "2.0", + "totalconfrsvnewadmped": "0.0", + "totalconfrsvnewadmadult": "0.0", + "totalconfrsvnewadm": "0.0", + "pctinptbedsocc": "0.7869", + "pctconfc19inptbeds": "0.0129", + "pctconffluinptbeds": "0.0003", + "pctconfrsvinptbeds": "0.0", + "pcticubedsocc": "0.7497", + "pctconfc19icubeds": "0.0126", + "pctconffluicubeds": "0.0", + "pctconfrsvicubeds": "0.0", + "pctconfc19newadmadult": "0.9811", + "pctconfc19newadmped": "0.0189", + "pctconfflunewadmadult": "0.0", + "pctconfflunewadmped": "0.0", + "numinptbedshosprep": "13", + "numinptbedsocchosprep": "13", + "numicubedshosprep": "13", + "numicubedsocchosprep": "13", + "totalconfc19hosppatshosprep": "13", + "totalconffluhosppatshosprep": "13", + "totalconfrsvhosppatshosprep": "1", + "totalconfc19icupatshosprep": "13", + "totalconffluicupatshosprep": "13", + "totalconfrsvicupatshosprep": "1", + "totalconfc19newadmpedhosprep": "12", + "totalconfc19newadmadulthosprep": "13", + "totalconfc19newadmhosprep": "13", + "totalconfflunewadmpedhosprep": "1", + "totalconfflunewadmadulthosprep": "1", + "totalconfflunewadmhosprep": "13", + "totalconfrsvnewadmpedhosprep": "1", + "totalconfrsvnewadmadulthosprep": "1", + "totalconfrsvnewadmhosprep": "1", + "pctinptbedsocchosprep": "13", + "pcticubedsocchosprep": "13", + "pctconfc19inptbedshosprep": "13", + "pctconffluinptbedshosprep": "13", + "pctconfrsvinptbedshosprep": "1", + "pctconfc19icubedshosprep": "13", + "pctconffluicubedshosprep": "13", + "pctconfrsvicubedshosprep": "1", + "numinptbedsperchosprep": "0.2549", + "numinptbedsoccperchosprep": "0.2549", + "numicubedsperchosprep": "0.2549", + "numicubedsoccperchosprep": "0.2549", + "totalconfc19hosppatsperc": "0.2549", + "totalconffluhosppatsperc": "0.2549", + "totalconfrsvhosppatsperc": "0.0196", + "totalconfc19icupatsperchosprep": "0.2549", + "totalconffluicupatsperchosprep": "0.2549", + "totalconfrsvicupatsperchosprep": "0.0196", + "totalconfc19newadmpedper": "0.2353", + "totalconfc19newadmadultp": "0.2549", + "totalconfc19newadmperchosprep": "0.2549", + "totalconfflunewadmpedper": "0.0196", + "totalconfflunewadmadultp": "0.0196", + "totalconfflunewadmperchosprep": "0.2549", + "totalconfrsvnewadmpedper": "0.0196", + "totalconfrsvnewadmadultp": "0.0196", + "totalconfrsvnewadmperchosprep": "0.0196", + "pctinptbedsoccperchosprep": "0.2549", + "pcticubedsoccperchosprep": "0.2549", + "pctconfc19inptbedsperchosprep": "0.2549", + "pctconffluinptbedsperchosprep": "0.2549", + "pctconfrsvinptbedsperchosprep": "0.0196", + "pctconfc19icubedsperchosprep": "0.2549", + "pctconffluicubedsperchosprep": "0.2549", + "pctconfrsvicubedsperchosprep": "0.0196", + "numinptbedsperchosprepabschg": "0.0196", + "numinptbedsoccperchospre": "0.0196", + "numicubedsperchosprepabschg": "0.0196", + "numicubedsoccperchosprepabschg": "0.0196", + "totalconfc19hosppatsperc_1": "0.0196", + "totalconffluhosppatsperc_1": "0.0196", + "totalconfrsvhosppatsperc_1": "0.0", + "totalconfc19icupatsperch": "0.0196", + "totalconffluicupatsperch": "0.0196", + "totalconfrsvicupatsperch": "0.0", + "totalconfc19newadmpedper_1": "0.0", + "totalconfc19newadmadultp_1": "0.0196", + "totalconfc19newadmpercho": "0.0196", + "totalconfflunewadmpedper_1": "0.0", + "totalconfflunewadmadultp_1": "0.0", + "totalconfflunewadmpercho": "0.0196", + "totalconfrsvnewadmpedper_1": "0.0", + "totalconfrsvnewadmadultp_1": "0.0", + "totalconfrsvnewadmpercho": "0.0", + "pctinptbedsoccperchospre": "0.0196", + "pcticubedsoccperchosprepabschg": "0.0196", + "pctconfc19inptbedspercho": "0.0196", + "pctconffluinptbedspercho": "0.0196", + "pctconfrsvinptbedspercho": "0.0", + "pctconfc19icubedsperchos": "0.0196", + "pctconffluicubedsperchos": "0.0196", + "pctconfrsvicubedsperchos": "0.0", + "numconfc19newadmped0to4per100k": "0.29", + "numconfc19newadmped5to17per100k": "0.0", + "totalconfc19newadmpedper100k": "0.07", + "numconfc19newadmadult18to49per100k": "0.24", + "totalconfc19newadmadultper100k": "1.09", + "totalconfc19newadmper100k": "0.86", + "totalconfflunewadmpedper100k": "0.0", + "totalconfflunewadmadultper100k": "0.0", + "totalconfflunewadmper100k": "0.03", + "totalconfrsvnewadmpedper100k": "0.0", + "totalconfrsvnewadmadultper100k": "0.0", + "totalconfrsvnewadmper100k": "0.0", + "totalconfc19newadmperchosprepabove80pct": "0", + "totalconfc19newadmperchosprepabove90pct": "0", + "totalconfflunewadmperchosprepabove80pct": "0", + "totalconfflunewadmperchosprepabove90pct": "0", + "totalconfrsvnewadmperchosprepabove80pct": "0", + "totalconfrsvnewadmperchosprepabove90pct": "0" + }, { "weekendingdate": "2021-08-21T00:00:00.000", "jurisdiction": "CO", diff --git a/nhsn/tests/test_data/prelim_page.json b/nhsn/tests/test_data/prelim_page.json index 374639f14..817fecd66 100644 --- a/nhsn/tests/test_data/prelim_page.json +++ b/nhsn/tests/test_data/prelim_page.json @@ -1,4 +1,163 @@ -[ +[{ + "weekendingdate": "2021-08-21T00:00:00.000", + "jurisdiction": "MI", + "numinptbeds": "20658.33", + "numinptbedsadult": "19048.19", + "numinptbedsped": "930.14", + "numinptbedsocc": "15875.6", + "numinptbedsoccadult": "14604.12", + "numinptbedsoccped": "608.76", + "numicubeds": "2870.62", + "numicubedsadult": "2265.48", + "numicubedsped": "175.71", + "numicubedsocc": "2153.88", + "numicubedsoccadult": "1733.88", + "numicubedsoccped": "138.86", + "numconfc19hosppatsadult": "249.76", + "numconfc19hosppatsped": "18.0", + "totalconfc19hosppats": "267.76", + "numconffluhosppatsadult": "0.29", + "numconffluhosppatsped": "0.0", + "totalconffluhosppats": "434.24", + "numconfrsvhosppatsadult": "1.14", + "numconfrsvhosppatsped": "0.0", + "totalconfrsvhosppats": "1.14", + "numconfc19icupatsadult": "31.71", + "numconfc19icupatsped": "1.86", + "totalconfc19icupats": "33.57", + "numconffluicupatsadult": "0.0", + "numconffluicupatsped": "0.0", + "totalconffluicupats": "44.29", + "numconfrsvicupatsadult": "0.0", + "numconfrsvicupatsped": "0.0", + "totalconfrsvicupats": "0.0", + "numconfc19newadmped0to4": "12.0", + "numconfc19newadmped5to17": "2.0", + "totalconfc19newadmped": "16.0", + "numconfc19newadmadult18to49": "32.0", + "totalconfc19newadmadult": "253.0", + "numconfc19newadmunk": "6.0", + "totalconfc19newadm": "269.0", + "totalconfflunewadmped": "0.0", + "totalconfflunewadmadult": "1.0", + "totalconfflunewadm": "523.0", + "totalconfrsvnewadmped": "0.0", + "totalconfrsvnewadmadult": "1.0", + "totalconfrsvnewadm": "1.0", + "pctinptbedsocc": "0.7685", + "pctconfc19inptbeds": "0.013", + "pctconffluinptbeds": "0.021", + "pctconfrsvinptbeds": "0.0001", + "pcticubedsocc": "0.7503", + "pctconfc19icubeds": "0.0117", + "pctconffluicubeds": "0.0154", + "pctconfrsvicubeds": "0.0", + "pctconfc19newadmadult": "0.9405", + "pctconfc19newadmped": "0.0595", + "pctconfflunewadmadult": "0.0019", + "pctconfflunewadmped": "0.0", + "pctconfrsvnewadmadult": "1.0", + "pctconfrsvnewadmped": "0.0", + "numinptbedshosprep": "152", + "numinptbedsocchosprep": "152", + "numicubedshosprep": "152", + "numicubedsocchosprep": "152", + "totalconfc19hosppatshosprep": "152", + "totalconffluhosppatshosprep": "152", + "totalconfrsvhosppatshosprep": "4", + "totalconfc19icupatshosprep": "152", + "totalconffluicupatshosprep": "152", + "totalconfrsvicupatshosprep": "4", + "totalconfc19newadmpedhosprep": "148", + "totalconfc19newadmadulthosprep": "152", + "totalconfc19newadmhosprep": "152", + "totalconfflunewadmpedhosprep": "4", + "totalconfflunewadmadulthosprep": "4", + "totalconfflunewadmhosprep": "152", + "totalconfrsvnewadmpedhosprep": "4", + "totalconfrsvnewadmadulthosprep": "4", + "totalconfrsvnewadmhosprep": "4", + "pctinptbedsocchosprep": "152", + "pcticubedsocchosprep": "152", + "pctconfc19inptbedshosprep": "152", + "pctconffluinptbedshosprep": "152", + "pctconfrsvinptbedshosprep": "4", + "pctconfc19icubedshosprep": "152", + "pctconffluicubedshosprep": "152", + "pctconfrsvicubedshosprep": "4", + "numinptbedsperchosprep": "0.9744", + "numinptbedsoccperchosprep": "0.9744", + "numicubedsperchosprep": "0.9744", + "numicubedsoccperchosprep": "0.9744", + "totalconfc19hosppatsperc": "0.9744", + "totalconffluhosppatsperc": "0.9744", + "totalconfrsvhosppatsperc": "0.0256", + "totalconfc19icupatsperchosprep": "0.9744", + "totalconffluicupatsperchosprep": "0.9744", + "totalconfrsvicupatsperchosprep": "0.0256", + "totalconfc19newadmpedper": "0.9487", + "totalconfc19newadmadultp": "0.9744", + "totalconfc19newadmperchosprep": "0.9744", + "totalconfflunewadmpedper": "0.0256", + "totalconfflunewadmadultp": "0.0256", + "totalconfflunewadmperchosprep": "0.9744", + "totalconfrsvnewadmpedper": "0.0256", + "totalconfrsvnewadmadultp": "0.0256", + "totalconfrsvnewadmperchosprep": "0.0256", + "pctinptbedsoccperchosprep": "0.9744", + "pcticubedsoccperchosprep": "0.9744", + "pctconfc19inptbedsperchosprep": "0.9744", + "pctconffluinptbedsperchosprep": "0.9744", + "pctconfrsvinptbedsperchosprep": "0.0256", + "pctconfc19icubedsperchosprep": "0.9744", + "pctconffluicubedsperchosprep": "0.9744", + "pctconfrsvicubedsperchosprep": "0.0256", + "numinptbedsperchosprepabschg": "0.0", + "numinptbedsoccperchospre": "0.0", + "numicubedsperchosprepabschg": "0.0", + "numicubedsoccperchosprepabschg": "0.0", + "totalconfc19hosppatsperc_1": "0.0", + "totalconffluhosppatsperc_1": "0.0", + "totalconfrsvhosppatsperc_1": "0.0064", + "totalconfc19icupatsperch": "0.0", + "totalconffluicupatsperch": "0.0", + "totalconfrsvicupatsperch": "0.0064", + "totalconfc19newadmpedper_1": "0.0", + "totalconfc19newadmadultp_1": "0.0", + "totalconfc19newadmpercho": "0.0", + "totalconfflunewadmpedper_1": "0.0", + "totalconfflunewadmadultp_1": "0.0", + "totalconfflunewadmpercho": "0.0", + "totalconfrsvnewadmpedper_1": "0.0064", + "totalconfrsvnewadmadultp_1": "0.0064", + "totalconfrsvnewadmpercho": "0.0064", + "pctinptbedsoccperchospre": "0.0", + "pcticubedsoccperchosprepabschg": "0.0", + "pctconfc19inptbedspercho": "0.0", + "pctconffluinptbedspercho": "0.0", + "pctconfrsvinptbedspercho": "0.0064", + "pctconfc19icubedsperchos": "0.0", + "pctconffluicubedsperchos": "0.0", + "pctconfrsvicubedsperchos": "0.0064", + "numconfc19newadmped0to4per100k": "2.26", + "numconfc19newadmped5to17per100k": "0.13", + "totalconfc19newadmpedper100k": "0.76", + "numconfc19newadmadult18to49per100k": "0.79", + "totalconfc19newadmadultper100k": "3.19", + "totalconfc19newadmper100k": "2.68", + "totalconfflunewadmpedper100k": "0.0", + "totalconfflunewadmadultper100k": "0.01", + "totalconfflunewadmper100k": "5.21", + "totalconfrsvnewadmpedper100k": "0.0", + "totalconfrsvnewadmadultper100k": "0.01", + "totalconfrsvnewadmper100k": "0.01", + "totalconfc19newadmperchosprepabove80pct": "1", + "totalconfc19newadmperchosprepabove90pct": "1", + "totalconfflunewadmperchosprepabove80pct": "1", + "totalconfflunewadmperchosprepabove90pct": "1", + "totalconfrsvnewadmperchosprepabove80pct": "0", + "totalconfrsvnewadmperchosprepabove90pct": "0" + }, { "weekendingdate": "2021-08-21T00:00:00.000", "jurisdiction": "CO", diff --git a/nhsn/tests/test_patch.py b/nhsn/tests/test_patch.py index 72da1e40c..ba0220fb5 100644 --- a/nhsn/tests/test_patch.py +++ b/nhsn/tests/test_patch.py @@ -13,7 +13,7 @@ from delphi_nhsn.patch import filter_source_files, patch from delphi_nhsn.constants import TOTAL_ADMISSION_COVID_COL, TOTAL_ADMISSION_FLU_COL, \ NUM_HOSP_REPORTING_FLU_COL, NUM_HOSP_REPORTING_COVID_COL, GEOS, TOTAL_ADMISSION_COVID, TOTAL_ADMISSION_FLU, \ - NUM_HOSP_REPORTING_COVID, NUM_HOSP_REPORTING_FLU + NUM_HOSP_REPORTING_COVID, NUM_HOSP_REPORTING_FLU, NUM_HOSP_REPORTING_RSV_COL, TOTAL_ADMISSION_RSV_COL from conftest import TEST_DATA, PRELIM_TEST_DATA, TEST_DIR class TestPatch: @@ -89,13 +89,17 @@ def generate_test_source_files(self): test_data = pd.DataFrame(TEST_DATA) test_data[TOTAL_ADMISSION_COVID_COL] = int(date) test_data[TOTAL_ADMISSION_FLU_COL] = int(date) + test_data[TOTAL_ADMISSION_RSV_COL] = int(date) test_data[NUM_HOSP_REPORTING_COVID_COL] = int(date) test_data[NUM_HOSP_REPORTING_FLU_COL] = int(date) + test_data[NUM_HOSP_REPORTING_RSV_COL] = int(date) test_prelim_data = pd.DataFrame(PRELIM_TEST_DATA) test_prelim_data[TOTAL_ADMISSION_COVID_COL] = int(date) test_prelim_data[TOTAL_ADMISSION_FLU_COL] = int(date) + test_prelim_data[TOTAL_ADMISSION_RSV_COL] = int(date) test_prelim_data[NUM_HOSP_REPORTING_COVID_COL] = int(date) test_prelim_data[NUM_HOSP_REPORTING_FLU_COL] = int(date) + test_prelim_data[NUM_HOSP_REPORTING_RSV_COL] = int(date) test_data = test_data.head(2) test_data.to_csv( diff --git a/nhsn/tests/test_pull.py b/nhsn/tests/test_pull.py index 91411e1d6..5f56f03d8 100644 --- a/nhsn/tests/test_pull.py +++ b/nhsn/tests/test_pull.py @@ -65,8 +65,11 @@ def test_pull_from_file(self, caplog, dataset, params_w_patch): expected_data = pd.DataFrame(dataset["test_data"]) df = pull_data_from_file(backup_dir, issue_date, logger=logger, prelim_flag=prelim_flag) - df = df.astype('str') - expected_data = expected_data.astype('str') + + # expected_data reads from dictionary and defaults all the columns as object data types + # compared to the method which pd.read_csv somewhat interprets numerical data types + expected_data = expected_data.astype(df.dtypes.to_dict()) + # expected_data = expected_data.astype('str') assert "Pulling data from file" in caplog.text pd.testing.assert_frame_equal(expected_data, df) @@ -95,6 +98,9 @@ def test_pull_nhsn_data_output(self, mock_create_backup, mock_socrata, caplog, p assert set(result.columns) == expected_columns for column in list(result.columns): + # some states don't report confirmed admissions rsv + if column == "confirmed_admissions_rsv_ew": + continue assert result[column].notnull().all(), f"{column} has rogue NaN" @@ -140,13 +146,13 @@ def test_pull_nhsn_data_backup(self, mock_socrata, caplog, params): os.remove(file) @patch("delphi_nhsn.pull.Socrata") - @patch("delphi_nhsn.pull.create_backup_csv") - def test_pull_prelim_nhsn_data_output(self, mock_create_backup, mock_socrata, caplog, params): + # @patch("delphi_nhsn.pull.create_backup_csv") + def test_pull_prelim_nhsn_data_output(self, mock_socrata, caplog, params): now = time.time() # Mock Socrata client and its get method mock_client = MagicMock() mock_socrata.return_value = mock_client - mock_client.get.side_effect = [TEST_DATA, []] + mock_client.get.side_effect = [PRELIM_TEST_DATA, []] mock_client.get_metadata.return_value = {"rowsUpdatedAt": now} @@ -157,12 +163,15 @@ def test_pull_prelim_nhsn_data_output(self, mock_create_backup, mock_socrata, ca logger = get_structured_logger() result = pull_preliminary_nhsn_data(test_token, backup_dir, custom_run, issue_date=None, logger=logger) - mock_create_backup.assert_called_once() + # mock_create_backup.assert_called_once() expected_columns = set(PRELIM_TYPE_DICT.keys()) assert set(result.columns) == expected_columns for column in list(result.columns): + # some states don't report confirmed admissions rsv + if column == "confirmed_admissions_rsv_ew_prelim": + continue assert result[column].notnull().all(), f"{column} has rogue NaN" @patch("delphi_nhsn.pull.Socrata") def test_pull_prelim_nhsn_data_backup(self, mock_socrata, caplog, params): From c06709c06883f5e8bbb25b3a5d51be02cc0bab87 Mon Sep 17 00:00:00 2001 From: Amaris Sim Date: Wed, 12 Feb 2025 11:06:27 -0500 Subject: [PATCH 2/8] lint --- nhsn/delphi_nhsn/run.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nhsn/delphi_nhsn/run.py b/nhsn/delphi_nhsn/run.py index 26f555311..404da3c34 100644 --- a/nhsn/delphi_nhsn/run.py +++ b/nhsn/delphi_nhsn/run.py @@ -66,7 +66,9 @@ def run_module(params, logger=None): signal_df_dict.update({signal: nhsn_df for signal in SIGNALS_MAP if signal in nhsn_df.columns}) # some of the source backups do not include for preliminary data if not preliminary_nhsn_df.empty: - signal_df_dict.update({signal: preliminary_nhsn_df for signal in PRELIM_SIGNALS_MAP if signal in preliminary_nhsn_df.columns}) + signal_df_dict.update( + {signal: preliminary_nhsn_df for signal in PRELIM_SIGNALS_MAP if signal in preliminary_nhsn_df.columns} + ) for geo, signals_df in product(GEOS, signal_df_dict.items()): signal, df_pull = signals_df From c1dadfc2c44967576b8c72a3dbd1877008c1b924 Mon Sep 17 00:00:00 2001 From: Amaris Sim Date: Wed, 12 Feb 2025 11:44:01 -0500 Subject: [PATCH 3/8] clean up and fix tests --- nhsn/delphi_nhsn/run.py | 5 +- nhsn/tests/test_data/20241212.csv.gz | Bin 3958 -> 4083 bytes nhsn/tests/test_data/20241212_prelim.csv.gz | Bin 3970 -> 4073 bytes nhsn/tests/test_data/page.json | 279 +++++++++++--------- nhsn/tests/test_data/prelim_page.json | 278 ++++++++++--------- nhsn/tests/test_patch.py | 4 +- nhsn/tests/test_pull.py | 7 +- 7 files changed, 328 insertions(+), 245 deletions(-) diff --git a/nhsn/delphi_nhsn/run.py b/nhsn/delphi_nhsn/run.py index 404da3c34..c61c5199d 100644 --- a/nhsn/delphi_nhsn/run.py +++ b/nhsn/delphi_nhsn/run.py @@ -63,11 +63,11 @@ def run_module(params, logger=None): geo_mapper = GeoMapper() signal_df_dict = dict() if not nhsn_df.empty: - signal_df_dict.update({signal: nhsn_df for signal in SIGNALS_MAP if signal in nhsn_df.columns}) + signal_df_dict.update({signal: nhsn_df for signal in SIGNALS_MAP}) # some of the source backups do not include for preliminary data if not preliminary_nhsn_df.empty: signal_df_dict.update( - {signal: preliminary_nhsn_df for signal in PRELIM_SIGNALS_MAP if signal in preliminary_nhsn_df.columns} + {signal: preliminary_nhsn_df for signal in PRELIM_SIGNALS_MAP} ) for geo, signals_df in product(GEOS, signal_df_dict.items()): @@ -94,6 +94,7 @@ def run_module(params, logger=None): df["se"] = np.nan df["sample_size"] = np.nan + dates = create_export_csv( df, geo_res=geo, diff --git a/nhsn/tests/test_data/20241212.csv.gz b/nhsn/tests/test_data/20241212.csv.gz index f2e6ce1822991848e2e9051333e24e7316883caa..088469c65278556e41f33417966731a5298b5e22 100644 GIT binary patch literal 4083 zcmVKOLHE(aoy)v_y>HPLIaIgcCKWb zc;RH7k}3LHx!0shqU^u#)7|sHU;vR!l#~`MeVoAn7|iSRIo%ED?+*{ZK0JQ>^!Uri z56=&`|Ni#n)7Oum?w>zBJ>DL_{q@u1=jT5?eEiy$KYaZ5+jCp-`Qc-G_;i2Y9bV2r z-FDLbw|#so9-Zy@+;JOxKkB<5L*G9={`sEjU!J~x{`}$j7`LmJ-p^0ZAAalVx2}Kw z?b~pOrv5ZUQ@?fn^7U`SA)5Nr5KVn^!+5~abF9_C+RX0_QnKZ+!M%IjSuCedc)^xA3 zuayTpMjP}zz01b(1=n95K6hl$*4lksYnpB&bG@`Rc6e!P3~V&!irvwWv!(;_qXG9< zF>lnjH*X%@%Y4-HH<>q&9%DZ08OZbIt>p~sw)XNxuaNIr8sv8^jR6csJEQ4h0E4lv z_}gXf%l#O>eaj|XquQ?f2+Gnv#PPY|q#kWUD++@vwVXd_ZYh0ypF2X(VTR=K8RNSd zk9WrK_>A%GkH7dIn>xU^O&#Q?V4uQ#3h?eKErf%7Y0W*J!107F)dngVc=oS6yx!;J*N8?TVb>ofLd^Fy) zpN)5bZQ$ItZ}y+O^K|FRou~hB?w|FyIOlHv_OCZzvtKyed@Oz;zEO=!NjD!tl|%1h zl4FW2y@IdWTB;mjkelbc_Ag~^$ECyNckKO9+;@|{crTIb>voQ6 znpTDwW^L#5&XvP4Uw_#*?8Y@O*3LV=VtBuQF!`*W1u69cw{X5r^Y7n(kKfct{^buh zavs(JoK{!$a+$sEoKv$%fx~JG=tLzI#SRQXj3DWbJm3nH*F@SFWq?=}{%TDL*F(GB zjNYP^@bf`U^i=Bm*e|f$MwKfRDyvl#HJHIUZb=0wMqMntHK8)Qf*pBoDr<6T9ii6s z?CMXyjB}2?TWjU@YK^6pvG0&8meFF@uhQFa_#W-_VNG!{<_ck zmfsY=|M|z?-%_O%K*{-*d;&1VNJ362FVTZfV=i%Ff8$&;>MD3Yg)BEz3n*vF;g*vt zECyC;0m?CA&7=^H5f{QMOFmaY8&g=I6jhO0iqq2xSBDgM+ymbj#Ec;#lm@n%7f>Wb z2$4|;tQxsRkIb`(^86w74{yME)HLT9kW;{v?XTHf;YKBHK1QJi{Xl1 zq4UZZuyFKDVy3&4?9wGN1@7Q-E{z4ts|7kWZkN<2L`hB3I#4HAx$28esnshH8r%TK zRF$xP^Ad9p~ND11a}5%RbVvaple`!R3$~c^IXCKUSG-*T8R5v zm&kn?KU@sf*8`osiIr2ZSKNU>KZ*y5Lyk4=?^EvbgnEdwd|QHT8msVDudL#IS=lD8 z;LjGF;kmba3N;QkkbD;LTFapd!0}eKk!&NDfR?-(#mnalhu6urC<|uZqQboYd|}79iXZ zEr9*qr{|fNbnl5R0RFP{p>V67w#5ea2n!QSc5c1Hc`Zqjg+v=fI*@FMpRSnQyx1!$n;-0^-wCkE9b_I$Wus~`wWD5kJ!@G1%VsX}=d zgr8Utt~fS9;KI=afqhAWB!~cDi)W3QBp3lXAe0HfqPQ_2SidCE1H`iu?+E)RDR`GK zR4RDYaMBkzn&5B1E(`e=csN1q#^FQY1yb~iZE~F8KgA7+0WfPlp5ki}A?nH|2OKYQ zAfA+yT@e|mE@sv+#0l_g@O(Hvl6x4+0h_aU$U(LQNJz1h1M^1xZkV9!G}we)0@iXg zIY`Fux>@BQ9dhu(`AW}zzSAsCD*$`VPUF7N#^xdk@gxc1DhY2Ke{;9H&E4jW4-Ze9 zMkYFoSmTIhcej3;rbAoxBP@tUDXd_OT}j+_gad+0Q34=Nlpt7UD*1OyJ>ddkF~~)IlPG3}b*>1g>Q9+(8BQP7=*5U22x9hfG6~>IM383@!S(!CLgi~@+_8}${0`hBmNRvrSrUdGg zM6nXMQXUqi)i6QH)sl)vg{@H?<$fe+cllhJr8Wsj+lL7#n zGd^jqIz8=ObzpVT0uIt*S^yp;4&)Z`IVBS$O29A%kB<_IOp_m}Azo@}b3+8n@~JQ5 zm+~O+^+9J&JR(+7R?9k>!H)EsS4$ObfkT%O%~^B&`y}@v1Uge}+==?M;ofr2y9kuQ zCW5$A+Nd36G7->5wvCBE@`nh#aK4gq{LW5!xtwSmuc8S8*a*C+3Y1m_@*oI!%i!jF zUpA5v3P%??Ozl*2(|KcYpRN%=>~@w53T+3gNrw9vKeRqT|!zgz3ejFB~9l? z(#(HwHXl%2Fyi+**Bx9k;!m2ttTw3~b;cXcigs1nFSdeYMEfCcAwV7z_}Y&&Q_6cH zNJHcrrnA`9@`7-oTGm(5r*#p?mI5-|1oH2!|5LMm4`5?`3h-4JEK1`2!-S&Ca912j z4v2&ZKvV1l-|%1%2O~bmQAPGv2|jr>YOtbul|0DS(j~ul*PG8;P2yAdR^CX zB7J2Ok-Tm~KodY@Yc*Q$-*OeZWJO8nUpFisyEo$Ioy0{$G^3-NmH5j+{4boZd>iIF zHOZXn;i8u_Tl=zSaw@m0dho%^fcf7r`uF{RdBf+_PJLlQU_av}=U?ZD>C{Dc^~YEc zxjW}a5&%CPTwkdw*C*reEa5PmJA;Hu?(nvtTN!YMSwNCxeQn^T)1KU?9KVl)M&%%7P#o*yA1ri zGB5zK$$%p+=D}13xx#iL{3)HgiYW?AWz`K8W)_5n0KZfgU_b~Q8Ca_wx})WOF2GpOQ>O>a=MEcY+CHtRWVi7Vt^d@qqgo z9)wKfQi3#y3Y3y$sX+$!muTFd;q1$z){>$zJ?IUNlC12O$RSDc-xWy?B`y?2CblR5 zD&kwO-eZIk`?Adg$9y6Gkb}*$scA+RvIf6Z7qUi@^&0!?2ksG{{EGN)62KUbF4Mrl z78j=dme{$=Mra#+oo{{6pRFGtZL?5c2;A~;ga77Sr;qXzKXn-yV3VJQl z^p9T82b!n<4f8d}(~6+Xx`KeJJIO$`|I*;z|4YNqEdxD}RR%m9cAbMt@O44oEVGl~ z9a-5EV=APp*5Fk~s0G~l5EGmS5E2oyta(Pn3WgT`hnMbjVJP$fgj+;f3P=ThJF&#b z9uY=l1Rc0sVYp53fRmR66}w|z52hDhFR-*Fbw$B+mFdBY3OHTsMp41eoYw!-psOA@ zR$Yv-uN=CB0Gf}^)QNy*Lup;`utx^N2p|nN9H{Z&U7Ae-xI9`6G(7*M#X#nf$J+bo zG4spw025P)9Cy>f+jdt2T6`X4gmSkx_9uUx4z|?Y|A`%a ldHChi)8p^x-I(5u>D`!KX-xkB009600|3j=*&dHf005L0;w1n8 literal 3958 zcmV-+4~g&}iwFqdyR2sd|1vN#H83(UGA?6tb^z>K%aR?raozhX`~jC0$V@!7Q%GCe z3)^c4)ohLE8Fe$rru6$hnWzT}1@Jaw7!EEvc)JRPS3Pj@;2@Fw=i}qAk53;yKK=6H z{mbL+f4_eI_~paLhnJ7fPq(M9fB*RO>E$nvAHKBd_aDCg_R?m2di>B{K0iEkmzNgE z+ZsK5-TSY}qjJ4ISKK<^ulnjo*ALH6e|@0xx92aPKD~cA`t9_wOA_7s%SQyH}AzfsNo`t<9d?&+MTz`K2)R94(Tc6{a(^MOo>!qx*!%JCXV52@)?2fve6&;8lb+~_u zd7*v1dGY93=DnWBWL`Xajd`zUAkT}pk~6H^+{=hwA>WlW$nQ!T0~qvnM$^Rr27O)e zw@-7QAI9+QOSXw?l-pGwL0QX(I6fDgW{Vk^25fB?wq%;1MrRR zyOItglfrSYEmF=U9aJWT<6c|9%-`Zb$MZiPv%>(rqFy=I8?pJQziGX$zY&{{`kU6X z{{Dp>(>s^nG5wcwf3V-;oV)$opKn|eFJ5jOGB4zBRKk+ujYW!HvI|kKDR`|VT&y$~ zy+S89_G#gAW43Be(-N0(qb1Qz^1@`*>Yvqc`=8WKP|@qX+M@Q4nsdZo;58#iQ;RPx zQ;tQG#CCE{wgO*LEoh&oWU^S0Dp8KvPy-WC_(wYt1E|1viDZRAI|#*qZ*~-MKa~IbAN8jXS5x@0m~oy`eVzIf zua8mm4w>?557|s*(8et>1HZ@%*>6S2OfKU@?3>IIo!NzyvbHXK`X#hE&Th@sPir~u z<|cK~Hy62cN;z%AJIHZ`wt~V-ImS5UC6x|Y-{1fL-(NSJulr4r```ch$6E@H0$!@z zVk*JACD1JZnqhH#3yH-CHIu**BfC%nxo2{Vfb7|&Xy31p6SrU*AWvnHaLXkF%$#8W z0W*aF-JXcG%fbn!qSnV~9D#T5aQ~b)EMtITYOw924jJ`n{P1KU^{>m6jOUX?JUD=s zlFkM`bJ9yLKJMR29vYPXCScN}M`1p$MwfPQ)9x15`%(rI{Rc7O-&)C z*1cAsj_^~l54jHGmdEx075^LGo5pbB(;MIatbCsbeD7m+hMAZ`&R&^3P=X`?WJ40K zAB>fqBm_-~ID9p*13~n2ULb#!??tm1R0&A~5(k?@d4yYHijX`C;ZkIg409ouq5hL* zMZ>ydpXGop=e$S@ZpKU#e4kc+O=@>N#Jy2FEWu!2@B|9_pxO(UK9q*>BlnDNqe`&K zGv!B7urud%DhM0r*A)%#GY>d93=RyQRz463lW!e4pXEK9NB`+7jug@T+d5U4*78lMPUq-qKpw#tks_?)xx=jKK z5DIb4=4~3VL6ZD73H+3kz!kxD67U)7*(L$cIVy%g`~WQ8G9)U{CLn}>B`Hv-q6MT1 zU9NKgR(~qrmGPq{*hO>|Gd?m4>;eO5B0DMJXXqJGi*XnNd^!qjs3zwl)Kb_mh~-0U z^-boIb@J)bbqes|G6kq^a(c?eM1M%i{k}Mbd=d84i2lhGfOW`u3P{yafQ3Xi1yrrc zyGhPv|4RF7dM5j|{#f>3+rLuwAEdwM&}($R$m?|fH~Oz6emN*5hoee&c{(&x!$tdZ zcY3!DO;e%G`hB)TBUg`9mM*079I{^#sq7D|Df>(H00~yY%H=t&zMkqCEp<&GY``q$ zJ%#$xsyW!I!rU@B$-(&6-@)jcc$Y@SpR26PV>0TuWv``84mIfK-q%wHKd+N~H3Gb@ z8b{i6(8l{A+YoqqWB#9&`Dx_t@vK6`+&wH1&Zhpso>=yUm~#XM!BHGk`~zZO)K&X4 zR5h#gcbE}W`6rkGL=gDVt3n3>A2UitvLnLW0{Vo$;Z^x3Ofos`LZ&R9THe<_ z*!OAW*s#b|`I~P?Pr>|{F`KW)n_nkqYSccPA)IKxI(sB{&YsoVBb(AWXunk2r@P8u z6f1i=R@(O$<)3EC|Jwc)%71rVZ#l7~L;0tP@{fvzS1A8!?{}5IbW*>$_EYIEub2K! ze|_HXw4aghMWldk{ACUG*{prQFYuObC zCRhIVq9v98!JXuV+@ji2y*$eP9%jH8j*xZR_;CC9rR3J{L$E21eHy=*21{E8ODg+g zO-jaOspx0(j%>6`bpve+gwD^IlI{BLo4yD6+sUXvzHu@x&L`@|r)b2!ucV!g`E$$sX(0B$r}L_@F9-GZ$WcWv z&zk~A<&C*{;N=h9srq@Otx&V#4=@)FBd(8iM-~v*MJw1{PYL44-1C9VlP|9N_p_5q z_H28<;$kKHXkN)uH+h5Y07IHsRipY`V<|>dFKLtdX-wK{Ju?5N-<>?R2re?+)h*TA z2f@@by-o47hC^)bor)MCeq;Kdl<9luR;DKjx+M!^Rh1932`)jLI8y%(squrFyi;UD zuV5KhRE|Ng>oyVQgW`N$k1{#sS>zBgVZD%WW94_%`QUc|*n`+LLF%lBV$UJxM&(2E zFVYgdFY2!GeOj3{p7y!cEh-R?uAy_?uZaKA$s7E!vdP*Ht4-d$_nCr9H%C{~XvQ8lyH)q5T0W{J0 zq{h^5ma$SlZy|6cxjFUMln_XbpCV~l-L7tt>BHLR@-=plt?EPYz3mabofqPK0UfN4l~b)z?i7mj)g zu)bz@H^?QP$0F5MvT#+wYkrMA$&tH=9;y*Nxsm#1aERN!?V8T(?rv#ZIjd`*4)(v( z-fS*?UUT3#dnZK2+Dvx;vn>Si`1G z?qeO%3jJ2-NR9Y8r6yY6f_Q9`K%aZ{>Qs$34(Wl#R<}-W!gms;F{Av}*f6*ei=}{sHaAq@GFjQyiv6hKDUhT273QH`=4*+Qa)|v z^1ZEGrdMv|9vvUCJ>dt~4aE`fR|+rs-&vd*&&vCu>i%M3H|o!^sC`}aYEXl(;XNzQ zOAV_5Ra5bQ37|G+e1@DoX@V*7|h~10KwPW{Vp}9N1qP?VWisVV& z_W!B46ZMxGzVF_`_kULE?;&i|&k*Len-rmGLD^n+7{N6fkrP79WN~It&jB+8WV?h2 zsQ?lp@?Im8u_gs8E9XOvI}75(8~~EdI-O*V1qaH#u7VjWiQuug!0ob!NP^Ohtl<#5 z051o-E0<^3*b=*>qPZ+0SLSyXokqc6?tN4igB4lKf)B+)DfWn;QRu!)i8|spA1AHh z3i}FRsPyCT;Ds8c?&8d;@6srUXW0H;5KmR!q4ZijB!AzpZwt3?2l3qN|DAd2$48Jd zZT|h^y4%v-j{6Ki3#Wm%Cw1q__{7&O-RhS6!4~xS@t2R!Pyb49X7px8Z)SAYjQ$S* Q0RR630GYofhdE0C0M#)cLI3~& diff --git a/nhsn/tests/test_data/20241212_prelim.csv.gz b/nhsn/tests/test_data/20241212_prelim.csv.gz index 7e89290fc9db03ab39ca5112626e86d6eff11466..f1afd8ce04dca885851a186df2af8f836740cd2a 100644 GIT binary patch literal 4073 zcmVK%aYx;5#8roC>FZ$XdpXRR#_xj zBw45A(b!gOYgE$6{{5V8@BxD0%pFfvN(+^)2wXgQ$Dt$!Km77# zxJ1)_SfXiPhkp9}+i;1d{jfyS-rO*L;OKWO&Hc|Om$-`G(xmUDYtl#EG&N$io?-97pl*Q*}i^mX^za}pI>5JG9po>;ehKqSfOosJ5E^#XUww)1 z?x$WY*J1n;-reK5TCRimCAu%cy}ZCHh+m@n64;lRzQpq-mamLabdz&GU)rQ=|J9ox zPHrsJ>-qHsJh6P&(_!SIbG&Mclw(f^m5a{tsx4ryf5ncDhu`kA!wb4$Tse*#Y4g!~ zm*cwiM%sL|-sL!3?@#QQ-njgR>0i$M**>##Zu;9_Cw3|3+53r#3wfq874K#W6U8iX zP7{UX!c06RmuE>6XK{1li8;kN%8?pu`a5Wgp0RtxCu8U+pMg^@ISI|zLC`><)}^M z^~yBaGtok#vo<2zo?=+`EWz27SgdeVwl%GcJp3Qpnm)4aTiD9*a{2e~^rN>~Yxpp*j{%2TKc;>lak!15JLD>pT8q4mJj{WdX9&ImK8LjnjpFpF z5r?L+r25Q!DTf^kA3lPad+C-|Jl@*kCU-FwJ8<#O6q03C5rY<2=rm)+Ni)tJS9rk= z|G)p;YrXAriQs?#^SdeK5+mR_B@SK{bDX>c#YmhYm6-gDHyb3Nf5!XgSb)t8Y)VFj z(j+m*1aM3_7nxI<5}r7?fu|TmIfkZ05@zU5!5(wO4uHI1BF1AsVwoYVG(+Ep08kQS z(y5%1s04#P0!BGd1lXvU(16b8V)Vk5G=!7ho>7Na5lTiw;40P6$_%0CmBGdivY2l9OD|b}ZNUIHGNr&*cFD7HuDDC(V z4@LgNyS}jpEjO210F^g3x82JNkENev?=!E(>u!MNppVd&v*)I;W+g4#-QWe&YWeCn z+Tt;7-A^>CL zIAM;OFD8hfk(oA9*+@(Y3&teg_@OqYE>?TaJC(%j%;3|SkK0JfGZ!c+JHt)_&Ezn9C_fSyIjY>m2Y?#yBC_}xwdTDOI47VI z@ET{OnF3Mp+Qcx!z=ANth!ZQ{2vZ6wi3HQe&mQKxBbUKQqD{##?-*Kkhc9Q)T?=w9 zMOm$@cZ8yqd&I=*M z1^yDM0~C+&j~+yQzJMiys+pzki$WaF);g(PR2I-9iAU)8w8CknlP+1$UaVe`v4 zRZ2*CFE#-Y=3YQvw0>2~d|O$S|FO+Kxy*W7b?EU{|KL`y5wh~eg%MLEGp`CI)97&T zlnR8B)8-V&{1wmU*%V)`v*66QQq8Wk;Y7z2NQ!pGP$|bI!iC3>Oy>QsP;MndHGd_J zy=H@gKRG#i58h0-(@f}r^YR5_TtF5-sg#rMU;L^-1Vc1LZ`5Rq))pT)2 za;1v^Q?t@V;G7gopa=jLzcLCHSTjCAr9}@ks+j}z%Nhuy0?$_?R>}=YE9?@Mg0{hY zcqJLHkky9}rG^X($gAWy5D*?6MXs?e7ZU8NxMI-*Xstz*xn!lOu3pI^#1mPlrKRyM zcyNk%7}vqP9H-DPa0hn7NTU=(vRL9;J6VXCq3l=BW+Mw)JeSmyMH*cEMi5IqRh700#Jk6U0eyxIIwyW z4z!_;ghY*66)4f52ETa9arU4F&`w3PBLV?I$HExm#gPy&QI^f>DRZEf9CQs1eO*;> zoc-C6sNpLV(E`0GTLdg*eRw%j<2k?g5d7jvFlQi=X9tZYY%-=tf7vD`K{K zs@xDxj)Yt}66g6hITESR(a>KKL6sT!+L1_EZgC_o*ME^C;RjyBp)EY)N%-46i6{1Y zrG$-wP6|p*bZ}gFLid56?$o(X$we)FOqC{xmza}9r~*}*!SFjY8Cjblo|bJi zN-2;-Dx-js#sQohYZpN$r3TH@s3o;gMT}SZ5?R7`Ikikt-G~xP6LgYLg(cEAn)upj zViJe*BLs-1bnkAuFG^-IsP@i3~(7dZwPf-0DDp3I1Pz=QhjC&}3cEv&GD^moo zJ(VefCW}2eEM0C@NlG}h2LAE&cb$pTxJZ6R_Ri1#Ws~}ge zz*6pxOf*Z4Kdt^o7ibkIJWB_bo+iA-o1_M`U0RV}qtzWnurFQte*a;%hi6CCK{%W6J7B%wY+u^k-?M-r? zy6bmNXW!w4o4ih(IF2|W)#NusKz(GefEEHX%kl@2EY;v$O+aMlD=Dz0*}Y4Z6q1EY zOUPtwR8N=5sXj&v5!0+mZO6(3>N$W1yr!)&8x_)Q0(^iRb8h?wcoB0OpX5+9N)-CK z>fUrn$St|F5){~!**t|5#mf->$AmPIfSija4*-c;Pp|9IA%YUK`VdNzXzPydDoM z;;w0;I@viij38Vc2cd>LAei+CqKn^X;%laf0gNk6I10JiI3?&IBL+`!O_;?L6{zqk zBITebUiB0#tl8PC=a3y$xE4M4pn!G7d(B_xBFY$fg6f5)4(bEzsf_|(B0^V3O<$jQkml9K9s>Ftx4#mo|ja0X#z=0_JMY$cxJvC^L$;198=C) zi%T1xSBhvtM|=K(&IyQ@?qE;sxgvyqv@cK2-#WwDfq2mduN0~>;sy)(7j~{nmPLca z!yf=tQp`!+W6xoshJ_2*LQHBwX|tDX5hto1yJj&|bz1m|&53|_(z*|=+2rd%r$g<+ zgRrwpmdw!mUSdks;v1i2l6qdw<=mWFg3oA2nWRlhlvGbUc^|%!!`Ds@Wk3#I<3yS) zNJ(>|md7p7tk9D&79X>vBRJ`?BQQgNg`t^cN+1s9h@6w2eh@H2j+Ge%u7V!x%_&wM zfZBs0rH9hh??{?-a9P7nDP)RDBC8K!hj$yFfU!c3eZ3~8yVFBD#*42RJ`>L9yqNQ> zl;t$35_Z4AhaU5Z7tD9bhhXs|I_0J7d0b9Ms|eiZP8U_`(B?92b+xm=47p7|FS&=Z zGQ+d;O+|<6!c6qnZ@I4oD;cUiXs#66NRJ`Km@?TRJJ{n9Oae#1#{|;?YKkZXJ-`ib7us97 z0rtJKGp^tYDC4ZOlSS0xr=b3|{$W52!Tty&2N+ebniK0WkIkenI}51J&yck_byY=m z(Z4Y~F@&?Dc@z!qAtVtyK4@n#LM(PfWIN_`avto=o=`^+7HOsB7{=yPJmEW};U#!# zY=4)oQV&i>CzpovpFO!`c6`jeWgipYlzxb(zleM34!8Z~L{-nfl|1vN#H83(UGA?6tb^z>K$#NvO5xx6cYATS4-8Y3#KG;6k zK08JvC5&|5}$n-hcb$xvu#9a9>|O-Q6{prxB=I2i<+! z<}by)bGbftTqobI+V1<*cTbN$+)@7N>FejuAD{Pmn|kT}{Pg_sm!^L0`ok~Zx>Hp3 zhbgN1wdJL*?_0Gz4W)ZS#mSau%UUg0LD%)Clz^dk9~#2FZfXEuH#LwSf_(_{A;6neY6x|k%2N;V-8|}Ox(4Gz zcsH+SG+hJnA-ad)UOwOm;zM)~fjz|Z5YIy_kBm_?lQBOJZPK-U^yRyQD+_fzzI=cS z(>E;*Mh1=jS!<*mTNfEhoSB?ePH^RF@M(IY>b)y_ScC`@M*SoqHJ86DNos&ncPG^ML&m$Trh4Xo`Okp z3=^kl=D-tk@Uu%3p;x4d$=f-}r7dTh=7ioZ%OFa#nK(o#o0&5XW^&fejD=)1xz>Uu znT=#DCh}2Rw6|Nwt^bK-QYi!+tX|c2PSG2! zBhux`yJgMnj1KWxPloa~r=C4`|GT;*uWY+=+u2>Ck>SLG z+ibGJJ37{rf#^6p1?vQV?4p7@1rZ0bHprz7dLqFt1Rw{6sP!^|$8<8SPGVFp|Nfo) zF_N8hLBPuByzTyfd5g zVa3dcU&P4$Y?fNQ-cn;GGw6#IINK|ynBuA;IyENKX~c??s-HQ|@PZZofB&1$dfRV^ z;D7(~y9qjvJRT}fA!hH)>>&_Ws35mI9kw@KrKe01(!I=C1)45_sFS%eGFCsUKBrsc&S#vvvK}Vonug*U~3Pz z;B5|_g9Vidv20~8Wscg&f|_nM8{I24A6+i??)y_ea? zzi@uMuXy!Dr{0t=j^7&T#4|5k;guqlI;3FPiyf0qB4q=B7RyK6gWz0A0cSCb#NJAg zM0Z2xSnzBk7g|6y?i7S7^d!81&H-b@I8X}-N_wUZyBkFaupt939ZP*;V6m305td3u z%_%8~^e-K#A%>XsMTSuGf*=<+P~>Cf1+RM6y7AC`)rB2$2gBv8)T&KAb8 zA(P_FQL2Js-mpt{hc72MEH%g(jO1Ng77ZLl{#vLzr2t5T!ad^-1pHoXHC#HMD`p7X zGJ}3qf^;6KAy@%Db6P0FuiW5@d(kiybhqyuVCy@vqn5JGsc?ffQaLC$*c^9mxH4X~ zvA=LqMZGy!TSdhTYSKG3mY37c4a{e5c)|Q1^mXxVXHMra_Tn__ZI!OaTm6Gey++9L z2jhB75!B<&6jY){- z$mo(ted`s9pf$J1D{aM_9)blU;$Z?90LNvt0T`Pa%5b9edSjy!XXK@MU*%v zp{b@^$in#(S%{@2(WH>Fx)DAKOSo(V`q{Fh%8eM1#S%5z$s(%h$#6{?vMAOmH>5Ro zBo=~L$|FZY`u81)^Y|MbiJctQ8p^jg5`3EjaB>5hhzyd8VXs(kO5hiU1IWB5VmfSvcOJ@KanuOiCoM7SzSCtf&JjCa+G4 zMuSqcJ1yvZyV_rU0b4l*$1aOEPRbAJWoucjs!BAt4KZ2OpvlqhHAFnFNKRbMsg)dj z(of*%ogUsgJy0*ocrrWmXe<>r~$N75N(J+K+v%;hInx#BA6(}&FaasqnaFK4t8x>RIs1z z*^$tYlAOGTQA5%YlhbfgN>n)ytO9Z1h7x14sfjTyK_ky2Wv-P3S1o0Ac(g>BK!V_xAc#$BlslHMT@zh+NQc8v# zQtkzmBo1KWu`~!e2{lL-L@cR|D!dDOjqizko;?aNqO7?HYsBpotH+6cb%iJZZ77CpB8+<|eKy%Z=L=IfD=md7 zoTOqc8BCp>gqMUT9K?ZvZJ%Il?Wu_5=A7z)6G65(v}OE4?0C!!O)5MAS64tTUV$dG z4XJ465`S9#jV90pkz1Ou8gGhAbUUobSLeBwLXV9&~R$fd#Vr=1`Ioe1J(^RM>#@Kho{w(ef4!YQjz58d@U zchV3qT;z4)#J_WAW@-MwlcpN1DG7*VY#{~KB(^o7kV4RKX)f7n zuc>SpIoeezL4_uCDZ$3od5C@GzVssv%(^YMq&|1H!}N5vYCR=?U<8fR!S<^e0mh|D z2Jt&1ym3ZoAzT=Nte4z!h$Mak#K)TmV@CFEM1l`7M<_361mUo6;2;7!KrJQ7XMs1e zamCYRGNO-BL&O9RBw}IV0r4Eb0|2rE0Z4%)n*bjm`IIWZ0bayh$0ylki4wWCEV?%w z5^_r}Ejs}=rIL6KKI)DP8={y#0lIyEiL6^T4uqhZMCh=|Kxj;f47Gj(7r!C3q!2p? zC$>HprS3KbFD#=$B0VgmkakkIGG0}2x^OBRb7G~qZKKM0aTzokw48Qgh-YGW*$TJ% z4w^5fHdJIxz9|P$jlSkMTwdrE@*{uF{aYa(lZC9+UA}Pdy-vD!{7*czwPZKIV3r+C4`7TE@ zritQYCs#2-N||Rw+qeh~bW} z?4ksyHrGM+^wqnVNMIUrTNZ^(=Ql#ZX@<>%vLr!RJOZ<1Y4%aokLiY&`UWjsp?z>l9ucP?M!iHyc+X7KE-%UId3&4 zY&21N^h z08j}&1#yopyM-DSE?^BYi3KIqHYSZYQSsO%iy^90!%wVE1iX{leW=YQTQ)i!N&^o< z_bzENL+*QtDHV&azmo|vU4lxf+O-6qQMWS1Iw_Gu+3jSldnbptP7b+44p!nsk}L>8 za-y2Y&CsmSlip|Rlcpm$$hIRegM)=3nPmzf4&Bb0g6w{XUWXres9YK-~CP~;SxI|t^Wc4AyF!d1>9;WH=nHB5ft-6{|e)z@PjX&MAp4V5>3)Seh^<TlNwGl5&a~WRh5ua7Wew zrtD(sZeMb%o4uC?{Ghv+^lycTG_a)Wv88d+p-4lH{;+*1LY_78U=9>fU}VkvUqQTe zifEy%6u~g{HTM-@B}274$(2GI$u=bKLn7T|2YcLuN#F=rA7ENQO%XXK8@S=^LVF80 zz`i%S#}!-wWt@a|QjB8plN0|MHm3Iu?h2Mm4n`HM=D@PeV>9XV&H|$I6J%`;O;J#t zXKT+4VRUOAd4;=kAx7OksCzM-&$>lq+ve2!3kVY^w^0}yM;&axRD0ec{`?;nH cYBCXjbge%oWB&sH0RR630ApM;5wl4E0I=wF8vp Date: Wed, 12 Feb 2025 13:54:43 -0500 Subject: [PATCH 4/8] more logging and lint --- nhsn/delphi_nhsn/pull.py | 5 ++++- nhsn/delphi_nhsn/run.py | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nhsn/delphi_nhsn/pull.py b/nhsn/delphi_nhsn/pull.py index 515547f67..8ef3f715d 100644 --- a/nhsn/delphi_nhsn/pull.py +++ b/nhsn/delphi_nhsn/pull.py @@ -58,7 +58,10 @@ def check_last_updated(socrata_token, dataset_id, logger): def pull_data(socrata_token: str, dataset_id: str, backup_dir: str, logger): """Pull data from Socrata API.""" client = Socrata("data.cdc.gov", socrata_token) - logger.info("Pulling data from Socrata API") + logger.info( + f"Pulling {'main' if dataset_id == MAIN_DATASET_ID else 'preliminary'} data from Socrata API", + dataset_id=dataset_id, + ) results = [] offset = 0 limit = 50000 # maximum limit allowed by SODA 2.0 diff --git a/nhsn/delphi_nhsn/run.py b/nhsn/delphi_nhsn/run.py index c61c5199d..b77f9fb36 100644 --- a/nhsn/delphi_nhsn/run.py +++ b/nhsn/delphi_nhsn/run.py @@ -66,9 +66,7 @@ def run_module(params, logger=None): signal_df_dict.update({signal: nhsn_df for signal in SIGNALS_MAP}) # some of the source backups do not include for preliminary data if not preliminary_nhsn_df.empty: - signal_df_dict.update( - {signal: preliminary_nhsn_df for signal in PRELIM_SIGNALS_MAP} - ) + signal_df_dict.update({signal: preliminary_nhsn_df for signal in PRELIM_SIGNALS_MAP}) for geo, signals_df in product(GEOS, signal_df_dict.items()): signal, df_pull = signals_df From ca0323a77461f19306eb7d121c53dba9c470f453 Mon Sep 17 00:00:00 2001 From: Amaris Sim Date: Thu, 13 Feb 2025 08:44:48 -0500 Subject: [PATCH 5/8] suggested changes and refactor pull_nhsn_data --- nhsn/delphi_nhsn/pull.py | 85 ++++------------ nhsn/delphi_nhsn/run.py | 6 +- nhsn/tests/test_data/expected_df.csv | 21 ++++ nhsn/tests/test_data/expected_df_prelim.csv | 20 ++++ nhsn/tests/test_patch.py | 2 +- nhsn/tests/test_pull.py | 101 +++++--------------- 6 files changed, 84 insertions(+), 151 deletions(-) create mode 100644 nhsn/tests/test_data/expected_df.csv create mode 100644 nhsn/tests/test_data/expected_df_prelim.csv diff --git a/nhsn/delphi_nhsn/pull.py b/nhsn/delphi_nhsn/pull.py index 8ef3f715d..d872a62b3 100644 --- a/nhsn/delphi_nhsn/pull.py +++ b/nhsn/delphi_nhsn/pull.py @@ -123,7 +123,9 @@ def pull_nhsn_data( backup_dir: str, custom_run: bool, issue_date: Optional[str], + preliminary: bool = False, logger: Optional[logging.Logger] = None, + ): """Pull the latest NHSN hospital admission data, and conforms it into a dataset. @@ -140,6 +142,10 @@ def pull_nhsn_data( Directory to which to save raw backup data custom_run: bool Flag indicating if the current run is a patch. If so, don't save any data to disk + preliminary: bool + Flag indicating if the grabbing main or preliminary data + issue_date: + date to indicate which backup file to pull for patching logger: Optional[logging.Logger] logger object @@ -148,22 +154,26 @@ def pull_nhsn_data( pd.DataFrame Dataframe as described above. """ + dataset_id = PRELIM_DATASET_ID if preliminary else MAIN_DATASET_ID # Pull data from Socrata API df = ( - pull_data(socrata_token, MAIN_DATASET_ID, backup_dir, logger) + pull_data(socrata_token, dataset_id, backup_dir, logger) if not custom_run - else pull_data_from_file(backup_dir, issue_date, logger, prelim_flag=False) + else pull_data_from_file(backup_dir, issue_date, logger, prelim_flag=preliminary) ) - recently_updated = True if custom_run else check_last_updated(socrata_token, MAIN_DATASET_ID, logger) + recently_updated = True if custom_run else check_last_updated(socrata_token, dataset_id, logger) + + type_dict = PRELIM_TYPE_DICT if preliminary else TYPE_DICT + keep_columns = list(type_dict.keys()) + filtered_type_dict = copy.deepcopy(type_dict) - keep_columns = list(TYPE_DICT.keys()) + signal_map = PRELIM_SIGNALS_MAP if preliminary else SIGNALS_MAP if not df.empty and recently_updated: df = df.rename(columns={"weekendingdate": "timestamp", "jurisdiction": "geo_id"}) - filtered_type_dict = copy.deepcopy(TYPE_DICT) - for signal, col_name in SIGNALS_MAP.items(): + for signal, col_name in signal_map.items(): # older backups don't have certain columns try: df[signal] = df[col_name] @@ -181,66 +191,3 @@ def pull_nhsn_data( df = pd.DataFrame(columns=keep_columns) return df - - -def pull_preliminary_nhsn_data( - socrata_token: str, - backup_dir: str, - custom_run: bool, - issue_date: Optional[str], - logger: Optional[logging.Logger] = None, -): - """Pull the latest preliminary NHSN hospital admission data, and conforms it into a dataset. - - The output dataset has: - - - Each row corresponds to a single observation - - Each row additionally has columns for the signals in SIGNALS - - Parameters - ---------- - socrata_token: str - My App Token for pulling the NHSN data - backup_dir: str - Directory to which to save raw backup data - custom_run: bool - Flag indicating if the current run is a patch. If so, don't save any data to disk - logger: Optional[logging.Logger] - logger object - - Returns - ------- - pd.DataFrame - Dataframe as described above. - """ - # Pull data from Socrata API - df = ( - pull_data(socrata_token, PRELIM_DATASET_ID, backup_dir, logger) - if not custom_run - else pull_data_from_file(backup_dir, issue_date, logger, prelim_flag=True) - ) - - keep_columns = list(PRELIM_TYPE_DICT.keys()) - recently_updated = True if custom_run else check_last_updated(socrata_token, PRELIM_DATASET_ID, logger) - - if not df.empty and recently_updated: - df = df.rename(columns={"weekendingdate": "timestamp", "jurisdiction": "geo_id"}) - filtered_type_dict = copy.deepcopy(PRELIM_TYPE_DICT) - - for signal, col_name in PRELIM_SIGNALS_MAP.items(): - try: - df[signal] = df[col_name] - except KeyError: - logger.info("column not available in data", col_name=col_name, signal=signal) - keep_columns.remove(signal) - del filtered_type_dict[signal] - - df = df[keep_columns] - df = df.astype(filtered_type_dict) - - df["geo_id"] = df["geo_id"].str.lower() - df.loc[df["geo_id"] == "usa", "geo_id"] = "us" - else: - df = pd.DataFrame(columns=keep_columns) - - return df diff --git a/nhsn/delphi_nhsn/run.py b/nhsn/delphi_nhsn/run.py index b77f9fb36..ad9be2384 100644 --- a/nhsn/delphi_nhsn/run.py +++ b/nhsn/delphi_nhsn/run.py @@ -24,7 +24,7 @@ from delphi_utils.export import create_export_csv from .constants import GEOS, PRELIM_SIGNALS_MAP, SIGNALS_MAP -from .pull import pull_nhsn_data, pull_preliminary_nhsn_data +from .pull import pull_nhsn_data def run_module(params, logger=None): @@ -56,8 +56,8 @@ def run_module(params, logger=None): export_start_date = export_start_date.strftime("%Y-%m-%d") nhsn_df = pull_nhsn_data(socrata_token, backup_dir, custom_run=custom_run, issue_date=issue_date, logger=logger) - preliminary_nhsn_df = pull_preliminary_nhsn_data( - socrata_token, backup_dir, custom_run=custom_run, issue_date=issue_date, logger=logger + preliminary_nhsn_df = pull_nhsn_data( + socrata_token, backup_dir, custom_run=custom_run, issue_date=issue_date, logger=logger, preliminary=True ) geo_mapper = GeoMapper() diff --git a/nhsn/tests/test_data/expected_df.csv b/nhsn/tests/test_data/expected_df.csv new file mode 100644 index 000000000..556a65b07 --- /dev/null +++ b/nhsn/tests/test_data/expected_df.csv @@ -0,0 +1,21 @@ +timestamp,geo_id,confirmed_admissions_covid_ew,confirmed_admissions_flu_ew,confirmed_admissions_rsv_ew,hosprep_confirmed_admissions_covid_ew,hosprep_confirmed_admissions_flu_ew,hosprep_confirmed_admissions_rsv_ew +2021-08-21,md,53.0,2.0,0.0,13.0,13.0,1.0 +2021-08-21,co,852.0,0.0,,92.0,78.0,0.0 +2021-08-21,us,10384.0,6049.0,84.0,5426.0,5426.0,469.0 +2021-08-28,co,835.0,1.0,,92.0,78.0,0.0 +2021-08-28,us,94596.0,262.0,,5391.0,4397.0,0.0 +2021-09-04,co,1000.0,3.0,,92.0,78.0,0.0 +2021-09-04,us,93241.0,282.0,,5392.0,4396.0,0.0 +2021-09-11,co,982.0,2.0,,92.0,78.0,0.0 +2021-09-11,us,88162.0,247.0,,5391.0,4377.0,0.0 +2021-09-18,co,955.0,0.0,,92.0,78.0,0.0 +2021-09-18,us,79169.0,261.0,,5394.0,4362.0,0.0 +2021-09-25,co,993.0,0.0,,92.0,78.0,0.0 +2021-09-25,us,67740.0,234.0,,5393.0,4368.0,0.0 +2021-10-02,co,970.0,0.0,,92.0,78.0,0.0 +2021-10-02,us,58076.0,253.0,,5395.0,4391.0,0.0 +2021-10-09,co,1079.0,1.0,,92.0,78.0,0.0 +2021-10-09,us,51744.0,341.0,,5396.0,4379.0,0.0 +2021-10-16,co,1231.0,0.0,,92.0,78.0,0.0 +2021-10-16,us,45978.0,266.0,,5394.0,4307.0,0.0 +2021-10-16,region 1,45978.0,266.0,,5394.0,4307.0,0.0 diff --git a/nhsn/tests/test_data/expected_df_prelim.csv b/nhsn/tests/test_data/expected_df_prelim.csv new file mode 100644 index 000000000..7a7fb367d --- /dev/null +++ b/nhsn/tests/test_data/expected_df_prelim.csv @@ -0,0 +1,20 @@ +timestamp,geo_id,confirmed_admissions_covid_ew_prelim,confirmed_admissions_flu_ew_prelim,confirmed_admissions_rsv_ew_prelim,hosprep_confirmed_admissions_covid_ew_prelim,hosprep_confirmed_admissions_flu_ew_prelim,hosprep_confirmed_admissions_rsv_ew_prelim +2021-08-21,mi,269.0,523.0,1.0,152.0,152.0,4.0 +2021-08-21,co,852.0,0.0,,92.0,78.0,0.0 +2021-08-21,us,8946.0,5576.0,61.0,5422.0,5422.0,485.0 +2021-08-28,co,835.0,1.0,,92.0,78.0,0.0 +2021-08-28,us,94596.0,262.0,,5391.0,4397.0,0.0 +2021-09-04,co,1000.0,3.0,,92.0,78.0,0.0 +2021-09-04,us,93241.0,282.0,,5392.0,4396.0,0.0 +2021-09-11,co,982.0,2.0,,92.0,78.0,0.0 +2021-09-11,us,88162.0,247.0,,5391.0,4377.0,0.0 +2021-09-18,co,955.0,0.0,,92.0,78.0,0.0 +2021-09-18,us,79169.0,261.0,,5394.0,4362.0,0.0 +2021-09-25,co,993.0,0.0,,92.0,78.0,0.0 +2021-09-25,us,67740.0,234.0,,5393.0,4368.0,0.0 +2021-10-02,co,970.0,0.0,,92.0,78.0,0.0 +2021-10-02,us,58076.0,253.0,,5395.0,4391.0,0.0 +2021-10-09,co,1079.0,1.0,,92.0,78.0,0.0 +2021-10-09,us,51744.0,341.0,,5396.0,4379.0,0.0 +2021-10-16,co,1231.0,0.0,,92.0,78.0,0.0 +2021-10-16,us,45978.0,266.0,,5394.0,4307.0,0.0 diff --git a/nhsn/tests/test_patch.py b/nhsn/tests/test_patch.py index 9f3f02d00..10b74c26d 100644 --- a/nhsn/tests/test_patch.py +++ b/nhsn/tests/test_patch.py @@ -13,7 +13,7 @@ from delphi_nhsn.patch import filter_source_files, patch from delphi_nhsn.constants import TOTAL_ADMISSION_COVID_COL, TOTAL_ADMISSION_FLU_COL, \ NUM_HOSP_REPORTING_FLU_COL, NUM_HOSP_REPORTING_COVID_COL, GEOS, TOTAL_ADMISSION_COVID, TOTAL_ADMISSION_FLU, \ - NUM_HOSP_REPORTING_COVID, NUM_HOSP_REPORTING_FLU, NUM_HOSP_REPORTING_RSV_COL, TOTAL_ADMISSION_RSV_COL + NUM_HOSP_REPORTING_RSV_COL, TOTAL_ADMISSION_RSV_COL from conftest import TEST_DATA, PRELIM_TEST_DATA, TEST_DIR class TestPatch: diff --git a/nhsn/tests/test_pull.py b/nhsn/tests/test_pull.py index cd30f0b40..f8e27868b 100644 --- a/nhsn/tests/test_pull.py +++ b/nhsn/tests/test_pull.py @@ -10,7 +10,7 @@ pull_nhsn_data, pull_data, pull_data_from_file, - pull_preliminary_nhsn_data, check_last_updated + check_last_updated ) from delphi_nhsn.constants import TYPE_DICT, PRELIM_TYPE_DICT, PRELIM_DATASET_ID, MAIN_DATASET_ID @@ -21,12 +21,16 @@ "test_data": TEST_DATA, "msg_prefix": "", "prelim_flag": False, + "expected_data": f"{TEST_DIR}/test_data/expected_df.csv", + "type_dict": TYPE_DICT, }, {"id":PRELIM_DATASET_ID, "test_data":PRELIM_TEST_DATA, "msg_prefix": "Preliminary ", "prelim_flag": True, + "expected_data": f"{TEST_DIR}/test_data/expected_df_prelim.csv", + "type_dict": PRELIM_TYPE_DICT, } ] @@ -75,123 +79,63 @@ def test_pull_from_file(self, caplog, dataset, params_w_patch): @patch("delphi_nhsn.pull.Socrata") @patch("delphi_nhsn.pull.create_backup_csv") - def test_pull_nhsn_data_output(self, mock_create_backup, mock_socrata, caplog, params): + @pytest.mark.parametrize('dataset', DATASETS, ids=["data", "prelim_data"]) + def test_pull_nhsn_data_output(self, mock_create_backup, mock_socrata, dataset, caplog, params): now = time.time() # Mock Socrata client and its get method mock_client = MagicMock() mock_socrata.return_value = mock_client - mock_client.get.side_effect = [TEST_DATA,[]] - + mock_client.get.side_effect = [dataset["test_data"],[]] mock_client.get_metadata.return_value = {"rowsUpdatedAt": now} backup_dir = params["common"]["backup_dir"] test_token = params["indicator"]["socrata_token"] custom_run = params["common"]["custom_run"] - logger = get_structured_logger() - result = pull_nhsn_data(test_token, backup_dir, custom_run, issue_date=None, logger=logger) + expected_df = pd.read_csv(dataset["expected_data"]) + + result = pull_nhsn_data(test_token, backup_dir, custom_run, issue_date=None, logger=logger, preliminary=dataset["prelim_flag"]) mock_create_backup.assert_called_once() - expected_columns = set(TYPE_DICT.keys()) + expected_columns = set(expected_df.columns) assert set(result.columns) == expected_columns for column in list(result.columns): # some states don't report confirmed admissions rsv - if column == "confirmed_admissions_rsv_ew": + if column == "confirmed_admissions_rsv_ew" and not dataset["prelim_flag"]: + continue + if column == "confirmed_admissions_rsv_ew_prelim" and dataset["prelim_flag"]: continue assert result[column].notnull().all(), f"{column} has rogue NaN" + expected_df = expected_df.astype(dataset["type_dict"]) - @patch("delphi_nhsn.pull.Socrata") - def test_pull_nhsn_data_backup(self, mock_socrata, caplog, params): - now = time.time() - # Mock Socrata client and its get method - mock_client = MagicMock() - mock_socrata.return_value = mock_client - mock_client.get.side_effect = [TEST_DATA, []] - - mock_client.get_metadata.return_value = {"rowsUpdatedAt": now} + pd.testing.assert_frame_equal(expected_df, result) - today = pd.Timestamp.today().strftime("%Y%m%d") - backup_dir = params["common"]["backup_dir"] - custom_run = params["common"]["custom_run"] - test_token = params["indicator"]["socrata_token"] - - # Load test data - expected_data = pd.DataFrame(TEST_DATA) - - logger = get_structured_logger() - # Call function with test token - pull_nhsn_data(test_token, backup_dir, custom_run, issue_date=None, logger=logger) - - # Check logger used: - assert "Backup file created" in caplog.text - - # Check that backup file was created - backup_files = glob.glob(f"{backup_dir}/{today}*") - assert len(backup_files) == 2, "Backup file was not created" - - for backup_file in backup_files: - if backup_file.endswith(".csv.gz"): - dtypes = expected_data.dtypes.to_dict() - actual_data = pd.read_csv(backup_file, dtype=dtypes) - else: - actual_data = pd.read_parquet(backup_file) - pd.testing.assert_frame_equal(expected_data, actual_data) - - # clean up - for file in backup_files: - os.remove(file) @patch("delphi_nhsn.pull.Socrata") - @patch("delphi_nhsn.pull.create_backup_csv") - def test_pull_prelim_nhsn_data_output(self, mock_create_backup, mock_socrata, caplog, params): + @pytest.mark.parametrize('dataset', DATASETS, ids=["data", "prelim_data"]) + def test_pull_nhsn_data_backup(self, mock_socrata, dataset, caplog, params): now = time.time() # Mock Socrata client and its get method mock_client = MagicMock() mock_socrata.return_value = mock_client - mock_client.get.side_effect = [PRELIM_TEST_DATA, []] + mock_client.get.side_effect = [dataset["test_data"], []] mock_client.get_metadata.return_value = {"rowsUpdatedAt": now} - backup_dir = params["common"]["backup_dir"] - test_token = params["indicator"]["socrata_token"] - custom_run = params["common"]["custom_run"] - - logger = get_structured_logger() - - result = pull_preliminary_nhsn_data(test_token, backup_dir, custom_run, issue_date=None, logger=logger) - mock_create_backup.assert_called_once() - - expected_columns = set(PRELIM_TYPE_DICT.keys()) - assert set(result.columns) == expected_columns - - for column in list(result.columns): - # some states don't report confirmed admissions rsv - if column == "confirmed_admissions_rsv_ew_prelim": - continue - assert result[column].notnull().all(), f"{column} has rogue NaN" - @patch("delphi_nhsn.pull.Socrata") - def test_pull_prelim_nhsn_data_backup(self, mock_socrata, caplog, params): - now = time.time() - # Mock Socrata client and its get method - mock_client = MagicMock() - mock_socrata.return_value = mock_client - mock_client.get.side_effect = [PRELIM_TEST_DATA, []] - - mock_client.get_metadata.return_value = {"rowsUpdatedAt": now} today = pd.Timestamp.today().strftime("%Y%m%d") backup_dir = params["common"]["backup_dir"] custom_run = params["common"]["custom_run"] test_token = params["indicator"]["socrata_token"] # Load test data - expected_data = pd.DataFrame(PRELIM_TEST_DATA) + expected_data = pd.DataFrame(dataset["test_data"]) logger = get_structured_logger() # Call function with test token - pull_preliminary_nhsn_data(test_token, backup_dir, custom_run, issue_date=None, logger=logger) + pull_nhsn_data(test_token, backup_dir, custom_run, issue_date=None, logger=logger, preliminary=dataset["prelim_flag"]) # Check logger used: assert "Backup file created" in caplog.text @@ -212,6 +156,7 @@ def test_pull_prelim_nhsn_data_backup(self, mock_socrata, caplog, params): for file in backup_files: os.remove(file) + @pytest.mark.parametrize('dataset', DATASETS, ids=["data", "prelim_data"]) @pytest.mark.parametrize("updatedAt", [time.time(), time.time() - 172800], ids=["updated", "stale"]) @patch("delphi_nhsn.pull.Socrata") From 09c3f7b6fb178d52fbc63e18d6498e4f620c224a Mon Sep 17 00:00:00 2001 From: nmdefries <42820733+nmdefries@users.noreply.github.com> Date: Thu, 13 Feb 2025 10:28:39 -0500 Subject: [PATCH 6/8] linting --- nhsn/delphi_nhsn/pull.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nhsn/delphi_nhsn/pull.py b/nhsn/delphi_nhsn/pull.py index d872a62b3..4d3c9c839 100644 --- a/nhsn/delphi_nhsn/pull.py +++ b/nhsn/delphi_nhsn/pull.py @@ -124,8 +124,7 @@ def pull_nhsn_data( custom_run: bool, issue_date: Optional[str], preliminary: bool = False, - logger: Optional[logging.Logger] = None, - + logger: Optional[logging.Logger] = None ): """Pull the latest NHSN hospital admission data, and conforms it into a dataset. From ed84720cc5a7e48de46c628797db6ce217932806 Mon Sep 17 00:00:00 2001 From: nmdefries <42820733+nmdefries@users.noreply.github.com> Date: Thu, 13 Feb 2025 10:39:57 -0500 Subject: [PATCH 7/8] trailing comma --- nhsn/delphi_nhsn/pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nhsn/delphi_nhsn/pull.py b/nhsn/delphi_nhsn/pull.py index 4d3c9c839..8fbf9de1d 100644 --- a/nhsn/delphi_nhsn/pull.py +++ b/nhsn/delphi_nhsn/pull.py @@ -124,7 +124,7 @@ def pull_nhsn_data( custom_run: bool, issue_date: Optional[str], preliminary: bool = False, - logger: Optional[logging.Logger] = None + logger: Optional[logging.Logger] = None, ): """Pull the latest NHSN hospital admission data, and conforms it into a dataset. From 939c35478b7428675789b4c7ed769823a8de1588 Mon Sep 17 00:00:00 2001 From: Amaris Sim Date: Thu, 13 Feb 2025 13:14:47 -0500 Subject: [PATCH 8/8] fixing backup overwrite --- nhsn/delphi_nhsn/pull.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nhsn/delphi_nhsn/pull.py b/nhsn/delphi_nhsn/pull.py index 8fbf9de1d..31164770e 100644 --- a/nhsn/delphi_nhsn/pull.py +++ b/nhsn/delphi_nhsn/pull.py @@ -83,7 +83,8 @@ def pull_data(socrata_token: str, dataset_id: str, backup_dir: str, logger): if results: df = pd.DataFrame.from_records(results) - create_backup_csv(df, backup_dir, False, logger=logger) + sensor = "prelim" if dataset_id == PRELIM_DATASET_ID else None + create_backup_csv(df, backup_dir, False, sensor=sensor, logger=logger) else: df = pd.DataFrame() return df