concat outside of loops

Author: nmdefries

doctor_visits/delphi_doctor_visits/sensor.py

@@ -60,18 +60,17 @@ def fill_dates(y_data, dates):
         last_date = dates[-1]
         cols = y_data.columns
 
+        df_list = [y_data]
         if first_date not in y_data.index:
-            y_data = pd.concat([
-                y_data,
+            df_list.append(
                 pd.DataFrame(dict.fromkeys(cols, 0.0), columns=cols, index=[first_date])
-            ])
+            )
         if last_date not in y_data.index:
-            y_data = pd.concat([
-                y_data,
+            df_list.append(
                 pd.DataFrame(dict.fromkeys(cols, 0.0), columns=cols, index=[last_date])
-            ])
+            )
 
-        y_data.sort_index(inplace=True)
+        y_data = pd.concat(df_list).sort_index()
         y_data = y_data.asfreq("D", fill_value=0)
         return y_data
 

quidel_covidtest/delphi_quidel_covidtest/data_tools.py

@@ -30,14 +30,18 @@ def fill_dates(y_data, first_date, last_date):
     Returns: dataframe containing all dates given
     """
     cols = y_data.columns
+
+    df_list = [y_data]
     if first_date not in y_data.index:
-        y_data = pd.concat([y_data, pd.DataFrame(dict.fromkeys(cols, 0.),
-                                            columns=cols, index=[first_date])])
+        df_list.append(
+            pd.DataFrame(dict.fromkeys(cols, 0.), columns=cols, index=[first_date])
+        )
     if last_date not in y_data.index:
-        y_data = pd.concat([y_data, pd.DataFrame(dict.fromkeys(cols, 0.),
-                                            columns=cols, index=[last_date])])
+        df_list.append(
+            pd.DataFrame(dict.fromkeys(cols, 0.), columns=cols, index=[last_date])
+        )
 
-    y_data.sort_index(inplace=True)
+    y_data = pd.concat(df_list).sort_index()
     y_data = y_data.asfreq('D', fill_value=0)
     y_data.fillna(0, inplace=True)
     return y_data

quidel_covidtest/delphi_quidel_covidtest/generate_sensor.py

@@ -27,8 +27,8 @@ def generate_sensor_for_nonparent_geo(state_groups, res_key, smooth, device,
     Returns:
         df: pd.DataFrame
     """
-    state_df = pd.DataFrame(columns=["geo_id", "val", "se", "sample_size", "timestamp"])
     state_list = list(state_groups.groups.keys())
+    df_list = []
     for state in state_list:
         state_group = state_groups.get_group(state)
         state_group = state_group.drop(columns=[res_key])
@@ -63,15 +63,15 @@ def generate_sensor_for_nonparent_geo(state_groups, res_key, smooth, device,
             stat = stat * 100
 
         se = se * 100
-        state_df = pd.concat([
-            state_df,
+        df_list.append(
             pd.DataFrame({"geo_id": state,
                          "timestamp": state_group.index,
                          "val": stat,
                          "se": se,
                          "sample_size": sample_size})
-        ])
-    return remove_null_samples(state_df)
+        )
+
+    return remove_null_samples(pd.concat(df_list))
 
 def generate_sensor_for_parent_geo(state_groups, data, res_key, smooth,
                                    device, first_date, last_date, suffix):
@@ -91,9 +91,9 @@ def generate_sensor_for_parent_geo(state_groups, data, res_key, smooth,
         df: pd.DataFrame
     """
     has_parent = True
-    res_df = pd.DataFrame(columns=["geo_id", "val", "se", "sample_size"])
     if res_key == "fips": # Add rest-of-state report for county level
         data = add_megacounties(data, smooth)
+    df_list = []
     for loc, res_group in data.groupby(res_key):
         parent_state = res_group['state_id'].values[0]
         try:
@@ -150,12 +150,12 @@ def generate_sensor_for_parent_geo(state_groups, data, res_key, smooth,
                 stat = stat * 100
 
         se = se * 100
-        res_df = pd.concat([
-            res_df,
+        df_list.append(
             pd.DataFrame({"geo_id": loc,
                          "timestamp": res_group.index,
                          "val": stat,
                          "se": se,
                          "sample_size": sample_size})
-        ])
-    return remove_null_samples(res_df)
+        )
+
+    return remove_null_samples(pd.concat(df_list))

quidel_covidtest/delphi_quidel_covidtest/pull.py

@@ -34,7 +34,6 @@ def get_from_s3(start_date, end_date, bucket, logger):
                                'State', 'Zip', 'PatientAge', 'Result1',
                                'Result2', 'OverallResult', 'StorageDate',
                                'fname']
-    df = pd.DataFrame(columns=selected_columns)
     s3_files = {}
     for obj in bucket.objects.all():
         if "-sars" in obj.key:
@@ -52,25 +51,30 @@ def get_from_s3(start_date, end_date, bucket, logger):
                 s3_files[received_date].append(obj.key)
 
     n_days = (end_date - start_date).days + 1
+    df_list = []
+    seen_files = set()
     for search_date in [start_date + timedelta(days=x) for x in range(n_days)]:
         if search_date in s3_files.keys():
-            # Avoid appending duplicate datasets
             logger.info(f"Pulling data received on {search_date.date()}")
 
             # Fetch data received on the same day
             for fn in s3_files[search_date]:
+                # Skip non-CSV files, such as directories
                 if ".csv" not in fn:
-                    continue  #Add to avoid that the folder name was readed as a fn.
-                if fn in set(df["fname"].values):
+                    continue
+                # Avoid appending duplicate datasets
+                if fn in seen_files:
                     continue
                 obj = bucket.Object(key=fn)
                 newdf = pd.read_csv(obj.get()["Body"],
                                     parse_dates=["StorageDate", "TestDate"],
                                     low_memory=False)
+                seen_files.add(fn)
                 newdf["fname"] = fn
-                df = pd.concat([df, newdf[selected_columns]])
+                df_list.append(newdf[selected_columns])
                 time_flag = search_date
-    return df, time_flag
+
+    return pd.concat(df_list), time_flag
 
 def fix_zipcode(df):
     """Fix zipcode that is 9 digit instead of 5 digit."""