From 57da8ce99fc7f8907af887c8bc03d217973929a8 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 29 Jun 2023 23:51:02 +0300 Subject: [PATCH 1/9] Created maintenance folder. Added new scripts for API Keys maintenance. Deleted: * src/acquisition/covidcast/generate_islatest_fix_sql.py * src/acquisition/covidcast/islatest_fix.sql * src/acquisition/covidcast/migrate_epidata_to_v4.py Moved from acquisition to maintenance: * src/acquisition/covidcast/covidcast_meta_cache_updater.py * src/acquisition/covidcast/delete_batch.py * src/acquisition/covidcast/signal_dash_data_generator.py --- .../covidcast/test_covidcast_meta_caching.py | 2 +- integrations/client/test_delphi_epidata.py | 2 +- .../server/test_covidcast_endpoints.py | 2 +- integrations/server/test_covidcast_meta.py | 2 +- .../covidcast/generate_islatest_fix_sql.py | 63 - src/acquisition/covidcast/islatest_fix.sql | 1774 ----------------- .../covidcast/migrate_epidata_to_v4.py | 187 -- .../covidcast_meta_cache_updater.py | 0 .../covidcast => maintenance}/delete_batch.py | 0 src/maintenance/remove_outdated_keys.py | 62 + .../signal_dash_data_generator.py | 0 src/maintenance/update_last_usage.py | 58 + .../test_covidcast_meta_cache_updater.py | 4 +- .../test_signal_dash_data_generator.py | 3 +- 14 files changed, 127 insertions(+), 2032 deletions(-) delete mode 100644 src/acquisition/covidcast/generate_islatest_fix_sql.py delete mode 100644 src/acquisition/covidcast/islatest_fix.sql delete mode 100644 src/acquisition/covidcast/migrate_epidata_to_v4.py rename src/{acquisition/covidcast => maintenance}/covidcast_meta_cache_updater.py (100%) rename src/{acquisition/covidcast => maintenance}/delete_batch.py (100%) create mode 100644 src/maintenance/remove_outdated_keys.py rename src/{acquisition/covidcast => maintenance}/signal_dash_data_generator.py (100%) create mode 100644 src/maintenance/update_last_usage.py diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index e746c4ef1..6e4c6378f 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -13,7 +13,7 @@ from delphi.epidata.client.delphi_epidata import Epidata import delphi.operations.secrets as secrets import delphi.epidata.acquisition.covidcast.database as live -from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main +from delphi.epidata.maintenance.covidcast_meta_cache_updater import main # py3tester coverage target (equivalent to `import *`) __test_target__ = ( diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 9698d1274..4ef1fa6a3 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -11,7 +11,7 @@ # third party import delphi.operations.secrets as secrets -from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main as update_covidcast_meta_cache +from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_covidcast_meta_cache from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow, FIPS, MSA from delphi.epidata.client.delphi_epidata import Epidata from delphi_utils import Nans diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 41b74ac03..c86bb10d6 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -9,7 +9,7 @@ import requests import pandas as pd -from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main as update_cache +from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_cache from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow # use the local instance of the Epidata API diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index 95a51e354..ad297f1e8 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -9,7 +9,7 @@ #first party from delphi_utils import Nans -from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main as update_cache +from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_cache import delphi.operations.secrets as secrets # use the local instance of the Epidata API diff --git a/src/acquisition/covidcast/generate_islatest_fix_sql.py b/src/acquisition/covidcast/generate_islatest_fix_sql.py deleted file mode 100644 index 115a7d131..000000000 --- a/src/acquisition/covidcast/generate_islatest_fix_sql.py +++ /dev/null @@ -1,63 +0,0 @@ -# what data to operate on -base_where_clause = "WHERE `source`='jhu-csse' AND `time_type`='day'" -### base_where_clause = "WHERE `source`='src2' AND `time_type`='day'" ### - - -# signal name construction -# NOTE: selecting these (unique) from the database takes 7-8 mins, so reconstructing here for efficiency -# TODO: maybe just put the damn UNIQUE query in here so you dont fat-finger it again george. -# also these hardcoded signals are unique to JHU data, or are at least not used by all sources. -signals = [] -for case in ('confirmed_', 'deaths_'): - for period in ('7dav_', ''): # NOTE: that is a V as in "7 Day AVerage", not a "Y" as in "7 DAY" - for count in ('cumulative_', 'incidence_'): - for typ in ('num', 'prop'): - signals.append(case+period+count+typ) -### signals = ['sig2'] ### -# variable to split on, 'time_value' is good because its high cardinality is suitable for chunking -PARTITION_VARIABLE = 'time_value' -PARTITION_SPLITS = [20200101 + i*100 for i in range(10)] # first day of the month for jan - oct 2020 in YYYYMMDD form -### PARTITION_SPLITS = [1,2] ### - - -print(''' --- --- run this as: --- python3 generate_islatest_fix_sql.py > islatest_fix.sql --- mysql -vvv -p epidata < islatest_fix.sql --- or: --- date ; (python3 generate_islatest_fix_sql.py | mysql -vvv -p epidata ) ; date --- -''') - -# create temp table -print("CREATE TABLE `islatest_fix` (`latest_id` INT(11) NOT NULL, PRIMARY KEY (`latest_id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8;") - -# find latest issue by partition (and by signal) and save primary ids into temp table -for partition_index in range(len(PARTITION_SPLITS)+1): - ge_condition = 'TRUE' if partition_index == 0 else f'`{PARTITION_VARIABLE}` >= {PARTITION_SPLITS[partition_index - 1]}' - l_condition = 'TRUE' if partition_index == len(PARTITION_SPLITS) else f'`{PARTITION_VARIABLE}` < {PARTITION_SPLITS[partition_index]}' - partition_condition = f'({ge_condition}) AND ({l_condition})' - for sig in signals: - where_clause = base_where_clause + " AND `signal`='%s' AND %s" % (sig, partition_condition) - - print(''' -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - ''' + where_clause + ''' - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); -''') - -# clear any current (potentially erroneous) is_latest_issue flags -print("UPDATE `covidcast` SET `is_latest_issue`=0 " + base_where_clause + " AND `is_latest_issue`=1;") - -# re-set proper is_latest_issue flags -print("UPDATE (SELECT `latest_id` FROM `islatest_fix`) xxx LEFT JOIN `covidcast` ON `xxx`.`latest_id`=`covidcast`.`id` SET `covidcast`.`is_latest_issue`=1;") - -# clean up temp table -print("-- TODO: drop this table") -print("-- DROP TABLE `islatest_fix`;") \ No newline at end of file diff --git a/src/acquisition/covidcast/islatest_fix.sql b/src/acquisition/covidcast/islatest_fix.sql deleted file mode 100644 index cd5475c74..000000000 --- a/src/acquisition/covidcast/islatest_fix.sql +++ /dev/null @@ -1,1774 +0,0 @@ - --- --- run this as: --- python3 generate_islatest_fix_sql.py > islatest_fix.sql --- mysql -vvv -p epidata < islatest_fix.sql --- or: --- date ; (python3 generate_islatest_fix_sql.py | mysql -vvv -p epidata ) ; date --- - -CREATE TABLE `islatest_fix` (`latest_id` INT(11) NOT NULL, PRIMARY KEY (`latest_id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_num' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_prop' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_num' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_prop' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_num' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_prop' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_num' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_prop' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_num' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_prop' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_num' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_prop' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_num' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_prop' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_num' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_prop' AND (TRUE) AND (`time_value` < 20200101) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_num' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_prop' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_num' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_prop' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_num' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_prop' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_num' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_prop' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_num' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_prop' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_num' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_prop' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_num' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_prop' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_num' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_prop' AND (`time_value` >= 20200101) AND (`time_value` < 20200201) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_num' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_prop' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_num' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_prop' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_num' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_prop' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_num' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_prop' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_num' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_prop' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_num' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_prop' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_num' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_prop' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_num' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_prop' AND (`time_value` >= 20200201) AND (`time_value` < 20200301) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_num' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_prop' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_num' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_prop' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_num' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_prop' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_num' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_prop' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_num' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_prop' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_num' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_prop' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_num' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_prop' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_num' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_prop' AND (`time_value` >= 20200301) AND (`time_value` < 20200401) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_num' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_prop' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_num' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_prop' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_num' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_prop' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_num' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_prop' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_num' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_prop' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_num' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_prop' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_num' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_prop' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_num' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_prop' AND (`time_value` >= 20200401) AND (`time_value` < 20200501) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_num' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_prop' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_num' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_prop' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_num' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_prop' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_num' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_prop' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_num' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_prop' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_num' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_prop' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_num' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_prop' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_num' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_prop' AND (`time_value` >= 20200501) AND (`time_value` < 20200601) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_num' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_prop' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_num' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_prop' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_num' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_prop' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_num' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_prop' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_num' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_prop' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_num' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_prop' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_num' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_prop' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_num' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_prop' AND (`time_value` >= 20200601) AND (`time_value` < 20200701) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_num' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_prop' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_num' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_prop' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_num' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_prop' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_num' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_prop' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_num' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_prop' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_num' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_prop' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_num' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_prop' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_num' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_prop' AND (`time_value` >= 20200701) AND (`time_value` < 20200801) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_num' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_prop' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_num' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_prop' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_num' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_prop' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_num' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_prop' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_num' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_prop' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_num' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_prop' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_num' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_prop' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_num' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_prop' AND (`time_value` >= 20200801) AND (`time_value` < 20200901) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_num' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_prop' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_num' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_prop' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_num' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_prop' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_num' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_prop' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_num' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_prop' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_num' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_prop' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_num' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_prop' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_num' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_prop' AND (`time_value` >= 20200901) AND (`time_value` < 20201001) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_num' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_cumulative_prop' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_num' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_7dav_incidence_prop' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_num' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_cumulative_prop' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_num' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='confirmed_incidence_prop' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_num' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_cumulative_prop' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_num' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_7dav_incidence_prop' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_num' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_cumulative_prop' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_num' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - - -INSERT INTO `islatest_fix` - SELECT id FROM - ( SELECT `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, MAX(`issue`) AS `issue` FROM `covidcast` - WHERE `source`='jhu-csse' AND `time_type`='day' AND `signal`='deaths_incidence_prop' AND (`time_value` >= 20201001) AND (TRUE) - GROUP BY `source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value` - ) b - LEFT JOIN `covidcast` a - USING (`source`, `signal`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`); - -UPDATE `covidcast` SET `is_latest_issue`=0 WHERE `source`='jhu-csse' AND `time_type`='day' AND `is_latest_issue`=1; -UPDATE (SELECT `latest_id` FROM `islatest_fix`) xxx LEFT JOIN `covidcast` ON `xxx`.`latest_id`=`covidcast`.`id` SET `covidcast`.`is_latest_issue`=1; --- TODO: drop this table --- DROP TABLE `islatest_fix`; diff --git a/src/acquisition/covidcast/migrate_epidata_to_v4.py b/src/acquisition/covidcast/migrate_epidata_to_v4.py deleted file mode 100644 index f5522337e..000000000 --- a/src/acquisition/covidcast/migrate_epidata_to_v4.py +++ /dev/null @@ -1,187 +0,0 @@ -import argparse -import sys -import time -from delphi.epidata.acquisition.covidcast.database import Database - -# run as: -# python3 -u -m delphi.epidata.acquisition.covidcast.migrate_epidata_to_v4 -# ("-u" allows unbuffered print statements so we can watch timing in closer-to-real-time) - - -#####import delphi.operations.secrets as secrets -#####secrets.db.host = '172.30.n.n' # aka 'epidata-db-qa-01' -#####secrets.db.epi = ('delphi', 'xxxxxxxx') -# ^ these are already set appropriately on qa-automation in/by the operations module ^ - - -# TODO: make cli flags for these two variables: -use_transaction_wrappers = False -use_autocommit = False - -# TODO: maybe output: was autocommit enabled? was table locking used? what isolation type was used? were indexes enabled? were uniqueness checks enabled? - -# TODO: consider dropping indexes before moving data and recreating them afterward - -''' - -mysql> select count(id) from epidata.covidcast; -+------------+ -| count(id) | -+------------+ -| 2647381579 | -+------------+ -1 row in set (13 min 49.32 sec) - -mysql> select max(id) from epidata.covidcast; -+------------+ -| max(id) | -+------------+ -| 3740757041 | -+------------+ -1 row in set (0.00 sec) - --- so ~71% coverage of actual rows per allocated ids ( 2647381579 / 3740757041 = .70771278379851347314 ) - -mysql> select time_value, issue from epidata.covidcast where id=3740757041; -+------------+----------+ -| time_value | issue | -+------------+----------+ -| 20210927 | 20210930 | -+------------+----------+ -1 row in set (0.01 sec) - -mysql> select now(); -+---------------------+ -| now() | -+---------------------+ -| 2022-05-16 16:45:34 | -+---------------------+ -1 row in set (0.00 sec) - -''' - -def start_tx(cursor): - cursor.execute('SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;') - cursor.execute('SET autocommit=0;') # starts a transaction as suggested in https://dev.mysql.com/doc/refman/8.0/en/lock-tables.html - # NOTE: locks must be specified for any aliases of table names that are used - cursor.execute('''LOCK TABLES epidata.covidcast AS cc READ, - epimetric_load WRITE, epimetric_load AS sl WRITE, - epimetric_full WRITE, - epimetric_latest WRITE, - signal_dim WRITE, signal_dim AS sd READ, - geo_dim WRITE, geo_dim AS gd READ;''') - cursor.execute('SET unique_checks=0;') - -def finish_tx(cursor): - cursor.execute('SET unique_checks=1;') - cursor.execute('COMMIT;') - cursor.execute('UNLOCK TABLES;') - - -def do_batches(db, start, upper_lim, batch_size): - # NOTE: upper_lim is not actually selected for ; make sure it exceeds any ids you want to include - batch_lower = start - - while batch_lower < upper_lim: - batch_upper = min(batch_lower + batch_size, upper_lim) - - # NOTE: first rows of column names are identical, second rows are for specifying a rename and a literal - batch_sql = f""" - INSERT INTO epimetric_load ( - `issue`, `source`, `signal`, geo_type, geo_value, time_type, time_value, `value`, stderr, sample_size, `lag`, value_updated_timestamp, is_latest_issue, missing_value, missing_stderr, missing_sample_size - ) SELECT - `issue`, `source`, `signal`, geo_type, geo_value, time_type, time_value, `value`, stderr, sample_size, `lag`, value_updated_timestamp, is_latest_issue, missing_value, missing_stderr, missing_sample_size - FROM epidata.covidcast AS cc - USE INDEX(`PRIMARY`) - WHERE {batch_lower} <= cc.id AND cc.id < {batch_upper}; """ - # TODO: use LIMIT instead of id range?? - # TODO: might it be worth adding "ORDER BY id ASC" ? - - if use_transaction_wrappers: - start_tx(db._cursor) - - print(f"-=-=-=-=-=-=-=- RUNNING BATCH STARTING AT {batch_lower} -=-=-=-=-=-=-=-") - print(f"-=-=-=-=-=-=-=- RUNNING ''INSERT INTO SELECT FROM''... ", end="") - t = time.time() - db._cursor.execute(batch_sql) - print(f"elapsed: {time.time()-t} sec, rows: {db._cursor.rowcount} -=-=-=-=-=-=-=-") - - t = time.time() - db.run_dbjobs() - print(f"-=-=-=-=-=-=-=- RAN db_jobs()... elapsed: {time.time()-t} sec -=-=-=-=-=-=-=-") - - print("-=-=-=-=-=-=-=- RUNNING commit()... ", end="") - t = time.time() - db.commit() - if use_transaction_wrappers: - finish_tx(db._cursor) - print(f"elapsed: {time.time()-t} sec -=-=-=-=-=-=-=-") - - print("\n\n") - # move pointer for next batch - batch_lower = batch_upper - - -def main(destination_schema, batch_size, start_id, upper_lim_override): - Database.DATABASE_NAME = destination_schema - db = Database() - db.connect() - if use_autocommit: - db._connection.autocommit = True - - if upper_lim_override: - upper_lim = upper_lim_override - else: - # find upper limit for data to be imported - db._cursor.execute("SELECT MAX(id) FROM epidata.covidcast;") - for (max_id,) in db._cursor: - upper_lim = 1 + max_id - - print(f"migrating data to schema '{destination_schema}', with batch size {batch_size} and {start_id} <= ids < {upper_lim}") - if start_id==0: - print("this WILL truncate any existing v4 tables") - print() - if input("type 'yes' to continue: ") != 'yes': - sys.exit('operation cancelled!') - - print(f"starting run at: {time.strftime('%c')}") - - if start_id==0: - # clear tables in the v4 schema - print("truncating tables...") - for table in "epimetric_load epimetric_latest epimetric_full geo_dim signal_dim".split(): - db._cursor.execute(f"TRUNCATE TABLE {table}") - db.commit() - start_id = 1 - - # run batch loop - do_batches(db, start_id, upper_lim, batch_size) - - # get table counts [the quick and dirty way] - print("-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-") - db._cursor.execute(f"SELECT MAX(epimetric_id) FROM epimetric_full;") - for (max_id,) in db._cursor: - print(f"epimetric_full: {max_id}") - db._cursor.execute(f"SELECT MAX(epimetric_id) FROM epimetric_latest;") - for (max_id,) in db._cursor: - print(f"epimetric_latest: {max_id} (this should be <= the number above)") - db._cursor.execute(f"SELECT COUNT(signal_key_id), MAX(signal_key_id) FROM signal_dim;") - for (count_id, max_id) in db._cursor: - print(f"signal_dim: count {count_id} / max {max_id}") - db._cursor.execute(f"SELECT COUNT(geo_key_id), MAX(geo_key_id) FROM geo_dim;") - for (count_id, max_id) in db._cursor: - print(f"geo_dim: count {count_id} / max {max_id}") - - return upper_lim - - -if __name__ == '__main__': - argparser = argparse.ArgumentParser() - argparser.add_argument('--destination_schema', type=str, default='covid') - argparser.add_argument('--batch_size', type=int, default=20_000_000) - argparser.add_argument('--start_id', type=int, default=0) - argparser.add_argument('--upper_lim_override', type=int) # should default to None - args = argparser.parse_args() - - upper_lim = main(args.destination_schema, args.batch_size, args.start_id, args.upper_lim_override) - print(f"the next execution of this program should include argument: --start_id={upper_lim}") diff --git a/src/acquisition/covidcast/covidcast_meta_cache_updater.py b/src/maintenance/covidcast_meta_cache_updater.py similarity index 100% rename from src/acquisition/covidcast/covidcast_meta_cache_updater.py rename to src/maintenance/covidcast_meta_cache_updater.py diff --git a/src/acquisition/covidcast/delete_batch.py b/src/maintenance/delete_batch.py similarity index 100% rename from src/acquisition/covidcast/delete_batch.py rename to src/maintenance/delete_batch.py diff --git a/src/maintenance/remove_outdated_keys.py b/src/maintenance/remove_outdated_keys.py new file mode 100644 index 000000000..f62555d32 --- /dev/null +++ b/src/maintenance/remove_outdated_keys.py @@ -0,0 +1,62 @@ +from collections import namedtuple + +import delphi.operations.secrets as secrets +import mysql.connector + + +API_USER_RECORD = namedtuple("APIUser", ("api_key", "email", "date_diff")) + + +def get_outdated_keys(cur): + cur.execute( + """ + SELECT + diff.api_key, + diff.email, + diff.date_diff + FROM ( + SELECT + api_key, + email, + created, + last_time_used, + ABS(TIMESTAMPDIFF(MONTH, last_time_used, created)) as date_diff + FROM api_user + ) diff + WHERE diff.date_diff >= 5; + + """ + ) + outdated_keys = cur.fetchall() + return outdated_keys + + +def remove_outdated_key(cur, api_key): + cur.execute( + f""" + DELETE FROM api_user WHERE api_key = "{api_key}" + """ + ) + + +def send_notification(api_key, email): + pass + + +def main(): + u, p = secrets.db.epi + cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host) + cur = cnx.cursor() + outdated_keys_list = [API_USER_RECORD(*item) for item in get_outdated_keys(cur)] + for item in outdated_keys_list: + if item.date_diff == 5: + send_notification(item.email) + else: + remove_outdated_key(cur, item.api_key) + cur.close() + cnx.commit() + cnx.close() + + +if __name__ == "__main__": + main() diff --git a/src/acquisition/covidcast/signal_dash_data_generator.py b/src/maintenance/signal_dash_data_generator.py similarity index 100% rename from src/acquisition/covidcast/signal_dash_data_generator.py rename to src/maintenance/signal_dash_data_generator.py diff --git a/src/maintenance/update_last_usage.py b/src/maintenance/update_last_usage.py new file mode 100644 index 000000000..c6fdac944 --- /dev/null +++ b/src/maintenance/update_last_usage.py @@ -0,0 +1,58 @@ +import os +from datetime import datetime as dtime + +import delphi.operations.secrets as secrets +import mysql.connector +import redis + +REDIS_HOST = os.environ.get("REDIS_HOST", "delphi_redis") +REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "1234") +LAST_USED_KEY_PATTERN = "*LAST_USED*" + + +def redis_get_last_time_used(redis_cli, pattern: str): + keys = redis_cli.keys(pattern=pattern) + key_val_pairs = { + str(key).split("/")[1]: dtime.strptime(str(redis_cli.get(key)), "%Y-%m-%d").date() for key in keys + } + return key_val_pairs + + +def db_get_last_time_used(api_key: str, cur): + cur.execute( + f""" + SELECT + last_time_used + FROM api_user + WHERE api_key = "{api_key}" + LIMIT 1 + """ + ) + last_time_used = cur.fetchone() + return last_time_used[0] + + +def main(): + redis_cli = redis.Redis(host=REDIS_HOST, password=REDIS_PASSWORD, decode_responses=True) + key_val_pairs = redis_get_last_time_used(redis_cli, LAST_USED_KEY_PATTERN) + u, p = secrets.db.epi + cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host) + cur = cnx.cursor() + for api_key, last_time_used in key_val_pairs.items(): + db_last_time_used = db_get_last_time_used(api_key, cur) + if db_last_time_used == None or last_time_used > db_last_time_used: + cur.execute( + f""" + UPDATE + api_user + SET last_time_used = "{last_time_used}" + WHERE api_key = "{api_key}" + """ + ) + cur.close() + cnx.commit() + cnx.close() + + +if __name__ == "__main__": + main() diff --git a/tests/acquisition/covidcast/test_covidcast_meta_cache_updater.py b/tests/acquisition/covidcast/test_covidcast_meta_cache_updater.py index 40a242e22..60b0f654c 100644 --- a/tests/acquisition/covidcast/test_covidcast_meta_cache_updater.py +++ b/tests/acquisition/covidcast/test_covidcast_meta_cache_updater.py @@ -8,11 +8,11 @@ # third party import pandas -from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import get_argument_parser, \ +from delphi.epidata.maintenance.covidcast_meta_cache_updater import get_argument_parser, \ main # py3tester coverage target __test_target__ = ( - 'delphi.epidata.acquisition.covidcast.' + 'delphi.epidata.maintenance.' 'covidcast_meta_cache_updater' ) diff --git a/tests/acquisition/covidcast/test_signal_dash_data_generator.py b/tests/acquisition/covidcast/test_signal_dash_data_generator.py index 5d0c81326..09bca3227 100644 --- a/tests/acquisition/covidcast/test_signal_dash_data_generator.py +++ b/tests/acquisition/covidcast/test_signal_dash_data_generator.py @@ -12,8 +12,7 @@ import pandas as pd # first party -from delphi.epidata.acquisition.covidcast import signal_dash_data_generator -from delphi.epidata.acquisition.covidcast.signal_dash_data_generator import ( +from delphi.epidata.maintenance.signal_dash_data_generator import ( get_argument_parser, Database, DashboardSignalStatus, From d63f63f45880a451360b657762f19da5490b0acb Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Fri, 30 Jun 2023 21:46:50 +0300 Subject: [PATCH 2/9] Updated send_notification() function --- src/maintenance/remove_outdated_keys.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/maintenance/remove_outdated_keys.py b/src/maintenance/remove_outdated_keys.py index f62555d32..728a0efc5 100644 --- a/src/maintenance/remove_outdated_keys.py +++ b/src/maintenance/remove_outdated_keys.py @@ -1,11 +1,20 @@ from collections import namedtuple +from smtplib import SMTP import delphi.operations.secrets as secrets import mysql.connector - +from delphi.epidata.server._config import API_KEY_REGISTRATION_FORM_LINK_LOCAL API_USER_RECORD = namedtuple("APIUser", ("api_key", "email", "date_diff")) +SMTP_HOST = "relay.andrew.cmu.edu" +SMTP_PORT = 25 + +EMAIL_SUBJECT = "Your API Key was deleted." +EMAIL_FROM = "noreply@andrew.cmu.edu" +EMAIL_MESSAGE = f"""Hi! \n Your API Key was removed due to inactivity. +To get new one, please use registration form ({API_KEY_REGISTRATION_FORM_LINK_LOCAL}) or contact us.""" + def get_outdated_keys(cur): cur.execute( @@ -39,8 +48,10 @@ def remove_outdated_key(cur, api_key): ) -def send_notification(api_key, email): - pass +def send_notification(to_addr): + BODY = "\r\n".join((f"FROM: {EMAIL_FROM}", f"TO: {to_addr}", f"Subject: {EMAIL_SUBJECT}", "", EMAIL_MESSAGE)) + smtp_server = SMTP(host=SMTP_HOST, port=SMTP_PORT) + smtp_server.sendmail(EMAIL_FROM, to_addr, BODY) def main(): From 5475310e4be6656ddabe50fa0883378a8ee90fc1 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 26 Jul 2023 21:56:46 +0300 Subject: [PATCH 3/9] Removed pattern arg in redis_get_last_time_used and replaced it with global LAST_USED_KEY_PATTERN. Deleted function and moved filtering login to the update query in --- src/maintenance/update_last_usage.py | 36 ++++++++-------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/src/maintenance/update_last_usage.py b/src/maintenance/update_last_usage.py index c6fdac944..2a024067d 100644 --- a/src/maintenance/update_last_usage.py +++ b/src/maintenance/update_last_usage.py @@ -10,28 +10,14 @@ LAST_USED_KEY_PATTERN = "*LAST_USED*" -def redis_get_last_time_used(redis_cli, pattern: str): - keys = redis_cli.keys(pattern=pattern) +def redis_get_last_time_used(redis_cli): + keys = redis_cli.keys(pattern=LAST_USED_KEY_PATTERN) key_val_pairs = { str(key).split("/")[1]: dtime.strptime(str(redis_cli.get(key)), "%Y-%m-%d").date() for key in keys } return key_val_pairs -def db_get_last_time_used(api_key: str, cur): - cur.execute( - f""" - SELECT - last_time_used - FROM api_user - WHERE api_key = "{api_key}" - LIMIT 1 - """ - ) - last_time_used = cur.fetchone() - return last_time_used[0] - - def main(): redis_cli = redis.Redis(host=REDIS_HOST, password=REDIS_PASSWORD, decode_responses=True) key_val_pairs = redis_get_last_time_used(redis_cli, LAST_USED_KEY_PATTERN) @@ -39,16 +25,14 @@ def main(): cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host) cur = cnx.cursor() for api_key, last_time_used in key_val_pairs.items(): - db_last_time_used = db_get_last_time_used(api_key, cur) - if db_last_time_used == None or last_time_used > db_last_time_used: - cur.execute( - f""" - UPDATE - api_user - SET last_time_used = "{last_time_used}" - WHERE api_key = "{api_key}" - """ - ) + cur.execute( + f""" + UPDATE + api_user + SET last_time_used = "{last_time_used}" + WHERE api_key = "{api_key}" AND (last_time_used < "{last_time_used}" OR last_time_used IS NULL) + """ + ) cur.close() cnx.commit() cnx.close() From c947c3d59941fc8446e8b3377e5c8bdc505fffa6 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 26 Jul 2023 21:21:05 +0300 Subject: [PATCH 4/9] Moved tests from tests/acquisition to tests/maintenance folder. --- tests/maintenance/__init__.py | 4 ++++ .../test_covidcast_meta_cache_updater.py | 0 .../test_signal_dash_data_generator.py | 0 3 files changed, 4 insertions(+) create mode 100644 tests/maintenance/__init__.py rename tests/{acquisition/covidcast => maintenance}/test_covidcast_meta_cache_updater.py (100%) rename tests/{acquisition/covidcast => maintenance}/test_signal_dash_data_generator.py (100%) diff --git a/tests/maintenance/__init__.py b/tests/maintenance/__init__.py new file mode 100644 index 000000000..e197f3ec4 --- /dev/null +++ b/tests/maintenance/__init__.py @@ -0,0 +1,4 @@ +import sys +import os + +sys.path.append(os.getcwd()) diff --git a/tests/acquisition/covidcast/test_covidcast_meta_cache_updater.py b/tests/maintenance/test_covidcast_meta_cache_updater.py similarity index 100% rename from tests/acquisition/covidcast/test_covidcast_meta_cache_updater.py rename to tests/maintenance/test_covidcast_meta_cache_updater.py diff --git a/tests/acquisition/covidcast/test_signal_dash_data_generator.py b/tests/maintenance/test_signal_dash_data_generator.py similarity index 100% rename from tests/acquisition/covidcast/test_signal_dash_data_generator.py rename to tests/maintenance/test_signal_dash_data_generator.py From 852312cd86724daf0af9565c9e681e88aefa68da Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 26 Jul 2023 21:43:11 +0300 Subject: [PATCH 5/9] Removed unused import --- tests/maintenance/test_covidcast_meta_cache_updater.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/maintenance/test_covidcast_meta_cache_updater.py b/tests/maintenance/test_covidcast_meta_cache_updater.py index 60b0f654c..318206ea4 100644 --- a/tests/maintenance/test_covidcast_meta_cache_updater.py +++ b/tests/maintenance/test_covidcast_meta_cache_updater.py @@ -5,9 +5,6 @@ import unittest from unittest.mock import MagicMock -# third party -import pandas - from delphi.epidata.maintenance.covidcast_meta_cache_updater import get_argument_parser, \ main # py3tester coverage target From 9497203207abdcbc18ff87c9970b0022dc26c2e2 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 27 Jul 2023 19:23:23 +0300 Subject: [PATCH 6/9] Changed variables names. Added additional level of notifications. --- src/maintenance/remove_outdated_keys.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/maintenance/remove_outdated_keys.py b/src/maintenance/remove_outdated_keys.py index 728a0efc5..b4483a149 100644 --- a/src/maintenance/remove_outdated_keys.py +++ b/src/maintenance/remove_outdated_keys.py @@ -5,18 +5,20 @@ import mysql.connector from delphi.epidata.server._config import API_KEY_REGISTRATION_FORM_LINK_LOCAL -API_USER_RECORD = namedtuple("APIUser", ("api_key", "email", "date_diff")) +ApiUserRecord = namedtuple("APIUserRecord", ("api_key", "email", "date_diff")) SMTP_HOST = "relay.andrew.cmu.edu" SMTP_PORT = 25 EMAIL_SUBJECT = "Your API Key was deleted." EMAIL_FROM = "noreply@andrew.cmu.edu" -EMAIL_MESSAGE = f"""Hi! \n Your API Key was removed due to inactivity. +ALERT_EMAIL_MESSAGE = f"""Hi! \n Your API Key is going to be removed due to inactivity. +To renew it, pelase use it within one month from now.""" +DELETED_EMAIL_MESSAGE = f"""Hi! \n Your API Key was removed due to inactivity. To get new one, please use registration form ({API_KEY_REGISTRATION_FORM_LINK_LOCAL}) or contact us.""" -def get_outdated_keys(cur): +def get_old_keys(cur): cur.execute( """ SELECT @@ -37,7 +39,7 @@ def get_outdated_keys(cur): """ ) outdated_keys = cur.fetchall() - return outdated_keys + return [ApiUserRecord(*item) for item in outdated_keys] def remove_outdated_key(cur, api_key): @@ -48,8 +50,9 @@ def remove_outdated_key(cur, api_key): ) -def send_notification(to_addr): - BODY = "\r\n".join((f"FROM: {EMAIL_FROM}", f"TO: {to_addr}", f"Subject: {EMAIL_SUBJECT}", "", EMAIL_MESSAGE)) +def send_notification(to_addr, alert=True): + message = ALERT_EMAIL_MESSAGE if alert else DELETED_EMAIL_MESSAGE + BODY = "\r\n".join((f"FROM: {EMAIL_FROM}", f"TO: {to_addr}", f"Subject: {EMAIL_SUBJECT}", "", message)) smtp_server = SMTP(host=SMTP_HOST, port=SMTP_PORT) smtp_server.sendmail(EMAIL_FROM, to_addr, BODY) @@ -58,12 +61,13 @@ def main(): u, p = secrets.db.epi cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host) cur = cnx.cursor() - outdated_keys_list = [API_USER_RECORD(*item) for item in get_outdated_keys(cur)] + outdated_keys_list = get_old_keys(cur) for item in outdated_keys_list: if item.date_diff == 5: send_notification(item.email) else: remove_outdated_key(cur, item.api_key) + send_notification(item.email, alert=False) cur.close() cnx.commit() cnx.close() From 5d7f0ce24db2b8f445e12b40ae65b2431edd2157 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 27 Jul 2023 19:24:16 +0300 Subject: [PATCH 7/9] Added Redis key removal if LAST_USAGE date is older than 'today' --- src/maintenance/update_last_usage.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/maintenance/update_last_usage.py b/src/maintenance/update_last_usage.py index 2a024067d..9dfe9c3dd 100644 --- a/src/maintenance/update_last_usage.py +++ b/src/maintenance/update_last_usage.py @@ -10,21 +10,19 @@ LAST_USED_KEY_PATTERN = "*LAST_USED*" -def redis_get_last_time_used(redis_cli): - keys = redis_cli.keys(pattern=LAST_USED_KEY_PATTERN) - key_val_pairs = { - str(key).split("/")[1]: dtime.strptime(str(redis_cli.get(key)), "%Y-%m-%d").date() for key in keys - } - return key_val_pairs - - def main(): redis_cli = redis.Redis(host=REDIS_HOST, password=REDIS_PASSWORD, decode_responses=True) - key_val_pairs = redis_get_last_time_used(redis_cli, LAST_USED_KEY_PATTERN) u, p = secrets.db.epi cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host) cur = cnx.cursor() - for api_key, last_time_used in key_val_pairs.items(): + + redis_keys = redis_cli.keys(pattern=LAST_USED_KEY_PATTERN) + today_date = dtime.today().date() + for key in redis_keys: + api_key, last_time_used = str(key).split("/")[1], dtime.strptime(str(redis_cli.get(key)), "%Y-%m-%d").date() + if last_time_used < today_date: + redis_cli.delete(key) + continue cur.execute( f""" UPDATE From c023a13ec337b58dea6cba8a3a7caa63a04a0174 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 31 Jul 2023 15:51:42 +0300 Subject: [PATCH 8/9] Simplified query to get keys where last_used date >= 5 month. Added .starttls() to the smtp server. --- src/maintenance/remove_outdated_keys.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/maintenance/remove_outdated_keys.py b/src/maintenance/remove_outdated_keys.py index b4483a149..56fea0cf9 100644 --- a/src/maintenance/remove_outdated_keys.py +++ b/src/maintenance/remove_outdated_keys.py @@ -21,21 +21,12 @@ def get_old_keys(cur): cur.execute( """ - SELECT - diff.api_key, - diff.email, - diff.date_diff - FROM ( - SELECT - api_key, - email, - created, - last_time_used, - ABS(TIMESTAMPDIFF(MONTH, last_time_used, created)) as date_diff - FROM api_user - ) diff - WHERE diff.date_diff >= 5; - + SELECT + api_key, + email, + TIMESTAMPDIFF(MONTH, last_time_used, NOW()) AS date_diff + FROM api_user + HAVING date_diff >= 5; """ ) outdated_keys = cur.fetchall() @@ -54,6 +45,7 @@ def send_notification(to_addr, alert=True): message = ALERT_EMAIL_MESSAGE if alert else DELETED_EMAIL_MESSAGE BODY = "\r\n".join((f"FROM: {EMAIL_FROM}", f"TO: {to_addr}", f"Subject: {EMAIL_SUBJECT}", "", message)) smtp_server = SMTP(host=SMTP_HOST, port=SMTP_PORT) + smtp_server.starttls() smtp_server.sendmail(EMAIL_FROM, to_addr, BODY) From 9957a3fda658f9452fade5b38e21962e6051f347 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 31 Jul 2023 15:52:44 +0300 Subject: [PATCH 9/9] Added Redis key removal after db update --- src/maintenance/update_last_usage.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/maintenance/update_last_usage.py b/src/maintenance/update_last_usage.py index 9dfe9c3dd..85596778a 100644 --- a/src/maintenance/update_last_usage.py +++ b/src/maintenance/update_last_usage.py @@ -20,9 +20,6 @@ def main(): today_date = dtime.today().date() for key in redis_keys: api_key, last_time_used = str(key).split("/")[1], dtime.strptime(str(redis_cli.get(key)), "%Y-%m-%d").date() - if last_time_used < today_date: - redis_cli.delete(key) - continue cur.execute( f""" UPDATE @@ -31,6 +28,7 @@ def main(): WHERE api_key = "{api_key}" AND (last_time_used < "{last_time_used}" OR last_time_used IS NULL) """ ) + redis_cli.delete(key) cur.close() cnx.commit() cnx.close()