Add mandatory meta_key check to server

* fix all tests by adding param where needed

Author: dshemetov

integrations/acquisition/covid_hosp/facility/test_scenarios.py

@@ -2,7 +2,7 @@
 
 # standard library
 import unittest
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 # first party
 from delphi.epidata.acquisition.covid_hosp.common.database import Database
@@ -19,6 +19,21 @@
 
 NEWLINE="\n"
 
+def _request(params):
+  """Request and parse epidata.
+
+  We default to GET since it has better caching and logging
+  capabilities, but fall back to POST if the request is too
+  long and returns a 414.
+  """
+  params.update({'meta_key': 'meta_secret'})
+  try:
+    return Epidata._request_with_retry(params).json()
+  except Exception as e:
+    return {'result': 0, 'message': 'error: ' + str(e)}
+
+
+@patch.object(Epidata, '_request', _request)
 class AcquisitionTests(unittest.TestCase):
 
   def setUp(self):

integrations/acquisition/covid_hosp/state_daily/test_scenarios.py

@@ -22,7 +22,21 @@
 __test_target__ = \
     'delphi.epidata.acquisition.covid_hosp.state_daily.update'
 
+def _request(params):
+  """Request and parse epidata.
 
+  We default to GET since it has better caching and logging
+  capabilities, but fall back to POST if the request is too
+  long and returns a 414.
+  """
+  params.update({'meta_key': 'meta_secret'})
+  try:
+    return Epidata._request_with_retry(params).json()
+  except Exception as e:
+    return {'result': 0, 'message': 'error: ' + str(e)}
+
+
+@patch.object(Epidata, '_request', _request)
 class AcquisitionTests(unittest.TestCase):
 
   def setUp(self):

integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py

@@ -2,7 +2,7 @@
 
 # standard library
 import unittest
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 # first party
 from delphi.epidata.acquisition.covid_hosp.common.database import Database
@@ -18,7 +18,21 @@
 __test_target__ = \
     'delphi.epidata.acquisition.covid_hosp.state_timeseries.update'
 
+def _request(params):
+  """Request and parse epidata.
 
+  We default to GET since it has better caching and logging
+  capabilities, but fall back to POST if the request is too
+  long and returns a 414.
+  """
+  params.update({'meta_key': 'meta_secret'})
+  try:
+    return Epidata._request_with_retry(params).json()
+  except Exception as e:
+    return {'result': 0, 'message': 'error: ' + str(e)}
+
+
+@patch.object(Epidata, '_request', _request)
 class AcquisitionTests(unittest.TestCase):
 
   def setUp(self):

integrations/acquisition/covidcast/test_covidcast_meta_caching.py

@@ -3,6 +3,7 @@
 # standard library
 import json
 import unittest
+from unittest.mock import patch
 
 # third party
 import mysql.connector
@@ -24,7 +25,21 @@
 # use the local instance of the Epidata API
 BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
 
+def _request(params):
+  """Request and parse epidata.
 
+  We default to GET since it has better caching and logging
+  capabilities, but fall back to POST if the request is too
+  long and returns a 414.
+  """
+  params.update({'meta_key': 'meta_secret'})
+  try:
+    return Epidata._request_with_retry(params).json()
+  except Exception as e:
+    return {'result': 0, 'message': 'error: ' + str(e)}
+
+
+@patch.object(Epidata, '_request', _request)
 class CovidcastMetaCacheTests(unittest.TestCase):
   """Tests covidcast metadata caching."""
 
@@ -144,7 +159,7 @@ def test_caching(self):
     self.cnx.commit()
 
     # fetch the cached version (manually)
-    params = {'endpoint': 'covidcast_meta', 'cached': 'true'}
+    params = {'endpoint': 'covidcast_meta', 'cached': 'true', 'meta_key': 'meta_secret'}
     response = requests.get(BASE_URL, params=params)
     response.raise_for_status()
     epidata4 = response.json()
@@ -167,7 +182,7 @@ def test_caching(self):
     self.cnx.commit()
 
     # fetch the cached version (manually)
-    params = {'endpoint': 'covidcast_meta', 'cached': 'true'}
+    params = {'endpoint': 'covidcast_meta', 'cached': 'true', 'meta_key': 'meta_secret'}
     response = requests.get(BASE_URL, params=params)
     response.raise_for_status()
     epidata5 = response.json()

integrations/acquisition/covidcast/test_csv_uploading.py

@@ -4,7 +4,7 @@
 from datetime import date
 import os
 import unittest
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 # third party
 import mysql.connector
@@ -20,7 +20,21 @@
 # py3tester coverage target (equivalent to `import *`)
 __test_target__ = 'delphi.epidata.acquisition.covidcast.csv_to_database'
 
+def _request(params):
+  """Request and parse epidata.
 
+  We default to GET since it has better caching and logging
+  capabilities, but fall back to POST if the request is too
+  long and returns a 414.
+  """
+  params.update({'meta_key': 'meta_secret'})
+  try:
+    return Epidata._request_with_retry(params).json()
+  except Exception as e:
+    return {'result': 0, 'message': 'error: ' + str(e)}
+
+
+@patch.object(Epidata, '_request', _request)
 class CsvUploadingTests(unittest.TestCase):
   """Tests covidcast CSV uploading."""
 

integrations/acquisition/covidcast/test_delete_batch.py

@@ -3,6 +3,7 @@
 # standard library
 from collections import namedtuple
 import unittest
+from unittest.mock import patch
 from os import path
 
 # third party
@@ -18,6 +19,21 @@
 
 Example = namedtuple("example", "given expected")
 
+def _request(params):
+  """Request and parse epidata.
+
+  We default to GET since it has better caching and logging
+  capabilities, but fall back to POST if the request is too
+  long and returns a 414.
+  """
+  params.update({'meta_key': 'meta_secret'})
+  try:
+    return Epidata._request_with_retry(params).json()
+  except Exception as e:
+    return {'result': 0, 'message': 'error: ' + str(e)}
+
+
+@patch.object(Epidata, '_request', _request)
 class DeleteBatch(unittest.TestCase):
     """Tests batch deletions"""
 

integrations/acquisition/covidcast/test_fill_is_latest_issue.py

@@ -15,7 +15,21 @@
 # py3tester coverage target (equivalent to `import *`)
 __test_target__ = 'delphi.epidata.acquisition.covidcast.fill_is_latest_issue'
 
+def _request(params):
+  """Request and parse epidata.
 
+  We default to GET since it has better caching and logging
+  capabilities, but fall back to POST if the request is too
+  long and returns a 414.
+  """
+  params.update({'meta_key': 'meta_secret'})
+  try:
+    return Epidata._request_with_retry(params).json()
+  except Exception as e:
+    return {'result': 0, 'message': 'error: ' + str(e)}
+
+
+@unittest.mock.patch.object(Epidata, '_request', _request)
 class FillIsLatestIssueTests(unittest.TestCase):
   """Tests filling is_latest_issue column"""
 

integrations/acquisition/covidcast_nowcast/test_csv_uploading.py

@@ -26,7 +26,21 @@
                                      issue=(date(2020, 4, 21), epi.Week.fromdate(date(2020, 4, 21)))
                                      )
 
+def _request(params):
+  """Request and parse epidata.
 
+  We default to GET since it has better caching and logging
+  capabilities, but fall back to POST if the request is too
+  long and returns a 414.
+  """
+  params.update({'meta_key': 'meta_secret'})
+  try:
+    return Epidata._request_with_retry(params).json()
+  except Exception as e:
+    return {'result': 0, 'message': 'error: ' + str(e)}
+
+
+@patch.object(Epidata, '_request', _request)
 class CsvUploadingTests(unittest.TestCase):
   """Tests covidcast nowcast CSV uploading."""
 

integrations/client/test_delphi_epidata.py

@@ -27,7 +27,21 @@ def wrapper(*args):
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
   return wrapper
 
+def _request(params):
+  """Request and parse epidata.
 
+  We default to GET since it has better caching and logging
+  capabilities, but fall back to POST if the request is too
+  long and returns a 414.
+  """
+  params.update({'meta_key': 'meta_secret'})
+  try:
+    return Epidata._request_with_retry(params).json()
+  except Exception as e:
+    return {'result': 0, 'message': 'error: ' + str(e)}
+
+
+@patch.object(Epidata, '_request', _request)
 class DelphiEpidataPythonClientTests(unittest.TestCase):
   """Tests the Python client."""
 
@@ -315,7 +329,7 @@ def test_retry_request(self, get):
       mock_response = MagicMock()
       mock_response.status_code = 200
       get.side_effect = [JSONDecodeError('Expecting value', "",  0), mock_response]
-      response = Epidata._request(None)
+      response = Epidata._request({})
       self.assertEqual(get.call_count, 2)
       self.assertEqual(response, mock_response.json())
 
@@ -326,7 +340,7 @@ def test_retry_request(self, get):
       get.side_effect = [JSONDecodeError('Expecting value', "",  0),
                          JSONDecodeError('Expecting value', "",  0),
                          mock_response]
-      response = Epidata._request(None)
+      response = Epidata._request({})
       self.assertEqual(get.call_count, 2)  # 2 from previous test + 2 from this one
       self.assertEqual(response,
                        {'result': 0, 'message': 'error: Expecting value: line 1 column 1 (char 0)'}
@@ -615,7 +629,8 @@ def test_async_epidata(self):
         'time_type': 'day',
         'geo_type': 'county',
         'geo_value': '11111',
-        'time_values': '20200414'
+        'time_values': '20200414',
+        'meta_key': 'meta_secret'
       },
       {
         'source': 'covidcast',
@@ -624,7 +639,8 @@ def test_async_epidata(self):
         'time_type': 'day',
         'geo_type': 'county',
         'geo_value': '00000',
-        'time_values': '20200414'
+        'time_values': '20200414',
+        'meta_key': 'meta_secret'
       }
     ]*12, batch_size=10)
     responses = [i[0] for i in test_output]
@@ -634,7 +650,7 @@ def test_async_epidata(self):
                       Epidata.covidcast('src', 'sig', 'day', 'county', 20200414, '00000')]*12
                      )
 
-  @fake_epidata_endpoint
+  @patch.object(Epidata, "BASE_URL", 'http://delphi_web_epidata/epidata/fake_api.php')
   def test_async_epidata_fail(self):
     with pytest.raises(ClientResponseError, match="404, message='NOT FOUND'"):
       Epidata.async_epidata([
@@ -645,6 +661,7 @@ def test_async_epidata_fail(self):
           'time_type': 'day',
           'geo_type': 'county',
           'geo_value': '11111',
-          'time_values': '20200414'
+          'time_values': '20200414',
+          'meta_key': 'meta_secret'
         }
       ])

integrations/server/test_covid_hosp.py

@@ -2,13 +2,28 @@
 
 # standard library
 import unittest
+from unittest.mock import patch
 
 # first party
 from delphi.epidata.acquisition.covid_hosp.state_timeseries.database import Database
 from delphi.epidata.client.delphi_epidata import Epidata
 import delphi.operations.secrets as secrets
 
+def _request(params):
+  """Request and parse epidata.
 
+  We default to GET since it has better caching and logging
+  capabilities, but fall back to POST if the request is too
+  long and returns a 414.
+  """
+  params.update({'meta_key': 'meta_secret'})
+  try:
+    return Epidata._request_with_retry(params).json()
+  except Exception as e:
+    return {'result': 0, 'message': 'error: ' + str(e)}
+
+
+@patch.object(Epidata, '_request', _request)
 class ServerTests(unittest.TestCase):
   """Tests the `covid_hosp` endpoint."""