Merge pull request #309 from chinandrew/patch-1

Change requests from GET to POST

Author: krivard

integrations/client/test_delphi_epidata.py

@@ -2,6 +2,7 @@
 
 # standard library
 import unittest
+from unittest.mock import patch, MagicMock
 
 # third party
 import mysql.connector
@@ -231,6 +232,29 @@ def test_covidcast(self):
           }],
           'message': 'success',
       })
+    with self.subTest(name='long request'):
+      # fetch data, without specifying issue or lag
+      response_1 = Epidata.covidcast(
+          'src', 'sig'*3000, 'day', 'county', 20200414, '01234')
+
+      # check result
+      self.assertEqual(response_1, {'message': 'no results', 'result': -2})
+
+  @patch('requests.post')
+  @patch('requests.get')
+  def test_request_method(self, get, post):
+    """Test that a GET request is default and POST is used if a 414 is returned."""
+    with self.subTest(name='get request'):
+      Epidata.covidcast('src', 'sig', 'day', 'county', 20200414, '01234')
+      get.assert_called_once()
+      post.assert_not_called()
+    with self.subTest(name='post request'):
+      mock_response = MagicMock()
+      mock_response.status_code = 414
+      get.return_value = mock_response
+      Epidata.covidcast('src', 'sig', 'day', 'county', 20200414, '01234')
+      self.assertEqual(get.call_count, 2)  # one from post test and one from get test
+      post.assert_called_once()
 
   def test_geo_value(self):
     """test different variants of geo types: single, *, multi."""

integrations/server/test_covidcast.py

@@ -79,6 +79,41 @@ def test_round_trip(self):
       'message': 'success',
     })
 
+  def test_uri_too_long(self):
+    """Test that a long request yields a 414 with GET but works with POST."""
+
+    # insert dummy data
+    self.cur.execute('''
+      insert into covidcast values
+        (0, 'src', 'sig', 'day', 'county', 20200414, '01234',
+          123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 1, False)
+    ''')
+    self.cnx.commit()
+
+    # make the request with GET
+    response = requests.get(BASE_URL, {
+      'source': 'covidcast',
+      'data_source': 'src'*10000,
+      'signal': 'sig',
+      'time_type': 'day',
+      'geo_type': 'county',
+      'time_values': 20200414,
+      'geo_value': '01234',
+    })
+    self.assertEqual(response.status_code, 414)
+
+    # make request with POST
+    response = requests.post(BASE_URL, {
+      'source': 'covidcast',
+      'data_source': 'src'*10000,
+      'signal': 'sig',
+      'time_type': 'day',
+      'geo_type': 'county',
+      'time_values': 20200414,
+      'geo_value': '01234',
+    })
+
+    self.assertEqual(response.status_code, 200)
 
   def test_csv_format(self):
     """Test generate csv data."""

src/client/delphi_epidata.py

@@ -52,10 +52,18 @@ def _list(values):
   # Helper function to request and parse epidata
   @staticmethod
   def _request(params):
-    """Request and parse epidata."""
+    """Request and parse epidata.
+
+    We default to GET since it has better caching and logging
+    capabilities, but fall back to POST if the request is too
+    long and returns a 414.
+    """
     try:
       # API call
-      return requests.get(Epidata.BASE_URL, params, headers=_HEADERS).json()
+      req = requests.get(Epidata.BASE_URL, params, headers=_HEADERS)
+      if req.status_code == 414:
+        req = requests.post(Epidata.BASE_URL, params, headers=_HEADERS)
+      return req.json()
     except Exception as e:
       # Something broke
       return {'result': 0, 'message': 'error: ' + str(e)}