py client: drop delphi_utils requirement, log to stderr instead (#1486)

* py client: drop bloated delphi_utils requirement, log to stderr instead

* Replace assertlogs with stderr capture

---------

Co-authored-by: Rostyslav Zatserkovnyi <[email protected]>

Author: melange396

integrations/client/test_delphi_epidata.py

@@ -49,6 +49,12 @@ def localSetUp(self):
     secrets.db.host = 'delphi_database_epidata'
     secrets.db.epi = ('user', 'pass')
 
+  @pytest.fixture(autouse=True)
+  def capsys(self, capsys):
+    """Hook capsys (stdout and stderr) into this test class."""
+
+    self.capsys = capsys
+
   def test_covidcast(self):
     """Test that the covidcast endpoint returns expected data."""
 
@@ -238,46 +244,46 @@ def raise_for_status(self): pass
 
     try:
       with self.subTest(name='test multiple GET'):
-        with self.assertLogs('delphi_epidata_client', level='INFO') as logs:
-          get.reset_mock()
-          get.return_value = MockResponse(b'{"key": "value"}', 200)
-          Epidata._request_with_retry("test_endpoint1", params={"key1": "value1"})
-          Epidata._request_with_retry("test_endpoint2", params={"key2": "value2"})
+        get.reset_mock()
+        get.return_value = MockResponse(b'{"key": "value"}', 200)
+        Epidata._request_with_retry("test_endpoint1", params={"key1": "value1"})
+        Epidata._request_with_retry("test_endpoint2", params={"key2": "value2"})
 
-        output = logs.output
+        captured = self.capsys.readouterr()
+        output = captured.err.splitlines()
         self.assertEqual(len(output), 4) # [request, response, request, response]
         self.assertIn("Sending GET request", output[0])
-        self.assertIn("\"url\": \"http://delphi_web_epidata/epidata/test_endpoint1/\"", output[0])
-        self.assertIn("\"params\": {\"key1\": \"value1\"}", output[0])
+        self.assertIn("\'url\': \'http://delphi_web_epidata/epidata/test_endpoint1/\'", output[0])
+        self.assertIn("\'params\': {\'key1\': \'value1\'}", output[0])
         self.assertIn("Received response", output[1])
-        self.assertIn("\"status_code\": 200", output[1])
-        self.assertIn("\"len\": 16", output[1])
+        self.assertIn("\'status_code\': 200", output[1])
+        self.assertIn("\'len\': 16", output[1])
         self.assertIn("Sending GET request", output[2])
-        self.assertIn("\"url\": \"http://delphi_web_epidata/epidata/test_endpoint2/\"", output[2])
-        self.assertIn("\"params\": {\"key2\": \"value2\"}", output[2])
+        self.assertIn("\'url\': \'http://delphi_web_epidata/epidata/test_endpoint2/\'", output[2])
+        self.assertIn("\'params\': {\'key2\': \'value2\'}", output[2])
         self.assertIn("Received response", output[3])
-        self.assertIn("\"status_code\": 200", output[3])
-        self.assertIn("\"len\": 16", output[3])
+        self.assertIn("\'status_code\': 200", output[3])
+        self.assertIn("\'len\': 16", output[3])
 
       with self.subTest(name='test GET and POST'):
-        with self.assertLogs('delphi_epidata_client', level='INFO') as logs:
-          get.reset_mock()
-          get.return_value = MockResponse(b'{"key": "value"}', 414)
-          post.reset_mock()
-          post.return_value = MockResponse(b'{"key": "value"}', 200)
-          Epidata._request_with_retry("test_endpoint3", params={"key3": "value3"})
-
-        output = logs.output
-        self.assertEqual(len(output), 3) # [request, response, request, response]
+        get.reset_mock()
+        get.return_value = MockResponse(b'{"key": "value"}', 414)
+        post.reset_mock()
+        post.return_value = MockResponse(b'{"key": "value"}', 200)
+        Epidata._request_with_retry("test_endpoint3", params={"key3": "value3"})
+
+        captured = self.capsys.readouterr()
+        output = captured.err.splitlines()
+        self.assertEqual(len(output), 3) # [request, retry, response]
         self.assertIn("Sending GET request", output[0])
-        self.assertIn("\"url\": \"http://delphi_web_epidata/epidata/test_endpoint3/\"", output[0])
-        self.assertIn("\"params\": {\"key3\": \"value3\"}", output[0])
+        self.assertIn("\'url\': \'http://delphi_web_epidata/epidata/test_endpoint3/\'", output[0])
+        self.assertIn("\'params\': {\'key3\': \'value3\'}", output[0])
         self.assertIn("Received 414 response, retrying as POST request", output[1])
-        self.assertIn("\"url\": \"http://delphi_web_epidata/epidata/test_endpoint3/\"", output[1])
-        self.assertIn("\"params\": {\"key3\": \"value3\"}", output[1])
+        self.assertIn("\'url\': \'http://delphi_web_epidata/epidata/test_endpoint3/\'", output[1])
+        self.assertIn("\'params\': {\'key3\': \'value3\'}", output[1])
         self.assertIn("Received response", output[2])
-        self.assertIn("\"status_code\": 200", output[2])
-        self.assertIn("\"len\": 16", output[2])
+        self.assertIn("\'status_code\': 200", output[2])
+        self.assertIn("\'len\': 16", output[2])
     finally: # make sure this global is always reset
       Epidata.debug = False
 
@@ -288,12 +294,12 @@ def test_sandbox(self, get, post):
     Epidata.debug = True
     Epidata.sandbox = True
     try:
-      with self.assertLogs('delphi_epidata_client', level='INFO') as logs:
-        Epidata.covidcast('src', 'sig', 'day', 'county', 20200414, '01234')
-      output = logs.output
+      Epidata.covidcast('src', 'sig', 'day', 'county', 20200414, '01234')
+      captured = self.capsys.readouterr()
+      output = captured.err.splitlines()
       self.assertEqual(len(output), 1)
       self.assertIn("Sending GET request", output[0])
-      self.assertIn("\"url\": \"http://delphi_web_epidata/epidata/covidcast/\"", output[0])
+      self.assertIn("\'url\': \'http://delphi_web_epidata/epidata/covidcast/\'", output[0])
       get.assert_not_called()
       post.assert_not_called()
     finally: # make sure these globals are always reset

src/client/delphi_epidata.py

@@ -8,6 +8,8 @@
  - Compatible with Python 2 and 3.
 """
 
+import sys
+
 # External modules
 import requests
 import time
@@ -16,8 +18,6 @@
 
 from aiohttp import ClientSession, TCPConnector, BasicAuth
 
-from delphi_utils.logger import get_structured_logger
-
 __version__ = "4.1.23"
 
 _HEADERS = {"user-agent": "delphi_epidata/" + __version__ + " (Python)"}
@@ -45,10 +45,15 @@ class Epidata:
 
     client_version = __version__
 
-    logger = get_structured_logger('delphi_epidata_client')
     debug = False # if True, prints extra logging statements
     sandbox = False # if True, will not execute any queries
 
+    @staticmethod
+    def log(evt, **kwargs):
+        kwargs['event'] = evt
+        kwargs['timestamp'] = time.strftime("%Y-%m-%d %H:%M:%S %z")
+        return sys.stderr.write(str(kwargs) + "\n")
+
     # Helper function to cast values and/or ranges to strings
     @staticmethod
     def _listitem(value):
@@ -72,7 +77,7 @@ def _request_with_retry(endpoint, params={}):
         """Make request with a retry if an exception is thrown."""
         request_url = f"{Epidata.BASE_URL}/{endpoint}/"
         if Epidata.debug:
-            Epidata.logger.info("Sending GET request", url=request_url, params=params, headers=_HEADERS, auth=Epidata.auth)
+            Epidata.log("Sending GET request", url=request_url, params=params, headers=_HEADERS, auth=Epidata.auth)
         if Epidata.sandbox:
             resp = requests.Response()
             resp._content = b'true'
@@ -81,10 +86,10 @@ def _request_with_retry(endpoint, params={}):
         req = requests.get(request_url, params, auth=Epidata.auth, headers=_HEADERS)
         if req.status_code == 414:
             if Epidata.debug:
-                Epidata.logger.info("Received 414 response, retrying as POST request", url=request_url, params=params, headers=_HEADERS)
+                Epidata.log("Received 414 response, retrying as POST request", url=request_url, params=params, headers=_HEADERS)
             req = requests.post(request_url, params, auth=Epidata.auth, headers=_HEADERS)
         if Epidata.debug:
-            Epidata.logger.info(
+            Epidata.log(
                 "Received response",
                 status_code=req.status_code,
                 len=len(req.content),

src/client/packaging/pypi/pyproject.toml

@@ -37,7 +37,7 @@ classifiers = [
     "Natural Language :: English",
     "Topic :: Scientific/Engineering :: Bio-Informatics",
 ]
-dependencies = ["aiohttp", "delphi-utils", "requests>=2.7.0", "tenacity"]
+dependencies = ["aiohttp", "requests>=2.7.0", "tenacity"]
 
 [project.urls]
 "Homepage" = "https://github.com/cmu-delphi/delphi-epidata"