Lazy load packages in compat for faster import. (#881)

* Streamline usage of ConfigParser for lazy loading.

* Do not import all of asyncio.

* Remove unused imports

* Defer import of urllib.request to when its actually needed.

* Test to ensure slow imports do not reappear.

* Use other configparser.

* Return test to original state.

* Skip test for python2.

* Update comment text.

Author: purple4reina

datadog/util/compat.py

@@ -5,9 +5,7 @@
 """
 Imports for compatibility with Python 2, Python 3 and Google App Engine.
 """
-from functools import wraps
 import logging
-import socket
 import sys
 
 # Logging
@@ -23,11 +21,25 @@
 if sys.version_info[0] >= 3:
     import builtins
     from collections import UserDict as IterableUserDict
-    import configparser
-    from configparser import ConfigParser
     from io import StringIO
-    from urllib.parse import urljoin, urlparse
-    import urllib.request as url_lib, urllib.error, urllib.parse
+    from urllib.parse import urlparse
+
+    class LazyLoader(object):
+        def __init__(self, module_name):
+            self.module_name = module_name
+
+        def __getattr__(self, name):
+            # defer the importing of the module to when one of its attributes
+            # is accessed
+            import importlib
+            mod = importlib.import_module(self.module_name)
+            return getattr(mod, name)
+
+    url_lib = LazyLoader('urllib.request')
+    configparser = LazyLoader('configparser')
+
+    def ConfigParser():
+        return configparser.ConfigParser()
 
     imap = map
     get_input = input
@@ -48,7 +60,7 @@ def iternext(iter):
     from cStringIO import StringIO
     from itertools import imap
     import urllib2 as url_lib
-    from urlparse import urljoin, urlparse
+    from urlparse import urlparse
     from UserDict import IterableUserDict
 
     get_input = raw_input
@@ -63,7 +75,7 @@ def iternext(iter):
 
 # Python >= 3.5
 if sys.version_info >= (3, 5):
-    from asyncio import iscoroutinefunction
+    from inspect import iscoroutinefunction
 # Others
 else:
 
@@ -76,9 +88,7 @@ def iscoroutinefunction(*args, **kwargs):
     from logging import NullHandler
 # Python 2.6.x
 else:
-    from logging import Handler
-
-    class NullHandler(Handler):
+    class NullHandler(logging.Handler):
         def emit(self, record):
             pass
 

tests/unit/util/test_compat.py

@@ -3,11 +3,13 @@
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2015-Present Datadog, Inc
 import logging
+import pytest
+import sys
 import unittest
 
 from mock import patch
 
-from datadog.util.compat import conditional_lru_cache, is_higher_py32
+from datadog.util.compat import conditional_lru_cache, is_higher_py32, is_p3k
 
 class TestConditionalLRUCache(unittest.TestCase):
     def test_normal_usage(self):
@@ -48,3 +50,36 @@ def test_function():
                 mock_debug.assert_called_once()
             else:
                 mock_debug.assert_not_called()
+
+@pytest.mark.skipif(not is_p3k(), reason='Python 3 only')
+def test_slow_imports(monkeypatch):
+    # We should lazy load certain modules to avoid slowing down the startup
+    # time when running in a serverless environment.  This test will fail if
+    # any of those modules are imported during the import of datadogpy.
+
+    blocklist = [
+        'configparser',
+        'email.mime.application',
+        'email.mime.multipart',
+        'importlib.metadata',
+        'importlib_metadata',
+        'logging.handlers',
+        'multiprocessing',
+        'urllib.request',
+    ]
+
+    class BlockListFinder:
+        def find_spec(self, fullname, *args):
+            for lib in blocklist:
+                if fullname == lib:
+                    raise ImportError('module %s was imported!' % fullname)
+            return None
+        find_module = find_spec  # Python 2
+
+    monkeypatch.setattr('sys.meta_path', [BlockListFinder()] + sys.meta_path)
+
+    for mod in sys.modules.copy():
+        if mod in blocklist or mod.startswith('datadog'):
+            del sys.modules[mod]
+
+    import datadog