PYTHON-4441 [v4.7] Use deferred imports instead of lazy module loading (#1650)

Author: blink1073

doc/changelog.rst

@@ -1,6 +1,22 @@
 Changelog
 =========
 
+Changes in Version 4.7.3
+-------------------------
+
+Version 4.7.3 has further fixes for lazily loading modules.
+
+- Use deferred imports instead of importlib lazy module loading.
+- Improve import time on Windows.
+
+Issues Resolved
+...............
+
+See the `PyMongo 4.7.3 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 4.7.3 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=39865
+
 Changes in Version 4.7.2
 -------------------------
 

pymongo/_gcp_helpers.py

@@ -16,10 +16,11 @@
 from __future__ import annotations
 
 from typing import Any
-from urllib.request import Request, urlopen
 
 
 def _get_gcp_response(resource: str, timeout: float = 5) -> dict[str, Any]:
+    from urllib.request import Request, urlopen
+
     url = "http://metadata/computeMetadata/v1/instance/service-accounts/default/identity"
     url += f"?audience={resource}"
     headers = {"Metadata-Flavor": "Google"}

pymongo/_lazy_import.py

@@ -15,15 +15,6 @@
 """MONGODB-AWS Authentication helpers."""
 from __future__ import annotations
 
-from pymongo._lazy_import import lazy_import
-
-try:
-    pymongo_auth_aws = lazy_import("pymongo_auth_aws")
-    _HAVE_MONGODB_AWS = True
-except ImportError:
-    _HAVE_MONGODB_AWS = False
-
-
 from typing import TYPE_CHECKING, Any, Mapping, Type
 
 import bson
@@ -38,11 +29,13 @@
 
 def _authenticate_aws(credentials: MongoCredential, conn: Connection) -> None:
     """Authenticate using MONGODB-AWS."""
-    if not _HAVE_MONGODB_AWS:
+    try:
+        import pymongo_auth_aws  # type:ignore[import]
+    except ImportError as e:
         raise ConfigurationError(
             "MONGODB-AWS authentication requires pymongo-auth-aws: "
             "install with: python -m pip install 'pymongo[aws]'"
-        )
+        ) from e
 
     # Delayed import.
     from pymongo_auth_aws.auth import (  # type:ignore[import]

pymongo/auth_aws.py

@@ -19,7 +19,6 @@
 
 from bson.codec_options import _parse_codec_options
 from pymongo import common
-from pymongo.auth import MongoCredential, _build_credentials_tuple
 from pymongo.compression_support import CompressionSettings
 from pymongo.errors import ConfigurationError
 from pymongo.monitoring import _EventListener, _EventListeners
@@ -36,6 +35,7 @@
 
 if TYPE_CHECKING:
     from bson.codec_options import CodecOptions
+    from pymongo.auth import MongoCredential
     from pymongo.encryption_options import AutoEncryptionOpts
     from pymongo.pyopenssl_context import SSLContext
     from pymongo.topology_description import _ServerSelector
@@ -48,6 +48,8 @@ def _parse_credentials(
     mechanism = options.get("authmechanism", "DEFAULT" if username else None)
     source = options.get("authsource")
     if username or mechanism:
+        from pymongo.auth import _build_credentials_tuple
+
         return _build_credentials_tuple(mechanism, source, username, password, options, database)
     return None
 

pymongo/client_options.py

@@ -40,8 +40,6 @@
 from bson.binary import UuidRepresentation
 from bson.codec_options import CodecOptions, DatetimeConversion, TypeRegistry
 from bson.raw_bson import RawBSONDocument
-from pymongo.auth import MECHANISMS
-from pymongo.auth_oidc import OIDCCallback
 from pymongo.compression_support import (
     validate_compressors,
     validate_zlib_compression_level,
@@ -380,6 +378,8 @@ def validate_read_preference_mode(dummy: Any, value: Any) -> _ServerMode:
 
 def validate_auth_mechanism(option: str, value: Any) -> str:
     """Validate the authMechanism URI option."""
+    from pymongo.auth import MECHANISMS
+
     if value not in MECHANISMS:
         raise ValueError(f"{option} must be in {tuple(MECHANISMS)}")
     return value
@@ -444,6 +444,8 @@ def validate_auth_mechanism_properties(option: str, value: Any) -> dict[str, Uni
             elif key in ["ALLOWED_HOSTS"] and isinstance(value, list):
                 props[key] = value
             elif key in ["OIDC_CALLBACK", "OIDC_HUMAN_CALLBACK"]:
+                from pymongo.auth_oidc import OIDCCallback
+
                 if not isinstance(value, OIDCCallback):
                     raise ValueError("callback must be an OIDCCallback object")
                 props[key] = value

pymongo/common.py

@@ -16,34 +16,39 @@
 import warnings
 from typing import Any, Iterable, Optional, Union
 
-from pymongo._lazy_import import lazy_import
 from pymongo.hello import HelloCompat
-from pymongo.monitoring import _SENSITIVE_COMMANDS
+from pymongo.helpers import _SENSITIVE_COMMANDS
 
-try:
-    snappy = lazy_import("snappy")
-    _HAVE_SNAPPY = True
-except ImportError:
-    # python-snappy isn't available.
-    _HAVE_SNAPPY = False
+_SUPPORTED_COMPRESSORS = {"snappy", "zlib", "zstd"}
+_NO_COMPRESSION = {HelloCompat.CMD, HelloCompat.LEGACY_CMD}
+_NO_COMPRESSION.update(_SENSITIVE_COMMANDS)
 
-try:
-    zlib = lazy_import("zlib")
 
-    _HAVE_ZLIB = True
-except ImportError:
-    # Python built without zlib support.
-    _HAVE_ZLIB = False
+def _have_snappy() -> bool:
+    try:
+        import snappy  # type:ignore[import]  # noqa: F401
 
-try:
-    zstandard = lazy_import("zstandard")
-    _HAVE_ZSTD = True
-except ImportError:
-    _HAVE_ZSTD = False
+        return True
+    except ImportError:
+        return False
 
-_SUPPORTED_COMPRESSORS = {"snappy", "zlib", "zstd"}
-_NO_COMPRESSION = {HelloCompat.CMD, HelloCompat.LEGACY_CMD}
-_NO_COMPRESSION.update(_SENSITIVE_COMMANDS)
+
+def _have_zlib() -> bool:
+    try:
+        import zlib  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
+def _have_zstd() -> bool:
+    try:
+        import zstandard  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
 
 
 def validate_compressors(dummy: Any, value: Union[str, Iterable[str]]) -> list[str]:
@@ -58,21 +63,21 @@ def validate_compressors(dummy: Any, value: Union[str, Iterable[str]]) -> list[s
         if compressor not in _SUPPORTED_COMPRESSORS:
             compressors.remove(compressor)
             warnings.warn(f"Unsupported compressor: {compressor}", stacklevel=2)
-        elif compressor == "snappy" and not _HAVE_SNAPPY:
+        elif compressor == "snappy" and not _have_snappy():
             compressors.remove(compressor)
             warnings.warn(
                 "Wire protocol compression with snappy is not available. "
                 "You must install the python-snappy module for snappy support.",
                 stacklevel=2,
             )
-        elif compressor == "zlib" and not _HAVE_ZLIB:
+        elif compressor == "zlib" and not _have_zlib():
             compressors.remove(compressor)
             warnings.warn(
                 "Wire protocol compression with zlib is not available. "
                 "The zlib module is not available.",
                 stacklevel=2,
             )
-        elif compressor == "zstd" and not _HAVE_ZSTD:
+        elif compressor == "zstd" and not _have_zstd():
             compressors.remove(compressor)
             warnings.warn(
                 "Wire protocol compression with zstandard is not available. "
@@ -117,6 +122,8 @@ class SnappyContext:
 
     @staticmethod
     def compress(data: bytes) -> bytes:
+        import snappy
+
         return snappy.compress(data)
 
 
@@ -127,6 +134,8 @@ def __init__(self, level: int):
         self.level = level
 
     def compress(self, data: bytes) -> bytes:
+        import zlib
+
         return zlib.compress(data, self.level)
 
 
@@ -137,6 +146,8 @@ class ZstdContext:
     def compress(data: bytes) -> bytes:
         # ZstdCompressor is not thread safe.
         # TODO: Use a pool?
+        import zstandard
+
         return zstandard.ZstdCompressor().compress(data)
 
 
@@ -146,12 +157,18 @@ def decompress(data: bytes, compressor_id: int) -> bytes:
         # https://github.com/andrix/python-snappy/issues/65
         # This only matters when data is a memoryview since
         # id(bytes(data)) == id(data) when data is a bytes.
+        import snappy
+
         return snappy.uncompress(bytes(data))
     elif compressor_id == ZlibContext.compressor_id:
+        import zlib
+
         return zlib.decompress(data)
     elif compressor_id == ZstdContext.compressor_id:
         # ZstdDecompressor is not thread safe.
         # TODO: Use a pool?
+        import zstandard
+
         return zstandard.ZstdDecompressor().decompress(data)
     else:
         raise ValueError("Unknown compressorId %d" % (compressor_id,))

pymongo/compression_support.py

@@ -93,6 +93,21 @@
 # Server code raised when authentication fails.
 _AUTHENTICATION_FAILURE_CODE: int = 18
 
+# Note - to avoid bugs from forgetting which if these is all lowercase and
+# which are camelCase, and at the same time avoid having to add a test for
+# every command, use all lowercase here and test against command_name.lower().
+_SENSITIVE_COMMANDS: set = {
+    "authenticate",
+    "saslstart",
+    "saslcontinue",
+    "getnonce",
+    "createuser",
+    "updateuser",
+    "copydbgetnonce",
+    "copydbsaslstart",
+    "copydb",
+}
+
 
 def _gen_index_name(keys: _IndexList) -> str:
     """Generate an index name from the set of fields it is over."""

pymongo/helpers.py

@@ -191,7 +191,7 @@ def connection_checked_in(self, event):
 
 from bson.objectid import ObjectId
 from pymongo.hello import Hello, HelloCompat
-from pymongo.helpers import _handle_exception
+from pymongo.helpers import _SENSITIVE_COMMANDS, _handle_exception
 from pymongo.typings import _Address, _DocumentOut
 
 if TYPE_CHECKING:
@@ -507,22 +507,6 @@ def register(listener: _EventListener) -> None:
         _LISTENERS.cmap_listeners.append(listener)
 
 
-# Note - to avoid bugs from forgetting which if these is all lowercase and
-# which are camelCase, and at the same time avoid having to add a test for
-# every command, use all lowercase here and test against command_name.lower().
-_SENSITIVE_COMMANDS: set = {
-    "authenticate",
-    "saslstart",
-    "saslcontinue",
-    "getnonce",
-    "createuser",
-    "updateuser",
-    "copydbgetnonce",
-    "copydbsaslstart",
-    "copydb",
-}
-
-
 # The "hello" command is also deemed sensitive when attempting speculative
 # authentication.
 def _is_speculative_authenticate(command_name: str, doc: Mapping[str, Any]) -> bool:

pymongo/monitoring.py

@@ -41,7 +41,7 @@
 
 import bson
 from bson import DEFAULT_CODEC_OPTIONS
-from pymongo import __version__, _csot, auth, helpers
+from pymongo import __version__, _csot, helpers
 from pymongo.client_session import _validate_session_write_concern
 from pymongo.common import (
     MAX_BSON_SIZE,
@@ -860,6 +860,8 @@ def _hello(
         if creds:
             if creds.mechanism == "DEFAULT" and creds.username:
                 cmd["saslSupportedMechs"] = creds.source + "." + creds.username
+            from pymongo import auth
+
             auth_ctx = auth._AuthContext.from_credentials(creds, self.address)
             if auth_ctx:
                 speculative_authenticate = auth_ctx.speculate_command()
@@ -1091,6 +1093,8 @@ def authenticate(self, reauthenticate: bool = False) -> None:
         if not self.ready:
             creds = self.opts._credentials
             if creds:
+                from pymongo import auth
+
                 auth.authenticate(creds, self, reauthenticate=reauthenticate)
             self.ready = True
             if self.enabled_for_cmap: