feat: Integrate google-resumable-media (#1283)

Integrate the google-resumable-media library into python-storage.

---------

Co-authored-by: cojenco <[email protected]>

Author: andrewsg

README.rst

@@ -37,6 +37,41 @@ Google APIs Client Libraries, in `Client Libraries Explained`_.
 .. _Storage Control API: https://cloud.google.com/storage/docs/reference/rpc/google.storage.control.v2
 .. _Client Libraries Explained: https://cloud.google.com/apis/docs/client-libraries-explained
 
+Major Version Release Notes
+---------------------------
+
+Preview Release
+~~~~~~~~~~~~~~~
+
+Python Storage 3.0 is currently in a preview state. If you experience that
+backwards compatibility for your application is broken with this release for any
+reason, please let us know through the Github issues system. Thank you.
+
+Exception Handling
+~~~~~~~~~~~~~~~~~~
+
+In Python Storage 3.0, the dependency `google-resumable-media` was integrated.
+The `google-resumable-media` dependency included exceptions
+`google.resumable_media.common.InvalidResponse` and
+`google.resumable_media.common.DataCorruption`, which were often imported
+directly in user application code. The replacements for these exceptions are
+`google.cloud.storage.exceptions.InvalidResponse` and
+`google.cloud.storage.exceptions.DataCorruption`. Please update application code
+to import and use these exceptions instead.
+
+For backwards compatibility, if `google-resumable-media` is installed, the new
+exceptions will be defined as subclasses of the old exceptions, so applications
+should continue to work without modification. This backwards compatibility
+feature may be removed in a future major version update.
+
+Some users may be using the original exception classes from the
+`google-resumable-media` library without explicitly importing that library. So
+as not to break user applications following this pattern,
+`google-resumable-media` is still in the list of dependencies in this package's
+setup.py file. Applications which do not import directly from
+`google-resumable-media` can safely disregard this dependency. This backwards
+compatibility feature will be removed in a future major version update.
+
 Quick Start
 -----------
 

docs/storage/exceptions.rst

@@ -0,0 +1,7 @@
+Exceptions
+~~~~~~~~~
+
+.. automodule:: google.cloud.storage.exceptions
+  :members:
+  :member-order: bysource
+

google/cloud/storage/_helpers.py

@@ -25,8 +25,8 @@
 from urllib.parse import urlunsplit
 from uuid import uuid4
 
-from google import resumable_media
 from google.auth import environment_vars
+from google.cloud.storage import _media
 from google.cloud.storage.constants import _DEFAULT_TIMEOUT
 from google.cloud.storage.retry import DEFAULT_RETRY
 from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED
@@ -635,7 +635,7 @@ def _bucket_bound_hostname_url(host, scheme=None):
 
 
 def _api_core_retry_to_resumable_media_retry(retry, num_retries=None):
-    """Convert google.api.core.Retry to google.resumable_media.RetryStrategy.
+    """Convert google.api.core.Retry to google.cloud.storage._media.RetryStrategy.
 
     Custom predicates are not translated.
 
@@ -647,7 +647,7 @@ def _api_core_retry_to_resumable_media_retry(retry, num_retries=None):
         supported for backwards compatibility and is mutually exclusive with
         `retry`.
 
-    :rtype: google.resumable_media.RetryStrategy
+    :rtype: google.cloud.storage._media.RetryStrategy
     :returns: A RetryStrategy with all applicable attributes copied from input,
               or a RetryStrategy with max_retries set to 0 if None was input.
     """
@@ -656,16 +656,16 @@ def _api_core_retry_to_resumable_media_retry(retry, num_retries=None):
         raise ValueError("num_retries and retry arguments are mutually exclusive")
 
     elif retry is not None:
-        return resumable_media.RetryStrategy(
+        return _media.RetryStrategy(
             max_sleep=retry._maximum,
             max_cumulative_retry=retry._deadline,
             initial_delay=retry._initial,
             multiplier=retry._multiplier,
         )
     elif num_retries is not None:
-        return resumable_media.RetryStrategy(max_retries=num_retries)
+        return _media.RetryStrategy(max_retries=num_retries)
     else:
-        return resumable_media.RetryStrategy(max_retries=0)
+        return _media.RetryStrategy(max_retries=0)
 
 
 def _get_invocation_id():

google/cloud/storage/_media/__init__.py

@@ -14,48 +14,23 @@
 
 """Utilities for Google Media Downloads and Resumable Uploads.
 
-This package has some general purposes modules, e.g.
-:mod:`~google.resumable_media.common`, but the majority of the
-public interface will be contained in subpackages.
-
 ===========
 Subpackages
 ===========
 
 Each subpackage is tailored to a specific transport library:
 
-* the :mod:`~google.resumable_media.requests` subpackage uses the ``requests``
+* the :mod:`~google.cloud.storage._media.requests` subpackage uses the ``requests``
   transport library.
 
 .. _requests: http://docs.python-requests.org/
-
-==========
-Installing
-==========
-
-To install with `pip`_:
-
-.. code-block:: console
-
-  $ pip install --upgrade google-resumable-media
-
-.. _pip: https://pip.pypa.io/
 """
 
-
-from google.resumable_media.common import DataCorruption
-from google.resumable_media.common import InvalidResponse
-from google.resumable_media.common import PERMANENT_REDIRECT
-from google.resumable_media.common import RetryStrategy
-from google.resumable_media.common import TOO_MANY_REQUESTS
-from google.resumable_media.common import UPLOAD_CHUNK_SIZE
+from google.cloud.storage._media.common import RetryStrategy
+from google.cloud.storage._media.common import UPLOAD_CHUNK_SIZE
 
 
 __all__ = [
-    "DataCorruption",
-    "InvalidResponse",
-    "PERMANENT_REDIRECT",
     "RetryStrategy",
-    "TOO_MANY_REQUESTS",
     "UPLOAD_CHUNK_SIZE",
 ]

google/cloud/storage/_media/_download.py

@@ -18,8 +18,9 @@
 import http.client
 import re
 
-from google.resumable_media import _helpers
-from google.resumable_media import common
+from google.cloud.storage._media import _helpers
+from google.cloud.storage._media import common
+from google.cloud.storage.exceptions import InvalidResponse
 
 
 _CONTENT_RANGE_RE = re.compile(
@@ -361,7 +362,7 @@ def _process_response(self, response):
             response (object): The HTTP response object (need headers).
 
         Raises:
-            ~google.resumable_media.common.InvalidResponse: If the number
+            ~google.cloud.storage.exceptions.InvalidResponse: If the number
                 of bytes in the body doesn't match the content length header.
 
         .. _sans-I/O: https://sans-io.readthedocs.io/
@@ -398,7 +399,7 @@ def _process_response(self, response):
             num_bytes = int(content_length)
             if len(response_body) != num_bytes:
                 self._make_invalid()
-                raise common.InvalidResponse(
+                raise InvalidResponse(
                     response,
                     "Response is different size than content-length",
                     "Expected",
@@ -508,7 +509,7 @@ def get_range_info(response, get_headers, callback=_helpers.do_nothing):
         Tuple[int, int, int]: The start byte, end byte and total bytes.
 
     Raises:
-        ~google.resumable_media.common.InvalidResponse: If the
+        ~google.cloud.storage.exceptions.InvalidResponse: If the
             ``Content-Range`` header is not of the form
             ``bytes {start}-{end}/{total}``.
     """
@@ -518,7 +519,7 @@ def get_range_info(response, get_headers, callback=_helpers.do_nothing):
     match = _CONTENT_RANGE_RE.match(content_range)
     if match is None:
         callback()
-        raise common.InvalidResponse(
+        raise InvalidResponse(
             response,
             "Unexpected content-range header",
             content_range,

google/cloud/storage/_media/_helpers.py

@@ -27,7 +27,8 @@
 from urllib.parse import urlsplit
 from urllib.parse import urlunsplit
 
-from google.resumable_media import common
+from google.cloud.storage._media import common
+from google.cloud.storage.exceptions import InvalidResponse
 
 
 RANGE_HEADER = "range"
@@ -70,15 +71,13 @@ def header_required(response, name, get_headers, callback=do_nothing):
         str: The desired header.
 
     Raises:
-        ~google.resumable_media.common.InvalidResponse: If the header
+        ~google.cloud.storage.exceptions.InvalidResponse: If the header
             is missing.
     """
     headers = get_headers(response)
     if name not in headers:
         callback()
-        raise common.InvalidResponse(
-            response, "Response headers must contain header", name
-        )
+        raise InvalidResponse(response, "Response headers must contain header", name)
 
     return headers[name]
 
@@ -98,14 +97,14 @@ def require_status_code(response, status_codes, get_status_code, callback=do_not
         int: The status code.
 
     Raises:
-        ~google.resumable_media.common.InvalidResponse: If the status code
+        ~google.cloud.storage.exceptions.InvalidResponse: If the status code
             is not one of the values in ``status_codes``.
     """
     status_code = get_status_code(response)
     if status_code not in status_codes:
         if status_code not in common.RETRYABLE:
             callback()
-        raise common.InvalidResponse(
+        raise InvalidResponse(
             response,
             "Request failed with status code",
             status_code,
@@ -298,7 +297,7 @@ def _parse_checksum_header(header_value, response, checksum_label):
         can be detected from the ``X-Goog-Hash`` header; otherwise, None.
 
     Raises:
-        ~google.resumable_media.common.InvalidResponse: If there are
+        ~google.cloud.storage.exceptions.InvalidResponse: If there are
             multiple checksums of the requested type in ``header_value``.
     """
     if header_value is None:
@@ -316,7 +315,7 @@ def _parse_checksum_header(header_value, response, checksum_label):
     elif len(matches) == 1:
         return matches[0]
     else:
-        raise common.InvalidResponse(
+        raise InvalidResponse(
             response,
             "X-Goog-Hash header had multiple ``{}`` values.".format(checksum_label),
             header_value,