Skip to content

Commit 42392ef

Browse files
authored
fix: do not spam the log with checksum related INFO messages when downloading using transfer_manager (#1357)
* fix: do not spam the log with checksum related INFO messages when downloading using transfer_manager `download_chunks_concurrently` function does not allow to set `checksum` field in `download_kwargs`. It also does not set it on its own so it takes the default value of `"md5"` (see `Blob._prep_and_do_download`). Because ranged downloads do not return checksums it results in a lot of INFO messages (tens/hundreds): ``` INFO google.resumable_media._helpers - No MD5 checksum was returned from the service while downloading ... (which happens for composite objects), so client-side content integrity checking is not being performed. ``` To fix it set the `checksum` field to `None` which means no checksum checking for individual chunks. Note that `transfer_manager` has its own checksum checking logic (enabled by `crc32c_checksum` argument) * fix tests
1 parent 8ec02c0 commit 42392ef

File tree

2 files changed

+3
-6
lines changed

2 files changed

+3
-6
lines changed

google/cloud/storage/transfer_manager.py

+2
Original file line numberDiff line numberDiff line change
@@ -885,6 +885,8 @@ def download_chunks_concurrently(
885885
"'checksum' is in download_kwargs, but is not supported because sliced downloads have a different checksum mechanism from regular downloads. Use the 'crc32c_checksum' argument on download_chunks_concurrently instead."
886886
)
887887

888+
download_kwargs = download_kwargs.copy()
889+
download_kwargs["checksum"] = None
888890
download_kwargs["command"] = "tm.download_sharded"
889891

890892
# We must know the size and the generation of the blob.

tests/unit/test_transfer_manager.py

+1-6
Original file line numberDiff line numberDiff line change
@@ -606,6 +606,7 @@ def test_download_chunks_concurrently():
606606

607607
expected_download_kwargs = EXPECTED_DOWNLOAD_KWARGS.copy()
608608
expected_download_kwargs["command"] = "tm.download_sharded"
609+
expected_download_kwargs["checksum"] = None
609610

610611
with mock.patch("google.cloud.storage.transfer_manager.open", mock.mock_open()):
611612
result = transfer_manager.download_chunks_concurrently(
@@ -636,9 +637,6 @@ def test_download_chunks_concurrently_with_crc32c():
636637
blob_mock.size = len(BLOB_CONTENTS)
637638
blob_mock.crc32c = "eOVVVw=="
638639

639-
expected_download_kwargs = EXPECTED_DOWNLOAD_KWARGS.copy()
640-
expected_download_kwargs["command"] = "tm.download_sharded"
641-
642640
def write_to_file(f, *args, **kwargs):
643641
f.write(BLOB_CHUNK)
644642

@@ -664,9 +662,6 @@ def test_download_chunks_concurrently_with_crc32c_failure():
664662
blob_mock.size = len(BLOB_CONTENTS)
665663
blob_mock.crc32c = "invalid"
666664

667-
expected_download_kwargs = EXPECTED_DOWNLOAD_KWARGS.copy()
668-
expected_download_kwargs["command"] = "tm.download_sharded"
669-
670665
def write_to_file(f, *args, **kwargs):
671666
f.write(BLOB_CHUNK)
672667

0 commit comments

Comments
 (0)