Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Draft] [Storage] Decompression #39740

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
+68 −3
Draft

[Draft] [Storage] Decompression #39740

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0bb3eaa
Repro
vincenttran-msft Feb 14, 2025
1d242f0
Merge branch 'main' into vincenttran/decompression
vincenttran-msft Feb 14, 2025
37efb36
Add test case
vincenttran-msft Feb 14, 2025
f57f65c
Pass decompress=False despite no side-effect
vincenttran-msft Feb 14, 2025
3bcd401
Failing testcase in sync
vincenttran-msft Feb 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions sdk/storage/azure-storage-blob/tests/test_common_blob_async.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3455,4 +3455,59 @@ async def test_upload_blob_partial_stream_chunked(self, **kwargs):
# Assert
result = await (await blob.download_blob()).readall()
assert result == data[:length]

@pytest.mark.live_test_only
@BlobPreparer()
async def test_download_blob_decompress(self, **kwargs):
# This test will currently fail against the current codebase
# AssertionError: assert b'hello from gzip' ==
# b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00'
# Becuase on decompress=False, data.response.read() does not respect turning off decompression. So result (LHS) comes back decompressed.

# Proposed fix:
# If we change how we access the content to: content = b"".join([d async for d in data]) in _download_async
# from await data.response.read() / content = cast(bytes, data.response.content)
# This will respect the decompress=

storage_account_name = kwargs.pop("storage_account_name")
storage_account_key = kwargs.pop("storage_account_key")

# Arrange
await self._setup(storage_account_name, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
compressed_data = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00'
decompressed_data = b"hello from gzip"
content_settings = ContentSettings(content_encoding='gzip')

# Act / Assert
await blob.upload_blob(data=compressed_data, content_settings=content_settings, overwrite=True)

downloaded = await blob.download_blob(decompress=True)
result = await downloaded.readall()
assert result == decompressed_data

downloaded = await blob.download_blob(decompress=False)
result = await downloaded.readall()
assert result == compressed_data
Copy link
Member Author

@vincenttran-msft vincenttran-msft Feb 14, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This scenario (no content validation, upload compressed data, but download specifying decompress=False) will fail here:

>       assert result == compressed_data
E       AssertionError: assert b'hello from gzip' == b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00'
E         
E         At index 0 diff: b'h' != b'\x1f'
E         
E         Full diff:
E         + (b'hello from gzip')
E         - (b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcf'
E         -  b'UH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00')

test_common_blob_async.py:3537: AssertionError

This is the download code for non-content validation path:

 await data.response.load_body() # data.response is type RestAioHttpTransportResponse
    content = cast(bytes, data.response.body())

The issue is that RestAioHttpTransportResponse did not respect decompress=False, so it decompressed the compressed data.


@pytest.mark.live_test_only
@BlobPreparer()
async def test_download_blob_decompress_md5(self, **kwargs):
storage_account_name = kwargs.pop("storage_account_name")
storage_account_key = kwargs.pop("storage_account_key")

# Arrange
await self._setup(storage_account_name, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
compressed_data = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00'
decompressed_data = b"hello from gzip"
content_settings = ContentSettings(content_encoding='gzip')

# Act / Assert
await blob.upload_blob(data=compressed_data, content_settings=content_settings, overwrite=True)
downloaded = await blob.download_blob(validate_content=True)
result = await downloaded.readall()
assert result == decompressed_data
# ------------------------------------------------------------------------------