Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Draft] [Storage] Decompression #39740

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
+68 −3
Draft

[Draft] [Storage] Decompression #39740

Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0bb3eaa
Repro
vincenttran-msft Feb 14, 2025
1d242f0
Merge branch 'main' into vincenttran/decompression
vincenttran-msft Feb 14, 2025
37efb36
Add test case
vincenttran-msft Feb 14, 2025
f57f65c
Pass decompress=False despite no side-effect
vincenttran-msft Feb 14, 2025
3bcd401
Failing testcase in sync
vincenttran-msft Feb 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 47 additions & 2 deletions sdk/storage/azure-storage-blob/tests/test_common_blob_async.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
class TestStorageCommonBlobAsync(AsyncStorageRecordedTestCase):
# --Helpers-----------------------------------------------------------------
async def _setup(self, storage_account_name, key):
self.bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"), credential=key)
self.bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"), credential=key, retry_total=0)
self.container_name = self.get_resource_name('utcontainer')
self.source_container_name = self.get_resource_name('utcontainersource')
self.byte_data = self.get_random_bytes(1024)
Expand Down Expand Up @@ -3510,4 +3510,49 @@ async def test_mock_transport_with_content_validation(self, **kwargs):

blob_data = await (await blob_client.download_blob(validate_content=True)).read()
assert blob_data == b"Hello Async World!" # data is fixed by mock transport
# ------------------------------------------------------------------------------

@pytest.mark.live_test_only
@BlobPreparer()
async def test_download_blob_decompress(self, **kwargs):
storage_account_name = kwargs.pop("storage_account_name")
storage_account_key = kwargs.pop("storage_account_key")

# Arrange
await self._setup(storage_account_name, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
compressed_data = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00'
decompressed_data = b"hello from gzip"
content_settings = ContentSettings(content_encoding='gzip')

# Act / Assert
await blob.upload_blob(data=compressed_data, content_settings=content_settings, overwrite=True)

downloaded = await blob.download_blob(decompress=True)
result = await downloaded.readall()
assert result == decompressed_data

downloaded = await blob.download_blob(decompress=False)
result = await downloaded.readall()
assert result == compressed_data
Copy link
Member Author

@vincenttran-msft vincenttran-msft Feb 14, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This scenario (no content validation, upload compressed data, but download specifying decompress=False) will fail here:

>       assert result == compressed_data
E       AssertionError: assert b'hello from gzip' == b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00'
E         
E         At index 0 diff: b'h' != b'\x1f'
E         
E         Full diff:
E         + (b'hello from gzip')
E         - (b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcf'
E         -  b'UH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00')

test_common_blob_async.py:3537: AssertionError

This is the download code for non-content validation path:

 await data.response.load_body() # data.response is type RestAioHttpTransportResponse
    content = cast(bytes, data.response.body())

The issue is that RestAioHttpTransportResponse did not respect decompress=False, so it decompressed the compressed data.


@pytest.mark.live_test_only
@BlobPreparer()
async def test_download_blob_decompress_md5(self, **kwargs):
storage_account_name = kwargs.pop("storage_account_name")
storage_account_key = kwargs.pop("storage_account_key")

# Arrange
await self._setup(storage_account_name, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
compressed_data = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00'
decompressed_data = b"hello from gzip"
content_settings = ContentSettings(content_encoding='gzip')

# Act / Assert
await blob.upload_blob(data=compressed_data, content_settings=content_settings, overwrite=True)
downloaded = await blob.download_blob(validate_content=True, decompress=False)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This scenario (validate_content=True, decompress=False), will not even return and will fail during the content validation step:

# response.http_response type is RestAioHttpTransportResponse
await response.http_response.load_body()  # Load the body in memory and close the socket
        except (StreamClosedError, StreamConsumedError):
            pass
        computed_md5 = response.http_request.headers.get('content-md5', None) or \
                       encode_base64(StorageContentValidation.get_content_md5(response.http_response.body()))

This is because computed_md5 will evaluate to encode_base64(StorageContentValidation.get_content_md5(response.http_response.body())),

where response.http_response.body() , because decompression is on by default (and our decompress=False had no effect), will compute the md5 against the decompressed data. Whereas the md5 returned at response.http_response.headers['content-md5'] is computed against the data uploaded (which was compressed).

Therefore, we need to be able to request the data back from Storage without auto-decompression, so that we can properly perform the MD5 check.

result = await downloaded.readall()
assert result == decompressed_data
# ------------------------------------------------------------------------------
Loading