fix(storage): track EOF in BlobReader to handle doubly-gzipped blobs (#18423)

Fixes #18423. When reading doubly-gzipped blobs (or objects with Content-Encoding: gzip), out-of-bounds byte range requests may return full payload bytes instead of HTTP 416 RequestRangeNotSatisfiable.

This change adds client-side _eof tracking to BlobReader. When a range download returns fewer bytes than requested (or raises 416), BlobReader sets _eof = True so subsequent read() calls immediately return b"" without initiating redundant HTTP range requests.

Author: ohmayr

packages/google-cloud-storage/google/cloud/storage/fileio.py

@@ -118,14 +118,15 @@ def __init__(self, blob, chunk_size=None, retry=DEFAULT_RETRY, **download_kwargs
         self._chunk_size = chunk_size or blob.chunk_size or DEFAULT_CHUNK_SIZE
         self._retry = retry
         self._download_kwargs = download_kwargs
+        self._eof = False
 
     def read(self, size=-1):
         self._checkClosed()  # Raises ValueError if closed.
 
         result = self._buffer.read(size)
         # If the read request demands more bytes than are buffered, fetch more.
         remaining_size = size - len(result)
-        if remaining_size > 0 or size < 0:
+        if (remaining_size > 0 or size < 0) and not self._eof:
             self._pos += self._buffer.tell()
             read_size = len(result)
 
@@ -142,17 +143,31 @@ def read(self, size=-1):
             # chunked downloads, and the server only knows the checksum of the
             # entire file.
             try:
-                result += self._blob.download_as_bytes(
+                downloaded = self._blob.download_as_bytes(
                     start=fetch_start,
                     end=fetch_end,
                     checksum=None,
                     retry=self._retry,
                     **self._download_kwargs,
                 )
+                # If fewer bytes were returned than requested for a range (len < fetch_end - fetch_start),
+                # or 0 bytes were returned, we have reached EOF. Tracking EOF client-side prevents
+                # infinite read loops for objects (such as doubly-gzipped files) where out-of-bounds
+                # range requests re-send body content instead of raising RequestRangeNotSatisfiable.
+                if (
+                    fetch_end is None
+                    or len(downloaded) == 0
+                    or (
+                        fetch_end is not None
+                        and len(downloaded) < (fetch_end - fetch_start)
+                    )
+                ):
+                    self._eof = True
+                result += downloaded
             except RequestRangeNotSatisfiable:
                 # We've reached the end of the file. Python file objects should
                 # return an empty response in this case, not raise an error.
-                pass
+                self._eof = True
 
             # If more bytes were read than is immediately needed, buffer the
             # remainder and then trim the result.
@@ -175,6 +190,7 @@ def seek(self, pos, whence=0):
         If the blob size is not already known it will call blob.reload().
         """
         self._checkClosed()  # Raises ValueError if closed.
+        self._eof = False
 
         if self._blob.size is None:
             reload_kwargs = {

packages/google-cloud-storage/tests/unit/test_fileio.py

@@ -159,6 +159,27 @@ def test_416_error_handled(self):
         reader = self._make_blob_reader(blob)
         self.assertEqual(reader.read(), b"")
 
+    def test_read_doubly_gzipped_eof(self):
+        blob = mock.Mock()
+        fake_data = b"x" * 100  # 100 bytes of test data
+
+        def download_side_effect(start=0, end=None, **_):
+            # For doubly-gzipped blobs (Content-Encoding: gzip), out-of-bounds
+            # range requests return HTTP 200 with the full transcoded body
+            # rather than HTTP 416. Simulate this GCS server behavior.
+            return fake_data
+
+        blob.download_as_bytes = mock.Mock(side_effect=download_side_effect)
+        reader = self._make_blob_reader(blob, chunk_size=1024)
+
+        # First read fetches 100 bytes (< 1024 chunk_size), setting client-side EOF
+        data1 = reader.read(1024)
+        self.assertEqual(data1, fake_data)
+
+        # Second read must return empty bytes without making additional HTTP requests
+        data2 = reader.read(1024)
+        self.assertEqual(data2, b"")
+
     def test_readline(self):
         blob = mock.Mock()
 
@@ -185,15 +206,15 @@ def read_from_fake_data(start=0, end=None, **_):
         blob.size = len(TEST_BINARY_DATA)
         reader.seek(0)
 
-        # Read all lines. The readlines algorithm will attempt to read past the end of the last line once to verify there is no more to read.
+        # Read all lines. With client-side EOF detection on short reads (chunk 6 returned 4 bytes < 10 requested), no extra call past EOF is made.
         self.assertEqual(b"".join(reader.readlines()), TEST_BINARY_DATA)
         blob.download_as_bytes.assert_called_with(
-            start=len(TEST_BINARY_DATA),
-            end=len(TEST_BINARY_DATA) + 10,
+            start=50,
+            end=60,
             checksum=None,
             retry=DEFAULT_RETRY,
         )
-        self.assertEqual(blob.download_as_bytes.call_count, 13)
+        self.assertEqual(blob.download_as_bytes.call_count, 12)
 
         reader.close()