Skip to content

Commit

Permalink
problem_report: reduce chunk size from 1 MB to 128 kB
Browse files Browse the repository at this point in the history
Use a consistant chunk size of 128 kB (instead of 1 MB) which is also
the maximum block size of Zstandard (`ZSTD_BLOCKSIZE_MAX`).
  • Loading branch information
bdrung committed Feb 28, 2025
1 parent 24f307b commit ed3b964
Showing 1 changed file with 5 additions and 5 deletions.
10 changes: 5 additions & 5 deletions problem_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,9 +201,9 @@ def __del__(self):
self._compressed_file.close()

def iter_compressed(self) -> Iterator[bytes]:
"""Iterate over the compressed content of the file in 1 MB chunks."""
"""Iterate over the compressed content of the file in chunks."""
while True:
block = self._compressed_file.read(1048576)
block = self._compressed_file.read(CHUNK_SIZE)

Check warning on line 206 in problem_report.py

View check run for this annotation

Codecov / codecov/patch

problem_report.py#L206

Added line #L206 was not covered by tests
if not block:
break
yield block
Expand Down Expand Up @@ -298,7 +298,7 @@ def write(self, file: typing.IO[bytes]) -> None:
if self.compressed_value.startswith(GZIP_HEADER_START):
with gzip.GzipFile(fileobj=io.BytesIO(self.compressed_value)) as gz:
while True:
block = gz.read(1048576)
block = gz.read(CHUNK_SIZE)
if not block:
break
file.write(block)
Expand Down Expand Up @@ -704,7 +704,7 @@ def _generate_compressed_chunks(self, key: str) -> Generator[bytes]:
# hard to change, pylint: disable=consider-using-with
f = open(value[0], "rb") # file name
while True:
block = f.read(1048576)
block = f.read(CHUNK_SIZE)
size += len(block)
crc = zlib.crc32(block, crc)
if limit is not None:
Expand Down Expand Up @@ -843,7 +843,7 @@ def write_mime(
out = io.BytesIO()
with gzip.GzipFile(k, mode="wb", fileobj=out, mtime=0) as gz:
while True:
block = f.read(1048576)
block = f.read(CHUNK_SIZE)

Check warning on line 846 in problem_report.py

View check run for this annotation

Codecov / codecov/patch

problem_report.py#L846

Added line #L846 was not covered by tests
if block:
gz.write(block)
else:
Expand Down

0 comments on commit ed3b964

Please sign in to comment.