From 8c3b56709df72dcca4351cb6556d392e3ddf037b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 10 Apr 2024 13:35:18 +0200 Subject: [PATCH 1/4] Set a new version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7fb82b6..8da9d85 100644 --- a/setup.py +++ b/setup.py @@ -124,7 +124,7 @@ def build_zlib_ng(): setup( name="zlib-ng", - version="0.4.2", + version="0.5.0-dev", description="Drop-in replacement for zlib and gzip modules using zlib-ng", author="Leiden University Medical Center", author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now From c124c5820392b0897cfd9a134c9b1b4cb05e0541 Mon Sep 17 00:00:00 2001 From: Boris Zikeyev <73240201+BorisOrca@users.noreply.github.com> Date: Tue, 16 Apr 2024 16:02:53 +0300 Subject: [PATCH 2/4] Fix ISIZE check to handle 32bit overflows --- CHANGELOG.rst | 4 ++++ src/zlib_ng/zlib_ngmodule.c | 5 +++-- tests/test_gzip_ng.py | 25 +++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7dc80dd..b6b8547 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 0.5.0-dev +----------------- ++ Fix a bug where files larger than 4GB could not be decompressed. + version 0.4.2 ----------------- + Fix a reference counting error that happened on module initialization and diff --git a/src/zlib_ng/zlib_ngmodule.c b/src/zlib_ng/zlib_ngmodule.c index f6d543d..ddeef4c 100644 --- a/src/zlib_ng/zlib_ngmodule.c +++ b/src/zlib_ng/zlib_ngmodule.c @@ -2590,8 +2590,9 @@ GzipReader_read_into_buffer(GzipReader *self, uint8_t *out_buffer, size_t out_bu return -1; } uint32_t length = load_u32_le(current_pos); - current_pos += 4; - if (length != self->zst.total_out) { + current_pos += 4; + // ISIZE is the length of the original data modulo 2^32 + if (length != (0xFFFFFFFFUL & self->zst.total_out)) { Py_BLOCK_THREADS; PyErr_SetString(BadGzipFile, "Incorrect length of data produced"); return -1; diff --git a/tests/test_gzip_ng.py b/tests/test_gzip_ng.py index a751d8e..abfd283 100644 --- a/tests/test_gzip_ng.py +++ b/tests/test_gzip_ng.py @@ -292,6 +292,31 @@ def test_decompress_incorrect_length(): error.match("Incorrect length of data produced") +def test_decompress_on_long_input(): + # Ensure that a compressed payload with length bigger than 2**32 (ISIZE is + # overflown) can be decompressed. To avoid writing the whole uncompressed payload + # into memory, the test writes the compressed data in chunks. The payload consists + # almost exclusively of zeros to achieve an exteremely efficient compression rate, + # so that the compressed data also fits in memory. + + buffered_stream = io.BytesIO() + n = 20 + block_size = 2**n + iterations = 2**(32 - n) + zeros_block = bytes(block_size) + + # To avoid writing the whole compressed data, we will write the compressed data + with gzip_ng.open(buffered_stream, "wb") as gz: + for _ in range(iterations): + gz.write(zeros_block) + gz.write(b"\x01" * 123) + buffered_stream.seek(0) + with gzip_ng.open(buffered_stream, "rb") as gz: + for _ in range(iterations): + assert zeros_block == gz.read(block_size) + assert gz.read() == b"\x01" * 123 + + def test_decompress_incorrect_checksum(): # Create a wrong checksum by using a non-default seed. wrong_checksum = zlib.crc32(DATA, 50) From 4a3f1a85909b3d3af97001650c534366efd4d263 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Apr 2024 15:05:35 +0200 Subject: [PATCH 3/4] Test on macos-14 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59170f2..51fa4ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,9 +62,9 @@ jobs: - "pypy-3.10" os: ["ubuntu-latest"] include: - - os: "macos-latest" # For m1 macos - python-version: "3.8" - - os: "macos-13" # for x86 macos + - os: "macos-14" # For m1 macos + python-version: "3.12" + - os: "macos-latest" # for x86 macos python-version: "3.8" - os: "windows-latest" python-version: "3.8" From 96424a28581e38c9cd71f173cbc3e246a4cb5462 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Apr 2024 15:36:56 +0200 Subject: [PATCH 4/4] Prepare 0.4.3 release --- CHANGELOG.rst | 2 +- setup.py | 2 +- src/zlib_ng/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b6b8547..742b50f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. -version 0.5.0-dev +version 0.4.3 ----------------- + Fix a bug where files larger than 4GB could not be decompressed. diff --git a/setup.py b/setup.py index 8da9d85..4c16c7f 100644 --- a/setup.py +++ b/setup.py @@ -124,7 +124,7 @@ def build_zlib_ng(): setup( name="zlib-ng", - version="0.5.0-dev", + version="0.4.3", description="Drop-in replacement for zlib and gzip modules using zlib-ng", author="Leiden University Medical Center", author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now diff --git a/src/zlib_ng/__init__.py b/src/zlib_ng/__init__.py index e696c44..c1e6f1b 100644 --- a/src/zlib_ng/__init__.py +++ b/src/zlib_ng/__init__.py @@ -5,4 +5,4 @@ # This file is part of python-zlib-ng which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. -__version__ = "0.4.2" +__version__ = "0.4.3"