Merge pull request #167 from pycompression/readallmemoryleak

Fix memory leak in GzipReader.readall

CHANGELOG.rst

@@ -7,6 +7,10 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 1.5.1
+-----------------
++ Fix a memory leak in the GzipReader.readall implementation.
+
 version 1.5.0
 -----------------
 + Make a special case for threads==1 in ``igzip_threaded.open`` for writing

benchmark_scripts/memory_leak_test.py

@@ -0,0 +1,19 @@
+import gc
+import resource
+import sys
+
+from isal import igzip
+
+for _ in range(10):
+    with igzip.open(sys.argv[1], "rb") as reader:
+        a = reader.read()
+        print(len(a))
+        gc.collect()
+        memory_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        memory_usage_mb = memory_usage / 1024
+        print(f"Maximum memory usage: {memory_usage_mb:.2f} MiB")
+        del a
+objects_and_size = [(sys.getsizeof(obj), type(obj)) for obj in
+                    gc.get_objects()]
+objects_and_size.sort(key=lambda x: x[0], reverse=True)
+print(objects_and_size[:10])

src/isal/isal_zlibmodule.c

@@ -1995,14 +1995,16 @@ GzipReader_readall(GzipReader *self, PyObject *Py_UNUSED(ignore))
             return NULL;
         }
         if (written_size == 0) {
+            Py_DECREF(chunk);
             break;
         }
         if (_PyBytes_Resize(&chunk, written_size) < 0) {
             Py_DECREF(chunk_list);
             return NULL;
         }
-        if (PyList_Append(chunk_list, chunk) < 0) {
-            Py_DECREF(chunk);
+        int ret = PyList_Append(chunk_list, chunk);
+        Py_DECREF(chunk);
+        if (ret < 0) {
             Py_DECREF(chunk_list);
             return NULL;
         }
@@ -2014,6 +2016,7 @@ GzipReader_readall(GzipReader *self, PyObject *Py_UNUSED(ignore))
     }
     PyObject *ret = _PyBytes_Join(empty_bytes, chunk_list);
     Py_DECREF(empty_bytes);
+    Py_DECREF(chunk_list);
     return ret;
 }
 

tox.ini

@@ -71,20 +71,24 @@ commands=
 
 [testenv:benchmark-all]
 deps=
+setenv =
 commands=
     python ./benchmark_scripts/benchmark.py --all
 
 [testenv:benchmark-functions]
 deps=
+setenv =
 commands=
     python ./benchmark_scripts/benchmark.py --functions
 
 [testenv:benchmark-gzip]
 deps=
+setenv =
 commands=
     python ./benchmark_scripts/benchmark.py --gzip
 
 [testenv:benchmark-checksums]
 deps=
+setenv =
 commands=
     python ./benchmark_scripts/benchmark.py --checksums