From e032c799707843cc9f603ef97f5ef7f3efdeca8f Mon Sep 17 00:00:00 2001 From: tbeu Date: Fri, 26 Jul 2019 14:00:12 +0200 Subject: [PATCH] Read compressed MAT file always in blocks As reported by https://github.com/tbeu/matio/issues/65 --- src/read_data.c | 56 +++++++++++++++++++++++++++++++++-------- src/read_data_impl.h | 60 ++++++++++++++++++++++---------------------- 2 files changed, 76 insertions(+), 40 deletions(-) diff --git a/src/read_data.c b/src/read_data.c index 14048393..ffb93409 100644 --- a/src/read_data.c +++ b/src/read_data.c @@ -45,10 +45,11 @@ const size_t block_size = READ_BLOCK_SIZE/data_size; \ if ( len <= block_size ) { \ bytesread += fread(v, data_size, len, (FILE*)mat->fp); \ - for ( j = 0; j < len; j++ ) { \ - data[j] = (T)v[j]; \ + for ( i = 0; i < len; i++ ) { \ + data[i] = (T)v[i]; \ } \ } else { \ + size_t j; \ for ( i = 0; i < len - block_size; i += block_size ) { \ bytesread += fread(v, data_size, block_size, (FILE*)mat->fp); \ for ( j = 0; j < block_size; j++ ) { \ @@ -70,10 +71,11 @@ const size_t block_size = READ_BLOCK_SIZE/data_size; \ if ( len <= block_size ) { \ bytesread += fread(v, data_size, len, (FILE*)mat->fp); \ - for ( j = 0; j < len; j++ ) { \ - data[j] = (T)SwapFunc(&v[j]); \ + for ( i = 0; i < len; i++ ) { \ + data[i] = (T)SwapFunc(&v[i]); \ } \ } else { \ + size_t j; \ for ( i = 0; i < len - block_size; i += block_size ) { \ bytesread += fread(v, data_size, block_size, (FILE*)mat->fp); \ for ( j = 0; j < block_size; j++ ) { \ @@ -95,18 +97,52 @@ #if defined(HAVE_ZLIB) #define READ_COMPRESSED_DATA_NOSWAP(T) \ do { \ - for ( i = 0; i < len; i++ ) { \ - InflateData(mat, z, &v, data_size); \ - data[i] = (T)v; \ + const size_t block_size = READ_BLOCK_SIZE/data_size; \ + if ( len <= block_size ) { \ + InflateData(mat, z, v, len*data_size); \ + for ( i = 0; i < len; i++ ) { \ + data[i] = (T)v[i]; \ + } \ + } else { \ + mat_uint32_t j; \ + len -= block_size; \ + for ( i = 0; i < len; i += block_size ) { \ + InflateData(mat, z, v, block_size*data_size); \ + for ( j = 0; j < block_size; j++ ) { \ + data[i + j] = (T)v[j]; \ + } \ + } \ + len -= (i - block_size); \ + InflateData(mat, z, v, len*data_size); \ + for ( j = 0; j < len; j++ ) { \ + data[i + j] = (T)v[j]; \ + } \ } \ } while (0) #define READ_COMPRESSED_DATA(T, SwapFunc) \ do { \ if ( mat->byteswap ) { \ - for ( i = 0; i < len; i++ ) { \ - InflateData(mat, z, &v, data_size); \ - data[i] = (T)SwapFunc(&v); \ + const size_t block_size = READ_BLOCK_SIZE/data_size; \ + if ( len <= block_size ) { \ + InflateData(mat, z, v, len*data_size); \ + for ( i = 0; i < len; i++ ) { \ + data[i] = (T)SwapFunc(&v[i]); \ + } \ + } else { \ + mat_uint32_t j; \ + len -= block_size; \ + for ( i = 0; i < len; i += block_size ) { \ + InflateData(mat, z, v, block_size*data_size); \ + for ( j = 0; j < block_size; j++ ) { \ + data[i + j] = (T)SwapFunc(&v[j]); \ + } \ + } \ + len -= (i - block_size); \ + InflateData(mat, z, v, len*data_size); \ + for ( j = 0; j < len; j++ ) { \ + data[i + j] = (T)SwapFunc(&v[j]); \ + } \ } \ } else { \ READ_COMPRESSED_DATA_NOSWAP(T); \ diff --git a/src/read_data_impl.h b/src/read_data_impl.h index 2f484129..fbe0958a 100644 --- a/src/read_data_impl.h +++ b/src/read_data_impl.h @@ -52,7 +52,7 @@ READ_TYPE_DOUBLE_DATA(mat_t *mat, READ_TYPE *data, size_t len) } } #else - size_t i, j; + size_t i; const size_t data_size = sizeof(double); double v[READ_BLOCK_SIZE/sizeof(double)]; READ_DATA(READ_TYPE, Mat_doubleSwap); @@ -73,7 +73,7 @@ READ_TYPE_SINGLE_DATA(mat_t *mat, READ_TYPE *data, size_t len) } } #else - size_t i, j; + size_t i; const size_t data_size = sizeof(float); float v[READ_BLOCK_SIZE/sizeof(float)]; READ_DATA(READ_TYPE, Mat_floatSwap); @@ -94,7 +94,7 @@ READ_TYPE_INT32_DATA(mat_t *mat, READ_TYPE *data, size_t len) } } #else - size_t i, j; + size_t i; const size_t data_size = sizeof(mat_int32_t); mat_int32_t v[READ_BLOCK_SIZE/sizeof(mat_int32_t)]; READ_DATA(READ_TYPE, Mat_int32Swap); @@ -115,7 +115,7 @@ READ_TYPE_UINT32_DATA(mat_t *mat, READ_TYPE *data, size_t len) } } #else - size_t i, j; + size_t i; const size_t data_size = sizeof(mat_uint32_t); mat_uint32_t v[READ_BLOCK_SIZE/sizeof(mat_uint32_t)]; READ_DATA(READ_TYPE, Mat_uint32Swap); @@ -136,7 +136,7 @@ READ_TYPE_INT16_DATA(mat_t *mat, READ_TYPE *data, size_t len) } } #else - size_t i, j; + size_t i; const size_t data_size = sizeof(mat_int16_t); mat_int16_t v[READ_BLOCK_SIZE/sizeof(mat_int16_t)]; READ_DATA(READ_TYPE, Mat_int16Swap); @@ -157,7 +157,7 @@ READ_TYPE_UINT16_DATA(mat_t *mat, READ_TYPE *data, size_t len) } } #else - size_t i, j; + size_t i; const size_t data_size = sizeof(mat_uint16_t); mat_uint16_t v[READ_BLOCK_SIZE/sizeof(mat_uint16_t)]; READ_DATA(READ_TYPE, Mat_uint16Swap); @@ -172,7 +172,7 @@ READ_TYPE_INT8_DATA(mat_t *mat, READ_TYPE *data, size_t len) #if READ_TYPE_TYPE == READ_TYPE_INT8 bytesread += fread(data, sizeof(mat_int8_t), len, (FILE*)mat->fp); #else - size_t i, j; + size_t i; const size_t data_size = sizeof(mat_int8_t); mat_int8_t v[READ_BLOCK_SIZE/sizeof(mat_int8_t)]; READ_DATA_NOSWAP(READ_TYPE); @@ -187,7 +187,7 @@ READ_TYPE_UINT8_DATA(mat_t *mat, READ_TYPE *data, size_t len) #if READ_TYPE_TYPE == READ_TYPE_UINT8 bytesread += fread(data, sizeof(mat_uint8_t), len, (FILE*)mat->fp); #else - size_t i, j; + size_t i; const size_t data_size = sizeof(mat_uint8_t); mat_uint8_t v[READ_BLOCK_SIZE/sizeof(mat_uint8_t)]; READ_DATA_NOSWAP(READ_TYPE); @@ -209,7 +209,7 @@ READ_TYPE_INT64_DATA(mat_t *mat, READ_TYPE *data, size_t len) } } #else - size_t i, j; + size_t i; const size_t data_size = sizeof(mat_int64_t); mat_int64_t v[READ_BLOCK_SIZE/sizeof(mat_int64_t)]; READ_DATA(READ_TYPE, Mat_int64Swap); @@ -232,7 +232,7 @@ READ_TYPE_UINT64_DATA(mat_t *mat, READ_TYPE *data, size_t len) } } #else - size_t i, j; + size_t i; const size_t data_size = sizeof(mat_uint64_t); mat_uint64_t v[READ_BLOCK_SIZE/sizeof(mat_uint64_t)]; READ_DATA(READ_TYPE, Mat_uint64Swap); @@ -349,9 +349,9 @@ READ_TYPE_DOUBLE_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len } } #else - size_t i, j; + mat_uint32_t i; const size_t data_size = sizeof(double); - double v;//[READ_BLOCK_SIZE/sizeof(double)]; + double v[READ_BLOCK_SIZE/sizeof(double)]; READ_COMPRESSED_DATA(READ_TYPE, Mat_doubleSwap); #endif return bytesread; @@ -370,9 +370,9 @@ READ_TYPE_SINGLE_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len } } #else - size_t i, j; + mat_uint32_t i; const size_t data_size = sizeof(float); - float v;//[READ_BLOCK_SIZE/sizeof(float)]; + float v[READ_BLOCK_SIZE/sizeof(float)]; READ_COMPRESSED_DATA(READ_TYPE, Mat_floatSwap); #endif return bytesread; @@ -392,9 +392,9 @@ READ_TYPE_INT64_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len) } } #else - size_t i, j; + mat_uint32_t i; const size_t data_size = sizeof(mat_int64_t); - mat_int64_t v;//[READ_BLOCK_SIZE/sizeof(mat_int64_t)]; + mat_int64_t v[READ_BLOCK_SIZE/sizeof(mat_int64_t)]; READ_COMPRESSED_DATA(READ_TYPE, Mat_int64Swap); #endif return bytesread; @@ -415,9 +415,9 @@ READ_TYPE_UINT64_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len } } #else - size_t i, j; + mat_uint32_t i; const size_t data_size = sizeof(mat_uint64_t); - mat_uint64_t v;//[READ_BLOCK_SIZE/sizeof(mat_uint64_t)]; + mat_uint64_t v[READ_BLOCK_SIZE/sizeof(mat_uint64_t)]; READ_COMPRESSED_DATA(READ_TYPE, Mat_uint64Swap); #endif return bytesread; @@ -437,9 +437,9 @@ READ_TYPE_INT32_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len) } } #else - size_t i, j; + mat_uint32_t i; const size_t data_size = sizeof(mat_int32_t); - mat_int32_t v;//[READ_BLOCK_SIZE/sizeof(mat_int32_t)]; + mat_int32_t v[READ_BLOCK_SIZE/sizeof(mat_int32_t)]; READ_COMPRESSED_DATA(READ_TYPE, Mat_int32Swap); #endif return bytesread; @@ -458,9 +458,9 @@ READ_TYPE_UINT32_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len } } #else - size_t i, j; + mat_uint32_t i; const size_t data_size = sizeof(mat_uint32_t); - mat_uint32_t v;//[READ_BLOCK_SIZE/sizeof(mat_uint32_t)]; + mat_uint32_t v[READ_BLOCK_SIZE/sizeof(mat_uint32_t)]; READ_COMPRESSED_DATA(READ_TYPE, Mat_uint32Swap); #endif return bytesread; @@ -479,9 +479,9 @@ READ_TYPE_INT16_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len) } } #else - size_t i, j; + mat_uint32_t i; const size_t data_size = sizeof(mat_int16_t); - mat_int16_t v;//[READ_BLOCK_SIZE/sizeof(mat_int16_t)]; + mat_int16_t v[READ_BLOCK_SIZE/sizeof(mat_int16_t)]; READ_COMPRESSED_DATA(READ_TYPE, Mat_int16Swap); #endif return bytesread; @@ -500,9 +500,9 @@ READ_TYPE_UINT16_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len } } #else - size_t i, j; + mat_uint32_t i; const size_t data_size = sizeof(mat_uint16_t); - mat_uint16_t v;//[READ_BLOCK_SIZE/sizeof(mat_uint16_t)]; + mat_uint16_t v[READ_BLOCK_SIZE/sizeof(mat_uint16_t)]; READ_COMPRESSED_DATA(READ_TYPE, Mat_uint16Swap); #endif return bytesread; @@ -515,9 +515,9 @@ READ_TYPE_INT8_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len) #if READ_TYPE_TYPE == READ_TYPE_INT8 bytesread = InflateData(mat, z, data, len*sizeof(mat_int8_t)); #else - size_t i, j; + mat_uint32_t i; const size_t data_size = sizeof(mat_int8_t); - mat_int8_t v;//[READ_BLOCK_SIZE/sizeof(mat_int8_t)]; + mat_int8_t v[READ_BLOCK_SIZE/sizeof(mat_int8_t)]; READ_COMPRESSED_DATA_NOSWAP(READ_TYPE); #endif return bytesread; @@ -530,9 +530,9 @@ READ_TYPE_UINT8_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len) #if READ_TYPE_TYPE == READ_TYPE_UINT8 bytesread = InflateData(mat, z, data, len*sizeof(mat_uint8_t)); #else - size_t i, j; + mat_uint32_t i; const size_t data_size = sizeof(mat_uint8_t); - mat_uint8_t v;//[READ_BLOCK_SIZE/sizeof(mat_uint8_t)]; + mat_uint8_t v[READ_BLOCK_SIZE/sizeof(mat_uint8_t)]; READ_COMPRESSED_DATA_NOSWAP(READ_TYPE); #endif return bytesread;