Skip to content

Commit

Permalink
Read compressed MAT file always in blocks
Browse files Browse the repository at this point in the history 
As reported by #65
  • Loading branch information
@tbeu
tbeu committed Jul 26, 2019
1 parent db29fe3 commit e032c79
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 40 deletions.
56 changes: 46 additions & 10 deletions src/read_data.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,11 @@
const size_t block_size = READ_BLOCK_SIZE/data_size; \
if ( len <= block_size ) { \
bytesread += fread(v, data_size, len, (FILE*)mat->fp); \
for ( j = 0; j < len; j++ ) { \
data[j] = (T)v[j]; \
for ( i = 0; i < len; i++ ) { \
data[i] = (T)v[i]; \
} \
} else { \
size_t j; \
for ( i = 0; i < len - block_size; i += block_size ) { \
bytesread += fread(v, data_size, block_size, (FILE*)mat->fp); \
for ( j = 0; j < block_size; j++ ) { \
Expand All @@ -70,10 +71,11 @@
const size_t block_size = READ_BLOCK_SIZE/data_size; \
if ( len <= block_size ) { \
bytesread += fread(v, data_size, len, (FILE*)mat->fp); \
for ( j = 0; j < len; j++ ) { \
data[j] = (T)SwapFunc(&v[j]); \
for ( i = 0; i < len; i++ ) { \
data[i] = (T)SwapFunc(&v[i]); \
} \
} else { \
size_t j; \
for ( i = 0; i < len - block_size; i += block_size ) { \
bytesread += fread(v, data_size, block_size, (FILE*)mat->fp); \
for ( j = 0; j < block_size; j++ ) { \
Expand All @@ -95,18 +97,52 @@
#if defined(HAVE_ZLIB)
#define READ_COMPRESSED_DATA_NOSWAP(T) \
do { \
for ( i = 0; i < len; i++ ) { \
InflateData(mat, z, &v, data_size); \
data[i] = (T)v; \
const size_t block_size = READ_BLOCK_SIZE/data_size; \
if ( len <= block_size ) { \
InflateData(mat, z, v, len*data_size); \
for ( i = 0; i < len; i++ ) { \
data[i] = (T)v[i]; \
} \
} else { \
mat_uint32_t j; \
len -= block_size; \
for ( i = 0; i < len; i += block_size ) { \
InflateData(mat, z, v, block_size*data_size); \
for ( j = 0; j < block_size; j++ ) { \
data[i + j] = (T)v[j]; \
} \
} \
len -= (i - block_size); \
InflateData(mat, z, v, len*data_size); \
for ( j = 0; j < len; j++ ) { \
data[i + j] = (T)v[j]; \
} \
} \
} while (0)

#define READ_COMPRESSED_DATA(T, SwapFunc) \
do { \
if ( mat->byteswap ) { \
for ( i = 0; i < len; i++ ) { \
InflateData(mat, z, &v, data_size); \
data[i] = (T)SwapFunc(&v); \
const size_t block_size = READ_BLOCK_SIZE/data_size; \
if ( len <= block_size ) { \
InflateData(mat, z, v, len*data_size); \
for ( i = 0; i < len; i++ ) { \
data[i] = (T)SwapFunc(&v[i]); \
} \
} else { \
mat_uint32_t j; \
len -= block_size; \
for ( i = 0; i < len; i += block_size ) { \
InflateData(mat, z, v, block_size*data_size); \
for ( j = 0; j < block_size; j++ ) { \
data[i + j] = (T)SwapFunc(&v[j]); \
} \
} \
len -= (i - block_size); \
InflateData(mat, z, v, len*data_size); \
for ( j = 0; j < len; j++ ) { \
data[i + j] = (T)SwapFunc(&v[j]); \
} \
} \
} else { \
READ_COMPRESSED_DATA_NOSWAP(T); \
Expand Down
60 changes: 30 additions & 30 deletions src/read_data_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ READ_TYPE_DOUBLE_DATA(mat_t *mat, READ_TYPE *data, size_t len)
}
}
#else
size_t i, j;
size_t i;
const size_t data_size = sizeof(double);
double v[READ_BLOCK_SIZE/sizeof(double)];
READ_DATA(READ_TYPE, Mat_doubleSwap);
Expand All @@ -73,7 +73,7 @@ READ_TYPE_SINGLE_DATA(mat_t *mat, READ_TYPE *data, size_t len)
}
}
#else
size_t i, j;
size_t i;
const size_t data_size = sizeof(float);
float v[READ_BLOCK_SIZE/sizeof(float)];
READ_DATA(READ_TYPE, Mat_floatSwap);
Expand All @@ -94,7 +94,7 @@ READ_TYPE_INT32_DATA(mat_t *mat, READ_TYPE *data, size_t len)
}
}
#else
size_t i, j;
size_t i;
const size_t data_size = sizeof(mat_int32_t);
mat_int32_t v[READ_BLOCK_SIZE/sizeof(mat_int32_t)];
READ_DATA(READ_TYPE, Mat_int32Swap);
Expand All @@ -115,7 +115,7 @@ READ_TYPE_UINT32_DATA(mat_t *mat, READ_TYPE *data, size_t len)
}
}
#else
size_t i, j;
size_t i;
const size_t data_size = sizeof(mat_uint32_t);
mat_uint32_t v[READ_BLOCK_SIZE/sizeof(mat_uint32_t)];
READ_DATA(READ_TYPE, Mat_uint32Swap);
Expand All @@ -136,7 +136,7 @@ READ_TYPE_INT16_DATA(mat_t *mat, READ_TYPE *data, size_t len)
}
}
#else
size_t i, j;
size_t i;
const size_t data_size = sizeof(mat_int16_t);
mat_int16_t v[READ_BLOCK_SIZE/sizeof(mat_int16_t)];
READ_DATA(READ_TYPE, Mat_int16Swap);
Expand All @@ -157,7 +157,7 @@ READ_TYPE_UINT16_DATA(mat_t *mat, READ_TYPE *data, size_t len)
}
}
#else
size_t i, j;
size_t i;
const size_t data_size = sizeof(mat_uint16_t);
mat_uint16_t v[READ_BLOCK_SIZE/sizeof(mat_uint16_t)];
READ_DATA(READ_TYPE, Mat_uint16Swap);
Expand All @@ -172,7 +172,7 @@ READ_TYPE_INT8_DATA(mat_t *mat, READ_TYPE *data, size_t len)
#if READ_TYPE_TYPE == READ_TYPE_INT8
bytesread += fread(data, sizeof(mat_int8_t), len, (FILE*)mat->fp);
#else
size_t i, j;
size_t i;
const size_t data_size = sizeof(mat_int8_t);
mat_int8_t v[READ_BLOCK_SIZE/sizeof(mat_int8_t)];
READ_DATA_NOSWAP(READ_TYPE);
Expand All @@ -187,7 +187,7 @@ READ_TYPE_UINT8_DATA(mat_t *mat, READ_TYPE *data, size_t len)
#if READ_TYPE_TYPE == READ_TYPE_UINT8
bytesread += fread(data, sizeof(mat_uint8_t), len, (FILE*)mat->fp);
#else
size_t i, j;
size_t i;
const size_t data_size = sizeof(mat_uint8_t);
mat_uint8_t v[READ_BLOCK_SIZE/sizeof(mat_uint8_t)];
READ_DATA_NOSWAP(READ_TYPE);
Expand All @@ -209,7 +209,7 @@ READ_TYPE_INT64_DATA(mat_t *mat, READ_TYPE *data, size_t len)
}
}
#else
size_t i, j;
size_t i;
const size_t data_size = sizeof(mat_int64_t);
mat_int64_t v[READ_BLOCK_SIZE/sizeof(mat_int64_t)];
READ_DATA(READ_TYPE, Mat_int64Swap);
Expand All @@ -232,7 +232,7 @@ READ_TYPE_UINT64_DATA(mat_t *mat, READ_TYPE *data, size_t len)
}
}
#else
size_t i, j;
size_t i;
const size_t data_size = sizeof(mat_uint64_t);
mat_uint64_t v[READ_BLOCK_SIZE/sizeof(mat_uint64_t)];
READ_DATA(READ_TYPE, Mat_uint64Swap);
Expand Down Expand Up @@ -349,9 +349,9 @@ READ_TYPE_DOUBLE_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len
}
}
#else
size_t i, j;
mat_uint32_t i;
const size_t data_size = sizeof(double);
double v;//[READ_BLOCK_SIZE/sizeof(double)];
double v[READ_BLOCK_SIZE/sizeof(double)];
READ_COMPRESSED_DATA(READ_TYPE, Mat_doubleSwap);
#endif
return bytesread;
Expand All @@ -370,9 +370,9 @@ READ_TYPE_SINGLE_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len
}
}
#else
size_t i, j;
mat_uint32_t i;
const size_t data_size = sizeof(float);
float v;//[READ_BLOCK_SIZE/sizeof(float)];
float v[READ_BLOCK_SIZE/sizeof(float)];
READ_COMPRESSED_DATA(READ_TYPE, Mat_floatSwap);
#endif
return bytesread;
Expand All @@ -392,9 +392,9 @@ READ_TYPE_INT64_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len)
}
}
#else
size_t i, j;
mat_uint32_t i;
const size_t data_size = sizeof(mat_int64_t);
mat_int64_t v;//[READ_BLOCK_SIZE/sizeof(mat_int64_t)];
mat_int64_t v[READ_BLOCK_SIZE/sizeof(mat_int64_t)];
READ_COMPRESSED_DATA(READ_TYPE, Mat_int64Swap);
#endif
return bytesread;
Expand All @@ -415,9 +415,9 @@ READ_TYPE_UINT64_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len
}
}
#else
size_t i, j;
mat_uint32_t i;
const size_t data_size = sizeof(mat_uint64_t);
mat_uint64_t v;//[READ_BLOCK_SIZE/sizeof(mat_uint64_t)];
mat_uint64_t v[READ_BLOCK_SIZE/sizeof(mat_uint64_t)];
READ_COMPRESSED_DATA(READ_TYPE, Mat_uint64Swap);
#endif
return bytesread;
Expand All @@ -437,9 +437,9 @@ READ_TYPE_INT32_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len)
}
}
#else
size_t i, j;
mat_uint32_t i;
const size_t data_size = sizeof(mat_int32_t);
mat_int32_t v;//[READ_BLOCK_SIZE/sizeof(mat_int32_t)];
mat_int32_t v[READ_BLOCK_SIZE/sizeof(mat_int32_t)];
READ_COMPRESSED_DATA(READ_TYPE, Mat_int32Swap);
#endif
return bytesread;
Expand All @@ -458,9 +458,9 @@ READ_TYPE_UINT32_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len
}
}
#else
size_t i, j;
mat_uint32_t i;
const size_t data_size = sizeof(mat_uint32_t);
mat_uint32_t v;//[READ_BLOCK_SIZE/sizeof(mat_uint32_t)];
mat_uint32_t v[READ_BLOCK_SIZE/sizeof(mat_uint32_t)];
READ_COMPRESSED_DATA(READ_TYPE, Mat_uint32Swap);
#endif
return bytesread;
Expand All @@ -479,9 +479,9 @@ READ_TYPE_INT16_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len)
}
}
#else
size_t i, j;
mat_uint32_t i;
const size_t data_size = sizeof(mat_int16_t);
mat_int16_t v;//[READ_BLOCK_SIZE/sizeof(mat_int16_t)];
mat_int16_t v[READ_BLOCK_SIZE/sizeof(mat_int16_t)];
READ_COMPRESSED_DATA(READ_TYPE, Mat_int16Swap);
#endif
return bytesread;
Expand All @@ -500,9 +500,9 @@ READ_TYPE_UINT16_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len
}
}
#else
size_t i, j;
mat_uint32_t i;
const size_t data_size = sizeof(mat_uint16_t);
mat_uint16_t v;//[READ_BLOCK_SIZE/sizeof(mat_uint16_t)];
mat_uint16_t v[READ_BLOCK_SIZE/sizeof(mat_uint16_t)];
READ_COMPRESSED_DATA(READ_TYPE, Mat_uint16Swap);
#endif
return bytesread;
Expand All @@ -515,9 +515,9 @@ READ_TYPE_INT8_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len)
#if READ_TYPE_TYPE == READ_TYPE_INT8
bytesread = InflateData(mat, z, data, len*sizeof(mat_int8_t));
#else
size_t i, j;
mat_uint32_t i;
const size_t data_size = sizeof(mat_int8_t);
mat_int8_t v;//[READ_BLOCK_SIZE/sizeof(mat_int8_t)];
mat_int8_t v[READ_BLOCK_SIZE/sizeof(mat_int8_t)];
READ_COMPRESSED_DATA_NOSWAP(READ_TYPE);
#endif
return bytesread;
Expand All @@ -530,9 +530,9 @@ READ_TYPE_UINT8_DATA(mat_t *mat, z_streamp z, READ_TYPE *data, mat_uint32_t len)
#if READ_TYPE_TYPE == READ_TYPE_UINT8
bytesread = InflateData(mat, z, data, len*sizeof(mat_uint8_t));
#else
size_t i, j;
mat_uint32_t i;
const size_t data_size = sizeof(mat_uint8_t);
mat_uint8_t v;//[READ_BLOCK_SIZE/sizeof(mat_uint8_t)];
mat_uint8_t v[READ_BLOCK_SIZE/sizeof(mat_uint8_t)];
READ_COMPRESSED_DATA_NOSWAP(READ_TYPE);
#endif
return bytesread;
Expand Down

0 comments on commit e032c79

Please sign in to comment.