Skip to content

Commit

Permalink
Create a masked record array if any of the columns are masked
Browse files Browse the repository at this point in the history
  • Loading branch information
timj committed Sep 30, 2024
1 parent 178fd9d commit ce3fe74
Showing 1 changed file with 9 additions and 2 deletions.
11 changes: 9 additions & 2 deletions python/lsst/daf/butler/formatters/parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,21 +231,28 @@ def arrow_to_numpy(arrow_table: pa.Table) -> np.ndarray:
-------
array : `numpy.ndarray` (N,)
Numpy array table with N rows and the same column names
as the input arrow table.
as the input arrow table. Will be masked records if any values
in the table are null.
"""
import numpy as np

numpy_dict = arrow_to_numpy_dict(arrow_table)

has_mask = False
dtype = []
for name, col in numpy_dict.items():
if len(shape := numpy_dict[name].shape) <= 1:
dtype.append((name, col.dtype))
else:
dtype.append((name, (col.dtype, shape[1:])))

array = np.rec.fromarrays(numpy_dict.values(), dtype=dtype)
if not has_mask and isinstance(col, np.ma.MaskedArray):
has_mask = True

if has_mask:
array = np.ma.mrecords.fromarrays(numpy_dict.values(), dtype=dtype)
else:
array = np.rec.fromarrays(numpy_dict.values(), dtype=dtype)
return array


Expand Down

0 comments on commit ce3fe74

Please sign in to comment.