From 4bc8d395dc3cb187a251f1c5a9fda8b9cd81f850 Mon Sep 17 00:00:00 2001 From: Aaron Brewster Date: Tue, 28 Feb 2023 16:43:52 -0800 Subject: [PATCH 1/8] NXmx: handle multidimensional arrays Data in NeXus can be 3 or 4 dimensional. 3D: Nimages by slow by fast 4D: Nimages by Nmodules by slow fast Slice image_size and reshape the raw_data in these cases --- newsfragments/XXX.bugfix | 1 + src/dxtbx/nexus/__init__.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 newsfragments/XXX.bugfix diff --git a/newsfragments/XXX.bugfix b/newsfragments/XXX.bugfix new file mode 100644 index 000000000..b1374168c --- /dev/null +++ b/newsfragments/XXX.bugfix @@ -0,0 +1 @@ +NeXus: handle dataset with multiple modules stored as multidimensional arrays diff --git a/src/dxtbx/nexus/__init__.py b/src/dxtbx/nexus/__init__.py index ccd84a7ec..610039881 100644 --- a/src/dxtbx/nexus/__init__.py +++ b/src/dxtbx/nexus/__init__.py @@ -404,9 +404,14 @@ def equipment_component_key(dependency): origin -= nxdetector.beam_center_y.magnitude * pixel_size[1] * slow_axis # dxtbx requires image size in the order fast, slow - which is the reverse of what - # is stored in module.data_size - image_size = cast(Tuple[int, int], tuple(map(int, module.data_size[::-1]))) - assert len(image_size) == 2 + # is stored in module.data_size. Additionally, data_size can have more than 2 + # dimensions, for multi-module detectors. So take the last two dimensions and reverse + # them. Examples: + # [1,2,3,4][-1:-3:-1] --> [4, 3] + # [1,2,3][-1:-3:-1] --> [3, 2] + # [1,2][-1:-3:-1] --> [2, 1] + image_size = cast(Tuple[int, int], tuple(map(int, module.data_size[-1:-3:-1]))) + assert len(image_size) == 2, image_size underload = ( float(nxdetector.underload_value) if nxdetector.underload_value is not None @@ -562,5 +567,8 @@ def get_raw_data( data_as_flex = _dataset_as_flex( sliced_outer, tuple(module_slices), bit_depth=bit_depth ) + data_as_flex.reshape( + flex.grid(data_as_flex.all()[-2:]) + ) # handle 3 or 4 dimension arrays all_data.append(data_as_flex) return tuple(all_data) From fa0ca7ff4f175e57ee79a92e41852f857fc3b7f8 Mon Sep 17 00:00:00 2001 From: DiamondLightSource-build-server Date: Wed, 1 Mar 2023 00:49:37 +0000 Subject: [PATCH 2/8] Rename newsfragments/XXX.bugfix to newsfragments/612.bugfix --- newsfragments/{XXX.bugfix => 612.bugfix} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{XXX.bugfix => 612.bugfix} (100%) diff --git a/newsfragments/XXX.bugfix b/newsfragments/612.bugfix similarity index 100% rename from newsfragments/XXX.bugfix rename to newsfragments/612.bugfix From 050c6af406a0e043b3ab243a967f94430012714c Mon Sep 17 00:00:00 2001 From: Aaron Brewster Date: Thu, 2 Mar 2023 17:38:53 -0800 Subject: [PATCH 3/8] Handle 3 or 4 dimension data in the mask --- src/dxtbx/nexus/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/dxtbx/nexus/__init__.py b/src/dxtbx/nexus/__init__.py index 610039881..8aa760153 100644 --- a/src/dxtbx/nexus/__init__.py +++ b/src/dxtbx/nexus/__init__.py @@ -480,13 +480,17 @@ def get_static_mask(nxdetector: nxmx.NXdetector) -> tuple[flex.bool, ...] | None pixel_mask = nxdetector.pixel_mask except KeyError: return None - if pixel_mask is None or not pixel_mask.size or pixel_mask.ndim != 2: + if pixel_mask is None or not pixel_mask.size: return None all_slices = get_detector_module_slices(nxdetector) - return tuple( - flumpy.from_numpy(np.ascontiguousarray(pixel_mask[slices])) == 0 - for slices in all_slices - ) + all_mask_slices = [] + for slices in all_slices: + mask_slice = flumpy.from_numpy(np.ascontiguousarray(pixel_mask[slices])) == 0 + mask_slice.reshape( + flex.grid(mask_slice.all()[-2:]) + ) # handle 3 or 4 dimension arrays + all_mask_slices.append(mask_slice) + return tuple(all_mask_slices) def _dataset_as_flex( From f091a12ea00c41434d5b6f483cd92a2e2c10f7a8 Mon Sep 17 00:00:00 2001 From: Richard Gildea Date: Thu, 4 May 2023 14:30:39 +0100 Subject: [PATCH 4/8] More explicit extraction of image_size Co-authored-by: Nicholas Devenish --- src/dxtbx/nexus/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/dxtbx/nexus/__init__.py b/src/dxtbx/nexus/__init__.py index 8aa760153..91f821f81 100644 --- a/src/dxtbx/nexus/__init__.py +++ b/src/dxtbx/nexus/__init__.py @@ -407,11 +407,10 @@ def equipment_component_key(dependency): # is stored in module.data_size. Additionally, data_size can have more than 2 # dimensions, for multi-module detectors. So take the last two dimensions and reverse # them. Examples: - # [1,2,3,4][-1:-3:-1] --> [4, 3] - # [1,2,3][-1:-3:-1] --> [3, 2] - # [1,2][-1:-3:-1] --> [2, 1] - image_size = cast(Tuple[int, int], tuple(map(int, module.data_size[-1:-3:-1]))) - assert len(image_size) == 2, image_size + # [1,2,3,4] --> (4, 3) + # [1,2,3] --> (3, 2) + # [1,2] --> (2, 1) + image_size = (int(module.data_size[-1]), int(module.data_size[-2])) underload = ( float(nxdetector.underload_value) if nxdetector.underload_value is not None From 0a600188bd31a8f50c399034d1c1674043acfa94 Mon Sep 17 00:00:00 2001 From: Richard Gildea Date: Thu, 4 May 2023 14:33:21 +0100 Subject: [PATCH 5/8] Remove unused imports --- src/dxtbx/nexus/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dxtbx/nexus/__init__.py b/src/dxtbx/nexus/__init__.py index 91f821f81..1fa32ab68 100644 --- a/src/dxtbx/nexus/__init__.py +++ b/src/dxtbx/nexus/__init__.py @@ -2,7 +2,7 @@ import itertools import logging -from typing import Literal, Optional, Tuple, cast +from typing import Literal, Optional import h5py import numpy as np From fe9308ee3dc9f2cfcc500c8a46567a385f1d13b2 Mon Sep 17 00:00:00 2001 From: Richard Gildea Date: Thu, 4 May 2023 15:18:33 +0100 Subject: [PATCH 6/8] Test case for 3D images --- tests/nexus/test_build_dxtbx_models.py | 103 +++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/tests/nexus/test_build_dxtbx_models.py b/tests/nexus/test_build_dxtbx_models.py index 8b3b684ac..82c33ca86 100644 --- a/tests/nexus/test_build_dxtbx_models.py +++ b/tests/nexus/test_build_dxtbx_models.py @@ -306,6 +306,109 @@ def test_get_dxtbx_detector_beam_center_fallback(nxmx_example): ) +@pytest.fixture +def detector_with_multiple_modules(): + + with h5py.File(" ", "w", **pytest.h5_in_memory) as f: + + detector = f.create_group("/entry/instrument/detector") + detector.attrs["NX_class"] = "NXdetector" + detector["beam_center_x"] = 2079.79727597266 + detector["beam_center_y"] = 2225.38773853771 + detector["count_time"] = 0.00285260857097799 + detector["depends_on"] = "/entry/instrument/detector/transformations/det_z" + detector["description"] = "Eiger 16M" + detector["distance"] = 0.237015940260233 + detector.create_dataset("data", data=np.zeros((100, 100))) + detector["sensor_material"] = "Silicon" + detector["sensor_thickness"] = 0.00045 + detector["sensor_thickness"].attrs["units"] = b"m" + detector["x_pixel_size"] = 7.5e-05 + detector["y_pixel_size"] = 7.5e-05 + detector["underload_value"] = 0 + detector["saturation_value"] = 9266 + detector["frame_time"] = 0.1 + detector["frame_time"].attrs["units"] = "s" + detector["bit_depth_readout"] = np.array(32) + mask = np.zeros((2, 100, 200), dtype="i8") + detector.create_dataset("pixel_mask", data=mask) + + detector_transformations = detector.create_group("transformations") + detector_transformations.attrs["NX_class"] = "NXtransformations" + det_z = detector_transformations.create_dataset("det_z", data=np.array([289.3])) + det_z.attrs["depends_on"] = b"." + det_z.attrs["transformation_type"] = b"translation" + det_z.attrs["units"] = b"mm" + det_z.attrs["vector"] = np.array([0.0, 0.0, 1.0]) + + def make_module(name, depends_on, data_origin, fast_direction, slow_direction): + module = detector.create_group(name) + module.attrs["NX_class"] = "NXdetector_module" + module.create_dataset("data_size", data=np.array([1, 100, 200])) + module.create_dataset("data_origin", data=np.array(data_origin)) + fast = module.create_dataset("fast_pixel_direction", data=0.075) + fast.attrs["transformation_type"] = "translation" + fast.attrs["depends_on"] = depends_on + fast.attrs["vector"] = np.array(fast_direction) + fast.attrs["units"] = "mm" + slow = module.create_dataset("slow_pixel_direction", data=0.075) + slow.attrs["transformation_type"] = "translation" + slow.attrs["depends_on"] = depends_on + slow.attrs["vector"] = np.array(slow_direction) + slow.attrs["units"] = "mm" + + make_module( + name="m0", + depends_on="/entry/instrument/detector/transformations/det_z", + data_origin=[0, 0, 0], + fast_direction=[-0.999998, -0.001781, 0], + slow_direction=[-0.001781, 0.999998, 0], + ) + make_module( + name="m1", + depends_on="/entry/instrument/detector/transformations/det_z", + data_origin=[1, 0, 0], + fast_direction=[-0.999998, -0.001781, 0], + slow_direction=[-0.001781, 0.999998, 0], + ) + + nxdata = f.create_group("/entry/data") + nxdata.attrs["NX_class"] = "NXdata" + nxdata.create_dataset( + "data", + data=np.array( + [np.full((2, 100, 200), i, dtype=np.int32) for i in range(3)] + ), + ) + nxdata.attrs["signal"] = "/entry/data/data" + + yield f + + +def test_get_dxtbx_detector_with_multiple_modules(detector_with_multiple_modules): + det = nxmx.NXdetector(detector_with_multiple_modules["/entry/instrument/detector"]) + wavelength = 1 + + detector = dxtbx.nexus.get_dxtbx_detector(det, wavelength) + assert len(detector) == 2 + for panel in detector: + assert panel.get_image_size() == (200, 100) + + nxdata = nxmx.NXdata(detector_with_multiple_modules["/entry/data"]) + for i in range(3): + raw_data = dxtbx.nexus.get_raw_data(nxdata, det, i) + assert len(raw_data) == 2 + for module_data in raw_data: + assert module_data.all() == (100, 200) + assert module_data.all_eq(i) + + mask = dxtbx.nexus.get_static_mask(det) + assert len(mask) == 2 + for module_mask in mask: + assert isinstance(module_mask, flex.bool) + assert module_mask.all() == (100, 200) + + @pytest.fixture def detector_with_two_theta(): with h5py.File(" ", "w", **pytest.h5_in_memory) as f: From 8fa0dee83f10ab950a233fef5fba4beaf7946301 Mon Sep 17 00:00:00 2001 From: Richard Gildea Date: Tue, 6 Jun 2023 13:42:37 +0100 Subject: [PATCH 7/8] Remove example --- src/dxtbx/nexus/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/dxtbx/nexus/__init__.py b/src/dxtbx/nexus/__init__.py index 1fa32ab68..6fc0d20c6 100644 --- a/src/dxtbx/nexus/__init__.py +++ b/src/dxtbx/nexus/__init__.py @@ -407,7 +407,6 @@ def equipment_component_key(dependency): # is stored in module.data_size. Additionally, data_size can have more than 2 # dimensions, for multi-module detectors. So take the last two dimensions and reverse # them. Examples: - # [1,2,3,4] --> (4, 3) # [1,2,3] --> (3, 2) # [1,2] --> (2, 1) image_size = (int(module.data_size[-1]), int(module.data_size[-2])) From 969c87d92b093936d4d93f48bd41ee6bd4ca6b42 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Fri, 11 Aug 2023 09:49:01 +0100 Subject: [PATCH 8/8] Update 612.bugfix --- newsfragments/612.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/612.bugfix b/newsfragments/612.bugfix index b1374168c..6f4e30c86 100644 --- a/newsfragments/612.bugfix +++ b/newsfragments/612.bugfix @@ -1 +1 @@ -NeXus: handle dataset with multiple modules stored as multidimensional arrays +NXmx files with multidimensional arrays (images, modules, or both) are now handled.