From 029416e503c8794f285ccdb008cbb77a94f4024c Mon Sep 17 00:00:00 2001
From: Richard Gildea <rjgildea@users.noreply.github.com>
Date: Tue, 11 Apr 2023 09:58:15 +0100
Subject: [PATCH 01/45] flumpy: add support for flex.miller_index (#618)

- Add flumpy.to_numpy support for flex.miller_index
- Add flumpy.miller_index_from_numpy

Co-authored-by: Nicholas Devenish <ndevenish@gmail.com>
---
 newsfragments/618.feature        |  1 +
 src/dxtbx/boost_python/flumpy.cc | 61 ++++++++++++++++++++++++++++----
 tests/test_flumpy.py             | 39 ++++++++++++++++++--
 3 files changed, 93 insertions(+), 8 deletions(-)
 create mode 100644 newsfragments/618.feature

diff --git a/newsfragments/618.feature b/newsfragments/618.feature
new file mode 100644
index 000000000..ebb9e858e
--- /dev/null
+++ b/newsfragments/618.feature
@@ -0,0 +1 @@
+``flumpy``: Add support for conversion of ``flex.miller_index`` arrays to/from numpy.
diff --git a/src/dxtbx/boost_python/flumpy.cc b/src/dxtbx/boost_python/flumpy.cc
index a289f4acd..12e5ebf94 100644
--- a/src/dxtbx/boost_python/flumpy.cc
+++ b/src/dxtbx/boost_python/flumpy.cc
@@ -24,6 +24,7 @@
 #include <scitbx/vec2.h>
 #include <scitbx/vec3.h>
 #include <scitbx/mat3.h>
+#include <cctbx/miller.h>
 
 using boost::optional;
 using std::cout;
@@ -43,7 +44,8 @@ using grid = af::versa<T, af::flex_grid<>>;
     grid<long>, grid<int8_t>, grid<int16_t>, grid<int32_t>, grid<int64_t>,            \
     grid<float>, grid<double>, grid<scitbx::vec3<double>>, grid<scitbx::vec3<int>>,   \
     grid<scitbx::vec2<double>>, grid<scitbx::mat3<double>>,                           \
-    grid<af::tiny<std::size_t, 2>>, grid<std::complex<double>>
+    grid<af::tiny<std::size_t, 2>>, grid<std::complex<double>>,                       \
+    grid<cctbx::miller::index<int>>
 // Unwrapped, and possibly doesn't make sense:
 // void wrap_flex_std_string(); - differently sized per element
 // void wrap_flex_sym_mat3_double(); - nonlinear memory layout
@@ -99,6 +101,31 @@ py::buffer_info get_buffer_specific(grid<scitbx::vec3<T>> flex) {
                          strides);
 }
 
+template <typename T>
+py::buffer_info get_buffer_specific(grid<cctbx::miller::index<T>> flex) {
+  std::vector<size_t> dim_sizes;
+  for (auto size : flex.accessor().all()) {
+    dim_sizes.push_back(size);
+  }
+  dim_sizes.push_back(3);
+
+  std::vector<size_t> strides;
+  for (int i = 0; i < dim_sizes.size(); ++i) {
+    auto stride = sizeof(T);
+    for (int j = i + 1; j < dim_sizes.size(); ++j) {
+      stride *= dim_sizes[j];
+    }
+    strides.push_back(stride);
+  }
+
+  return py::buffer_info(&flex.front(),
+                         sizeof(T),
+                         py::format_descriptor<T>::format(),
+                         dim_sizes.size(),
+                         dim_sizes,
+                         strides);
+}
+
 template <typename T>
 py::buffer_info get_buffer_specific(grid<scitbx::vec2<T>> flex) {
   std::vector<size_t> dim_sizes;
@@ -486,6 +513,11 @@ py::object from_numpy(py::object array) {
 /// More structured arrays need to be explicitly requested
 template <template <class> class VecType>
 py::object vec_from_numpy(py::array np_array) {
+  // Check that this array is contiguous
+  if (!is_array_c_contiguous(np_array)) {
+    throw ERR_NON_CONTIGUOUS;
+  }
+
   static_assert(VecType<int>::fixed_size == 2 || VecType<int>::fixed_size == 3,
                 "Only vec2/vec3 supported");
   // Only accept arrays whose last dimension is the size of this object
@@ -495,6 +527,11 @@ py::object vec_from_numpy(py::array np_array) {
   }
 
   auto dtype = np_array.attr("dtype").attr("char").cast<char>();
+  // If we are on windows, where int and long are the same size - prefer 'i'.
+  // This is because flex arrays are only bound to 'int' on this platform.
+  if (dtype == 'l' && (sizeof(long) == sizeof(int))) {
+    dtype = 'i';
+  }
 
   std::string accepted_types = VecType<int>::fixed_size == 2 ? "dQ" : "di";
   if (accepted_types.find(dtype) == std::string::npos) {
@@ -526,11 +563,6 @@ py::object vec_from_numpy(py::array np_array) {
 /// Decide which sized vector we want to convert to, and hand off to the
 /// specialization
 py::object vecs_from_numpy(py::array np_array) {
-  // Check that this array is contiguous
-  if (!is_array_c_contiguous(np_array)) {
-    throw ERR_NON_CONTIGUOUS;
-  }
-
   if (np_array.shape(np_array.ndim() - 1) == 3) {
     return vec_from_numpy<scitbx::vec3>(np_array);
   } else if (np_array.shape(np_array.ndim() - 1) == 2) {
@@ -540,6 +572,19 @@ py::object vecs_from_numpy(py::array np_array) {
     "Invalid input array: last numpy dimension must be 2 or 3 to convert to vector");
 }
 
+/// Create a versa<miller_index> from numpy
+py::object miller_index_from_numpy(py::array np_array) {
+  // miller_index is ONLY bound as 'int', but on windows we accept the same-sized long
+  auto dtype = np_array.attr("dtype").attr("char").cast<char>();
+  std::string accepted_types = sizeof(long) == sizeof(int) ? "il" : "i";
+  if (accepted_types.find(dtype) == std::string::npos) {
+    throw std::invalid_argument(
+      std::string("miller_index only supports int32 or intc types - cannot convert '")
+      + std::to_string(dtype) + "'");
+  }
+  return vec_from_numpy<cctbx::miller::index>(np_array);
+}
+
 py::object mat3_from_numpy(py::array np_array) {
   // Check that this array is contiguous
   if (!is_array_c_contiguous(np_array)) {
@@ -586,8 +631,12 @@ PYBIND11_MODULE(dxtbx_flumpy, m) {
         &vecs_from_numpy,
         "Convert a numpy object to a flex.vec2 or .vec3, depending on input array");
   m.def("mat3_from_numpy", &mat3_from_numpy, "Convert a numpy object to a flex.mat3");
+  m.def("miller_index_from_numpy",
+        &miller_index_from_numpy,
+        "Convert a numpy object to a flex.miller_index");
 
   // Make sure that we have imported flex - cannot do boost::python conversions
   // otherwise
   pybind11::module::import("scitbx.array_family.flex");
+  pybind11::module::import("cctbx.array_family.flex");
 }
diff --git a/tests/test_flumpy.py b/tests/test_flumpy.py
index 9598a193f..cd09e3c09 100644
--- a/tests/test_flumpy.py
+++ b/tests/test_flumpy.py
@@ -5,6 +5,7 @@
 import numpy as np
 import pytest
 
+import cctbx.array_family.flex as cctbx_flex
 import scitbx.array_family.flex as flex
 
 import dxtbx.flumpy as flumpy
@@ -28,6 +29,7 @@
     "vec3_int": "i",
     "vec2_double": "d",
     "tiny_size_t_2": "Q",
+    "miller_index": "i",
 }
 
 # On POSIX platforms, long and q are the same and numpy tends to give q's
@@ -174,7 +176,9 @@ def test_reverse_bool():
     assert fo.count(True) == npo.sum()
 
 
-@pytest.mark.parametrize("flex_vec", [flex.vec3_double, flex.vec3_int])
+@pytest.mark.parametrize(
+    "flex_vec", [flex.vec3_double, flex.vec3_int, cctbx_flex.miller_index]
+)
 def test_vec3(flex_vec):
     basic_vector = [(i, i * 2, i * 3) for i in range(10)]
     fo = flex_vec(basic_vector)
@@ -186,7 +190,9 @@ def test_vec3(flex_vec):
     assert (as_np == fo).all()
 
 
-@pytest.mark.parametrize("flex_vec", [flex.vec3_double, flex.vec3_int])
+@pytest.mark.parametrize(
+    "flex_vec", [flex.vec3_double, flex.vec3_int, cctbx_flex.miller_index]
+)
 def test_reverse_vec3(flex_vec):
     dtype = lookup_flex_type_to_numpy[flex_vec.__name__]
     no = np.zeros((5, 3), dtype=dtype)
@@ -199,6 +205,35 @@ def test_reverse_vec3(flex_vec):
         flumpy.vec_from_numpy(no.reshape((1, 15)))
 
 
+@pytest.mark.parametrize("dtype", [np.int32, np.intc])
+def test_reverse_vec3_dtype(dtype):
+    no = np.zeros((5, 3), dtype=dtype)
+    fo = flumpy.vec_from_numpy(no)
+    assert fo.all() == (5,)
+    fo[0] = (1, 2, 3)
+    assert (no[0] == (1, 2, 3)).all()
+
+
+@pytest.mark.parametrize("dtype", [np.int32, np.intc, int])
+def test_reverse_miller_index(dtype):
+    hkl = np.array([(1, 0, 0), (0, 1, 0), (0, 0, 1)], dtype=dtype)
+    if dtype is int and np.dtype("l").itemsize != np.dtype("i").itemsize:
+        with pytest.raises(ValueError):
+            flumpy.miller_index_from_numpy(hkl)
+    else:
+        mi = flumpy.miller_index_from_numpy(hkl)
+        assert isinstance(mi, cctbx_flex.miller_index)
+        assert len(mi) == 3
+        for i in range(len(mi)):
+            assert (mi[i] == hkl[i]).all()
+
+        with pytest.raises(ValueError):
+            flumpy.miller_index_from_numpy(hkl.reshape((1, 9)))
+
+        with pytest.raises(ValueError):
+            flumpy.miller_index_from_numpy(hkl.astype(float))
+
+
 @pytest.mark.parametrize("flex_vec", [flex.vec2_double, flex.tiny_size_t_2])
 def test_vec2(flex_vec):
     basic_vector = [(i, i * 2) for i in range(10)]

From 292a8c83f66759963af383ac3af66cc73044068a Mon Sep 17 00:00:00 2001
From: Nicholas Devenish <ndevenish@gmail.com>
Date: Tue, 11 Apr 2023 12:27:53 +0100
Subject: [PATCH 02/45] MNT: Hide system/cctbx_project warnings when building
 dxtbx

---
 SConscript | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/SConscript b/SConscript
index 1ac9eb1a5..5b69aa630 100644
--- a/SConscript
+++ b/SConscript
@@ -2,6 +2,7 @@ import sys
 import os
 import libtbx.load_env
 from libtbx.env_config import get_boost_library_with_python_version
+from pathlib import Path
 
 try:
     # Check if we have a "modernized" pycbf
@@ -31,6 +32,7 @@ env_etc.dxtbx_common_includes = [
 
 Import("env_no_includes_boost_python_ext")
 env = env_no_includes_boost_python_ext.Clone()
+
 if libtbx.env.build_options.use_conda:
     boost_python = get_boost_library_with_python_version(
         "boost_python", env_etc.conda_libpath
@@ -51,7 +53,6 @@ env_etc.dxtbx_hdf5_libs = ["hdf5"]
 env_etc.dxtbx_hdf5_lib_paths = []
 
 if sys.platform == "win32" and env_etc.compiler == "win32_cl":
-
     if libtbx.env.build_options.use_conda:
         env_etc.dxtbx_hdf5_libs = ["hdf5"]
         env_etc.cppdefines = {"H5_BUILT_AS_DYNAMIC_LIB": 1}
@@ -116,6 +117,13 @@ else:
 if not env_etc.no_boost_python and hasattr(env_etc, "boost_adaptbx_include"):
     Import("env_no_includes_boost_python_ext")
     env = env_no_includes_boost_python_ext.Clone()
+
+    # Don't surface warnings from system or cctbx_project headers
+    system_includes = [x for x in env_etc.conda_cpppath if x]
+    system_includes.append(str(Path(env_etc.scitbx_dist).parent))
+    env.Append(CXXFLAGS=[f"-isystem{x}" for x in system_includes])
+    env.Append(SHCXXFLAGS=[f"-isystem{x}" for x in system_includes])
+
     env_etc.enable_more_warnings(env=env)
     env_etc.include_registry.append(
         env=env,

From b70ad250c88f6ab826dafab4ddbee9712f41a6cb Mon Sep 17 00:00:00 2001
From: DiamondLightSource-build-server
 <DiamondLightSource-build-server@users.noreply.github.com>
Date: Wed, 12 Apr 2023 15:05:38 +0100
Subject: [PATCH 03/45] dxtbx 3.14.0 Changelog

    towncrier --name=dxtbx --version='3.14.0'
---
 CHANGELOG.rst             | 22 ++++++++++++++++++++++
 newsfragments/604.misc    |  1 -
 newsfragments/607.bugfix  |  1 -
 newsfragments/608.misc    |  1 -
 newsfragments/609.misc    |  1 -
 newsfragments/610.bugfix  |  1 -
 newsfragments/611.misc    |  1 -
 newsfragments/614.misc    |  1 -
 newsfragments/618.feature |  1 -
 9 files changed, 22 insertions(+), 8 deletions(-)
 delete mode 100644 newsfragments/604.misc
 delete mode 100644 newsfragments/607.bugfix
 delete mode 100644 newsfragments/608.misc
 delete mode 100644 newsfragments/609.misc
 delete mode 100644 newsfragments/610.bugfix
 delete mode 100644 newsfragments/611.misc
 delete mode 100644 newsfragments/614.misc
 delete mode 100644 newsfragments/618.feature

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 4aefadce2..734063096 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,3 +1,25 @@
+dxtbx 3.14.0 (2023-04-12)
+=========================
+
+Features
+--------
+
+- ``flumpy``: Add support for conversion of ``flex.miller_index`` arrays to/from numpy. (`#618 <https://github.com/cctbx/dxtbx/issues/618>`_)
+
+
+Bugfixes
+--------
+
+- Flumpy: Prefer returning ``flex.int`` instead of ``flex.long`` when they are the same size. This solves ambiguous behaviour when reading images on Windows platforms. (`#607 <https://github.com/cctbx/dxtbx/issues/607>`_)
+- ``dxtbx.plot_detector_models``: Fix display of multiple single-panel detector models. (`#610 <https://github.com/cctbx/dxtbx/issues/610>`_)
+
+
+Misc
+----
+
+- `#604 <https://github.com/cctbx/dxtbx/issues/604>`_, `#608 <https://github.com/cctbx/dxtbx/issues/608>`_, `#609 <https://github.com/cctbx/dxtbx/issues/609>`_, `#611 <https://github.com/cctbx/dxtbx/issues/611>`_, `#614 <https://github.com/cctbx/dxtbx/issues/614>`_
+
+
 dxtbx 3.13.0 (2023-01-26)
 =========================
 
diff --git a/newsfragments/604.misc b/newsfragments/604.misc
deleted file mode 100644
index 42aff4a48..000000000
--- a/newsfragments/604.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update base prebuilt CCTBX, enable windows builds.
diff --git a/newsfragments/607.bugfix b/newsfragments/607.bugfix
deleted file mode 100644
index 6fa543542..000000000
--- a/newsfragments/607.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Flumpy: Prefer to return ``flex.int`` instead of ``flex.long`` when these are the same size. This solves ambiguous behaviour when reading images on Windows platforms.
diff --git a/newsfragments/608.misc b/newsfragments/608.misc
deleted file mode 100644
index d28139468..000000000
--- a/newsfragments/608.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix NXmx tests attempting to read locked file, on Windows.
diff --git a/newsfragments/609.misc b/newsfragments/609.misc
deleted file mode 100644
index 3358d8186..000000000
--- a/newsfragments/609.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix case from #607 where we didn't properly handle casting uint with data loss.
diff --git a/newsfragments/610.bugfix b/newsfragments/610.bugfix
deleted file mode 100644
index 30359a78c..000000000
--- a/newsfragments/610.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-``dxtbx.plot_detector_models``: fix display of multiple single-panel detector models
diff --git a/newsfragments/611.misc b/newsfragments/611.misc
deleted file mode 100644
index 7044a3bdd..000000000
--- a/newsfragments/611.misc
+++ /dev/null
@@ -1 +0,0 @@
-Ignore deprecated ``pkg_resources.declare_namespace`` warnings in pytest
diff --git a/newsfragments/614.misc b/newsfragments/614.misc
deleted file mode 100644
index 3cb1f6854..000000000
--- a/newsfragments/614.misc
+++ /dev/null
@@ -1 +0,0 @@
-Tidy up dxtbx_model_ext namespacing, so that there are no name clashes when merged.
diff --git a/newsfragments/618.feature b/newsfragments/618.feature
deleted file mode 100644
index ebb9e858e..000000000
--- a/newsfragments/618.feature
+++ /dev/null
@@ -1 +0,0 @@
-``flumpy``: Add support for conversion of ``flex.miller_index`` arrays to/from numpy.

From b864c2390adfbe53ac83368e1cf019134f3eafba Mon Sep 17 00:00:00 2001
From: DiamondLightSource-build-server
 <DiamondLightSource-build-server@users.noreply.github.com>
Date: Wed, 12 Apr 2023 15:05:39 +0100
Subject: [PATCH 04/45] =?UTF-8?q?Bump=20version:=203.14.dev=20=E2=86=92=20?=
 =?UTF-8?q?3.15.dev?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 setup.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 8196f0b31..d3de94e98 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.14.dev
+current_version = 3.15.dev
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<release>[a-z]+)?(?P<patch>\d+)?
diff --git a/setup.py b/setup.py
index 7686d4819..dd44af70c 100644
--- a/setup.py
+++ b/setup.py
@@ -8,7 +8,7 @@
 
 # Static version number which is updated by bump2version
 # Do not change this manually - use 'bump2version <major/minor/patch/release>'
-__version_tag__ = "3.14.dev"
+__version_tag__ = "3.15.dev"
 
 setup_kwargs = {
     "name": "dxtbx",

From af7cf7de7137b9a46d3e47cfc40386919650050b Mon Sep 17 00:00:00 2001
From: David McDonagh <60879630+toastisme@users.noreply.github.com>
Date: Fri, 21 Apr 2023 15:46:10 +0100
Subject: [PATCH 05/45] Add array family to install env (#625)

* Add array_family to install environment.

---------

Co-authored-by: DiamondLightSource-build-server <DiamondLightSource-build-server@users.noreply.github.com>
Co-authored-by: Nicholas Devenish <ndevenish@gmail.com>
---
 newsfragments/625.misc | 1 +
 setup.py               | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 newsfragments/625.misc

diff --git a/newsfragments/625.misc b/newsfragments/625.misc
new file mode 100644
index 000000000..ae1df20fe
--- /dev/null
+++ b/newsfragments/625.misc
@@ -0,0 +1 @@
+Add array_family to install environment
diff --git a/setup.py b/setup.py
index dd44af70c..bb8b3d8fd 100644
--- a/setup.py
+++ b/setup.py
@@ -26,7 +26,7 @@
     "package_dir": {"": "src"},
     "package_data": {
         "": ["*"],
-        "dxtbx": ["boost_python/*", "example/*", "py.typed"],
+        "dxtbx": ["array_family/*", "boost_python/*", "example/*", "py.typed"],
         "dxtbx.format": ["boost_python/*"],
         "dxtbx.masking": ["boost_python/*"],
         "dxtbx.model": ["boost_python/*"],

From 2554dc45521709f68e9f44841bae9ea6f7d2b885 Mon Sep 17 00:00:00 2001
From: Graeme Winter <graeme.winter@gmail.com>
Date: Thu, 4 May 2023 16:07:48 +0100
Subject: [PATCH 06/45] Add axis inversion (#623)

Add axis inversion

If we are inverting the rotation angles so that the direction of rotation is
positive we should also invert the rotation axis to ensure that this is
consistent.

Fixes #2398
---
 newsfragments/623.bugfix      | 2 ++
 src/dxtbx/model/goniometer.py | 8 ++++++++
 2 files changed, 10 insertions(+)
 create mode 100644 newsfragments/623.bugfix

diff --git a/newsfragments/623.bugfix b/newsfragments/623.bugfix
new file mode 100644
index 000000000..c836ee0eb
--- /dev/null
+++ b/newsfragments/623.bugfix
@@ -0,0 +1,2 @@
+FormatCBFFull: if rotation angles are decreasing not increasing we invert the angles, now invert rotation axis as well.
+
diff --git a/src/dxtbx/model/goniometer.py b/src/dxtbx/model/goniometer.py
index 3cde825aa..be4016d25 100644
--- a/src/dxtbx/model/goniometer.py
+++ b/src/dxtbx/model/goniometer.py
@@ -396,6 +396,7 @@ def imgCIF_H(cbf_handle):
         axis_vectors = {}
         angles = {}
         scan_axis = None
+        scan_axis_reversed = False
         cbf_handle.find_category(b"axis")
         for i in range(cbf_handle.count_rows()):
             cbf_handle.find_column(b"equipment")
@@ -431,6 +432,8 @@ def imgCIF_H(cbf_handle):
                         "More than one scan axis is defined: not currently supported."
                     )
                 scan_axis = axis_name
+                if increment < 0:
+                    scan_axis_reversed = True
             cbf_handle.next_row()
         if not len(axis_vectors) == len(angles):
             raise ValueError(
@@ -454,6 +457,11 @@ def imgCIF_H(cbf_handle):
         # If no scan_axis, probably a still shot ⇒ scan axis arbitrary, set to 0.
         scan_axis = ordered_axes.index(scan_axis) if scan_axis else 0
 
+        # invert axis since the axis _values_ will be inverted in scan.py::imgCIF_H
+        if scan_axis_reversed:
+            a0, a1, a2 = axis_vectors[scan_axis]
+            axis_vectors[scan_axis] = (-a0, -a1, -a2)
+
         # construct a multi-axis goniometer
         gonio = GoniometerFactory.multi_axis(
             axis_vectors, angles, flex.std_string(ordered_axes), scan_axis

From 43c169a4a7527ba954451dd8cfe16e7d75e99ee8 Mon Sep 17 00:00:00 2001
From: Graeme Winter <graeme.winter@gmail.com>
Date: Mon, 15 May 2023 12:10:28 +0100
Subject: [PATCH 07/45] Handle the new DECTRIS file writer image_bit_depth
 (#632)

They now write a (non-standard) bit_depth_image field, in addition to
bit_depth_readout. This handles this case.


Co-authored-by: Richard Gildea <richard.gildea@diamond.ac.uk>
---
 newsfragments/632.bugfix                      |  1 +
 src/dxtbx/format/FormatNXmxEigerFilewriter.py | 37 ++++++++++++++++---
 2 files changed, 32 insertions(+), 6 deletions(-)
 create mode 100644 newsfragments/632.bugfix

diff --git a/newsfragments/632.bugfix b/newsfragments/632.bugfix
new file mode 100644
index 000000000..de123cfa9
--- /dev/null
+++ b/newsfragments/632.bugfix
@@ -0,0 +1 @@
+Compatibility fix for the DECTRIS Eiger FileWriter. Recent FileWriter versions split bit depth metadata into two separate items, ``bit_depth_readout`` from the NXmx standard, and the new ``bit_depth_image`` field. This adds support for the latter, and now passes the metadata through into image conversion.
diff --git a/src/dxtbx/format/FormatNXmxEigerFilewriter.py b/src/dxtbx/format/FormatNXmxEigerFilewriter.py
index e06240d11..7e8eb2924 100644
--- a/src/dxtbx/format/FormatNXmxEigerFilewriter.py
+++ b/src/dxtbx/format/FormatNXmxEigerFilewriter.py
@@ -28,6 +28,19 @@ def __init__(self, image_file, **kwargs):
         """Initialise the image structure from the given file."""
         super().__init__(image_file, **kwargs)
 
+    def _start(self):
+        super()._start()
+        try:
+            # This is (currently) a DECTRIS-specific non-standard item that
+            # we will use in preference to bit_depth_readout (see below)
+            self._bit_depth_image = int(
+                self._cached_file_handle["/entry/instrument/detector/bit_depth_image"][
+                    ()
+                ]
+            )
+        except KeyError:
+            self._bit_depth_image = None
+
     def _get_nxmx(self, fh: h5py.File):
         nxmx_obj = nxmx.NXmx(fh)
         nxentry = nxmx_obj.entries[0]
@@ -47,14 +60,21 @@ def get_raw_data(self, index):
         nxmx_obj = self._get_nxmx(self._cached_file_handle)
         nxdata = nxmx_obj.entries[0].data[0]
         nxdetector = nxmx_obj.entries[0].instruments[0].detectors[0]
-        raw_data = get_raw_data(nxdata, nxdetector, index)
-        if self._bit_depth_readout:
+
+        # Prefer bit_depth_image over bit_depth_readout since the former
+        # actually corresponds to the bit depth of the images as stored on
+        # disk. See also:
+        #   https://www.dectris.com/support/downloads/header-docs/nexus/
+        bit_depth = self._bit_depth_image or self._bit_depth_readout
+        raw_data = get_raw_data(nxdata, nxdetector, index, bit_depth)
+
+        if bit_depth:
             # if 32 bit then it is a signed int, I think if 8, 16 then it is
             # unsigned with the highest two values assigned as masking values
-            if self._bit_depth_readout == 32:
+            if bit_depth == 32:
                 top = 2**31
             else:
-                top = 2**self._bit_depth_readout
+                top = 2**bit_depth
             for data in raw_data:
                 d1d = data.as_1d()
                 d1d.set_selected(d1d == top - 1, -1)
@@ -63,7 +83,10 @@ def get_raw_data(self, index):
 
 
 def get_raw_data(
-    nxdata: nxmx.NXdata, nxdetector: nxmx.NXdetector, index: int
+    nxdata: nxmx.NXdata,
+    nxdetector: nxmx.NXdetector,
+    index: int,
+    bit_depth: int | None = None,
 ) -> tuple[flex.float | flex.double | flex.int, ...]:
     """Return the raw data for an NXdetector.
 
@@ -85,6 +108,8 @@ def get_raw_data(
     all_data = []
     sliced_outer = data[index]
     for module_slices in get_detector_module_slices(nxdetector):
-        data_as_flex = _dataset_as_flex(sliced_outer, tuple(module_slices))
+        data_as_flex = _dataset_as_flex(
+            sliced_outer, tuple(module_slices), bit_depth=bit_depth
+        )
         all_data.append(data_as_flex)
     return tuple(all_data)

From 59405f6b38e86fc958fd2d9d3c19cd8645f8c8e8 Mon Sep 17 00:00:00 2001
From: DiamondLightSource-build-server
 <DiamondLightSource-build-server@users.noreply.github.com>
Date: Tue, 16 May 2023 04:38:19 +0100
Subject: [PATCH 08/45] DIALS 3.14.2 Changelog

---
 CHANGELOG.rst            | 9 +++++++++
 newsfragments/632.bugfix | 1 -
 2 files changed, 9 insertions(+), 1 deletion(-)
 delete mode 100644 newsfragments/632.bugfix

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 734063096..7a5d11abc 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,3 +1,12 @@
+DIALS 3.14.2 (2023-05-16)
+=========================
+
+Bugfixes
+--------
+
+- Compatibility fix for the DECTRIS Eiger FileWriter. Recent FileWriter versions split bit depth metadata into two separate items, ``bit_depth_readout`` from the NXmx standard, and the new ``bit_depth_image`` field. This adds support for the latter, and now passes the metadata through into image conversion. (`#632 <https://github.com/cctbx/dxtbx/issues/632>`_)
+
+
 dxtbx 3.14.0 (2023-04-12)
 =========================
 
diff --git a/newsfragments/632.bugfix b/newsfragments/632.bugfix
deleted file mode 100644
index de123cfa9..000000000
--- a/newsfragments/632.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Compatibility fix for the DECTRIS Eiger FileWriter. Recent FileWriter versions split bit depth metadata into two separate items, ``bit_depth_readout`` from the NXmx standard, and the new ``bit_depth_image`` field. This adds support for the latter, and now passes the metadata through into image conversion.

From 31480aec2b2a81b71c895e01bf4db203e1b1fe7f Mon Sep 17 00:00:00 2001
From: Nicholas Devenish <ndevenish@gmail.com>
Date: Tue, 16 May 2023 17:20:01 +0100
Subject: [PATCH 09/45] Include-guard references to
 ref_pickle_double_buffered.h (#636)

This was done in DIALS, which refers to this file in multiple places,
but not here. DIALS will be updated to point to this version of the
include file.
---
 newsfragments/636.misc                              | 1 +
 src/dxtbx/array_family/flex_table_suite.h           | 3 ++-
 src/dxtbx/array_family/ref_pickle_double_buffered.h | 8 ++++++++
 3 files changed, 11 insertions(+), 1 deletion(-)
 create mode 100644 newsfragments/636.misc
 create mode 100644 src/dxtbx/array_family/ref_pickle_double_buffered.h

diff --git a/newsfragments/636.misc b/newsfragments/636.misc
new file mode 100644
index 000000000..16c9a8683
--- /dev/null
+++ b/newsfragments/636.misc
@@ -0,0 +1 @@
+Use a wrapper for ref_pickle_double_buffered.h that correctly guards against multiple inclusion.
diff --git a/src/dxtbx/array_family/flex_table_suite.h b/src/dxtbx/array_family/flex_table_suite.h
index a8e500b19..b7132717b 100644
--- a/src/dxtbx/array_family/flex_table_suite.h
+++ b/src/dxtbx/array_family/flex_table_suite.h
@@ -12,7 +12,6 @@
 #include <boost/python/suite/indexing/vector_indexing_suite.hpp>
 #include <boost/mpl/for_each.hpp>
 #include <scitbx/array_family/flex_types.h>
-#include <scitbx/array_family/boost_python/ref_pickle_double_buffered.h>
 #include <scitbx/boost_python/slice.h>
 #include <scitbx/boost_python/utils.h>
 #include <dxtbx/array_family/flex_table.h>
@@ -20,6 +19,8 @@
 #include <scitbx/array_family/versa.h>
 #include <dxtbx/error.h>
 
+#include "ref_pickle_double_buffered.h"
+
 namespace dxtbx { namespace af { namespace flex_table_suite {
 
   using namespace boost::python;
diff --git a/src/dxtbx/array_family/ref_pickle_double_buffered.h b/src/dxtbx/array_family/ref_pickle_double_buffered.h
new file mode 100644
index 000000000..12477ea56
--- /dev/null
+++ b/src/dxtbx/array_family/ref_pickle_double_buffered.h
@@ -0,0 +1,8 @@
+#ifndef SCITBX_ARRAY_FAMILY_BOOST_PYTHON_REF_PICKLE_DOUBLE_BUFFERED_H_WRAPPER
+#define SCITBX_ARRAY_FAMILY_BOOST_PYTHON_REF_PICKLE_DOUBLE_BUFFERED_H_WRAPPER
+
+// This header does not have an include guard
+
+#include <scitbx/array_family/boost_python/ref_pickle_double_buffered.h>
+
+#endif
\ No newline at end of file

From 842d45939160d47829e03776a98f671f15ae16a2 Mon Sep 17 00:00:00 2001
From: David Waterman <dagewa@users.noreply.github.com>
Date: Fri, 2 Jun 2023 17:08:47 +0100
Subject: [PATCH 10/45] Bruker general Photon format (#637)

Extend support for Bruker Photon detectors (.sfrm files) to include Photon-III by using a general gain calculation based on values in the header.

Assume all such detectors are on "home" sources, and create an unpolarised beam model.
---
 newsfragments/637.feature                     |  1 +
 ...rukerPhotonII.py => FormatBrukerPhoton.py} | 50 ++++++++++---------
 2 files changed, 28 insertions(+), 23 deletions(-)
 create mode 100644 newsfragments/637.feature
 rename src/dxtbx/format/{FormatBrukerPhotonII.py => FormatBrukerPhoton.py} (87%)

diff --git a/newsfragments/637.feature b/newsfragments/637.feature
new file mode 100644
index 000000000..8b6c05f00
--- /dev/null
+++ b/newsfragments/637.feature
@@ -0,0 +1 @@
+Support for Bruker Photon detectors has been extended to include Photon-III
diff --git a/src/dxtbx/format/FormatBrukerPhotonII.py b/src/dxtbx/format/FormatBrukerPhoton.py
similarity index 87%
rename from src/dxtbx/format/FormatBrukerPhotonII.py
rename to src/dxtbx/format/FormatBrukerPhoton.py
index 073f54bfd..7ccb2b4e7 100644
--- a/src/dxtbx/format/FormatBrukerPhotonII.py
+++ b/src/dxtbx/format/FormatBrukerPhoton.py
@@ -18,7 +18,7 @@
 from dxtbx.format.FormatBruker import FormatBruker
 
 
-class FormatBrukerPhotonII(FormatBruker):
+class FormatBrukerPhoton(FormatBruker):
     @staticmethod
     def understand(image_file):
 
@@ -32,6 +32,7 @@ def understand(image_file):
         dettype = header_dic.get("DETTYPE")
         if dettype is None:
             return False
+        # We support Photon-II and Photon-III detectors
         if not dettype.startswith("CMOS-PHOTONII"):
             return False
 
@@ -44,9 +45,9 @@ def _start(self):
         except OSError:
             return False
 
-        self.header_dict = FormatBrukerPhotonII.parse_header(header_lines)
+        self.header_dict = FormatBrukerPhoton.parse_header(header_lines)
 
-        # The Photon II format can't currently use BrukerImage, see
+        # The Photon II/III format can't currently use BrukerImage, see
         # https://github.com/cctbx/cctbx_project/issues/65
         # from iotbx.detectors.bruker import BrukerImage
         # self.detectorbase = BrukerImage(self._image_file)
@@ -73,24 +74,21 @@ def _goniometer(self):
             axes, angles, names, scan_axis
         )
 
-    @staticmethod
-    def _estimate_gain(wavelength):
-        """Estimate the detector gain based on values provided by Bruker. Each ADU
-        corresponds to 36.6 electrons. The X-ray conversion results in deposited
-        charge according to the following table for typical home sources:
-
-        In (0.5136 A): 359.6893 e/X-ray
-        Ag (0.5609 A): 329.3748 e/X-ray
-        Mo (0.7107 A): 259.9139 e/X-ray
-        Ga (1.3414 A): 137.6781 e/X-ray
-        Cu (1.5418 A): 119.8156 e/X-ray
-
-        This fits the linear model (1/G) = -0.0000193358 + 0.1981607255 * wavelength
-        extremely well.
+    def _calculate_gain(self, wavelength):
+        """The CCDPARM header item contains 5 items:
+            1. readnoise
+            2. e/ADU
+            3. e/photon
+            4. bias
+            5. full scale
+        The gain in ADU/X-ray is given by (e/photon) / (e/ADU).
         """
-        inv_gain = -0.0000193358 + 0.1981607255 * wavelength
-        assert inv_gain > 0.1
-        return 1.0 / inv_gain
+        ccdparm = self.header_dict["CCDPARM"].split()
+        e_ADU = float(ccdparm[1])
+        e_photon = float(ccdparm[2])
+        if e_ADU == 0:
+            return 1.0
+        return e_photon / e_ADU
 
     def _detector(self):
         # goniometer angles in ANGLES are 2-theta, omega, phi, chi (FIXED)
@@ -123,7 +121,7 @@ def _detector(self):
 
         # Not a CCD, but is an integrating detector. Photon II has a 90 um Gadox
         # scintillator.
-        gain = self._estimate_gain(float(self.header_dict["WAVELEN"].split()[0]))
+        gain = self._calculate_gain(float(self.header_dict["WAVELEN"].split()[0]))
         return self._detector_factory.complex(
             "CCD",
             origin.elems,
@@ -136,9 +134,15 @@ def _detector(self):
         )
 
     def _beam(self):
+        """Assume home source, so make unpolarized beam"""
         wavelength = float(self.header_dict["WAVELEN"].split()[0])
 
-        return self._beam_factory.simple(wavelength)
+        return self._beam_factory.make_polarized_beam(
+            sample_to_source=(0.0, 0.0, 1.0),
+            wavelength=wavelength,
+            polarization=(0, 1, 0),
+            polarization_fraction=0.5,
+        )
 
     def _scan(self):
 
@@ -257,4 +261,4 @@ def get_raw_data(self):
 
 if __name__ == "__main__":
     for arg in sys.argv[1:]:
-        print(FormatBrukerPhotonII.understand(arg))
+        print(FormatBrukerPhoton.understand(arg))

From 666288c04b7e89f9294371a769c3bc370639d05f Mon Sep 17 00:00:00 2001
From: "Aaron S. Brewster" <asbrewster@lbl.gov>
Date: Mon, 12 Jun 2023 01:29:21 -0700
Subject: [PATCH 11/45] Handle no conda for system_includes (#630)

The libtbx env_etc.conda_cpppath doesn't exist if not in a conda environment.
---
 SConscript               | 2 +-
 newsfragments/630.bugfix | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 newsfragments/630.bugfix

diff --git a/SConscript b/SConscript
index 5b69aa630..fe962f34f 100644
--- a/SConscript
+++ b/SConscript
@@ -119,7 +119,7 @@ if not env_etc.no_boost_python and hasattr(env_etc, "boost_adaptbx_include"):
     env = env_no_includes_boost_python_ext.Clone()
 
     # Don't surface warnings from system or cctbx_project headers
-    system_includes = [x for x in env_etc.conda_cpppath if x]
+    system_includes = [x for x in env_etc.conda_cpppath if x] if libtbx.env.build_options.use_conda else []
     system_includes.append(str(Path(env_etc.scitbx_dist).parent))
     env.Append(CXXFLAGS=[f"-isystem{x}" for x in system_includes])
     env.Append(SHCXXFLAGS=[f"-isystem{x}" for x in system_includes])
diff --git a/newsfragments/630.bugfix b/newsfragments/630.bugfix
new file mode 100644
index 000000000..e3d028c8d
--- /dev/null
+++ b/newsfragments/630.bugfix
@@ -0,0 +1 @@
+Bugfix for environments without conda

From ca47d2d90da6aaad5860636245ecf16cb3283496 Mon Sep 17 00:00:00 2001
From: James Hester <jamesrhester@gmail.com>
Date: Mon, 12 Jun 2023 18:56:09 +1000
Subject: [PATCH 12/45] FormatSMVRigakuSaturn: Recognise multi-axis crystal
 goniometers (#617)

FormatSMVRigakuSaturn class previously assumed a single-axis crystal
goniometer. When there are multiple axes, and these are at non-zero
positions during a scan, inter-scan relationships will be incorrect.

Copies the code already present for multi-axis crystal goniometers in
FormatSMVRigakuEiger and add a check for a single axis, in which case
the previous behaviour is preserved.

Closes #616 .
---
 AUTHORS                                   |  1 +
 newsfragments/617.bugfix                  |  1 +
 src/dxtbx/format/FormatSMVRigakuSaturn.py | 40 +++++++++++++++++++++--
 3 files changed, 39 insertions(+), 3 deletions(-)
 create mode 100644 newsfragments/617.bugfix

diff --git a/AUTHORS b/AUTHORS
index 5dc2d823f..4ba6be00e 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -14,6 +14,7 @@ Graeme Winter
 Ian Rees
 Iris Young
 James Beilsten-Edmands
+James Hester
 James Holton
 James Parkhurst
 Johan Hattne
diff --git a/newsfragments/617.bugfix b/newsfragments/617.bugfix
new file mode 100644
index 000000000..9f0717cc5
--- /dev/null
+++ b/newsfragments/617.bugfix
@@ -0,0 +1 @@
+Rigaku Saturn SMV images with multi-axis crystal goniometers are now handledi, instead of being silently ignored. With thanks to James Hester for this contribution.
diff --git a/src/dxtbx/format/FormatSMVRigakuSaturn.py b/src/dxtbx/format/FormatSMVRigakuSaturn.py
index c44343a2a..ecd80d0f3 100644
--- a/src/dxtbx/format/FormatSMVRigakuSaturn.py
+++ b/src/dxtbx/format/FormatSMVRigakuSaturn.py
@@ -10,6 +10,7 @@
 
 from iotbx.detectors.saturn import SaturnImage
 from scitbx import matrix
+from scitbx.array_family import flex
 
 from dxtbx.format.FormatSMVRigaku import FormatSMVRigaku
 
@@ -85,12 +86,45 @@ def _goniometer(self):
         """Initialize the structure for the goniometer - this will need to
         correctly compose the axes given in the image header. In this case
         this is made rather straightforward as the image header has the
-        calculated rotation axis stored in it. We could work from the
+        calculated rotation axis stored in it. We work from the
         rest of the header and construct a goniometer model."""
 
-        axis = tuple(map(float, self._header_dictionary["ROTATION_VECTOR"].split()))
+        head_dict = self._header_dictionary
+
+        names = [e.strip() for e in head_dict["CRYSTAL_GONIO_NAMES"].split()]
+        values = [float(e) for e in head_dict["CRYSTAL_GONIO_VALUES"].split()]
+        units = [e.strip() for e in head_dict["CRYSTAL_GONIO_UNITS"].split()]
+        axis_elts = [float(e) for e in head_dict["CRYSTAL_GONIO_VECTORS"].split()]
+
+        rot_axis = tuple(map(float, head_dict["ROTATION_VECTOR"].split()))
+        scan_axis = head_dict["ROTATION_AXIS_NAME"].strip()
+        axes = [matrix.col(axis_elts[3 * j : 3 * (j + 1)]) for j in range(len(units))]
+
+        # Take only elements that have corresponding units of 'deg' (which is
+        # probably all of them).
+        filt = [e == "deg" for e in units]
+        values = [e for e, f in zip(values, filt) if f]
+        names = [e for e, f in zip(names, filt) if f]
+        axes = [e for e, f in zip(axes, filt) if f]
+
+        # Multi-axis gonio requires axes in order as viewed from crystal to gonio
+        # base. Assume the SMV header records them in reverse order.
+
+        axes = flex.vec3_double(reversed(axes))
+        names = flex.std_string(reversed(names))
+        values = flex.double(reversed(values))
+        scan_axis = flex.first_index(names, scan_axis)
+        assert scan_axis is not None
+        gonio = self._goniometer_factory.make_multi_axis_goniometer(
+            axes, values, names, scan_axis
+        )
+
+        # The calculated rotation axis is also recorded in the header. We
+        # use this to check that the goniometer is as expected
+        for e1, e2 in zip(rot_axis, gonio.get_rotation_axis()):
+            assert abs(e1 - e2) < 1e-6
 
-        return self._goniometer_factory.known_axis(axis)
+        return gonio
 
     def _detector(self):
         """Return a model for the detector, allowing for two-theta offsets

From e78fe37dd2c4d1e2e9cceb15b7be141e6d398942 Mon Sep 17 00:00:00 2001
From: Richard Gildea <rjgildea@users.noreply.github.com>
Date: Mon, 12 Jun 2023 14:34:06 +0100
Subject: [PATCH 13/45] Use orderedset.OrderedSet (#639)

Elsewhere in dials we already use this in place of the
libtbx.containers version, so use it here too.
---
 .azure-pipelines/ci-conda-env.txt | 1 +
 newsfragments/639.misc            | 1 +
 src/dxtbx/model/__init__.py       | 3 ++-
 3 files changed, 4 insertions(+), 1 deletion(-)
 create mode 100644 newsfragments/639.misc

diff --git a/.azure-pipelines/ci-conda-env.txt b/.azure-pipelines/ci-conda-env.txt
index 78a2235a6..bb73e6c19 100644
--- a/.azure-pipelines/ci-conda-env.txt
+++ b/.azure-pipelines/ci-conda-env.txt
@@ -18,6 +18,7 @@ conda-forge::matplotlib-base
 conda-forge::mrcfile
 conda-forge::numpy
 conda-forge::nxmx
+conda-forge::orderedset
 conda-forge::pillow>=5.4.1
 conda-forge::pint
 conda-forge::pip
diff --git a/newsfragments/639.misc b/newsfragments/639.misc
new file mode 100644
index 000000000..958670b9d
--- /dev/null
+++ b/newsfragments/639.misc
@@ -0,0 +1 @@
+Use ``OrderedSet`` from ``orderedset`` rather than ``libtbx.containers``.
diff --git a/src/dxtbx/model/__init__.py b/src/dxtbx/model/__init__.py
index 69048e762..26a2fc4d8 100644
--- a/src/dxtbx/model/__init__.py
+++ b/src/dxtbx/model/__init__.py
@@ -5,11 +5,12 @@
 import os
 import sys
 
+from orderedset import OrderedSet
+
 import boost_adaptbx.boost.python
 import cctbx.crystal
 import cctbx.sgtbx
 import cctbx.uctbx
-from libtbx.containers import OrderedSet
 from scitbx import matrix
 from scitbx.array_family import flex
 

From 2bbcc86a34df982ca7dce93199e2a22ccb3e61e3 Mon Sep 17 00:00:00 2001
From: Nicholas Devenish <ndevenish@gmail.com>
Date: Tue, 13 Jun 2023 14:26:01 +0100
Subject: [PATCH 14/45] MNT: Update CCTBX prebuilt version for CI

---
 .azure-pipelines/ci-conda-env.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.azure-pipelines/ci-conda-env.txt b/.azure-pipelines/ci-conda-env.txt
index bb73e6c19..38629fda4 100644
--- a/.azure-pipelines/ci-conda-env.txt
+++ b/.azure-pipelines/ci-conda-env.txt
@@ -2,7 +2,7 @@ conda-forge::boost
 conda-forge::boost-cpp
 conda-forge::bzip2
 conda-forge::c-compiler<1.5
-conda-forge::cctbx-base==2022.12
+conda-forge::cctbx-base==2023.5
 conda-forge::conda
 conda-forge::cxx-compiler<1.5
 conda-forge::python-dateutil

From d0871c707c5c40edaa2ca453203163242d613025 Mon Sep 17 00:00:00 2001
From: DiamondLightSource-build-server
 <DiamondLightSource-build-server@users.noreply.github.com>
Date: Tue, 13 Jun 2023 14:32:09 +0100
Subject: [PATCH 15/45] dxtbx 3.15.0 Changelog

    towncrier --name=dxtbx --version='3.15.0'
---
 CHANGELOG.rst             | 23 +++++++++++++++++++++++
 newsfragments/617.bugfix  |  1 -
 newsfragments/623.bugfix  |  2 --
 newsfragments/625.misc    |  1 -
 newsfragments/630.bugfix  |  1 -
 newsfragments/636.misc    |  1 -
 newsfragments/637.feature |  1 -
 newsfragments/639.misc    |  1 -
 8 files changed, 23 insertions(+), 8 deletions(-)
 delete mode 100644 newsfragments/617.bugfix
 delete mode 100644 newsfragments/623.bugfix
 delete mode 100644 newsfragments/625.misc
 delete mode 100644 newsfragments/630.bugfix
 delete mode 100644 newsfragments/636.misc
 delete mode 100644 newsfragments/637.feature
 delete mode 100644 newsfragments/639.misc

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 7a5d11abc..dddf08c6e 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,3 +1,26 @@
+dxtbx 3.15.0 (2023-06-13)
+=========================
+
+Features
+--------
+
+- Support for Bruker Photon detectors has been extended to include Photon-III. (`#637 <https://github.com/cctbx/dxtbx/issues/637>`_)
+
+
+Bugfixes
+--------
+
+- Rigaku Saturn SMV images with multi-axis crystal goniometers are now handledi, instead of being silently ignored. With thanks to James Hester for this contribution. (`#617 <https://github.com/cctbx/dxtbx/issues/617>`_)
+- FormatCBFFull: If rotation angles are decreasing, then invert the rotation axis as well as the angles, to be consistent. (`#623 <https://github.com/cctbx/dxtbx/issues/623>`_)
+- Bugfix for CCTBX bootstrapped environments, without conda. (`#630 <https://github.com/cctbx/dxtbx/issues/630>`_)
+
+
+Misc
+----
+
+- `#625 <https://github.com/cctbx/dxtbx/issues/625>`_, `#636 <https://github.com/cctbx/dxtbx/issues/636>`_, `#639 <https://github.com/cctbx/dxtbx/issues/639>`_
+
+
 DIALS 3.14.2 (2023-05-16)
 =========================
 
diff --git a/newsfragments/617.bugfix b/newsfragments/617.bugfix
deleted file mode 100644
index 9f0717cc5..000000000
--- a/newsfragments/617.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Rigaku Saturn SMV images with multi-axis crystal goniometers are now handledi, instead of being silently ignored. With thanks to James Hester for this contribution.
diff --git a/newsfragments/623.bugfix b/newsfragments/623.bugfix
deleted file mode 100644
index c836ee0eb..000000000
--- a/newsfragments/623.bugfix
+++ /dev/null
@@ -1,2 +0,0 @@
-FormatCBFFull: if rotation angles are decreasing not increasing we invert the angles, now invert rotation axis as well.
-
diff --git a/newsfragments/625.misc b/newsfragments/625.misc
deleted file mode 100644
index ae1df20fe..000000000
--- a/newsfragments/625.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add array_family to install environment
diff --git a/newsfragments/630.bugfix b/newsfragments/630.bugfix
deleted file mode 100644
index e3d028c8d..000000000
--- a/newsfragments/630.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Bugfix for environments without conda
diff --git a/newsfragments/636.misc b/newsfragments/636.misc
deleted file mode 100644
index 16c9a8683..000000000
--- a/newsfragments/636.misc
+++ /dev/null
@@ -1 +0,0 @@
-Use a wrapper for ref_pickle_double_buffered.h that correctly guards against multiple inclusion.
diff --git a/newsfragments/637.feature b/newsfragments/637.feature
deleted file mode 100644
index 8b6c05f00..000000000
--- a/newsfragments/637.feature
+++ /dev/null
@@ -1 +0,0 @@
-Support for Bruker Photon detectors has been extended to include Photon-III
diff --git a/newsfragments/639.misc b/newsfragments/639.misc
deleted file mode 100644
index 958670b9d..000000000
--- a/newsfragments/639.misc
+++ /dev/null
@@ -1 +0,0 @@
-Use ``OrderedSet`` from ``orderedset`` rather than ``libtbx.containers``.

From 700bb2e33157d03050374f422b42e63e4701e36f Mon Sep 17 00:00:00 2001
From: DiamondLightSource-build-server
 <DiamondLightSource-build-server@users.noreply.github.com>
Date: Tue, 13 Jun 2023 14:32:09 +0100
Subject: [PATCH 16/45] =?UTF-8?q?Bump=20version:=203.15.dev=20=E2=86=92=20?=
 =?UTF-8?q?3.16.dev?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 setup.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index d3de94e98..64494b80e 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.15.dev
+current_version = 3.16.dev
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<release>[a-z]+)?(?P<patch>\d+)?
diff --git a/setup.py b/setup.py
index bb8b3d8fd..8213d9921 100644
--- a/setup.py
+++ b/setup.py
@@ -8,7 +8,7 @@
 
 # Static version number which is updated by bump2version
 # Do not change this manually - use 'bump2version <major/minor/patch/release>'
-__version_tag__ = "3.15.dev"
+__version_tag__ = "3.16.dev"
 
 setup_kwargs = {
     "name": "dxtbx",

From e7b27b4a26e1bf0d2d1d1684a30cb8bb9d50dbaa Mon Sep 17 00:00:00 2001
From: Richard Gildea <rjgildea@users.noreply.github.com>
Date: Fri, 16 Jun 2023 09:22:58 +0100
Subject: [PATCH 17/45] Remove dxtbx procrunner dependency (#640)

There is no longer any reason to use procrunner in place of built-in subprocess
---
 .azure-pipelines/ci-conda-env.txt  |  1 -
 libtbx_config                      |  1 -
 newsfragments/640.misc             |  1 +
 tests/command_line/test_average.py | 14 ++++++++------
 4 files changed, 9 insertions(+), 8 deletions(-)
 create mode 100644 newsfragments/640.misc

diff --git a/.azure-pipelines/ci-conda-env.txt b/.azure-pipelines/ci-conda-env.txt
index 38629fda4..a9ffd554e 100644
--- a/.azure-pipelines/ci-conda-env.txt
+++ b/.azure-pipelines/ci-conda-env.txt
@@ -22,7 +22,6 @@ conda-forge::orderedset
 conda-forge::pillow>=5.4.1
 conda-forge::pint
 conda-forge::pip
-conda-forge::procrunner
 conda-forge::psutil
 conda-forge::pyrtf
 conda-forge::pybind11
diff --git a/libtbx_config b/libtbx_config
index 0f9b1ae2e..e511ce2b2 100644
--- a/libtbx_config
+++ b/libtbx_config
@@ -3,7 +3,6 @@
     "python_required": [
         "dials-data>=2.0.30",
         "pint",
-        "procrunner>=1.0.2",
         "pytest>=4.5,<5.0",
     ],
 }
diff --git a/newsfragments/640.misc b/newsfragments/640.misc
new file mode 100644
index 000000000..8183d5860
--- /dev/null
+++ b/newsfragments/640.misc
@@ -0,0 +1 @@
+Use Python built-in ``subprocess`` in place of external dependency ``procrunner`` for running CLI tests.
diff --git a/tests/command_line/test_average.py b/tests/command_line/test_average.py
index 38d1c901e..bedbc2d48 100644
--- a/tests/command_line/test_average.py
+++ b/tests/command_line/test_average.py
@@ -1,13 +1,15 @@
 from __future__ import annotations
 
-import procrunner
+import shutil
+import subprocess
+
 import pytest
 
 import dxtbx
 
 
 @pytest.mark.parametrize("use_mpi", [True, False])
-def test_average(dials_data, tmpdir, use_mpi):
+def test_average(dials_data, tmp_path, use_mpi):
     # averager uses cbf handling code in the xfel module
     pytest.importorskip("xfel")
 
@@ -22,16 +24,16 @@ def test_average(dials_data, tmpdir, use_mpi):
         command = "mpirun"
         mpargs = "-n 2 dxtbx.image_average --mpi=True".split()
     else:
-        command = "dxtbx.image_average"
+        command = shutil.which("dxtbx.image_average")
         mpargs = "-n 2".split()
-    result = procrunner.run(
+    result = subprocess.run(
         [command] + mpargs + "-v -a avg.cbf -s stddev.cbf -m max.cbf".split() + [data],
-        working_directory=tmpdir,
+        cwd=tmp_path,
     )
     assert not result.returncode and not result.stderr
 
     h5 = dxtbx.load(data).get_detector()
-    cbf = dxtbx.load(tmpdir.join("avg.cbf")).get_detector()
+    cbf = dxtbx.load(tmp_path / "avg.cbf").get_detector()
 
     assert h5.is_similar_to(cbf)
     assert h5[0].get_gain() == cbf[0].get_gain()

From f5e835c1934d9ed5126b31ee796cc33bc90ba7e1 Mon Sep 17 00:00:00 2001
From: Richard Gildea <rjgildea@users.noreply.github.com>
Date: Tue, 27 Jun 2023 14:08:30 +0100
Subject: [PATCH 18/45] Fix for nxmx import overwritten by local variable
 (#641)

* Unbreak help test
* explicitly set program name
* minor change to docstring

* Fix for nxmx import overwritten by local variable

Introduced by #597

Another case were bugs have crept in through lack of tests (see #470)

* news

* Rename newsfragments/XXX.bugfix to newsfragments/641.bugfix
---
 newsfragments/641.bugfix              |  1 +
 src/dxtbx/command_line/dlsnxs2cbf.py  |  4 ++--
 src/dxtbx/util/dlsnxs2cbf.py          | 15 ++++++++-------
 tests/command_line/test_dlsnxs2cbf.py |  5 ++---
 4 files changed, 13 insertions(+), 12 deletions(-)
 create mode 100644 newsfragments/641.bugfix

diff --git a/newsfragments/641.bugfix b/newsfragments/641.bugfix
new file mode 100644
index 000000000..3012098b5
--- /dev/null
+++ b/newsfragments/641.bugfix
@@ -0,0 +1 @@
+``dxtbx.dlsnxs2cbf``: Fix import overwritten by local variable.
diff --git a/src/dxtbx/command_line/dlsnxs2cbf.py b/src/dxtbx/command_line/dlsnxs2cbf.py
index 7c6cc4dbc..742fb2534 100644
--- a/src/dxtbx/command_line/dlsnxs2cbf.py
+++ b/src/dxtbx/command_line/dlsnxs2cbf.py
@@ -2,7 +2,7 @@
 Convert a NXmx-format NeXus file to a set of CBF-format image files.
 
 Note that this tool does not produce full imgCIF-format files, only
-Dectris-style mini-CBF files consisting of a plain text simplified
+DECTRIS-style mini-CBF files consisting of a plain text simplified
 header and the binary compressed image data.  The simplified header
 does not contain a full description of the experiment geometry and some
 metadata, including the detector orientation, are unspecified.  As
@@ -18,7 +18,7 @@
 
 import dxtbx.util.dlsnxs2cbf
 
-parser = argparse.ArgumentParser(description=__doc__)
+parser = argparse.ArgumentParser(description=__doc__, prog="dxtbx.dlsnxs2cbf")
 parser.add_argument(
     "nexus_file", metavar="nexus-file", help="Input NeXus file.", type=Path
 )
diff --git a/src/dxtbx/util/dlsnxs2cbf.py b/src/dxtbx/util/dlsnxs2cbf.py
index a78e2f584..011afe42d 100644
--- a/src/dxtbx/util/dlsnxs2cbf.py
+++ b/src/dxtbx/util/dlsnxs2cbf.py
@@ -12,11 +12,12 @@
 from tqdm import tqdm
 
 import dxtbx.model
+import dxtbx.nexus
 from dxtbx.ext import compress
 
 
-def compute_cbf_header(nxmx: nxmx.NXmx, nn: int):
-    nxentry = nxmx.entries[0]
+def compute_cbf_header(nxmx_obj: nxmx.NXmx, nn: int):
+    nxentry = nxmx_obj.entries[0]
     nxsample = nxentry.samples[0]
     nxinstrument = nxentry.instruments[0]
     nxdetector = nxinstrument.detectors[0]
@@ -145,11 +146,11 @@ def make_cbf(
     with h5py.File(in_name) as f:
         start_tag = binascii.unhexlify("0c1a04d5")
 
-        nxmx = nxmx.NXmx(f)
-        nxsample = nxmx.entries[0].samples[0]
-        nxinstrument = nxmx.entries[0].instruments[0]
+        nxmx_obj = nxmx.NXmx(f)
+        nxsample = nxmx_obj.entries[0].samples[0]
+        nxinstrument = nxmx_obj.entries[0].instruments[0]
         nxdetector = nxinstrument.detectors[0]
-        nxdata = nxmx.entries[0].data[0]
+        nxdata = nxmx_obj.entries[0].data[0]
 
         dependency_chain = nxmx.get_dependency_chain(nxsample.depends_on)
         scan_axis = None
@@ -183,7 +184,7 @@ def make_cbf(
 
         print(f"Writing images to {template}{'#' * num_digits}.cbf:")
         for j in tqdm(range(num_images), unit=" images"):
-            header = compute_cbf_header(nxmx, j)
+            header = compute_cbf_header(nxmx_obj, j)
             (data,) = dxtbx.nexus.get_raw_data(nxdata, nxdetector, j)
             if bit_depth_readout:
                 # if 32 bit then it is a signed int, I think if 8, 16 then it is
diff --git a/tests/command_line/test_dlsnxs2cbf.py b/tests/command_line/test_dlsnxs2cbf.py
index 5bf7673fd..954b3947c 100644
--- a/tests/command_line/test_dlsnxs2cbf.py
+++ b/tests/command_line/test_dlsnxs2cbf.py
@@ -69,10 +69,9 @@ def test_dlsnxs2cbf_deleted_axis(dials_data, tmp_path, remove_axis):
     make_cbf(tmp_path / master, template=str(tmp_path / "image_%04d.cbf"))
 
 
-@pytest.mark.xfail(reason="Broken for old data while collecting new data")
 def test_dlsnxs2cbf_help(capsys):
     with pytest.raises(SystemExit):
         run(["-h"])
     captured = capsys.readouterr()
-    assert parser.description in captured.out
-    assert "Template cbf output name e.g. 'image_%04d.cbf'" in captured.out
+    assert parser.description.splitlines()[0] in captured.out
+    assert "usage: dxtbx.dlsnxs2cbf" in captured.out

From 133ca8e4466889a8e21f7d3aed0cc77739f46f97 Mon Sep 17 00:00:00 2001
From: Nicholas Devenish <ndevenish@gmail.com>
Date: Tue, 27 Jun 2023 14:22:04 +0100
Subject: [PATCH 19/45] MNT: Skip tests removed from DLS filesystem

---
 tests/format/test_FormatNXmxDLS16M.py    | 2 ++
 tests/format/test_FormatNXmxDLS16MI03.py | 1 +
 2 files changed, 3 insertions(+)

diff --git a/tests/format/test_FormatNXmxDLS16M.py b/tests/format/test_FormatNXmxDLS16M.py
index be60d4681..56ecc1434 100644
--- a/tests/format/test_FormatNXmxDLS16M.py
+++ b/tests/format/test_FormatNXmxDLS16M.py
@@ -98,6 +98,7 @@ def test_rotation_scan_i04_2022_run_5(master_h5):
     assert masker.get_mask(imageset.get_detector(), 100)[0].count(False) == 1110799
 
 
+@pytest.mark.skip(reason="Data erased from CM visit")
 @pytest.mark.parametrize(
     "master_h5",
     [
@@ -116,6 +117,7 @@ def test_masked_i04_32bit(master_h5):
     assert flex.max(imageset[0][0]) != 0x7FFFFFFF
 
 
+@pytest.mark.skip(reason="Data erased from CM visit")
 @pytest.mark.parametrize(
     "master_h5",
     [
diff --git a/tests/format/test_FormatNXmxDLS16MI03.py b/tests/format/test_FormatNXmxDLS16MI03.py
index 0dc5f4833..6a71dc3b6 100644
--- a/tests/format/test_FormatNXmxDLS16MI03.py
+++ b/tests/format/test_FormatNXmxDLS16MI03.py
@@ -11,6 +11,7 @@
 pytest.importorskip("h5py")
 
 
+@pytest.mark.skip(reason="Data erased from CM visit")
 @pytest.mark.parametrize(
     "master_h5,masked_count",
     [

From 6c6d665901eff6a1fbe7fcf1312adadf58424d1d Mon Sep 17 00:00:00 2001
From: DiamondLightSource-build-server
 <DiamondLightSource-build-server@users.noreply.github.com>
Date: Thu, 29 Jun 2023 09:43:58 +0100
Subject: [PATCH 20/45] DIALS 3.15.1 Changelog

---
 CHANGELOG.rst            | 9 +++++++++
 newsfragments/641.bugfix | 1 -
 2 files changed, 9 insertions(+), 1 deletion(-)
 delete mode 100644 newsfragments/641.bugfix

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index dddf08c6e..101191acd 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,3 +1,12 @@
+DIALS 3.15.1 (2023-06-29)
+=========================
+
+Bugfixes
+--------
+
+- ``dxtbx.dlsnxs2cbf``: Fix import overwritten by local variable. (`#641 <https://github.com/cctbx/dxtbx/issues/641>`_)
+
+
 dxtbx 3.15.0 (2023-06-13)
 =========================
 
diff --git a/newsfragments/641.bugfix b/newsfragments/641.bugfix
deleted file mode 100644
index 3012098b5..000000000
--- a/newsfragments/641.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-``dxtbx.dlsnxs2cbf``: Fix import overwritten by local variable.

From 71d4ec04c95739c061d12b551d9cb9f86811134f Mon Sep 17 00:00:00 2001
From: Nicholas Devenish <ndevenish@gmail.com>
Date: Thu, 29 Jun 2023 22:24:38 +0100
Subject: [PATCH 21/45] Use importlib instead of pkg_resources (#643)

pkg_resources is now deprecated, as importlib has covered
most use-cases since python 3.8. This removes the code
causing warnings in downstream tests.

Also suppress numpy 'product' warnings when running pytest.
---
 libtbx_refresh.py                  | 11 +++--------
 newsfragments/643.misc             |  1 +
 pytest.ini                         |  1 +
 src/dxtbx/format/Registry.py       |  5 ++---
 src/dxtbx/model/experiment_list.py |  6 ++----
 src/dxtbx/model/profile.py         |  5 ++---
 6 files changed, 11 insertions(+), 18 deletions(-)
 create mode 100644 newsfragments/643.misc

diff --git a/libtbx_refresh.py b/libtbx_refresh.py
index f80786519..eef867e0f 100644
--- a/libtbx_refresh.py
+++ b/libtbx_refresh.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import contextlib
+import importlib
 import inspect
 import io
 import os
@@ -17,11 +18,6 @@
 except ModuleNotFoundError:
     pass
 
-try:
-    import pkg_resources
-except ModuleNotFoundError:
-    pkg_resources = None
-
 
 def _install_setup_readonly_fallback(package_name: str):
     """
@@ -67,9 +63,8 @@ def _install_setup_readonly_fallback(package_name: str):
     if import_path not in sys.path:
         sys.path.insert(0, import_path)
 
-    # ...and add to the existing pkg_resources working_set
-    if pkg_resources:
-        pkg_resources.working_set.add_entry(import_path)
+    # ...and make sure it is picked up by the import system
+    importlib.invalidate_caches()
 
     # This is already generated by this point, but will get picked up
     # on the second libtbx.refresh.
diff --git a/newsfragments/643.misc b/newsfragments/643.misc
new file mode 100644
index 000000000..0780b3d13
--- /dev/null
+++ b/newsfragments/643.misc
@@ -0,0 +1 @@
+Move from deprecated pkg_resources to importlib.
diff --git a/pytest.ini b/pytest.ini
index a3adf716c..a89237729 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -5,6 +5,7 @@ filterwarnings =
     ignore:numpy.dtype size changed:RuntimeWarning
     ignore:Datablocks are deprecated:UserWarning
     ignore:Deprecated call to `pkg_resources.declare_namespace:DeprecationWarning
+    ignore:`product` is deprecated as of NumPy:DeprecationWarning:h5py|numpy
 junit_family = legacy
 markers =
     regression: dxtbx regression test
diff --git a/src/dxtbx/format/Registry.py b/src/dxtbx/format/Registry.py
index b7034a22e..03dc39e0d 100644
--- a/src/dxtbx/format/Registry.py
+++ b/src/dxtbx/format/Registry.py
@@ -5,12 +5,11 @@
 """
 from __future__ import annotations
 
+import importlib.metadata
 import os
 import typing
 from typing import Callable
 
-import pkg_resources
-
 from dxtbx.util import get_url_scheme
 
 if typing.TYPE_CHECKING:
@@ -35,7 +34,7 @@ def get_format_class_index() -> dict[str, tuple[Callable[[], type[Format]], list
     """
     if not hasattr(get_format_class_index, "cache"):
         class_index = {}
-        for e in pkg_resources.iter_entry_points("dxtbx.format"):
+        for e in importlib.metadata.entry_points()["dxtbx.format"]:
             if ":" in e.name:
                 format_name, base_classes_str = e.name.split(":", 1)
                 base_classes = tuple(base_classes_str.split(","))
diff --git a/src/dxtbx/model/experiment_list.py b/src/dxtbx/model/experiment_list.py
index 9770129ed..d415e1ccd 100644
--- a/src/dxtbx/model/experiment_list.py
+++ b/src/dxtbx/model/experiment_list.py
@@ -3,6 +3,7 @@
 import collections
 import copy
 import errno
+import importlib.metadata
 import itertools
 import json
 import logging
@@ -11,8 +12,6 @@
 import pickle
 from typing import Any, Callable, Generator, Iterable
 
-import pkg_resources
-
 import dxtbx.datablock
 from dxtbx.datablock import (
     BeamComparison,
@@ -323,7 +322,6 @@ def decode(self):
         # a sensible experiment.
         el = ExperimentList()
         for eobj in self._obj["experiment"]:
-
             # Get the models
             identifier = eobj.get("identifier", "")
             beam = self._lookup_model("beam", eobj)
@@ -467,7 +465,7 @@ def _lookup_model(self, name, experiment_dict):
     @staticmethod
     def _scaling_model_from_dict(obj):
         """Get the scaling model from a dictionary."""
-        for entry_point in pkg_resources.iter_entry_points("dxtbx.scaling_model_ext"):
+        for entry_point in importlib.metadata.entry_points()["dxtbx.scaling_model_ext"]:
             if entry_point.name == obj["__id__"]:
                 return entry_point.load().from_dict(obj)
 
diff --git a/src/dxtbx/model/profile.py b/src/dxtbx/model/profile.py
index 0fbb0434b..ba3852586 100644
--- a/src/dxtbx/model/profile.py
+++ b/src/dxtbx/model/profile.py
@@ -1,9 +1,8 @@
 from __future__ import annotations
 
+import importlib.metadata
 import logging
 
-import pkg_resources
-
 
 class ProfileModelFactory:
     """
@@ -17,7 +16,7 @@ def from_dict(obj):
         """
         if obj is None:
             return None
-        for entry_point in pkg_resources.iter_entry_points("dxtbx.profile_model"):
+        for entry_point in importlib.metadata.entry_points()["dxtbx.profile_model"]:
             if entry_point.name == obj["__id__"]:
                 return entry_point.load().from_dict(obj)
         logging.getLogger("dxtbx.model.profile").warn(

From 1625cfa4c2236505ab0d4ba15905483cdd4dbb38 Mon Sep 17 00:00:00 2001
From: Richard Gildea <rjgildea@users.noreply.github.com>
Date: Fri, 30 Jun 2023 09:42:21 +0100
Subject: [PATCH 22/45] Drop Python 3.8 support (#642)

Upstream dependencies (e.g. numpy, scipy) have now dropped Python 3.8
support in latest releases.

Add Python 3.11 support.

See also https://dials.github.io/kb/proposals/dc3

---------

Co-authored-by: Nicholas Devenish <ndevenish@gmail.com>
---
 .azure-pipelines/azure-pipelines.yml | 20 +++++++++++---------
 .azure-pipelines/bootstrap.py        |  2 +-
 newsfragments/642.misc               |  1 +
 setup.cfg                            |  3 ++-
 4 files changed, 15 insertions(+), 11 deletions(-)
 create mode 100644 newsfragments/642.misc

diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml
index ccd5c6918..c5ec71f44 100644
--- a/.azure-pipelines/azure-pipelines.yml
+++ b/.azure-pipelines/azure-pipelines.yml
@@ -14,11 +14,11 @@ stages:
     steps:
       - checkout: none
 
-      # Use Python >=3.8 for syntax validation
+      # Use Python >=3.9 for syntax validation
       - task: UsePythonVersion@0
         displayName: Set up python
         inputs:
-          versionSpec: 3.8
+          versionSpec: 3.9
 
       # Run syntax validation on a shallow clone
       - bash: |
@@ -78,7 +78,7 @@ stages:
         vmImage: ubuntu-20.04
       timeoutInMinutes: 60
       variables:
-        PYTHON_VERSION: 3.8
+        PYTHON_VERSION: 3.9
       steps:
       - template: unix-build.yml
 
@@ -88,10 +88,10 @@ stages:
         vmImage: ubuntu-20.04
       strategy:
         matrix:
-          python38:
-            PYTHON_VERSION: 3.8
           python39:
             PYTHON_VERSION: 3.9
+          python311:
+            PYTHON_VERSION: 3.11
       timeoutInMinutes: 60
       steps:
       - template: unix-build.yml
@@ -101,10 +101,10 @@ stages:
         vmImage: macOS-latest
       strategy:
         matrix:
-          python38:
-            PYTHON_VERSION: 3.8
           python39:
             PYTHON_VERSION: 3.9
+          python311:
+            PYTHON_VERSION: 3.11
       timeoutInMinutes: 60
       steps:
       - template: unix-build.yml
@@ -114,8 +114,10 @@ stages:
         vmImage: windows-2019
       strategy:
         matrix:
-          python38:
-            PYTHON_VERSION: 3.8
+          python39:
+            PYTHON_VERSION: 3.9
+          python311:
+            PYTHON_VERSION: 3.11
       timeoutInMinutes: 20
       steps:
       - template: windows-build.yml
diff --git a/.azure-pipelines/bootstrap.py b/.azure-pipelines/bootstrap.py
index d475d7709..becd7a72b 100644
--- a/.azure-pipelines/bootstrap.py
+++ b/.azure-pipelines/bootstrap.py
@@ -847,7 +847,7 @@ def run():
         "--python",
         help="Install this minor version of Python (default: %(default)s)",
         default="3.9",
-        choices=("3.8", "3.9", "3.10"),
+        choices=("3.9", "3.10", "3.11"),
     )
     parser.add_argument(
         "--branch",
diff --git a/newsfragments/642.misc b/newsfragments/642.misc
new file mode 100644
index 000000000..b8a2436d1
--- /dev/null
+++ b/newsfragments/642.misc
@@ -0,0 +1 @@
+Drop Python 3.8 support in line with https://dials.github.io/kb/proposals/dc3. Add Python 3.11 support.
diff --git a/setup.cfg b/setup.cfg
index 8c0ba29aa..3a93e3ec8 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -8,8 +8,9 @@ classifiers =
     Operating System :: Microsoft :: Windows
     Operating System :: POSIX :: Linux
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
 
 [flake8]
 # Black disagrees with flake8 on a few points. Ignore those.

From 930be0957804d84be170e921e34c9633b92b5e4f Mon Sep 17 00:00:00 2001
From: David Waterman <dagewa@users.noreply.github.com>
Date: Tue, 25 Jul 2023 09:54:50 +0100
Subject: [PATCH 23/45] Easier override of panel geometry (#644)

When setting panels by PHIL, merge multiple definitions first by id. This fixes #299.
---
 newsfragments/299.bugfix    |  1 +
 src/dxtbx/model/detector.py | 48 +++++++++++++++++++++++++++++++++++--
 2 files changed, 47 insertions(+), 2 deletions(-)
 create mode 100644 newsfragments/299.bugfix

diff --git a/newsfragments/299.bugfix b/newsfragments/299.bugfix
new file mode 100644
index 000000000..5a8abfc77
--- /dev/null
+++ b/newsfragments/299.bugfix
@@ -0,0 +1 @@
+Panel geometry definitions in PHIL are merged by panel id before constructing panels
diff --git a/src/dxtbx/model/detector.py b/src/dxtbx/model/detector.py
index c98828c72..c046ca8d9 100644
--- a/src/dxtbx/model/detector.py
+++ b/src/dxtbx/model/detector.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+from copy import deepcopy
 
 import pycbf
 
@@ -212,6 +213,42 @@
 )
 
 
+def merge_panel_scope_extracts_by_id(panel_params):
+
+    id_to_params = {}
+    for (i, params) in enumerate(panel_params):
+        if params.id not in id_to_params:
+            id_to_params[params.id] = [
+                i,
+            ]
+        else:
+            id_to_params[params.id].append(i)
+
+    merged_params = []
+    for params_set in id_to_params.values():
+        params0 = deepcopy(panel_params[params_set[0]])
+        for i in params_set[1:]:
+            params1 = deepcopy(panel_params[i])
+            for key in params1.__dict__:
+                if (
+                    key.startswith("_")
+                    or key == "id"
+                    or params0.__dict__[key] == params1.__dict__[key]
+                ):
+                    continue
+                if (
+                    params0.__dict__[key] is not None
+                    and params1.__dict__[key] is not None
+                ):
+                    raise RuntimeError(
+                        f"Multiple definitions for {key} for panel id={params0.id}"
+                    )
+                if params0.__dict__[key] is None:
+                    params0.__dict__[key] = params1.__dict__[key]
+        merged_params.append(params0)
+    return merged_params
+
+
 class DetectorFactory:
     """A factory class for detector objects, which will encapsulate standard
     detector designs to make it a little easier to get started with these. In
@@ -229,7 +266,11 @@ def generate_from_phil(params, beam=None):
 
         # Create a list of panels
         panel_list = {}
-        for panel_params in params.detector.panel:
+
+        # merge panel params by id first
+        merged = merge_panel_scope_extracts_by_id(params.detector.panel)
+
+        for panel_params in merged:
             panel = Panel()
             if panel_params.name is not None:
                 panel.set_name(panel_params.name)
@@ -335,8 +376,11 @@ def overwrite_from_phil(params, detector, beam=None):
         Overwrite from phil parameters
 
         """
+        # merge panel params by id first
+        merged = merge_panel_scope_extracts_by_id(params.detector.panel)
+
         # Override any panel parameters
-        for panel_params in params.detector.panel:
+        for panel_params in merged:
             panel = detector[panel_params.id]
             if panel_params.name is not None:
                 panel.set_name(panel_params.name)

From 436a1b6a03298338a4027c4ca374b73480adff14 Mon Sep 17 00:00:00 2001
From: Nicholas Devenish <ndevenish@gmail.com>
Date: Tue, 25 Jul 2023 18:42:06 +0100
Subject: [PATCH 24/45] Flumpy: Don't allow converting 1D vectors to vec2/3
 (#648)

Previously, a size (3,) array would have erroneously converted
to a 0x3 length vec3.

This is now explicitly forbidden; you must pass at least a (0, 3)
multidimensional array.

Fixes #439.
---
 newsfragments/439.bugfix         |  1 +
 src/dxtbx/boost_python/flumpy.cc |  7 +++++++
 tests/test_flumpy.py             | 10 +++++++++-
 3 files changed, 17 insertions(+), 1 deletion(-)
 create mode 100644 newsfragments/439.bugfix

diff --git a/newsfragments/439.bugfix b/newsfragments/439.bugfix
new file mode 100644
index 000000000..a97cc5116
--- /dev/null
+++ b/newsfragments/439.bugfix
@@ -0,0 +1 @@
+``flumpy``: Fix case where incorrect ``flex.vec2``, ``flex.vec3`` could be generated.
diff --git a/src/dxtbx/boost_python/flumpy.cc b/src/dxtbx/boost_python/flumpy.cc
index 12e5ebf94..d7ce34b20 100644
--- a/src/dxtbx/boost_python/flumpy.cc
+++ b/src/dxtbx/boost_python/flumpy.cc
@@ -520,6 +520,13 @@ py::object vec_from_numpy(py::array np_array) {
 
   static_assert(VecType<int>::fixed_size == 2 || VecType<int>::fixed_size == 3,
                 "Only vec2/vec3 supported");
+
+  // Only accept arrays that have a dimension higher than 1 - we want
+  // numpy.array([1,2,3]) to fail but numpy.array([[1,2,3]]) to work
+  if (np_array.ndim() == 1) {
+    throw std::invalid_argument("Array for conversion to vec must be multidimensional");
+  }
+
   // Only accept arrays whose last dimension is the size of this object
   if (np_array.shape(np_array.ndim() - 1) != VecType<int>::fixed_size) {
     throw std::invalid_argument("Input array last dimension is not size "
diff --git a/tests/test_flumpy.py b/tests/test_flumpy.py
index cd09e3c09..88ef9f8cc 100644
--- a/tests/test_flumpy.py
+++ b/tests/test_flumpy.py
@@ -136,7 +136,7 @@ def test_reverse_numeric_2d(flex_numeric):
 
 
 def test_numeric_4d(flex_numeric):
-    #  Check that we can think fourth-dimnesionally
+    # Check that we can think fourth-dimensionally
     grid = flex.grid(1, 9, 8, 5)
     fo = flex_numeric(grid)
     assert fo.nd() == 4
@@ -402,3 +402,11 @@ def test_int_long_degeneracy():
     npo[0] = 42
     assert all(fo[x] == npo[x] for x in range(4))
     assert fo[0] == 42
+
+
+def test_single_entry_vec_from_numpy():
+    with pytest.raises(ValueError):
+        flumpy.vec_from_numpy(np.array([2.0, 3.0, 4.0]))
+    ao = flumpy.vec_from_numpy(np.array([[2.0, 3.0, 4.0]]))
+    assert len(ao) == 1
+    assert ao[0] == pytest.approx([2.0, 3.0, 4.0])

From 180fac839980dd912abf918be34c99f86d56ab62 Mon Sep 17 00:00:00 2001
From: David Waterman <dagewa@users.noreply.github.com>
Date: Wed, 26 Jul 2023 14:17:57 +0100
Subject: [PATCH 25/45] FormatROD (#645)

Partial support for Rigaku Oxford Diffraction format images.
---------

Co-authored-by: Takanori Nakane <nakane.t@gmail.com>
Co-authored-by: Nicholas Devenish <ndevenish@gmail.com>
---
 newsfragments/645.feature     |   1 +
 src/dxtbx/format/FormatROD.py | 459 ++++++++++++++++++++++++++++++++++
 2 files changed, 460 insertions(+)
 create mode 100644 newsfragments/645.feature
 create mode 100644 src/dxtbx/format/FormatROD.py

diff --git a/newsfragments/645.feature b/newsfragments/645.feature
new file mode 100644
index 000000000..fb38a5121
--- /dev/null
+++ b/newsfragments/645.feature
@@ -0,0 +1 @@
+Add partial support for the Rigaku Oxford Diffraction file format.
diff --git a/src/dxtbx/format/FormatROD.py b/src/dxtbx/format/FormatROD.py
new file mode 100644
index 000000000..417f99f1a
--- /dev/null
+++ b/src/dxtbx/format/FormatROD.py
@@ -0,0 +1,459 @@
+"""Support for the Rigaku Oxford Diffraction image format
+
+NB: Rigaku datasets may use a non-zero-padded image incremental serial number.
+At present, this is not compatible with assumptions in dxtbx. In order to
+import these datasets, the images should be renumbered first.
+
+See https://github.com/cctbx/dxtbx/issues/646 for details."""
+
+# Takanori Nakane took David Waterman's code to parse headers from
+#  https://github.com/cctbx/cctbx_project/commit/b95467f3b2a70a37eeb820ea294128a32551700c
+# and heavily modified it. The original commit in the cctbx_project repository is orphan now.
+from __future__ import annotations
+
+__author__ = "David Waterman, Takanori Nakane"
+__copyright__ = (
+    "Copyright 2018 United Kingdom Research and Innovation & 2022 Takanori Nakane"
+)
+__license__ = "BSD 3-clause"
+
+import re
+import struct
+
+import numpy as np
+
+from scitbx.array_family import flex
+
+from dxtbx.format.Format import Format
+
+
+class FormatROD(Format):
+    """An image reading class for Rigaku Oxford Diffraction images.
+
+    The Rigaku Oxford Diffraction format is used by CrysAlis(Pro).
+    """
+
+    def __init__(self, image_file, **kwargs):
+        """Initialise the image structure from the given file."""
+
+        from dxtbx import IncorrectFormatError
+
+        if not self.understand(image_file):
+            raise IncorrectFormatError(self, image_file)
+
+        Format.__init__(self, image_file, **kwargs)
+
+    @staticmethod
+    def understand(image_file):
+        with FormatROD.open_file(image_file, "rb") as f:
+            try:
+                hdr = f.read(256).decode("ascii")
+            except UnicodeDecodeError:
+                return False
+
+        lines = hdr.splitlines()
+        if len(lines) < 2:
+            return False
+
+        vers = lines[0].split()
+        if len(vers) < 2 or vers[0] != "OD" or vers[1] != "SAPPHIRE":
+            return False
+
+        compression = lines[1].split("=")
+        if compression[0] != "COMPRESSION":
+            return False
+
+        return True
+
+    @staticmethod
+    def _read_ascii_header(image_file):
+        """Read the ASCII header comprising the first 256 bytes of the file"""
+
+        hd = {}
+        with FormatROD.open_file(image_file, "rb") as f:
+            hdr = f.read(256).decode("ascii")
+        lines = hdr.splitlines()
+
+        vers = lines[0].split()
+        if len(vers) < 2 or vers[0] != "OD" or vers[1] != "SAPPHIRE":
+            raise ValueError("Wrong header format")
+        hd["version"] = float(vers[-1])
+
+        compression = lines[1].split("=")
+        if compression[0] != "COMPRESSION":
+            raise ValueError("Wrong header format")
+        hd["compression"] = compression[1]
+
+        # Extract definitions from the 3rd - 5th line
+        defn = re.compile(r"([A-Z]+=[ 0-9]+)")
+        for line in lines[2:5]:
+            sizes = defn.findall(line)
+            for s in sizes:
+                n, v = s.split("=")
+                hd[n] = int(v)
+
+        hd["time"] = lines[5].split("TIME=")[-1].strip("\x1a").rstrip()
+
+        return hd
+
+    @staticmethod
+    def _read_binary_header(
+        image_file,
+        offset=256,
+        nbytes=5120,
+        general_nbytes=512,
+        special_nbytes=768,
+        km4gonio_nbytes=1024,
+        statistics_nbytes=512,
+        history_nbytes=2048,
+    ):
+        """Read the most relevant parameters from the binary header"""
+
+        with FormatROD.open_file(image_file, "rb") as f:
+            # General section
+            f.seek(offset)
+            bin_x, bin_y = struct.unpack("<hh", f.read(4))
+            f.seek(offset + 22)
+            chip_npx_x, chip_npx_y, im_npx_x, im_npx_y = struct.unpack(
+                "<hhhh", f.read(8)
+            )
+            f.seek(offset + 36)
+            num_points = struct.unpack("<I", f.read(4))[0]
+            if num_points != im_npx_x * im_npx_y:
+                raise ValueError("Cannot interpret binary header")
+
+            # Special section
+            f.seek(offset + general_nbytes + 56)
+            gain = struct.unpack("<d", f.read(8))[0]
+            f.seek(offset + general_nbytes + 464)
+            overflow_flag, overflow_after_remeasure_flag = struct.unpack(
+                "<hh", f.read(4)
+            )
+            f.seek(offset + general_nbytes + 472)
+            overflow_threshold = struct.unpack("<l", f.read(4))[0]
+            f.seek(offset + general_nbytes + 480)
+            exposure_time_sec, overflow_time_sec = struct.unpack("<dd", f.read(16))
+            f.seek(offset + general_nbytes + 548)
+            detector_type = struct.unpack("<l", f.read(4))[0]
+            f.seek(offset + general_nbytes + 568)
+            real_px_size_x, real_px_size_y = struct.unpack("<dd", f.read(16))
+
+            # Goniometer section
+            f.seek(offset + general_nbytes + special_nbytes + 284)
+            # angles for OMEGA, THETA, CHI(=KAPPA), PHI,
+            # OMEGA_PRIME (also called DETECTOR_AXIS; what's this?), THETA_PRIME
+            start_angles_steps = struct.unpack("<llllllllll", f.read(40))
+            end_angles_steps = struct.unpack("<llllllllll", f.read(40))
+            f.seek(offset + general_nbytes + special_nbytes + 368)
+            step_to_rad = struct.unpack("<dddddddddd", f.read(80))
+            f.seek(offset + general_nbytes + special_nbytes + 552)
+            # FIXME: I don't know what these are. Isn't the beam along e1 by definition??
+            beam_rotn_around_e2, beam_rotn_around_e3 = struct.unpack("<dd", f.read(16))
+            alpha1_wavelength = struct.unpack("<d", f.read(8))[0]
+            f.seek(offset + general_nbytes + special_nbytes + 640)
+            # detector rotation in degrees along e1, e2, e3
+            detector_rotns = struct.unpack("<ddd", f.read(24))
+            # FIXME: direct beam position when all angles are zero?
+            origin_px_x, origin_px_y = struct.unpack("<dd", f.read(16))
+            # alpha and beta are angles between KAPPA(=CHI) and THETA, and e3.
+            angles_in_deg = struct.unpack(
+                "<dddd", f.read(32)
+            )  # alpha, beta, gamma, delta
+            f.seek(offset + general_nbytes + special_nbytes + 712)
+            distance_mm = struct.unpack("<d", f.read(8))[0]
+
+        return {
+            "bin_x": bin_x,
+            "bin_y": bin_y,
+            "chip_npx_x": chip_npx_x,
+            "chip_npx_y": im_npx_y,
+            "im_npx_x": im_npx_x,
+            "im_npx_y": im_npx_y,
+            "gain": gain,
+            "overflow_flag": overflow_flag,
+            "overflow_after_remeasure_flag": overflow_after_remeasure_flag,
+            "overflow_threshold": overflow_threshold,
+            "exposure_time_sec": exposure_time_sec,
+            "overflow_time_sec": overflow_time_sec,
+            "detector_type": detector_type,
+            "real_px_size_x": real_px_size_x,
+            "real_px_size_y": real_px_size_y,
+            "start_angles_steps": start_angles_steps,
+            "end_angles_steps": end_angles_steps,
+            "step_to_rad": step_to_rad,
+            "beam_rotn_around_e2": beam_rotn_around_e2,
+            "beam_rotn_around_e3": beam_rotn_around_e3,
+            "alpha1_wavelength": alpha1_wavelength,
+            "detector_rotns": detector_rotns,
+            "origin_px_x": origin_px_x,
+            "origin_px_y": origin_px_y,
+            "angles_in_deg": angles_in_deg,
+            "distance_mm": distance_mm,
+        }
+
+    def _start(self):
+        """Open the image file, read useful metadata into an internal dictionary
+        self._header_dictionary"""
+
+        self._txt_header = self._read_ascii_header(self._image_file)
+        self._bin_header = self._read_binary_header(self._image_file)
+
+        return
+
+    # Rigaku/Oxford Geometry:
+    # - e3: parallel to OMEGA (and THETA)
+    # - e1: crystal to source
+    # - e2: completes the right handed coordinate system
+
+    def get_goniometer(self, index=None):
+        return Format.get_goniometer(self)
+
+    def _goniometer(self):
+        # FIXME: sometimes XDS.INP generated by CrysAlisPro has
+        #  tiny deviationis from (0, 1, 0). I don't know how to calculate it.
+        # DIALS' third axis points to the opposite of XDS; thus the direction will be opposite.
+        direction = (0.0, -1.0, 0.0)
+
+        # FIXME: represent kappa axis in fixed_rotation
+        return self._goniometer_factory.known_axis(direction)
+
+    def get_beam(self, index=None):
+        return Format.get_beam(self)
+
+    def _beam(self):
+        return self._beam_factory.make_polarized_beam(
+            sample_to_source=(0.0, 0.0, 1.0),
+            wavelength=self._bin_header["alpha1_wavelength"],
+            polarization=(0, 1, 0),
+            polarization_fraction=0.5,
+        )
+
+    def get_detector(self, index=None):
+        return Format.get_detector(self)
+
+    def _detector(self):
+        gonio_angles = np.array(self._bin_header["start_angles_steps"]) * np.array(
+            self._bin_header["step_to_rad"]
+        )
+        detector_rotns_rad = np.array(self._bin_header["detector_rotns"]) / 180 * np.pi
+        rot_e1 = np.array(
+            [
+                np.cos(detector_rotns_rad[0]),
+                np.sin(detector_rotns_rad[0]),
+                0,
+                -np.sin(detector_rotns_rad[0]),
+                np.cos(detector_rotns_rad[0]),
+                0,
+                0,
+                0,
+                1,
+            ]
+        ).reshape(3, 3)
+        rot_e2 = np.array(
+            [
+                1,
+                0,
+                0,
+                0,
+                np.cos(detector_rotns_rad[1]),
+                np.sin(detector_rotns_rad[1]),
+                0,
+                -np.sin(detector_rotns_rad[1]),
+                np.cos(detector_rotns_rad[1]),
+            ]
+        ).reshape(3, 3)
+        rot_theta = np.array(
+            [
+                np.cos(gonio_angles[1]),
+                0,
+                np.sin(gonio_angles[1]),
+                0,
+                1,
+                0,
+                -np.sin(gonio_angles[1]),
+                0,
+                np.cos(gonio_angles[1]),
+            ]
+        ).reshape(3, 3)
+        detector_axes = rot_theta.dot(rot_e2.dot(rot_e1))
+        # The third axis points to the opposite of XDS.
+        detector_axes[2, :] *= -1
+
+        pixel_size_x = self._bin_header["real_px_size_x"]
+        pixel_size_y = self._bin_header["real_px_size_y"]
+        origin_at_zero = np.array(
+            [
+                -self._bin_header["origin_px_x"] * pixel_size_x,
+                +self._bin_header["origin_px_y"] * pixel_size_y,
+                +self._bin_header["distance_mm"],
+            ]
+        )
+
+        # FIXME: this formula seem to give the right answer but I don't know why.
+        # XDS exporter in IPR's CrysAlisPro seems broken. It writes the same ORGX/Y
+        # regardless of the theta angles.
+        origin = detector_axes.dot(origin_at_zero)
+
+        detector = self._detector_factory.make_detector(
+            "PAD",
+            detector_axes[:, 0],
+            -detector_axes[:, 1],
+            origin,
+            (pixel_size_x, pixel_size_y),
+            (self._txt_header["NX"], self._txt_header["NY"]),
+            (0, self._bin_header["overflow_threshold"]),
+        )  # not sure about min
+
+        return detector
+
+    def get_scan(self, index=None):
+        return Format.get_scan(self)
+
+    def _scan(self):
+        """Return the scan information for this image."""
+
+        for axis in [0, 3]:  # 0 - OMEGA, 3 - PHI: FIXME: is it always PHI scan?
+            start_angle = (
+                self._bin_header["start_angles_steps"][axis]
+                * self._bin_header["step_to_rad"][axis]
+                / np.pi
+                * 180
+            )
+            end_angle = (
+                self._bin_header["end_angles_steps"][axis]
+                * self._bin_header["step_to_rad"][axis]
+                / np.pi
+                * 180
+            )
+            if start_angle != end_angle:
+                break
+
+        return self._scan_factory.single_file(
+            filename=self._image_file,
+            exposure_times=self._bin_header["exposure_time_sec"],
+            osc_start=start_angle,
+            osc_width=end_angle - start_angle,
+            epoch=None,
+        )
+
+    def get_raw_data(self):
+        comp = self._txt_header["compression"].strip()
+        if comp.startswith("TY6"):
+            return self._get_raw_data_ty6()
+        else:
+            raise NotImplementedError("Can't handle compression: {0}".format(comp))
+
+    def _get_raw_data_ty6(self):
+        offset = self._txt_header["NHEADER"]
+        nx = self._txt_header["NX"]
+        ny = self._txt_header["NY"]
+        with open(self._image_file, "rb") as f:
+            f.seek(offset)
+            lbytesincompressedfield = struct.unpack("<l", f.read(4))[0]
+            linedata = np.fromfile(f, dtype=np.uint8, count=lbytesincompressedfield)
+            offsets = struct.unpack("<%dI" % ny, f.read(4 * ny))
+
+            image = np.zeros((ny, nx), dtype=np.int32)
+            for iy in range(ny):
+                image[iy, :] = self.decode_TY6_oneline(linedata[offsets[iy] :], nx)
+            return flex.int(image)
+
+    def decode_TY6_oneline(self, linedata, w):
+        """Decompress TY6 encoded pixels for a single line.
+        w is the number of pixels in the fast axis."""
+
+        BLOCKSIZE = 8
+        SHORT_OVERFLOW = np.int32(254)
+        LONG_OVERFLOW = 255
+        SHORT_OVERFLOW_SIGNED = SHORT_OVERFLOW - 127
+        LONG_OVERFLOW_SIGNED = LONG_OVERFLOW - 127
+
+        ipos = 0
+        opos = 0
+        ret = np.zeros(w, dtype=np.int32)
+
+        nblock = (w - 1) // (BLOCKSIZE * 2)
+        nrest = (w - 1) % (BLOCKSIZE * 2)
+
+        firstpx = linedata[ipos]
+        ipos += 1
+        if firstpx < SHORT_OVERFLOW:
+            ret[opos] = firstpx - 127
+        elif firstpx == LONG_OVERFLOW:
+            ret[opos] = linedata[ipos : (ipos + 4)].view(np.int32)[0]
+            ipos += 4
+        else:
+            ret[opos] = linedata[ipos : (ipos + 2)].view(np.int16)[0]
+            ipos += 2
+        opos += 1
+
+        for k in range(nblock):
+            bittype = linedata[ipos]
+            nbits = (bittype & 15, (bittype >> 4) & 15)
+            ipos += 1
+            # ipos_bit = ipos * 8
+
+            for i in range(2):
+                nbit = nbits[i]
+                # 0: 0
+                # 1: 1 or 0
+                # 2: -1, 0, 1, 2
+                # 3: -3, -2, -1, 0, 1, 2, 3, 4 (= b111 -3 = 7 - 3)
+                zero_at = 0
+                if nbit > 1:
+                    zero_at = (1 << (nbit - 1)) - 1
+
+                v = 0
+                for j in range(nbit):
+                    v |= linedata[ipos] << (8 * j)
+                    ipos += 1
+
+                mask = (1 << nbit) - 1
+                for j in range(BLOCKSIZE):
+                    ret[opos] = ((v >> (nbit * j)) & mask) - zero_at
+                    opos += 1
+
+            for i in range(opos - BLOCKSIZE * 2, opos):
+                offset = ret[i]
+
+                # surprisingly, this IF is very slow!
+                if offset >= SHORT_OVERFLOW_SIGNED:
+                    if offset >= LONG_OVERFLOW_SIGNED:
+                        offset = linedata[ipos : (ipos + 4)].view(np.int32)[0]
+                        ipos += 4
+                    else:
+                        offset = linedata[ipos : (ipos + 2)].view(np.int16)[0]
+                        ipos += 2
+
+                ret[i] = offset + ret[i - 1]
+
+        for i in range(nrest):
+            px = linedata[ipos]
+            ipos += 1
+            if px < SHORT_OVERFLOW:
+                ret[opos] = ret[opos - 1] + px - 127
+            elif px == LONG_OVERFLOW:
+                ret[opos] = (
+                    ret[opos - 1] + linedata[ipos : (ipos + 4)].view(np.int32)[0]
+                )
+                ipos += 4
+            else:
+                ret[opos] = (
+                    ret[opos - 1] + linedata[ipos : (ipos + 2)].view(np.int16)[0]
+                )
+                ipos += 2
+            opos += 1
+
+        return ret
+
+
+if __name__ == "__main__":
+    import sys
+
+    for arg in sys.argv[1:]:
+        if FormatROD.understand(arg):
+            reader = FormatROD(arg)
+            print(reader._txt_header)
+            print(reader._bin_header)
+        else:
+            print("Unsupported format")

From e9b88d273389b5fdaa2afc37fa7940905fbb5cb9 Mon Sep 17 00:00:00 2001
From: Nicholas Devenish <ndevenish@gmail.com>
Date: Wed, 26 Jul 2023 14:41:57 +0100
Subject: [PATCH 26/45] Check Registry against refl/expt files (#650)

This was previously done implicitly, but only when dials_regression
was present. This makes the test explicit, and works with
dials-data (so that it will work on e.g. CI with restricted access
to test cases).
---
 newsfragments/650.misc      |  1 +
 tests/format/test_format.py | 12 ++++++++++++
 2 files changed, 13 insertions(+)
 create mode 100644 newsfragments/650.misc
 create mode 100644 tests/format/test_format.py

diff --git a/newsfragments/650.misc b/newsfragments/650.misc
new file mode 100644
index 000000000..94f0ed982
--- /dev/null
+++ b/newsfragments/650.misc
@@ -0,0 +1 @@
+Format classes are now tested against invalid binary data with dials-data, for when dials-regression is not present.
diff --git a/tests/format/test_format.py b/tests/format/test_format.py
new file mode 100644
index 000000000..3b951ba8d
--- /dev/null
+++ b/tests/format/test_format.py
@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from dxtbx.format import Registry
+
+
+def test_reading_refl_failure(dials_data):
+    test_data = dials_data("centroid_test_data", pathlib=True)
+
+    # Without dials_regression, none of the dials-data tests check for this "invalid binary data" case
+    assert Registry.get_format_class_for_file(test_data / "indexed.refl") is None
+    # Check .expt while here
+    assert Registry.get_format_class_for_file(test_data / "indexed.expt") is None

From 69169ff5a0e144c445f29677bed0b18fd6d8d0d4 Mon Sep 17 00:00:00 2001
From: David Waterman <dagewa@users.noreply.github.com>
Date: Wed, 26 Jul 2023 16:58:34 +0100
Subject: [PATCH 27/45] Make Imageset slices consistently index from 0

Without this, slices sometimes needed to be indexed by their original
image index, even inside the slice.

The previous workaround in 67aab2 (#485) for negative offset was
reacting to the erroneous behaviour introduced by including batch
offsets in imagesequence slicing.
---
 newsfragments/633.bugfix           |  1 +
 src/dxtbx/imageset.py              | 23 +++--------------------
 src/dxtbx/model/experiment_list.py |  3 +--
 tests/test_imageset.py             |  8 +++++++-
 4 files changed, 12 insertions(+), 23 deletions(-)
 create mode 100644 newsfragments/633.bugfix

diff --git a/newsfragments/633.bugfix b/newsfragments/633.bugfix
new file mode 100644
index 000000000..7ce5110e5
--- /dev/null
+++ b/newsfragments/633.bugfix
@@ -0,0 +1 @@
+Slicing of imageset objects is now consistently 0-based, including for the sliced data accessor. Previously, the data accessor had to be accessed with the original index offsets.
diff --git a/src/dxtbx/imageset.py b/src/dxtbx/imageset.py
index 574779956..85fd2200b 100644
--- a/src/dxtbx/imageset.py
+++ b/src/dxtbx/imageset.py
@@ -300,29 +300,12 @@ def __getitem__(self, item):
         if not isinstance(item, slice):
             return self.get_corrected_data(item)
         else:
-            offset = self.get_scan().get_batch_offset()
             if item.step is not None:
                 raise IndexError("Sequences must be sequential")
 
-            # nasty workaround for https://github.com/dials/dials/issues/1153
-            # slices with -1 in them are meaningful :-/ so grab the original
-            # constructor arguments of the slice object.
-            # item.start and item.stop may have been compromised at this point.
-            if offset < 0:
-                start, stop, step = item.__reduce__()[1]
-                if start is None:
-                    start = 0
-                else:
-                    start -= offset
-                if stop is None:
-                    stop = len(self)
-                else:
-                    stop -= offset
-            else:
-                start = item.start or 0
-                stop = item.stop or (len(self) + offset)
-                start -= offset
-                stop -= offset
+            start = item.start or 0
+            stop = item.stop or len(self)
+
             if self.data().has_single_file_reader():
                 reader = self.reader().copy(self.reader().paths(), stop - start)
             else:
diff --git a/src/dxtbx/model/experiment_list.py b/src/dxtbx/model/experiment_list.py
index d415e1ccd..ec836614f 100644
--- a/src/dxtbx/model/experiment_list.py
+++ b/src/dxtbx/model/experiment_list.py
@@ -625,8 +625,7 @@ def from_sequence_and_crystal(imageset, crystal, load_models=True):
             # if imagesequence is still images, make one experiment for each
             # all referencing into the same image set
             if imageset.get_scan().is_still():
-                start, end = imageset.get_scan().get_array_range()
-                for j in range(start, end):
+                for j in range(len(imageset)):
                     subset = imageset[j : j + 1]
                     experiments.append(
                         Experiment(
diff --git a/tests/test_imageset.py b/tests/test_imageset.py
index 9285db95d..7c54d65b7 100644
--- a/tests/test_imageset.py
+++ b/tests/test_imageset.py
@@ -387,6 +387,11 @@ def tst_get_item(self, sequence):
         with pytest.raises(RuntimeError):
             _ = sequence2[5]
 
+        # Check data access matches expected images from the slice
+        panel_data1 = sequence[3][0]
+        panel_data2 = sequence2[0][0]
+        assert panel_data1.all_eq(panel_data2)
+
         assert len(sequence2) == 4
         assert_can_get_detectorbase(sequence2, range(0, 4), 5)
         self.tst_get_models(sequence2, range(0, 4), 5)
@@ -396,11 +401,12 @@ def tst_get_item(self, sequence):
         with pytest.raises(IndexError):
             _ = sequence[3:7:2]
 
+        # Batch offset should not affect slicing of imagesequence
         # Simulate a scan starting from image 0
         sequence_ = copy.deepcopy(sequence)
         sequence_.get_scan().set_batch_offset(-1)
         sequence3 = sequence_[3:7]
-        assert sequence3.get_array_range() == (4, 8)
+        assert sequence3.get_array_range() == (3, 7)
 
     @staticmethod
     def tst_paths(sequence, filenames1):

From 0a068b56df0cc05cd5ea5e5ebe2f0c203e341e54 Mon Sep 17 00:00:00 2001
From: Nicholas Devenish <ndevenish@gmail.com>
Date: Thu, 27 Jul 2023 14:52:33 +0100
Subject: [PATCH 28/45] MNT: Update CMake modules path relative to script

Without this, dxtbx did not work as a subdirectory in
a larger CMakeLists project.
---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c286d39e2..96c6b76d5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.20 FATAL_ERROR)
 project(dxtbx LANGUAGES C CXX)
 
 # Add the included modules
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/")
 
 # General cmake environment configuration
 include(SetDefaultBuildRelWithDebInfo) # Default builds to release with debug info

From 64bcb3240a3e300fe7c7fa2f0de7a7aa931d9ef3 Mon Sep 17 00:00:00 2001
From: Christian Becke <christianbecke@gmail.com>
Date: Mon, 31 Jul 2023 10:58:00 +0200
Subject: [PATCH 29/45] Fixes for EMBL beamlines at PETRA (#626)

The E-32-0017 Eiger moved to P13 on 2021-5-22, where the goniometer axis no
longer needs overriding.

Check for that date in the MiniCBF, and check for the serial matching the date
range.

Co-authored-by: Christian Becke <christian.becke@fu-berlin.de>
Co-authored-by: Nicholas Devenish <ndevenish@gmail.com>
---
 newsfragments/626.feature                      |  1 +
 src/dxtbx/format/FormatCBFMini.py              | 15 +++++++++++++++
 src/dxtbx/format/FormatCBFMiniEigerPetraP14.py | 13 ++++++++++++-
 3 files changed, 28 insertions(+), 1 deletion(-)
 create mode 100644 newsfragments/626.feature

diff --git a/newsfragments/626.feature b/newsfragments/626.feature
new file mode 100644
index 000000000..c28cc4293
--- /dev/null
+++ b/newsfragments/626.feature
@@ -0,0 +1 @@
+Update Format handling to reflect move of Eiger detector from PETRA P14 to P13.
diff --git a/src/dxtbx/format/FormatCBFMini.py b/src/dxtbx/format/FormatCBFMini.py
index 4561665fd..39060f84a 100644
--- a/src/dxtbx/format/FormatCBFMini.py
+++ b/src/dxtbx/format/FormatCBFMini.py
@@ -8,6 +8,7 @@
 from __future__ import annotations
 
 import binascii
+import datetime
 import os
 import pathlib
 import sys
@@ -73,6 +74,20 @@ def __init__(self, image_file, **kwargs):
         self._raw_data = None
         super().__init__(image_file, **kwargs)
 
+    @staticmethod
+    def _get_timestamp_from_raw_header(
+        header: str | list[str],
+    ) -> datetime.datetime | None:
+        """Given a raw header, or lines from, attempt to extract the timestamp field"""
+        if isinstance(header, str):
+            header = header.splitlines()
+        timestamp = None
+        for record in header:
+            if len(record[1:].split()) <= 2 and record.count(":") == 2:
+                timestamp = datetime.datetime.fromisoformat(record[1:].strip())
+                break
+        return timestamp
+
     def _start(self):
         """Open the image file, read the image header, copy it into a
         dictionary for future reference."""
diff --git a/src/dxtbx/format/FormatCBFMiniEigerPetraP14.py b/src/dxtbx/format/FormatCBFMiniEigerPetraP14.py
index 530948590..36b5cce91 100644
--- a/src/dxtbx/format/FormatCBFMiniEigerPetraP14.py
+++ b/src/dxtbx/format/FormatCBFMiniEigerPetraP14.py
@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import datetime
 import sys
 
 from dxtbx.format.FormatCBFMiniEiger import FormatCBFMiniEiger
@@ -19,11 +20,21 @@ def understand(image_file):
 
         header = FormatCBFMiniEiger.get_cbf_header(image_file)
 
+        # Valid from 22nd May 2021
+        expected_serial = "E-32-0129"
+        if timestamp := FormatCBFMiniEiger._get_timestamp_from_raw_header(header):
+            # We have a timestamp. Let's see what detector we should expect
+
+            # Before 22nd May 2021
+            if timestamp < datetime.datetime(2021, 5, 22):
+                expected_serial = "E-32-0107"
+
+        # Find the line recording detector serial, and check
         for record in header.split("\n"):
             if (
                 "# detector" in record.lower()
                 and "eiger" in record.lower()
-                and "E-32-0107" in record
+                and expected_serial in record
             ):
                 return True
 

From ef5f9abab7c08d46f2bff43ad497e44fe3604048 Mon Sep 17 00:00:00 2001
From: David McDonagh <60879630+toastisme@users.noreply.github.com>
Date: Wed, 2 Aug 2023 14:15:44 +0100
Subject: [PATCH 30/45] Add polychromatic beam (#621)

Add new Beam class "PolychromaticBeam" for polychromatic/multi-wavelength/wide bandpass experiments.

---------

Co-authored-by: DiamondLightSource-build-server <DiamondLightSource-build-server@users.noreply.github.com>
Co-authored-by: Nicholas Devenish <ndevenish@gmail.com>
---
 newsfragments/621.feature            |   1 +
 src/dxtbx/dxtbx_model_ext.pyi        |  29 ++++
 src/dxtbx/model/__init__.py          |   3 +
 src/dxtbx/model/beam.h               | 191 +++++++++++++++++++++++++--
 src/dxtbx/model/beam.py              | 145 +++++++++++++-------
 src/dxtbx/model/boost_python/beam.cc | 130 ++++++++++++++++++
 tests/model/test_beam.py             |  94 ++++++++++++-
 7 files changed, 535 insertions(+), 58 deletions(-)
 create mode 100644 newsfragments/621.feature

diff --git a/newsfragments/621.feature b/newsfragments/621.feature
new file mode 100644
index 000000000..394ff8e74
--- /dev/null
+++ b/newsfragments/621.feature
@@ -0,0 +1 @@
+Add new Beam class "PolychromaticBeam" for polychromatic/multi-wavelength/wide bandpass experiments.
diff --git a/src/dxtbx/dxtbx_model_ext.pyi b/src/dxtbx/dxtbx_model_ext.pyi
index 4cbee00fb..4eb5e1530 100644
--- a/src/dxtbx/dxtbx_model_ext.pyi
+++ b/src/dxtbx/dxtbx_model_ext.pyi
@@ -114,6 +114,35 @@ class Beam(BeamBase):
     def from_dict(data: Dict) -> Beam: ...
     def to_dict(self) -> Dict: ...
 
+class PolychromaticBeam(Beam):
+    @overload
+    def __init__(self, beam: PolychromaticBeam) -> None: ...
+    @overload
+    def __init__(self, direction: Vec3Float) -> None: ...
+    @overload
+    def __init__(
+        self,
+        direction: Vec3Float,
+        divergence: float,
+        sigma_divergence: float,
+        deg: bool = ...,
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        direction: Vec3Float,
+        divergence: float,
+        sigma_divergence: float,
+        polarization_normal: Vec3Float,
+        polarization_fraction: float,
+        flux: float,
+        transmission: float,
+        deg: bool = ...,
+    ) -> None: ...
+    @staticmethod
+    def from_dict(data: Dict) -> PolychromaticBeam: ...
+    def to_dict(self) -> Dict: ...
+
 class CrystalBase:
     @property
     def num_scan_points(self) -> int: ...
diff --git a/src/dxtbx/model/__init__.py b/src/dxtbx/model/__init__.py
index 26a2fc4d8..a7e3ada72 100644
--- a/src/dxtbx/model/__init__.py
+++ b/src/dxtbx/model/__init__.py
@@ -45,6 +45,7 @@
         OffsetPxMmStrategy,
         Panel,
         ParallaxCorrectedPxMmStrategy,
+        PolychromaticBeam,
         PxMmStrategy,
         Scan,
         ScanBase,
@@ -80,6 +81,7 @@
         OffsetPxMmStrategy,
         Panel,
         ParallaxCorrectedPxMmStrategy,
+        PolychromaticBeam,
         PxMmStrategy,
         Scan,
         ScanBase,
@@ -97,6 +99,7 @@
 __all__ = (
     "Beam",
     "BeamBase",
+    "PolychromaticBeam",
     "BeamFactory",
     "Crystal",
     "CrystalBase",
diff --git a/src/dxtbx/model/beam.h b/src/dxtbx/model/beam.h
index 9903341ec..a8d72f33c 100644
--- a/src/dxtbx/model/beam.h
+++ b/src/dxtbx/model/beam.h
@@ -188,7 +188,7 @@ namespace dxtbx { namespace model {
       return direction_;
     }
 
-    double get_wavelength() const {
+    virtual double get_wavelength() const {
       return wavelength_;
     }
 
@@ -207,16 +207,16 @@ namespace dxtbx { namespace model {
       direction_ = direction.normalize();
     }
 
-    void set_wavelength(double wavelength) {
+    virtual void set_wavelength(double wavelength) {
       wavelength_ = wavelength;
     }
 
-    vec3<double> get_s0() const {
+    virtual vec3<double> get_s0() const {
       DXTBX_ASSERT(wavelength_ != 0.0);
       return -direction_ * 1.0 / wavelength_;
     }
 
-    void set_s0(vec3<double> s0) {
+    virtual void set_s0(vec3<double> s0) {
       DXTBX_ASSERT(s0.length() > 0);
       direction_ = -s0.normalize();
       wavelength_ = 1.0 / s0.length();
@@ -293,7 +293,7 @@ namespace dxtbx { namespace model {
       s0_at_scan_points_.clear();
     }
 
-    bool operator==(const BeamBase &rhs) const {
+    virtual bool operator==(const BeamBase &rhs) const {
       double eps = 1.0e-6;
 
       // scan-varying model checks
@@ -327,11 +327,11 @@ namespace dxtbx { namespace model {
                   <= eps;
     }
 
-    bool is_similar_to(const BeamBase &rhs,
-                       double wavelength_tolerance,
-                       double direction_tolerance,
-                       double polarization_normal_tolerance,
-                       double polarization_fraction_tolerance) const {
+    virtual bool is_similar_to(const BeamBase &rhs,
+                               double wavelength_tolerance,
+                               double direction_tolerance,
+                               double polarization_normal_tolerance,
+                               double polarization_fraction_tolerance) const {
       // scan varying model checks
       if (get_num_scan_points() != rhs.get_num_scan_points()) {
         return false;
@@ -375,8 +375,7 @@ namespace dxtbx { namespace model {
 
     friend std::ostream &operator<<(std::ostream &os, const Beam &b);
 
-  private:
-    double wavelength_;
+  protected:
     vec3<double> direction_;
     double divergence_;
     double sigma_divergence_;
@@ -384,6 +383,9 @@ namespace dxtbx { namespace model {
     double polarization_fraction_;
     double flux_;
     double transmission_;
+
+  private:
+    double wavelength_;
     scitbx::af::shared<vec3<double> > s0_at_scan_points_;
   };
 
@@ -403,6 +405,171 @@ namespace dxtbx { namespace model {
     return os;
   }
 
+  class PolychromaticBeam : public Beam {
+  public:
+    PolychromaticBeam() {
+      set_direction(vec3<double>(0.0, 0.0, 1.0));
+      set_divergence(0.0);
+      set_sigma_divergence(0.0);
+      set_polarization_normal(vec3<double>(0.0, 1.0, 0.0));
+      set_polarization_fraction(0.5);
+      set_flux(0);
+      set_transmission(1.0);
+    }
+
+    /**
+     * @param direction The beam direction pointing sample to source
+     */
+    PolychromaticBeam(vec3<double> direction) {
+      DXTBX_ASSERT(direction.length() > 0);
+      direction_ = direction.normalize();
+      set_divergence(0.0);
+      set_sigma_divergence(0.0);
+      set_polarization_normal(vec3<double>(0.0, 1.0, 0.0));
+      set_polarization_fraction(0.5);
+      set_flux(0);
+      set_transmission(1.0);
+    }
+
+    /**
+     * @param direction The beam direction pointing sample to source
+     * @param divergence The beam divergence
+     * @param sigma_divergence The standard deviation of the beam divergence
+     */
+    PolychromaticBeam(vec3<double> direction,
+                      double divergence,
+                      double sigma_divergence) {
+      DXTBX_ASSERT(direction.length() > 0);
+      direction_ = direction.normalize();
+      set_divergence(divergence);
+      set_sigma_divergence(sigma_divergence);
+      set_polarization_normal(vec3<double>(0.0, 1.0, 0.0));
+      set_polarization_fraction(0.5);
+      set_flux(0);
+      set_transmission(1.0);
+    }
+
+    /**
+     * @param direction The beam direction pointing sample to source
+     * @param divergence The beam divergence
+     * @param sigma_divergence The standard deviation of the beam divergence
+     * @param polarization_normal The polarization plane
+     * @param polarization_fraction The polarization fraction
+     * @param flux The beam flux
+     * @param transmission The beam transmission
+     */
+    PolychromaticBeam(vec3<double> direction,
+                      double divergence,
+                      double sigma_divergence,
+                      vec3<double> polarization_normal,
+                      double polarization_fraction,
+                      double flux,
+                      double transmission) {
+      DXTBX_ASSERT(direction.length() > 0);
+      direction_ = direction.normalize();
+      set_divergence(divergence);
+      set_sigma_divergence(sigma_divergence);
+      set_polarization_normal(polarization_normal);
+      set_polarization_fraction(polarization_fraction);
+      set_flux(flux);
+      set_transmission(transmission);
+    }
+
+    double get_wavelength() const {
+      throw DXTBX_ERROR("PolychromaticBeam has no fixed wavelength");
+      return -1.;
+    }
+
+    void set_wavelength(double wavelength) {
+      throw DXTBX_ERROR("PolychromaticBeam has no fixed wavelength");
+    }
+
+    vec3<double> get_s0() const {
+      throw DXTBX_ERROR("PolychromaticBeam has no fixed s0");
+      return vec3<double>(0., 0., 0.);
+    }
+
+    void set_s0(vec3<double> s0) {
+      throw DXTBX_ERROR("PolychromaticBeam has no fixed s0");
+    }
+
+    std::size_t get_num_scan_points() const {
+      throw DXTBX_ERROR("PolychromaticBeam has no fixed s0");
+      return 1;
+    }
+
+    void set_s0_at_scan_points(const scitbx::af::const_ref<vec3<double> > &s0) {
+      throw DXTBX_ERROR("PolychromaticBeam has no fixed s0");
+    }
+
+    scitbx::af::shared<vec3<double> > get_s0_at_scan_points() const {
+      throw DXTBX_ERROR("PolychromaticBeam has no fixed s0");
+      return scitbx::af::shared<vec3<double> >(1, (0., 0., 0.));
+    }
+
+    vec3<double> get_s0_at_scan_point(std::size_t index) const {
+      throw DXTBX_ERROR("PolychromaticBeam has no fixed s0");
+      return vec3<double>(0., 0., 0.);
+    }
+
+    void reset_scan_points() {
+      throw DXTBX_ERROR("PolychromaticBeam has no fixed s0");
+    }
+
+    bool operator==(const BeamBase &rhs) const {
+      double eps = 1.0e-6;
+
+      return std::abs(angle_safe(direction_, rhs.get_sample_to_source_direction()))
+               <= eps
+             && std::abs(divergence_ - rhs.get_divergence()) <= eps
+             && std::abs(sigma_divergence_ - rhs.get_sigma_divergence()) <= eps
+             && std::abs(
+                  angle_safe(polarization_normal_, rhs.get_polarization_normal()))
+                  <= eps
+             && std::abs(polarization_fraction_ - rhs.get_polarization_fraction())
+                  <= eps;
+    }
+
+    bool is_similar_to(const BeamBase &rhs,
+                       double wavelength_tolerance,
+                       double direction_tolerance,
+                       double polarization_normal_tolerance,
+                       double polarization_fraction_tolerance) const {
+      return is_similar_to(rhs,
+                           direction_tolerance,
+                           polarization_normal_tolerance,
+                           polarization_fraction_tolerance);
+    }
+
+    bool is_similar_to(const BeamBase &rhs,
+                       double direction_tolerance,
+                       double polarization_normal_tolerance,
+                       double polarization_fraction_tolerance) const {
+      return std::abs(angle_safe(direction_, rhs.get_sample_to_source_direction()))
+               <= direction_tolerance
+             && std::abs(
+                  angle_safe(polarization_normal_, rhs.get_polarization_normal()))
+                  <= polarization_normal_tolerance
+             && std::abs(polarization_fraction_ - rhs.get_polarization_fraction())
+                  <= polarization_fraction_tolerance;
+    }
+  };
+
+  /** Print beam information */
+  inline std::ostream &operator<<(std::ostream &os, const PolychromaticBeam &b) {
+    os << "Beam:\n";
+    os << "    sample to source direction : "
+       << b.get_sample_to_source_direction().const_ref() << "\n";
+    os << "    divergence: " << b.get_divergence() << "\n";
+    os << "    sigma divergence: " << b.get_sigma_divergence() << "\n";
+    os << "    polarization normal: " << b.get_polarization_normal().const_ref()
+       << "\n";
+    os << "    polarization fraction: " << b.get_polarization_fraction() << "\n";
+    os << "    flux: " << b.get_flux() << "\n";
+    os << "    transmission: " << b.get_transmission() << "\n";
+    return os;
+  }
+
 }}  // namespace dxtbx::model
 
 #endif  // DXTBX_MODEL_BEAM_H
diff --git a/src/dxtbx/model/beam.py b/src/dxtbx/model/beam.py
index 870dd9c4b..7c94cffa7 100644
--- a/src/dxtbx/model/beam.py
+++ b/src/dxtbx/model/beam.py
@@ -1,15 +1,18 @@
 from __future__ import annotations
 
 import math
+from typing import Tuple
 
 import pycbf
 
 import libtbx.phil
 
 try:
-    from ..dxtbx_model_ext import Beam
+    from ..dxtbx_model_ext import Beam, PolychromaticBeam
 except ModuleNotFoundError:
-    from dxtbx_model_ext import Beam  # type: ignore
+    from dxtbx_model_ext import Beam, PolychromaticBeam  # type: ignore
+
+Vec3Float = Tuple[float, float, float]
 
 beam_phil_scope = libtbx.phil.parse(
     """
@@ -17,6 +20,10 @@
     .expert_level = 1
     .short_caption = "Beam overrides"
   {
+    type = *monochromatic polychromatic
+      .type = choice
+      .help = "Override the beam type"
+      .short_caption = "beam_type"
 
     wavelength = None
       .type = float
@@ -27,6 +34,14 @@
       .help = "Override the sample to source direction"
       .short_caption = "Sample to source direction"
 
+    divergence = None
+        .type = float
+        .help = "Override the beam divergence"
+
+    sigma_divergence = None
+        .type = float
+        .help = "Override the beam sigma divergence"
+
     polarization_normal = None
       .type = floats(size=3)
       .help = "Override the polarization normal"
@@ -57,62 +72,77 @@ class BeamFactory:
     will be used, otherwise simplified descriptions can be applied."""
 
     @staticmethod
-    def from_phil(params, reference=None):
+    def from_phil(
+        params: libtbx.phil.scope_extract,
+        reference: Beam | PolychromaticBeam = None,
+    ) -> Beam | PolychromaticBeam:
         """
         Convert the phil parameters into a beam model
-
         """
+
         # Check the input
         if reference is None:
-            beam = Beam()
+            beam = (
+                PolychromaticBeam() if params.beam.type == "polychromatic" else Beam()
+            )
         else:
             beam = reference
 
         # Set the parameters
-        if params.beam.wavelength is not None:
-            beam.set_wavelength(params.beam.wavelength)
-        elif reference is None:
-            raise RuntimeError("No wavelength set")
+        if params.beam.type == "monochromatic":
+            if params.beam.wavelength is not None:
+                beam.set_wavelength(params.beam.wavelength)
+            elif reference is None:
+                raise RuntimeError("No wavelength set")
         if params.beam.direction is not None:
             beam.set_direction(params.beam.direction)
         elif reference is None:
             raise RuntimeError("No beam direction set")
+
+        if params.beam.divergence is not None:
+            beam.set_divergence(params.beam.divergence)
+        if params.beam.sigma_divergence is not None:
+            beam.set_sigma_divergence(params.beam.sigma_divergence)
         if params.beam.polarization_normal is not None:
             beam.set_polarization_normal(params.beam.polarization_normal)
         if params.beam.polarization_fraction is not None:
             beam.set_polarization_fraction(params.beam.polarization_fraction)
+        if params.beam.transmission is not None:
+            beam.set_transmission(params.beam.transmission)
+        if params.beam.flux is not None:
+            beam.set_flux(params.beam.flux)
 
         return beam
 
     @staticmethod
-    def from_dict(d, t=None):
-        """Convert the dictionary to a beam model
+    def from_dict(dict: dict, template: dict = None) -> Beam | PolychromaticBeam:
+        """Convert the dictionary to a beam model"""
 
-        Params:
-            d The dictionary of parameters
-            t The template dictionary to use
+        if template is not None:
+            if "__id__" in dict and "__id__" in template:
+                assert (
+                    dict["__id__"] == template["__id__"]
+                ), "Beam and template dictionaries are not the same type."
 
-        Returns:
-            The beam model
-        """
-        if d is None and t is None:
+        if dict is None and template is None:
             return None
-        joint = t.copy() if t else {}
-        joint.update(d)
+        joint = template.copy() if template else {}
+        joint.update(dict)
 
         # Create the model from the joint dictionary
+        if joint.get("__id__") == "polychromatic":
+            return PolychromaticBeam.from_dict(joint)
         return Beam.from_dict(joint)
 
     @staticmethod
     def make_beam(
-        sample_to_source=None,
-        wavelength=None,
-        s0=None,
-        unit_s0=None,
-        divergence=None,
-        sigma_divergence=None,
-    ):
-
+        sample_to_source: Vec3Float = None,
+        wavelength: float = None,
+        s0: Vec3Float = None,
+        unit_s0: Vec3Float = None,
+        divergence: float = None,
+        sigma_divergence: float = None,
+    ) -> Beam:
         if divergence is None or sigma_divergence is None:
             divergence = 0.0
             sigma_divergence = 0.0
@@ -137,19 +167,41 @@ def make_beam(
             assert s0
             return Beam(tuple(map(float, s0)))
 
+    @staticmethod
+    def make_polychromatic_beam(
+        direction: Vec3Float,
+        divergence: float = 0.0,
+        sigma_divergence: float = 0.0,
+        polarization_normal: Vec3Float = (0.0, 1.0, 0.0),
+        polarization_fraction: float = 0.5,
+        flux: float = 0.0,
+        transmission: float = 1.0,
+        deg: bool = True,
+    ) -> PolychromaticBeam:
+        return PolychromaticBeam(
+            tuple(map(float, direction)),
+            float(divergence),
+            float(sigma_divergence),
+            tuple(map(float, polarization_normal)),
+            float(polarization_fraction),
+            float(flux),
+            float(transmission),
+            bool(deg),
+        )
+
     @staticmethod
     def make_polarized_beam(
-        sample_to_source=None,
-        wavelength=None,
-        s0=None,
-        unit_s0=None,
-        polarization=None,
-        polarization_fraction=None,
-        divergence=None,
-        sigma_divergence=None,
-        flux=None,
-        transmission=None,
-    ):
+        sample_to_source: Vec3Float = None,
+        wavelength: float = None,
+        s0: Vec3Float = None,
+        unit_s0: Vec3Float = None,
+        polarization: Vec3Float = None,
+        polarization_fraction: float = None,
+        divergence: float = None,
+        sigma_divergence: float = None,
+        flux: float = None,
+        transmission: float = None,
+    ) -> Beam:
         assert polarization
         assert 0.0 <= polarization_fraction <= 1.0
 
@@ -199,7 +251,7 @@ def make_polarized_beam(
             )
 
     @staticmethod
-    def simple(wavelength):
+    def simple(wavelength: float) -> Beam:
         """Construct a beam object on the principle that the beam is aligned
         with the +z axis, as is quite normal. Also assume the beam has
         polarization fraction 0.999 and is polarized in the x-z plane, unless
@@ -219,7 +271,7 @@ def simple(wavelength):
             )
 
     @staticmethod
-    def simple_directional(sample_to_source, wavelength):
+    def simple_directional(sample_to_source: Vec3Float, wavelength: float) -> Beam:
         """Construct a beam with direction and wavelength."""
 
         if wavelength > 0.05:
@@ -236,8 +288,11 @@ def simple_directional(sample_to_source, wavelength):
 
     @staticmethod
     def complex(
-        sample_to_source, polarization_fraction, polarization_plane_normal, wavelength
-    ):
+        sample_to_source: Vec3Float,
+        polarization_fraction: float,
+        polarization_plane_normal: Vec3Float,
+        wavelength: float,
+    ) -> Beam:
         """Full access to the constructor for cases where we do know everything
         that we need..."""
 
@@ -249,7 +304,7 @@ def complex(
         )
 
     @staticmethod
-    def imgCIF(cif_file):
+    def imgCIF(cif_file: str) -> Beam:
         """Initialize a detector model from an imgCIF file. N.B. the
         definition of the polarization plane is not completely helpful
         in this - it is the angle between the polarization plane and the
@@ -263,7 +318,7 @@ def imgCIF(cif_file):
         return result
 
     @staticmethod
-    def imgCIF_H(cbf_handle):
+    def imgCIF_H(cbf_handle: pycbf.cbf_handle_struct) -> Beam:
         """Initialize a detector model from an imgCIF file. N.B. the
         definition of the polarization plane is not completely helpful
         in this - it is the angle between the polarization plane and the
diff --git a/src/dxtbx/model/boost_python/beam.cc b/src/dxtbx/model/boost_python/beam.cc
index bf709d526..45bec9f95 100644
--- a/src/dxtbx/model/boost_python/beam.cc
+++ b/src/dxtbx/model/boost_python/beam.cc
@@ -25,6 +25,8 @@ namespace dxtbx { namespace model { namespace boost_python {
     using scitbx::deg_as_rad;
     using scitbx::rad_as_deg;
 
+    // Beam
+
     std::string beam_to_string(const Beam &beam) {
       std::stringstream ss;
       ss << beam;
@@ -180,6 +182,7 @@ namespace dxtbx { namespace model { namespace boost_python {
   template <>
   boost::python::dict to_dict<Beam>(const Beam &obj) {
     boost::python::dict result;
+    result["__id__"] = "monochromatic";
     result["direction"] = obj.get_sample_to_source_direction();
     result["wavelength"] = obj.get_wavelength();
     result["divergence"] = rad_as_deg(obj.get_divergence());
@@ -221,6 +224,102 @@ namespace dxtbx { namespace model { namespace boost_python {
     return b;
   }
 
+  // PolychromaticBeam
+
+  std::string PolychromaticBeam_to_string(const PolychromaticBeam &beam) {
+    std::stringstream ss;
+    ss << beam;
+    return ss.str();
+  }
+
+  struct PolychromaticBeamPickleSuite : boost::python::pickle_suite {
+    static boost::python::tuple getinitargs(const PolychromaticBeam &obj) {
+      return boost::python::make_tuple(obj.get_sample_to_source_direction(),
+                                       obj.get_divergence(),
+                                       obj.get_sigma_divergence(),
+                                       obj.get_polarization_normal(),
+                                       obj.get_polarization_fraction(),
+                                       obj.get_flux(),
+                                       obj.get_transmission());
+    }
+  };
+
+  static PolychromaticBeam *make_PolychromaticBeam(vec3<double> direction) {
+    return new PolychromaticBeam(direction);
+  }
+
+  static PolychromaticBeam *make_PolychromaticBeam_w_divergence(vec3<double> direction,
+                                                                double divergence,
+                                                                double sigma_divergence,
+                                                                bool deg) {
+    PolychromaticBeam *beam = NULL;
+    if (deg) {
+      beam = new PolychromaticBeam(
+        direction, deg_as_rad(divergence), deg_as_rad(sigma_divergence));
+    } else {
+      beam = new PolychromaticBeam(direction, divergence, sigma_divergence);
+    }
+    return beam;
+  }
+
+  static PolychromaticBeam *make_PolychromaticBeam_w_all(
+    vec3<double> direction,
+    double divergence,
+    double sigma_divergence,
+    vec3<double> polarization_normal,
+    double polarization_fraction,
+    double flux,
+    double transmission,
+    bool deg) {
+    PolychromaticBeam *beam = NULL;
+    if (deg) {
+      beam = new PolychromaticBeam(direction,
+                                   deg_as_rad(divergence),
+                                   deg_as_rad(sigma_divergence),
+                                   polarization_normal,
+                                   polarization_fraction,
+                                   flux,
+                                   transmission);
+    } else {
+      beam = new PolychromaticBeam(direction,
+                                   divergence,
+                                   sigma_divergence,
+                                   polarization_normal,
+                                   polarization_fraction,
+                                   flux,
+                                   transmission);
+    }
+    return beam;
+  }
+
+  template <>
+  boost::python::dict to_dict<PolychromaticBeam>(const PolychromaticBeam &obj) {
+    boost::python::dict result;
+    result["__id__"] = "polychromatic";
+    result["direction"] = obj.get_sample_to_source_direction();
+    result["divergence"] = rad_as_deg(obj.get_divergence());
+    result["sigma_divergence"] = rad_as_deg(obj.get_sigma_divergence());
+    result["polarization_normal"] = obj.get_polarization_normal();
+    result["polarization_fraction"] = obj.get_polarization_fraction();
+    result["flux"] = obj.get_flux();
+    result["transmission"] = obj.get_transmission();
+    return result;
+  }
+
+  template <>
+  PolychromaticBeam *from_dict<PolychromaticBeam>(boost::python::dict obj) {
+    PolychromaticBeam *b = new PolychromaticBeam(
+      boost::python::extract<vec3<double> >(obj["direction"]),
+      deg_as_rad(boost::python::extract<double>(obj.get("divergence", 0.0))),
+      deg_as_rad(boost::python::extract<double>(obj.get("sigma_divergence", 0.0))),
+      boost::python::extract<vec3<double> >(
+        obj.get("polarization_normal", vec3<double>(0.0, 1.0, 0.0))),
+      boost::python::extract<double>(obj.get("polarization_fraction", 0.999)),
+      boost::python::extract<double>(obj.get("flux", 0)),
+      boost::python::extract<double>(obj.get("transmission", 1)));
+    return b;
+  }
+
   void export_beam() {
     using namespace beam_detail;
 
@@ -305,6 +404,37 @@ namespace dxtbx { namespace model { namespace boost_python {
       .staticmethod("from_dict")
       .def_pickle(BeamPickleSuite());
 
+    class_<PolychromaticBeam, std::shared_ptr<PolychromaticBeam>, bases<Beam> >(
+      "PolychromaticBeam")
+      .def("__init__",
+           make_constructor(
+             &make_PolychromaticBeam, default_call_policies(), (arg("direction"))))
+      .def("__init__",
+           make_constructor(&make_PolychromaticBeam_w_divergence,
+                            default_call_policies(),
+                            (arg("direction"),
+                             arg("divergence"),
+                             arg("sigma_divergence"),
+                             arg("deg") = true)))
+      .def("__init__",
+           make_constructor(&make_PolychromaticBeam_w_all,
+                            default_call_policies(),
+                            (arg("direction"),
+                             arg("divergence"),
+                             arg("sigma_divergence"),
+                             arg("polarization_normal"),
+                             arg("polarization_fraction"),
+                             arg("flux"),
+                             arg("transmission"),
+                             arg("deg") = true)))
+      .def("__str__", &PolychromaticBeam_to_string)
+      .def("to_dict", &to_dict<PolychromaticBeam>)
+      .def("from_dict",
+           &from_dict<PolychromaticBeam>,
+           return_value_policy<manage_new_object>())
+      .staticmethod("from_dict")
+      .def_pickle(PolychromaticBeamPickleSuite());
+
     scitbx::af::boost_python::flex_wrapper<Beam>::plain("flex_Beam");
   }
 
diff --git a/tests/model/test_beam.py b/tests/model/test_beam.py
index 7f1ef5b9d..2f83a790c 100644
--- a/tests/model/test_beam.py
+++ b/tests/model/test_beam.py
@@ -5,7 +5,7 @@
 from libtbx.phil import parse
 from scitbx import matrix
 
-from dxtbx.model import Beam
+from dxtbx.model import Beam, PolychromaticBeam
 from dxtbx.model.beam import BeamFactory, beam_phil_scope
 
 
@@ -176,3 +176,95 @@ def test_beam_object_comparison():
 def test_beam_self_serialization():
     beam = Beam()
     assert beam == BeamFactory.from_dict(beam.to_dict())
+
+
+def test_polychromatic_beam_from_phil():
+    params = beam_phil_scope.fetch(
+        parse(
+            """
+    beam {
+      type = polychromatic
+      direction = (0., 0., 1.)
+      divergence = 0.2
+      sigma_divergence = 0.3
+      polarization_normal = (0., -1., 0.)
+      polarization_fraction = .65
+      transmission = .5
+      flux = .75
+    }
+    """
+        )
+    ).extract()
+
+    beam = BeamFactory.from_phil(params)
+    assert isinstance(beam, PolychromaticBeam)
+
+    assert beam.get_sample_to_source_direction() == pytest.approx((0.0, 0.0, 1.0))
+    assert beam.get_divergence() == pytest.approx(0.2)
+    assert beam.get_sigma_divergence() == pytest.approx(0.3)
+    assert beam.get_polarization_normal() == pytest.approx((0.0, -1.0, 0.0))
+    assert beam.get_polarization_fraction() == pytest.approx(0.65)
+    assert beam.get_transmission() == pytest.approx(0.5)
+    assert beam.get_flux() == pytest.approx(0.75)
+
+
+def test_polychromatic_beam_from_dict():
+    beam = PolychromaticBeam()
+    assert beam == BeamFactory.from_dict(beam.to_dict())
+
+
+def test_make_polychromatic_beam():
+
+    direction = (0.0, 0.0, 1.0)
+    divergence = 0.2
+    sigma_divergence = 0.3
+    polarization_normal = (0.0, -1.0, 0.0)
+    polarization_fraction = 0.65
+    transmission = 0.5
+    flux = 0.75
+
+    beam = BeamFactory.make_polychromatic_beam(
+        direction=direction,
+        divergence=divergence,
+        sigma_divergence=sigma_divergence,
+        polarization_normal=polarization_normal,
+        polarization_fraction=polarization_fraction,
+        transmission=transmission,
+        flux=flux,
+    )
+
+    assert beam.get_sample_to_source_direction() == pytest.approx((0.0, 0.0, 1.0))
+    assert beam.get_divergence() == pytest.approx(0.2)
+    assert beam.get_sigma_divergence() == pytest.approx(0.3)
+    assert beam.get_polarization_normal() == pytest.approx((0.0, -1.0, 0.0))
+    assert beam.get_polarization_fraction() == pytest.approx(0.65)
+    assert beam.get_transmission() == pytest.approx(0.5)
+    assert beam.get_flux() == pytest.approx(0.75)
+
+
+def test_polychromatic_beam_wavelength_guards():
+    beam = PolychromaticBeam()
+    with pytest.raises(RuntimeError):
+        _ = beam.get_wavelength()
+    with pytest.raises(RuntimeError):
+        _ = beam.get_s0()
+    with pytest.raises(RuntimeError):
+        _ = beam.get_num_scan_points()
+    with pytest.raises(RuntimeError):
+        _ = beam.get_s0_at_scan_points()
+    with pytest.raises(RuntimeError):
+        _ = beam.get_s0_at_scan_point(0)
+    with pytest.raises(RuntimeError):
+        beam.reset_scan_points()
+    with pytest.raises(RuntimeError):
+        beam.set_wavelength(1.0)
+    with pytest.raises(RuntimeError):
+        beam.set_s0((0.0, 0.0, 0.1))
+
+
+def test_polychromatic_beam_str():
+    beam = PolychromaticBeam()
+    assert (
+        beam.__str__()
+        == "Beam:\n    sample to source direction : {0,0,1}\n    divergence: 0\n    sigma divergence: 0\n    polarization normal: {0,1,0}\n    polarization fraction: 0.5\n    flux: 0\n    transmission: 1\n"
+    )

From 5c27616fc62890a751cbf33468a0a446c8f16a3d Mon Sep 17 00:00:00 2001
From: David Waterman <dagewa@users.noreply.github.com>
Date: Thu, 3 Aug 2023 16:59:27 +0100
Subject: [PATCH 31/45] FormatROD - Multi-axis goniometer, faster decompression
 (#653)

Co-authored-by: Takanori Nakane <nakane.t@gmail.com>
---
 newsfragments/653.feature             |   1 +
 src/dxtbx/boost_python/compression.cc | 126 +++++++++++++++++
 src/dxtbx/boost_python/compression.h  |   6 +
 src/dxtbx/boost_python/ext.cpp        |  21 +++
 src/dxtbx/format/FormatROD.py         | 196 ++++++++++++++++----------
 5 files changed, 272 insertions(+), 78 deletions(-)
 create mode 100644 newsfragments/653.feature

diff --git a/newsfragments/653.feature b/newsfragments/653.feature
new file mode 100644
index 000000000..22c7d8c0e
--- /dev/null
+++ b/newsfragments/653.feature
@@ -0,0 +1 @@
+``FormatROD``: include support for multi-axis goniometers and faster decompression.
diff --git a/src/dxtbx/boost_python/compression.cc b/src/dxtbx/boost_python/compression.cc
index ff5d91c5f..6957d1bd1 100644
--- a/src/dxtbx/boost_python/compression.cc
+++ b/src/dxtbx/boost_python/compression.cc
@@ -164,3 +164,129 @@ unsigned int dxtbx::boost_python::cbf_decompress(const char *packed,
 
   return values - original;
 }
+
+inline uint32_t read_uint32_from_bytearray(const char *buf) {
+  // `char` can be signed or unsigned depending on the platform.
+  // For bit shift operations, we need unsigned values.
+  // If `char` on the platform is signed, converting directly to "unsigned int" can
+  // produce huge numbers because modulo 2^n is taken by the integral conversion
+  // rules. Thus, we have to explicitly cast to `unsigned char` first.
+  // Then the automatic integral promotion converts them to `int`.
+  // Note that the unsigned to signed conversion is implementation-dependent
+  // and might not produce the intended result if two's complement is not used.
+  // Fortunately, DIALS targets only two's complement.
+  //    https://github.com/cctbx/dxtbx/issues/11#issuecomment-1657809645
+  // Moreover, C++20 standarized this:
+  //    https://stackoverflow.com/questions/54947427/going-from-signed-integers-to-unsigned-integers-and-vice-versa-in-c20
+
+  return ((unsigned char)buf[0]) | (((unsigned char)buf[1]) << 8)
+         | (((unsigned char)buf[2]) << 16) | (((unsigned char)buf[3]) << 24);
+}
+
+inline uint16_t read_uint16_from_bytearray(const char *buf) {
+  return ((unsigned char)buf[0]) | ((unsigned char)buf[1] << 8);
+}
+
+void dxtbx::boost_python::rod_TY6_decompress(int *const ret,
+                                             const char *const buf_data,
+                                             const char *const buf_offsets,
+                                             const int slow,
+                                             const int fast) {
+  const size_t BLOCKSIZE = 8;             // Codes below assume this is at most 8
+  const signed int SHORT_OVERFLOW = 127;  // after 127 is subtracted
+  const signed int LONG_OVERFLOW = 128;
+
+  const size_t nblock = (fast - 1) / (BLOCKSIZE * 2);
+  const size_t nrest = (fast - 1) % (BLOCKSIZE * 2);
+
+  for (size_t iy = 0; iy < slow; iy++) {
+    size_t ipos = read_uint32_from_bytearray(buf_offsets + iy * sizeof(uint32_t));
+    size_t opos = fast * iy;
+
+    // Values from -127 to +126 (inclusive) are stored with an offset of 127
+    // as 0 to 253. 254 and 255 mark short and long overflows.
+    // Other values ("overflows") are represented in two's complement.
+
+    int firstpx = (unsigned char)buf_data[ipos++] - 127;
+    if (firstpx == LONG_OVERFLOW) {
+      // See comments in read_uint32_from_bytearray() about
+      // the safety of the unsigned to signed conversion.
+      firstpx = (signed int)read_uint32_from_bytearray(buf_data + ipos);
+      ipos += 4;
+    } else if (firstpx == SHORT_OVERFLOW) {
+      firstpx = (signed short)read_uint16_from_bytearray(buf_data + ipos);
+      ipos += 2;
+    }
+    ret[opos++] = firstpx;
+
+    // For every two blocks
+    for (int k = 0; k < nblock; k++) {
+      const size_t bittypes = buf_data[ipos++];
+      const size_t nbits[2] = {bittypes & 15, (bittypes >> 4) & 15};
+
+      // One pixel is stored using `nbit` bits.
+      // Although `nbit` itself is stored using 4 bits,
+      // only values 1 (0001b) to 8 (1000b) are allowed.
+      // Negative values are encoded as follows. (Not 2's complement!)
+      // - When nbit = 1, the pixel value is 0 or 1
+      // - When nbit = 2, the pixel value is -1, 0, 1, 2
+      // - When nbit = 3, the pixel value is -3, -2, 1, 0, 1, 2, 3, 4
+      // - When nbit - 8, the pixel value is -127, -126, ...,
+      //   127 (== // SHORT_OVERFLOW), 128 (== LONG_OVERFLOW)
+
+      // Load values
+      for (int i = 0; i < 2; i++) {
+        const size_t nbit = nbits[i];
+        assert(nbit >= 0 && nbit <= 8);
+
+        int zero_at = 0;
+        if (nbit > 1) {
+          zero_at = (1 << (nbit - 1)) - 1;
+        }
+
+        // Since nbit is at most 8, 8 * 8 (= BLOCKSIZE) = 64 bits are sufficient.
+        unsigned long long v = 0;
+        for (int j = 0; j < nbit; j++) {
+          // Implicit promotion is only up to 32 bits, not 64 bits so we have to be
+          // explicit.
+          v |= (long long)((unsigned char)buf_data[ipos++]) << (BLOCKSIZE * j);
+        }
+
+        const unsigned long long mask = (1 << nbit) - 1;
+        for (int j = 0; j < BLOCKSIZE; j++) {
+          ret[opos++] = ((v >> (nbit * j)) & mask) - zero_at;
+        }
+      }
+
+      // Apply differences. Load more values when overflown.
+      for (size_t i = opos - 2 * BLOCKSIZE; i < opos; i++) {
+        int offset = ret[i];
+
+        if (offset == LONG_OVERFLOW) {
+          offset = (signed int)read_uint32_from_bytearray(buf_data + ipos);
+          ipos += 4;
+        } else if (offset == SHORT_OVERFLOW) {
+          offset = (signed short)read_uint16_from_bytearray(buf_data + ipos);
+          ipos += 2;
+        }
+
+        ret[i] = offset + ret[i - 1];
+      }
+    }
+
+    for (int i = 0; i < nrest; i++) {
+      int offset = (unsigned char)buf_data[ipos++] - 127;
+
+      if (offset == LONG_OVERFLOW) {
+        offset = (signed int)read_uint32_from_bytearray(buf_data + ipos);
+        ipos += 4;
+      } else if (offset == SHORT_OVERFLOW) {
+        offset = (signed short)read_uint16_from_bytearray(buf_data + ipos);
+        ipos += 2;
+      }
+
+      ret[opos] = ret[opos - 1] + offset;
+      opos++;
+    }
+  }
+}
diff --git a/src/dxtbx/boost_python/compression.h b/src/dxtbx/boost_python/compression.h
index d046b3965..b50b5e73f 100644
--- a/src/dxtbx/boost_python/compression.h
+++ b/src/dxtbx/boost_python/compression.h
@@ -6,6 +6,12 @@
 namespace dxtbx { namespace boost_python {
   unsigned int cbf_decompress(const char*, std::size_t, int*, const std::size_t);
   std::vector<char> cbf_compress(const int*, const std::size_t&);
+  // Decompress Rigaku Oxford diffractometer TY6 compression
+  void rod_TY6_decompress(int* const,
+                          const char* const,
+                          const char* const,
+                          const int,
+                          const int);
 }}  // namespace dxtbx::boost_python
 
 #endif
diff --git a/src/dxtbx/boost_python/ext.cpp b/src/dxtbx/boost_python/ext.cpp
index 5273eb69e..70c69c06b 100644
--- a/src/dxtbx/boost_python/ext.cpp
+++ b/src/dxtbx/boost_python/ext.cpp
@@ -193,6 +193,24 @@ namespace dxtbx { namespace boost_python {
     return PyBytes_FromStringAndSize(&*packed.begin(), packed.size());
   }
 
+  // Python entry point to decompress Rigaku Oxford Diffractometer TY6 compression
+  scitbx::af::flex_int uncompress_rod_TY6(const boost::python::object &data,
+                                          const boost::python::object &offsets,
+                                          const int &slow,
+                                          const int &fast) {
+    // Cannot I extract const char* directly?
+    std::string str_data = boost::python::extract<std::string>(data);
+    std::string str_offsets = boost::python::extract<std::string>(offsets);
+
+    scitbx::af::flex_int z((scitbx::af::flex_grid<>(slow, fast)),
+                           scitbx::af::init_functor_null<int>());
+
+    dxtbx::boost_python::rod_TY6_decompress(
+      z.begin(), str_data.c_str(), str_offsets.c_str(), slow, fast);
+
+    return z;
+  }
+
   void init_module() {
     using namespace boost::python;
     def("read_uint8", read_uint8, (arg("file"), arg("count")));
@@ -206,6 +224,9 @@ namespace dxtbx { namespace boost_python {
     def("is_big_endian", is_big_endian);
     def("uncompress", &uncompress, (arg_("packed"), arg_("slow"), arg_("fast")));
     def("compress", &compress);
+    def("uncompress_rod_TY6",
+        &uncompress_rod_TY6,
+        (arg_("data"), arg_("offsets"), arg_("slow"), arg_("fast")));
   }
 
   BOOST_PYTHON_MODULE(dxtbx_ext) {
diff --git a/src/dxtbx/format/FormatROD.py b/src/dxtbx/format/FormatROD.py
index 417f99f1a..5ce6bd814 100644
--- a/src/dxtbx/format/FormatROD.py
+++ b/src/dxtbx/format/FormatROD.py
@@ -12,9 +12,7 @@
 from __future__ import annotations
 
 __author__ = "David Waterman, Takanori Nakane"
-__copyright__ = (
-    "Copyright 2018 United Kingdom Research and Innovation & 2022 Takanori Nakane"
-)
+__copyright__ = "Copyright 2018-2023 United Kingdom Research and Innovation & 2022-2023 Takanori Nakane"
 __license__ = "BSD 3-clause"
 
 import re
@@ -23,7 +21,9 @@
 import numpy as np
 
 from scitbx.array_family import flex
+from scitbx.math import r3_rotation_axis_and_angle_as_matrix
 
+from dxtbx.ext import uncompress_rod_TY6
 from dxtbx.format.Format import Format
 
 
@@ -153,7 +153,7 @@ def _read_binary_header(
             f.seek(offset + general_nbytes + special_nbytes + 640)
             # detector rotation in degrees along e1, e2, e3
             detector_rotns = struct.unpack("<ddd", f.read(24))
-            # FIXME: direct beam position when all angles are zero?
+            # direct beam position when all angles are zero (FIXME: not completely sure)
             origin_px_x, origin_px_y = struct.unpack("<dd", f.read(16))
             # alpha and beta are angles between KAPPA(=CHI) and THETA, and e3.
             angles_in_deg = struct.unpack(
@@ -198,24 +198,86 @@ def _start(self):
         self._txt_header = self._read_ascii_header(self._image_file)
         self._bin_header = self._read_binary_header(self._image_file)
 
+        self._gonio_start_angles = (
+            np.array(self._bin_header["start_angles_steps"])
+            * np.array(self._bin_header["step_to_rad"])
+            / np.pi
+            * 180
+        )
+        self._gonio_end_angles = (
+            np.array(self._bin_header["end_angles_steps"])
+            * np.array(self._bin_header["step_to_rad"])
+            / np.pi
+            * 180
+        )
+
+        self._scan_axis = -1
+        for axis in [0, 3]:  # 0 - OMEGA, 3 - PHI; the default is omega scan
+            if self._gonio_start_angles[axis] != self._gonio_end_angles[axis]:
+                self._scan_axis = axis
+                break
+
         return
 
-    # Rigaku/Oxford Geometry:
-    # - e3: parallel to OMEGA (and THETA)
-    # - e1: crystal to source
+    # Rigaku/Oxford coordinate system:
+    # - e3: parallel to OMEGA (and THETA), upwards
+    # - e1: along the beam, from the crystal to the source
     # - e2: completes the right handed coordinate system
+    #
+    # The detector origin is at the lower left corner looking from the crystal (or the source).
+    # - slow axis: (roughly) vertical and towards the ceiling
+    # - fast axis: (roughly) horizontal and towards right (at THETA = 0)
+    #
+    # Importantly, positive THETA, OMEGA, PHI rotations are CLOCKWISE looking down from the ceiling,
+    # while positive CHI=KAPPA rotation is CLOCKWISE looking from the detector at OMEGA=0, THETA=0.
+    # See specifications uploaded in https://github.com/cctbx/dxtbx/issues/11#issue-434741559.
+    #
+    # In this dxtbx coordinate system:
+    # - Z: along the beam, from the crystal to the source
+    # - Y: vertical, towards the ceiling
+    # - X: completes the right handed coordinate system
+    # Thus, Z = e1, Y = e3, X = e2.
+    #
+    # The coordinate system of XDS.INP generated by CrysAlisPro:
+    # - Z: along the beam, from the source to the crystal
+    # - Y: vertical, towards the ground
+    # - X: completes the right handed coordinate system
+    # Thus, Y and Z are pointing the opposite directions from DIALS.
 
     def get_goniometer(self, index=None):
         return Format.get_goniometer(self)
 
     def _goniometer(self):
+        if self._scan_axis == -1:
+            raise NotImplementedError("Still shots not implemented yet.")
+        elif self._scan_axis == 0:  # OMEGA
+            dxtbx_scan_axis = 2
+        elif self._scan_axis == 3:  # PHI
+            dxtbx_scan_axis = 0
+        else:
+            pass  # should not happen
+
         # FIXME: sometimes XDS.INP generated by CrysAlisPro has
-        #  tiny deviationis from (0, 1, 0). I don't know how to calculate it.
-        # DIALS' third axis points to the opposite of XDS; thus the direction will be opposite.
-        direction = (0.0, -1.0, 0.0)
+        #  tiny deviations from (0, 1, 0). I don't know how to calculate it.
+        alpha_rad = self._bin_header["angles_in_deg"][0] * np.pi / 180
+        axes = flex.vec3_double(
+            ((0, -1, 0), (0, -np.cos(alpha_rad), np.sin(alpha_rad)), (0, -1, 0))
+        )
 
-        # FIXME: represent kappa axis in fixed_rotation
-        return self._goniometer_factory.known_axis(direction)
+        # angles[self._scan_axis] is not used anyway
+        # angles are in degrees!
+        angles = flex.double(
+            (
+                self._gonio_start_angles[3],
+                self._gonio_start_angles[2],
+                self._gonio_start_angles[0],
+            )
+        )
+        names = flex.std_string(("PHI", "KAPPA=CHI", "OMEGA"))
+
+        return self._goniometer_factory.make_multi_axis_goniometer(
+            axes, angles, names, dxtbx_scan_axis
+        )
 
     def get_beam(self, index=None):
         return Format.get_beam(self)
@@ -232,77 +294,49 @@ def get_detector(self, index=None):
         return Format.get_detector(self)
 
     def _detector(self):
-        gonio_angles = np.array(self._bin_header["start_angles_steps"]) * np.array(
-            self._bin_header["step_to_rad"]
-        )
+        theta_rad = self._gonio_start_angles[1] / 180 * np.pi
         detector_rotns_rad = np.array(self._bin_header["detector_rotns"]) / 180 * np.pi
+
+        # I don't know why only rot_e1 is clockwise but this matches
+        # DIRECTION_OF_DETECTOR_X-AXIS/Y-AXIS in XDS.INP from CrysAlisPro.
+        # Note that XDS.INP's directions of Y and Z are opposite from ours.
         rot_e1 = np.array(
-            [
-                np.cos(detector_rotns_rad[0]),
-                np.sin(detector_rotns_rad[0]),
-                0,
-                -np.sin(detector_rotns_rad[0]),
-                np.cos(detector_rotns_rad[0]),
-                0,
-                0,
-                0,
-                1,
-            ]
-        ).reshape(3, 3)
+            r3_rotation_axis_and_angle_as_matrix([0, 0, 1], detector_rotns_rad[0])
+        ).reshape(
+            3, 3
+        )  # clockwise along e1 = Z
         rot_e2 = np.array(
-            [
-                1,
-                0,
-                0,
-                0,
-                np.cos(detector_rotns_rad[1]),
-                np.sin(detector_rotns_rad[1]),
-                0,
-                -np.sin(detector_rotns_rad[1]),
-                np.cos(detector_rotns_rad[1]),
-            ]
-        ).reshape(3, 3)
+            r3_rotation_axis_and_angle_as_matrix([-1, 0, 0], detector_rotns_rad[1])
+        ).reshape(
+            3, 3
+        )  # ANTI-clockwise along e2 = X
         rot_theta = np.array(
-            [
-                np.cos(gonio_angles[1]),
-                0,
-                np.sin(gonio_angles[1]),
-                0,
-                1,
-                0,
-                -np.sin(gonio_angles[1]),
-                0,
-                np.cos(gonio_angles[1]),
-            ]
-        ).reshape(3, 3)
+            r3_rotation_axis_and_angle_as_matrix([0, -1, 0], theta_rad)
+        ).reshape(
+            3, 3
+        )  # ANTI-clockwise along e3 = Y
         detector_axes = rot_theta.dot(rot_e2.dot(rot_e1))
-        # The third axis points to the opposite of XDS.
-        detector_axes[2, :] *= -1
 
         pixel_size_x = self._bin_header["real_px_size_x"]
         pixel_size_y = self._bin_header["real_px_size_y"]
         origin_at_zero = np.array(
             [
                 -self._bin_header["origin_px_x"] * pixel_size_x,
-                +self._bin_header["origin_px_y"] * pixel_size_y,
-                +self._bin_header["distance_mm"],
+                -self._bin_header["origin_px_y"] * pixel_size_y,
+                -self._bin_header["distance_mm"],
             ]
         )
-
-        # FIXME: this formula seem to give the right answer but I don't know why.
-        # XDS exporter in IPR's CrysAlisPro seems broken. It writes the same ORGX/Y
-        # regardless of the theta angles.
         origin = detector_axes.dot(origin_at_zero)
 
         detector = self._detector_factory.make_detector(
             "PAD",
             detector_axes[:, 0],
-            -detector_axes[:, 1],
+            detector_axes[:, 1],
             origin,
             (pixel_size_x, pixel_size_y),
             (self._txt_header["NX"], self._txt_header["NY"]),
             (0, self._bin_header["overflow_threshold"]),
-        )  # not sure about min
+        )
 
         return detector
 
@@ -312,21 +346,8 @@ def get_scan(self, index=None):
     def _scan(self):
         """Return the scan information for this image."""
 
-        for axis in [0, 3]:  # 0 - OMEGA, 3 - PHI: FIXME: is it always PHI scan?
-            start_angle = (
-                self._bin_header["start_angles_steps"][axis]
-                * self._bin_header["step_to_rad"][axis]
-                / np.pi
-                * 180
-            )
-            end_angle = (
-                self._bin_header["end_angles_steps"][axis]
-                * self._bin_header["step_to_rad"][axis]
-                / np.pi
-                * 180
-            )
-            if start_angle != end_angle:
-                break
+        start_angle = self._gonio_start_angles[self._scan_axis]
+        end_angle = self._gonio_end_angles[self._scan_axis]
 
         return self._scan_factory.single_file(
             filename=self._image_file,
@@ -339,10 +360,23 @@ def _scan(self):
     def get_raw_data(self):
         comp = self._txt_header["compression"].strip()
         if comp.startswith("TY6"):
-            return self._get_raw_data_ty6()
+            return self._get_raw_data_ty6_native()
         else:
             raise NotImplementedError("Can't handle compression: {0}".format(comp))
 
+    def _get_raw_data_ty6_native(self):
+        offset = self._txt_header["NHEADER"]
+        nx = self._txt_header["NX"]
+        ny = self._txt_header["NY"]
+        with open(self._image_file, "rb") as f:
+            f.seek(offset)
+            lbytesincompressedfield = struct.unpack("<l", f.read(4))[0]
+            linedata = f.read(lbytesincompressedfield)
+            offsets = f.read(4 * ny)
+
+            return uncompress_rod_TY6(linedata, offsets, ny, nx)
+
+    # Python implementation
     def _get_raw_data_ty6(self):
         offset = self._txt_header["NHEADER"]
         nx = self._txt_header["NX"]
@@ -391,7 +425,6 @@ def decode_TY6_oneline(self, linedata, w):
             bittype = linedata[ipos]
             nbits = (bittype & 15, (bittype >> 4) & 15)
             ipos += 1
-            # ipos_bit = ipos * 8
 
             for i in range(2):
                 nbit = nbits[i]
@@ -455,5 +488,12 @@ def decode_TY6_oneline(self, linedata, w):
             reader = FormatROD(arg)
             print(reader._txt_header)
             print(reader._bin_header)
+            print()
+            print(
+                "Starting angles in degrees (OMEGA, THETA, KAPPA=CHI, PHI, OMEGA PRIME, THETA PRIME)"
+            )
+            print(reader._gonio_start_angles)
+            print("Ending angles in degrees")
+            print(reader._gonio_end_angles)
         else:
             print("Unsupported format")

From 4783b2c79a82150a3eec7fd5cc99eaab5ce67340 Mon Sep 17 00:00:00 2001
From: David Waterman <dagewa@users.noreply.github.com>
Date: Fri, 4 Aug 2023 16:48:36 +0100
Subject: [PATCH 32/45] Add probe type to the Beam model (#647)

At the moment this just adds the additional datum and allows
serialisation etc. Old experiment files where this is not set are
interpreted as having Probe.xray. Some electron diffraction format
classes are updated here to record Probe.electron.

Co-authored-by: davidmcdonagh <davidnmcdonagh@gmail.com>
---
 newsfragments/647.feature               |  1 +
 src/dxtbx/dxtbx_model_ext.pyi           |  4 ++
 src/dxtbx/format/FormatGatanDM4.py      |  2 +
 src/dxtbx/format/FormatMRC.py           |  2 +
 src/dxtbx/format/FormatSMVTimePix_SU.py |  2 +
 src/dxtbx/model/beam.h                  | 90 +++++++++++++++++++++----
 src/dxtbx/model/beam.py                 | 25 ++++++-
 src/dxtbx/model/boost_python/beam.cc    | 44 +++++++++---
 tests/model/test_beam.py                | 19 +++++-
 9 files changed, 164 insertions(+), 25 deletions(-)
 create mode 100644 newsfragments/647.feature

diff --git a/newsfragments/647.feature b/newsfragments/647.feature
new file mode 100644
index 000000000..0ff6b0846
--- /dev/null
+++ b/newsfragments/647.feature
@@ -0,0 +1 @@
+The ``Beam`` model now has a ``probe`` value to keep track of the type of radiation.
diff --git a/src/dxtbx/dxtbx_model_ext.pyi b/src/dxtbx/dxtbx_model_ext.pyi
index 4eb5e1530..8a4a85934 100644
--- a/src/dxtbx/dxtbx_model_ext.pyi
+++ b/src/dxtbx/dxtbx_model_ext.pyi
@@ -22,6 +22,8 @@ from scitbx.array_family import shared as flex_shared
 # Attempt to use the stub typing for flex-inheritance
 from scitbx.array_family.flex import FlexPlain
 
+from dxtbx_model_ext import Probe  # type: ignore
+
 # TypeVar for the set of Experiment models that can be joint-accepted
 # - profile, imageset and scalingmodel are handled as 'object'
 TExperimentModel = TypeVar(
@@ -113,6 +115,8 @@ class Beam(BeamBase):
     @staticmethod
     def from_dict(data: Dict) -> Beam: ...
     def to_dict(self) -> Dict: ...
+    @staticmethod
+    def get_probe_from_name(name: str) -> Probe: ...
 
 class PolychromaticBeam(Beam):
     @overload
diff --git a/src/dxtbx/format/FormatGatanDM4.py b/src/dxtbx/format/FormatGatanDM4.py
index d5cb31181..acb93f1ef 100644
--- a/src/dxtbx/format/FormatGatanDM4.py
+++ b/src/dxtbx/format/FormatGatanDM4.py
@@ -21,6 +21,7 @@
 )
 from dxtbx.format.Format import Format
 from dxtbx.format.FormatMultiImage import FormatMultiImage
+from dxtbx.model.beam import Probe
 
 
 def read_tag(f, byteorder):
@@ -358,6 +359,7 @@ def _beam(self):
             wavelength=wavelength,
             polarization=(0, 1, 0),
             polarization_fraction=0.5,
+            probe=Probe.electron,
         )
 
     def _scan(self):
diff --git a/src/dxtbx/format/FormatMRC.py b/src/dxtbx/format/FormatMRC.py
index 0b9b06226..b7d9b30bb 100644
--- a/src/dxtbx/format/FormatMRC.py
+++ b/src/dxtbx/format/FormatMRC.py
@@ -19,6 +19,7 @@
 from dxtbx.format.Format import Format
 from dxtbx.format.FormatMultiImage import FormatMultiImage
 from dxtbx.model import ScanFactory
+from dxtbx.model.beam import Probe
 
 logger = logging.getLogger("dials")
 
@@ -226,6 +227,7 @@ def _beam(self):
             wavelength=wavelength,
             polarization=(0, 1, 0),
             polarization_fraction=0.5,
+            probe=Probe.electron,
         )
 
 
diff --git a/src/dxtbx/format/FormatSMVTimePix_SU.py b/src/dxtbx/format/FormatSMVTimePix_SU.py
index 223061836..b3492ff56 100644
--- a/src/dxtbx/format/FormatSMVTimePix_SU.py
+++ b/src/dxtbx/format/FormatSMVTimePix_SU.py
@@ -12,6 +12,7 @@
 from scitbx import matrix
 
 from dxtbx.format.FormatSMV import FormatSMV
+from dxtbx.model.beam import Probe
 from dxtbx.model.detector import Detector
 
 
@@ -76,6 +77,7 @@ def _beam(self):
             wavelength=wavelength,
             polarization=(0, 1, 0),
             polarization_fraction=0.5,
+            probe=Probe.electron,
         )
 
     def _scan(self):
diff --git a/src/dxtbx/model/beam.h b/src/dxtbx/model/beam.h
index a8d72f33c..6df5325d1 100644
--- a/src/dxtbx/model/beam.h
+++ b/src/dxtbx/model/beam.h
@@ -13,6 +13,7 @@
 
 #include <iostream>
 #include <cmath>
+#include <string>
 #include <scitbx/vec3.h>
 #include <scitbx/array_family/shared.h>
 #include <scitbx/array_family/simple_io.h>
@@ -24,6 +25,9 @@ namespace dxtbx { namespace model {
 
   using scitbx::vec3;
 
+  // probe type enumeration
+  enum Probe { xray = 1, electron = 2, neutron = 3 };
+
   /** Base class for beam objects */
   class BeamBase {
   public:
@@ -44,6 +48,8 @@ namespace dxtbx { namespace model {
     virtual std::size_t get_num_scan_points() const = 0;
     virtual scitbx::af::shared<vec3<double> > get_s0_at_scan_points() const = 0;
     virtual vec3<double> get_s0_at_scan_point(std::size_t index) const = 0;
+    virtual Probe get_probe() const = 0;
+    virtual std::string get_probe_name() const = 0;
 
     virtual void set_direction(vec3<double> direction) = 0;
     virtual void set_wavelength(double wavelength) = 0;
@@ -59,6 +65,7 @@ namespace dxtbx { namespace model {
     virtual void set_transmission(double transmission) = 0;
     virtual void set_s0_at_scan_points(
       const scitbx::af::const_ref<vec3<double> > &s0) = 0;
+    virtual void set_probe(Probe probe) = 0;
 
     virtual void reset_scan_points() = 0;
     virtual bool is_similar_to(const BeamBase &rhs,
@@ -82,7 +89,8 @@ namespace dxtbx { namespace model {
           polarization_normal_(0.0, 1.0, 0.0),
           polarization_fraction_(0.999),
           flux_(0),
-          transmission_(1.0) {}
+          transmission_(1.0),
+          probe_(Probe::xray) {}
 
     /**
      * @param s0 The incident beam vector.
@@ -93,7 +101,8 @@ namespace dxtbx { namespace model {
           polarization_normal_(0.0, 1.0, 0.0),
           polarization_fraction_(0.999),
           flux_(0),
-          transmission_(1.0) {
+          transmission_(1.0),
+          probe_(Probe::xray) {
       DXTBX_ASSERT(s0.length() > 0);
       wavelength_ = 1.0 / s0.length();
       direction_ = -s0.normalize();
@@ -110,7 +119,8 @@ namespace dxtbx { namespace model {
           polarization_normal_(0.0, 1.0, 0.0),
           polarization_fraction_(0.999),
           flux_(0),
-          transmission_(1.0) {
+          transmission_(1.0),
+          probe_(Probe::xray) {
       DXTBX_ASSERT(direction.length() > 0);
       direction_ = direction.normalize();
     }
@@ -126,7 +136,8 @@ namespace dxtbx { namespace model {
           polarization_normal_(0.0, 1.0, 0.0),
           polarization_fraction_(0.999),
           flux_(0),
-          transmission_(1.0) {
+          transmission_(1.0),
+          probe_(Probe::xray) {
       DXTBX_ASSERT(s0.length() > 0);
       wavelength_ = 1.0 / s0.length();
       direction_ = -s0.normalize();
@@ -148,7 +159,8 @@ namespace dxtbx { namespace model {
           polarization_normal_(0.0, 1.0, 0.0),
           polarization_fraction_(0.999),
           flux_(0),
-          transmission_(1.0) {
+          transmission_(1.0),
+          probe_(Probe::xray) {
       DXTBX_ASSERT(direction.length() > 0);
       direction_ = direction.normalize();
     }
@@ -162,6 +174,7 @@ namespace dxtbx { namespace model {
      * @param polarization_fraction The polarization fraction
      * @param flux The beam flux
      * @param transmission The beam transmission
+     * @param probe The probe value
      */
     Beam(vec3<double> direction,
          double wavelength,
@@ -170,14 +183,16 @@ namespace dxtbx { namespace model {
          vec3<double> polarization_normal,
          double polarization_fraction,
          double flux,
-         double transmission)
+         double transmission,
+         Probe probe)
         : wavelength_(wavelength),
           divergence_(divergence),
           sigma_divergence_(sigma_divergence),
           polarization_normal_(polarization_normal),
           polarization_fraction_(polarization_fraction),
           flux_(flux),
-          transmission_(transmission) {
+          transmission_(transmission),
+          probe_(probe) {
       DXTBX_ASSERT(direction.length() > 0);
       direction_ = direction.normalize();
     }
@@ -289,6 +304,44 @@ namespace dxtbx { namespace model {
       return s0_at_scan_points_[index];
     }
 
+    Probe get_probe() const {
+      return probe_;
+    }
+
+    std::string get_probe_name() const {
+      // Return a name that matches NeXus definitions from
+      // https://manual.nexusformat.org/classes/base_classes/NXsource.html
+      switch (probe_) {
+      case xray:
+        return std::string("x-ray");
+      case electron:
+        return std::string("electron");
+      case neutron:
+        return std::string("neutron");
+      default:
+        DXTBX_ERROR("Unknown probe type");
+      }
+    }
+
+    static Probe get_probe_from_name(const std::string probe) {
+      // Return a Probe matched to NeXus definitions from
+      // https://manual.nexusformat.org/classes/base_classes/NXsource.html
+
+      if (probe == "x-ray") {
+        return Probe::xray;
+      } else if (probe == "electron") {
+        return Probe::electron;
+      } else if (probe == "neutron") {
+        return Probe::neutron;
+      }
+
+      DXTBX_ERROR("Unknown probe " + probe);
+    }
+
+    void set_probe(Probe probe) {
+      probe_ = probe;
+    }
+
     void reset_scan_points() {
       s0_at_scan_points_.clear();
     }
@@ -324,7 +377,8 @@ namespace dxtbx { namespace model {
                   angle_safe(polarization_normal_, rhs.get_polarization_normal()))
                   <= eps
              && std::abs(polarization_fraction_ - rhs.get_polarization_fraction())
-                  <= eps;
+                  <= eps
+             && (probe_ == rhs.get_probe());
     }
 
     virtual bool is_similar_to(const BeamBase &rhs,
@@ -361,7 +415,8 @@ namespace dxtbx { namespace model {
                   angle_safe(polarization_normal_, rhs.get_polarization_normal()))
                   <= polarization_normal_tolerance
              && std::abs(polarization_fraction_ - rhs.get_polarization_fraction())
-                  <= polarization_fraction_tolerance;
+                  <= polarization_fraction_tolerance
+             && (probe_ == rhs.get_probe());
     }
 
     bool operator!=(const BeamBase &rhs) const {
@@ -383,6 +438,7 @@ namespace dxtbx { namespace model {
     double polarization_fraction_;
     double flux_;
     double transmission_;
+    Probe probe_;
 
   private:
     double wavelength_;
@@ -392,6 +448,7 @@ namespace dxtbx { namespace model {
   /** Print beam information */
   inline std::ostream &operator<<(std::ostream &os, const Beam &b) {
     os << "Beam:\n";
+    os << "    probe: " << b.get_probe_name() << "\n";
     os << "    wavelength: " << b.get_wavelength() << "\n";
     os << "    sample to source direction : "
        << b.get_sample_to_source_direction().const_ref() << "\n";
@@ -415,6 +472,7 @@ namespace dxtbx { namespace model {
       set_polarization_fraction(0.5);
       set_flux(0);
       set_transmission(1.0);
+      set_probe(Probe::xray);
     }
 
     /**
@@ -429,6 +487,7 @@ namespace dxtbx { namespace model {
       set_polarization_fraction(0.5);
       set_flux(0);
       set_transmission(1.0);
+      set_probe(Probe::xray);
     }
 
     /**
@@ -447,6 +506,7 @@ namespace dxtbx { namespace model {
       set_polarization_fraction(0.5);
       set_flux(0);
       set_transmission(1.0);
+      set_probe(Probe::xray);
     }
 
     /**
@@ -457,6 +517,7 @@ namespace dxtbx { namespace model {
      * @param polarization_fraction The polarization fraction
      * @param flux The beam flux
      * @param transmission The beam transmission
+     * @param probe The probe value
      */
     PolychromaticBeam(vec3<double> direction,
                       double divergence,
@@ -464,7 +525,8 @@ namespace dxtbx { namespace model {
                       vec3<double> polarization_normal,
                       double polarization_fraction,
                       double flux,
-                      double transmission) {
+                      double transmission,
+                      Probe probe) {
       DXTBX_ASSERT(direction.length() > 0);
       direction_ = direction.normalize();
       set_divergence(divergence);
@@ -473,6 +535,7 @@ namespace dxtbx { namespace model {
       set_polarization_fraction(polarization_fraction);
       set_flux(flux);
       set_transmission(transmission);
+      set_probe(probe);
     }
 
     double get_wavelength() const {
@@ -527,7 +590,8 @@ namespace dxtbx { namespace model {
                   angle_safe(polarization_normal_, rhs.get_polarization_normal()))
                   <= eps
              && std::abs(polarization_fraction_ - rhs.get_polarization_fraction())
-                  <= eps;
+                  <= eps
+             && (probe_ == rhs.get_probe());
     }
 
     bool is_similar_to(const BeamBase &rhs,
@@ -551,13 +615,15 @@ namespace dxtbx { namespace model {
                   angle_safe(polarization_normal_, rhs.get_polarization_normal()))
                   <= polarization_normal_tolerance
              && std::abs(polarization_fraction_ - rhs.get_polarization_fraction())
-                  <= polarization_fraction_tolerance;
+                  <= polarization_fraction_tolerance
+             && (probe_ == rhs.get_probe());
     }
   };
 
   /** Print beam information */
   inline std::ostream &operator<<(std::ostream &os, const PolychromaticBeam &b) {
     os << "Beam:\n";
+    os << "    probe: " << b.get_probe_name() << "\n";
     os << "    sample to source direction : "
        << b.get_sample_to_source_direction().const_ref() << "\n";
     os << "    divergence: " << b.get_divergence() << "\n";
diff --git a/src/dxtbx/model/beam.py b/src/dxtbx/model/beam.py
index 7c94cffa7..a7360e9e5 100644
--- a/src/dxtbx/model/beam.py
+++ b/src/dxtbx/model/beam.py
@@ -8,9 +8,9 @@
 import libtbx.phil
 
 try:
-    from ..dxtbx_model_ext import Beam, PolychromaticBeam
+    from ..dxtbx_model_ext import Beam, PolychromaticBeam, Probe
 except ModuleNotFoundError:
-    from dxtbx_model_ext import Beam, PolychromaticBeam  # type: ignore
+    from dxtbx_model_ext import Beam, PolychromaticBeam, Probe  # type: ignore
 
 Vec3Float = Tuple[float, float, float]
 
@@ -25,6 +25,11 @@
       .help = "Override the beam type"
       .short_caption = "beam_type"
 
+    probe = *x-ray electron neutron
+      .type = choice
+      .help = "Override the beam probe"
+      .short_caption = "beam_probe"
+
     wavelength = None
       .type = float
       .help = "Override the beam wavelength"
@@ -74,7 +79,7 @@ class BeamFactory:
     @staticmethod
     def from_phil(
         params: libtbx.phil.scope_extract,
-        reference: Beam | PolychromaticBeam = None,
+        reference: Beam | PolychromaticBeam | None = None,
     ) -> Beam | PolychromaticBeam:
         """
         Convert the phil parameters into a beam model
@@ -111,6 +116,7 @@ def from_phil(
             beam.set_transmission(params.beam.transmission)
         if params.beam.flux is not None:
             beam.set_flux(params.beam.flux)
+        beam.set_probe(Beam.get_probe_from_name(params.beam.probe))
 
         return beam
 
@@ -130,8 +136,11 @@ def from_dict(dict: dict, template: dict = None) -> Beam | PolychromaticBeam:
         joint.update(dict)
 
         # Create the model from the joint dictionary
+        if "probe" not in joint:
+            joint["probe"] = "x-ray"
         if joint.get("__id__") == "polychromatic":
             return PolychromaticBeam.from_dict(joint)
+
         return Beam.from_dict(joint)
 
     @staticmethod
@@ -176,6 +185,7 @@ def make_polychromatic_beam(
         polarization_fraction: float = 0.5,
         flux: float = 0.0,
         transmission: float = 1.0,
+        probe: Probe = Probe.xray,
         deg: bool = True,
     ) -> PolychromaticBeam:
         return PolychromaticBeam(
@@ -186,6 +196,7 @@ def make_polychromatic_beam(
             float(polarization_fraction),
             float(flux),
             float(transmission),
+            probe,
             bool(deg),
         )
 
@@ -201,6 +212,7 @@ def make_polarized_beam(
         sigma_divergence: float = None,
         flux: float = None,
         transmission: float = None,
+        probe: Probe = Probe.xray,
     ) -> Beam:
         assert polarization
         assert 0.0 <= polarization_fraction <= 1.0
@@ -225,6 +237,7 @@ def make_polarized_beam(
                 float(polarization_fraction),
                 float(flux),
                 float(transmission),
+                probe,
             )
         elif unit_s0:
             assert wavelength
@@ -237,17 +250,23 @@ def make_polarized_beam(
                 float(polarization_fraction),
                 float(flux),
                 float(transmission),
+                probe,
             )
         else:
             assert s0
+            sum_sq_s0 = s0[0] ** 2 + s0[1] ** 2 + s0[2] ** 2
+            assert sum_sq_s0 > 0
+            wavelength = 1.0 / math.sqrt(sum_sq_s0)
             return Beam(
                 tuple(map(float, s0)),
+                wavelength,
                 float(divergence),
                 float(sigma_divergence),
                 tuple(map(float, polarization)),
                 float(polarization_fraction),
                 float(flux),
                 float(transmission),
+                probe,
             )
 
     @staticmethod
diff --git a/src/dxtbx/model/boost_python/beam.cc b/src/dxtbx/model/boost_python/beam.cc
index 45bec9f95..86cdab927 100644
--- a/src/dxtbx/model/boost_python/beam.cc
+++ b/src/dxtbx/model/boost_python/beam.cc
@@ -17,6 +17,7 @@
 #include <dxtbx/model/beam.h>
 #include <dxtbx/model/boost_python/to_from_dict.h>
 #include <scitbx/array_family/boost_python/flex_wrapper.h>
+#include <boost/python/enum.hpp>
 
 namespace dxtbx { namespace model { namespace boost_python {
   namespace beam_detail {
@@ -42,7 +43,8 @@ namespace dxtbx { namespace model { namespace boost_python {
                                          obj.get_polarization_normal(),
                                          obj.get_polarization_fraction(),
                                          obj.get_flux(),
-                                         obj.get_transmission());
+                                         obj.get_transmission(),
+                                         obj.get_probe());
       }
 
       static boost::python::tuple getstate(boost::python::object obj) {
@@ -109,6 +111,7 @@ namespace dxtbx { namespace model { namespace boost_python {
                                  double polarization_fraction,
                                  double flux,
                                  double transmission,
+                                 Probe probe,
                                  bool deg) {
       Beam *beam = NULL;
       if (deg) {
@@ -119,7 +122,8 @@ namespace dxtbx { namespace model { namespace boost_python {
                         polarization_normal,
                         polarization_fraction,
                         flux,
-                        transmission);
+                        transmission,
+                        probe);
       } else {
         beam = new Beam(sample_to_source,
                         wavelength,
@@ -128,7 +132,8 @@ namespace dxtbx { namespace model { namespace boost_python {
                         polarization_normal,
                         polarization_fraction,
                         flux,
-                        transmission);
+                        transmission,
+                        probe);
       }
       return beam;
     }
@@ -191,6 +196,7 @@ namespace dxtbx { namespace model { namespace boost_python {
     result["polarization_fraction"] = obj.get_polarization_fraction();
     result["flux"] = obj.get_flux();
     result["transmission"] = obj.get_transmission();
+    result["probe"] = obj.get_probe_name();
     if (obj.get_num_scan_points() > 0) {
       boost::python::list l;
       scitbx::af::shared<vec3<double> > s0_at_scan_points = obj.get_s0_at_scan_points();
@@ -215,7 +221,9 @@ namespace dxtbx { namespace model { namespace boost_python {
         obj.get("polarization_normal", vec3<double>(0.0, 1.0, 0.0))),
       boost::python::extract<double>(obj.get("polarization_fraction", 0.999)),
       boost::python::extract<double>(obj.get("flux", 0)),
-      boost::python::extract<double>(obj.get("transmission", 1)));
+      boost::python::extract<double>(obj.get("transmission", 1)),
+      Beam::get_probe_from_name(
+        boost::python::extract<std::string>(obj.get("probe", "x-ray"))));
     if (obj.has_key("s0_at_scan_points")) {
       boost::python::list s0_at_scan_points =
         boost::python::extract<boost::python::list>(obj["s0_at_scan_points"]);
@@ -240,7 +248,8 @@ namespace dxtbx { namespace model { namespace boost_python {
                                        obj.get_polarization_normal(),
                                        obj.get_polarization_fraction(),
                                        obj.get_flux(),
-                                       obj.get_transmission());
+                                       obj.get_transmission(),
+                                       obj.get_probe());
     }
   };
 
@@ -270,6 +279,7 @@ namespace dxtbx { namespace model { namespace boost_python {
     double polarization_fraction,
     double flux,
     double transmission,
+    Probe probe,
     bool deg) {
     PolychromaticBeam *beam = NULL;
     if (deg) {
@@ -279,7 +289,8 @@ namespace dxtbx { namespace model { namespace boost_python {
                                    polarization_normal,
                                    polarization_fraction,
                                    flux,
-                                   transmission);
+                                   transmission,
+                                   probe);
     } else {
       beam = new PolychromaticBeam(direction,
                                    divergence,
@@ -287,7 +298,8 @@ namespace dxtbx { namespace model { namespace boost_python {
                                    polarization_normal,
                                    polarization_fraction,
                                    flux,
-                                   transmission);
+                                   transmission,
+                                   probe);
     }
     return beam;
   }
@@ -303,6 +315,7 @@ namespace dxtbx { namespace model { namespace boost_python {
     result["polarization_fraction"] = obj.get_polarization_fraction();
     result["flux"] = obj.get_flux();
     result["transmission"] = obj.get_transmission();
+    result["probe"] = obj.get_probe_name();
     return result;
   }
 
@@ -316,13 +329,20 @@ namespace dxtbx { namespace model { namespace boost_python {
         obj.get("polarization_normal", vec3<double>(0.0, 1.0, 0.0))),
       boost::python::extract<double>(obj.get("polarization_fraction", 0.999)),
       boost::python::extract<double>(obj.get("flux", 0)),
-      boost::python::extract<double>(obj.get("transmission", 1)));
+      boost::python::extract<double>(obj.get("transmission", 1)),
+      Beam::get_probe_from_name(
+        boost::python::extract<std::string>(obj.get("probe", "x-ray"))));
     return b;
   }
 
   void export_beam() {
     using namespace beam_detail;
 
+    enum_<Probe>("Probe")
+      .value("xray", xray)
+      .value("electron", electron)
+      .value("neutron", neutron);
+
     class_<BeamBase, boost::noncopyable>("BeamBase", no_init)
       .def("get_sample_to_source_direction", &BeamBase::get_sample_to_source_direction)
       .def("set_direction", &BeamBase::set_direction)
@@ -353,6 +373,9 @@ namespace dxtbx { namespace model { namespace boost_python {
       .def("set_s0_at_scan_points", &Beam_set_s0_at_scan_points_from_list)
       .def("get_s0_at_scan_points", &BeamBase::get_s0_at_scan_points)
       .def("get_s0_at_scan_point", &BeamBase::get_s0_at_scan_point)
+      .def("get_probe", &BeamBase::get_probe)
+      .def("get_probe_name", &BeamBase::get_probe_name)
+      .def("set_probe", &BeamBase::set_probe)
       .def("reset_scan_points", &BeamBase::reset_scan_points)
       .def("rotate_around_origin",
            &rotate_around_origin,
@@ -397,15 +420,19 @@ namespace dxtbx { namespace model { namespace boost_python {
                              arg("polarization_fraction"),
                              arg("flux"),
                              arg("transmission"),
+                             arg("probe") = Probe::xray,
                              arg("deg") = true)))
       .def("__str__", &beam_to_string)
       .def("to_dict", &to_dict<Beam>)
       .def("from_dict", &from_dict<Beam>, return_value_policy<manage_new_object>())
       .staticmethod("from_dict")
+      .def("get_probe_from_name", &Beam::get_probe_from_name)
+      .staticmethod("get_probe_from_name")
       .def_pickle(BeamPickleSuite());
 
     class_<PolychromaticBeam, std::shared_ptr<PolychromaticBeam>, bases<Beam> >(
       "PolychromaticBeam")
+      .def(init<const PolychromaticBeam &>())
       .def("__init__",
            make_constructor(
              &make_PolychromaticBeam, default_call_policies(), (arg("direction"))))
@@ -426,6 +453,7 @@ namespace dxtbx { namespace model { namespace boost_python {
                              arg("polarization_fraction"),
                              arg("flux"),
                              arg("transmission"),
+                             arg("probe") = Probe::xray,
                              arg("deg") = true)))
       .def("__str__", &PolychromaticBeam_to_string)
       .def("to_dict", &to_dict<PolychromaticBeam>)
diff --git a/tests/model/test_beam.py b/tests/model/test_beam.py
index 2f83a790c..40659f4d4 100644
--- a/tests/model/test_beam.py
+++ b/tests/model/test_beam.py
@@ -6,7 +6,7 @@
 from scitbx import matrix
 
 from dxtbx.model import Beam, PolychromaticBeam
-from dxtbx.model.beam import BeamFactory, beam_phil_scope
+from dxtbx.model.beam import BeamFactory, Probe, beam_phil_scope
 
 
 def test_setting_direction_and_wavelength():
@@ -102,6 +102,18 @@ def test_from_phil():
     assert b3.get_polarization_fraction() == 0.5
     assert b3.get_polarization_normal() == (1.0, 0.0, 0.0)
 
+    params3 = beam_phil_scope.fetch(
+        parse(
+            """
+    beam {
+        probe = electron
+    }
+  """
+        )
+    ).extract()
+    b4 = BeamFactory.from_phil(params3, reference)
+    assert b4.get_probe() == Probe.electron
+
 
 def test_scan_varying():
     direction = matrix.col((0.013142, 0.002200, 1.450476))
@@ -222,6 +234,7 @@ def test_make_polychromatic_beam():
     polarization_fraction = 0.65
     transmission = 0.5
     flux = 0.75
+    probe = Probe.neutron
 
     beam = BeamFactory.make_polychromatic_beam(
         direction=direction,
@@ -231,6 +244,7 @@ def test_make_polychromatic_beam():
         polarization_fraction=polarization_fraction,
         transmission=transmission,
         flux=flux,
+        probe=probe,
     )
 
     assert beam.get_sample_to_source_direction() == pytest.approx((0.0, 0.0, 1.0))
@@ -240,6 +254,7 @@ def test_make_polychromatic_beam():
     assert beam.get_polarization_fraction() == pytest.approx(0.65)
     assert beam.get_transmission() == pytest.approx(0.5)
     assert beam.get_flux() == pytest.approx(0.75)
+    assert beam.get_probe() == Probe.neutron
 
 
 def test_polychromatic_beam_wavelength_guards():
@@ -266,5 +281,5 @@ def test_polychromatic_beam_str():
     beam = PolychromaticBeam()
     assert (
         beam.__str__()
-        == "Beam:\n    sample to source direction : {0,0,1}\n    divergence: 0\n    sigma divergence: 0\n    polarization normal: {0,1,0}\n    polarization fraction: 0.5\n    flux: 0\n    transmission: 1\n"
+        == "Beam:\n    probe: x-ray\n    sample to source direction : {0,0,1}\n    divergence: 0\n    sigma divergence: 0\n    polarization normal: {0,1,0}\n    polarization fraction: 0.5\n    flux: 0\n    transmission: 1\n"
     )

From d516f2044c1c4568f8ecb57217c95d73924cf47e Mon Sep 17 00:00:00 2001
From: Graeme Winter <graeme.winter@gmail.com>
Date: Tue, 8 Aug 2023 14:09:39 +0100
Subject: [PATCH 33/45] Fix bit depth for Eiger / NXmx for i19-2 (#652)

Fix bit depth for Eiger / NXmx for i19-2

Add bit depth check from image metadata as implemented on other Diamond NXmx
installations. Fixes dials/dials#2473
---
 newsfragments/652.bugfix               | 1 +
 src/dxtbx/format/FormatNXmxDLSI19_2.py | 7 ++++++-
 2 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 newsfragments/652.bugfix

diff --git a/newsfragments/652.bugfix b/newsfragments/652.bugfix
new file mode 100644
index 000000000..ec7635b59
--- /dev/null
+++ b/newsfragments/652.bugfix
@@ -0,0 +1 @@
+``dxtbx``: add fix for Eiger / NXmx data from i19-2 to correctly assign the image bit depth
diff --git a/src/dxtbx/format/FormatNXmxDLSI19_2.py b/src/dxtbx/format/FormatNXmxDLSI19_2.py
index 6d7f0c7ad..255330eb4 100644
--- a/src/dxtbx/format/FormatNXmxDLSI19_2.py
+++ b/src/dxtbx/format/FormatNXmxDLSI19_2.py
@@ -8,7 +8,7 @@
 from libtbx import Auto
 
 from dxtbx.format.FormatNXmx import FormatNXmx
-from dxtbx.format.FormatNXmxDLS import FormatNXmxDLS
+from dxtbx.format.FormatNXmxDLS import FormatNXmxDLS, get_bit_depth_from_meta
 from dxtbx.masking import GoniometerMaskerFactory
 
 
@@ -41,6 +41,11 @@ def __init__(self, image_file, **kwargs):
         """Initialise the image structure from the given file."""
         self._dynamic_shadowing = self.has_dynamic_shadowing(**kwargs)
         super().__init__(image_file, **kwargs)
+        try:
+            self._bit_depth_readout = get_bit_depth_from_meta(self._meta)
+        except Exception:
+            self._bit_depth_readout = 16
+
 
     def get_goniometer_shadow_masker(self, goniometer=None):
         """Apply the dynamic mask for a diamond anvil cell with a 76° aperture."""

From 39c49235e26a09779ffd0c5f5bef7c64e6d5d3d3 Mon Sep 17 00:00:00 2001
From: Graeme Winter <graeme.winter@gmail.com>
Date: Thu, 10 Aug 2023 16:32:35 +0100
Subject: [PATCH 34/45] ESRF ID23-2 support (#651)

---
 newsfragments/651.feature                     |  1 +
 .../FormatNXmxEigerFilewriterESRFID232.py     | 34 +++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 newsfragments/651.feature
 create mode 100644 src/dxtbx/format/FormatNXmxEigerFilewriterESRFID232.py

diff --git a/newsfragments/651.feature b/newsfragments/651.feature
new file mode 100644
index 000000000..2f790c707
--- /dev/null
+++ b/newsfragments/651.feature
@@ -0,0 +1 @@
+Add support for Eiger 9M on ESRF ID23-2, which has an undeclared vertical goniometer.
diff --git a/src/dxtbx/format/FormatNXmxEigerFilewriterESRFID232.py b/src/dxtbx/format/FormatNXmxEigerFilewriterESRFID232.py
new file mode 100644
index 000000000..58d2fcb60
--- /dev/null
+++ b/src/dxtbx/format/FormatNXmxEigerFilewriterESRFID232.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import re
+
+import h5py
+
+from dxtbx.format.FormatNXmxEigerFilewriter import FormatNXmxEigerFilewriter
+
+DATA_FILE_RE = re.compile(r"data_\d{6}")
+
+
+class FormatNXmxEigerFilewriterESRFID232(FormatNXmxEigerFilewriter):
+    _cached_file_handle = None
+
+    @staticmethod
+    def understand(image_file):
+        with h5py.File(image_file) as handle:
+            if "/entry/instrument/detector/detector_number" in handle:
+                if (
+                    handle["/entry/instrument/detector/detector_number"][()]
+                    == b"E-18-0133"
+                ):
+                    return True
+        return False
+
+    def __init__(self, image_file, **kwargs):
+        """Initialise the image structure from the given file."""
+        super().__init__(image_file, **kwargs)
+
+    def _start(self):
+        super()._start()
+
+    def _goniometer(self):
+        return self._goniometer_factory.known_axis((0, 1, 0))

From 206c7cb2c1549db95350f8074a7e087394c596fd Mon Sep 17 00:00:00 2001
From: David Waterman <dagewa@users.noreply.github.com>
Date: Thu, 10 Aug 2023 16:33:10 +0100
Subject: [PATCH 35/45] FormatCBFMiniEigerChessID7B2 (#649)

---
 newsfragments/649.feature                     |  1 +
 .../format/FormatCBFMiniEigerChessID7B2.py    | 30 +++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 newsfragments/649.feature
 create mode 100644 src/dxtbx/format/FormatCBFMiniEigerChessID7B2.py

diff --git a/newsfragments/649.feature b/newsfragments/649.feature
new file mode 100644
index 000000000..eb736b9a4
--- /dev/null
+++ b/newsfragments/649.feature
@@ -0,0 +1 @@
+Add CBFMini support for the EIGER2 16M detector at CHESS beamline ID7B2, which has an inverted rotation axis.
diff --git a/src/dxtbx/format/FormatCBFMiniEigerChessID7B2.py b/src/dxtbx/format/FormatCBFMiniEigerChessID7B2.py
new file mode 100644
index 000000000..9bcee5b52
--- /dev/null
+++ b/src/dxtbx/format/FormatCBFMiniEigerChessID7B2.py
@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import sys
+
+from dxtbx.format.FormatCBFMiniEiger import FormatCBFMiniEiger
+
+
+class FormatCBFMiniEigerChessID7B2(FormatCBFMiniEiger):
+    """A class for reading mini CBF format Eiger16M images for S/N E-32-0123
+    installed at CHESS ID7B2, which has an inverted goniometer axis."""
+
+    @staticmethod
+    def understand(image_file):
+        """Check to see if this looks like an Eiger mini CBF format image,
+        i.e. we can make sense of it."""
+
+        header = FormatCBFMiniEiger.get_cbf_header(image_file)
+        for record in header.split("\n"):
+            if "# Detector: Dectris EIGER2 Si 16M, S/N E-32-0123" in record:
+                return True
+
+        return False
+
+    def _goniometer(self):
+        return self._goniometer_factory.known_axis((-1, 0, 0))
+
+
+if __name__ == "__main__":
+    for arg in sys.argv[1:]:
+        print(FormatCBFMiniEigerChessID7B2.understand(arg))

From 3bc6e67337a0935496e99edb4e6eee25deff9c64 Mon Sep 17 00:00:00 2001
From: "Aaron S. Brewster" <asbrewster@lbl.gov>
Date: Fri, 11 Aug 2023 02:05:16 -0700
Subject: [PATCH 36/45] NXmx: handle multidimensional arrays (#612)

Data in NeXus can be 3 or 4 dimensional.
3D: Nimages by slow by fast
4D: Nimages by Nmodules by slow fast

Slice image_size and reshape the raw_data in these cases.

Co-authored-by: Richard Gildea <rjgildea@users.noreply.github.com>
---
 newsfragments/612.bugfix               |   1 +
 src/dxtbx/nexus/__init__.py            |  28 ++++---
 tests/nexus/test_build_dxtbx_models.py | 103 +++++++++++++++++++++++++
 3 files changed, 123 insertions(+), 9 deletions(-)
 create mode 100644 newsfragments/612.bugfix

diff --git a/newsfragments/612.bugfix b/newsfragments/612.bugfix
new file mode 100644
index 000000000..6f4e30c86
--- /dev/null
+++ b/newsfragments/612.bugfix
@@ -0,0 +1 @@
+NXmx files with multidimensional arrays (images, modules, or both) are now handled.
diff --git a/src/dxtbx/nexus/__init__.py b/src/dxtbx/nexus/__init__.py
index ccd84a7ec..6fc0d20c6 100644
--- a/src/dxtbx/nexus/__init__.py
+++ b/src/dxtbx/nexus/__init__.py
@@ -2,7 +2,7 @@
 
 import itertools
 import logging
-from typing import Literal, Optional, Tuple, cast
+from typing import Literal, Optional
 
 import h5py
 import numpy as np
@@ -404,9 +404,12 @@ def equipment_component_key(dependency):
                 origin -= nxdetector.beam_center_y.magnitude * pixel_size[1] * slow_axis
 
         # dxtbx requires image size in the order fast, slow - which is the reverse of what
-        # is stored in module.data_size
-        image_size = cast(Tuple[int, int], tuple(map(int, module.data_size[::-1])))
-        assert len(image_size) == 2
+        # is stored in module.data_size. Additionally, data_size can have more than 2
+        # dimensions, for multi-module detectors. So take the last two dimensions and reverse
+        # them.  Examples:
+        # [1,2,3]   --> (3, 2)
+        # [1,2]     --> (2, 1)
+        image_size = (int(module.data_size[-1]), int(module.data_size[-2]))
         underload = (
             float(nxdetector.underload_value)
             if nxdetector.underload_value is not None
@@ -475,13 +478,17 @@ def get_static_mask(nxdetector: nxmx.NXdetector) -> tuple[flex.bool, ...] | None
         pixel_mask = nxdetector.pixel_mask
     except KeyError:
         return None
-    if pixel_mask is None or not pixel_mask.size or pixel_mask.ndim != 2:
+    if pixel_mask is None or not pixel_mask.size:
         return None
     all_slices = get_detector_module_slices(nxdetector)
-    return tuple(
-        flumpy.from_numpy(np.ascontiguousarray(pixel_mask[slices])) == 0
-        for slices in all_slices
-    )
+    all_mask_slices = []
+    for slices in all_slices:
+        mask_slice = flumpy.from_numpy(np.ascontiguousarray(pixel_mask[slices])) == 0
+        mask_slice.reshape(
+            flex.grid(mask_slice.all()[-2:])
+        )  # handle 3 or 4 dimension arrays
+        all_mask_slices.append(mask_slice)
+    return tuple(all_mask_slices)
 
 
 def _dataset_as_flex(
@@ -562,5 +569,8 @@ def get_raw_data(
         data_as_flex = _dataset_as_flex(
             sliced_outer, tuple(module_slices), bit_depth=bit_depth
         )
+        data_as_flex.reshape(
+            flex.grid(data_as_flex.all()[-2:])
+        )  # handle 3 or 4 dimension arrays
         all_data.append(data_as_flex)
     return tuple(all_data)
diff --git a/tests/nexus/test_build_dxtbx_models.py b/tests/nexus/test_build_dxtbx_models.py
index 8b3b684ac..82c33ca86 100644
--- a/tests/nexus/test_build_dxtbx_models.py
+++ b/tests/nexus/test_build_dxtbx_models.py
@@ -306,6 +306,109 @@ def test_get_dxtbx_detector_beam_center_fallback(nxmx_example):
     )
 
 
+@pytest.fixture
+def detector_with_multiple_modules():
+
+    with h5py.File(" ", "w", **pytest.h5_in_memory) as f:
+
+        detector = f.create_group("/entry/instrument/detector")
+        detector.attrs["NX_class"] = "NXdetector"
+        detector["beam_center_x"] = 2079.79727597266
+        detector["beam_center_y"] = 2225.38773853771
+        detector["count_time"] = 0.00285260857097799
+        detector["depends_on"] = "/entry/instrument/detector/transformations/det_z"
+        detector["description"] = "Eiger 16M"
+        detector["distance"] = 0.237015940260233
+        detector.create_dataset("data", data=np.zeros((100, 100)))
+        detector["sensor_material"] = "Silicon"
+        detector["sensor_thickness"] = 0.00045
+        detector["sensor_thickness"].attrs["units"] = b"m"
+        detector["x_pixel_size"] = 7.5e-05
+        detector["y_pixel_size"] = 7.5e-05
+        detector["underload_value"] = 0
+        detector["saturation_value"] = 9266
+        detector["frame_time"] = 0.1
+        detector["frame_time"].attrs["units"] = "s"
+        detector["bit_depth_readout"] = np.array(32)
+        mask = np.zeros((2, 100, 200), dtype="i8")
+        detector.create_dataset("pixel_mask", data=mask)
+
+        detector_transformations = detector.create_group("transformations")
+        detector_transformations.attrs["NX_class"] = "NXtransformations"
+        det_z = detector_transformations.create_dataset("det_z", data=np.array([289.3]))
+        det_z.attrs["depends_on"] = b"."
+        det_z.attrs["transformation_type"] = b"translation"
+        det_z.attrs["units"] = b"mm"
+        det_z.attrs["vector"] = np.array([0.0, 0.0, 1.0])
+
+        def make_module(name, depends_on, data_origin, fast_direction, slow_direction):
+            module = detector.create_group(name)
+            module.attrs["NX_class"] = "NXdetector_module"
+            module.create_dataset("data_size", data=np.array([1, 100, 200]))
+            module.create_dataset("data_origin", data=np.array(data_origin))
+            fast = module.create_dataset("fast_pixel_direction", data=0.075)
+            fast.attrs["transformation_type"] = "translation"
+            fast.attrs["depends_on"] = depends_on
+            fast.attrs["vector"] = np.array(fast_direction)
+            fast.attrs["units"] = "mm"
+            slow = module.create_dataset("slow_pixel_direction", data=0.075)
+            slow.attrs["transformation_type"] = "translation"
+            slow.attrs["depends_on"] = depends_on
+            slow.attrs["vector"] = np.array(slow_direction)
+            slow.attrs["units"] = "mm"
+
+        make_module(
+            name="m0",
+            depends_on="/entry/instrument/detector/transformations/det_z",
+            data_origin=[0, 0, 0],
+            fast_direction=[-0.999998, -0.001781, 0],
+            slow_direction=[-0.001781, 0.999998, 0],
+        )
+        make_module(
+            name="m1",
+            depends_on="/entry/instrument/detector/transformations/det_z",
+            data_origin=[1, 0, 0],
+            fast_direction=[-0.999998, -0.001781, 0],
+            slow_direction=[-0.001781, 0.999998, 0],
+        )
+
+        nxdata = f.create_group("/entry/data")
+        nxdata.attrs["NX_class"] = "NXdata"
+        nxdata.create_dataset(
+            "data",
+            data=np.array(
+                [np.full((2, 100, 200), i, dtype=np.int32) for i in range(3)]
+            ),
+        )
+        nxdata.attrs["signal"] = "/entry/data/data"
+
+        yield f
+
+
+def test_get_dxtbx_detector_with_multiple_modules(detector_with_multiple_modules):
+    det = nxmx.NXdetector(detector_with_multiple_modules["/entry/instrument/detector"])
+    wavelength = 1
+
+    detector = dxtbx.nexus.get_dxtbx_detector(det, wavelength)
+    assert len(detector) == 2
+    for panel in detector:
+        assert panel.get_image_size() == (200, 100)
+
+    nxdata = nxmx.NXdata(detector_with_multiple_modules["/entry/data"])
+    for i in range(3):
+        raw_data = dxtbx.nexus.get_raw_data(nxdata, det, i)
+        assert len(raw_data) == 2
+        for module_data in raw_data:
+            assert module_data.all() == (100, 200)
+            assert module_data.all_eq(i)
+
+    mask = dxtbx.nexus.get_static_mask(det)
+    assert len(mask) == 2
+    for module_mask in mask:
+        assert isinstance(module_mask, flex.bool)
+        assert module_mask.all() == (100, 200)
+
+
 @pytest.fixture
 def detector_with_two_theta():
     with h5py.File(" ", "w", **pytest.h5_in_memory) as f:

From 5e079c7929194b2502ef796590ddbb48b3ca2bef Mon Sep 17 00:00:00 2001
From: Nicholas Devenish <ndevenish@gmail.com>
Date: Fri, 11 Aug 2023 13:14:14 +0100
Subject: [PATCH 37/45] Update CI prebuilt version of CCTBX (#655)

---
 .azure-pipelines/ci-conda-env.txt | 2 +-
 newsfragments/655.misc            | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 newsfragments/655.misc

diff --git a/.azure-pipelines/ci-conda-env.txt b/.azure-pipelines/ci-conda-env.txt
index a9ffd554e..5cc5c4fdd 100644
--- a/.azure-pipelines/ci-conda-env.txt
+++ b/.azure-pipelines/ci-conda-env.txt
@@ -2,7 +2,7 @@ conda-forge::boost
 conda-forge::boost-cpp
 conda-forge::bzip2
 conda-forge::c-compiler<1.5
-conda-forge::cctbx-base==2023.5
+conda-forge::cctbx-base==2023.7
 conda-forge::conda
 conda-forge::cxx-compiler<1.5
 conda-forge::python-dateutil
diff --git a/newsfragments/655.misc b/newsfragments/655.misc
new file mode 100644
index 000000000..57176f832
--- /dev/null
+++ b/newsfragments/655.misc
@@ -0,0 +1 @@
+Update prebuilt CI CCTBX version.

From eda7169c68a720d9f6739b563a1529572121c6b2 Mon Sep 17 00:00:00 2001
From: DiamondLightSource-build-server
 <DiamondLightSource-build-server@users.noreply.github.com>
Date: Sun, 13 Aug 2023 05:11:09 +0100
Subject: [PATCH 38/45] Clean Clutter

---
 src/dxtbx/format/FormatNXmxDLSI19_2.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/dxtbx/format/FormatNXmxDLSI19_2.py b/src/dxtbx/format/FormatNXmxDLSI19_2.py
index 255330eb4..5617fa360 100644
--- a/src/dxtbx/format/FormatNXmxDLSI19_2.py
+++ b/src/dxtbx/format/FormatNXmxDLSI19_2.py
@@ -46,7 +46,6 @@ def __init__(self, image_file, **kwargs):
         except Exception:
             self._bit_depth_readout = 16
 
-
     def get_goniometer_shadow_masker(self, goniometer=None):
         """Apply the dynamic mask for a diamond anvil cell with a 76° aperture."""
         if goniometer is None:

From f464b576e1fd6cdef55ebade4c0e70a5c9038b67 Mon Sep 17 00:00:00 2001
From: DiamondLightSource-build-server
 <DiamondLightSource-build-server@users.noreply.github.com>
Date: Mon, 14 Aug 2023 10:25:01 +0100
Subject: [PATCH 39/45] dxtbx 3.16.0 Changelog

    towncrier --name=dxtbx --version='3.16.0'
---
 CHANGELOG.rst             | 30 ++++++++++++++++++++++++++++++
 newsfragments/299.bugfix  |  1 -
 newsfragments/439.bugfix  |  1 -
 newsfragments/612.bugfix  |  1 -
 newsfragments/621.feature |  1 -
 newsfragments/626.feature |  1 -
 newsfragments/633.bugfix  |  1 -
 newsfragments/640.misc    |  1 -
 newsfragments/642.misc    |  1 -
 newsfragments/643.misc    |  1 -
 newsfragments/645.feature |  1 -
 newsfragments/647.feature |  1 -
 newsfragments/649.feature |  1 -
 newsfragments/650.misc    |  1 -
 newsfragments/651.feature |  1 -
 newsfragments/652.bugfix  |  1 -
 newsfragments/653.feature |  1 -
 newsfragments/655.misc    |  1 -
 18 files changed, 30 insertions(+), 17 deletions(-)
 delete mode 100644 newsfragments/299.bugfix
 delete mode 100644 newsfragments/439.bugfix
 delete mode 100644 newsfragments/612.bugfix
 delete mode 100644 newsfragments/621.feature
 delete mode 100644 newsfragments/626.feature
 delete mode 100644 newsfragments/633.bugfix
 delete mode 100644 newsfragments/640.misc
 delete mode 100644 newsfragments/642.misc
 delete mode 100644 newsfragments/643.misc
 delete mode 100644 newsfragments/645.feature
 delete mode 100644 newsfragments/647.feature
 delete mode 100644 newsfragments/649.feature
 delete mode 100644 newsfragments/650.misc
 delete mode 100644 newsfragments/651.feature
 delete mode 100644 newsfragments/652.bugfix
 delete mode 100644 newsfragments/653.feature
 delete mode 100644 newsfragments/655.misc

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 101191acd..6e5f63187 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,3 +1,33 @@
+dxtbx 3.16.0 (2023-08-14)
+=========================
+
+Features
+--------
+
+- Add new Beam class ``dxtbx.model.PolychromaticBeam``, for polychromatic/multi-wavelength/wide bandpass experiments. (`#621 <https://github.com/cctbx/dxtbx/issues/621>`_)
+- Formats: Reflect move of Eiger detector from PETRA P14 to P13. (`#626 <https://github.com/cctbx/dxtbx/issues/626>`_)
+- The ``model.Beam`` object now has a ``probe`` value to keep track of the type of radiation. (`#647 <https://github.com/cctbx/dxtbx/issues/647>`_)
+- Formats: CBFMini support for the EIGER2 16M detector at CHESS beamline ID7B2, which has an inverted rotation axis. (`#649 <https://github.com/cctbx/dxtbx/issues/649>`_)
+- Formats: Support for Eiger 9M on ESRF ID23-2, which has an undeclared vertical goniometer. (`#651 <https://github.com/cctbx/dxtbx/issues/651>`_)
+- Formats: Partial support for the Rigaku Oxford Diffraction file format, including support for multi-axis goniometers and faster decompression. (`#645 <https://github.com/cctbx/dxtbx/issues/645>`_) (`#653 <https://github.com/cctbx/dxtbx/issues/653>`_)
+
+
+Bugfixes
+--------
+
+- Panel geometry definitions in PHIL are merged by panel id *before* constructing panels. (`#299 <https://github.com/cctbx/dxtbx/issues/299>`_)
+- ``flumpy``: Fix case where incorrect ``flex.vec2``, ``flex.vec3`` could be generated. (`#439 <https://github.com/cctbx/dxtbx/issues/439>`_)
+- NXmx files with multidimensional arrays (images, modules, or both) are now handled. (`#612 <https://github.com/cctbx/dxtbx/issues/612>`_)
+- Slicing of imageset objects is now consistently 0-based, including for the sliced data accessor. Previously, the data accessor had to be accessed with the original index offsets. (`#633 <https://github.com/cctbx/dxtbx/issues/633>`_)
+- Formats: Add fix for Eiger / NXmx data from DLS i19-2, to correctly assign the image bit depth. (`#652 <https://github.com/cctbx/dxtbx/issues/652>`_)
+
+
+Misc
+----
+
+- `#640 <https://github.com/cctbx/dxtbx/issues/640>`_, `#642 <https://github.com/cctbx/dxtbx/issues/642>`_, `#643 <https://github.com/cctbx/dxtbx/issues/643>`_, `#645 <https://github.com/cctbx/dxtbx/issues/645>`_, `#650 <https://github.com/cctbx/dxtbx/issues/650>`_, `#655 <https://github.com/cctbx/dxtbx/issues/655>`_
+
+
 DIALS 3.15.1 (2023-06-29)
 =========================
 
diff --git a/newsfragments/299.bugfix b/newsfragments/299.bugfix
deleted file mode 100644
index 5a8abfc77..000000000
--- a/newsfragments/299.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Panel geometry definitions in PHIL are merged by panel id before constructing panels
diff --git a/newsfragments/439.bugfix b/newsfragments/439.bugfix
deleted file mode 100644
index a97cc5116..000000000
--- a/newsfragments/439.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-``flumpy``: Fix case where incorrect ``flex.vec2``, ``flex.vec3`` could be generated.
diff --git a/newsfragments/612.bugfix b/newsfragments/612.bugfix
deleted file mode 100644
index 6f4e30c86..000000000
--- a/newsfragments/612.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-NXmx files with multidimensional arrays (images, modules, or both) are now handled.
diff --git a/newsfragments/621.feature b/newsfragments/621.feature
deleted file mode 100644
index 394ff8e74..000000000
--- a/newsfragments/621.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add new Beam class "PolychromaticBeam" for polychromatic/multi-wavelength/wide bandpass experiments.
diff --git a/newsfragments/626.feature b/newsfragments/626.feature
deleted file mode 100644
index c28cc4293..000000000
--- a/newsfragments/626.feature
+++ /dev/null
@@ -1 +0,0 @@
-Update Format handling to reflect move of Eiger detector from PETRA P14 to P13.
diff --git a/newsfragments/633.bugfix b/newsfragments/633.bugfix
deleted file mode 100644
index 7ce5110e5..000000000
--- a/newsfragments/633.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Slicing of imageset objects is now consistently 0-based, including for the sliced data accessor. Previously, the data accessor had to be accessed with the original index offsets.
diff --git a/newsfragments/640.misc b/newsfragments/640.misc
deleted file mode 100644
index 8183d5860..000000000
--- a/newsfragments/640.misc
+++ /dev/null
@@ -1 +0,0 @@
-Use Python built-in ``subprocess`` in place of external dependency ``procrunner`` for running CLI tests.
diff --git a/newsfragments/642.misc b/newsfragments/642.misc
deleted file mode 100644
index b8a2436d1..000000000
--- a/newsfragments/642.misc
+++ /dev/null
@@ -1 +0,0 @@
-Drop Python 3.8 support in line with https://dials.github.io/kb/proposals/dc3. Add Python 3.11 support.
diff --git a/newsfragments/643.misc b/newsfragments/643.misc
deleted file mode 100644
index 0780b3d13..000000000
--- a/newsfragments/643.misc
+++ /dev/null
@@ -1 +0,0 @@
-Move from deprecated pkg_resources to importlib.
diff --git a/newsfragments/645.feature b/newsfragments/645.feature
deleted file mode 100644
index fb38a5121..000000000
--- a/newsfragments/645.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add partial support for the Rigaku Oxford Diffraction file format.
diff --git a/newsfragments/647.feature b/newsfragments/647.feature
deleted file mode 100644
index 0ff6b0846..000000000
--- a/newsfragments/647.feature
+++ /dev/null
@@ -1 +0,0 @@
-The ``Beam`` model now has a ``probe`` value to keep track of the type of radiation.
diff --git a/newsfragments/649.feature b/newsfragments/649.feature
deleted file mode 100644
index eb736b9a4..000000000
--- a/newsfragments/649.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add CBFMini support for the EIGER2 16M detector at CHESS beamline ID7B2, which has an inverted rotation axis.
diff --git a/newsfragments/650.misc b/newsfragments/650.misc
deleted file mode 100644
index 94f0ed982..000000000
--- a/newsfragments/650.misc
+++ /dev/null
@@ -1 +0,0 @@
-Format classes are now tested against invalid binary data with dials-data, for when dials-regression is not present.
diff --git a/newsfragments/651.feature b/newsfragments/651.feature
deleted file mode 100644
index 2f790c707..000000000
--- a/newsfragments/651.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add support for Eiger 9M on ESRF ID23-2, which has an undeclared vertical goniometer.
diff --git a/newsfragments/652.bugfix b/newsfragments/652.bugfix
deleted file mode 100644
index ec7635b59..000000000
--- a/newsfragments/652.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-``dxtbx``: add fix for Eiger / NXmx data from i19-2 to correctly assign the image bit depth
diff --git a/newsfragments/653.feature b/newsfragments/653.feature
deleted file mode 100644
index 22c7d8c0e..000000000
--- a/newsfragments/653.feature
+++ /dev/null
@@ -1 +0,0 @@
-``FormatROD``: include support for multi-axis goniometers and faster decompression.
diff --git a/newsfragments/655.misc b/newsfragments/655.misc
deleted file mode 100644
index 57176f832..000000000
--- a/newsfragments/655.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update prebuilt CI CCTBX version.

From df3e0b69c85b9f9b318c5c8c528a0dd09cb78430 Mon Sep 17 00:00:00 2001
From: DiamondLightSource-build-server
 <DiamondLightSource-build-server@users.noreply.github.com>
Date: Mon, 14 Aug 2023 10:25:02 +0100
Subject: [PATCH 40/45] =?UTF-8?q?Bump=20version:=203.16.dev=20=E2=86=92=20?=
 =?UTF-8?q?3.17.dev?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 setup.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 64494b80e..9417f73e2 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.16.dev
+current_version = 3.17.dev
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<release>[a-z]+)?(?P<patch>\d+)?
diff --git a/setup.py b/setup.py
index 8213d9921..8fbd4a3e7 100644
--- a/setup.py
+++ b/setup.py
@@ -8,7 +8,7 @@
 
 # Static version number which is updated by bump2version
 # Do not change this manually - use 'bump2version <major/minor/patch/release>'
-__version_tag__ = "3.16.dev"
+__version_tag__ = "3.17.dev"
 
 setup_kwargs = {
     "name": "dxtbx",

From 0554ef39d31bf62804c24c598771c20d84ce9b21 Mon Sep 17 00:00:00 2001
From: "Aaron S. Brewster" <asbrewster@lbl.gov>
Date: Mon, 14 Aug 2023 11:26:01 -0700
Subject: [PATCH 41/45] NXmx general writer (#615)

Writes any data that dxtbx can import, out to NeXus NXmx .
---
 newsfragments/615.feature       |   1 +
 src/dxtbx/format/nxmx_writer.py | 844 ++++++++++++++++++++++++++++++++
 src/dxtbx/imageset.py           |   5 +-
 src/dxtbx/nexus/__init__.py     |  13 +-
 tests/nexus/test_nxmx_writer.py | 111 +++++
 5 files changed, 972 insertions(+), 2 deletions(-)
 create mode 100644 newsfragments/615.feature
 create mode 100644 src/dxtbx/format/nxmx_writer.py
 create mode 100644 tests/nexus/test_nxmx_writer.py

diff --git a/newsfragments/615.feature b/newsfragments/615.feature
new file mode 100644
index 000000000..b13456787
--- /dev/null
+++ b/newsfragments/615.feature
@@ -0,0 +1 @@
+Add nxmx_writer, a tool for converting any data dxtbx can read to NeXus data
diff --git a/src/dxtbx/format/nxmx_writer.py b/src/dxtbx/format/nxmx_writer.py
new file mode 100644
index 000000000..c9dfccd2b
--- /dev/null
+++ b/src/dxtbx/format/nxmx_writer.py
@@ -0,0 +1,844 @@
+"""
+Note, scans and gonios not supported here. This writer essentially writes still images
+
+Example to write 10 cbfs to a single NXmx file:
+writer = NXmxWriter("*.cbf")
+writer.write_nxmx("example.h5")
+"""
+
+from __future__ import annotations
+
+import datetime
+import h5py
+import numpy as np
+import os
+import sys
+
+from cctbx import factor_ev_angstrom
+from dials.util.options import ArgumentParser, flatten_experiments
+from dxtbx import flumpy
+from libtbx import easy_pickle
+from libtbx.phil import parse
+from libtbx.utils import Sorry
+from scitbx import matrix
+from scitbx.array_family import flex
+from xfel.cftbx.detector.cspad_cbf_tbx import basis, angle_and_axis
+
+help_message = """
+Create a NeXus file from either an experiment list or a set of image files
+
+Uses NXmx as documented here:
+https://manual.nexusformat.org/classes/applications/NXmx.html
+and in Bernstein et. al. (2020):
+https://doi.org/10.1107/S2052252520008672
+"""
+
+phil_scope = parse(
+    """
+  output_file = None
+    .type = path
+    .help = output file path
+  wavelength = None
+    .type = float
+    .help = If not provided, use data from files provided.
+  mask_file = None
+    .type = str
+    .help = Path to file with bad pixel mask in DIALS mask format
+  trusted_range = None
+    .type = floats(size=2)
+    .help = Override the trusted range
+  compression = gzip
+    .type = str
+    .help = Compression to apply to the data
+  dtype = None
+    .type = str
+    .help = Override the data type. If data is floats and an integer type is specified, \
+            the data is first rounded.
+  nexus_details {
+    instrument_name = None
+      .type = str
+      .help = Name of instrument. Consistency with the controlled vocabulary beamline naming in \
+              https://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Items/_diffrn_source.pdbx_synchrotron_beamline.html \
+              and https://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Items/_diffrn_source.type.html \
+              is highly recommended.
+    instrument_short_name = None
+      .type = str
+      .help = short name for instrument, perhaps the acronym
+    source_name = None
+      .type = str
+      .help = Name of the neutron or x-ray storage ring/facility. Consistency with the naming in \
+              https://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Items/_diffrn_source.pdbx_synchrotron_site.html \
+              controlled vocabulary is highly recommended.
+    source_short_name = None
+      .type = str
+      .help = short name for source, perhaps the acronym
+    start_time = None
+      .type = str
+      .help = ISO 8601 time/date of the first data point collected in UTC, \
+              using the Z suffix to avoid confusion with local time
+    end_time = None
+      .type = str
+      .help = ISO 8601 time/date of the last data point collected in UTC, \
+              using the Z suffix to avoid confusion with local time. \
+              This field should only be filled when the value is accurately \
+              observed. If the data collection aborts or otherwise prevents \
+              accurate recording of the end_time, this field should be omitted
+    end_time_estimated = None
+      .type = str
+      .help = ISO 8601 time/date of the last data point collected in UTC, \
+              using the Z suffix to avoid confusion with local time. \
+              This field may be filled with a value estimated before an \
+              observed value is avilable.
+    count_time = None
+      .type = float
+      .help = Elapsed actual counting time
+    frame_time = None
+      .type = float
+      .help = This is time for each frame. This is exposure_time + readout time \
+              in seconds.
+    sample_name = None
+      .type = str
+      .help = Descriptive name of sample
+    total_flux = None
+      .type = float
+      .help = flux incident on beam plane in photons per second
+  }
+"""
+)
+
+# Conversion from the imgCIF coordinate system conventionally used by dxtbx to
+# the McStas coordinate system as used by NeXus:
+#   https://www.iucr.org/__data/iucr/cifdic_html/2/cif_img.dic/Caxis.html
+#   https://manual.nexusformat.org/design.html#design-coordinatesystem
+IMGCIF_TO_MCSTAS = matrix.diag([-1, 1, -1])
+
+
+class NXmxWriter:
+    """Class for writing NXmx NeXus files from any dxtbx-supported format class"""
+
+    def __init__(self, params, experiments=None, imageset=None):
+        self.params = params
+        if experiments or imageset:
+            self.setup(experiments, imageset)
+        self.handle = None
+
+    def setup(self, experiments=None, imageset=None):
+        assert [experiments, imageset].count(
+            None
+        ) == 1, "Supply either experiments or imagset, not both"
+        if experiments:
+            self.imagesets = experiments.imagesets()
+            assert len(experiments.detectors()) == 1, "Multiple detectors not supported"
+            self.detector = experiments.detectors()[0]
+            self.beams = experiments.beams()
+            assert len(experiments.scans()) <= 1
+            assert len(experiments.goniometers()) <= 1
+            self.scan = experiments[0].scan
+            self.goniometer = experiments[0].goniometer
+
+            if self.scan or self.goniometer:
+                assert self.scan and self.goniometer
+                assert len(experiments) == 1
+        else:
+            self.imagesets = [imageset]
+            self.detector = imageset.get_detector(0)
+            self.beams = [imageset.get_beam(i) for i in range(len(imageset))]
+
+            self.scan = imageset.get_scan(0)
+            self.goniometer = imageset.get_goniometer(0)
+        self.construct_entry()
+
+    def construct_entry(self):
+        output_file_name = (
+            self.params.output_file
+            if self.params.output_file is not None
+            else "converted.h5"
+        )
+        self.handle = h5py.File(output_file_name, "w")
+        f = self.handle
+        f.attrs["NX_class"] = "NXroot"
+        f.attrs["file_name"] = os.path.basename(output_file_name)
+        f.attrs["file_time"] = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
+        f.attrs["HDF5_Version"] = h5py.version.hdf5_version
+        entry = f.create_group("entry")
+        entry.attrs["NX_class"] = "NXentry"
+        if self.params.nexus_details.start_time:
+            entry["start_time"] = self.params.nexus_details.start_time
+        if self.params.nexus_details.end_time:
+            entry["end_time"] = self.params.nexus_details.end_time
+        if self.params.nexus_details.end_time_estimated:
+            entry["end_time_estimated"] = self.params.nexus_details.end_time_estimated
+
+        # --> definition
+        self._create_scalar(entry, "definition", "S4", np.string_("NXmx"))
+
+        # --> sample
+        sample = self.handle["entry"].create_group("sample")
+        sample.attrs["NX_class"] = "NXsample"
+        if self.params.nexus_details.sample_name:
+            sample["name"] = self.params.nexus_details.sample_name
+        sample["depends_on"] = "."  # Will be overriden if a scan and gonio are added
+
+    def __call__(self, experiments=None, imageset=None):
+        if experiments or imageset:
+            self.setup(experiments, imageset)
+        self.validate()
+        self.construct_detector()
+        self.add_all_beams()
+        self.append_all_frames()
+        self.add_scan_and_gonio()
+
+    def validate(self):
+        if not self.params.nexus_details.instrument_name:
+            raise ValueError("instrument_name is required.")
+        if not self.params.nexus_details.source_name:
+            raise ValueError("source_name is required.")
+        if not self.params.nexus_details.start_time:
+            raise ValueError("start_time is required.")
+        if not self.params.nexus_details.end_time_estimated:
+            raise ValueError("end_time_estimated is required.")
+        if not self.params.nexus_details.sample_name:
+            raise ValueError("sample_name is required.")
+
+    def get_metrology_dict(self):
+        """Build a metrology dictionary.  This dictionary maps hierarchy keys to basis
+        objects. A hierarchy key looks like this (0,1,2), where the entries are
+        levels in a hierarchy and the numbers refer to a panel or group within that
+        level"""
+        metro = {}
+
+        def recursive_setup_dict(panelgroup, key):
+            metro[key] = basis(panelgroup=panelgroup)
+            if panelgroup.is_panel():
+                return
+            for i, child in enumerate(panelgroup):
+                childkey = tuple(list(key) + [i])
+                recursive_setup_dict(child, childkey)
+
+        recursive_setup_dict(self.detector.hierarchy(), (0,))
+        return metro
+
+    def _create_scalar(self, handle, path, dtype, value):
+        dataset = handle.create_dataset(path, (), dtype=dtype)
+        dataset[()] = value
+
+    def create_vector(self, handle, name, value, **attributes):
+        handle.create_dataset(name, (1,), data=[value], dtype="f")
+        for key, attribute in attributes.items():
+            handle[name].attrs[key] = attribute
+
+    def add_frame_shift(self, handle, basis):
+        """Add an axis representing a frame shift (a rotation axis with an offset)"""
+        angle, axis = angle_and_axis(basis)
+
+        if angle == 0:
+            axis = matrix.col((0, 0, 1))
+
+        if basis.include_translation:
+            translation = basis.translation
+        else:
+            translation = matrix.col((0, 0, 0))
+
+        axis = IMGCIF_TO_MCSTAS * axis
+        translation = IMGCIF_TO_MCSTAS * translation
+
+        self.create_vector(
+            handle,
+            os.path.basename(basis.name),
+            angle,
+            depends_on=basis.depends_on,
+            equipment="detector",
+            equipment_component=basis.equipment_component,
+            transformation_type="rotation",
+            units="deg",
+            vector=axis,
+            offset=translation,
+            offset_units="mm",
+        )
+
+    def construct_detector(self, detector=None):
+        """
+        Hierarchical structure of master nexus file. Format information available here
+        http://download.nexusformat.org/sphinx/classes/base_classes/NXdetector_module.html#nxdetector-module
+        --> entry
+          --> data
+          --> definition (leaf)
+          --> instrument
+          --> sample
+        """
+        if not self.handle:
+            # setup was skipped, so not using imagesets/experiments
+            self.construct_entry()
+            assert detector
+            self.detector = detector
+        if not detector:
+            detector = self.detector
+
+        # set up the metrology dictionary to include axis names, pixel sizes, and so forth
+        metro = self.get_metrology_dict()
+
+        def panel_group_from_key(key):
+            # Find the node that a hierarchy key refers to
+            if len(key) == 1:
+                assert key[0] == 0
+                return detector.hierarchy()
+
+            node = detector.hierarchy()
+            for i in key[1:]:
+                node = node[i]
+            return node
+
+        def level_string(key):
+            # Example for key (0,1,2). "L0M0_L1M1_L2M2", where L is level and M is module
+            return "_".join(["L%dM%d" % (l, m) for l, m in enumerate(key)])
+
+        def recursive_setup_basis_dict(key, parent_name="", panel_id=0):
+            # Set up NeXus axis names, including equipment components and depends_on chains
+            basis = metro[key]
+            node = panel_group_from_key(key)
+            nodename = level_string(key)
+            if basis.name:
+                dname = os.path.split(basis.name)[1]
+            else:
+                basis.name = dname = "AXIS_" + nodename
+
+            if node.is_panel():
+                panel_id += 1
+
+            if len(key) == 1:
+                assert key == (0,)  # only one detector allowed for now
+                basis.depends_on = "."
+            else:
+                basis.depends_on = parent_name
+
+            basis.equipment_component = "detector_level_%d" % (len(key) - 1)
+            basis.axis_name = dname
+            if not node.is_panel():
+                for c, child in enumerate(node):
+                    panel_id = recursive_setup_basis_dict(
+                        tuple(list(key) + [c]), dname, panel_id
+                    )
+            return panel_id
+
+        f = self.handle
+        entry = f["entry"]
+        recursive_setup_basis_dict((0,))
+
+        # --> source
+        source = entry.create_group("source")
+        source.attrs["NX_class"] = "NXsource"
+        source["name"] = self.params.nexus_details.source_name
+        if self.params.nexus_details.source_short_name:
+            source["name"].attrs[
+                "short_name"
+            ] = self.params.nexus_details.source_short_name
+        # --> instrument
+        instrument = entry.create_group("instrument")
+        instrument.attrs["NX_class"] = "NXinstrument"
+        instrument["name"] = self.params.nexus_details.instrument_name
+        if self.params.nexus_details.instrument_short_name:
+            instrument["name"].attrs[
+                "short_name"
+            ] = self.params.nexus_details.instrument_short_name
+        beam = instrument.create_group("beam")
+        beam.attrs["NX_class"] = "NXbeam"
+        if self.params.nexus_details.total_flux:
+            self._create_scalar(
+                beam, "total_flux", "f", self.params.nexus_details.total_flux
+            )
+            beam["total_flux"].attrs["units"] = "Hz"
+            beam.attrs["flux"] = "total_flux"
+
+        det_group = instrument.create_group("detector_group")
+        det_group.attrs["NX_class"] = "NXdetector_group"
+
+        det_group.create_dataset("group_index", data=list(range(1, 3)), dtype="i")
+        data = [np.string_("detector"), np.string_("detector")]
+        det_group.create_dataset("group_names", (2,), data=data, dtype="S12")
+        det_group.create_dataset("group_parent", (2,), data=[-1, 1], dtype="i")
+        det_group.create_dataset("group_type", (2,), data=[1, 2], dtype="i")
+        det = instrument.create_group("detector")
+        det.attrs["NX_class"] = "NXdetector"
+        det["description"] = "Detector converted from DIALS models"
+        det["depends_on"] = "/entry/instrument/detector/transformations/AXIS_RAIL"
+        det["gain_setting"] = "auto"
+        assert len(set([p.get_material() for p in detector])) == 1
+        assert len(set([p.get_thickness() for p in detector])) == 1
+        det["sensor_material"] = detector[0].get_material()
+        self._create_scalar(
+            det, "sensor_thickness", "f", detector[0].get_thickness() * 1000
+        )
+        det["sensor_thickness"].attrs["units"] = "microns"
+        if self.params.nexus_details.count_time is not None:
+            self._create_scalar(
+                det, "count_time", "f", self.params.nexus_details.count_time
+            )
+            det["count_time"].attrs["units"] = "us"
+        if self.params.nexus_details.frame_time is not None:
+            self._create_scalar(
+                det, "frame_time", "f", self.params.nexus_details.frame_time
+            )
+            det["frame_time"].attrs["units"] = "s"
+
+        if self.params.mask_file is not None:
+            mask = easy_pickle.load(self.params.mask_file)
+            if len(mask) > 1:
+                mask = [flumpy.to_numpy(m) for m in mask]
+                mask = np.stack(mask)
+            else:
+                mask = flumpy.to_numpy(mask[0])
+            mask = (~mask).astype(np.int32)
+            det.create_dataset("pixel_mask", mask.shape, data=mask, dtype=mask.dtype)
+
+        transformations = det.create_group("transformations")
+        transformations.attrs["NX_class"] = "NXtransformations"
+
+        if self.params.trusted_range is None:
+            assert len(set([p.get_trusted_range() for p in detector])) == 1
+            trusted_min, trusted_max = detector[0].get_trusted_range()
+        else:
+            trusted_min, trusted_max = self.params.trusted_range
+        # DIALS definitions match up with NXmx
+        det.create_dataset("underload_value", (1,), data=[trusted_min], dtype="int32")
+        det.create_dataset("saturation_value", (1,), data=[trusted_max], dtype="int32")
+
+        def find_panel_id(panel):
+            for i in range(len(detector)):
+                if detector[i].get_name() == panel.get_name():
+                    return i
+
+        # Create a series of axis describing frame shifts from each level of the detector to the next
+        for key in sorted(metro):
+            basis = metro[key]
+            node = panel_group_from_key(key)
+
+            if node.is_panel():
+                aname = level_string(key)
+                fast = (
+                    IMGCIF_TO_MCSTAS
+                    * basis.orientation.unit_quaternion_as_r3_rotation_matrix()
+                    * matrix.col((1.0, 0.0, 0.0))
+                )
+                slow = (
+                    IMGCIF_TO_MCSTAS
+                    * basis.orientation.unit_quaternion_as_r3_rotation_matrix()
+                    * matrix.col((0.0, 1.0, 0.0))
+                )
+
+                module = det.create_group(aname)
+                module.attrs["NX_class"] = "NXdetector_module"
+
+                if len(detector) > 1:
+                    panel_id = find_panel_id(node)
+                    module.create_dataset(
+                        "data_origin", (3,), data=[panel_id, 0, 0], dtype="i"
+                    )
+                    module.create_dataset(
+                        "data_size",
+                        (3,),
+                        data=[1, *list(reversed(node.get_image_size()))],
+                        dtype="i",
+                    )
+                else:
+                    module.create_dataset("data_origin", (2,), data=[0, 0], dtype="i")
+                    module.create_dataset(
+                        "data_size",
+                        (2,),
+                        data=list(reversed(node.get_image_size())),
+                        dtype="i",
+                    )
+
+                self.create_vector(
+                    module,
+                    "fast_pixel_direction",
+                    node.get_pixel_size()[1],
+                    depends_on=transformations.name
+                    + "/"
+                    + os.path.basename(basis.depends_on),
+                    transformation_type="translation",
+                    units="mm",
+                    vector=fast,
+                    offset=IMGCIF_TO_MCSTAS * basis.translation,
+                    offset_units="mm",
+                )
+                self.create_vector(
+                    module,
+                    "slow_pixel_direction",
+                    node.get_pixel_size()[0],
+                    depends_on=transformations.name
+                    + "/"
+                    + os.path.basename(basis.depends_on),
+                    transformation_type="translation",
+                    units="mm",
+                    vector=slow,
+                    offset=IMGCIF_TO_MCSTAS * basis.translation,
+                    offset_units="mm",
+                )
+            else:
+                self.add_frame_shift(transformations, basis)
+
+    def add_all_beams(self):
+        spectra = []
+        for imageset in self.imagesets:
+            spectra.extend(
+                [
+                    imageset.get_spectrum(i)
+                    for i in range(len(imageset))
+                    if imageset.get_spectrum(i)
+                ]
+            )
+        self.add_beams(self.beams, spectra)
+
+    def add_beams(self, beams=None, spectra=None):
+        handle = self.handle["entry/instrument/beam"]
+        if self.params.wavelength:
+            assert beams is None and spectra is None
+            handle.create_dataset(
+                "incident_wavelength", (1,), data=self.params.wavelength, dtype="f8"
+            )
+        else:
+            assert beams
+            if spectra:
+                assert len(beams) == len(spectra)
+                if len(beams) > 1:
+                    test = None
+                    for spectrum in spectra[1:]:
+                        if test:
+                            test &= (
+                                spectrum.get_energies_eV()
+                                == spectra[0].get_energies_eV()
+                            )
+                        else:
+                            test = (
+                                spectrum.get_energies_eV()
+                                == spectra[0].get_energies_eV()
+                            )
+                    matching_channels = test.count(False) == 0
+
+                    if matching_channels:
+                        spectra_x = (
+                            factor_ev_angstrom
+                            / spectra[0].get_energies_eV().as_numpy_array()
+                        )
+                    else:
+                        spectra_x = np.stack(
+                            [
+                                factor_ev_angstrom
+                                / s.get_energies_eV().as_numpy_array()
+                                for s in spectra
+                            ]
+                        )
+                    spectra_y = np.stack(
+                        [s.get_weights().as_numpy_array() for s in spectra]
+                    )
+                else:
+                    spectra_x = (
+                        factor_ev_angstrom
+                        / spectra[0].get_energies_eV().as_numpy_array()
+                    )
+                    spectra_y = spectra[0].get_weights().as_numpy_array()
+
+                matching_weighted_wavelengths = (
+                    spectra[0].get_weighted_wavelength() == beams[0].get_wavelength()
+                )
+                if matching_weighted_wavelengths:
+                    handle.create_dataset(
+                        "incident_wavelength",
+                        spectra_x.shape,
+                        data=spectra_x,
+                        dtype=spectra_x.dtype,
+                    )
+                    handle.create_dataset(
+                        "incident_wavelength_weights",
+                        spectra_y.shape,
+                        data=spectra_y,
+                        dtype=spectra_y.dtype,
+                    )
+                else:
+                    wavelengths = np.array(
+                        [beam.get_wavelength() for beam in self.beams],
+                        dtype="f8",
+                    )
+                    handle.create_dataset(
+                        "incident_wavelength",
+                        wavelengths.shape,
+                        data=wavelengths,
+                        dtype=wavelengths.dtype,
+                    )
+                    handle.create_dataset(
+                        "incident_wavelength_1Dspectrum",
+                        spectra_x.shape,
+                        data=spectra_x,
+                        dtype=spectra_x.dtype,
+                    )
+                    handle.create_dataset(
+                        "incident_wavelength_1Dspectrum_weights",
+                        spectra_y.shape,
+                        data=spectra_y,
+                        dtype=spectra_y.dtype,
+                    )
+                    handle["incident_wavelength_1Dspectrum"].attrs["units"] = "angstrom"
+                    handle["incident_wavelength"].attrs[
+                        "variant"
+                    ] = "incident_wavelength_1Dspectrum"
+            else:
+                if len(beams) > 1:
+                    wavelengths = np.array(
+                        [beam.get_wavelength() for beam in self.beams],
+                        dtype="f8",
+                    )
+                    handle.create_dataset(
+                        "incident_wavelength",
+                        wavelengths.shape,
+                        data=wavelengths,
+                        dtype=wavelengths.dtype,
+                    )
+                else:
+                    wavelength = self.beams[0].get_wavelength()
+                    handle.create_dataset(
+                        "incident_wavelength", (1,), data=wavelength, dtype="f8"
+                    )
+        handle["incident_wavelength"].attrs["units"] = "angstrom"
+
+    def add_beam_in_sequence(self, beam, spectrum=None):
+        """
+        Alternate way to add beams, assuming adding them one at a time
+
+        For spectra, this usecase assumes only one set of energy channels and no variants
+        """
+        handle = self.handle["entry/instrument/beam"]
+        if spectrum:
+            spectrum_x = (
+                factor_ev_angstrom / spectrum.get_energies_eV().as_numpy_array()
+            )
+            spectrum_y = spectrum.get_weights().as_numpy_array()
+
+            if "incident_wavelength" in handle:
+                weights = handle["incident_wavelength_weights"]
+                weights.resize(weights.shape[0] + 1, axis=0)
+            else:
+                handle.create_dataset(
+                    "incident_wavelength",
+                    spectrum_x.shape,
+                    data=spectrum_x,
+                    dtype=spectrum_x.dtype,
+                )
+                weights = handle.create_dataset(
+                    "incident_wavelength_weights",
+                    (1, len(spectrum_y)),
+                    maxshape=(None, len(spectrum_y)),
+                    dtype=spectrum_y.dtype,
+                )
+            weights[-1] = spectrum_y
+        else:
+            wavelength = beam.get_wavelength()
+            if "incident_wavelength" in handle:
+                dset = handle["incident_wavelength"]
+                dset.resize(dset.shape[0] + 1, axis=0)
+            else:
+                dset = handle.create_dataset(
+                    "incident_wavelength", (1,), maxshape=(None,), dtype="f8"
+                )
+            dset[-1] = wavelength
+        handle["incident_wavelength"].attrs["units"] = "angstrom"
+
+    def append_all_frames(self):
+        """
+        Given a h5py handle, append all the data. from the imagesets in the
+        original experiment list.
+        """
+        for imageset in self.imagesets:
+            for i in range(len(imageset)):
+                self.append_frame(data=imageset[i])
+
+    def append_frame(self, index=None, data=None):
+        """
+        Append some data. Can specify either
+        the index into imageset zero, or the data itself.
+        """
+        if data is None:
+            if index is None:
+                data = self.imagesets[0][0]
+            else:
+                data = self.imagesets[0][index]
+        if not isinstance(data, tuple):
+            data = (data,)
+
+        if len(data) > 1:
+            assert len(set([d.focus() for d in data])) == 1
+            shape = len(data), data[0].focus()[0], data[0].focus()[1]
+        else:
+            shape = data[0].focus()[0], data[0].focus()[1]
+
+        dataisint = []
+        for panel_data in data:
+            assert len(panel_data.focus()) == 2
+            if isinstance(panel_data, flex.int):
+                dataisint.append(True)
+            elif isinstance(panel_data, flex.double):
+                dataisint.append(False)
+            else:
+                raise TypeError("Ints or doubles are required")
+        assert all(dataisint) or not any(dataisint), "Mix of ints and doubles found"
+        dataisint = all(dataisint)
+
+        if self.params.dtype:
+            if self.params.dtype in ["int", "float"]:
+                dtype = int if self.params.dtype == "int" else float
+            else:
+                dtype = np.dtype(self.params.dtype)
+            if not dataisint and np.issubdtype(dtype, np.integer):
+                data = [p.iround() for p in data]
+        else:
+            dtype = int if dataisint else float
+
+        det = self.handle["entry/instrument/detector"]
+        entry = self.handle["entry"]
+        if "data" in entry:
+            data_group = entry["data"]
+            dset = data_group["data"]
+            dset.resize(dset.shape[0] + 1, axis=0)
+        else:
+            data_group = entry.create_group("data")
+            data_group.attrs["NX_class"] = "NXdata"
+            dset = data_group.create_dataset(
+                "data",
+                (1, *shape),
+                maxshape=(None, *shape),
+                dtype=dtype,
+                compression=self.params.compression,
+            )
+
+        if len(data) > 1:
+            data = [flumpy.to_numpy(d) for d in data]
+            data = np.stack(data)
+        else:
+            data = flumpy.to_numpy(data[0])
+        dset[-1:] = data.astype(dtype)
+
+        if "bit_depth_readout" not in det:
+            self._create_scalar(det, "bit_depth_readout", "i", dset.dtype.itemsize * 8)
+
+    def add_scan_and_gonio(self, scan=None, gonio=None):
+        if scan is None or gonio is None:
+
+            assert scan is None and gonio is None
+            scan = self.scan
+            gonio = self.goniometer
+        if scan is None or gonio is None:
+            return
+
+        sample = self.handle["entry/sample"]
+        if "depends_on" in sample:
+            del sample["depends_on"]
+        multi_axis = hasattr(gonio, "get_axes")
+
+        if multi_axis:
+            root = gonio.get_names()[0]
+        else:
+            root = "omega"
+        sample["depends_on"] = "/entry/sample/transformations/%s" % (root)
+
+        transformations = sample.create_group("transformations")
+        transformations.attrs["NX_class"] = "NXtransformations"
+
+        def setup_axis(name, vector, main_axis=False):
+            if main_axis:
+                angles = np.array(
+                    [
+                        scan.get_angle_from_array_index(i)
+                        for i in range(*scan.get_array_range())
+                    ]
+                )
+            else:
+                angles = [0]
+            axis = transformations.create_dataset(name, data=angles)
+
+            if main_axis:
+                oscillation_angle = scan.get_oscillation()[1]
+                oscillation = np.array([oscillation_angle] * len(scan))
+                transformations.create_dataset(
+                    name + "_increment_set", data=oscillation
+                )
+
+                end = angles + oscillation
+                transformations.create_dataset(name + "_end", data=end)
+            axis.attrs["transformation_type"] = "rotation"
+            axis.attrs["units"] = "deg"
+            axis.attrs["vector"] = IMGCIF_TO_MCSTAS.as_numpy_array() @ np.array(vector)
+            return axis
+
+        if multi_axis:
+            for axis_number in range(len(gonio.get_axes())):
+                if axis_number == gonio.get_scan_axis():
+                    axis = setup_axis(
+                        gonio.get_names()[axis_number],
+                        gonio.get_axes()[axis_number],
+                        main_axis=True,
+                    )
+                else:
+                    axis = setup_axis(
+                        gonio.get_names()[axis_number],
+                        gonio.get_axes()[axis_number],
+                        main_axis=False,
+                    )
+
+                if axis_number == len(gonio.get_axes()) - 1:
+                    axis.attrs["depends_on"] = "."
+                else:
+                    axis.attrs["depends_on"] = "/entry/sample/transformations/%s" % (
+                        gonio.get_names()[axis_number + 1]
+                    )
+        else:
+            setup_axis("omega", gonio.get_rotation_axis(), main_axis=True)
+
+        assert len(set(scan.get_exposure_times())) == 1
+        if self.params.nexus_details.frame_time is None:
+            self._create_scalar(
+                self.handle["entry/instrument/detector"],
+                "frame_time",
+                "f",
+                scan.get_exposure_times()[0],
+            )
+            self.handle["entry/instrument/detector/frame_time"].attrs["units"] = "s"
+
+    def setup_multi_crystal(self, multi_crystal_group_name):
+        self.handle["entry/sample"].create_group(multi_crystal_group_name)
+
+    def add_crystal(self, multi_crystal_group_name, crystal):
+        from dials.utils.nexus import nx_mx
+
+        # Assume this is the next crystal in the list, added after the last image
+        # was added
+        n_images = self.handle["entry/data/data"].shape[0]
+        nx_mx.dump_crystal(
+            self.handle["entry/sample/" + multi_crystal_group_name],
+            crystal,
+            None,
+            "crystal%d" % (n_images - 1),
+        )
+
+
+def run(args):
+    usage = "dials.python nxmx_writer.py <experiments OR image files>"
+    parser = ArgumentParser(
+        usage=usage,
+        sort_options=True,
+        phil=phil_scope,
+        read_experiments=True,
+        read_experiments_from_images=True,
+        epilog=help_message,
+    )
+    params, options = parser.parse_args(args=args, show_diff_phil=True)
+
+    experiments = flatten_experiments(params.input.experiments)
+
+    try:
+        NXmxWriter(params)(experiments)
+    except ValueError as e:
+        print("Missing information. Run with -c to see full parameter description")
+        raise Sorry(e)
+
+
+if __name__ == "__main__":
+    run(sys.argv[1:])
diff --git a/src/dxtbx/imageset.py b/src/dxtbx/imageset.py
index 85fd2200b..564ced681 100644
--- a/src/dxtbx/imageset.py
+++ b/src/dxtbx/imageset.py
@@ -149,7 +149,10 @@ def get_spectrum(self, index):
             format_instance = self.get_format_class().get_instance(
                 self.get_path(index), **kwargs
             )
-        return format_instance.get_spectrum(self.indices()[index])
+        try:
+            return format_instance.get_spectrum(self.indices()[index])
+        except TypeError:
+            return format_instance.get_spectrum()
 
     def params(self):
         """Get the parameters"""
diff --git a/src/dxtbx/nexus/__init__.py b/src/dxtbx/nexus/__init__.py
index 6fc0d20c6..6b28d9dc7 100644
--- a/src/dxtbx/nexus/__init__.py
+++ b/src/dxtbx/nexus/__init__.py
@@ -390,7 +390,18 @@ def equipment_component_key(dependency):
                 module.fast_pixel_direction.depends_on
             )
             A = nxmx.get_cumulative_transformation(dependency_chain)
-            origin = MCSTAS_TO_IMGCIF @ A[0, :3, 3]
+
+            origin = MCSTAS_TO_IMGCIF @ (
+                (
+                    module.fast_pixel_direction.offset.to("mm").magnitude
+                    if module.fast_pixel_direction.offset is not None
+                    else np.array([0.0, 0.0, 0.0])
+                    + module.slow_pixel_direction.offset.to("mm").magnitude
+                    if module.slow_pixel_direction.offset is not None
+                    else np.array([0.0, 0.0, 0.0])
+                )
+                + A[0, :3, 3]
+            )
 
             if (
                 origin[0] == 0
diff --git a/tests/nexus/test_nxmx_writer.py b/tests/nexus/test_nxmx_writer.py
new file mode 100644
index 000000000..fb9f60273
--- /dev/null
+++ b/tests/nexus/test_nxmx_writer.py
@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import glob
+
+import h5py
+import pytest
+
+from libtbx.phil import parse
+from libtbx.test_utils import approx_equal
+
+from dxtbx.format.nexus import h5str
+from dxtbx.model.experiment_list import ExperimentListFactory
+
+pytest.importorskip("dials")
+pytest.importorskip("xfel")
+
+from dxtbx.format.nxmx_writer import NXmxWriter, phil_scope
+
+
+def test_writer_jf16M(dials_data, tmpdir):
+    h5path = (
+        dials_data("lysozyme_JF16M_4img", pathlib=True)
+        / "lyso009a_0087.JF07T32V01_master_4img.h5"
+    )
+    with h5py.File(h5path) as handle:
+        output_file = tmpdir / "4img.h5"
+        instrument_name = h5str(handle["entry/instrument/name"][()])
+        source_name = h5str(handle["entry/source/name"][()])
+        start_time = h5str(handle["entry/start_time"][()])
+        end_time_estimated = h5str(handle["entry/end_time_estimated"][()])
+        sample_name = h5str(handle["entry/sample/name"][()])
+        params = phil_scope.fetch(
+            parse(
+                f"""
+        output_file = {output_file}
+        nexus_details {{
+          instrument_name = {instrument_name}
+          source_name = {source_name}
+          start_time = {start_time}
+          end_time_estimated = {end_time_estimated}
+          sample_name = {sample_name}
+        }}
+        """
+            )
+        ).extract()
+
+    expts1 = ExperimentListFactory.from_filenames([h5path])
+
+    NXmxWriter(params)(expts1)
+
+    expts2 = ExperimentListFactory.from_filenames([output_file])
+
+    def recursive_test(pg1, pg2):
+        assert pg1.is_similar_to(pg2)
+        for c1, c2 in zip(pg1, pg2):
+            recursive_test(c1, c2)
+
+    for d1, d2 in zip(expts1.detectors(), expts2.detectors()):
+        recursive_test(d1.hierarchy(), d2.hierarchy())
+
+    for b1, b2 in zip(expts1.beams(), expts2.beams()):
+        assert approx_equal(b1.get_wavelength(), b2.get_wavelength())
+        assert approx_equal(b2.get_s0(), b2.get_s0())
+
+    assert not (any(expts2.scans()))
+    assert not (any(expts2.goniometers()))
+
+
+def test_writer_x4wide(dials_data, tmpdir):
+    cbfspath = dials_data("x4wide", pathlib=True) / "X4_wide_M1S4_2_000*.cbf"
+
+    output_file = tmpdir / "x4wide_10img.h5"
+    params = phil_scope.fetch(
+        parse(
+            f"""
+    output_file = {output_file}
+    nexus_details {{
+      instrument_name = PILATUS
+      source_name =  PILATUS 6M-F, S/N 60-0114-F
+      start_time = 2013-02-08T13:23:40.833
+      end_time_estimated = 2013-02-08T13:24:00.000
+      sample_name = sample_x4wide
+    }}
+    """
+        )
+    ).extract()
+
+    expts1 = ExperimentListFactory.from_filenames(glob.glob(str(cbfspath)))
+
+    NXmxWriter(params)(expts1)
+
+    expts2 = ExperimentListFactory.from_filenames([tmpdir / "x4wide_10img.h5"])
+
+    def recursive_test(pg1, pg2):
+        assert pg1.is_similar_to(pg2)
+        for c1, c2 in zip(pg1, pg2):
+            recursive_test(c1, c2)
+
+    for d1, d2 in zip(expts1.detectors(), expts2.detectors()):
+        recursive_test(d1.hierarchy(), d2.hierarchy())
+
+    for b1, b2 in zip(expts1.beams(), expts2.beams()):
+        assert approx_equal(b1.get_wavelength(), b2.get_wavelength())
+        assert approx_equal(b2.get_s0(), b2.get_s0())
+
+    for scan1, scan2 in zip(expts1.scans(), expts2.scans()):
+        assert approx_equal(scan1.get_array_range(), scan2.get_array_range())
+        assert approx_equal(scan1.get_oscillation(), scan2.get_oscillation())
+
+    for gonio1, gonio2 in zip(expts1.goniometers(), expts2.goniometers()):
+        assert approx_equal(gonio1.get_rotation_axis(), gonio2.get_rotation_axis())

From 730b8d563386e1e8a65342925701e89e327e783b Mon Sep 17 00:00:00 2001
From: Nicholas Devenish <ndevenish@gmail.com>
Date: Tue, 15 Aug 2023 12:16:48 +0100
Subject: [PATCH 42/45] Fix undefined behaviour from unthrown error (#656)

Added in code review for 4783b2c, but without throw.
---
 newsfragments/656.bugfix | 1 +
 src/dxtbx/model/beam.h   | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)
 create mode 100644 newsfragments/656.bugfix

diff --git a/newsfragments/656.bugfix b/newsfragments/656.bugfix
new file mode 100644
index 000000000..6696aae3a
--- /dev/null
+++ b/newsfragments/656.bugfix
@@ -0,0 +1 @@
+Fix situation where a bad ``Beam.probe`` could cause undefined behaviour.
diff --git a/src/dxtbx/model/beam.h b/src/dxtbx/model/beam.h
index 6df5325d1..7f9cefb37 100644
--- a/src/dxtbx/model/beam.h
+++ b/src/dxtbx/model/beam.h
@@ -319,7 +319,7 @@ namespace dxtbx { namespace model {
       case neutron:
         return std::string("neutron");
       default:
-        DXTBX_ERROR("Unknown probe type");
+        throw DXTBX_ERROR("Unknown probe type");
       }
     }
 
@@ -335,7 +335,7 @@ namespace dxtbx { namespace model {
         return Probe::neutron;
       }
 
-      DXTBX_ERROR("Unknown probe " + probe);
+      throw DXTBX_ERROR("Unknown probe " + probe);
     }
 
     void set_probe(Probe probe) {

From 831061fe2acdbe74fa4f09f526d7b5a03fe6a1cb Mon Sep 17 00:00:00 2001
From: David McDonagh <60879630+toastisme@users.noreply.github.com>
Date: Wed, 16 Aug 2023 14:23:45 +0100
Subject: [PATCH 43/45] Add sample to source distance to Beam (#622)

* Add sample_to_source_distance to Beam

---------

Co-authored-by: DiamondLightSource-build-server <DiamondLightSource-build-server@users.noreply.github.com>
Co-authored-by: Nicholas Devenish <ndevenish@gmail.com>
---
 newsfragments/622.misc               |   1 +
 src/dxtbx/dxtbx_model_ext.pyi        |  22 +++
 src/dxtbx/model/beam.h               | 207 +++++++++++++++++++++++++--
 src/dxtbx/model/beam.py              |   8 ++
 src/dxtbx/model/boost_python/beam.cc |  96 ++++++++++++-
 tests/model/test_beam.py             |   9 +-
 6 files changed, 326 insertions(+), 17 deletions(-)
 create mode 100644 newsfragments/622.misc

diff --git a/newsfragments/622.misc b/newsfragments/622.misc
new file mode 100644
index 000000000..033071027
--- /dev/null
+++ b/newsfragments/622.misc
@@ -0,0 +1 @@
+Add sample_to_source_distance to Beam
diff --git a/src/dxtbx/dxtbx_model_ext.pyi b/src/dxtbx/dxtbx_model_ext.pyi
index 8a4a85934..9ca38cffd 100644
--- a/src/dxtbx/dxtbx_model_ext.pyi
+++ b/src/dxtbx/dxtbx_model_ext.pyi
@@ -57,6 +57,10 @@ class BeamBase:
         self, points: Union[Tuple[Vec3Float], List[Vec3Float]]
     ) -> None: ...
     def get_sample_to_source_direction(self) -> Vec3Float: ...
+    def get_sample_to_source_distance(self) -> float: ...
+    def set_sample_to_source_distance(
+        self, sample_to_source_distance: float
+    ) -> None: ...
     def get_sigma_divergence(self) -> float: ...
     def set_sigma_divergence(self, sigma_divergence: float) -> None: ...
     def get_transmission(self) -> float: ...
@@ -110,6 +114,22 @@ class Beam(BeamBase):
         polarization_fraction: float,
         flux: float,
         transmission: float,
+        probe: Probe = ...,
+        deg: bool = ...,
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        direction: Vec3Float,
+        wavelength: float,
+        divergence: float,
+        sigma_divergence: float,
+        polarization_normal: Vec3Float,
+        polarization_fraction: float,
+        flux: float,
+        transmission: float,
+        probe: Probe = ...,
+        sample_to_source_distance: float = ...,
         deg: bool = ...,
     ) -> None: ...
     @staticmethod
@@ -141,6 +161,8 @@ class PolychromaticBeam(Beam):
         polarization_fraction: float,
         flux: float,
         transmission: float,
+        probe: Probe = ...,
+        sample_to_source_distance: float = ...,
         deg: bool = ...,
     ) -> None: ...
     @staticmethod
diff --git a/src/dxtbx/model/beam.h b/src/dxtbx/model/beam.h
index 7f9cefb37..42aaffe07 100644
--- a/src/dxtbx/model/beam.h
+++ b/src/dxtbx/model/beam.h
@@ -34,6 +34,7 @@ namespace dxtbx { namespace model {
     virtual ~BeamBase() {}
 
     virtual vec3<double> get_sample_to_source_direction() const = 0;
+    virtual double get_sample_to_source_distance() const = 0;
     virtual double get_wavelength() const = 0;
     virtual double get_divergence() const = 0;
     // Get the standard deviation of the beam divergence
@@ -66,13 +67,19 @@ namespace dxtbx { namespace model {
     virtual void set_s0_at_scan_points(
       const scitbx::af::const_ref<vec3<double> > &s0) = 0;
     virtual void set_probe(Probe probe) = 0;
+    virtual void set_sample_to_source_distance(double sample_to_source_distance) = 0;
 
     virtual void reset_scan_points() = 0;
     virtual bool is_similar_to(const BeamBase &rhs,
                                double wavelength_tolerance,
                                double direction_tolerance,
                                double polarization_normal_tolerance,
-                               double polarization_fraction_tolerance) const = 0;
+                               double polarization_fraction_tolerance,
+                               double divergence_tolerance,
+                               double sigma_divergence_tolerance,
+                               double flux_tolerance,
+                               double transmission_tolerance,
+                               double sample_to_source_distance_tolerance) const = 0;
     virtual void rotate_around_origin(vec3<double> axis, double angle) = 0;
     virtual bool operator!=(const BeamBase &rhs) const = 0;
     virtual bool operator==(const BeamBase &rhs) const = 0;
@@ -90,7 +97,8 @@ namespace dxtbx { namespace model {
           polarization_fraction_(0.999),
           flux_(0),
           transmission_(1.0),
-          probe_(Probe::xray) {}
+          probe_(Probe::xray),
+          sample_to_source_distance_(0.) {}
 
     /**
      * @param s0 The incident beam vector.
@@ -102,7 +110,8 @@ namespace dxtbx { namespace model {
           polarization_fraction_(0.999),
           flux_(0),
           transmission_(1.0),
-          probe_(Probe::xray) {
+          probe_(Probe::xray),
+          sample_to_source_distance_(0.) {
       DXTBX_ASSERT(s0.length() > 0);
       wavelength_ = 1.0 / s0.length();
       direction_ = -s0.normalize();
@@ -120,7 +129,8 @@ namespace dxtbx { namespace model {
           polarization_fraction_(0.999),
           flux_(0),
           transmission_(1.0),
-          probe_(Probe::xray) {
+          probe_(Probe::xray),
+          sample_to_source_distance_(0.) {
       DXTBX_ASSERT(direction.length() > 0);
       direction_ = direction.normalize();
     }
@@ -137,7 +147,8 @@ namespace dxtbx { namespace model {
           polarization_fraction_(0.999),
           flux_(0),
           transmission_(1.0),
-          probe_(Probe::xray) {
+          probe_(Probe::xray),
+          sample_to_source_distance_(0.) {
       DXTBX_ASSERT(s0.length() > 0);
       wavelength_ = 1.0 / s0.length();
       direction_ = -s0.normalize();
@@ -160,7 +171,8 @@ namespace dxtbx { namespace model {
           polarization_fraction_(0.999),
           flux_(0),
           transmission_(1.0),
-          probe_(Probe::xray) {
+          probe_(Probe::xray),
+          sample_to_source_distance_(0.) {
       DXTBX_ASSERT(direction.length() > 0);
       direction_ = direction.normalize();
     }
@@ -192,7 +204,43 @@ namespace dxtbx { namespace model {
           polarization_fraction_(polarization_fraction),
           flux_(flux),
           transmission_(transmission),
-          probe_(probe) {
+          probe_(probe),
+          sample_to_source_distance_(0.) {
+      DXTBX_ASSERT(direction.length() > 0);
+      direction_ = direction.normalize();
+    }
+
+    /**
+     * @param direction The beam direction vector from source to sample
+     * @param wavelength The wavelength of the beam
+     * @param divergence The beam divergence
+     * @param sigma_divergence The standard deviation of the beam divergence
+     * @param polarization_normal The polarization plane
+     * @param polarization_fraction The polarization fraction
+     * @param flux The beam flux
+     * @param transmission The beam transmission
+     * @param probe The probe value
+     * @param sample_to_source_distance (mm)
+     */
+    Beam(vec3<double> direction,
+         double wavelength,
+         double divergence,
+         double sigma_divergence,
+         vec3<double> polarization_normal,
+         double polarization_fraction,
+         double flux,
+         double transmission,
+         Probe probe,
+         double sample_to_source_distance)
+        : wavelength_(wavelength),
+          divergence_(divergence),
+          sigma_divergence_(sigma_divergence),
+          polarization_normal_(polarization_normal),
+          polarization_fraction_(polarization_fraction),
+          flux_(flux),
+          transmission_(transmission),
+          probe_(probe),
+          sample_to_source_distance_(sample_to_source_distance) {
       DXTBX_ASSERT(direction.length() > 0);
       direction_ = direction.normalize();
     }
@@ -346,6 +394,17 @@ namespace dxtbx { namespace model {
       s0_at_scan_points_.clear();
     }
 
+    /* Distance from sample to source in mm */
+    double get_sample_to_source_distance() const {
+      return sample_to_source_distance_;
+    }
+
+    /* Distance from sample to source in mm */
+    void set_sample_to_source_distance(double sample_to_source_distance) {
+      DXTBX_ASSERT(sample_to_source_distance >= 0.);
+      sample_to_source_distance_ = sample_to_source_distance;
+    }
+
     virtual bool operator==(const BeamBase &rhs) const {
       double eps = 1.0e-6;
 
@@ -378,6 +437,11 @@ namespace dxtbx { namespace model {
                   <= eps
              && std::abs(polarization_fraction_ - rhs.get_polarization_fraction())
                   <= eps
+             && std::abs(flux_ - rhs.get_flux()) <= eps
+             && std::abs(transmission_ - rhs.get_transmission()) <= eps
+             && std::abs(sample_to_source_distance_
+                         - rhs.get_sample_to_source_distance())
+                  <= eps
              && (probe_ == rhs.get_probe());
     }
 
@@ -419,6 +483,57 @@ namespace dxtbx { namespace model {
              && (probe_ == rhs.get_probe());
     }
 
+    virtual bool is_similar_to(const BeamBase &rhs,
+                               double wavelength_tolerance,
+                               double direction_tolerance,
+                               double polarization_normal_tolerance,
+                               double polarization_fraction_tolerance,
+                               double divergence_tolerance = 1e-6,
+                               double sigma_divergence_tolerance = 1e-6,
+                               double flux_tolerance = 1e-6,
+                               double transmission_tolerance = 1e-6,
+                               double sample_to_source_tolerance = 1e-6) const {
+      // scan varying model checks
+      if (get_num_scan_points() != rhs.get_num_scan_points()) {
+        return false;
+      }
+      for (std::size_t i = 0; i < get_num_scan_points(); ++i) {
+        vec3<double> s0_a = get_s0_at_scan_point(i);
+        vec3<double> s0_b = rhs.get_s0_at_scan_point(i);
+
+        vec3<double> us0_a = s0_a.normalize();
+        vec3<double> us0_b = s0_b.normalize();
+        if (std::abs(angle_safe(us0_a, us0_b)) > direction_tolerance) {
+          return false;
+        }
+
+        double wavelength_a = 1.0 / s0_a.length();
+        double wavelength_b = 1.0 / s0_b.length();
+        if (std::abs(wavelength_a - wavelength_b) > wavelength_tolerance) {
+          return false;
+        }
+      }
+
+      // static model checks
+      return std::abs(angle_safe(direction_, rhs.get_sample_to_source_direction()))
+               <= direction_tolerance
+             && std::abs(wavelength_ - rhs.get_wavelength()) <= wavelength_tolerance
+             && std::abs(
+                  angle_safe(polarization_normal_, rhs.get_polarization_normal()))
+                  <= polarization_normal_tolerance
+             && std::abs(polarization_fraction_ - rhs.get_polarization_fraction())
+                  <= polarization_fraction_tolerance
+             && std::abs(divergence_ - rhs.get_divergence()) <= divergence_tolerance
+             && std::abs(sigma_divergence_ - rhs.get_sigma_divergence())
+                  <= sigma_divergence_tolerance
+             && std::abs(flux_ - rhs.get_flux()) <= flux_tolerance
+             && std::abs(transmission_ - rhs.get_transmission())
+                  <= transmission_tolerance
+             && std::abs(sample_to_source_distance_
+                         - rhs.get_sample_to_source_distance())
+                  <= sample_to_source_tolerance;
+    }
+
     bool operator!=(const BeamBase &rhs) const {
       return !(*this == rhs);
     }
@@ -439,6 +554,7 @@ namespace dxtbx { namespace model {
     double flux_;
     double transmission_;
     Probe probe_;
+    double sample_to_source_distance_;
 
   private:
     double wavelength_;
@@ -459,9 +575,10 @@ namespace dxtbx { namespace model {
     os << "    polarization fraction: " << b.get_polarization_fraction() << "\n";
     os << "    flux: " << b.get_flux() << "\n";
     os << "    transmission: " << b.get_transmission() << "\n";
+    os << "    sample to source distance: " << b.get_sample_to_source_distance()
+       << "\n";
     return os;
   }
-
   class PolychromaticBeam : public Beam {
   public:
     PolychromaticBeam() {
@@ -473,6 +590,7 @@ namespace dxtbx { namespace model {
       set_flux(0);
       set_transmission(1.0);
       set_probe(Probe::xray);
+      set_sample_to_source_distance(0.0);
     }
 
     /**
@@ -488,6 +606,24 @@ namespace dxtbx { namespace model {
       set_flux(0);
       set_transmission(1.0);
       set_probe(Probe::xray);
+      set_sample_to_source_distance(0.0);
+    }
+
+    /**
+     * @param direction The beam direction pointing source to sample
+     * @param sample_to_source_distance (mm)
+     */
+    PolychromaticBeam(vec3<double> direction, double sample_to_source_distance) {
+      DXTBX_ASSERT(direction.length() > 0);
+      direction_ = direction.normalize();
+      set_sample_to_source_distance(sample_to_source_distance);
+      set_divergence(0.0);
+      set_sigma_divergence(0.0);
+      set_polarization_normal(vec3<double>(0.0, 1.0, 0.0));
+      set_polarization_fraction(0.999);
+      set_flux(0);
+      set_transmission(1.0);
+      set_probe(Probe::xray);
     }
 
     /**
@@ -507,6 +643,7 @@ namespace dxtbx { namespace model {
       set_flux(0);
       set_transmission(1.0);
       set_probe(Probe::xray);
+      set_sample_to_source_distance(0.0);
     }
 
     /**
@@ -536,6 +673,39 @@ namespace dxtbx { namespace model {
       set_flux(flux);
       set_transmission(transmission);
       set_probe(probe);
+      set_sample_to_source_distance(0.0);
+    }
+
+    /**
+     * @param direction The beam direction pointing source to sample
+     * @param divergence The beam divergence
+     * @param sigma_divergence The standard deviation of the beam divergence
+     * @param polarization_normal The polarization plane
+     * @param polarization_fraction The polarization fraction
+     * @param flux The beam flux
+     * @param transmission The beam transmission
+     * @param probe The probe value
+     * @param sample_to_source_distance (mm)
+     */
+    PolychromaticBeam(vec3<double> direction,
+                      double divergence,
+                      double sigma_divergence,
+                      vec3<double> polarization_normal,
+                      double polarization_fraction,
+                      double flux,
+                      double transmission,
+                      Probe probe,
+                      double sample_to_source_distance) {
+      DXTBX_ASSERT(direction.length() > 0);
+      direction_ = direction.normalize();
+      set_divergence(divergence);
+      set_sigma_divergence(sigma_divergence);
+      set_polarization_normal(polarization_normal);
+      set_polarization_fraction(polarization_fraction);
+      set_flux(flux);
+      set_transmission(transmission);
+      set_probe(probe);
+      set_sample_to_source_distance(sample_to_source_distance);
     }
 
     double get_wavelength() const {
@@ -591,6 +761,11 @@ namespace dxtbx { namespace model {
                   <= eps
              && std::abs(polarization_fraction_ - rhs.get_polarization_fraction())
                   <= eps
+             && std::abs(flux_ - rhs.get_flux()) <= eps
+             && std::abs(transmission_ - rhs.get_transmission()) <= eps
+             && std::abs(sample_to_source_distance_
+                         - rhs.get_sample_to_source_distance())
+                  <= eps
              && (probe_ == rhs.get_probe());
     }
 
@@ -608,7 +783,12 @@ namespace dxtbx { namespace model {
     bool is_similar_to(const BeamBase &rhs,
                        double direction_tolerance,
                        double polarization_normal_tolerance,
-                       double polarization_fraction_tolerance) const {
+                       double polarization_fraction_tolerance,
+                       double divergence_tolerance = 1e-6,
+                       double sigma_divergence_tolerance = 1e-6,
+                       double flux_tolerance = 1e-6,
+                       double transmission_tolerance = 1e-6,
+                       double sample_to_source_tolerance = 1e-6) const {
       return std::abs(angle_safe(direction_, rhs.get_sample_to_source_direction()))
                <= direction_tolerance
              && std::abs(
@@ -616,6 +796,12 @@ namespace dxtbx { namespace model {
                   <= polarization_normal_tolerance
              && std::abs(polarization_fraction_ - rhs.get_polarization_fraction())
                   <= polarization_fraction_tolerance
+             && std::abs(flux_ - rhs.get_flux()) <= flux_tolerance
+             && std::abs(transmission_ - rhs.get_transmission())
+                  <= transmission_tolerance
+             && std::abs(sample_to_source_distance_
+                         - rhs.get_sample_to_source_distance())
+                  <= sample_to_source_tolerance
              && (probe_ == rhs.get_probe());
     }
   };
@@ -633,9 +819,10 @@ namespace dxtbx { namespace model {
     os << "    polarization fraction: " << b.get_polarization_fraction() << "\n";
     os << "    flux: " << b.get_flux() << "\n";
     os << "    transmission: " << b.get_transmission() << "\n";
+    os << "    sample to source distance : " << b.get_sample_to_source_distance()
+       << "\n";
     return os;
   }
-
 }}  // namespace dxtbx::model
 
 #endif  // DXTBX_MODEL_BEAM_H
diff --git a/src/dxtbx/model/beam.py b/src/dxtbx/model/beam.py
index a7360e9e5..799d9be32 100644
--- a/src/dxtbx/model/beam.py
+++ b/src/dxtbx/model/beam.py
@@ -66,6 +66,10 @@
         .type = float
         .help = "Override the flux"
         .short_caption = "flux"
+
+    sample_to_source_distance = None
+        .type = float
+        .help = "Override the distance between sample and source (mm)"
   }
 """
 )
@@ -116,6 +120,8 @@ def from_phil(
             beam.set_transmission(params.beam.transmission)
         if params.beam.flux is not None:
             beam.set_flux(params.beam.flux)
+        if params.beam.sample_to_source_distance is not None:
+            beam.set_sample_to_source_distance(params.beam.sample_to_source_distance)
         beam.set_probe(Beam.get_probe_from_name(params.beam.probe))
 
         return beam
@@ -186,6 +192,7 @@ def make_polychromatic_beam(
         flux: float = 0.0,
         transmission: float = 1.0,
         probe: Probe = Probe.xray,
+        sample_to_source_distance: float = 0.0,
         deg: bool = True,
     ) -> PolychromaticBeam:
         return PolychromaticBeam(
@@ -197,6 +204,7 @@ def make_polychromatic_beam(
             float(flux),
             float(transmission),
             probe,
+            float(sample_to_source_distance),
             bool(deg),
         )
 
diff --git a/src/dxtbx/model/boost_python/beam.cc b/src/dxtbx/model/boost_python/beam.cc
index 86cdab927..0cb9ea4c5 100644
--- a/src/dxtbx/model/boost_python/beam.cc
+++ b/src/dxtbx/model/boost_python/beam.cc
@@ -44,7 +44,8 @@ namespace dxtbx { namespace model { namespace boost_python {
                                          obj.get_polarization_fraction(),
                                          obj.get_flux(),
                                          obj.get_transmission(),
-                                         obj.get_probe());
+                                         obj.get_probe(),
+                                         obj.get_sample_to_source_distance());
       }
 
       static boost::python::tuple getstate(boost::python::object obj) {
@@ -138,6 +139,44 @@ namespace dxtbx { namespace model { namespace boost_python {
       return beam;
     }
 
+    static Beam *make_beam_w_sample_to_source_distance(vec3<double> sample_to_source,
+                                                       double wavelength,
+                                                       double divergence,
+                                                       double sigma_divergence,
+                                                       vec3<double> polarization_normal,
+                                                       double polarization_fraction,
+                                                       double flux,
+                                                       double transmission,
+                                                       Probe probe,
+                                                       double sample_to_source_distance,
+                                                       bool deg) {
+      Beam *beam = NULL;
+      if (deg) {
+        beam = new Beam(sample_to_source,
+                        wavelength,
+                        deg_as_rad(divergence),
+                        deg_as_rad(sigma_divergence),
+                        polarization_normal,
+                        polarization_fraction,
+                        flux,
+                        transmission,
+                        probe,
+                        sample_to_source_distance);
+      } else {
+        beam = new Beam(sample_to_source,
+                        wavelength,
+                        divergence,
+                        sigma_divergence,
+                        polarization_normal,
+                        polarization_fraction,
+                        flux,
+                        transmission,
+                        probe,
+                        sample_to_source_distance);
+      }
+      return beam;
+    }
+
     static double get_divergence(const Beam &beam, bool deg) {
       double divergence = beam.get_divergence();
       return deg ? rad_as_deg(divergence) : divergence;
@@ -197,6 +236,7 @@ namespace dxtbx { namespace model { namespace boost_python {
     result["flux"] = obj.get_flux();
     result["transmission"] = obj.get_transmission();
     result["probe"] = obj.get_probe_name();
+    result["sample_to_source_distance"] = obj.get_sample_to_source_distance();
     if (obj.get_num_scan_points() > 0) {
       boost::python::list l;
       scitbx::af::shared<vec3<double> > s0_at_scan_points = obj.get_s0_at_scan_points();
@@ -229,6 +269,10 @@ namespace dxtbx { namespace model { namespace boost_python {
         boost::python::extract<boost::python::list>(obj["s0_at_scan_points"]);
       beam_detail::Beam_set_s0_at_scan_points_from_list(*b, s0_at_scan_points);
     }
+    if (obj.has_key("sample_to_source_distance")) {
+      double val = boost::python::extract<double>(obj["sample_to_source_distance"]);
+      b->set_sample_to_source_distance(val);
+    }
     return b;
   }
 
@@ -249,7 +293,8 @@ namespace dxtbx { namespace model { namespace boost_python {
                                        obj.get_polarization_fraction(),
                                        obj.get_flux(),
                                        obj.get_transmission(),
-                                       obj.get_probe());
+                                       obj.get_probe(),
+                                       obj.get_sample_to_source_distance());
     }
   };
 
@@ -271,6 +316,12 @@ namespace dxtbx { namespace model { namespace boost_python {
     return beam;
   }
 
+  static PolychromaticBeam *make_PolychromaticBeam_w_sample_to_source_distance(
+    vec3<double> direction,
+    double sample_to_source_distance) {
+    return new PolychromaticBeam(direction, sample_to_source_distance);
+  }
+
   static PolychromaticBeam *make_PolychromaticBeam_w_all(
     vec3<double> direction,
     double divergence,
@@ -280,6 +331,7 @@ namespace dxtbx { namespace model { namespace boost_python {
     double flux,
     double transmission,
     Probe probe,
+    double sample_to_source_distance,
     bool deg) {
     PolychromaticBeam *beam = NULL;
     if (deg) {
@@ -290,7 +342,8 @@ namespace dxtbx { namespace model { namespace boost_python {
                                    polarization_fraction,
                                    flux,
                                    transmission,
-                                   probe);
+                                   probe,
+                                   sample_to_source_distance);
     } else {
       beam = new PolychromaticBeam(direction,
                                    divergence,
@@ -299,7 +352,8 @@ namespace dxtbx { namespace model { namespace boost_python {
                                    polarization_fraction,
                                    flux,
                                    transmission,
-                                   probe);
+                                   probe,
+                                   sample_to_source_distance);
     }
     return beam;
   }
@@ -316,6 +370,7 @@ namespace dxtbx { namespace model { namespace boost_python {
     result["flux"] = obj.get_flux();
     result["transmission"] = obj.get_transmission();
     result["probe"] = obj.get_probe_name();
+    result["sample_to_source_distance"] = obj.get_sample_to_source_distance();
     return result;
   }
 
@@ -331,7 +386,8 @@ namespace dxtbx { namespace model { namespace boost_python {
       boost::python::extract<double>(obj.get("flux", 0)),
       boost::python::extract<double>(obj.get("transmission", 1)),
       Beam::get_probe_from_name(
-        boost::python::extract<std::string>(obj.get("probe", "x-ray"))));
+        boost::python::extract<std::string>(obj.get("probe", "x-ray"))),
+      boost::python::extract<double>(obj.get("sample_to_source_distance", 0.)));
     return b;
   }
 
@@ -380,6 +436,10 @@ namespace dxtbx { namespace model { namespace boost_python {
       .def("rotate_around_origin",
            &rotate_around_origin,
            (arg("axis"), arg("angle"), arg("deg") = true))
+      .def("get_sample_to_source_distance", &BeamBase::get_sample_to_source_distance)
+      .def("set_sample_to_source_distance",
+           &BeamBase::set_sample_to_source_distance,
+           (arg("sample_to_source_distance")))
       .def("__eq__", &BeamBase::operator==)
       .def("__ne__", &BeamBase::operator!=)
       .def("is_similar_to",
@@ -388,7 +448,12 @@ namespace dxtbx { namespace model { namespace boost_python {
             arg("wavelength_tolerance") = 1e-6,
             arg("direction_tolerance") = 1e-6,
             arg("polarization_normal_tolerance") = 1e-6,
-            arg("polarization_fraction_tolerance") = 1e-6));
+            arg("polarization_fraction_tolerance") = 1e-6,
+            arg("divergence") = 1e-6,
+            arg("sigma_divergence") = 1e-6,
+            arg("flux_tolerance") = 1e-6,
+            arg("transmission_tolerance") = 1e-6,
+            arg("sample_to_source_distance_tolerance") = 1e-6));
 
     // Export Beam : BeamBase
     class_<Beam, std::shared_ptr<Beam>, bases<BeamBase> >("Beam")
@@ -422,6 +487,20 @@ namespace dxtbx { namespace model { namespace boost_python {
                              arg("transmission"),
                              arg("probe") = Probe::xray,
                              arg("deg") = true)))
+      .def("__init__",
+           make_constructor(&make_beam_w_sample_to_source_distance,
+                            default_call_policies(),
+                            (arg("direction"),
+                             arg("wavelength"),
+                             arg("divergence"),
+                             arg("sigma_divergence"),
+                             arg("polarization_normal"),
+                             arg("polarization_fraction"),
+                             arg("flux"),
+                             arg("transmission"),
+                             arg("probe") = Probe::xray,
+                             arg("sample_to_source_distance") = 0,
+                             arg("deg") = true)))
       .def("__str__", &beam_to_string)
       .def("to_dict", &to_dict<Beam>)
       .def("from_dict", &from_dict<Beam>, return_value_policy<manage_new_object>())
@@ -433,6 +512,10 @@ namespace dxtbx { namespace model { namespace boost_python {
     class_<PolychromaticBeam, std::shared_ptr<PolychromaticBeam>, bases<Beam> >(
       "PolychromaticBeam")
       .def(init<const PolychromaticBeam &>())
+      .def("__init__",
+           make_constructor(&make_PolychromaticBeam_w_sample_to_source_distance,
+                            default_call_policies(),
+                            (arg("direction"), arg("sample_to_source_distance"))))
       .def("__init__",
            make_constructor(
              &make_PolychromaticBeam, default_call_policies(), (arg("direction"))))
@@ -454,6 +537,7 @@ namespace dxtbx { namespace model { namespace boost_python {
                              arg("flux"),
                              arg("transmission"),
                              arg("probe") = Probe::xray,
+                             arg("sample_to_source_distance") = 0,
                              arg("deg") = true)))
       .def("__str__", &PolychromaticBeam_to_string)
       .def("to_dict", &to_dict<PolychromaticBeam>)
diff --git a/tests/model/test_beam.py b/tests/model/test_beam.py
index 40659f4d4..5aea58b1b 100644
--- a/tests/model/test_beam.py
+++ b/tests/model/test_beam.py
@@ -107,12 +107,14 @@ def test_from_phil():
             """
     beam {
         probe = electron
+        sample_to_source_distance = 4000
     }
   """
         )
     ).extract()
     b4 = BeamFactory.from_phil(params3, reference)
     assert b4.get_probe() == Probe.electron
+    assert b4.get_sample_to_source_distance() == pytest.approx(4000)
 
 
 def test_scan_varying():
@@ -203,6 +205,7 @@ def test_polychromatic_beam_from_phil():
       polarization_fraction = .65
       transmission = .5
       flux = .75
+      sample_to_source_distance = 5000
     }
     """
         )
@@ -218,6 +221,7 @@ def test_polychromatic_beam_from_phil():
     assert beam.get_polarization_fraction() == pytest.approx(0.65)
     assert beam.get_transmission() == pytest.approx(0.5)
     assert beam.get_flux() == pytest.approx(0.75)
+    assert beam.get_sample_to_source_distance() == pytest.approx(5000)
 
 
 def test_polychromatic_beam_from_dict():
@@ -235,6 +239,7 @@ def test_make_polychromatic_beam():
     transmission = 0.5
     flux = 0.75
     probe = Probe.neutron
+    sample_to_source_distance = 8500
 
     beam = BeamFactory.make_polychromatic_beam(
         direction=direction,
@@ -245,6 +250,7 @@ def test_make_polychromatic_beam():
         transmission=transmission,
         flux=flux,
         probe=probe,
+        sample_to_source_distance=sample_to_source_distance,
     )
 
     assert beam.get_sample_to_source_direction() == pytest.approx((0.0, 0.0, 1.0))
@@ -255,6 +261,7 @@ def test_make_polychromatic_beam():
     assert beam.get_transmission() == pytest.approx(0.5)
     assert beam.get_flux() == pytest.approx(0.75)
     assert beam.get_probe() == Probe.neutron
+    assert beam.get_sample_to_source_distance() == pytest.approx(8500.0)
 
 
 def test_polychromatic_beam_wavelength_guards():
@@ -281,5 +288,5 @@ def test_polychromatic_beam_str():
     beam = PolychromaticBeam()
     assert (
         beam.__str__()
-        == "Beam:\n    probe: x-ray\n    sample to source direction : {0,0,1}\n    divergence: 0\n    sigma divergence: 0\n    polarization normal: {0,1,0}\n    polarization fraction: 0.5\n    flux: 0\n    transmission: 1\n"
+        == "Beam:\n    probe: x-ray\n    sample to source direction : {0,0,1}\n    divergence: 0\n    sigma divergence: 0\n    polarization normal: {0,1,0}\n    polarization fraction: 0.5\n    flux: 0\n    transmission: 1\n    sample to source distance : 0\n"
     )

From 6185cc389f693cf8776173ceff70be93477289b9 Mon Sep 17 00:00:00 2001
From: Graeme Winter <graeme.winter@gmail.com>
Date: Thu, 17 Aug 2023 11:50:04 +0100
Subject: [PATCH 44/45] Remove Datablock (#570)

Datablock has been deprecated for several years now. The dxtbx.datablock
module has been left in as an alias to the functionality now moved to
dxtbx.experiment_list, but will raise a deprecation warning pointing to
the new place.

Co-authored-by: Aaron Brewster <asbrewster@lbl.gov>
Co-authored-by: David Waterman <dgwaterman@gmail.com>
Co-authored-by: Nicholas Devenish <ndevenish@gmail.com>
---
 newsfragments/504.removal                     |   1 +
 pytest.ini                                    |   1 -
 .../display_parallax_correction.py            |  10 +-
 .../command_line/plot_detector_models.py      |  19 +-
 src/dxtbx/datablock.py                        | 900 +-----------------
 src/dxtbx/model/__init__.py                   |  44 +-
 src/dxtbx/model/experiment_list.py            | 194 ++--
 src/dxtbx/serialize/load.py                   |  16 -
 src/dxtbx/util/__init__.py                    |  11 +
 tests/model/test_experiment_list.py           |  59 --
 tests/test_datablock.py                       | 117 ---
 11 files changed, 177 insertions(+), 1195 deletions(-)
 create mode 100644 newsfragments/504.removal

diff --git a/newsfragments/504.removal b/newsfragments/504.removal
new file mode 100644
index 000000000..1500b053e
--- /dev/null
+++ b/newsfragments/504.removal
@@ -0,0 +1 @@
+dxtbx: remove legacy datablock object (obsolete for several years)
diff --git a/pytest.ini b/pytest.ini
index a89237729..5d6319d5e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -3,7 +3,6 @@ addopts = -rsxX
 filterwarnings =
     ignore:the matrix subclass is not the recommended way:PendingDeprecationWarning
     ignore:numpy.dtype size changed:RuntimeWarning
-    ignore:Datablocks are deprecated:UserWarning
     ignore:Deprecated call to `pkg_resources.declare_namespace:DeprecationWarning
     ignore:`product` is deprecated as of NumPy:DeprecationWarning:h5py|numpy
 junit_family = legacy
diff --git a/src/dxtbx/command_line/display_parallax_correction.py b/src/dxtbx/command_line/display_parallax_correction.py
index 5f85805b9..02e2b7abb 100644
--- a/src/dxtbx/command_line/display_parallax_correction.py
+++ b/src/dxtbx/command_line/display_parallax_correction.py
@@ -7,15 +7,17 @@
 from scitbx.array_family import flex
 
 import dxtbx.util
-from dxtbx.datablock import DataBlockFactory
 from dxtbx.model import ParallaxCorrectedPxMmStrategy
+from dxtbx.model.experiment_list import ExperimentListFactory
 
 
 def run(args=None):
     dxtbx.util.encode_output_as_utf8()
-    datablocks = DataBlockFactory.from_args(args or sys.argv[1:])
-    assert len(datablocks) == 1
-    detectors = datablocks[0].unique_detectors()
+    experiments = ExperimentListFactory.from_args(
+        args or sys.argv[1:], check_format=False
+    )
+    assert len(experiments) == 1
+    detectors = experiments.detectors()
     assert len(detectors) == 1
     detector = detectors[0]
     assert len(detector) == 1
diff --git a/src/dxtbx/command_line/plot_detector_models.py b/src/dxtbx/command_line/plot_detector_models.py
index f26ab6f4b..4034e7417 100644
--- a/src/dxtbx/command_line/plot_detector_models.py
+++ b/src/dxtbx/command_line/plot_detector_models.py
@@ -17,12 +17,11 @@
 from scitbx.matrix import col
 
 import dxtbx.util
-from dxtbx.datablock import DataBlockFactory
 from dxtbx.model.detector_helpers import get_detector_projection_2d_axes
 from dxtbx.model.experiment_list import ExperimentListFactory
 
 usage = """Plot dxtbx detector models. Provide multiple json files if desired
-Example: dxtbx.plot_detector_models datablock1.json datablock2.json
+Example: dxtbx.plot_detector_models experiments1.json experiments2.json
 """
 
 
@@ -181,16 +180,12 @@ def run(args=None):
 
         # read the data and get the detector models
         try:
-            datablocks = DataBlockFactory.from_json_file(file_name, check_format=False)
-            detectors = sum((db.unique_detectors() for db in datablocks), [])
-        except Exception:
-            try:
-                experiments = ExperimentListFactory.from_json_file(
-                    file_name, check_format=False
-                )
-            except ValueError:
-                experiments = ExperimentListFactory.from_filenames([file_name])
-            detectors = experiments.detectors()
+            experiments = ExperimentListFactory.from_json_file(
+                file_name, check_format=False
+            )
+        except ValueError:
+            experiments = ExperimentListFactory.from_filenames([file_name])
+        detectors = experiments.detectors()
         if not params.plot_all_detectors:
             detectors = detectors[0:1]
         for detector in detectors:
diff --git a/src/dxtbx/datablock.py b/src/dxtbx/datablock.py
index ab37c389f..436f52ea3 100644
--- a/src/dxtbx/datablock.py
+++ b/src/dxtbx/datablock.py
@@ -1,888 +1,24 @@
 from __future__ import annotations
 
-import collections
-import json
-import logging
-import os.path
-import pickle
 import warnings
 
-import libtbx
-
-import dxtbx.imageset
-import dxtbx.model
-from dxtbx.format.FormatMultiImage import FormatMultiImage
-from dxtbx.format.image import ImageBool, ImageDouble
-from dxtbx.format.Registry import get_format_class_for_file
-from dxtbx.sequence_filenames import (
-    locate_files_matching_template_string,
-    template_string_number_index,
+from dxtbx.experiment_list import (
+    BeamComparison,
+    DetectorComparison,
+    FormatChecker,
+    GoniometerComparison,
 )
-from dxtbx.serialize import load
-from dxtbx.serialize.filename import resolve_path
-
-logger = logging.getLogger(__name__)
-
-
-class DataBlock:
-    """High level container for blocks of sequences and imagesets."""
-
-    def __init__(self, imagesets=None):
-        """Instantiate from a list of imagesets."""
-
-        warnings.warn(
-            "Datablocks are deprecated; please use ExperimentLists instead",
-            UserWarning,
-            stacklevel=2,
-        )
-
-        # Try to get a format class
-        self._format_class = None
-        self._imagesets = []
-
-        # Load imagesets
-        if imagesets is not None:
-            for iset in imagesets:
-                self.append(iset)
-
-    def append(self, imageset):
-        """Add an imageset to the block."""
-        if self._format_class is None:
-            self._format_class = imageset.get_format_class()
-        elif not self._format_class == imageset.get_format_class():
-            raise TypeError("Can not mix image format classes in one datablock")
-        self._imagesets.append(imageset)
-
-    def extend(self, datablock):
-        """Add two datablocks."""
-        for iset in datablock:
-            self.append(iset)
-
-    def format_class(self):
-        """Return the format class."""
-        return self._format_class
-
-    def extract_stills(self):
-        """Extract all the still imagesets"""
-        return list(self.iter_stills())
-
-    def extract_sequences(self):
-        """Extract all the sequences from the block."""
-        return list(self.iter_sequences())
-
-    def extract_imagesets(self):
-        """Extract all imagesets."""
-        return list(self._imagesets)
-
-    def num_images(self):
-        """Get the number of images."""
-        return sum(len(iset) for iset in self._imagesets)
-
-    def __len__(self):
-        """The number of image sets."""
-        return len(self._imagesets)
-
-    def __eq__(self, rhs):
-        """Check if two blocks are the same."""
-        return (
-            self._format_class == rhs._format_class
-            and self._imagesets == rhs._imagesets
-        )
-
-    def __ne__(self, rhs):
-        """Check if two blocks are not equal."""
-        return not self.__eq__(rhs)
-
-    def __iter__(self):
-        """Iterate through the imagesets."""
-        yield from self._imagesets
-
-    def iter_sequences(self):
-        """Iterate over sequence groups."""
-        for iset in self._imagesets:
-            if isinstance(iset, dxtbx.imageset.ImageSequence):
-                yield iset
-
-    def iter_stills(self):
-        """Iterate over still groups."""
-        for iset in self._imagesets:
-            if not isinstance(iset, dxtbx.imageset.ImageSequence):
-                yield iset
-
-    def _find_unique_items(self, item_name, filter_none=False):
-        """Return a list of unique beams, detectors, ... in order.
-        Optionally filter out None values (unless they came in via
-        an ImageSequence)."""
-        items = {}
-        for imageset in self._imagesets:
-            getter_function = getattr(imageset, "get_" + item_name)
-            if isinstance(imageset, dxtbx.imageset.ImageSequence):
-                items[getter_function()] = None
-            else:
-                for i in range(len(imageset)):
-                    item = getter_function(i)
-                    if not filter_none or item is not None:
-                        items[item] = None
-        return list(items)
-
-    def unique_beams(self):
-        return self._find_unique_items("beam")
-
-    def unique_detectors(self):
-        return self._find_unique_items("detector")
-
-    def unique_goniometers(self):
-        return self._find_unique_items("goniometer", filter_none=True)
-
-    def unique_scans(self):
-        return self._find_unique_items("scan", filter_none=True)
-
-    def to_dict(self):
-        """Convert the datablock to a dictionary"""
-
-        def abspath_or_none(filename):
-            if filename is None or filename == "":
-                return None
-            return os.path.abspath(filename)
-
-        # Get a list of all the unique models
-        b = list(self.unique_beams())
-        d = list(self.unique_detectors())
-        g = list(self.unique_goniometers())
-        s = list(self.unique_scans())
-
-        # Create the data block dictionary
-        result = {
-            "__id__": "DataBlock",
-            "imageset": [],
-        }
-
-        # Loop through all the imagesets
-        for iset in self._imagesets:
-            if isinstance(iset, dxtbx.imageset.ImageSequence):
-                if iset.reader().is_single_file_reader():
-                    result["imageset"].append(
-                        {
-                            "__id__": "ImageSequence",
-                            "master": os.path.abspath(iset.reader().master_path()),
-                            "mask": abspath_or_none(iset.external_lookup.mask.filename),
-                            "gain": abspath_or_none(iset.external_lookup.gain.filename),
-                            "pedestal": abspath_or_none(
-                                iset.external_lookup.pedestal.filename
-                            ),
-                            "dx": abspath_or_none(iset.external_lookup.dx.filename),
-                            "dy": abspath_or_none(iset.external_lookup.dy.filename),
-                            "beam": b.index(iset.get_beam()),
-                            "detector": d.index(iset.get_detector()),
-                            "goniometer": g.index(iset.get_goniometer()),
-                            "scan": s.index(iset.get_scan()),
-                            "images": list(iset.indices()),
-                            "params": iset.params(),
-                        }
-                    )
-                else:
-                    result["imageset"].append(
-                        {
-                            "__id__": "ImageSequence",
-                            "template": os.path.abspath(iset.get_template()),
-                            "mask": abspath_or_none(iset.external_lookup.mask.filename),
-                            "gain": abspath_or_none(iset.external_lookup.gain.filename),
-                            "pedestal": abspath_or_none(
-                                iset.external_lookup.pedestal.filename
-                            ),
-                            "dx": abspath_or_none(iset.external_lookup.dx.filename),
-                            "dy": abspath_or_none(iset.external_lookup.dy.filename),
-                            "beam": b.index(iset.get_beam()),
-                            "detector": d.index(iset.get_detector()),
-                            "goniometer": g.index(iset.get_goniometer()),
-                            "scan": s.index(iset.get_scan()),
-                            "params": iset.params(),
-                        }
-                    )
-            else:
-                imageset = {}
-                if isinstance(iset, dxtbx.imageset.ImageGrid):
-                    imageset["__id__"] = "ImageGrid"
-                    imageset["grid_size"] = iset.get_grid_size()
-                else:
-                    imageset["__id__"] = "ImageSet"
-                image_list = []
-                for i in range(len(iset)):
-                    image_dict = {
-                        "filename": os.path.abspath(iset.get_path(i)),
-                        "gain": abspath_or_none(iset.external_lookup.gain.filename),
-                        "pedestal": abspath_or_none(
-                            iset.external_lookup.pedestal.filename
-                        ),
-                        "dx": abspath_or_none(iset.external_lookup.dx.filename),
-                        "dy": abspath_or_none(iset.external_lookup.dy.filename),
-                    }
-                    if iset.reader().is_single_file_reader():
-                        image_dict["image"] = iset.indices()[i]
-                    try:
-                        image_dict["beam"] = b.index(iset.get_beam(i))
-                    except Exception:
-                        pass
-                    try:
-                        image_dict["detector"] = d.index(iset.get_detector())
-                    except Exception:
-                        pass
-                    try:
-                        image_dict["goniometer"] = g.index(iset.get_goniometer())
-                    except Exception:
-                        pass
-                    try:
-                        image_dict["scan"] = s.index(iset.get_scan(i))
-                    except Exception:
-                        pass
-                    image_list.append(image_dict)
-                imageset["mask"] = abspath_or_none(iset.external_lookup.mask.filename)
-                imageset["images"] = image_list
-                imageset["params"] = iset.params()
-                result["imageset"].append(imageset)
-
-        # Add the models to the dictionary
-        result["beam"] = [bb.to_dict() for bb in b]
-        result["detector"] = [dd.to_dict() for dd in d]
-        result["goniometer"] = [gg.to_dict() for gg in g]
-        result["scan"] = [ss.to_dict() for ss in s]
-
-        return result
-
-
-class FormatChecker:
-    """A helper class to speed up identifying the correct image format by first
-    trying the last format that was used."""
-
-    def __init__(self):
-        """Set the format class to none."""
-        self._format_class = None
-
-    def find_format(self, filename):
-        """Search the registry for the image format class.
-        Where possible use the last seen format class as a prioritisation hint.
-        """
-        if self._format_class:
-            self._format_class = get_format_class_for_file(
-                filename, format_hint=self._format_class.__name__
-            )
-        else:
-            self._format_class = get_format_class_for_file(filename)
-        if self._format_class:
-            logger.debug("Using %s for %s", self._format_class.__name__, filename)
-        else:
-            logger.debug("No format class found for %s", filename)
-        return self._format_class
-
-    def iter_groups(self, filenames):
-        group_format = None
-        group_fnames = []
-        for filename in filenames:
-            fmt = self.find_format(filename)
-            if fmt == group_format:
-                group_fnames.append(filename)
-            else:
-                if group_fnames:
-                    yield group_format, group_fnames
-                group_fnames = [filename]
-                group_format = fmt
-            if fmt is not None:
-                logger.debug("Using %s for %s", fmt.__name__, filename)
-        if group_fnames:
-            yield group_format, group_fnames
-
-
-class DataBlockTemplateImporter:
-    """A class to import a datablock from a template."""
-
-    def __init__(self, templates, **kwargs):
-        """Import the datablocks from the given templates."""
-        assert "verbose" not in kwargs, "The verbose parameter has been removed"
-        assert len(templates) > 0
-
-        self.datablocks = []
-
-        # A function to append or create a new datablock
-        def append_to_datablocks(iset):
-            try:
-                self.datablocks[-1].append(iset)
-            except (IndexError, TypeError):
-                # This happens when we already have a datablock with a different format
-                self.datablocks.append(DataBlock([iset]))
-            logger.debug("Added imageset to datablock %d", len(self.datablocks) - 1)
-
-        # For each template do an import
-        for template in templates:
-            template = os.path.normpath(template)
-            paths = sorted(locate_files_matching_template_string(template))
-            logger.debug("The following files matched the template string:")
-            if len(paths) > 0:
-                for p in paths:
-                    logger.debug(" %s", p)
-            else:
-                logger.debug(" No files found")
-
-            # Check if we've matched any filenames
-            if len(paths) == 0:
-                raise ValueError('Template "%s" does not match any files' % template)
-
-            # Get the format from the first image
-            fmt = FormatChecker().find_format(paths[0])
-            if fmt is None:
-                raise ValueError("Image file %s format is unknown" % paths[0])
-            elif fmt.is_abstract():
-                raise ValueError(
-                    f"Image file {paths[0]} appears to be a '{type(fmt).__name__}', but this is an abstract Format"
-                )
-            else:
-                imageset = self._create_imageset(fmt, template, paths, **kwargs)
-                append_to_datablocks(imageset)
-
-    def _create_imageset(self, format_class, template, filenames, **kwargs):
-        """Create a multi file sequence or imageset."""
-        # Get the image range
-        index = slice(*template_string_number_index(template))
-        image_range = kwargs.get("image_range")
-        if image_range:
-            first, last = image_range
-        else:
-            first = int(filenames[0][index])
-            last = int(filenames[-1][index])
-
-        # Check all images in range are present - if allowed
-        if not kwargs.get("allow_incomplete_sequences", False):
-            all_numbers = {int(f[index]) for f in filenames}
-            missing = set(range(first, last + 1)) - all_numbers
-            if missing:
-                raise ValueError(
-                    "Missing image{} {} from imageset ({}-{})".format(
-                        "s" if len(missing) > 1 else "",
-                        ", ".join(str(x) for x in sorted(missing)),
-                        first,
-                        last,
-                    )
-                )
-
-        # Read the image
-        fmt = format_class(filenames[0], **(kwargs.get("format_kwargs", {})))
-
-        # Get the meta data from the format
-        b = fmt.get_beam()
-        d = fmt.get_detector()
-        g = fmt.get_goniometer()
-        s = fmt.get_scan()
-
-        # Update the image range
-        s.set_image_range((first, last))
-
-        # Get the image range
-        image_range = s.get_image_range()
-        image_range = (image_range[0], image_range[1] + 1)
 
-        # Create the sequence
-        imageset = dxtbx.imageset.ImageSetFactory.make_sequence(
-            template,
-            list(range(*image_range)),
-            format_class,
-            b,
-            d,
-            g,
-            s,
-            format_kwargs=kwargs.get("format_kwargs"),
-        )
-
-        return imageset
-
-
-class InvalidDataBlockError(RuntimeError):
-    """
-    Indicates an error whilst validating the experiment list.
-
-    This means that there is some structural problem that prevents the given data
-    from representing a well-formed experiment list. This doesn't indicate e.g.
-    some problem with the data or model consistency.
-    """
-
-
-def datablocks_from_dict(obj, check_format=True, directory=None):
-    """Get the datablocks from the dictionary."""
-
-    # If we have a list, extract for each dictionary in the list
-    if isinstance(obj, list):
-        return [datablocks_from_dict(dd, check_format, directory) for dd in obj]
-    elif not isinstance(obj, dict):
-        raise InvalidDataBlockError(
-            "Unexpected datablock type {} instead of dict".format(type(obj))
-        )
-    # Make sure the id signature is correct
-    if not obj.get("__id__") == "DataBlock":
-        raise InvalidDataBlockError(
-            "Expected __id__ 'DataBlock', but found {}".format(repr(obj.get("__id__")))
-        )
-
-    # Get the list of models
-    blist = obj.get("beam", [])
-    dlist = obj.get("detector", [])
-    glist = obj.get("goniometer", [])
-    slist = obj.get("scan", [])
-
-    def load_models(obj):
-        try:
-            beam = dxtbx.model.BeamFactory.from_dict(blist[obj["beam"]])
-        except Exception:
-            beam = None
-        try:
-            dobj = dlist[obj["detector"]]
-            detector = dxtbx.model.DetectorFactory.from_dict(dobj)
-        except Exception:
-            detector = None
-        try:
-            gonio = dxtbx.model.GoniometerFactory.from_dict(glist[obj["goniometer"]])
-        except Exception:
-            gonio = None
-        try:
-            scan = dxtbx.model.ScanFactory.from_dict(slist[obj["scan"]])
-        except Exception:
-            scan = None
-        return beam, detector, gonio, scan
-
-    # Loop through all the imagesets
-    imagesets = []
-    for imageset in obj["imageset"]:
-        ident = imageset["__id__"]
-        if "params" in imageset:
-            format_kwargs = imageset["params"]
-        else:
-            format_kwargs = {}
-        if ident == "ImageSequence" or ident == "ImageSweep":
-            beam, detector, gonio, scan = load_models(imageset)
-            if "template" in imageset:
-                template = resolve_path(imageset["template"], directory=directory)
-                i0, i1 = scan.get_image_range()
-                iset = dxtbx.imageset.ImageSetFactory.make_sequence(
-                    template,
-                    list(range(i0, i1 + 1)),
-                    None,
-                    beam,
-                    detector,
-                    gonio,
-                    scan,
-                    check_format,
-                    format_kwargs=format_kwargs,
-                )
-                if "mask" in imageset and imageset["mask"] is not None:
-                    imageset["mask"] = resolve_path(
-                        imageset["mask"], directory=directory
-                    )
-                    iset.external_lookup.mask.filename = imageset["mask"]
-                    if check_format:
-                        with open(imageset["mask"], "rb") as infile:
-                            iset.external_lookup.mask.data = ImageBool(
-                                pickle.load(infile, encoding="bytes")
-                            )
-                if "gain" in imageset and imageset["gain"] is not None:
-                    imageset["gain"] = resolve_path(
-                        imageset["gain"], directory=directory
-                    )
-                    iset.external_lookup.gain.filename = imageset["gain"]
-                    if check_format:
-                        with open(imageset["gain"], "rb") as infile:
-                            iset.external_lookup.gain.data = ImageDouble(
-                                pickle.load(infile, encoding="bytes")
-                            )
-                if "pedestal" in imageset and imageset["pedestal"] is not None:
-                    imageset["pedestal"] = resolve_path(
-                        imageset["pedestal"], directory=directory
-                    )
-                    iset.external_lookup.pedestal.filename = imageset["pedestal"]
-                    if check_format:
-                        with open(imageset["pedestal"], "rb") as infile:
-                            iset.external_lookup.pedestal.data = ImageDouble(
-                                pickle.load(infile, encoding="bytes")
-                            )
-                if "dx" in imageset and imageset["dx"] is not None:
-                    imageset["dx"] = resolve_path(imageset["dx"], directory=directory)
-                    iset.external_lookup.dx.filename = imageset["dx"]
-                    with open(imageset["dx"], "rb") as infile:
-                        iset.external_lookup.dx.data = ImageDouble(
-                            pickle.load(infile, encoding="bytes")
-                        )
-                if "dy" in imageset and imageset["dy"] is not None:
-                    imageset["dy"] = resolve_path(imageset["dy"], directory=directory)
-                    iset.external_lookup.dy.filename = imageset["dy"]
-                    with open(imageset["dy"], "rb") as infile:
-                        iset.external_lookup.dy.data = ImageDouble(
-                            pickle.load(infile, encoding="bytes")
-                        )
-                iset.update_detector_px_mm_data()
-            elif "master" in imageset:
-                template = resolve_path(imageset["master"], directory=directory)
-                i0, i1 = scan.get_image_range()
-                if not check_format:
-                    format_class = FormatMultiImage
-                else:
-                    format_class = None
-                iset = dxtbx.imageset.ImageSetFactory.make_sequence(
-                    template,
-                    list(range(i0, i1 + 1)),
-                    format_class=format_class,
-                    beam=beam,
-                    detector=detector,
-                    goniometer=gonio,
-                    scan=scan,
-                    check_format=check_format,
-                    format_kwargs=format_kwargs,
-                )
-                if "mask" in imageset and imageset["mask"] is not None:
-                    imageset["mask"] = resolve_path(imageset["mask"], directory)
-                    iset.external_lookup.mask.filename = imageset["mask"]
-                    if check_format:
-                        with open(imageset["mask"], "rb") as infile:
-                            iset.external_lookup.mask.data = ImageBool(
-                                pickle.load(infile, encoding="bytes")
-                            )
-                if "gain" in imageset and imageset["gain"] is not None:
-                    imageset["gain"] = resolve_path(imageset["gain"], directory)
-                    iset.external_lookup.gain.filename = imageset["gain"]
-                    if check_format:
-                        with open(imageset["gain"], "rb") as infile:
-                            iset.external_lookup.gain.data = ImageDouble(
-                                pickle.load(infile, encoding="bytes")
-                            )
-                if "pedestal" in imageset and imageset["pedestal"] is not None:
-                    imageset["pedestal"] = resolve_path(imageset["pedestal"], directory)
-                    iset.external_lookup.pedestal.filename = imageset["pedestal"]
-                    if check_format:
-                        with open(imageset["pedestal"], "rb") as infile:
-                            iset.external_lookup.pedestal.data = ImageDouble(
-                                pickle.load(infile, encoding="bytes")
-                            )
-                if "dx" in imageset and imageset["dx"] is not None:
-                    imageset["dx"] = resolve_path(imageset["dx"], directory)
-                    iset.external_lookup.dx.filename = imageset["dx"]
-                    with open(imageset["dx"], "rb") as infile:
-                        iset.external_lookup.dx.data = ImageDouble(
-                            pickle.load(infile, encoding="bytes")
-                        )
-                if "dy" in imageset and imageset["dy"] is not None:
-                    imageset["dy"] = resolve_path(imageset["dy"], directory)
-                    iset.external_lookup.dy.filename = imageset["dy"]
-                    with open(imageset["dy"], "rb") as infile:
-                        iset.external_lookup.dy.data = ImageDouble(
-                            pickle.load(infile, encoding="bytes")
-                        )
-                iset.update_detector_px_mm_data()
-            imagesets.append(iset)
-        elif ident == "ImageSet" or ident == "ImageGrid":
-            filenames = [image["filename"] for image in imageset["images"]]
-            indices = [
-                image["image"] for image in imageset["images"] if "image" in image
-            ]
-            assert len(indices) == 0 or len(indices) == len(filenames)
-            iset = dxtbx.imageset.ImageSetFactory.make_imageset(
-                filenames, None, check_format, indices, format_kwargs=format_kwargs
-            )
-            if ident == "ImageGrid":
-                grid_size = imageset["grid_size"]
-                iset = dxtbx.imageset.ImageGrid.from_imageset(iset, grid_size)
-            for i, image in enumerate(imageset["images"]):
-                beam, detector, gonio, scan = load_models(image)
-                iset.set_beam(beam, i)
-                iset.set_detector(detector, i)
-                iset.set_goniometer(gonio, i)
-                iset.set_scan(scan, i)
-            if "mask" in imageset and imageset["mask"] is not None:
-                imageset["mask"] = resolve_path(imageset["mask"], directory)
-                iset.external_lookup.mask.filename = imageset["mask"]
-                if check_format:
-                    with open(imageset["mask"], "rb") as infile:
-                        iset.external_lookup.mask.data = ImageBool(
-                            pickle.load(infile, encoding="bytes")
-                        )
-            if "gain" in imageset and imageset["gain"] is not None:
-                imageset["gain"] = resolve_path(imageset["gain"], directory)
-                iset.external_lookup.gain.filename = imageset["gain"]
-                if check_format:
-                    with open(imageset["gain"], "rb") as infile:
-                        iset.external_lookup.gain.data = ImageDouble(
-                            pickle.load(infile, encoding="bytes")
-                        )
-            if "pedestal" in imageset and imageset["pedestal"] is not None:
-                imageset["pedestal"] = resolve_path(imageset["pedestal"], directory)
-                iset.external_lookup.pedestal.filename = imageset["pedestal"]
-                if check_format:
-                    with open(imageset["pedestal"], "rb") as infile:
-                        iset.external_lookup.pedestal.data = ImageDouble(
-                            pickle.load(infile, encoding="bytes")
-                        )
-            if "dx" in imageset and imageset["dx"] is not None:
-                imageset["dx"] = resolve_path(imageset["dx"], directory)
-                iset.external_lookup.dx.filename = imageset["dx"]
-                with open(imageset["dx"], "rb") as infile:
-                    iset.external_lookup.dx.data = ImageDouble(
-                        pickle.load(infile, encoding="bytes")
-                    )
-            if "dy" in imageset and imageset["dy"] is not None:
-                imageset["dy"] = resolve_path(imageset["dy"], directory)
-                iset.external_lookup.dy.filename = imageset["dy"]
-                with open(imageset["dy"], "rb") as infile:
-                    iset.external_lookup.dy.data = ImageDouble(
-                        pickle.load(infile, encoding="bytes")
-                    )
-                iset.update_detector_px_mm_data()
-            imagesets.append(iset)
-        else:
-            raise RuntimeError("expected ImageSet/ImageSequence, got %s" % ident)
-
-    return DataBlock(imagesets)
-
-
-def datablocks_from_imagesets(imagesets):
-    """Load a list of datablocks from imagesets."""
-
-    datablocks = []
-    if not isinstance(imagesets, list):
-        imagesets = [imagesets]
-    for imageset in imagesets:
-        try:
-            datablocks[-1].append(imageset)
-        except (IndexError, AssertionError):
-            datablocks.append(DataBlock([imageset]))
-    return datablocks
-
-
-class DataBlockFactory:
-    """Class for creating DataBlock instances"""
-
-    @staticmethod
-    def from_args(
-        args,
-        unhandled=None,
-        compare_beam=None,
-        compare_detector=None,
-        compare_goniometer=None,
-        scan_tolerance=None,
-        format_kwargs=None,
-    ):
-        """Try to load datablocks from any recognized format."""
-
-        if unhandled is None:
-            unhandled = []
-        unhandled1 = []
-
-        # Try as image files
-        datablocks = DataBlockFactory.from_filenames(
-            args,
-            unhandled=unhandled1,
-            compare_beam=compare_beam,
-            compare_detector=compare_detector,
-            compare_goniometer=compare_goniometer,
-            scan_tolerance=scan_tolerance,
-            format_kwargs=format_kwargs,
-        )
-
-        # Try as serialized formats
-        for filename in unhandled1:
-            try:
-                datablocks.extend(DataBlockFactory.from_serialized_format(filename))
-                logger.debug("Loaded datablocks(s) from %s", filename)
-            except Exception:
-                unhandled.append(filename)
-
-        return datablocks
-
-    @staticmethod
-    def from_filenames(
-        filenames,
-        unhandled=None,
-        compare_beam=None,
-        compare_detector=None,
-        compare_goniometer=None,
-        scan_tolerance=None,
-        format_kwargs=None,
-    ):
-        """Create a list of data blocks from a list of directory or file names."""
-        from dxtbx.model.experiment_list import ExperimentList, ExperimentListFactory
-
-        expts = ExperimentListFactory.from_filenames(
-            filenames,
-            unhandled=unhandled,
-            compare_beam=compare_beam,
-            compare_detector=compare_detector,
-            compare_goniometer=compare_goniometer,
-            scan_tolerance=scan_tolerance,
-            format_kwargs=format_kwargs,
-        )
-        datablocks = []
-
-        # Datablocks can only contain imageset with a shared format class, therefore
-        # group the experiments by format class
-        format_groups = collections.defaultdict(ExperimentList)
-        for expt in expts:
-            format_groups[expt.imageset.get_format_class()].append(expt)
-
-        # Generate a datablock for each format group
-        for format_group, format_expts in format_groups.items():
-            datablocks.extend(format_expts.to_datablocks())
-
-        return datablocks
-
-    @staticmethod
-    def from_dict(obj, check_format=True, directory=None):
-        """Create a datablock from a dictionary."""
-        return datablocks_from_dict(obj, check_format, directory)
-
-    @staticmethod
-    def from_json(string, check_format=True, directory=None):
-        """Decode a datablock from JSON string."""
-        return DataBlockFactory.from_dict(
-            json.loads(string),
-            check_format=check_format,
-            directory=directory,
-        )
-
-    @staticmethod
-    def from_json_file(filename, check_format=True):
-        """Decode a datablock from a JSON file."""
-        filename = os.path.abspath(filename)
-        directory = os.path.dirname(filename)
-        with open(filename) as infile:
-            return DataBlockFactory.from_json(
-                infile.read(), check_format=check_format, directory=directory
-            )
-
-    @staticmethod
-    def from_pickle_file(filename):
-        """Decode a datablock from a pickle file."""
-        with open(filename, "rb") as infile:
-            obj = pickle.load(infile)
-            if isinstance(obj, list):
-                assert all(isinstance(db, DataBlock) for db in obj)
-            else:
-                assert isinstance(obj, DataBlock)
-            return obj
-
-    @staticmethod
-    def from_imageset(imagesets):
-        """Load a datablock from a list of imagesets."""
-        return datablocks_from_imagesets(imagesets)
-
-    @staticmethod
-    def from_imageset_json_file(filename):
-        """Load a datablock from a sequence file."""
-        # Load the imageset and create a datablock from the filenames
-        imageset = load.imageset(filename)
-        return DataBlockFactory.from_imageset(imageset)
-
-    @staticmethod
-    def from_serialized_format(filename, check_format=True):
-        """Load a datablock from serialized formats."""
-
-        # First try as JSON format
-        try:
-            return DataBlockFactory.from_json_file(filename, check_format)
-        except Exception:
-            pass
-
-        # Now try as pickle format
-        try:
-            return DataBlockFactory.from_pickle_file(filename)
-        except Exception:
-            pass
-
-        # Now try as imageset json files
-        return DataBlockFactory.from_imageset_json_file(filename)
-
-    @staticmethod
-    def from_in_memory(images, indices=None):
-        """Function to instantiate data block from in memory imageset."""
-        return DataBlock(
-            [
-                dxtbx.imageset.ImageSet(
-                    dxtbx.imageset.ImageSetData(dxtbx.imageset.MemReader(images)),
-                    indices,
-                )
-            ]
-        )
-
-
-class AutoEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, libtbx.AutoType):
-            return "Auto"
-        # Let the base class default method raise the TypeError
-        return json.JSONEncoder.default(self, obj)
-
-
-class BeamComparison:
-    """A class to provide simple beam comparison"""
-
-    def __init__(
-        self,
-        wavelength_tolerance=1e-6,
-        direction_tolerance=1e-6,
-        polarization_normal_tolerance=1e-6,
-        polarization_fraction_tolerance=1e-6,
-    ):
-        self.wavelength_tolerance = wavelength_tolerance
-        self.direction_tolerance = direction_tolerance
-        self.polarization_normal_tolerance = polarization_normal_tolerance
-        self.polarization_fraction_tolerance = polarization_fraction_tolerance
-
-    def __call__(self, a, b):
-        if a is None and b is None:
-            return True
-        return a.is_similar_to(
-            b,
-            wavelength_tolerance=self.wavelength_tolerance,
-            direction_tolerance=self.direction_tolerance,
-            polarization_normal_tolerance=self.polarization_normal_tolerance,
-            polarization_fraction_tolerance=self.polarization_fraction_tolerance,
-        )
-
-
-class DetectorComparison:
-    """A class to provide simple detector comparison"""
-
-    def __init__(
-        self, fast_axis_tolerance=1e-6, slow_axis_tolerance=1e-6, origin_tolerance=1e-6
-    ):
-        self.fast_axis_tolerance = fast_axis_tolerance
-        self.slow_axis_tolerance = slow_axis_tolerance
-        self.origin_tolerance = origin_tolerance
-
-    def __call__(self, a, b):
-        if a is None and b is None:
-            return True
-        return a.is_similar_to(
-            b,
-            fast_axis_tolerance=self.fast_axis_tolerance,
-            slow_axis_tolerance=self.slow_axis_tolerance,
-            origin_tolerance=self.origin_tolerance,
-        )
-
-
-class GoniometerComparison:
-    """A class to provide simple goniometer comparison"""
-
-    def __init__(
-        self,
-        rotation_axis_tolerance=1e-6,
-        fixed_rotation_tolerance=1e-6,
-        setting_rotation_tolerance=1e-6,
-    ):
-        self.rotation_axis_tolerance = rotation_axis_tolerance
-        self.fixed_rotation_tolerance = fixed_rotation_tolerance
-        self.setting_rotation_tolerance = setting_rotation_tolerance
-
-    def __call__(self, a, b):
-        if a is None and b is None:
-            return True
-        elif a is None or b is None:
-            return False
-        return a.is_similar_to(
-            b,
-            rotation_axis_tolerance=self.rotation_axis_tolerance,
-            fixed_rotation_tolerance=self.fixed_rotation_tolerance,
-            setting_rotation_tolerance=self.setting_rotation_tolerance,
-        )
+__all__ = [
+    "BeamComparison",
+    "DetectorComparison",
+    "FormatChecker",
+    "GoniometerComparison",
+]
+
+# Remove this module after DIALS 3.17
+warnings.warn(
+    "dxtbx.datablock is deprecated; please import from dxtbx.experiment_list instead",
+    UserWarning,
+    stacklevel=2,
+)
diff --git a/src/dxtbx/model/__init__.py b/src/dxtbx/model/__init__.py
index a7e3ada72..38f9de359 100644
--- a/src/dxtbx/model/__init__.py
+++ b/src/dxtbx/model/__init__.py
@@ -21,7 +21,7 @@
 from dxtbx.model.goniometer import GoniometerFactory
 from dxtbx.model.profile import ProfileModelFactory
 from dxtbx.model.scan import ScanFactory
-from dxtbx.util import format_float_with_standard_uncertainty
+from dxtbx.util import AutoEncoder, format_float_with_standard_uncertainty
 
 try:
     from ..dxtbx_model_ext import (
@@ -711,44 +711,6 @@ def get_template(imset):
 
         return result
 
-    def to_datablocks(self):
-        """Return the experiment list as a datablock list.
-        This assumes that the experiment contains 1 datablock."""
-        # Datablock depends on model/__init__
-        from dxtbx.datablock import DataBlockFactory
-
-        # Convert the experiment list to dict
-        obj = self.to_dict()
-        # Convert the dictionary to a datablock dictionary
-        obj["__id__"] = "DataBlock"
-        for e in obj["experiment"]:
-            iid = e["imageset"]
-            imageset = obj["imageset"][iid]
-            if "beam" in e:
-                imageset["beam"] = e["beam"]
-            if "detector" in e:
-                imageset["detector"] = e["detector"]
-            if "goniometer" in e:
-                imageset["goniometer"] = e["goniometer"]
-            if "scan" in e:
-                imageset["scan"] = e["scan"]
-
-            if imageset["__id__"] in ("ImageSet", "ImageGrid"):
-                image_list = []
-                for file_index, filename in enumerate(imageset["images"]):
-                    image_dict = {
-                        "filename": filename,
-                        "image": file_index,
-                    }
-                    image_list.append(image_dict)
-                imageset["images"] = image_list
-
-        # Remove the experiments
-        del obj["experiment"]
-
-        # Create the datablock
-        return DataBlockFactory.from_dict([obj])
-
     def nullify_all_single_file_reader_format_instances(self):
         """
         Parallel reading of HDF5 from the same handle is not allowed. Python
@@ -767,7 +729,6 @@ def as_json(self, filename=None, compact=False, split=False):
 
         # Split into separate files
         if filename is not None and split:
-
             # Get lists of models by filename
             basepath = os.path.splitext(filename)[0]
             ilist = [
@@ -842,9 +803,6 @@ def as_json(self, filename=None, compact=False, split=False):
         else:
             to_write = [(filename, dictionary)]
 
-        # Datablock depends on model/__init__
-        from dxtbx.datablock import AutoEncoder
-
         for fname, obj in to_write:
             if compact:
                 separators = (",", ":")
diff --git a/src/dxtbx/model/experiment_list.py b/src/dxtbx/model/experiment_list.py
index ec836614f..b357d99da 100644
--- a/src/dxtbx/model/experiment_list.py
+++ b/src/dxtbx/model/experiment_list.py
@@ -12,16 +12,11 @@
 import pickle
 from typing import Any, Callable, Generator, Iterable
 
-import dxtbx.datablock
-from dxtbx.datablock import (
-    BeamComparison,
-    DataBlockFactory,
-    DetectorComparison,
-    GoniometerComparison,
-)
+import dxtbx
 from dxtbx.format.Format import Format
 from dxtbx.format.FormatMultiImage import FormatMultiImage
 from dxtbx.format.image import ImageBool, ImageDouble
+from dxtbx.format.Registry import get_format_class_for_file
 from dxtbx.imageset import ImageGrid, ImageSequence, ImageSet, ImageSetFactory
 from dxtbx.model import (
     BeamFactory,
@@ -64,6 +59,122 @@ class InvalidExperimentListError(RuntimeError):
     """
 
 
+class FormatChecker:
+    """A helper class to speed up identifying the correct image format by first
+    trying the last format that was used."""
+
+    def __init__(self):
+        """Set the format class to none."""
+        self._format_class = None
+
+    def find_format(self, filename):
+        """Search the registry for the image format class.
+        Where possible use the last seen format class as a prioritisation hint.
+        """
+        if self._format_class:
+            self._format_class = get_format_class_for_file(
+                filename, format_hint=self._format_class.__name__
+            )
+        else:
+            self._format_class = get_format_class_for_file(filename)
+        if self._format_class:
+            logger.debug("Using %s for %s", self._format_class.__name__, filename)
+        else:
+            logger.debug("No format class found for %s", filename)
+        return self._format_class
+
+    def iter_groups(self, filenames):
+        group_format = None
+        group_fnames = []
+        for filename in filenames:
+            fmt = self.find_format(filename)
+            if fmt == group_format:
+                group_fnames.append(filename)
+            else:
+                if group_fnames:
+                    yield group_format, group_fnames
+                group_fnames = [filename]
+                group_format = fmt
+            if fmt is not None:
+                logger.debug("Using %s for %s", fmt.__name__, filename)
+        if group_fnames:
+            yield group_format, group_fnames
+
+
+class BeamComparison:
+    """A class to provide simple beam comparison"""
+
+    def __init__(
+        self,
+        wavelength_tolerance=1e-6,
+        direction_tolerance=1e-6,
+        polarization_normal_tolerance=1e-6,
+        polarization_fraction_tolerance=1e-6,
+    ):
+        self.wavelength_tolerance = wavelength_tolerance
+        self.direction_tolerance = direction_tolerance
+        self.polarization_normal_tolerance = polarization_normal_tolerance
+        self.polarization_fraction_tolerance = polarization_fraction_tolerance
+
+    def __call__(self, a, b):
+        if a is None and b is None:
+            return True
+        return a.is_similar_to(
+            b,
+            wavelength_tolerance=self.wavelength_tolerance,
+            direction_tolerance=self.direction_tolerance,
+            polarization_normal_tolerance=self.polarization_normal_tolerance,
+            polarization_fraction_tolerance=self.polarization_fraction_tolerance,
+        )
+
+
+class DetectorComparison:
+    """A class to provide simple detector comparison"""
+
+    def __init__(
+        self, fast_axis_tolerance=1e-6, slow_axis_tolerance=1e-6, origin_tolerance=1e-6
+    ):
+        self.fast_axis_tolerance = fast_axis_tolerance
+        self.slow_axis_tolerance = slow_axis_tolerance
+        self.origin_tolerance = origin_tolerance
+
+    def __call__(self, a, b):
+        if a is None and b is None:
+            return True
+        return a.is_similar_to(
+            b,
+            fast_axis_tolerance=self.fast_axis_tolerance,
+            slow_axis_tolerance=self.slow_axis_tolerance,
+            origin_tolerance=self.origin_tolerance,
+        )
+
+
+class GoniometerComparison:
+    """A class to provide simple goniometer comparison"""
+
+    def __init__(
+        self,
+        rotation_axis_tolerance=1e-6,
+        fixed_rotation_tolerance=1e-6,
+        setting_rotation_tolerance=1e-6,
+    ):
+        self.rotation_axis_tolerance = rotation_axis_tolerance
+        self.fixed_rotation_tolerance = fixed_rotation_tolerance
+        self.setting_rotation_tolerance = setting_rotation_tolerance
+
+    def __call__(self, a, b):
+        if a is None and b is None:
+            return True
+        elif a is None or b is None:
+            return False
+        return a.is_similar_to(
+            b,
+            rotation_axis_tolerance=self.rotation_axis_tolerance,
+            fixed_rotation_tolerance=self.fixed_rotation_tolerance,
+            setting_rotation_tolerance=self.setting_rotation_tolerance,
+        )
+
+
 class ExperimentListDict:
     """A helper class for serializing the experiment list to dictionary (needed
     to save the experiment list to JSON format."""
@@ -484,7 +595,9 @@ class ExperimentListFactory:
     """A class to help instantiate experiment lists."""
 
     @staticmethod
-    def from_args(args, unhandled=None):
+    def from_args(
+        args: list[str], unhandled: list[str] | None = None, check_format: bool = True
+    ) -> ExperimentList:
         """Try to load serialised experiments from any recognised format."""
 
         # Create a list for unhandled arguments
@@ -497,12 +610,15 @@ def from_args(args, unhandled=None):
         for filename in args:
             try:
                 experiments.extend(
-                    ExperimentListFactory.from_serialized_format(filename)
+                    ExperimentListFactory.from_serialized_format(
+                        filename, check_format=check_format
+                    )
                 )
                 logger.debug(f"Loaded experiments from {filename}")
             except Exception as e:
                 logger.debug(f"Could not load experiments from {filename}: {e}")
                 unhandled.append(filename)
+                raise
 
         return experiments
 
@@ -522,7 +638,7 @@ def from_filenames(
 
         # Process each file given by this path list
         to_process = _openingpathiterator(filenames)
-        find_format = dxtbx.datablock.FormatChecker()
+        find_format = FormatChecker()
 
         format_groups = collections.defaultdict(list)
         if format_kwargs is None:
@@ -567,7 +683,7 @@ def from_filenames(
         # - Any consecutive still frames that share any metadata with the
         #   previous still fram get collected into one ImageSet
 
-        # Treat each format as a separate datablock
+        # Treat each format as a separate block of data
         for format_class, records in format_groups.items():
             if issubclass(format_class, FormatMultiImage):
                 for imageset in records:
@@ -590,8 +706,7 @@ def from_filenames(
                 _convert_to_imagesets(records, format_class, format_kwargs)
             )
 
-            # Validate this datablock and store it
-            assert imagesets, "Datablock got no imagesets?"
+            assert imagesets, "Got no imagesets when constructing ExperimentList?"
             for imageset in imagesets:
                 experiments.extend(
                     ExperimentListFactory.from_imageset_and_crystal(
@@ -677,36 +792,6 @@ def from_stills_and_crystal(imageset, crystal, load_models=True):
                 )
         return experiments
 
-    @staticmethod
-    def from_datablock_and_crystal(datablock, crystal, load_models=True):
-        """Load an experiment list from a datablock."""
-
-        # Initialise the experiment list
-        experiments = ExperimentList()
-
-        # If we have a list, loop through
-        if isinstance(datablock, list):
-            for db in datablock:
-                experiments.extend(
-                    ExperimentListFactory.from_datablock_and_crystal(
-                        db, crystal, load_models
-                    )
-                )
-            return experiments
-
-        # Add all the imagesets
-        for imageset in datablock.extract_imagesets():
-            experiments.extend(
-                ExperimentListFactory.from_imageset_and_crystal(
-                    imageset, crystal, load_models
-                )
-            )
-
-        # Check the list is consistent
-        assert experiments.is_consistent()
-
-        return experiments
-
     @staticmethod
     def from_dict(obj, check_format=True, directory=None):
         """
@@ -724,22 +809,9 @@ def from_dict(obj, check_format=True, directory=None):
             ExperimentList: The dictionary converted
         """
 
-        try:
-            experiments = ExperimentList()
-            for db in DataBlockFactory.from_dict(
-                obj, check_format=check_format, directory=directory
-            ):
-                experiments.extend(
-                    ExperimentListFactory.from_datablock_and_crystal(db, None)
-                )
-        except Exception:
-            experiments = None
-
-        # Decode the experiments from the dictionary
-        if experiments is None:
-            experiments = ExperimentListDict(
-                obj, check_format=check_format, directory=directory
-            ).decode()
+        experiments = ExperimentListDict(
+            obj, check_format=check_format, directory=directory
+        ).decode()
 
         # Check the list is consistent
         assert experiments.is_consistent()
@@ -806,7 +878,7 @@ def from_templates(templates, **kwargs):
         assert len(templates) > 0
 
         experiments = ExperimentList()
-        find_format = dxtbx.datablock.FormatChecker()
+        find_format = FormatChecker()
 
         # For each template do an import
         for template in templates:
@@ -893,7 +965,7 @@ def from_templates(templates, **kwargs):
 class ImageMetadataRecord:
     """Object to store metadata information.
 
-    This is used whilst building the datablocks.  The metadata for each
+    This is used whilst building the experiment lists. The metadata for each
     image can be read once, and then any grouping/deduplication can happen
     later, without re-opening the original file.
     """
diff --git a/src/dxtbx/serialize/load.py b/src/dxtbx/serialize/load.py
index 4437fd49b..bd0e85642 100644
--- a/src/dxtbx/serialize/load.py
+++ b/src/dxtbx/serialize/load.py
@@ -24,22 +24,6 @@ def imageset(filename):
         return imageset_from_dict(json.load(infile), directory=directory)
 
 
-def datablock(filename, check_format=True):
-    """Load a given JSON or pickle file.
-
-    Params:
-      filename The input filename
-
-    Returns:
-      The datablock
-
-    """
-    # Resolve recursive import
-    from dxtbx.datablock import DataBlockFactory
-
-    return DataBlockFactory.from_serialized_format(filename, check_format=check_format)
-
-
 def crystal(infile):
     """Load the given JSON file.
 
diff --git a/src/dxtbx/util/__init__.py b/src/dxtbx/util/__init__.py
index 5cad10dc2..d0d0cd799 100644
--- a/src/dxtbx/util/__init__.py
+++ b/src/dxtbx/util/__init__.py
@@ -1,10 +1,21 @@
 from __future__ import annotations
 
+import json
 import math
 import os
 import sys
 from urllib.parse import urlparse
 
+import libtbx
+
+
+class AutoEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, libtbx.AutoType):
+            return "Auto"
+        # Let the base class default method raise the TypeError
+        return json.JSONEncoder.default(self, obj)
+
 
 def encode_output_as_utf8() -> None:
     """
diff --git a/tests/model/test_experiment_list.py b/tests/model/test_experiment_list.py
index cca54c12d..2046114f5 100644
--- a/tests/model/test_experiment_list.py
+++ b/tests/model/test_experiment_list.py
@@ -13,7 +13,6 @@
 
 import dxtbx
 import dxtbx.model.experiment_list
-from dxtbx.datablock import DataBlockFactory
 from dxtbx.format.Format import Format
 from dxtbx.imageset import ImageSetFactory
 from dxtbx.model import (
@@ -531,64 +530,6 @@ def test_experimentlist_factory_from_sequence():
     assert experiments[0].crystal
 
 
-def test_experimentlist_factory_from_datablock():
-    filenames = ["filename_%01d.cbf" % (i + 1) for i in range(0, 2)]
-
-    imageset = Format.get_imageset(
-        filenames,
-        beam=Beam(),
-        detector=Detector(),
-        goniometer=Goniometer(),
-        scan=Scan((1, 2), (0, 1)),
-        as_sequence=True,
-    )
-
-    crystal = Crystal((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol="P1")
-
-    datablock = DataBlockFactory.from_imageset(imageset)
-    assert datablock[0].format_class()
-
-    experiments = ExperimentListFactory.from_datablock_and_crystal(datablock, crystal)
-
-    assert len(experiments) == 1
-    assert experiments[0].imageset
-    assert experiments[0].beam
-    assert experiments[0].detector is not None
-    assert experiments[0].goniometer
-    assert experiments[0].scan
-    assert experiments[0].crystal
-
-
-def test_experimentlist_to_datablock_imageset(dials_data):
-    filenames = [
-        str(f)
-        for f in dials_data("thaumatin_grid_scan", pathlib=False).listdir(
-            "thau_3_2_*.cbf.bz2"
-        )
-    ]
-    imageset = Format.get_imageset(filenames, as_imageset=True)
-    expts = ExperimentListFactory.from_imageset_and_crystal(imageset, crystal=None)
-    datablocks = expts.to_datablocks()
-    assert len(datablocks) == 1
-    assert datablocks[0].num_images() == len(expts) == len(imageset)
-    assert len(datablocks[0].extract_imagesets()) == len(expts)
-
-
-def test_experimentlist_to_datablock_centroid_test_data(dials_data):
-    filenames = [
-        str(f)
-        for f in dials_data("centroid_test_data", pathlib=False).listdir(
-            "centroid_*.cbf"
-        )
-    ]
-    expts = ExperimentListFactory.from_filenames(filenames)
-    datablocks = expts.to_datablocks()
-    assert len(datablocks) == 1
-    assert datablocks[0].num_images() == len(expts[0].imageset)
-    assert len(datablocks[0].extract_imagesets()) == len(expts)
-    assert len(datablocks[0].extract_imagesets()[0]) == len(expts[0].imageset)
-
-
 def test_experimentlist_dumper_dump_formats(monkeypatch, dials_regression, tmp_path):
     # Get all the filenames
     filename1 = os.path.join(
diff --git a/tests/test_datablock.py b/tests/test_datablock.py
index 04a735e5a..a0335ec04 100644
--- a/tests/test_datablock.py
+++ b/tests/test_datablock.py
@@ -1,14 +1,10 @@
 from __future__ import annotations
 
-import json
 import os
 import pickle
 
 import pytest
 
-from dxtbx.datablock import DataBlockFactory
-from dxtbx.imageset import ImageSequence
-
 
 @pytest.fixture(scope="session")
 def centroid_test_data(dials_regression):
@@ -58,118 +54,5 @@ def multiple_block_filenames(single_sequence_filenames, all_image_examples):
     return single_sequence_filenames + all_image_examples
 
 
-def encode_json_then_decode(obj, check_format=True):
-    string = json.dumps([db.to_dict() for db in obj], ensure_ascii=True)
-    return DataBlockFactory.from_json(string, check_format=check_format)
-
-
 def pickle_then_unpickle(obj):
     return pickle.loads(pickle.dumps(obj))
-
-
-def test_pickling(multiple_block_filenames):
-    blocks1 = DataBlockFactory.from_filenames(multiple_block_filenames)
-    blocks2 = pickle_then_unpickle(blocks1)
-    assert len(blocks2) == len(blocks1)
-    for b1, b2 in zip(blocks1, blocks2):
-        assert b1.format_class() == b2.format_class()
-        assert b1 == b2
-    assert blocks1 == blocks2
-
-
-def test_json(multiple_block_filenames):
-    blocks1 = DataBlockFactory.from_filenames(multiple_block_filenames)
-    blocks2 = encode_json_then_decode(blocks1)
-    assert len(blocks2) == len(blocks1)
-    for b1, b2 in zip(blocks1, blocks2):
-        assert b1.format_class() == b2.format_class()
-        assert b1 == b2
-    assert blocks1 == blocks2
-
-
-def test_json2(multiple_block_filenames):
-    blocks1 = DataBlockFactory.from_filenames(multiple_block_filenames)
-    blocks2 = encode_json_then_decode(blocks1, check_format=False)
-    assert len(blocks2) == len(blocks1)
-    for b1, b2 in zip(blocks1, blocks2):
-        for im1, im2 in zip(b1.extract_imagesets(), b2.extract_imagesets()):
-            assert len(im1) == len(im2)
-            if isinstance(im1, ImageSequence):
-                assert isinstance(im2, ImageSequence)
-                assert im1.get_beam() == im2.get_beam()
-                assert im1.get_detector() == im2.get_detector()
-                assert im1.get_goniometer() == im2.get_goniometer()
-                assert im1.get_scan() == im2.get_scan()
-            else:
-                assert not isinstance(im2, ImageSequence)
-                for i in range(len(im1)):
-                    assert im1.get_beam(i) == im2.get_beam(i)
-                    assert im1.get_detector(i) == im2.get_detector(i)
-
-
-def test_with_bad_external_lookup(centroid_test_data):
-    filename = os.path.join(centroid_test_data, "datablock_with_bad_lookup.json")
-    blocks = DataBlockFactory.from_json_file(filename, check_format=False)
-    assert len(blocks) == 1
-    imageset = blocks[0].extract_imagesets()[0]
-    assert imageset.external_lookup.mask.filename is not None
-    assert imageset.external_lookup.gain.filename is not None
-    assert imageset.external_lookup.pedestal.filename is not None
-    assert imageset.external_lookup.mask.data.empty()
-    assert imageset.external_lookup.gain.data.empty()
-    assert imageset.external_lookup.pedestal.data.empty()
-
-    blocks = encode_json_then_decode(blocks, check_format=False)
-    assert len(blocks) == 1
-    imageset = blocks[0].extract_imagesets()[0]
-    assert imageset.external_lookup.mask.filename is not None
-    assert imageset.external_lookup.gain.filename is not None
-    assert imageset.external_lookup.pedestal.filename is not None
-    assert imageset.external_lookup.mask.data.empty()
-    assert imageset.external_lookup.gain.data.empty()
-    assert imageset.external_lookup.pedestal.data.empty()
-
-
-def test_with_external_lookup(centroid_test_data):
-    filename = os.path.join(centroid_test_data, "datablock_with_lookup.json")
-    blocks = DataBlockFactory.from_json_file(filename)
-    assert len(blocks) == 1
-    imageset = blocks[0].extract_imagesets()[0]
-    assert not imageset.external_lookup.mask.data.empty()
-    assert not imageset.external_lookup.gain.data.empty()
-    assert not imageset.external_lookup.pedestal.data.empty()
-    assert imageset.external_lookup.mask.filename is not None
-    assert imageset.external_lookup.gain.filename is not None
-    assert imageset.external_lookup.pedestal.filename is not None
-    assert imageset.external_lookup.mask.data.tile(0).data().all_eq(True)
-    assert imageset.external_lookup.gain.data.tile(0).data().all_eq(1)
-    assert imageset.external_lookup.pedestal.data.tile(0).data().all_eq(0)
-
-    blocks = encode_json_then_decode(blocks)
-    assert len(blocks) == 1
-    imageset = blocks[0].extract_imagesets()[0]
-    assert not imageset.external_lookup.mask.data.empty()
-    assert not imageset.external_lookup.gain.data.empty()
-    assert not imageset.external_lookup.pedestal.data.empty()
-    assert imageset.external_lookup.mask.filename is not None
-    assert imageset.external_lookup.gain.filename is not None
-    assert imageset.external_lookup.pedestal.filename is not None
-    assert imageset.external_lookup.mask.data.tile(0).data().all_eq(True)
-    assert imageset.external_lookup.gain.data.tile(0).data().all_eq(1)
-    assert imageset.external_lookup.pedestal.data.tile(0).data().all_eq(0)
-
-
-@pytest.mark.xfail(
-    raises=AssertionError, reason="https://github.com/cctbx/dxtbx/issues/336"
-)
-def test_single_image_datablock(dials_regression):
-    path = os.path.join(
-        dials_regression,
-        "image_examples",
-        "LCLS_cspad_nexus",
-        "idx-20130301060858401.cbf",
-    )
-    datablocks_cbf = DataBlockFactory.from_filenames([path])
-    datablock_cbf = datablocks_cbf[0]
-    imageset_cbf = datablock_cbf.extract_imagesets()[0]
-    assert imageset_cbf.get_detector(0) is not None

From 9657a1625e8d08842d8c7554cbd42f367b6faac5 Mon Sep 17 00:00:00 2001
From: DiamondLightSource-build-server
 <DiamondLightSource-build-server@users.noreply.github.com>
Date: Thu, 17 Aug 2023 20:08:48 +0100
Subject: [PATCH 45/45] Clean Clutter

---
 src/dxtbx/format/nxmx_writer.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/dxtbx/format/nxmx_writer.py b/src/dxtbx/format/nxmx_writer.py
index c9dfccd2b..58e705484 100644
--- a/src/dxtbx/format/nxmx_writer.py
+++ b/src/dxtbx/format/nxmx_writer.py
@@ -9,20 +9,23 @@
 from __future__ import annotations
 
 import datetime
-import h5py
-import numpy as np
 import os
 import sys
 
+import h5py
+import numpy as np
+
 from cctbx import factor_ev_angstrom
-from dials.util.options import ArgumentParser, flatten_experiments
-from dxtbx import flumpy
 from libtbx import easy_pickle
 from libtbx.phil import parse
 from libtbx.utils import Sorry
 from scitbx import matrix
 from scitbx.array_family import flex
-from xfel.cftbx.detector.cspad_cbf_tbx import basis, angle_and_axis
+from xfel.cftbx.detector.cspad_cbf_tbx import angle_and_axis, basis
+
+from dials.util.options import ArgumentParser, flatten_experiments
+
+from dxtbx import flumpy
 
 help_message = """
 Create a NeXus file from either an experiment list or a set of image files