Merge branch 'main' into add_properties_to_scan

cctbx · Aug 17, 2023 · aebe73e · aebe73e
2 parents 75772ad + 9657a16
commit aebe73e
Show file tree

Hide file tree

Showing 74 changed files with 3,467 additions and 1,412 deletions.
diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml
@@ -14,11 +14,11 @@ stages:
     steps:
       - checkout: none
 
-      # Use Python >=3.8 for syntax validation
+      # Use Python >=3.9 for syntax validation
       - task: UsePythonVersion@0
         displayName: Set up python
         inputs:
-          versionSpec: 3.8
+          versionSpec: 3.9
 
       # Run syntax validation on a shallow clone
       - bash: |
@@ -78,7 +78,7 @@ stages:
         vmImage: ubuntu-20.04
       timeoutInMinutes: 60
       variables:
-        PYTHON_VERSION: 3.8
+        PYTHON_VERSION: 3.9
       steps:
       - template: unix-build.yml
 
@@ -88,10 +88,10 @@ stages:
         vmImage: ubuntu-20.04
       strategy:
         matrix:
-          python38:
-            PYTHON_VERSION: 3.8
           python39:
             PYTHON_VERSION: 3.9
+          python311:
+            PYTHON_VERSION: 3.11
       timeoutInMinutes: 60
       steps:
       - template: unix-build.yml
@@ -101,10 +101,10 @@ stages:
         vmImage: macOS-latest
       strategy:
         matrix:
-          python38:
-            PYTHON_VERSION: 3.8
           python39:
             PYTHON_VERSION: 3.9
+          python311:
+            PYTHON_VERSION: 3.11
       timeoutInMinutes: 60
       steps:
       - template: unix-build.yml
@@ -114,8 +114,10 @@ stages:
         vmImage: windows-2019
       strategy:
         matrix:
-          python38:
-            PYTHON_VERSION: 3.8
+          python39:
+            PYTHON_VERSION: 3.9
+          python311:
+            PYTHON_VERSION: 3.11
       timeoutInMinutes: 20
       steps:
       - template: windows-build.yml

diff --git a/.azure-pipelines/bootstrap.py b/.azure-pipelines/bootstrap.py
@@ -847,7 +847,7 @@ def run():
         "--python",
         help="Install this minor version of Python (default: %(default)s)",
         default="3.9",
-        choices=("3.8", "3.9", "3.10"),
+        choices=("3.9", "3.10", "3.11"),
     )
     parser.add_argument(
         "--branch",

diff --git a/.azure-pipelines/ci-conda-env.txt b/.azure-pipelines/ci-conda-env.txt
@@ -2,7 +2,7 @@ conda-forge::boost
 conda-forge::boost-cpp
 conda-forge::bzip2
 conda-forge::c-compiler<1.5
-conda-forge::cctbx-base==2022.12
+conda-forge::cctbx-base==2023.7
 conda-forge::conda
 conda-forge::cxx-compiler<1.5
 conda-forge::python-dateutil
@@ -18,10 +18,10 @@ conda-forge::matplotlib-base
 conda-forge::mrcfile
 conda-forge::numpy
 conda-forge::nxmx
+conda-forge::orderedset
 conda-forge::pillow>=5.4.1
 conda-forge::pint
 conda-forge::pip
-conda-forge::procrunner
 conda-forge::psutil
 conda-forge::pyrtf
 conda-forge::pybind11

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.14.dev
+current_version = 3.17.dev
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<release>[a-z]+)?(?P<patch>\d+)?

diff --git a/AUTHORS b/AUTHORS
@@ -14,6 +14,7 @@ Graeme Winter
 Ian Rees
 Iris Young
 James Beilsten-Edmands
+James Hester
 James Holton
 James Parkhurst
 Johan Hattne

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,3 +1,96 @@
+dxtbx 3.16.0 (2023-08-14)
+=========================
+
+Features
+--------
+
+- Add new Beam class ``dxtbx.model.PolychromaticBeam``, for polychromatic/multi-wavelength/wide bandpass experiments. (`#621 <https://github.com/cctbx/dxtbx/issues/621>`_)
+- Formats: Reflect move of Eiger detector from PETRA P14 to P13. (`#626 <https://github.com/cctbx/dxtbx/issues/626>`_)
+- The ``model.Beam`` object now has a ``probe`` value to keep track of the type of radiation. (`#647 <https://github.com/cctbx/dxtbx/issues/647>`_)
+- Formats: CBFMini support for the EIGER2 16M detector at CHESS beamline ID7B2, which has an inverted rotation axis. (`#649 <https://github.com/cctbx/dxtbx/issues/649>`_)
+- Formats: Support for Eiger 9M on ESRF ID23-2, which has an undeclared vertical goniometer. (`#651 <https://github.com/cctbx/dxtbx/issues/651>`_)
+- Formats: Partial support for the Rigaku Oxford Diffraction file format, including support for multi-axis goniometers and faster decompression. (`#645 <https://github.com/cctbx/dxtbx/issues/645>`_) (`#653 <https://github.com/cctbx/dxtbx/issues/653>`_)
+
+
+Bugfixes
+--------
+
+- Panel geometry definitions in PHIL are merged by panel id *before* constructing panels. (`#299 <https://github.com/cctbx/dxtbx/issues/299>`_)
+- ``flumpy``: Fix case where incorrect ``flex.vec2``, ``flex.vec3`` could be generated. (`#439 <https://github.com/cctbx/dxtbx/issues/439>`_)
+- NXmx files with multidimensional arrays (images, modules, or both) are now handled. (`#612 <https://github.com/cctbx/dxtbx/issues/612>`_)
+- Slicing of imageset objects is now consistently 0-based, including for the sliced data accessor. Previously, the data accessor had to be accessed with the original index offsets. (`#633 <https://github.com/cctbx/dxtbx/issues/633>`_)
+- Formats: Add fix for Eiger / NXmx data from DLS i19-2, to correctly assign the image bit depth. (`#652 <https://github.com/cctbx/dxtbx/issues/652>`_)
+
+
+Misc
+----
+
+- `#640 <https://github.com/cctbx/dxtbx/issues/640>`_, `#642 <https://github.com/cctbx/dxtbx/issues/642>`_, `#643 <https://github.com/cctbx/dxtbx/issues/643>`_, `#645 <https://github.com/cctbx/dxtbx/issues/645>`_, `#650 <https://github.com/cctbx/dxtbx/issues/650>`_, `#655 <https://github.com/cctbx/dxtbx/issues/655>`_
+
+
+DIALS 3.15.1 (2023-06-29)
+=========================
+
+Bugfixes
+--------
+
+- ``dxtbx.dlsnxs2cbf``: Fix import overwritten by local variable. (`#641 <https://github.com/cctbx/dxtbx/issues/641>`_)
+
+
+dxtbx 3.15.0 (2023-06-13)
+=========================
+
+Features
+--------
+
+- Support for Bruker Photon detectors has been extended to include Photon-III. (`#637 <https://github.com/cctbx/dxtbx/issues/637>`_)
+
+
+Bugfixes
+--------
+
+- Rigaku Saturn SMV images with multi-axis crystal goniometers are now handledi, instead of being silently ignored. With thanks to James Hester for this contribution. (`#617 <https://github.com/cctbx/dxtbx/issues/617>`_)
+- FormatCBFFull: If rotation angles are decreasing, then invert the rotation axis as well as the angles, to be consistent. (`#623 <https://github.com/cctbx/dxtbx/issues/623>`_)
+- Bugfix for CCTBX bootstrapped environments, without conda. (`#630 <https://github.com/cctbx/dxtbx/issues/630>`_)
+
+
+Misc
+----
+
+- `#625 <https://github.com/cctbx/dxtbx/issues/625>`_, `#636 <https://github.com/cctbx/dxtbx/issues/636>`_, `#639 <https://github.com/cctbx/dxtbx/issues/639>`_
+
+
+DIALS 3.14.2 (2023-05-16)
+=========================
+
+Bugfixes
+--------
+
+- Compatibility fix for the DECTRIS Eiger FileWriter. Recent FileWriter versions split bit depth metadata into two separate items, ``bit_depth_readout`` from the NXmx standard, and the new ``bit_depth_image`` field. This adds support for the latter, and now passes the metadata through into image conversion. (`#632 <https://github.com/cctbx/dxtbx/issues/632>`_)
+
+
+dxtbx 3.14.0 (2023-04-12)
+=========================
+
+Features
+--------
+
+- ``flumpy``: Add support for conversion of ``flex.miller_index`` arrays to/from numpy. (`#618 <https://github.com/cctbx/dxtbx/issues/618>`_)
+
+
+Bugfixes
+--------
+
+- Flumpy: Prefer returning ``flex.int`` instead of ``flex.long`` when they are the same size. This solves ambiguous behaviour when reading images on Windows platforms. (`#607 <https://github.com/cctbx/dxtbx/issues/607>`_)
+- ``dxtbx.plot_detector_models``: Fix display of multiple single-panel detector models. (`#610 <https://github.com/cctbx/dxtbx/issues/610>`_)
+
+
+Misc
+----
+
+- `#604 <https://github.com/cctbx/dxtbx/issues/604>`_, `#608 <https://github.com/cctbx/dxtbx/issues/608>`_, `#609 <https://github.com/cctbx/dxtbx/issues/609>`_, `#611 <https://github.com/cctbx/dxtbx/issues/611>`_, `#614 <https://github.com/cctbx/dxtbx/issues/614>`_
+
+
 dxtbx 3.13.0 (2023-01-26)
 =========================
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.20 FATAL_ERROR)
 project(dxtbx LANGUAGES C CXX)
 
 # Add the included modules
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/")
 
 # General cmake environment configuration
 include(SetDefaultBuildRelWithDebInfo) # Default builds to release with debug info

diff --git a/SConscript b/SConscript
@@ -2,6 +2,7 @@ import sys
 import os
 import libtbx.load_env
 from libtbx.env_config import get_boost_library_with_python_version
+from pathlib import Path
 
 try:
     # Check if we have a "modernized" pycbf
@@ -31,6 +32,7 @@ env_etc.dxtbx_common_includes = [
 
 Import("env_no_includes_boost_python_ext")
 env = env_no_includes_boost_python_ext.Clone()
+
 if libtbx.env.build_options.use_conda:
     boost_python = get_boost_library_with_python_version(
         "boost_python", env_etc.conda_libpath
@@ -51,7 +53,6 @@ env_etc.dxtbx_hdf5_libs = ["hdf5"]
 env_etc.dxtbx_hdf5_lib_paths = []
 
 if sys.platform == "win32" and env_etc.compiler == "win32_cl":
-
     if libtbx.env.build_options.use_conda:
         env_etc.dxtbx_hdf5_libs = ["hdf5"]
         env_etc.cppdefines = {"H5_BUILT_AS_DYNAMIC_LIB": 1}
@@ -116,6 +117,13 @@ else:
 if not env_etc.no_boost_python and hasattr(env_etc, "boost_adaptbx_include"):
     Import("env_no_includes_boost_python_ext")
     env = env_no_includes_boost_python_ext.Clone()
+
+    # Don't surface warnings from system or cctbx_project headers
+    system_includes = [x for x in env_etc.conda_cpppath if x] if libtbx.env.build_options.use_conda else []
+    system_includes.append(str(Path(env_etc.scitbx_dist).parent))
+    env.Append(CXXFLAGS=[f"-isystem{x}" for x in system_includes])
+    env.Append(SHCXXFLAGS=[f"-isystem{x}" for x in system_includes])
+
     env_etc.enable_more_warnings(env=env)
     env_etc.include_registry.append(
         env=env,

diff --git a/libtbx_config b/libtbx_config
@@ -3,7 +3,6 @@
     "python_required": [
         "dials-data>=2.0.30",
         "pint",
-        "procrunner>=1.0.2",
         "pytest>=4.5,<5.0",
     ],
 }
diff --git a/libtbx_refresh.py b/libtbx_refresh.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import contextlib
+import importlib
 import inspect
 import io
 import os
@@ -17,11 +18,6 @@
 except ModuleNotFoundError:
     pass
 
-try:
-    import pkg_resources
-except ModuleNotFoundError:
-    pkg_resources = None
-
 
 def _install_setup_readonly_fallback(package_name: str):
     """
@@ -67,9 +63,8 @@ def _install_setup_readonly_fallback(package_name: str):
     if import_path not in sys.path:
         sys.path.insert(0, import_path)
 
-    # ...and add to the existing pkg_resources working_set
-    if pkg_resources:
-        pkg_resources.working_set.add_entry(import_path)
+    # ...and make sure it is picked up by the import system
+    importlib.invalidate_caches()
 
     # This is already generated by this point, but will get picked up
     # on the second libtbx.refresh.

diff --git a/newsfragments/504.removal b/newsfragments/504.removal
@@ -0,0 +1 @@
+dxtbx: remove legacy datablock object (obsolete for several years)
diff --git a/newsfragments/604.misc b/newsfragments/604.misc
diff --git a/newsfragments/607.bugfix b/newsfragments/607.bugfix
diff --git a/newsfragments/608.misc b/newsfragments/608.misc
diff --git a/newsfragments/609.misc b/newsfragments/609.misc
diff --git a/newsfragments/610.bugfix b/newsfragments/610.bugfix
diff --git a/newsfragments/611.misc b/newsfragments/611.misc
diff --git a/newsfragments/614.misc b/newsfragments/614.misc
diff --git a/newsfragments/615.feature b/newsfragments/615.feature
@@ -0,0 +1 @@
+Add nxmx_writer, a tool for converting any data dxtbx can read to NeXus data
diff --git a/newsfragments/622.misc b/newsfragments/622.misc
@@ -0,0 +1 @@
+Add sample_to_source_distance to Beam
diff --git a/newsfragments/656.bugfix b/newsfragments/656.bugfix
@@ -0,0 +1 @@
+Fix situation where a bad ``Beam.probe`` could cause undefined behaviour.
diff --git a/pytest.ini b/pytest.ini
@@ -3,8 +3,8 @@ addopts = -rsxX
 filterwarnings =
     ignore:the matrix subclass is not the recommended way:PendingDeprecationWarning
     ignore:numpy.dtype size changed:RuntimeWarning
-    ignore:Datablocks are deprecated:UserWarning
     ignore:Deprecated call to `pkg_resources.declare_namespace:DeprecationWarning
+    ignore:`product` is deprecated as of NumPy:DeprecationWarning:h5py|numpy
 junit_family = legacy
 markers =
     regression: dxtbx regression test
diff --git a/setup.cfg b/setup.cfg
@@ -8,8 +8,9 @@ classifiers =
     Operating System :: Microsoft :: Windows
     Operating System :: POSIX :: Linux
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
 
 [flake8]
 # Black disagrees with flake8 on a few points. Ignore those.

diff --git a/setup.py b/setup.py
@@ -8,7 +8,7 @@
 
 # Static version number which is updated by bump2version
 # Do not change this manually - use 'bump2version <major/minor/patch/release>'
-__version_tag__ = "3.14.dev"
+__version_tag__ = "3.17.dev"
 
 setup_kwargs = {
     "name": "dxtbx",

diff --git a/src/dxtbx/array_family/flex_table_suite.h b/src/dxtbx/array_family/flex_table_suite.h
@@ -19,6 +19,8 @@
 #include <scitbx/array_family/versa.h>
 #include <dxtbx/error.h>
 
+#include "ref_pickle_double_buffered.h"
+
 namespace dxtbx { namespace af { namespace flex_table_suite {
 
   using namespace boost::python;

diff --git a/src/dxtbx/array_family/ref_pickle_double_buffered.h b/src/dxtbx/array_family/ref_pickle_double_buffered.h
@@ -0,0 +1,8 @@
+#ifndef SCITBX_ARRAY_FAMILY_BOOST_PYTHON_REF_PICKLE_DOUBLE_BUFFERED_H_WRAPPER
+#define SCITBX_ARRAY_FAMILY_BOOST_PYTHON_REF_PICKLE_DOUBLE_BUFFERED_H_WRAPPER
+
+// This header does not have an include guard
+
+#include <scitbx/array_family/boost_python/ref_pickle_double_buffered.h>
+
+#endif