Merge branch 'bluesky:main' into smse_typesense

.github/workflows/ci.yml

@@ -27,7 +27,6 @@ jobs:
     strategy:
       matrix:
         python-version:
-        - "3.8"
         - "3.9"
         - "3.10"
         - "3.11"

CHANGELOG.md

@@ -3,6 +3,29 @@ Write the date in place of the "Unreleased" in the case a new version is release
 
 # Changelog
 
+## v0.1.0b11 (2024-11-14)
+
+### Added
+
+- Add adapters for reading back assets with the image/jpeg and
+  multipart/related;type=image/jpeg mimetypes.
+- Automatic reshaping of tiff data by the adapter to account for
+  extra/missing singleton dimension
+ - Add a check for the `openpyxcl` module when importing excel serializer.
+
+### Changed
+
+- Drop support for Python 3.8, which is reached end of life
+  upstream on 7 October 2024.
+- Do not require SQL database URIs to specify a "driver" (Python
+  library to be used for connecting).
+
+### Fixed
+
+- A regression in the container broke support for `tiled register ...` and
+  `tiled serve directory ...`. When these became client-side operations, the
+  container needed to add the client-side dependencies to support them.
+
 ## v0.1.0b10 (2024-10-11)
 
 - Add kwarg to client logout to auto-clear default identity.

Dockerfile

@@ -37,7 +37,10 @@ COPY . .
 
 # Skip building the UI here because we already did it in the stage
 # above using a node container.
-RUN TILED_BUILD_SKIP_UI=1 pip install '.[server]'
+# Include server and client depedencies here because this container may be used
+# for `tiled register ...` and `tiled server directory ...` which invokes
+# client-side code.
+RUN TILED_BUILD_SKIP_UI=1 pip install '.[all]'
 
 # FROM base as test
 #

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.2"
 services:
   tiled:
-    image: ghcr.io/bluesky/tiled:v0.1.0b10
+    image: ghcr.io/bluesky/tiled:v0.1.0b11
     environment:
       - TILED_SINGLE_USER_API_KEY=${TILED_SINGLE_USER_API_KEY}
     ports:

pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 maintainers = [
   { name = "Brookhaven National Laboratory", email = "[email protected]" },
 ]
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 
 # All dependencies are optional; it depends on whether you are running
 # a client or server (or both) and what data structures you care about.
@@ -23,8 +23,6 @@ classifiers = [
   "Development Status :: 4 - Beta",
   "License :: OSI Approved :: BSD License",
   "Programming Language :: Python :: 3 :: Only",
-  "Programming Language :: Python :: 3.7",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
@@ -152,7 +150,7 @@ dev = [
     "ipython",
     "ldap3",
     "matplotlib",
-    "mistune <2.0.0",  # temporary while sphinx sorts this out,
+    "mistune",
     "myst-parser",
     "numpydoc",
     "pre-commit",

tiled/_tests/conftest.py

@@ -13,6 +13,7 @@
 from ..catalog import from_uri, in_memory
 from ..client.base import BaseClient
 from ..server.settings import get_settings
+from ..utils import ensure_specified_sql_driver
 from .utils import enter_username_password as utils_enter_uname_passwd
 from .utils import temp_postgres
 
@@ -152,7 +153,7 @@ async def postgresql_with_example_data_adapter(request, tmpdir):
     if uri.endswith("/"):
         uri = uri[:-1]
     uri_with_database_name = f"{uri}/{DATABASE_NAME}"
-    engine = create_async_engine(uri_with_database_name)
+    engine = create_async_engine(ensure_specified_sql_driver(uri_with_database_name))
     try:
         async with engine.connect():
             pass

tiled/_tests/test_directory_walker.py

@@ -16,10 +16,10 @@
 from ..client import Context, from_context
 from ..client.register import (
     Settings,
-    group_tiff_sequences,
+    group_image_sequences,
     identity,
     register,
-    register_tiff_sequence,
+    register_image_sequence,
     skip_all,
     strip_suffixes,
 )
@@ -155,14 +155,14 @@ async def test_skip_all_in_combination(tmpdir):
     # With skip_all, directories and tiff sequence are registered, but individual files are not
     with Context.from_app(build_app(catalog)) as context:
         client = from_context(context)
-        await register(client, tmpdir, walkers=[group_tiff_sequences, skip_all])
+        await register(client, tmpdir, walkers=[group_image_sequences, skip_all])
         assert list(client) == ["one"]
         assert "image" in client["one"]
 
 
 @pytest.mark.asyncio
 async def test_tiff_seq_custom_sorting(tmpdir):
-    "Register TIFFs that are not in alphanumeric order."
+    "Register images that are not in alphanumeric order."
     N = 10
     ordering = list(range(N))
     random.Random(0).shuffle(ordering)
@@ -177,7 +177,7 @@ async def test_tiff_seq_custom_sorting(tmpdir):
     catalog = in_memory(writable_storage=tmpdir)
     with Context.from_app(build_app(catalog)) as context:
         client = from_context(context)
-        await register_tiff_sequence(
+        await register_image_sequence(
             client,
             "image",
             files,

tiled/_tests/test_jpeg.py

@@ -0,0 +1,189 @@
+from pathlib import Path
+
+import numpy
+import pytest
+from PIL import Image
+
+from ..adapters.jpeg import JPEGAdapter, JPEGSequenceAdapter
+from ..adapters.mapping import MapAdapter
+from ..catalog import in_memory
+from ..client import Context, from_context
+from ..client.register import IMG_SEQUENCE_EMPTY_NAME_ROOT, register
+from ..server.app import build_app
+from ..utils import ensure_uri
+
+COLOR_SHAPE = (11, 17, 3)
+
+
+@pytest.fixture(scope="module")
+def client(tmpdir_module):
+    sequence_directory = Path(tmpdir_module, "sequence")
+    sequence_directory.mkdir()
+    filepaths = []
+    for i in range(3):
+        # JPEGs can only be 8 bit ints
+        data = numpy.random.randint(0, 255, (5, 7), dtype="uint8")
+        filepath = sequence_directory / f"temp{i:05}.jpeg"
+        Image.fromarray(data).convert("L").save(filepath)
+        filepaths.append(filepath)
+    color_data = numpy.random.randint(0, 255, COLOR_SHAPE, dtype="uint8")
+    path = Path(tmpdir_module, "color.jpeg")
+    Image.fromarray(color_data).convert("RGB").save(path)
+
+    tree = MapAdapter(
+        {
+            "color": JPEGAdapter(ensure_uri(path)),
+            "sequence": JPEGSequenceAdapter.from_uris(
+                [ensure_uri(filepath) for filepath in filepaths]
+            ),
+        }
+    )
+    app = build_app(tree)
+    with Context.from_app(app) as context:
+        client = from_context(context)
+        yield client
+
+
+@pytest.mark.parametrize(
+    "slice_input, correct_shape",
+    [
+        (None, (3, 5, 7)),
+        (0, (5, 7)),
+        (slice(0, 3, 2), (2, 5, 7)),
+        ((1, slice(0, 3), slice(0, 3)), (3, 3)),
+        ((slice(0, 3), slice(0, 3), slice(0, 3)), (3, 3, 3)),
+        ((..., 0, 0), (3,)),
+        ((0, slice(0, 1), slice(0, 2), ...), (1, 2)),
+        ((0, ..., slice(0, 2)), (5, 2)),
+        ((..., slice(0, 1)), (3, 5, 1)),
+    ],
+)
+def test_jpeg_sequence(client, slice_input, correct_shape):
+    arr = client["sequence"].read(slice=slice_input)
+    assert arr.shape == correct_shape
+
+
+@pytest.mark.parametrize("block_input, correct_shape", [((0, 0, 0), (1, 5, 7))])
+def test_jpeg_sequence_block(client, block_input, correct_shape):
+    arr = client["sequence"].read_block(block_input)
+    assert arr.shape == correct_shape
+
+
+@pytest.mark.asyncio
+async def test_jpeg_sequence_order(tmpdir):
+    """
+    directory/
+      00001.jpeg
+      00002.jpeg
+      ...
+      00010.jpeg
+    """
+    data = numpy.ones((4, 5))
+    num_files = 10
+    for i in range(num_files):
+        Image.fromarray(data * i).convert("L").save(Path(tmpdir / f"image{i:05}.jpeg"))
+
+    adapter = in_memory(readable_storage=[tmpdir])
+    with Context.from_app(build_app(adapter)) as context:
+        client = from_context(context)
+        await register(client, tmpdir)
+        for i in range(num_files):
+            numpy.testing.assert_equal(client["image"][i], data * i)
+
+
+@pytest.mark.asyncio
+async def test_jpeg_sequence_with_directory_walker(tmpdir):
+    """
+    directory/
+      00001.jpeg
+      00002.jpeg
+      ...
+      00010.jpeg
+      single_image.jpeg
+      image00001.jpeg
+      image00002.jpeg
+      ...
+      image00010.jpeg
+      other_image00001.jpeg
+      other_image00002.jpeg
+      ...
+      other_image00010.jpeg
+      other_image2_00001.jpeg
+      other_image2_00002.jpeg
+      ...
+      other_image2_00010.jpeg
+      other_file1.csv
+      other_file2.csv
+      stuff.csv
+    """
+    data = numpy.random.randint(0, 255, (3, 5), dtype="uint8")
+    for i in range(10):
+        Image.fromarray(data).convert("L").save(Path(tmpdir / f"image{i:05}.jpeg"))
+        Image.fromarray(data).convert("L").save(
+            Path(tmpdir / f"other_image{i:05}.jpeg")
+        )
+        Image.fromarray(data).convert("L").save(Path(tmpdir / f"{i:05}.jpeg"))
+        Image.fromarray(data).convert("L").save(
+            Path(tmpdir / f"other_image2_{i:05}.jpeg")
+        )
+    Image.fromarray(data).save(Path(tmpdir / "single_image.jpeg"))
+    for target in ["stuff.csv", "other_file1.csv", "other_file2.csv"]:
+        with open(Path(tmpdir / target), "w") as file:
+            file.write(
+                """
+a,b,c
+1,2,3
+"""
+            )
+    adapter = in_memory(readable_storage=[tmpdir])
+    with Context.from_app(build_app(adapter)) as context:
+        client = from_context(context)
+        await register(client, tmpdir)
+        # Single image is its own node.
+        assert client["single_image"].shape == (3, 5)
+        # Each sequence is grouped into a node.
+        assert client[IMG_SEQUENCE_EMPTY_NAME_ROOT].shape == (10, 3, 5)
+        assert client["image"].shape == (10, 3, 5)
+        assert client["other_image"].shape == (10, 3, 5)
+        assert client["other_image2_"].shape == (10, 3, 5)
+        # The sequence grouping digit-only files appears with a uuid
+        named_keys = [
+            "single_image",
+            "image",
+            "other_image",
+            "other_image2_",
+            "other_file1",
+            "other_file2",
+            "stuff",
+        ]
+        no_name_keys = [key for key in client.keys() if key not in named_keys]
+        # There is only a single one of this type
+        assert len(no_name_keys) == 1
+        assert client[no_name_keys[0]].shape == (10, 3, 5)
+        # Other files are single nodes.
+        assert client["stuff"].columns == ["a", "b", "c"]
+        assert client["other_file1"].columns == ["a", "b", "c"]
+        assert client["other_file2"].columns == ["a", "b", "c"]
+
+
+def test_rgb(client):
+    "Test an RGB JPEG."
+    arr = client["color"].read()
+    assert arr.shape == COLOR_SHAPE
+
+
+def test_jpeg_sequence_cache(client):
+    from numpy.testing import assert_raises
+
+    # The two requests go through the same method in the server (read_block) to
+    # call the same object
+    indexed_array = client["sequence"][0]
+    read_array = client["sequence"].read(0)
+
+    # Using a different index to confirm that the previous cache doesn't affect the new array
+    other_read_array = client["sequence"].read(1)
+
+    numpy.testing.assert_equal(indexed_array, read_array)
+    assert_raises(
+        AssertionError, numpy.testing.assert_equal, read_array, other_read_array
+    )