Release 1.0.0-rc3 (#760)

* deploy: e5851e0

* deploy: fa0d20f

---------

Co-authored-by: DriesSchaumont <[email protected]>

.github/workflows/main-build.yml

@@ -1,6 +1,6 @@
 name: Build
 concurrency: 
-  group: ${{ github.workflow }}-${{ github.event.inputs.deploy_branch && format('{0}_build', github.ref_name) || github.event.inputs.deploy_branch }}
+  group: ${{ github.workflow }}-${{ inputs.deploy_branch && inputs.deploy_branch || format('{0}_build', github.ref_name) }}
   cancel-in-progress: true
 
 on:
@@ -20,11 +20,10 @@ on:
       target_tag:
         type: string
         required: false
-        default: main_build
         description: |
-          Version tag of containers to use. Is `main_build` by default.
-          Can be used in combination with 'push_containers' to re-use existing docker images
-          or set the tag for new builds.
+          Version tag of containers to use. Defaults to name of the branch that triggered the workflow, 
+          suffixed by "_build". Can be used in combination with 'push_containers' (by unchecking it)
+          to re-use existing docker images or set the tag for new builds.
       deploy_to_viash_hub:
         type: boolean
         required: false
@@ -48,8 +47,10 @@ on:
       target_tag:
         type: string
         required: false
-        default: main_build
-        description: Version tag of existing containers to use. Is `main_build` by default.
+        description: |
+          Version tag of containers to use. Defaults to name of the branch that triggered the workflow, 
+          suffixed by "_build". Can be used in combination with 'push_containers' (by setting it to 'false')
+          to re-use existing docker images or set the tag for new builds.
       deploy_branch:
         type: string
         required: false
@@ -95,6 +96,16 @@ jobs:
       DEPLOY_BRANCH: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}
 
     steps:
+    - name: Check input arguments
+      run: |
+        input_version="${{ inputs.version || format('{0}_build', github.ref_name) }}"
+        target_tag="${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}"
+        should_push="${{inputs.push_containers }}"
+        if [ "$input_version" != "$target_tag" ] && [ "$should_push" == "true" ]; then
+          echo "When trying to push new docker images, the tag for the components must be equal to the target_tag for the docker images."
+          exit 1
+        fi
+
     - name: Keep symlinks as-is
       run: |
         git config --global core.symlinks true
@@ -142,7 +153,7 @@ jobs:
       with:
         config_mod: |
           .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
-          .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }}'
+          .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}'
         parallel: true
         query: ^(?!workflows)
 
@@ -246,7 +257,7 @@ jobs:
       with:
         config_mod: |
           .functionality.version := " ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}"
-          .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }}'
+          .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}'
           .platforms[.type == 'docker'].target_organization := 'openpipelines-bio/openpipeline'
           .platforms[.type == 'docker'].target_registry := 'viash-hub.com:5050'
           .platforms[.type == 'docker'].target_image_source := 'https://viash-hub.com/openpipelines-bio/openpipeline'
@@ -335,7 +346,7 @@ jobs:
       if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }}
       run: |
         viash ns exec -s ${{ matrix.component.dir }} --apply_platform -p docker \
-        'docker tag ghcr.io/openpipelines-bio/{namespace}_{functionality-name}:${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }} viash-hub.com:5050/openpipelines-bio/openpipeline/{namespace}_{functionality-name}:${{ inputs.version || format('{0}_build', github.ref_name) }}'
+        'docker tag ghcr.io/openpipelines-bio/{namespace}${{matrix.component.namespace_separator}}{functionality-name}:${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }} viash-hub.com:5050/openpipelines-bio/openpipeline/{namespace}${{matrix.component.namespace_separator}}{functionality-name}:${{ inputs.version || format('{0}_build', github.ref_name) }}'
   
     - name: Push container to Viash-Hub
       if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }}

.github/workflows/release-build.yml

@@ -158,7 +158,7 @@ jobs:
       run: |
         viash test \
           "${{ matrix.component.config }}" \
-          --config_mod ".platforms[.type == 'docker'].image := 'ghcr.io/openpipelines-bio/${{ matrix.component.namespace }}_${{ matrix.component.name }}:${{ github.event.inputs.version_tag }}'" \
+          --config_mod ".platforms[.type == 'docker'].image := 'ghcr.io/openpipelines-bio/${{ matrix.component.namespace }}${{matrix.component.namespace_separator}}${{ matrix.component.name }}:${{ github.event.inputs.version_tag }}'" \
           --config_mod ".platforms[.type == 'docker'].setup := []" \
-          --cpus 2 \
-          --memory "6gb"
+          --cpus 4 \
+          --memory "12gb"

.github/workflows/viash-test.yml

@@ -54,13 +54,6 @@ jobs:
         dest_path: resources_test
         cache_key_prefix: resources_test__
 
-    - name: Get changed files
-      id: changed-files
-      uses: tj-actions/changed-files@v42
-      with:
-        separator: ";"
-        diff_relative: true
-
     - id: ns_list
       uses: viash-io/viash-actions/ns-list@v5
       with:
@@ -174,6 +167,6 @@ jobs:
       run: |
         viash test \
           "${{ matrix.component.config }}" \
-          --cpus 2 \
-          --memory "6gb"
+          --cpus 4 \
+          --memory "12gb"
 

CHANGELOG.md

@@ -1,3 +1,20 @@
+# openpipelines 1.0.0-rc3
+
+## BREAKING CHANGES
+
+* Docker image names now use `/` instead of `_` between the name of the component and the namespace (PR #712).
+
+## BUG FIXES
+
+* `rna_singlesample`: fixed a bug where selecting the column for the filtering with mitochondrial fractions 
+  using `obs_name_mitochondrial_fraction` was done with the wrong column name, causing `ValueError` (PR #743).
+
+* Fix publishing in `process_samples` and `process_batches` (PR #759).
+
+## NEW FUNCTIONALITY
+
+* `dimred/tsne` component: Added a tSNE dimensionality reduction component (PR #742).
+
 # openpipelines 1.0.0-rc2
 
 ## BUG FIXES

_viash.yaml

@@ -9,7 +9,7 @@ config_mods: |
   .functionality.arguments[.multiple == true].multiple_sep := ";"
   .functionality.argument_groups[true].arguments[.multiple == true].multiple_sep := ";"
   .functionality.test_resources += {path: 'src/base/openpipelinetestutils', dest: 'openpipelinetestutils'}
-  .platforms[.type == 'docker'].namespace_separator := '_'
+  .platforms[.type == 'docker'].namespace_separator := '/'
   .platforms[.type == 'docker'].target_registry := 'ghcr.io'
   .platforms[.type == 'docker'].target_organization := 'openpipelines-bio'
   .platforms[.type == 'docker'].target_image_source := 'https://github.com/openpipelines-bio/openpipeline'

src/authors/jakub_majercik.yaml

@@ -0,0 +1,11 @@
+name: Jakub Majercik
+info:
+  role: Contributor
+  links:
+    email: [email protected]
+    github: jakubmajercik
+    linkedin: jakubmajercik
+  organizations:
+    - name: Data Intuitive
+      href: https://www.data-intuitive.com
+      role: Bioinformatics Engineer

src/base/openpipelinetestutils/asserters.py

@@ -2,17 +2,18 @@
 import anndata
 import pandas as pd
 import numpy as np
-from scipy.sparse import issparse
+from scipy.sparse import issparse, spmatrix
 from mudata import MuData
 from pathlib import Path
 from pandas.testing import assert_frame_equal
 from typing import Literal
 from .typing import AnnotationObjectOrPathLike
+from functools import singledispatch
 
 
 def _read_if_needed(anndata_mudata_path_or_obj):
     if isinstance(anndata_mudata_path_or_obj, (str, Path)):
-        return mudata.read(anndata_mudata_path_or_obj)
+        return mudata.read(str(anndata_mudata_path_or_obj)) # TODO: remove when mudata fixes PAth bug
     if isinstance(anndata_mudata_path_or_obj, (mudata.MuData, anndata.AnnData)):
         return anndata_mudata_path_or_obj.copy()
     raise AssertionError("Expected 'Path', 'str' to MuData/AnnData "
@@ -64,6 +65,12 @@ def assert_var_names_equal(left: AnnotationObjectOrPathLike, right: AnnotationOb
             assert_var_names_equal(modality, right[mod_name])
 
 
+def _assert_frame_equal(left, right, sort=False, *args, **kwargs):
+    if sort:
+        left, right = left.sort_index(inplace=False), right.sort_index(inplace=False)
+        left, right = left.sort_index(axis=1, inplace=False), right.sort_index(axis=1, inplace=False)
+    assert_frame_equal(left, right, *args, **kwargs) 
+
 def assert_annotation_frame_equal(annotation_attr: Literal["obs", "var"], 
                                    left: AnnotationObjectOrPathLike, right: AnnotationObjectOrPathLike, 
                                    sort=False, *args, **kwargs):
@@ -72,9 +79,7 @@ def assert_annotation_frame_equal(annotation_attr: Literal["obs", "var"],
     left, right = _read_if_needed(left), _read_if_needed(right)
     _assert_same_annotation_object_class(left, right)
     left_frame, right_frame = getattr(left, annotation_attr), getattr(right, annotation_attr)
-    if sort:
-        left_frame, right_frame = left_frame.sort_index(inplace=False), right_frame.sort_index(inplace=False)
-    assert_frame_equal(left_frame, right_frame, *args, **kwargs)
+    _assert_frame_equal(left_frame, right_frame, sort=sort, *args, **kwargs)
     if isinstance(left, MuData):
         assert_mudata_modality_keys_equal(left, right)
         for mod_name, modality in left.mod.items(): 
@@ -123,13 +128,49 @@ def assert_layers_equal(left: AnnotationObjectOrPathLike,
             assert_layers_equal(modality, right[mod_name])
 
 
+
+def assert_multidimensional_annotation_equal(annotation_attr: Literal["obsm", "varm"],
+                                             left, right, sort=False):
+    if not annotation_attr in ("obsm", "varm"):
+        raise ValueError("annotation_attr should be 'obsm', or 'varm'")
+    left, right = _read_if_needed(left), _read_if_needed(right)
+    _assert_same_annotation_object_class(left, right)
+
+    @singledispatch
+    def _assert_multidimensional_value_equal(left, right, **kwargs):
+        raise NotImplementedError("Unregistered type found while asserting")
+
+    @_assert_multidimensional_value_equal.register
+    def _(left: pd.DataFrame, right, **kwargs):
+        _assert_frame_equal(left, right, **kwargs)
+
+    @_assert_multidimensional_value_equal.register(np.ndarray)
+    @_assert_multidimensional_value_equal.register(spmatrix)
+    def _(left, right, **kwargs):
+        # Cannot sort sparse and dense matrices so ignore sort param
+        _assert_layer_equal(left, right)
+
+    left_dict, right_dict = getattr(left, annotation_attr), getattr(right, annotation_attr)
+    left_keys, right_keys = left_dict.keys(), right_dict.keys()
+    assert left_keys == right_keys, f"Keys of {annotation_attr} differ:\n[left]:{left_keys}\n[right]:{right_keys}"
+    for left_key, left_value in left_dict.items():
+        _assert_multidimensional_value_equal(left_value, right_dict[left_key], sort=sort)
+    if isinstance(left, MuData):
+        assert_mudata_modality_keys_equal(left, right)
+        for mod_name, modality in left.mod.items(): 
+            assert_multidimensional_annotation_equal(annotation_attr ,modality, right[mod_name], sort=sort)
+
+
 def assert_annotation_objects_equal(left: AnnotationObjectOrPathLike,
                                     right: AnnotationObjectOrPathLike,
-                                    check_data=True):
+                                    check_data=True,
+                                    sort=True):
     left, right = _read_if_needed(left), _read_if_needed(right)
     _assert_same_annotation_object_class(left, right)
     assert_shape_equal(left, right)
-    assert_annotation_frame_equal("obs", left, right)
-    assert_annotation_frame_equal("var", left, right)
+    assert_annotation_frame_equal("obs", left, right, sort=sort)
+    assert_annotation_frame_equal("var", left, right, sort=sort)
+    assert_multidimensional_annotation_equal("varm", left, right, sort=sort)
+    assert_multidimensional_annotation_equal("obsm", left, right, sort=sort)
     if check_data:
-        assert_layers_equal(left, right)
+        assert_layers_equal(left, right)

src/base/openpipelinetestutils/fixtures.py

@@ -1,5 +1,8 @@
 from uuid import uuid4
 import pytest
+import pandas as pd
+import anndata as ad
+import mudata as md
 
 @pytest.fixture
 def random_path(tmp_path):
@@ -20,4 +23,41 @@ def wrapper(mudata_obj):
         output_path = random_h5mu_path()
         mudata_obj.write(output_path)
         return output_path
-    return wrapper
+    return wrapper
+
+@pytest.fixture
+def small_anndata_1():
+    df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], index=["obs1", "obs2"], columns=["var1", "var2", "var3"])
+    obs = pd.DataFrame([["A"], ["B"]], index=df.index, columns=["Obs"])
+    var = pd.DataFrame([["a"], ["b"], ["c"]], index=df.columns, columns=["Feat"])
+    ad1 = ad.AnnData(df, obs=obs, var=var)
+    return ad1
+
+@pytest.fixture
+def small_anndata_2():
+    df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], index=["obs1", "obs2"], columns=["var4", "var5", "var6"])
+    obs2 = pd.DataFrame(["C", "D"], index=df.index, columns=["Obs"])
+    var2 = pd.DataFrame(["d", "e", "g"], index=df.columns, columns=["Feat"])
+    ad2 = ad.AnnData(df, obs=obs2, var=var2)
+    return ad2
+
+@pytest.fixture
+def small_mudata(small_anndata_1, small_anndata_2):
+    return md.MuData({'mod1': small_anndata_1, 'mod2': small_anndata_2})
+
+@pytest.fixture
+def small_mudata_path(small_mudata, write_mudata_to_file):
+    return write_mudata_to_file(small_mudata)
+
+@pytest.fixture
+def split_small_mudata_path(small_mudata_mod1_path, small_mudata_mod2_path):
+    return small_mudata_mod1_path, small_mudata_mod2_path
+
+@pytest.fixture
+def small_mudata_mod1_path(small_mudata, write_mudata_to_file):
+    return write_mudata_to_file(md.MuData({'mod1': small_mudata.mod['mod1']}))
+
+@pytest.fixture
+def small_mudata_mod2_path(small_mudata, write_mudata_to_file):
+    return write_mudata_to_file(md.MuData({'mod2': small_mudata.mod['mod2']}))
+

src/convert/from_h5ad_to_h5mu/config.vsh.yaml

@@ -48,9 +48,7 @@ platforms:
           - procps
       - type: python
         __merge__: /src/base/requirements/anndata_mudata.yaml
-    test_setup:
-      - type: python
-        __merge__: [ /src/base/requirements/viashpy.yaml, .]
+    __merge__: [ /src/base/requirements/python_test_setup.yaml, .]
   - type: nextflow
     directives:
       label: [lowmem, singlecpu]

src/convert/from_h5ad_to_h5mu/test.py

@@ -1,31 +1,33 @@
 import sys
 import pytest
 import mudata as mu
+from openpipelinetestutils.asserters import assert_annotation_objects_equal
 
 ## VIASH START
 meta = {
-    'resources_dir': 'resources_test'
+    'resources_dir': 'resources_test',
+    'executable': './target/docker/convert/from_h5ad_to_h5mu/from_h5ad_to_h5mu',
+    'config': './src/convert/from_h5ad_to_h5mu/config.vsh.yaml'
 }
 ## VIASH END
 
 input = meta["resources_dir"] + "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"
 
-def test_run(run_component, tmp_path):
+def test_run(run_component, random_h5mu_path, random_path):
     mdata = mu.read_h5mu(input)
-
-    tmp_rna = tmp_path / "rna.h5ad"
-    tmp_prot = tmp_path / "prot.h5ad"
+    tmp_rna = random_path(extension="h5ad")
+    tmp_prot = random_path(extension="h5ad")
     mdata.mod["rna"].write_h5ad(tmp_rna)
     mdata.mod["prot"].write_h5ad(tmp_prot)
 
-    tmp_output = tmp_path / "output.h5mu"
+    tmp_output = random_h5mu_path()
 
     cmd_pars = [
         "--modality", "rna",
-        "--input", str(tmp_rna),
+        "--input", tmp_rna,
         "--modality", "prot",
-        "--input", str(tmp_prot),
-        "--output", str(tmp_output),
+        "--input", tmp_prot,
+        "--output", tmp_output,
         "--output_compression", "gzip"
     ]
     run_component(cmd_pars)
@@ -34,8 +36,10 @@ def test_run(run_component, tmp_path):
 
     mdata2 = mu.read_h5mu(tmp_output)
 
-    assert "rna" in mdata2.mod, "Resulting mudata should contain rna modality"
-    assert "prot" in mdata2.mod, "Resulting mudata should contain rna modality"
+    assert list(mdata2.mod.keys()) == ["rna", "prot"]
+
+    assert_annotation_objects_equal(mdata2.mod["rna"], tmp_rna)
+    assert_annotation_objects_equal(mdata2.mod["prot"], tmp_prot)
 
 if __name__ == "__main__":
     sys.exit(pytest.main([__file__]))

src/convert/from_h5mu_to_h5ad/config.vsh.yaml

@@ -46,9 +46,7 @@ platforms:
           - procps
       - type: python
         __merge__: [/src/base/requirements/anndata_mudata.yaml]
-    test_setup:
-      - type: python
-        __merge__: [ /src/base/requirements/viashpy.yaml, .]
+    __merge__: [ /src/base/requirements/python_test_setup.yaml, . ]
   - type: nextflow
     directives:
       label: [lowmem, singlecpu]