Backport PR #852 to 0.12.x (#854)

openpipelines-bio · Aug 9, 2024 · f0e836d · f0e836d
1 parent 196a69d
commit f0e836d
Show file tree

Hide file tree

Showing 513 changed files with 10,320 additions and 2,275 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,9 @@
+# openpipelines 0.12.7
+
+## BUG FIXES
+
+* `qc/calculate_qc_metrics`: increase total counts accuracy with low precision floating dtypes as input layer (PR # , backported from PR #852).
+
 # openpipelines 0.12.6
 
 ## BUG FIXES

diff --git a/src/qc/calculate_qc_metrics/script.py b/src/qc/calculate_qc_metrics/script.py
@@ -49,7 +49,15 @@ def main():
     num_nonzero_obs = layer.getnnz(axis=0)
     obs_mean, _  = mean_variance_axis(layer, axis=0)
     pct_dropout = (1 - num_nonzero_obs / layer.shape[0]) * 100
-    total_counts_obs = np.ravel(layer.sum(axis=0))
+    # from the np.sum documentation: 
+    # Especially when summing a large number of lower precision floating point numbers,
+    # such as float32, numerical errors can become significant. In such cases it can
+    # be advisable to use dtype="float64" to use a higher precision for the output.
+    layer_with_type = layer
+    if np.issubdtype(layer.dtype, np.floating) and np.can_cast(layer.dtype, np.float64, casting="safe"):
+        # 'safe' casting makes sure not to cast np.float128 or anything else to a lower precision dtype
+        layer_with_type = layer.astype(np.float64)
+    total_counts_obs = np.ravel(layer_with_type.sum(axis=0))
 
     # obs statistics
     num_nonzero_vars = layer.getnnz(axis=1)

diff --git a/src/qc/calculate_qc_metrics/test.py b/src/qc/calculate_qc_metrics/test.py
@@ -2,13 +2,15 @@
 import pytest
 from pathlib import Path
 import mudata as md
+import anndata as ad
 import numpy as np
 import scanpy as sc
 import pandas as pd
 import scipy
 import uuid
 from pandas.testing import assert_series_equal
 from itertools import product
+from uuid import uuid4
 
 
 ## VIASH START
@@ -20,6 +22,28 @@
 }
 ## VIASH END
 
+@pytest.fixture
+def random_h5mu_path(tmp_path):
+    def wrapper():
+        return tmp_path / f"{uuid4()}.h5mu"
+    return wrapper
+
+
+@pytest.fixture
+def input_mudata_random():
+    rng = np.random.default_rng(seed=1)
+    random_counts = scipy.sparse.random(50000, 100,
+                                        density=0.8,
+                                        format='csr',
+                                        dtype=np.uint32,
+                                        random_state=rng)
+    good_dtype=random_counts.astype(np.float32)
+    del random_counts 
+    mod1 = ad.AnnData(X=good_dtype, 
+                      obs=pd.DataFrame(index=pd.RangeIndex(50000)),
+                      var=pd.DataFrame(index=pd.RangeIndex(100)))
+    return md.MuData({"mod1": mod1})
+
 
 @pytest.fixture
 def input_path():
@@ -155,7 +179,8 @@ def test_compare_scanpy(run_component,
     for from_var, to_var in vars_to_compare.items():
         assert_series_equal(component_var[from_var],
                             scanpy_var[to_var],
-                            check_names=False)
+                            check_names=False,
+                            check_dtype=False)
 
 
     scanpy_obs =  input_mudata.mod['rna'].obs
@@ -171,7 +196,30 @@ def test_compare_scanpy(run_component,
     for from_obs, to_obs in obs_to_compare.items():
         assert_series_equal(component_obs[from_obs],
                             scanpy_obs[to_obs],
-                            check_names=False)
+                            check_names=False,
+                            check_dtype=False)
+
+
+def test_total_counts_less_precision_dtype(run_component, input_mudata_random, random_h5mu_path):
+    input_path = random_h5mu_path()
+    input_mudata_random.write(input_path) 
+    output_path = random_h5mu_path()
+    run_component([
+        "--input", input_path,
+        "--output", output_path,
+        "--modality", "mod1",
+    ])
+    output_data = md.read_h5mu(output_path)
+    matrix_good_type = input_mudata_random.mod['mod1'].X
+    var_names = input_mudata_random.var_names
+    obs_names = input_mudata_random.obs_names
+    del input_mudata_random
+    input_df = pd.DataFrame(matrix_good_type.todense(),
+                            columns=var_names,
+                            index=obs_names)
+    total_sums_manual = input_df.to_numpy().sum(axis=0, dtype=np.float128)
+    total_counts = output_data.mod['mod1'].var['total_counts']
+    np.testing.assert_allclose(total_sums_manual, total_counts.to_numpy())
 
 
 if __name__ == "__main__":

diff --git a/target/docker/annotate/popv/.config.vsh.yaml b/target/docker/annotate/popv/.config.vsh.yaml
@@ -1,7 +1,7 @@
 functionality:
   name: "popv"
   namespace: "annotate"
-  version: "0.12.6"
+  version: "0.12.7"
   authors:
   - name: "Matthias Beyens"
     roles:
@@ -341,6 +341,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/annotate/popv"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/annotate/popv/popv"
   viash_version: "0.7.5"
-  git_commit: "8bf9c29a2c1223c2e4139d0372743537798fd3ea"
+  git_commit: "f6f4912a84a12ea49353aeb070947bc4ec73b41c"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "0.12.5-3-g8bf9c29a2c"
+  git_tag: "0.12.6-2-gf6f4912a84"
diff --git a/target/docker/annotate/popv/popv b/target/docker/annotate/popv/popv
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# popv 0.12.6
+# popv 0.12.7
 # 
 # This wrapper script is auto-generated by viash 0.7.5 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -159,7 +159,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "popv 0.12.6"
+  echo "popv 0.12.7"
   echo ""
   echo "Performs popular major vote cell typing on single cell sequence data using"
   echo "multiple algorithms. Note that this is a one-shot version of PopV."
@@ -488,10 +488,10 @@ RUN cd /opt && git clone --depth 1 https://github.com/YosefLab/PopV.git && \
 
 LABEL org.opencontainers.image.authors="Matthias Beyens, Robrecht Cannoodt"
 LABEL org.opencontainers.image.description="Companion container for running component annotate popv"
-LABEL org.opencontainers.image.created="2024-02-03T20:49:49Z"
+LABEL org.opencontainers.image.created="2024-08-09T13:18:14Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="8bf9c29a2c1223c2e4139d0372743537798fd3ea"
-LABEL org.opencontainers.image.version="0.12.6"
+LABEL org.opencontainers.image.revision="f6f4912a84a12ea49353aeb070947bc4ec73b41c"
+LABEL org.opencontainers.image.version="0.12.7"
 
 VIASHDOCKER
 }
@@ -642,7 +642,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "popv 0.12.6"
+            echo "popv 0.12.7"
             exit
             ;;
         --input)

diff --git a/target/docker/cluster/leiden/.config.vsh.yaml b/target/docker/cluster/leiden/.config.vsh.yaml
@@ -1,7 +1,7 @@
 functionality:
   name: "leiden"
   namespace: "cluster"
-  version: "0.12.6"
+  version: "0.12.7"
   authors:
   - name: "Dries De Maeyer"
     roles:
@@ -214,6 +214,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/cluster/leiden"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/cluster/leiden/leiden"
   viash_version: "0.7.5"
-  git_commit: "8bf9c29a2c1223c2e4139d0372743537798fd3ea"
+  git_commit: "f6f4912a84a12ea49353aeb070947bc4ec73b41c"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "0.12.5-3-g8bf9c29a2c"
+  git_tag: "0.12.6-2-gf6f4912a84"
diff --git a/target/docker/cluster/leiden/leiden b/target/docker/cluster/leiden/leiden
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# leiden 0.12.6
+# leiden 0.12.7
 # 
 # This wrapper script is auto-generated by viash 0.7.5 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -158,7 +158,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "leiden 0.12.6"
+  echo "leiden 0.12.7"
   echo ""
   echo "Cluster cells using the Leiden algorithm [Traag18] implemented in the Scanpy"
   echo "framework [Wolf18]."
@@ -445,10 +445,10 @@ RUN pip install --upgrade pip && \
 
 LABEL org.opencontainers.image.authors="Dries De Maeyer"
 LABEL org.opencontainers.image.description="Companion container for running component cluster leiden"
-LABEL org.opencontainers.image.created="2024-02-03T20:49:48Z"
+LABEL org.opencontainers.image.created="2024-08-09T13:18:17Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="8bf9c29a2c1223c2e4139d0372743537798fd3ea"
-LABEL org.opencontainers.image.version="0.12.6"
+LABEL org.opencontainers.image.revision="f6f4912a84a12ea49353aeb070947bc4ec73b41c"
+LABEL org.opencontainers.image.version="0.12.7"
 
 VIASHDOCKER
 }
@@ -599,7 +599,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "leiden 0.12.6"
+            echo "leiden 0.12.7"
             exit
             ;;
         --input)

diff --git a/target/docker/compression/compress_h5mu/.config.vsh.yaml b/target/docker/compression/compress_h5mu/.config.vsh.yaml
@@ -1,7 +1,7 @@
 functionality:
   name: "compress_h5mu"
   namespace: "compression"
-  version: "0.12.6"
+  version: "0.12.7"
   authors:
   - name: "Dries Schaumont"
     roles:
@@ -162,6 +162,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/compression/compress_h5mu"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/compression/compress_h5mu/compress_h5mu"
   viash_version: "0.7.5"
-  git_commit: "8bf9c29a2c1223c2e4139d0372743537798fd3ea"
+  git_commit: "f6f4912a84a12ea49353aeb070947bc4ec73b41c"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "0.12.5-3-g8bf9c29a2c"
+  git_tag: "0.12.6-2-gf6f4912a84"
diff --git a/target/docker/compression/compress_h5mu/compress_h5mu b/target/docker/compression/compress_h5mu/compress_h5mu
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# compress_h5mu 0.12.6
+# compress_h5mu 0.12.7
 # 
 # This wrapper script is auto-generated by viash 0.7.5 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -158,7 +158,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "compress_h5mu 0.12.6"
+  echo "compress_h5mu 0.12.7"
   echo ""
   echo "Compress a MuData file."
   echo ""
@@ -408,10 +408,10 @@ RUN pip install --upgrade pip && \
 
 LABEL org.opencontainers.image.authors="Dries Schaumont"
 LABEL org.opencontainers.image.description="Companion container for running component compression compress_h5mu"
-LABEL org.opencontainers.image.created="2024-02-03T20:49:49Z"
+LABEL org.opencontainers.image.created="2024-08-09T13:18:14Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="8bf9c29a2c1223c2e4139d0372743537798fd3ea"
-LABEL org.opencontainers.image.version="0.12.6"
+LABEL org.opencontainers.image.revision="f6f4912a84a12ea49353aeb070947bc4ec73b41c"
+LABEL org.opencontainers.image.version="0.12.7"
 
 VIASHDOCKER
 }
@@ -562,7 +562,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "compress_h5mu 0.12.6"
+            echo "compress_h5mu 0.12.7"
             exit
             ;;
         --input)

diff --git a/target/docker/compression/tar_extract/.config.vsh.yaml b/target/docker/compression/tar_extract/.config.vsh.yaml
@@ -1,7 +1,7 @@
 functionality:
   name: "tar_extract"
   namespace: "compression"
-  version: "0.12.6"
+  version: "0.12.7"
   arguments:
   - type: "file"
     name: "--input"
@@ -101,6 +101,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/compression/tar_extract"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/compression/tar_extract/tar_extract"
   viash_version: "0.7.5"
-  git_commit: "8bf9c29a2c1223c2e4139d0372743537798fd3ea"
+  git_commit: "f6f4912a84a12ea49353aeb070947bc4ec73b41c"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "0.12.5-3-g8bf9c29a2c"
+  git_tag: "0.12.6-2-gf6f4912a84"
diff --git a/target/docker/compression/tar_extract/tar_extract b/target/docker/compression/tar_extract/tar_extract
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# tar_extract 0.12.6
+# tar_extract 0.12.7
 # 
 # This wrapper script is auto-generated by viash 0.7.5 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
@@ -155,7 +155,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 # ViashHelp: Display helpful explanation about this executable
 function ViashHelp {
-  echo "tar_extract 0.12.6"
+  echo "tar_extract 0.12.7"
   echo ""
   echo "Extract files from a tar archive"
   echo ""
@@ -406,10 +406,10 @@ ENTRYPOINT []
  
 RUN :
 LABEL org.opencontainers.image.description="Companion container for running component compression tar_extract"
-LABEL org.opencontainers.image.created="2024-02-03T20:49:49Z"
+LABEL org.opencontainers.image.created="2024-08-09T13:18:15Z"
 LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline"
-LABEL org.opencontainers.image.revision="8bf9c29a2c1223c2e4139d0372743537798fd3ea"
-LABEL org.opencontainers.image.version="0.12.6"
+LABEL org.opencontainers.image.revision="f6f4912a84a12ea49353aeb070947bc4ec73b41c"
+LABEL org.opencontainers.image.version="0.12.7"
 
 VIASHDOCKER
 }
@@ -560,7 +560,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "tar_extract 0.12.6"
+            echo "tar_extract 0.12.7"
             exit
             ;;
         --input)

diff --git a/target/docker/convert/from_10xh5_to_h5mu/.config.vsh.yaml b/target/docker/convert/from_10xh5_to_h5mu/.config.vsh.yaml
@@ -1,7 +1,7 @@
 functionality:
   name: "from_10xh5_to_h5mu"
   namespace: "convert"
-  version: "0.12.6"
+  version: "0.12.7"
   authors:
   - name: "Robrecht Cannoodt"
     roles:
@@ -267,6 +267,6 @@ info:
   output: "/home/runner/work/openpipeline/openpipeline/target/docker/convert/from_10xh5_to_h5mu"
   executable: "/home/runner/work/openpipeline/openpipeline/target/docker/convert/from_10xh5_to_h5mu/from_10xh5_to_h5mu"
   viash_version: "0.7.5"
-  git_commit: "8bf9c29a2c1223c2e4139d0372743537798fd3ea"
+  git_commit: "f6f4912a84a12ea49353aeb070947bc4ec73b41c"
   git_remote: "https://github.com/openpipelines-bio/openpipeline"
-  git_tag: "0.12.5-3-g8bf9c29a2c"
+  git_tag: "0.12.6-2-gf6f4912a84"