Merging staging branch into prod branch

.github/dependabot.yml

@@ -1,114 +1,114 @@
-# version: 2
-# updates:
-#   - package-ecosystem: "github-actions"
-#     directory: "/"
-#     open-pull-requests-limit: 3
-#     schedule:
-#       interval: "weekly"
-#       day: "sunday"
-#     assignees:
-#       - "seve"
-#   - package-ecosystem: pip
-#     directory: "/python_dependencies/backend/"
-#     open-pull-requests-limit: 3
-#     schedule:
-#       interval: "daily"
-#     assignees:
-#       - "Bento007"
-#     groups:
-#       database:
-#         patterns:
-#           - "alembic"
-#           - "sqlalchemy"
-#           - "psycopg2"
-#       server:
-#         patterns:
-#           - "flask"
-#           - "gunicorn"
-#           - "setproctitle"
-#           - "connexion"
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    assignees:
+      - "seve"
+  - package-ecosystem: pip
+    directory: "/python_dependencies/backend/"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "daily"
+    assignees:
+      - "Bento007"
+    groups:
+      database:
+        patterns:
+          - "alembic"
+          - "sqlalchemy"
+          - "psycopg2"
+      server:
+        patterns:
+          - "flask"
+          - "gunicorn"
+          - "setproctitle"
+          - "connexion"
 
-#   - package-ecosystem: pip
-#     directory: "/python_dependencies/wmg/"
-#     open-pull-requests-limit: 3
-#     schedule:
-#       interval: "weekly"
-#       day: "sunday"
-#     assignees:
-#       - "prathapsridharan"
-#   - package-ecosystem: pip
-#     directory: "/python_dependencies/upload_handler/"
-#     open-pull-requests-limit: 3
-#     schedule:
-#       interval: "weekly"
-#       day: "sunday"
-#     assignees:
-#       - "ebezzi"
-#   - package-ecosystem: pip
-#     directory: "/python_dependencies/submissions/"
-#     open-pull-requests-limit: 3
-#     schedule:
-#       interval: "weekly"
-#       day: "sunday"
-#     assignees:
-#       - "nayib-jose-gloria"
-#   - package-ecosystem: pip
-#     directory: "/python_dependencies/cellguide_pipeline/"
-#     open-pull-requests-limit: 3
-#     schedule:
-#       interval: "weekly"
-#       day: "sunday"
-#     assignees:
-#       - "atarashansky"
-#   - package-ecosystem: pip
-#     directory: "/python_dependencies/processing/"
-#     open-pull-requests-limit: 3
-#     schedule:
-#       interval: "weekly"
-#       day: "sunday"
-#     assignees:
-#       - "Bento007"
-#     groups:
-#       database:
-#         patterns:
-#           - "sqlalchemy*"
-#           - "psycopg2*"
-#       non-major-updates:
-#         patterns:
-#           - "*"
-#         applies-to: version-updates
-#         update-types:
-#           - patch
-#           - minor
-#         exclude-patterns:
-#           # The following packages do not have major versions. So a minor version updates may contain breaking changes.
-#           - "anndata"
-#           - "tiledb"
-#           - "numba"
-#           - "s3fs"
-#   - package-ecosystem: pip
-#     directory: "/python_dependencies/common/"
-#     open-pull-requests-limit: 3
-#     schedule:
-#       interval: "weekly"
-#       day: "sunday"
-#     assignees:
-#       - "nayib-jose-gloria"
-#     groups:
-#       formatters:
-#         patterns:
-#           - "black"
-#           - "ruff"
-#       test-runners:
-#         patterns:
-#           - "pytest"
-#           - "coverage"
-#           - "allure"
-#   - package-ecosystem: npm
-#     directory: "/frontend/"
-#     open-pull-requests-limit: 3
-#     schedule:
-#       interval: "weekly"
-#       day: "sunday"
-#     assignees:
-#       - "tihuan"
+  - package-ecosystem: pip
+    directory: "/python_dependencies/wmg/"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    assignees:
+      - "prathapsridharan"
+  - package-ecosystem: pip
+    directory: "/python_dependencies/upload_handler/"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    assignees:
+      - "ebezzi"
+  - package-ecosystem: pip
+    directory: "/python_dependencies/submissions/"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    assignees:
+      - "nayib-jose-gloria"
+  - package-ecosystem: pip
+    directory: "/python_dependencies/cellguide_pipeline/"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    assignees:
+      - "atarashansky"
+  - package-ecosystem: pip
+    directory: "/python_dependencies/processing/"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    assignees:
+      - "Bento007"
+    groups:
+      database:
+        patterns:
+          - "sqlalchemy*"
+          - "psycopg2*"
+      non-major-updates:
+        patterns:
+          - "*"
+        applies-to: version-updates
+        update-types:
+          - patch
+          - minor
+        exclude-patterns:
+          # The following packages do not have major versions. So a minor version updates may contain breaking changes.
+          - "anndata"
+          - "tiledb"
+          - "numba"
+          - "s3fs"
+  - package-ecosystem: pip
+    directory: "/python_dependencies/common/"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    assignees:
+      - "nayib-jose-gloria"
+    groups:
+      formatters:
+        patterns:
+          - "black"
+          - "ruff"
+      test-runners:
+        patterns:
+          - "pytest"
+          - "coverage"
+          - "allure"
+  - package-ecosystem: npm
+    directory: "/frontend/"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    assignees:
+      - "tihuan"

Makefile

@@ -38,7 +38,7 @@ cellguide-pipeline-unittest:
 
 .PHONY: functional-test
 functional-test:
-	python3 -m pytest tests/functional/ --rootdir=. --verbose
+	python3 -m pytest tests/functional/ --rootdir=. --verbose -n auto
 
 .PHONY: prod-performance-test
 prod-performance-test:

backend/common/census_cube/data/query.py

@@ -97,11 +97,9 @@ def cell_counts_diffexp_df(self, criteria: BaseQueryCriteria) -> DataFrame:
     @tracer.wrap(
         name="expression_summary_and_cell_counts_diffexp", service="de-api", resource="_query", span_type="de-api"
     )
-    def expression_summary_and_cell_counts_diffexp(self, criteria: BaseQueryCriteria) -> tuple[DataFrame, DataFrame]:
-        use_simple = not any(
-            depluralize(key) not in cell_counts_indexed_dims and values for key, values in dict(criteria).items()
-        )
-
+    def expression_summary_and_cell_counts_diffexp(
+        self, criteria: BaseQueryCriteria, use_simple: bool
+    ) -> tuple[DataFrame, DataFrame]:
         cell_counts_diffexp_df = self.cell_counts_diffexp_df(criteria)
         cell_counts_group_id_key = "group_id_simple" if use_simple else "group_id"
         cube = (
@@ -209,6 +207,12 @@ def list_grouped_primary_filter_dimensions_term_ids(
         )
 
 
+def should_use_simple_group_ids(criteria: BaseQueryCriteria):
+    return not any(
+        depluralize(key) not in cell_counts_indexed_dims and values for key, values in dict(criteria).items()
+    )
+
+
 def depluralize(attr_name):
     return attr_name[:-1] if attr_name[-1] == "s" else attr_name
 

backend/de/api/v1.py

@@ -10,7 +10,7 @@
 
 from backend.common.census_cube.data.criteria import BaseQueryCriteria
 from backend.common.census_cube.data.ontology_labels import gene_term_label, ontology_term_label
-from backend.common.census_cube.data.query import CensusCubeQuery
+from backend.common.census_cube.data.query import CensusCubeQuery, should_use_simple_group_ids
 from backend.common.census_cube.data.schemas.cube_schema_diffexp import cell_counts_logical_dims_exclude_dataset_id
 from backend.common.census_cube.data.snapshot import CensusCubeSnapshot, load_snapshot
 from backend.common.census_cube.utils import ancestors, descendants
@@ -248,8 +248,19 @@ def run_differential_expression(
             set(sum([descendants(i) for i in criteria2.cell_type_ontology_term_ids], []))
         )
 
-    es1, cell_counts1 = q.expression_summary_and_cell_counts_diffexp(criteria1)
-    es2, cell_counts2 = q.expression_summary_and_cell_counts_diffexp(criteria2)
+    if exclude_overlapping_cells == "retainBoth":
+        # If we are not excluding overlapping cells (retainBoth), we can use the simple group IDs where applicable.
+        es1, cell_counts1 = q.expression_summary_and_cell_counts_diffexp(
+            criteria1, should_use_simple_group_ids(criteria1)
+        )
+        es2, cell_counts2 = q.expression_summary_and_cell_counts_diffexp(
+            criteria2, should_use_simple_group_ids(criteria2)
+        )
+    else:
+        # If we are excluding overlapping cells, we can only use the simple group IDs if both groups are eligible.
+        use_simple_group_ids = should_use_simple_group_ids(criteria1) and should_use_simple_group_ids(criteria2)
+        es1, cell_counts1 = q.expression_summary_and_cell_counts_diffexp(criteria1, use_simple_group_ids)
+        es2, cell_counts2 = q.expression_summary_and_cell_counts_diffexp(criteria2, use_simple_group_ids)
 
     n_cells1 = cell_counts1["n_total_cells"].sum()
     n_cells2 = cell_counts2["n_total_cells"].sum()

backend/layers/processing/h5ad_data_file.py

@@ -7,21 +7,21 @@
 import numpy as np
 import tiledb
 
-from backend.common.utils.color_conversion_utils import (
-    ColorFormatException,
-    convert_anndata_category_colors_to_cxg_category_colors,
-)
 from backend.common.utils.corpora_constants import CorporaConstants
 from backend.common.utils.cxg_constants import CxgConstants
-from backend.common.utils.matrix_utils import is_matrix_sparse
 from backend.common.utils.tiledb import consolidation_buffer_size
+from backend.layers.processing.utils.color_conversion_utils import (
+    ColorFormatException,
+    convert_anndata_category_colors_to_cxg_category_colors,
+)
 from backend.layers.processing.utils.cxg_generation_utils import (
     convert_dataframe_to_cxg_array,
     convert_dictionary_to_cxg_group,
     convert_matrices_to_cxg_arrays,
     convert_ndarray_to_cxg_dense_array,
     convert_uns_to_cxg_group,
 )
+from backend.layers.processing.utils.matrix_utils import is_matrix_sparse
 
 
 class H5ADDataFile:

backend/layers/processing/utils/cxg_generation_utils.py

@@ -7,8 +7,8 @@
 import tiledb
 
 from backend.common.constants import SPATIAL_KEYS_EXCLUDE, UNS_SPATIAL_KEY
-from backend.common.utils.type_conversion_utils import get_dtype_and_schema_of_array
 from backend.layers.processing.utils.spatial import SpatialDataProcessor
+from backend.layers.processing.utils.type_conversion_utils import get_dtype_and_schema_of_array
 
 
 def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata", ctx=None):

frontend/src/views/CellGuide/components/CellGuideCard/components/common/HelpTooltip/index.tsx

@@ -35,9 +35,10 @@ interface Props {
       element: JSX.Element;
     } | null>
   >;
-  title: string;
+  title?: string;
   skinnyMode?: boolean;
   extraContent?: JSX.Element;
+  width?: "wide" | "default";
 }
 const HelpTooltip = ({
   text,
@@ -48,12 +49,13 @@ const HelpTooltip = ({
   title,
   skinnyMode = false,
   extraContent,
+  width = "wide",
 }: Props) => {
   return (
     <Tooltip
       sdsStyle={dark ? "dark" : "light"}
       placement={placement}
-      width="wide"
+      width={width}
       arrow
       title={!skinnyMode && <StyledTooltip>{text}</StyledTooltip>}
       slotProps={getSlotProps(dark)}
@@ -67,7 +69,7 @@ const HelpTooltip = ({
         <ExtraContentWrapper>
           <StyledQuestionMarkIcon
             onClick={() => {
-              if (skinnyMode && setTooltipContent) {
+              if (skinnyMode && setTooltipContent && title) {
                 setTooltipContent({
                   title: title,
                   element: <StyledTooltip>{text}</StyledTooltip>,