Merging staging branch into prod branch

chanzuckerberg · Oct 3, 2024 · cb38f31 · cb38f31
2 parents 3da9a8d + bf9d9f3
commit cb38f31
Show file tree

Hide file tree

Showing 9 changed files with 334 additions and 169 deletions.
diff --git a/.github/workflows/build-images-and-create-deployment.yml b/.github/workflows/build-images-and-create-deployment.yml
@@ -45,7 +45,7 @@ jobs:
         uses: avakar/create-deployment@v1
         # To stop deployment to a specific DEPLOYMENT_STAGE remove it from condition below.
         # The DEPLOYMENT_STAGE that should be present are dev, stage, prod.
-        if: env.DEPLOYMENT_STAGE == 'prod' || env.DEPLOYMENT_STAGE == 'stage' || env.DEPLOYMENT_STAGE == 'dev'
+        if: env.DEPLOYMENT_STAGE == 'prod' || env.DEPLOYMENT_STAGE == 'stage'
         with:
           auto_merge: false
           environment: ${{ env.DEPLOYMENT_STAGE }}

diff --git a/backend/layers/processing/process_validate.py b/backend/layers/processing/process_validate.py
@@ -147,7 +147,7 @@ def extract_metadata(self, filename) -> DatasetMetadata:
             range(stride, layer_for_mean_genes_per_cell.shape[0] + stride, stride),
             strict=False,
         ):
-            chunk = layer_for_mean_genes_per_cell[bounds[0] : bounds[1], filter_gene_vars]
+            chunk = layer_for_mean_genes_per_cell[bounds[0] : bounds[1], :][:, filter_gene_vars]
             numerator += chunk.nnz if hasattr(chunk, "nnz") else numpy.count_nonzero(chunk)
             denominator += chunk.shape[0]
 

diff --git a/backend/layers/processing/schema_migration.py b/backend/layers/processing/schema_migration.py
@@ -15,7 +15,11 @@
     CollectionVersion,
     CollectionVersionId,
     DatasetArtifactType,
+    DatasetConversionStatus,
     DatasetProcessingStatus,
+    DatasetStatusKey,
+    DatasetUploadStatus,
+    DatasetValidationStatus,
     DatasetVersionId,
 )
 from backend.layers.processing import logger
@@ -228,6 +232,25 @@ def log_errors_and_cleanup(self, collection_version_id: str) -> list:
             key_prefix = self.get_key_prefix(previous_dataset_version_id)
             object_keys_to_delete.append(f"{key_prefix}/migrated.h5ad")
             if dataset.status.processing_status != DatasetProcessingStatus.SUCCESS:
+                # If only rds failure, set rds status to skipped + processing status to successful and do not rollback
+                if (
+                    dataset.status.rds_status == DatasetConversionStatus.FAILED
+                    and dataset.status.upload_status == DatasetUploadStatus.UPLOADED
+                    and dataset.status.validation_status == DatasetValidationStatus.VALID
+                    and dataset.status.cxg_status == DatasetConversionStatus.UPLOADED
+                    and dataset.status.h5ad_status == DatasetConversionStatus.UPLOADED
+                ):
+                    self.business_logic.update_dataset_version_status(
+                        dataset.version_id,
+                        DatasetStatusKey.RDS,
+                        DatasetConversionStatus.SKIPPED,
+                    )
+                    self.business_logic.update_dataset_version_status(
+                        dataset.version_id,
+                        DatasetStatusKey.PROCESSING,
+                        DatasetProcessingStatus.SUCCESS,
+                    )
+                    continue
                 error = {
                     "message": dataset.status.validation_message,
                     "dataset_status": dataset.status.to_dict(),

diff --git a/frontend/src/components/common/Filter/common/constants.ts b/frontend/src/components/common/Filter/common/constants.ts
@@ -26,78 +26,56 @@ export const DEVELOPMENT_STAGE_ONTOLOGY_TERM_SET: OntologyTermSet = {
         {
           label: "Embryonic human (0–56 days)",
           ontology_term_id: "HsapDv:0000002",
+        },
+        {
+          label: "Fetal (>56 days–birth)",
+          ontology_term_id: "HsapDv:0000037",
+        },
+      ],
+    },
+    {
+      label: "Postnatal human",
+      ontology_term_id: "HsapDv:0010000",
+      children: [
+        {
+          label: "Pediatric (0-15 years)",
+          ontology_term_id: "HsapDv:0000264",
           children: [
             {
-              label: "Carnegie (CS1)",
-              ontology_term_id: "HsapDv:0000003",
-            },
-            {
-              label: "Cleavage (CS2)",
-              ontology_term_id: "HsapDv:0000004",
-            },
-            {
-              label: "Blastula (CS3–5)",
-              ontology_term_id: "HsapDv:0000006",
+              label: "Newborn human (0-28 days)",
+              ontology_term_id: "HsapDv:0000262",
             },
             {
-              label: "Gastrula (CS6)",
-              ontology_term_id: "HsapDv:0000010",
+              label: "Infant (28 days-1 year)",
+              ontology_term_id: "HsapDv:0000261",
             },
             {
-              label: "Neurula (CS7–8)",
-              ontology_term_id: "HsapDv:0000012",
+              label: "Child (1-5 years)",
+              ontology_term_id: "HsapDv:0000265",
             },
             {
-              label: "Organogenesis (CS9–23)",
-              ontology_term_id: "HsapDv:0000015",
+              label: "Juvenile (5-15 years)",
+              ontology_term_id: "HsapDv:0000271",
             },
           ],
         },
-        {
-          label: "Fetal (>56 days–birth)",
-          ontology_term_id: "HsapDv:0000037",
-        },
       ],
     },
     {
-      label: "Immature (0–12 years)",
-      ontology_term_id: "HsapDv:0000080",
+      label: "Adult (15+ years)",
+      ontology_term_id: "HsapDv:0000258",
       children: [
         {
-          label: "Newborn human (0–1 month)",
-          ontology_term_id: "HsapDv:0000082",
-        },
-        {
-          label: "Infant (1–23 months)",
-          ontology_term_id: "HsapDv:0000083",
+          label: "Young adult (15-40 years)",
+          ontology_term_id: "HsapDv:0000266",
         },
         {
-          label: "Child (2–12 years)",
-          ontology_term_id: "HsapDv:0000081",
-        },
-      ],
-    },
-    {
-      label: "Mature (13+ years)",
-      ontology_term_id: "HsapDv:0000204",
-      children: [
-        {
-          label: "Adolescent (13–19 years)",
-          ontology_term_id: "HsapDv:0000086",
+          label: "Middle aged (40-60 years)",
+          ontology_term_id: "HsapDv:0000267",
         },
         {
-          label: "Human adult (19+ years)",
-          ontology_term_id: "HsapDv:0000087",
-          children: [
-            {
-              label: "Early adulthood (19–45 years)",
-              ontology_term_id: "HsapDv:0000088",
-            },
-            {
-              label: "Late adulthood (45+ years)",
-              ontology_term_id: "HsapDv:0000091",
-            },
-          ],
+          label: "Late adult (60+ years)",
+          ontology_term_id: "HsapDv:0000227",
         },
       ],
     },
@@ -110,32 +88,6 @@ export const DEVELOPMENT_STAGE_ONTOLOGY_TERM_SET: OntologyTermSet = {
         {
           label: "Embryonic mouse",
           ontology_term_id: "MmusDv:0000002",
-          children: [
-            {
-              label: "Thelier stage 1 (TS1)",
-              ontology_term_id: "MmusDv:0000003",
-            },
-            {
-              label: "Cleavage (TS2–3)",
-              ontology_term_id: "MmusDv:0000004",
-            },
-            {
-              label: "Blastula (TS4–8)",
-              ontology_term_id: "MmusDv:0000007",
-            },
-            {
-              label: "Gastrula (TS9–10)",
-              ontology_term_id: "MmusDv:0000013",
-            },
-            {
-              label: "Thelier stage 11 (TS11)",
-              ontology_term_id: "MmusDv:0000017",
-            },
-            {
-              label: "Organogenesis (TS11–22)",
-              ontology_term_id: "MmusDv:0000018",
-            },
-          ],
         },
         {
           label: "Fetal (TS23–26)",
@@ -144,34 +96,34 @@ export const DEVELOPMENT_STAGE_ONTOLOGY_TERM_SET: OntologyTermSet = {
       ],
     },
     {
-      label: "Post-partum (Birth+)",
+      label: "Postnatal mouse",
       ontology_term_id: "MmusDv:0000092",
       children: [
         {
-          label: "Immature (0–6 weeks)",
+          label: "Immature (0-6 weeks)",
           ontology_term_id: "MmusDv:0000043",
-          children: [
-            {
-              label: "Thelier stage 27 (0–3 days)",
-              ontology_term_id: "MmusDv:0000036",
-            },
-            {
-              label: "Premature (3 days–6 weeks)",
-              ontology_term_id: "MmusDv:0000112",
-            },
-          ],
         },
         {
           label: "Mature (6+ weeks)",
           ontology_term_id: "MmusDv:0000110",
           children: [
             {
-              label: "Early adulthood (6 weeks–7 months)",
-              ontology_term_id: "MmusDv:0000061",
+              label: "Prime adult (6 weeks-18 months)",
+              ontology_term_id: "MmusDv:0000136",
+              children: [
+                {
+                  label: "Young adult (6 weeks-10 months)",
+                  ontology_term_id: "MmusDv:0000153",
+                },
+                {
+                  label: "Middle aged (10-18 months)",
+                  ontology_term_id: "MmusDv:0000135",
+                },
+              ],
             },
             {
-              label: "Late adulthood (7+ months)",
-              ontology_term_id: "MmusDv:0000097",
+              label: "Late adult (18+ months)",
+              ontology_term_id: "MmusDv:0000134",
             },
           ],
         },