Merging staging branch into prod branch

.github/dependabot.yml

@@ -1,101 +1,114 @@
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    open-pull-requests-limit: 3
-    schedule:
-      interval: "weekly"
-      day: "sunday"
-    assignees:
-      - "seve"
-  - package-ecosystem: pip
-    directory: "/python_dependencies/backend/"
-    open-pull-requests-limit: 3
-    schedule:
-      interval: "daily"
-    assignees:
-      - "Bento007"
-  - package-ecosystem: pip
-    directory: "/python_dependencies/wmg/"
-    open-pull-requests-limit: 3
-    schedule:
-      interval: "weekly"
-      day: "sunday"
-    assignees:
-      - "prathapsridharan"
-  - package-ecosystem: pip
-    directory: "/python_dependencies/upload_handler/"
-    open-pull-requests-limit: 3
-    schedule:
-      interval: "weekly"
-      day: "sunday"
-    assignees:
-      - "ebezzi"
-  - package-ecosystem: pip
-    directory: "/python_dependencies/submissions/"
-    open-pull-requests-limit: 3
-    schedule:
-      interval: "weekly"
-      day: "sunday"
-    assignees:
-      - "nayib-jose-gloria"
-  - package-ecosystem: pip
-    directory: "/python_dependencies/cellguide_pipeline/"
-    open-pull-requests-limit: 3
-    schedule:
-      interval: "weekly"
-      day: "sunday"
-    assignees:
-      - "atarashansky"
-  - package-ecosystem: pip
-    directory: "/python_dependencies/processing/"
-    open-pull-requests-limit: 3
-    schedule:
-      interval: "weekly"
-      day: "sunday"
-    assignees:
-      - "Bento007"
-    groups:
-      database:
-        patterns:
-          - "sqlalchemy*"
-          - "psycopg2*"
-      non-major-updates:
-        patterns:
-          - "*"
-        applies-to: version-updates
-        update-types:
-          - patch
-          - minor
-        exclude-patterns:
-          # The following packages do not have major versions. So a minor version updates may contain breaking changes.
-          - "anndata"
-          - "tiledb"
-          - "numba"
-          - "s3fs"
-  - package-ecosystem: pip
-    directory: "/python_dependencies/common/"
-    open-pull-requests-limit: 3
-    schedule:
-      interval: "weekly"
-      day: "sunday"
-    assignees:
-      - "nayib-jose-gloria"
-    groups:
-      formatters:
-        patterns:
-          - "black"
-          - "ruff"
-      test-runners:
-        patterns:
-          - "pytest"
-          - "coverage"
-          - "allure"
-  - package-ecosystem: npm
-    directory: "/frontend/"
-    open-pull-requests-limit: 3
-    schedule:
-      interval: "weekly"
-      day: "sunday"
-    assignees:
-      - "tihuan"
+# version: 2
+# updates:
+#   - package-ecosystem: "github-actions"
+#     directory: "/"
+#     open-pull-requests-limit: 3
+#     schedule:
+#       interval: "weekly"
+#       day: "sunday"
+#     assignees:
+#       - "seve"
+#   - package-ecosystem: pip
+#     directory: "/python_dependencies/backend/"
+#     open-pull-requests-limit: 3
+#     schedule:
+#       interval: "daily"
+#     assignees:
+#       - "Bento007"
+#     groups:
+#       database:
+#         patterns:
+#           - "alembic"
+#           - "sqlalchemy"
+#           - "psycopg2"
+#       server:
+#         patterns:
+#           - "flask"
+#           - "gunicorn"
+#           - "setproctitle"
+#           - "connexion"
+
+#   - package-ecosystem: pip
+#     directory: "/python_dependencies/wmg/"
+#     open-pull-requests-limit: 3
+#     schedule:
+#       interval: "weekly"
+#       day: "sunday"
+#     assignees:
+#       - "prathapsridharan"
+#   - package-ecosystem: pip
+#     directory: "/python_dependencies/upload_handler/"
+#     open-pull-requests-limit: 3
+#     schedule:
+#       interval: "weekly"
+#       day: "sunday"
+#     assignees:
+#       - "ebezzi"
+#   - package-ecosystem: pip
+#     directory: "/python_dependencies/submissions/"
+#     open-pull-requests-limit: 3
+#     schedule:
+#       interval: "weekly"
+#       day: "sunday"
+#     assignees:
+#       - "nayib-jose-gloria"
+#   - package-ecosystem: pip
+#     directory: "/python_dependencies/cellguide_pipeline/"
+#     open-pull-requests-limit: 3
+#     schedule:
+#       interval: "weekly"
+#       day: "sunday"
+#     assignees:
+#       - "atarashansky"
+#   - package-ecosystem: pip
+#     directory: "/python_dependencies/processing/"
+#     open-pull-requests-limit: 3
+#     schedule:
+#       interval: "weekly"
+#       day: "sunday"
+#     assignees:
+#       - "Bento007"
+#     groups:
+#       database:
+#         patterns:
+#           - "sqlalchemy*"
+#           - "psycopg2*"
+#       non-major-updates:
+#         patterns:
+#           - "*"
+#         applies-to: version-updates
+#         update-types:
+#           - patch
+#           - minor
+#         exclude-patterns:
+#           # The following packages do not have major versions. So a minor version updates may contain breaking changes.
+#           - "anndata"
+#           - "tiledb"
+#           - "numba"
+#           - "s3fs"
+#   - package-ecosystem: pip
+#     directory: "/python_dependencies/common/"
+#     open-pull-requests-limit: 3
+#     schedule:
+#       interval: "weekly"
+#       day: "sunday"
+#     assignees:
+#       - "nayib-jose-gloria"
+#     groups:
+#       formatters:
+#         patterns:
+#           - "black"
+#           - "ruff"
+#       test-runners:
+#         patterns:
+#           - "pytest"
+#           - "coverage"
+#           - "allure"
+#   - package-ecosystem: npm
+#     directory: "/frontend/"
+#     open-pull-requests-limit: 3
+#     schedule:
+#       interval: "weekly"
+#       day: "sunday"
+#     assignees:
+#       - "tihuan"

Dockerfile.backend

@@ -5,6 +5,9 @@ ENV EXPORT_ENV_VARS_TO_LAMBDA="APP_NAME DEPLOYMENT_STAGE"
 ENV LC_ALL=C.UTF-8
 ENV DEBIAN_FRONTEND=noninteractive
 
+# Update AWS root Certificates
+ADD https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem /etc/ssl/certs/rds-global-bundle.pem
+
 RUN apt-get update && \
     apt-get install -y python3 libhdf5-dev python3-h5py gettext moreutils build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp llvm jq && \
     rm -rf /var/lib/apt/lists/*

Dockerfile.backend_de

@@ -5,6 +5,9 @@ ENV EXPORT_ENV_VARS_TO_LAMBDA="APP_NAME DEPLOYMENT_STAGE"
 ENV LC_ALL=C.UTF-8
 ENV DEBIAN_FRONTEND=noninteractive
 
+# Update AWS root Certificates
+ADD https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem /etc/ssl/certs/rds-global-bundle.pem
+
 RUN apt-get update && \
     apt-get install -y python3 libhdf5-dev python3-h5py gettext moreutils build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp llvm jq && \
     rm -rf /var/lib/apt/lists/*

Dockerfile.backend_wmg

@@ -5,6 +5,9 @@ ENV EXPORT_ENV_VARS_TO_LAMBDA="APP_NAME DEPLOYMENT_STAGE"
 ENV LC_ALL=C.UTF-8
 ENV DEBIAN_FRONTEND=noninteractive
 
+# Update AWS root Certificates
+ADD https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem /etc/ssl/certs/rds-global-bundle.pem
+
 RUN apt-get update && \
     apt-get install -y python3 libhdf5-dev python3-h5py gettext moreutils build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp llvm jq && \
     rm -rf /var/lib/apt/lists/*

Dockerfile.cellguide_pipeline

@@ -1,6 +1,10 @@
 ARG BASE_TAG=branch-main
 
 FROM python:3.10
+
+# Update AWS root Certificates
+ADD https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem /etc/ssl/certs/rds-global-bundle.pem
+
 RUN /usr/local/bin/python -m pip install --upgrade pip && \
     apt update && apt -y install graphviz graphviz-dev && \
     rm -rf /var/lib/apt/lists/*

Dockerfile.dataset_submissions

@@ -1,5 +1,8 @@
 FROM public.ecr.aws/lambda/python:3.9
 
+# Update AWS root Certificates
+ADD https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem /etc/ssl/certs/rds-global-bundle.pem
+
 COPY backend/layers/processing/submissions .
 COPY backend/layers ./backend/layers
 

Dockerfile.processing

@@ -2,6 +2,9 @@ ARG BASE_TAG=branch-main
 
 FROM ghcr.io/chanzuckerberg/corpora-upload-base:$BASE_TAG
 
+# Update AWS root Certificates
+ADD https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem /etc/ssl/certs/rds-global-bundle.pem
+
 # Install cellxgene so we get the remote server that has the converter in it
 # The cellxgene install script expects executables named python and pip, not python3 and pip3
 RUN apt-get update && \

Dockerfile.upload_failures

@@ -1,5 +1,8 @@
 FROM public.ecr.aws/lambda/python:3.8
 
+# Update AWS root Certificates
+ADD https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem /etc/ssl/certs/rds-global-bundle.pem
+
 COPY backend/layers/processing/upload_failures .
 COPY backend/layers ./backend/layers
 COPY backend/portal ./backend/portal

Dockerfile.upload_success

@@ -1,5 +1,8 @@
 FROM public.ecr.aws/lambda/python:3.8
 
+# Update AWS root Certificates
+ADD https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem /etc/ssl/certs/rds-global-bundle.pem
+
 COPY backend/layers/processing/upload_failures backend/layers/processing/upload_failures
 COPY backend/layers/processing/upload_success .
 COPY backend/layers ./backend/layers

Dockerfile.wmg_pipeline

@@ -1,6 +1,10 @@
 ARG BASE_TAG=branch-main
 
 FROM python:3.10
+
+# Update AWS root Certificates
+ADD https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem /etc/ssl/certs/rds-global-bundle.pem
+
 RUN /usr/local/bin/python -m pip install --upgrade pip && \
     apt update && apt -y install graphviz graphviz-dev && \
     rm -rf /var/lib/apt/lists/* && \

backend/curation/api/curation-api.yml

@@ -472,6 +472,43 @@ paths:
           $ref: "#/components/responses/404"
         "413":
           $ref: "#/components/responses/413"
+    patch:
+      tags:
+        - Collection
+        - Dataset
+      summary: Update a Dataset's metadata.
+      security:
+        - curatorAccess: []
+      description: >-
+        Update a Dataset's title and title of corresponding dataset artifacts.
+      operationId: backend.curation.api.v1.curation.collections.collection_id.datasets.dataset_id.actions.patch
+      parameters:
+        - $ref: "#/components/parameters/path_collection_id"
+        - $ref: "#/components/parameters/path_dataset_id"
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                title:
+                  type: string
+                  description: The title of the Dataset.
+      responses:
+        "202":
+          $ref: "#/components/responses/202"
+        "400":
+          $ref: "#/components/responses/400"
+        "401":
+          $ref: "#/components/responses/401"
+        "403":
+          $ref: "#/components/responses/403"
+        "404":
+          $ref: "#/components/responses/404"
+        "405":
+          $ref: "#/components/responses/405"
+        "410":
+          $ref: "#/components/responses/410"
 
   /v1/collections/{collection_id}/s3-upload-credentials:
     get:

backend/curation/api/v1/curation/auth/token.py

@@ -1,3 +1,4 @@
+import requests
 from flask import make_response, request
 from jose import JWTError
 
@@ -14,9 +15,15 @@ def post():
         token_info = api_key.verify(user_api_key, config.api_key_secret)
     except JWTError:
         raise UnauthorizedError(detail="The API key is invalid") from None
-    else:
-        identity = auth0_management_session.get_user_api_key_identity(token_info["sub"])
-        if not identity:
-            raise NotFoundHTTPException(detail="The API key is no longer valid.")
+    identity = auth0_management_session.get_user_api_key_identity(token_info["sub"])
+    if not identity:
+        raise NotFoundHTTPException(detail="The API key is no longer valid.")
+    try:
         token = auth0_management_session.generate_access_token(identity["profileData"]["email"], user_api_key)
-        return make_response(token, 201)
+    except requests.exceptions.HTTPError as error:
+        if error.response.status_code == 403:
+            raise UnauthorizedError(
+                detail="This API key is old. use the latest API key or generate a new one."
+            ) from error
+        raise error
+    return make_response(token, 201)