Merge branch 'main' into gill/HJ-290/action-center-to-show-results

ethyca · Jan 7, 2025 · e4ff656 · e4ff656
2 parents 20ae9a0 + b805c8c
commit e4ff656
Show file tree

Hide file tree

Showing 13 changed files with 109 additions and 79 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -17,6 +17,8 @@ _Write some things here about the changes and any potential caveats_
 * [ ] Issue requirements met
 * [ ] All CI pipelines succeeded
 * [ ] `CHANGELOG.md` updated
+  * [ ] Add a https://github.com/ethyca/fides/labels/db-migration label to the entry if your change includes a DB migration
+  * [ ] Add a https://github.com/ethyca/fides/labels/high-risk label to the entry if your change includes a high-risk change (i.e. potential for performance impact or unexpected regression) that should be flagged
 * Followup issues:
   * [ ] Followup issues created (include link)
   * [ ] No followup issues

diff --git a/.github/workflows/backend_checks.yml b/.github/workflows/backend_checks.yml
@@ -75,43 +75,6 @@ jobs:
           path: /tmp/python-${{ matrix.python_version }}.tar
           retention-days: 1
 
-  ###################
-  ## Static Checks ##
-  ###################
-  Static-Checks:
-    strategy:
-      matrix:
-        session_name:
-          [
-            '"isort(check)"',
-            '"black(check)"',
-            "mypy",
-            "pylint",
-            "xenon",
-            "check_install",
-            '"pytest(nox)"',
-          ]
-    runs-on: ubuntu-latest
-    continue-on-error: true
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set Up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
-          cache: "pip"
-
-      - name: Install Nox
-        run: pip install nox>=2022
-
-      - name: Install Dev Requirements
-        run: pip install -r dev-requirements.txt
-
-      - name: Run Static Check
-        run: nox -s ${{ matrix.session_name }}
-
   ##################
   ## Performance ##
   ##################

diff --git a/.github/workflows/static_checks.yml b/.github/workflows/static_checks.yml
@@ -0,0 +1,53 @@
+name: Backend Static Code Checks
+
+on:
+  pull_request:
+  push:
+    branches:
+      - "main"
+      - "release-**"
+
+env:
+  IMAGE: ethyca/fides:local
+  DEFAULT_PYTHON_VERSION: "3.10.13"
+  # Docker auth with read-only permissions.
+  DOCKER_USER: ${{ secrets.DOCKER_USER }}
+  DOCKER_RO_TOKEN: ${{ secrets.DOCKER_RO_TOKEN }}
+
+jobs:
+  ###################
+  ## Static Checks ##
+  ###################
+  Static-Checks:
+    strategy:
+      matrix:
+        session_name:
+          [
+            '"isort(check)"',
+            '"black(check)"',
+            "mypy",
+            "pylint",
+            "xenon",
+            "check_install",
+            '"pytest(nox)"',
+          ]
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set Up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+          cache: "pip"
+
+      - name: Install Nox
+        run: pip install nox>=2022
+
+      - name: Install Dev Requirements
+        run: pip install -r dev-requirements.txt
+
+      - name: Run Static Check
+        run: nox -s ${{ matrix.session_name }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,10 @@ The types of changes are:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+Changes can also be flagged with a GitHub label for tracking purposes. The URL of the label should be put at the end of the entry. The possible labels are:
+- https://github.com/ethyca/fides/labels/high-risk: to indicate that a change is a "high-risk" change that could potentially lead to unanticipated regressions or degradations
+- https://github.com/ethyca/fides/labels/db-migration: to indicate that a given change includes a DB migration
+
 ## [Unreleased](https://github.com/ethyca/fides/compare/2.52.0...main)
 
 ### Added
@@ -29,7 +33,7 @@ The types of changes are:
 - Added event based communication example to the Cookie House sample app [#5597](https://github.com/ethyca/fides/pull/5597)
 - Added new erasure tests for BigQuery Enterprise [#5554](https://github.com/ethyca/fides/pull/5554)
 - Added new `has_next` parameter for the `link` pagination strategy [#5596](https://github.com/ethyca/fides/pull/5596)
-- Added a `DBCache` model for database-backed caching [#5613](https://github.com/ethyca/fides/pull/5613)
+- Added a `DBCache` model for database-backed caching [#5613](https://github.com/ethyca/fides/pull/5613) https://github.com/ethyca/fides/labels/db-migration
 - Adds "reclassify" button to discovery result tables [#5574](https://github.com/ethyca/fides/pull/5574)
 - Added support for exporting datamaps with column renaming, reordering and visibility options [#5543](https://github.com/ethyca/fides/pull/5543)
 
@@ -98,7 +102,7 @@ The types of changes are:
 - Allow hiding systems via a `hidden` parameter and add two flags on the `/system` api endpoint; `show_hidden` and `dnd_relevant`, to display only systems with integrations [#5484](https://github.com/ethyca/fides/pull/5484)
 - The CMP override `fides_privacy_policy_url` will now apply even if the `fides_override_language` doesn't match [#5515](https://github.com/ethyca/fides/pull/5515)
 - Updated POST taxonomy endpoints to handle creating resources without specifying fides_key [#5468](https://github.com/ethyca/fides/pull/5468)
-- Disabled connection pooling for task workers and added retries and keep-alive configurations for database connections [#5448](https://github.com/ethyca/fides/pull/5448)
+- Disabled connection pooling for task workers and added retries and keep-alive configurations for database connections [#5448](https://github.com/ethyca/fides/pull/5448) https://github.com/ethyca/fides/labels/high-risk
 - Added timeout handling in the UI for async discovery monitor-related queries [#5519](https://github.com/ethyca/fides/pull/5519)
 
 ### Developer Experience

diff --git a/noxfiles/ci_nox.py b/noxfiles/ci_nox.py
@@ -95,13 +95,19 @@ def xenon(session: nox.Session) -> None:
         "src",
         "tests",
         "scripts",
-        "--max-absolute B",
-        "--max-modules B",
-        "--max-average A",
-        "--ignore 'data, docs'",
-        "--exclude src/fides/_version.py",
+        "--max-absolute=B",
+        "--max-modules=B",
+        "--max-average=A",
+        "--ignore=data,docs",
+        "--exclude=src/fides/_version.py",
+    )
+    session.run(*command, success_codes=[0, 1])
+    session.warn(
+        "Note: This command was malformed so it's been failing to report complexity issues."
+    )
+    session.warn(
+        "Intentionally suppressing the error status code for now to slowly work through the issues."
     )
-    session.run(*command)
 
 
 ##################

diff --git a/src/fides/api/api/v1/endpoints/dataset_endpoints.py b/src/fides/api/api/v1/endpoints/dataset_endpoints.py
@@ -98,7 +98,7 @@
 def _get_connection_config(
     connection_key: FidesKey, db: Session = Depends(deps.get_db)
 ) -> ConnectionConfig:
-    logger.info("Finding connection config with key '{}'", connection_key)
+    logger.debug("Finding connection config with key '{}'", connection_key)
     connection_config = ConnectionConfig.get_by(db, field="key", value=connection_key)
     if not connection_config:
         raise HTTPException(
@@ -509,7 +509,7 @@ def get_datasets(
     Soon to be deprecated.
     """
 
-    logger.info(
+    logger.debug(
         "Finding all datasets for connection '{}' with pagination params {}",
         connection_config.key,
         params,
@@ -544,7 +544,7 @@ def get_dataset(
     Soon to be deprecated
     """
 
-    logger.info(
+    logger.debug(
         "Finding dataset '{}' for connection '{}'", fides_key, connection_config.key
     )
     dataset_config = DatasetConfig.filter(
@@ -574,7 +574,7 @@ def get_dataset_configs(
 ) -> AbstractPage[DatasetConfig]:
     """Returns all Dataset Configs attached to current Connection Config."""
 
-    logger.info(
+    logger.debug(
         "Finding all dataset configs for connection '{}' with pagination params {}",
         connection_config.key,
         params,
@@ -598,7 +598,7 @@ def get_dataset_config(
 ) -> DatasetConfig:
     """Returns the specific Dataset Config linked to the Connection Config."""
 
-    logger.info(
+    logger.debug(
         "Finding dataset config '{}' for connection '{}'",
         fides_key,
         connection_config.key,
@@ -669,7 +669,7 @@ def get_ctl_datasets(
     Returns all CTL datasets .
     """
 
-    logger.info(
+    logger.debug(
         f"Finding all datasets {remove_saas_datasets=} {only_unlinked_datasets=}"
     )
     filters = []

diff --git a/src/fides/api/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/api/v1/endpoints/messaging_endpoints.py
@@ -182,7 +182,7 @@ def get_active_default_config(*, db: Session = Depends(deps.get_db)) -> Messagin
     """
     Retrieves the active default messaging config.
     """
-    logger.info("Finding active default messaging config")
+    logger.debug("Finding active default messaging config")
     try:
         messaging_config = MessagingConfig.get_active_default(db)
     except ValueError:
@@ -421,7 +421,7 @@ def get_configs(
     """
     Retrieves configs for messaging.
     """
-    logger.info(
+    logger.debug(
         "Finding all messaging configurations with pagination params {}", params
     )
     return paginate(
@@ -441,7 +441,7 @@ def get_config_by_key(
     """
     Retrieves configs for messaging service by key.
     """
-    logger.info("Finding messaging config with key '{}'", config_key)
+    logger.debug("Finding messaging config with key '{}'", config_key)
 
     try:
         return get_messaging_config_by_key(db=db, key=config_key)
@@ -463,7 +463,7 @@ def get_default_config_by_type(
     """
     Retrieves default config for messaging service by type.
     """
-    logger.info("Finding default messaging config of type '{}'", service_type)
+    logger.debug("Finding default messaging config of type '{}'", service_type)
 
     messaging_config = MessagingConfig.get_by_type(db, service_type)
     if not messaging_config:
@@ -620,7 +620,7 @@ def get_default_messaging_template(
     """
     Retrieves default messaging template by template type.
     """
-    logger.info(
+    logger.debug(
         "Finding default messaging template of template type '{}'", template_type
     )
     try:
@@ -645,7 +645,7 @@ def get_messaging_template_by_id(
     """
     Retrieves messaging template by template tid.
     """
-    logger.info("Finding messaging template with id '{}'", template_id)
+    logger.debug("Finding messaging template with id '{}'", template_id)
 
     try:
         messaging_template = get_template_by_id(db, template_id)

diff --git a/src/fides/api/api/v1/endpoints/policy_endpoints.py b/src/fides/api/api/v1/endpoints/policy_endpoints.py
@@ -56,14 +56,14 @@ def get_policy_list(
     """
     Return a paginated list of all Policy records in this system
     """
-    logger.info("Finding all policies with pagination params '{}'", params)
+    logger.debug("Finding all policies with pagination params '{}'", params)
     policies = Policy.query(db=db).order_by(Policy.created_at.desc())
     return paginate(policies, params=params)
 
 
 def get_policy_or_error(db: Session, policy_key: FidesKey) -> Policy:
     """Helper method to load Policy or throw a 404"""
-    logger.info("Finding policy with key '{}'", policy_key)
+    logger.debug("Finding policy with key '{}'", policy_key)
     policy = Policy.get_by(db=db, field="key", value=policy_key)
     if not policy:
         raise HTTPException(
@@ -160,7 +160,7 @@ def get_rule_or_error(db: Session, policy_key: FidesKey, rule_key: FidesKey) ->
     Also throws a 404 if a `Policy` with the given key can't be found.
     """
     policy = get_policy_or_error(db, policy_key)
-    logger.info("Finding rule with key '{}'", rule_key)
+    logger.debug("Finding rule with key '{}'", rule_key)
     rule = Rule.filter(
         db=db,
         conditions=((Rule.policy_id == policy.id) & (Rule.key == rule_key)),
@@ -191,7 +191,7 @@ def get_rule_list(
     Throws a 404 if the given `Policy` can't be found.
     """
     policy = get_policy_or_error(db, policy_key)
-    logger.info(
+    logger.debug(
         "Finding all rules for policy {} with pagination params '{}'",
         policy_key,
         params,
@@ -382,7 +382,7 @@ def get_rule_target_or_error(
     Helper method to load Rule Target or throw a 404.
     Also throws a 404 if a `Policy` or `Rule` with the given keys can't be found.
     """
-    logger.info("Finding rule target with key '{}'", rule_target_key)
+    logger.debug("Finding rule target with key '{}'", rule_target_key)
     rule: Rule = get_rule_or_error(db, policy_key, rule_key)
     rule_target = RuleTarget.filter(
         db=db,
@@ -417,7 +417,7 @@ def get_rule_target_list(
     Throws a 404 if the given `Rule` or `Policy` can't be found.
     """
     rule = get_rule_or_error(db, policy_key, rule_key)
-    logger.info(
+    logger.debug(
         "Finding all rule targets for rule {} with pagination params '{}'",
         rule_key,
         params,

diff --git a/src/fides/api/api/v1/endpoints/policy_webhook_endpoints.py b/src/fides/api/api/v1/endpoints/policy_webhook_endpoints.py
@@ -50,7 +50,7 @@ def get_policy_pre_execution_webhooks(
     """
     policy = get_policy_or_error(db, policy_key)
 
-    logger.info(
+    logger.debug(
         "Finding all Pre-Execution Webhooks for Policy '{}' with pagination params '{}'",
         policy.key,
         params,
@@ -76,7 +76,7 @@ def get_policy_post_execution_webhooks(
     """
     policy = get_policy_or_error(db, policy_key)
 
-    logger.info(
+    logger.debug(
         "Finding all Post-Execution Webhooks for Policy '{}' with pagination params '{}'",
         policy.key,
         params,
@@ -218,7 +218,7 @@ def get_policy_webhook_or_error(
 
     Also verifies that the webhook belongs to the given Policy.
     """
-    logger.info(
+    logger.debug(
         "Finding {}-Execution Webhook with key '{}' for Policy '{}'",
         webhook_cls.prefix.capitalize(),
         webhook_key,

diff --git a/src/fides/api/api/v1/endpoints/pre_approval_webhook_endpoints.py b/src/fides/api/api/v1/endpoints/pre_approval_webhook_endpoints.py
@@ -39,7 +39,9 @@ def get_pre_approval_webhook_list(
     """
     Return a paginated list of all PreApprovalWebhook records in this system
     """
-    logger.info("Finding all pre_approval webhooks with pagination params '{}'", params)
+    logger.debug(
+        "Finding all pre_approval webhooks with pagination params '{}'", params
+    )
     pre_approval_webhooks = PreApprovalWebhook.query(db=db).order_by(
         PreApprovalWebhook.created_at.desc()
     )
@@ -50,7 +52,7 @@ def get_pre_approval_webhook_or_error(
     db: Session, webhook_key: FidesKey
 ) -> PreApprovalWebhook:
     """Helper method to load PreApprovalWebhook or throw a 404"""
-    logger.info("Finding PreApprovalWebhook with key '{}'", webhook_key)
+    logger.debug("Finding PreApprovalWebhook with key '{}'", webhook_key)
     pre_approval_webhook = PreApprovalWebhook.get_by(
         db=db, field="key", value=webhook_key
     )