From 037f9bef0676a5037c4483d182919571e3285b91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Mon, 13 Jan 2025 11:03:36 -0600 Subject: [PATCH] HJ-181 - Updaeting connection configs so Datahub is only visible on fidesplus --- src/fides/api/models/connectionconfig.py | 47 ++++++++++++++++++- .../connection_secrets_datahub.py | 13 ----- .../enums/system_type.py | 5 +- src/fides/api/util/connection_type.py | 20 +++++--- 4 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/fides/api/models/connectionconfig.py b/src/fides/api/models/connectionconfig.py index 2dc518cd44..6eade61c51 100644 --- a/src/fides/api/models/connectionconfig.py +++ b/src/fides/api/models/connectionconfig.py @@ -2,7 +2,7 @@ import enum from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type +from typing import TYPE_CHECKING, Any, List, Optional, Type from loguru import logger from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, String, event @@ -72,7 +72,7 @@ def human_readable(self) -> str: """Human-readable mapping for ConnectionTypes Add to this mapping if you add a new ConnectionType """ - readable_mapping: Dict[str, str] = { + readable_mapping: dict[str, str] = { ConnectionType.attentive_email.value: "Attentive Email", ConnectionType.bigquery.value: "BigQuery", ConnectionType.datahub.value: "DataHub", @@ -108,6 +108,49 @@ def human_readable(self) -> str: "Add new ConnectionType to human_readable mapping" ) + @property + def is_fidesplus(self) -> bool: + fidesplus_only_mapping: list[str] = [ + ConnectionType.datahub.value + ] + return self.value in fidesplus_only_mapping + + @property + def system_type(self) -> SystemType: + from fides.api.schemas.connection_configuration.enums.system_type import SystemType + + system_type_mapping: dict[str, SystemType] = { + ConnectionType.attentive_email.value: SystemType.email, + ConnectionType.bigquery.value: SystemType.database, + ConnectionType.datahub.value: SystemType.data_catalog, + ConnectionType.dynamic_erasure_email.value: SystemType.email, + ConnectionType.dynamodb.value: SystemType.database, + ConnectionType.fides.value: SystemType.database, # What's the actualy SystemType here? + ConnectionType.generic_consent_email.value: SystemType.email, + ConnectionType.generic_erasure_email.value: SystemType.email, + ConnectionType.google_cloud_sql_mysql.value: SystemType.database, + ConnectionType.google_cloud_sql_postgres.value: SystemType.database, + ConnectionType.https.value: SystemType.database, # What's the actualy SystemType here? + ConnectionType.manual_webhook.value: SystemType.manual, + ConnectionType.manual.value: SystemType.manual, + ConnectionType.mariadb.value: SystemType.database, + ConnectionType.mongodb.value: SystemType.database, + ConnectionType.mssql.value: SystemType.database, + ConnectionType.mysql.value: SystemType.database, + ConnectionType.postgres.value: SystemType.database, + ConnectionType.rds_mysql.value: SystemType.database, + ConnectionType.rds_postgres.value: SystemType.database, + ConnectionType.redshift.value: SystemType.database, + ConnectionType.s3.value: SystemType.database, + ConnectionType.saas.value: SystemType.saas, + ConnectionType.scylla.value: SystemType.database, + ConnectionType.snowflake.value: SystemType.database, + ConnectionType.sovrn.value: SystemType.email, + ConnectionType.timescale.value: SystemType.database, + } + + return system_type_mapping[self.value] + class AccessLevel(enum.Enum): """ diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py b/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py index bfda71ed50..54ae1ac4ab 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py @@ -10,14 +10,6 @@ ) -class PeriodicIntegrationFrequency(Enum): - """Enum for periodic integration frequency""" - - daily = "daily" - weekly = "weekly" - monthly = "monthly" - - class DatahubSchema(ConnectionConfigSecretsSchema): datahub_server_url: AnyHttpUrlStringRemovesSlash = Field( title="DataHub Server URL", @@ -28,11 +20,6 @@ class DatahubSchema(ConnectionConfigSecretsSchema): description="The token used to authenticate with your DataHub server.", json_schema_extra={"sensitive": True}, ) - frequency: PeriodicIntegrationFrequency = Field( - title="Frequency", - description="The frequency at which the integration should run. Defaults to daily.", - default=PeriodicIntegrationFrequency.daily, - ) _required_components: ClassVar[List[str]] = ["datahub_server_url", "datahub_token"] diff --git a/src/fides/api/schemas/connection_configuration/enums/system_type.py b/src/fides/api/schemas/connection_configuration/enums/system_type.py index 275a63f3f5..dad517822e 100644 --- a/src/fides/api/schemas/connection_configuration/enums/system_type.py +++ b/src/fides/api/schemas/connection_configuration/enums/system_type.py @@ -2,7 +2,8 @@ class SystemType(Enum): - saas = "saas" + data_catalog = "data_catalog" database = "database" - manual = "manual" email = "email" + manual = "manual" + saas = "saas" diff --git a/src/fides/api/util/connection_type.py b/src/fides/api/util/connection_type.py index 5edae3135a..286831942c 100644 --- a/src/fides/api/util/connection_type.py +++ b/src/fides/api/util/connection_type.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib from typing import Any, Dict, Set from fides.api.common_exceptions import NoSuchConnectionTypeSecretSchemaError @@ -194,6 +195,12 @@ def get_connection_types( system_type: SystemType | None = None, action_types: Set[ActionType] = SUPPORTED_ACTION_TYPES, ) -> list[ConnectionSystemTypeMap]: + + def check_fidesplus(conn_type: ConnectionType) -> bool: + """Check if the given connection type is a Fidesplus connector.""" + fidesplus_found = importlib.util.find_spec("fidesplus") + return conn_type.is_fidesplus and fidesplus_found + def is_match(elem: str) -> bool: """If a search query param was included, is it a substring of an available connector type?""" return search.lower() in elem.lower() if search else True @@ -220,9 +227,9 @@ def saas_request_type_filter(connection_type: str) -> bool: if (system_type == SystemType.database or system_type is None) and ( ActionType.access in action_types or ActionType.erasure in action_types ): - database_types: list[str] = sorted( + database_types: list[ConnectionType] = sorted( [ - conn_type.value + conn_type for conn_type in ConnectionType if conn_type not in [ @@ -237,15 +244,15 @@ def saas_request_type_filter(connection_type: str) -> bool: ConnectionType.saas, ConnectionType.sovrn, ] - and is_match(conn_type.value) + and is_match(conn_type.value) and check_fidesplus(conn_type) ] - ) + , key=lambda x: x.value) connection_system_types.extend( [ ConnectionSystemTypeMap( identifier=item, - type=SystemType.database, - human_readable=ConnectionType(item).human_readable, + type=item.system_type, + human_readable=item.human_readable, supported_actions=[ActionType.access, ActionType.erasure], ) for item in database_types @@ -336,4 +343,5 @@ def saas_request_type_filter(connection_type: str) -> bool: for email_type in email_types ] ) + return connection_system_types