HJ 181 - Datahub groundwork and UI

clients/admin-ui/src/features/integrations/ConnectionCategory.ts

@@ -1,4 +1,5 @@
 export enum ConnectionCategory {
+  DATA_CATALOG = "Data Catalog",
   DATA_WAREHOUSE = "Data Warehouse",
   DATABASE = "Database",
 }

clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx

@@ -2,6 +2,7 @@ import { ReactNode } from "react";
 
 import { ConnectionCategory } from "~/features/integrations/ConnectionCategory";
 import BIGQUERY_TYPE_INFO from "~/features/integrations/integration-type-info/bigqueryInfo";
+import DATAHUB_TYPE_INFO from "~/features/integrations/integration-type-info/datahubInfo";
 import DYNAMO_TYPE_INFO from "~/features/integrations/integration-type-info/dynamoInfo";
 import GOOGLE_CLOUD_SQL_MYSQL_TYPE_INFO from "~/features/integrations/integration-type-info/googleCloudSQLMySQLInfo";
 import GOOGLE_CLOUD_SQL_POSTGRES_TYPE_INFO from "~/features/integrations/integration-type-info/googleCloudSQLPostgresInfo";
@@ -27,6 +28,7 @@ export type IntegrationTypeInfo = {
 
 const INTEGRATION_TYPE_MAP: { [K in ConnectionType]?: IntegrationTypeInfo } = {
   [ConnectionType.BIGQUERY]: BIGQUERY_TYPE_INFO,
+  [ConnectionType.DATAHUB]: DATAHUB_TYPE_INFO,
   [ConnectionType.DYNAMODB]: DYNAMO_TYPE_INFO,
   [ConnectionType.GOOGLE_CLOUD_SQL_MYSQL]: GOOGLE_CLOUD_SQL_MYSQL_TYPE_INFO,
   [ConnectionType.GOOGLE_CLOUD_SQL_POSTGRES]:

clients/admin-ui/src/features/integrations/integration-type-info/datahubInfo.tsx

@@ -0,0 +1,67 @@
+import { ListItem } from "fidesui";
+
+import {
+  InfoHeading,
+  InfoText,
+  InfoUnorderedList,
+} from "~/features/common/copy/components";
+import ShowMoreContent from "~/features/common/copy/ShowMoreContent";
+import { ConnectionCategory } from "~/features/integrations/ConnectionCategory";
+import { AccessLevel, ConnectionType } from "~/types/api";
+
+export const DATAHUB_PLACEHOLDER = {
+  name: "Datahub",
+  key: "datahub_placeholder",
+  connection_type: ConnectionType.DATAHUB,
+  access: AccessLevel.READ,
+  created_at: "",
+};
+
+export const DATAHUB_TAGS = ["Data catalog"];
+
+export const DatahubOverview = () => (
+  <>
+    <InfoHeading text="Overview" />
+    <InfoText>
+      Datahub is a cloud-based data warehousing platform designed for handling
+      large-scale data storage and analytics. It enables organizations to store,
+      manage, and analyze massive amounts of data efficiently, offering features
+      like scalability, performance, and flexibility.
+    </InfoText>
+    <ShowMoreContent>
+      <InfoHeading text="Categories" />
+      <InfoUnorderedList>
+        <ListItem>Database</ListItem>
+        <ListItem>SQL database</ListItem>
+        <ListItem>Storage system</ListItem>
+        <ListItem>Data detection</ListItem>
+        <ListItem>Data discovery</ListItem>
+      </InfoUnorderedList>
+      <InfoHeading text="Permissions" />
+      <InfoUnorderedList>
+        <ListItem>CREATE ROLE my_monitor_role;</ListItem>
+        <ListItem>
+          GRANT USAGE ON DATABASE DATABASE_1 TO ROLE my_monitor_role;
+        </ListItem>
+        <ListItem>
+          GRANT USAGE ON SCHEMA DATABASE_1.TEST_SCHEMA TO ROLE my_monitor_role;
+        </ListItem>
+        <ListItem>
+          GRANT SELECT ON ALL TABLES IN SCHEMA DATABASE_1.TEST_SCHEMA TO ROLE
+          my_monitor_role;
+        </ListItem>
+        <ListItem>CREATE USER test_user PASSWORD=&apos;***&apos;;</ListItem>
+        <ListItem>GRANT ROLE my_monitor_role TO USER test_user;</ListItem>
+      </InfoUnorderedList>
+    </ShowMoreContent>
+  </>
+);
+
+const DATAHUB_TYPE_INFO = {
+  placeholder: DATAHUB_PLACEHOLDER,
+  category: ConnectionCategory.DATA_CATALOG,
+  overview: <DatahubOverview />,
+  tags: DATAHUB_TAGS,
+};
+
+export default DATAHUB_TYPE_INFO;

clients/admin-ui/src/features/integrations/useIntegrationFilterTabs.tsx

@@ -5,6 +5,7 @@ import { IntegrationTypeInfo } from "~/features/integrations/add-integration/all
 export enum IntegrationFilterTabs {
   ALL = "All",
   DATABASE = "Database",
+  DATA_CATALOG = "Data Catalog",
   DATA_WAREHOUSE = "Data Warehouse",
 }
 

src/fides/api/models/connectionconfig.py

@@ -2,7 +2,7 @@
 
 import enum
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type
+from typing import TYPE_CHECKING, Any, List, Optional, Type
 
 from loguru import logger
 from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, String, event
@@ -72,7 +72,7 @@ def human_readable(self) -> str:
         """Human-readable mapping for ConnectionTypes
         Add to this mapping if you add a new ConnectionType
         """
-        readable_mapping: Dict[str, str] = {
+        readable_mapping: dict[str, str] = {
             ConnectionType.attentive_email.value: "Attentive Email",
             ConnectionType.bigquery.value: "BigQuery",
             ConnectionType.datahub.value: "DataHub",
@@ -108,6 +108,49 @@ def human_readable(self) -> str:
                 "Add new ConnectionType to human_readable mapping"
             )
 
+    @property
+    def is_fidesplus(self) -> bool:
+        fidesplus_only_mapping: list[str] = [
+            ConnectionType.datahub.value
+        ]
+        return self.value in fidesplus_only_mapping
+
+    @property
+    def system_type(self) -> SystemType:
+        from fides.api.schemas.connection_configuration.enums.system_type import SystemType
+
+        system_type_mapping: dict[str, SystemType] = {
+            ConnectionType.attentive_email.value: SystemType.email,
+            ConnectionType.bigquery.value: SystemType.database,
+            ConnectionType.datahub.value: SystemType.data_catalog,
+            ConnectionType.dynamic_erasure_email.value: SystemType.email,
+            ConnectionType.dynamodb.value: SystemType.database,
+            ConnectionType.fides.value: SystemType.database,  # What's the actualy SystemType here?
+            ConnectionType.generic_consent_email.value: SystemType.email,
+            ConnectionType.generic_erasure_email.value: SystemType.email,
+            ConnectionType.google_cloud_sql_mysql.value: SystemType.database,
+            ConnectionType.google_cloud_sql_postgres.value: SystemType.database,
+            ConnectionType.https.value: SystemType.database,  # What's the actualy SystemType here?
+            ConnectionType.manual_webhook.value: SystemType.manual,
+            ConnectionType.manual.value: SystemType.manual,
+            ConnectionType.mariadb.value: SystemType.database,
+            ConnectionType.mongodb.value: SystemType.database,
+            ConnectionType.mssql.value: SystemType.database,
+            ConnectionType.mysql.value: SystemType.database,
+            ConnectionType.postgres.value: SystemType.database,
+            ConnectionType.rds_mysql.value: SystemType.database,
+            ConnectionType.rds_postgres.value: SystemType.database,
+            ConnectionType.redshift.value: SystemType.database,
+            ConnectionType.s3.value: SystemType.database,
+            ConnectionType.saas.value: SystemType.saas,
+            ConnectionType.scylla.value: SystemType.database,
+            ConnectionType.snowflake.value: SystemType.database,
+            ConnectionType.sovrn.value: SystemType.email,
+            ConnectionType.timescale.value: SystemType.database,
+        }
+
+        return system_type_mapping[self.value]
+
 
 class AccessLevel(enum.Enum):
     """

src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py

@@ -10,14 +10,6 @@
 )
 
 
-class PeriodicIntegrationFrequency(Enum):
-    """Enum for periodic integration frequency"""
-
-    daily = "daily"
-    weekly = "weekly"
-    monthly = "monthly"
-
-
 class DatahubSchema(ConnectionConfigSecretsSchema):
     datahub_server_url: AnyHttpUrlStringRemovesSlash = Field(
         title="DataHub Server URL",
@@ -28,11 +20,6 @@ class DatahubSchema(ConnectionConfigSecretsSchema):
         description="The token used to authenticate with your DataHub server.",
         json_schema_extra={"sensitive": True},
     )
-    frequency: PeriodicIntegrationFrequency = Field(
-        title="Frequency",
-        description="The frequency at which the integration should run. Defaults to daily.",
-        default=PeriodicIntegrationFrequency.daily,
-    )
 
     _required_components: ClassVar[List[str]] = ["datahub_server_url", "datahub_token"]
 

src/fides/api/schemas/connection_configuration/enums/system_type.py

@@ -2,7 +2,8 @@
 
 
 class SystemType(Enum):
-    saas = "saas"
+    data_catalog = "data_catalog"
     database = "database"
-    manual = "manual"
     email = "email"
+    manual = "manual"
+    saas = "saas"

src/fides/api/util/connection_type.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import importlib
 from typing import Any, Dict, Set
 
 from fides.api.common_exceptions import NoSuchConnectionTypeSecretSchemaError
@@ -194,6 +195,12 @@ def get_connection_types(
     system_type: SystemType | None = None,
     action_types: Set[ActionType] = SUPPORTED_ACTION_TYPES,
 ) -> list[ConnectionSystemTypeMap]:
+
+    def check_fidesplus(conn_type: ConnectionType) -> bool:
+        """Check if the given connection type is a Fidesplus connector."""
+        fidesplus_found = importlib.util.find_spec("fidesplus")
+        return conn_type.is_fidesplus and fidesplus_found
+
     def is_match(elem: str) -> bool:
         """If a search query param was included, is it a substring of an available connector type?"""
         return search.lower() in elem.lower() if search else True
@@ -220,9 +227,9 @@ def saas_request_type_filter(connection_type: str) -> bool:
     if (system_type == SystemType.database or system_type is None) and (
         ActionType.access in action_types or ActionType.erasure in action_types
     ):
-        database_types: list[str] = sorted(
+        database_types: list[ConnectionType] = sorted(
             [
-                conn_type.value
+                conn_type
                 for conn_type in ConnectionType
                 if conn_type
                 not in [
@@ -237,15 +244,15 @@ def saas_request_type_filter(connection_type: str) -> bool:
                     ConnectionType.saas,
                     ConnectionType.sovrn,
                 ]
-                and is_match(conn_type.value)
+                and is_match(conn_type.value) and check_fidesplus(conn_type)
             ]
-        )
+        , key=lambda x: x.value)
         connection_system_types.extend(
             [
                 ConnectionSystemTypeMap(
                     identifier=item,
-                    type=SystemType.database,
-                    human_readable=ConnectionType(item).human_readable,
+                    type=item.system_type,
+                    human_readable=item.human_readable,
                     supported_actions=[ActionType.access, ActionType.erasure],
                 )
                 for item in database_types
@@ -336,4 +343,5 @@ def saas_request_type_filter(connection_type: str) -> bool:
                 for email_type in email_types
             ]
         )
+
     return connection_system_types