From 2d9f1ef27026454fac915d635d9d0b41936b0916 Mon Sep 17 00:00:00 2001 From: Jason Gill <jason.gill@ethyca.com> Date: Wed, 8 Jan 2025 10:49:14 -0700 Subject: [PATCH 1/9] Saved report bug fixes (#5649) --- CHANGELOG.md | 4 ++ .../admin-ui/cypress/e2e/datamap-report.cy.ts | 44 +++++++++++-- .../reporting/DatamapReportFilterModal.tsx | 2 +- .../datamap/reporting/DatamapReportTable.tsx | 65 +++++++++++++++---- .../reporting/datamap-report-context.tsx | 22 ++++--- .../src/features/datamap/reporting/utils.ts | 24 +++---- 6 files changed, 119 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 284a623081..3e7d248787 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,10 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ### Added - Added cache-clearing methods to the `DBCache` model to allow deleting cache entries [#5629](https://github.com/ethyca/fides/pull/5629) +### Fixed +- Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) +- Fixed column ordering issue in the Data Map report [#5649](https://github.com/ethyca/fides/pull/5649) +- Fixed issue where the Data Map report filter dialog was missing an Accordion item label [#5649](https://github.com/ethyca/fides/pull/5649) ## [2.52.0](https://github.com/ethyca/fides/compare/2.51.2...2.52.0) diff --git a/clients/admin-ui/cypress/e2e/datamap-report.cy.ts b/clients/admin-ui/cypress/e2e/datamap-report.cy.ts index 7d15b5a230..40c96cbe54 100644 --- a/clients/admin-ui/cypress/e2e/datamap-report.cy.ts +++ b/clients/admin-ui/cypress/e2e/datamap-report.cy.ts @@ -323,6 +323,15 @@ describe("Data map report table", () => { it("should filter the table by making a selection", () => { cy.getByTestId("filter-multiple-systems-btn").click(); cy.getByTestId("datamap-report-filter-modal").should("be.visible"); + cy.getByTestId("filter-modal-accordion-button") + .eq(0) + .should("have.text", "Data use"); + cy.getByTestId("filter-modal-accordion-button") + .eq(1) + .should("have.text", "Data categories"); + cy.getByTestId("filter-modal-accordion-button") + .eq(2) + .should("have.text", "Data subject"); cy.getByTestId("filter-modal-accordion-button").eq(1).click(); cy.getByTestId("filter-modal-checkbox-tree-categories").should( "be.visible", @@ -389,14 +398,15 @@ describe("Data map report table", () => { cy.get("#toast-datamap-report-toast") .should("be.visible") .should("have.attr", "data-status", "success"); - cy.getByTestId("custom-reports-trigger") - .should("contain.text", "My Custom Report") - .click(); + cy.getByTestId("custom-reports-trigger").should( + "contain.text", + "My Custom Report", + ); cy.getByTestId("fidesTable").within(() => { // reordering applied to report cy.get("thead th").eq(2).should("contain.text", "Legal name"); // column visibility applied to report - cy.get("thead th").eq(4).should("not.contain.text", "Data subject"); + cy.getByTestId("column-data_subjects").should("not.exist"); }); cy.getByTestId("group-by-menu").should( "contain.text", @@ -442,10 +452,36 @@ describe("Data map report table", () => { cy.getByTestId("custom-reports-reset-button").click(); cy.getByTestId("apply-report-button").click(); cy.getByTestId("custom-reports-popover").should("not.be.visible"); + cy.getByTestId("custom-reports-trigger").should( "contain.text", "Reports", ); + cy.getByTestId("fidesTable").within(() => { + // reordering reverted + cy.get("thead th").eq(2).should("contain.text", "Data categories"); + // column visibility restored + cy.getByTestId("column-data_subjects").should("exist"); + }); + cy.getByTestId("group-by-menu").should("contain.text", "Group by system"); + cy.getByTestId("more-menu").click(); + cy.getByTestId("edit-columns-btn").click(); + cy.get("button#data_subjects").should( + "have.attr", + "aria-checked", + "true", + ); + cy.getByTestId("column-settings-close-button").click(); + cy.getByTestId("filter-multiple-systems-btn").click(); + cy.getByTestId("datamap-report-filter-modal") + .should("be.visible") + .within(() => { + cy.getByTestId("filter-modal-accordion-button").eq(0).click(); + cy.getByTestId("checkbox-Analytics").within(() => { + cy.get("[data-checked]").should("not.exist"); + }); + cy.getByTestId("standard-dialog-close-btn").click(); + }); }); it("should allow the user cancel a report selection", () => { cy.wait("@getCustomReportsMinimal"); diff --git a/clients/admin-ui/src/features/datamap/reporting/DatamapReportFilterModal.tsx b/clients/admin-ui/src/features/datamap/reporting/DatamapReportFilterModal.tsx index fc67fb016f..1bded963b2 100644 --- a/clients/admin-ui/src/features/datamap/reporting/DatamapReportFilterModal.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/DatamapReportFilterModal.tsx @@ -148,7 +148,7 @@ export const DatamapReportFilterModal = ({ data-testid="datamap-report-filter-modal" > <Accordion allowToggle> - <FilterModalAccordionItem label={columnNameMap.data_use}> + <FilterModalAccordionItem label={columnNameMap.data_uses}> <CheckboxTree nodes={dataUseNodes} selected={checkedUses} diff --git a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx index 0b0164ce12..d621bb1bf0 100644 --- a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx @@ -60,7 +60,11 @@ import { import { CustomReportTemplates } from "../../common/custom-reports/CustomReportTemplates"; import { DATAMAP_LOCAL_STORAGE_KEYS, DEFAULT_COLUMN_NAMES } from "./constants"; import { DatamapReportWithCustomFields as DatamapReport } from "./datamap-report"; -import { useDatamapReport } from "./datamap-report-context"; +import { + DEFAULT_COLUMN_FILTERS, + DEFAULT_COLUMN_VISIBILITY, + useDatamapReport, +} from "./datamap-report-context"; import { getDatamapReportColumns, getDefaultColumn, @@ -223,14 +227,6 @@ export const DatamapReportTable = () => { ], ); - useEffect(() => { - if (datamapReport?.items?.length) { - const columnIDs = Object.keys(datamapReport.items[0]); - setColumnOrder(getColumnOrder(groupBy, columnIDs)); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [groupBy, datamapReport]); - const { isOpen: isColumnSettingsOpen, onOpen: onColumnSettingsOpen, @@ -306,6 +302,20 @@ export const DatamapReportTable = () => { }, }); + useEffect(() => { + if (groupBy && !!tableInstance) { + if (tableInstance.getState().columnOrder.length === 0) { + const tableColumnIds = tableInstance.getAllColumns().map((c) => c.id); + setColumnOrder(getColumnOrder(groupBy, tableColumnIds)); + } else { + setColumnOrder( + getColumnOrder(groupBy, tableInstance.getState().columnOrder), + ); + } + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [groupBy, tableInstance]); + useEffect(() => { // changing the groupBy should wait until the data is loaded to update the grouping const newGrouping = getGrouping(groupBy); @@ -345,12 +355,41 @@ export const DatamapReportTable = () => { const handleSavedReport = ( savedReport: CustomReportResponse | null, - resetForm: ( + resetColumnNameForm: ( nextState?: Partial<FormikState<Record<string, string>>> | undefined, ) => void, ) => { + if (!savedReport && !savedCustomReportId) { + return; + } if (!savedReport) { - setSavedCustomReportId(""); + try { + setSavedCustomReportId(""); + + /* NOTE: we can't just use tableInstance.reset() here because it will reset the table to the initial state, which is likely to include report settings that were saved in the user's local storage. Instead, we need to reset each individual setting to its default value. */ + + // reset column visibility (must happen before updating order) + setColumnVisibility(DEFAULT_COLUMN_VISIBILITY); + tableInstance.toggleAllColumnsVisible(true); + tableInstance.setColumnVisibility(DEFAULT_COLUMN_VISIBILITY); + + // reset column order (must happen prior to updating groupBy) + setColumnOrder([]); + tableInstance.setColumnOrder([]); + + // reset groupBy and filters (will automatically update the tableinstance) + setGroupBy(DATAMAP_GROUPING.SYSTEM_DATA_USE); + setSelectedFilters(DEFAULT_COLUMN_FILTERS); + + // reset column names + setColumnNameMapOverrides({}); + resetColumnNameForm({ values: {} }); + } catch (error: any) { + toast({ + status: "error", + description: "There was a problem resetting the report.", + }); + } return; } try { @@ -369,8 +408,8 @@ export const DatamapReportTable = () => { ); if (savedGroupBy) { + // No need to manually update the tableInstance here; setting the groupBy will trigger the useEffect to update the grouping. setGroupBy(savedGroupBy); - tableInstance.setGrouping(getGrouping(savedGroupBy)); } if (savedFilters) { setSelectedFilters(savedFilters); @@ -394,7 +433,7 @@ export const DatamapReportTable = () => { }, ); setColumnNameMapOverrides(columnNameMap); - resetForm({ values: columnNameMap }); + resetColumnNameForm({ values: columnNameMap }); } setSavedCustomReportId(savedReport.id); toast({ diff --git a/clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx b/clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx index 3051bfd243..883f331fdf 100644 --- a/clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx @@ -12,6 +12,17 @@ import { DATAMAP_GROUPING } from "~/types/api"; import { DatamapReportFilterSelections } from "../types"; import { COLUMN_IDS, DATAMAP_LOCAL_STORAGE_KEYS } from "./constants"; +export const DEFAULT_COLUMN_VISIBILITY = { + [COLUMN_IDS.SYSTEM_UNDECLARED_DATA_CATEGORIES]: false, + [COLUMN_IDS.DATA_USE_UNDECLARED_DATA_CATEGORIES]: false, +}; + +export const DEFAULT_COLUMN_FILTERS = { + dataUses: [], + dataSubjects: [], + dataCategories: [], +}; + interface DatamapReportContextProps { savedCustomReportId: string; setSavedCustomReportId: Dispatch<SetStateAction<string>>; @@ -51,11 +62,7 @@ export const DatamapReportProvider = ({ const [selectedFilters, setSelectedFilters] = useLocalStorage<DatamapReportFilterSelections>( DATAMAP_LOCAL_STORAGE_KEYS.FILTERS, - { - dataUses: [], - dataSubjects: [], - dataCategories: [], - }, + DEFAULT_COLUMN_FILTERS, ); const [columnOrder, setColumnOrder] = useLocalStorage<string[]>( @@ -65,10 +72,7 @@ export const DatamapReportProvider = ({ const [columnVisibility, setColumnVisibility] = useLocalStorage< Record<string, boolean> - >(DATAMAP_LOCAL_STORAGE_KEYS.COLUMN_VISIBILITY, { - [COLUMN_IDS.SYSTEM_UNDECLARED_DATA_CATEGORIES]: false, - [COLUMN_IDS.DATA_USE_UNDECLARED_DATA_CATEGORIES]: false, - }); + >(DATAMAP_LOCAL_STORAGE_KEYS.COLUMN_VISIBILITY, DEFAULT_COLUMN_VISIBILITY); const [columnSizing, setColumnSizing] = useLocalStorage< Record<string, number> diff --git a/clients/admin-ui/src/features/datamap/reporting/utils.ts b/clients/admin-ui/src/features/datamap/reporting/utils.ts index 6c7b910134..343dce7969 100644 --- a/clients/admin-ui/src/features/datamap/reporting/utils.ts +++ b/clients/admin-ui/src/features/datamap/reporting/utils.ts @@ -12,10 +12,7 @@ export const getGrouping = (groupBy?: DATAMAP_GROUPING) => { } }; -export const getColumnOrder = ( - groupBy: DATAMAP_GROUPING, - columnIDs: string[], -) => { +export const getPrefixColumns = (groupBy: DATAMAP_GROUPING) => { let columnOrder: string[] = []; if (DATAMAP_GROUPING.SYSTEM_DATA_USE === groupBy) { columnOrder = [COLUMN_IDS.SYSTEM_NAME, COLUMN_IDS.DATA_USE]; @@ -23,6 +20,14 @@ export const getColumnOrder = ( if (DATAMAP_GROUPING.DATA_USE_SYSTEM === groupBy) { columnOrder = [COLUMN_IDS.DATA_USE, COLUMN_IDS.SYSTEM_NAME]; } + return columnOrder; +}; + +export const getColumnOrder = ( + groupBy: DATAMAP_GROUPING, + columnIDs: string[], +) => { + let columnOrder: string[] = getPrefixColumns(groupBy); columnOrder = columnOrder.concat( columnIDs.filter( (columnID) => @@ -31,14 +36,3 @@ export const getColumnOrder = ( ); return columnOrder; }; - -export const getPrefixColumns = (groupBy: DATAMAP_GROUPING) => { - let columnOrder: string[] = []; - if (DATAMAP_GROUPING.SYSTEM_DATA_USE === groupBy) { - columnOrder = [COLUMN_IDS.SYSTEM_NAME, COLUMN_IDS.DATA_USE]; - } - if (DATAMAP_GROUPING.DATA_USE_SYSTEM === groupBy) { - columnOrder = [COLUMN_IDS.DATA_USE, COLUMN_IDS.SYSTEM_NAME]; - } - return columnOrder; -}; From 483d7984e996af447e37e1bd4cc59d05af9821db Mon Sep 17 00:00:00 2001 From: Jason Gill <jason.gill@ethyca.com> Date: Wed, 8 Jan 2025 11:49:18 -0700 Subject: [PATCH 2/9] Action Center results MVP (#5622) --- CHANGELOG.md | 2 +- .../admin-ui/cypress/e2e/action-center.cy.ts | 119 ++++++++++++ .../results/aggregate-results.json | 40 ++++ clients/admin-ui/cypress/support/stubs.ts | 13 ++ clients/admin-ui/package.json | 5 +- .../src/features/common/SearchBar.tsx | 2 +- .../admin-ui/src/features/common/api.slice.ts | 1 + .../src/features/common/nav/v2/nav-config.ts | 7 + .../src/features/common/nav/v2/routes.ts | 1 + .../common/table/v2/PaginationBar.tsx | 4 +- clients/admin-ui/src/features/common/utils.ts | 6 +- .../action-center/DisabledMonitorPage.tsx | 28 +++ .../action-center/EmptyMonitorResult.tsx | 15 ++ .../action-center/MonitorResult.tsx | 97 ++++++++++ .../action-center/actionCenter.slice.tsx | 24 +++ .../action-center/types.ts | 17 ++ .../features/locations/LocationManagement.tsx | 1 - .../locations/RegulationManagement.tsx | 1 - clients/admin-ui/src/flags.json | 6 + .../action-center/[monitorId]/index.tsx | 5 + .../data-discovery/action-center/index.tsx | 176 ++++++++++++++++++ clients/admin-ui/src/theme/global.scss | 14 ++ clients/fidesui/src/index.ts | 6 + clients/package-lock.json | 37 ++-- 24 files changed, 601 insertions(+), 26 deletions(-) create mode 100644 clients/admin-ui/cypress/e2e/action-center.cy.ts create mode 100644 clients/admin-ui/cypress/fixtures/detection-discovery/results/aggregate-results.json create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts create mode 100644 clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx create mode 100644 clients/admin-ui/src/pages/data-discovery/action-center/index.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e7d248787..1862215a98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ## [Unreleased](https://github.com/ethyca/fides/compare/2.52.0...main) ### Added +- Added Action Center MVP behind new feature flag [#5622](https://github.com/ethyca/fides/pull/5622) - Added cache-clearing methods to the `DBCache` model to allow deleting cache entries [#5629](https://github.com/ethyca/fides/pull/5629) ### Fixed @@ -29,7 +30,6 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - Fixed column ordering issue in the Data Map report [#5649](https://github.com/ethyca/fides/pull/5649) - Fixed issue where the Data Map report filter dialog was missing an Accordion item label [#5649](https://github.com/ethyca/fides/pull/5649) - ## [2.52.0](https://github.com/ethyca/fides/compare/2.51.2...2.52.0) ### Added diff --git a/clients/admin-ui/cypress/e2e/action-center.cy.ts b/clients/admin-ui/cypress/e2e/action-center.cy.ts new file mode 100644 index 0000000000..b3541de4a7 --- /dev/null +++ b/clients/admin-ui/cypress/e2e/action-center.cy.ts @@ -0,0 +1,119 @@ +import { stubActionCenter, stubPlus } from "cypress/support/stubs"; + +import { + ACTION_CENTER_ROUTE, + INTEGRATION_MANAGEMENT_ROUTE, +} from "~/features/common/nav/v2/routes"; + +describe("Action center", () => { + beforeEach(() => { + cy.login(); + stubPlus(true); + stubActionCenter(); + }); + + describe("disabled web monitor", () => { + beforeEach(() => { + cy.intercept("GET", "/api/v1/config*", { + body: { + detection_discovery: { + website_monitor_enabled: false, + }, + }, + }).as("getTranslationConfig"); + cy.visit(ACTION_CENTER_ROUTE); + }); + it("should display a message that the web monitor is disabled", () => { + cy.wait("@getTranslationConfig"); + cy.contains("currently disabled").should("exist"); + }); + }); + + describe("empty action center", () => { + beforeEach(() => { + cy.intercept("GET", "/api/v1/plus/discovery-monitor/aggregate-results*", { + fixture: "empty-pagination.json", + }).as("getMonitorResults"); + cy.visit(ACTION_CENTER_ROUTE); + }); + it("should display empty state", () => { + cy.wait("@getMonitorResults"); + cy.get("[data-testid='search-bar']").should("exist"); + cy.get(`[class*='ant-empty'] [class*='ant-empty-image']`).should("exist"); + cy.get( + `[class*='ant-empty'] a[href="${INTEGRATION_MANAGEMENT_ROUTE}"]`, + ).should("exist"); + }); + }); + + describe("Action center monitor results", () => { + const webMonitorKey = "my_web_monitor_2"; + const integrationMonitorKey = "My_New_BQ_Monitor"; + beforeEach(() => { + cy.visit(ACTION_CENTER_ROUTE); + }); + it("should render the current monitor results", () => { + cy.get("[data-testid='Action center']").should("exist"); + cy.wait("@getMonitorResults"); + cy.get("[data-testid*='monitor-result-']").should("have.length", 3); + cy.get("[data-testid^='monitor-result-']").each((result) => { + const monitorKey = result + .attr("data-testid") + .replace("monitor-result-", ""); + // linked title + cy.wrap(result) + .contains("assets detected") + .should("have.attr", "href", `${ACTION_CENTER_ROUTE}/${monitorKey}`); + // last monitored relative date with real date in tooltip + cy.wrap(result) + .find("[data-testid='monitor-date']") + .contains(" ago") + .realHover(); + cy.get(".ant-tooltip-inner").should("contain", "December"); + }); + // description + cy.getByTestId(`monitor-result-${webMonitorKey}`).should( + "contain", + "92 Browser Requests, 5 Cookies detected.", + ); + // monitor name + cy.getByTestId(`monitor-result-${webMonitorKey}`).should( + "contain", + "my web monitor 2", + ); + }); + it("should have appropriate actions for web monitors", () => { + cy.wait("@getMonitorResults"); + // Add button + // TODO: [HJ-337] uncomment when Add button is implemented + // cy.getByTestId(`add-button-${webMonitorKey}`).should("exist"); + // Review button + cy.getByTestId(`review-button-${webMonitorKey}`).should( + "have.attr", + "href", + `${ACTION_CENTER_ROUTE}/${webMonitorKey}`, + ); + }); + it.skip("Should have appropriate actions for Integrations monitors", () => { + cy.wait("@getMonitorResults"); + // Classify button + cy.getByTestId(`review-button-${integrationMonitorKey}`).should( + "have.attr", + "href", + `${ACTION_CENTER_ROUTE}/${integrationMonitorKey}`, + ); + // Ignore button + cy.getByTestId(`ignore-button-${integrationMonitorKey}`).should("exist"); + }); + it.skip("Should have appropriate actions for SSO monitors", () => { + cy.wait("@getMonitorResults"); + // Add button + cy.getByTestId(`add-button-${webMonitorKey}`).should("exist"); + // Ignore button + cy.getByTestId(`ignore-button-${webMonitorKey}`).should("exist"); + }); + it.skip("Should paginate results", () => { + // TODO: mock pagination and also test skeleton loading state + }); + }); +}); diff --git a/clients/admin-ui/cypress/fixtures/detection-discovery/results/aggregate-results.json b/clients/admin-ui/cypress/fixtures/detection-discovery/results/aggregate-results.json new file mode 100644 index 0000000000..0a870f84e1 --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/detection-discovery/results/aggregate-results.json @@ -0,0 +1,40 @@ +{ + "items": [ + { + "name": "my web monitor 2", + "key": "my_web_monitor_2", + "last_monitored": "2024-12-17T17:31:20.791014Z", + "updates": { + "Browser Request": 92, + "Cookie": 5 + }, + "total_updates": 97 + }, + { + "name": "my web monitor 1", + "key": "my_web_monitor_1", + "last_monitored": "2024-12-17T17:31:02.319068Z", + "updates": { + "Browser Request": 201, + "Cookie": 24 + }, + "total_updates": 225 + }, + { + "name": "My New BQ Monitor", + "key": "My_New_BQ_Monitor", + "last_monitored": "2024-12-16T20:04:16.824025Z", + "updates": { + "Database": 2, + "Field": 216, + "Schema": 13, + "Table": 22 + }, + "total_updates": 253 + } + ], + "total": 3, + "page": 1, + "size": 25, + "pages": 1 +} diff --git a/clients/admin-ui/cypress/support/stubs.ts b/clients/admin-ui/cypress/support/stubs.ts index bb2e40537c..2bc49dd917 100644 --- a/clients/admin-ui/cypress/support/stubs.ts +++ b/clients/admin-ui/cypress/support/stubs.ts @@ -502,3 +502,16 @@ export const stubFidesCloud = () => { domain_verification_records: [], }).as("getFidesCloud"); }; + +export const stubActionCenter = () => { + cy.intercept("GET", "/api/v1/config*", { + body: { + detection_discovery: { + website_monitor_enabled: true, + }, + }, + }).as("getTranslationConfig"); + cy.intercept("GET", "/api/v1/plus/discovery-monitor/aggregate-results*", { + fixture: "detection-discovery/results/aggregate-results", + }).as("getMonitorResults"); +}; diff --git a/clients/admin-ui/package.json b/clients/admin-ui/package.json index 1fd75f5903..34dd8bbb4d 100644 --- a/clients/admin-ui/package.json +++ b/clients/admin-ui/package.json @@ -30,6 +30,7 @@ }, "dependencies": { "@ant-design/cssinjs": "^1.21.0", + "@date-fns/tz": "^1.2.0", "@fontsource/inter": "^4.5.15", "@monaco-editor/react": "^4.6.0", "@reduxjs/toolkit": "^1.9.3", @@ -40,8 +41,8 @@ "cytoscape": "^3.30.0", "cytoscape-klay": "^3.1.4", "d3-hierarchy": "^3.1.2", - "date-fns": "^2.29.3", - "date-fns-tz": "^2.0.0", + "date-fns": "^4.1.0", + "date-fns-tz": "^3.2.0", "eslint-plugin-tailwindcss": "^3.17.4", "fides-js": "^0.0.1", "fidesui": "*", diff --git a/clients/admin-ui/src/features/common/SearchBar.tsx b/clients/admin-ui/src/features/common/SearchBar.tsx index 973ca254ed..248d34e44b 100644 --- a/clients/admin-ui/src/features/common/SearchBar.tsx +++ b/clients/admin-ui/src/features/common/SearchBar.tsx @@ -24,7 +24,7 @@ const SearchBar = ({ onChange(event.target.value); return ( - <Space.Compact className="w-96"> + <Space.Compact className="w-96" data-testid="search-bar"> <Input autoComplete="off" value={search} diff --git a/clients/admin-ui/src/features/common/api.slice.ts b/clients/admin-ui/src/features/common/api.slice.ts index 4a6c5a9f1e..971ff2a766 100644 --- a/clients/admin-ui/src/features/common/api.slice.ts +++ b/clients/admin-ui/src/features/common/api.slice.ts @@ -41,6 +41,7 @@ export const baseApi = createApi({ "Languages", "Locations", "Messaging Templates", + "Monitor Summary", "Dictionary", "System Vendors", "Latest Scan", diff --git a/clients/admin-ui/src/features/common/nav/v2/nav-config.ts b/clients/admin-ui/src/features/common/nav/v2/nav-config.ts index 4b5fba1402..ed8ae94d28 100644 --- a/clients/admin-ui/src/features/common/nav/v2/nav-config.ts +++ b/clients/admin-ui/src/features/common/nav/v2/nav-config.ts @@ -38,6 +38,13 @@ export const NAV_CONFIG: NavConfigGroup[] = [ { title: "Detection & Discovery", routes: [ + { + title: "Action center", + path: routes.ACTION_CENTER_ROUTE, + scopes: [], + requiresFlag: "webMonitor", + requiresPlus: true, + }, { title: "Activity", path: routes.DETECTION_DISCOVERY_ACTIVITY_ROUTE, diff --git a/clients/admin-ui/src/features/common/nav/v2/routes.ts b/clients/admin-ui/src/features/common/nav/v2/routes.ts index b043b16fc9..2286ce49a0 100644 --- a/clients/admin-ui/src/features/common/nav/v2/routes.ts +++ b/clients/admin-ui/src/features/common/nav/v2/routes.ts @@ -20,6 +20,7 @@ export const DATASET_COLLECTION_SUBFIELD_DETAIL_ROUTE = "/dataset/[datasetId]/[collectionName]/[...subfieldNames]"; // Detection and discovery +export const ACTION_CENTER_ROUTE = "/data-discovery/action-center"; export const DETECTION_DISCOVERY_ACTIVITY_ROUTE = "/data-discovery/activity"; export const DATA_DETECTION_ROUTE = "/data-discovery/detection"; export const DATA_DETECTION_ROUTE_DETAIL = diff --git a/clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx b/clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx index 49623716db..457e6fbfdc 100644 --- a/clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx +++ b/clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx @@ -44,7 +44,7 @@ export const useServerSidePagination = () => { const defaultPageIndex = 1; const [pageSize, setPageSize] = useState(PAGE_SIZES[0]); const [pageIndex, setPageIndex] = useState<number>(defaultPageIndex); - const [totalPages, setTotalPages] = useState<number | null>(); + const [totalPages, setTotalPages] = useState<number | null | undefined>(1); const onPreviousPageClick = useCallback(() => { setPageIndex((prev) => prev - 1); }, [setPageIndex]); @@ -53,7 +53,7 @@ export const useServerSidePagination = () => { setPageIndex((prev) => prev + 1); }, [setPageIndex]); const isNextPageDisabled = useMemo( - () => pageIndex === totalPages, + () => !!totalPages && (pageIndex === totalPages || totalPages < 2), [pageIndex, totalPages], ); diff --git a/clients/admin-ui/src/features/common/utils.ts b/clients/admin-ui/src/features/common/utils.ts index acc4d86588..ab18bc6803 100644 --- a/clients/admin-ui/src/features/common/utils.ts +++ b/clients/admin-ui/src/features/common/utils.ts @@ -32,7 +32,7 @@ export const debounce = (fn: (props?: any) => void, ms = 0) => { }; export const formatDate = (value: string | number | Date): string => - format(new Date(value), "MMMM d, Y, KK:mm:ss z"); + format(new Date(value), "MMMM d, y, KK:mm:ss z"); export const utf8ToB64 = (str: string): string => window.btoa(unescape(encodeURIComponent(str))); @@ -116,3 +116,7 @@ export const getOptionsFromMap = <T = string>( label: value, value: key, })); + +export const getWebsiteIconUrl = (hostname: string) => { + return `https://icons.duckduckgo.com/ip3/${hostname}.ico`; +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx new file mode 100644 index 0000000000..0cac2e4d62 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx @@ -0,0 +1,28 @@ +import { AntAlert as Alert, AntFlex as Flex, Spinner } from "fidesui"; + +import Layout from "~/features/common/Layout"; + +interface DisabledMonitorPageProps { + isConfigLoading: boolean; +} + +const DISABLED_MONITOR_MESSAGE = "Action center is currently disabled."; + +export const DisabledMonitorPage = ({ + isConfigLoading, +}: DisabledMonitorPageProps) => ( + <Layout title="Action center" mainProps={{ className: "h-full" }}> + <Flex justify="center" align="center" className="h-full"> + {isConfigLoading ? ( + <Spinner color="minos.500" /> + ) : ( + <Alert + message="Coming soon..." + description={DISABLED_MONITOR_MESSAGE} + type="info" + showIcon + /> + )} + </Flex> + </Layout> +); diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx new file mode 100644 index 0000000000..f878a958b8 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx @@ -0,0 +1,15 @@ +import { AntButton as Button, AntEmpty as Empty } from "fidesui"; +import NextLink from "next/link"; + +import { INTEGRATION_MANAGEMENT_ROUTE } from "~/features/common/nav/v2/routes"; + +export const EmptyMonitorResult = () => ( + <Empty + image={Empty.PRESENTED_IMAGE_SIMPLE} + description="All caught up! Set up an integration monitor to track your infrastructure in greater detail." + > + <NextLink href={INTEGRATION_MANAGEMENT_ROUTE} passHref legacyBehavior> + <Button type="primary">Visit integrations</Button> + </NextLink> + </Empty> +); diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx new file mode 100644 index 0000000000..24c8c49ac4 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx @@ -0,0 +1,97 @@ +import { formatDistance } from "date-fns"; +import { + AntAvatar as Avatar, + AntFlex as Flex, + AntList as List, + AntListItemProps as ListItemProps, + AntSkeleton as Skeleton, + AntTooltip as Tooltip, + AntTypography as Typography, + Icons, +} from "fidesui"; +import NextLink from "next/link"; + +import { ACTION_CENTER_ROUTE } from "~/features/common/nav/v2/routes"; +import { formatDate, getWebsiteIconUrl } from "~/features/common/utils"; + +import { MonitorSummary } from "./types"; + +const { Text } = Typography; + +interface MonitorResultProps extends ListItemProps { + monitorSummary: MonitorSummary; + showSkeleton?: boolean; +} + +export const MonitorResult = ({ + monitorSummary, + showSkeleton, + ...props +}: MonitorResultProps) => { + if (!monitorSummary) { + return null; + } + + const { + name, + property, + total_updates: totalUpdates, + updates, + last_monitored: lastMonitored, + warning, + key, + } = monitorSummary; + + const assetCountString = Object.entries(updates) + .map((update) => { + return `${update[1]} ${update[0]}s`; + }) + .join(", "); + + const lastMonitoredDistance = lastMonitored + ? formatDistance(new Date(lastMonitored), new Date(), { + addSuffix: true, + }) + : undefined; + + const iconUrl = property ? getWebsiteIconUrl(property) : undefined; + + return ( + <List.Item data-testid={`monitor-result-${key}`} {...props}> + <Skeleton avatar title={false} loading={showSkeleton} active> + <List.Item.Meta + avatar={!!iconUrl && <Avatar src={iconUrl} size="small" />} + title={ + <NextLink + href={`${ACTION_CENTER_ROUTE}/${key}`} + className="whitespace-nowrap" + > + {`${totalUpdates} assets detected${property ? `on ${property}` : ""}`} + {!!warning && ( + <Tooltip + title={typeof warning === "string" ? warning : undefined} + > + <Icons.WarningAltFilled + className="ml-1 inline-block align-middle" + style={{ color: "var(--fidesui-error)" }} + /> + </Tooltip> + )} + </NextLink> + } + description={`${assetCountString} detected.`} + /> + <Flex className="gap-12"> + <Text style={{ maxWidth: 300 }} ellipsis={{ tooltip: name }}> + {name} + </Text> + {!!lastMonitoredDistance && ( + <Tooltip title={formatDate(lastMonitored)}> + <Text data-testid="monitor-date">{lastMonitoredDistance}</Text> + </Tooltip> + )} + </Flex> + </Skeleton> + </List.Item> + ); +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx new file mode 100644 index 0000000000..6d217a0c4b --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx @@ -0,0 +1,24 @@ +import { baseApi } from "~/features/common/api.slice"; + +import { MonitorSummaryPaginatedResponse } from "./types"; + +const actionCenterApi = baseApi.injectEndpoints({ + endpoints: (build) => ({ + getMonitorSummary: build.query< + MonitorSummaryPaginatedResponse, + { + pageIndex?: number; + pageSize?: number; + search?: string; + } + >({ + query: ({ pageIndex = 1, pageSize = 20, search }) => ({ + url: `/plus/discovery-monitor/aggregate-results`, + params: { page: pageIndex, size: pageSize, search }, + }), + providesTags: ["Monitor Summary"], + }), + }), +}); + +export const { useGetMonitorSummaryQuery } = actionCenterApi; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts new file mode 100644 index 0000000000..e33d824d58 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts @@ -0,0 +1,17 @@ +// TODO: [HJ-334] remove these in favor of autogenerated types from the API +export interface MonitorSummary { + updates: Record<string, number>; + property?: string; + last_monitored: string | number; + key: string; + name: string; + total_updates: number; + warning?: boolean | string; +} + +export interface MonitorSummaryPaginatedResponse { + items: MonitorSummary[]; + page: number; + size: number; + total: number; +} diff --git a/clients/admin-ui/src/features/locations/LocationManagement.tsx b/clients/admin-ui/src/features/locations/LocationManagement.tsx index 40931434d1..ac244ebf8e 100644 --- a/clients/admin-ui/src/features/locations/LocationManagement.tsx +++ b/clients/admin-ui/src/features/locations/LocationManagement.tsx @@ -98,7 +98,6 @@ const LocationManagement = ({ data }: { data: LocationRegulationResponse }) => { placeholder="Search" search={search} onClear={() => setSearch("")} - data-testid="search-bar" /> </Box> <SimpleGrid columns={{ base: 1, md: 2, xl: 3 }} spacing={6} width="100%"> diff --git a/clients/admin-ui/src/features/locations/RegulationManagement.tsx b/clients/admin-ui/src/features/locations/RegulationManagement.tsx index 795d2b77ff..980801e3fc 100644 --- a/clients/admin-ui/src/features/locations/RegulationManagement.tsx +++ b/clients/admin-ui/src/features/locations/RegulationManagement.tsx @@ -103,7 +103,6 @@ const RegulationManagement = ({ placeholder="Search" search={search} onClear={() => setSearch("")} - data-testid="search-bar" /> </Box> <SimpleGrid columns={{ base: 1, md: 2, xl: 3 }} spacing={6} width="100%"> diff --git a/clients/admin-ui/src/flags.json b/clients/admin-ui/src/flags.json index f33a2e81ee..5f2b32cea2 100644 --- a/clients/admin-ui/src/flags.json +++ b/clients/admin-ui/src/flags.json @@ -36,6 +36,12 @@ "test": true, "production": false }, + "webMonitor": { + "description": "Monitor websites for activity", + "development": true, + "test": true, + "production": false + }, "ssoAuthentication": { "description": "SSO Authentication Providers (OpenID)", "development": true, diff --git a/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx b/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx new file mode 100644 index 0000000000..6f07a74600 --- /dev/null +++ b/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx @@ -0,0 +1,5 @@ +const MonitorResultSystems = () => { + return <div>Monitor Result Systems FPO</div>; +}; + +export default MonitorResultSystems; diff --git a/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx b/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx new file mode 100644 index 0000000000..6edb23321c --- /dev/null +++ b/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx @@ -0,0 +1,176 @@ +import { + AntButton as Button, + AntDivider as Divider, + AntFlex as Flex, + AntList as List, + useToast, +} from "fidesui"; +import NextLink from "next/link"; +import { useCallback, useEffect, useState } from "react"; + +import Layout from "~/features/common/Layout"; +import { ACTION_CENTER_ROUTE } from "~/features/common/nav/v2/routes"; +import PageHeader from "~/features/common/PageHeader"; +import { + PaginationBar, + useServerSidePagination, +} from "~/features/common/table/v2"; +import { useGetMonitorSummaryQuery } from "~/features/data-discovery-and-detection/action-center/actionCenter.slice"; +import { DisabledMonitorPage } from "~/features/data-discovery-and-detection/action-center/DisabledMonitorPage"; +import { EmptyMonitorResult } from "~/features/data-discovery-and-detection/action-center/EmptyMonitorResult"; +import { MonitorResult } from "~/features/data-discovery-and-detection/action-center/MonitorResult"; +import { MonitorSummary } from "~/features/data-discovery-and-detection/action-center/types"; +import { SearchInput } from "~/features/data-discovery-and-detection/SearchInput"; +import { useGetConfigurationSettingsQuery } from "~/features/privacy-requests"; + +const ActionCenterPage = () => { + const toast = useToast(); + const { + PAGE_SIZES, + pageSize, + setPageSize, + onPreviousPageClick, + isPreviousPageDisabled, + onNextPageClick, + isNextPageDisabled, + startRange, + endRange, + pageIndex, + setTotalPages, + resetPageIndexToDefault, + } = useServerSidePagination(); + const [searchQuery, setSearchQuery] = useState(""); + const { data: appConfig, isLoading: isConfigLoading } = + useGetConfigurationSettingsQuery({ + api_set: false, + }); + const webMonitorEnabled = + !!appConfig?.detection_discovery?.website_monitor_enabled; + + useEffect(() => { + resetPageIndexToDefault(); + }, [searchQuery, resetPageIndexToDefault]); + + const { data, isError, isLoading, isFetching } = useGetMonitorSummaryQuery( + { + pageIndex, + pageSize, + search: searchQuery, + }, + { skip: isConfigLoading || !webMonitorEnabled }, + ); + + useEffect(() => { + if (isError && !!toast && webMonitorEnabled) { + toast({ + title: "Error fetching data", + description: "Please try again later", + status: "error", + }); + } + }, [isError, toast, webMonitorEnabled]); + + useEffect(() => { + if (data) { + setTotalPages(data.total || 1); + } + }, [data, setTotalPages]); + + const results = data?.items || []; + const loadingResults = isFetching + ? (Array.from({ length: pageSize }, (_, index) => ({ + key: index.toString(), + updates: [], + last_monitored: null, + })) as any[]) + : []; + + // TODO: [HJ-337] Add button functionality + + // const handleAdd = (monidorId: string) => { + // console.log("Add report", monidorId); + // }; + + const getWebsiteMonitorActions = useCallback( + (monitorKey: string) => [ + // <Button + // key="add" + // type="link" + // className="p-0" + // onClick={() => { + // handleAdd(monitorKey); + // }} + // data-testid={`add-button-${monitorKey}`} + // > + // Add + // </Button>, + <NextLink + key="review" + href={`${ACTION_CENTER_ROUTE}/${monitorKey}`} + passHref + legacyBehavior + > + <Button + type="link" + className="p-0" + data-testid={`review-button-${monitorKey}`} + > + Review + </Button> + </NextLink>, + ], + [], + ); + + if (!webMonitorEnabled) { + return <DisabledMonitorPage isConfigLoading={isConfigLoading} />; + } + + return ( + <Layout title="Action center"> + <PageHeader + heading="Action center" + breadcrumbItems={[{ title: "All activity" }]} + /> + + <Flex className="justify-between py-6"> + <SearchInput value={searchQuery} onChange={setSearchQuery} /> + </Flex> + + <List + loading={isLoading} + dataSource={results || loadingResults} + locale={{ + emptyText: <EmptyMonitorResult />, + }} + renderItem={(summary: MonitorSummary) => ( + <MonitorResult + showSkeleton={isFetching} + key={summary.key} + monitorSummary={summary} + actions={getWebsiteMonitorActions(summary.key)} // TODO: when monitor type becomes available, use it to determine actions. Defaulting to website monitor actions for now. + /> + )} + /> + + {!!results && !!data?.total && data.total > pageSize && ( + <> + <Divider className="mb-6 mt-0" /> + <PaginationBar + totalRows={data?.total || 0} + pageSizes={PAGE_SIZES} + setPageSize={setPageSize} + onPreviousPageClick={onPreviousPageClick} + isPreviousPageDisabled={isPreviousPageDisabled || isFetching} + onNextPageClick={onNextPageClick} + isNextPageDisabled={isNextPageDisabled || isFetching} + startRange={startRange} + endRange={endRange} + /> + </> + )} + </Layout> + ); +}; + +export default ActionCenterPage; diff --git a/clients/admin-ui/src/theme/global.scss b/clients/admin-ui/src/theme/global.scss index 5c7aa053e1..d4357518ce 100644 --- a/clients/admin-ui/src/theme/global.scss +++ b/clients/admin-ui/src/theme/global.scss @@ -1,5 +1,19 @@ @import "fidesui/src/palette/palette.module.scss"; +/** + * Chakra removes heading font weight, wheras Ant assumes browser defaults. + * This sets the font weight for headings back to the browser default for Ant support. + * Remove this once Chakra has been removed. + */ +h1, +h2, +h3, +h4, +h5, +h6 { + font-weight: bold; +} + /** * Adds the color variables from the palette to the root element */ diff --git a/clients/fidesui/src/index.ts b/clients/fidesui/src/index.ts index 6179fa6da0..25652dce60 100644 --- a/clients/fidesui/src/index.ts +++ b/clients/fidesui/src/index.ts @@ -11,6 +11,7 @@ export type { FlexProps as AntFlexProps, FormInstance as AntFormInstance, InputProps as AntInputProps, + ListProps as AntListProps, SelectProps as AntSelectProps, SwitchProps as AntSwitchProps, GetProps, @@ -18,19 +19,23 @@ export type { } from "antd/lib"; export { Alert as AntAlert, + Avatar as AntAvatar, Breadcrumb as AntBreadcrumb, Button as AntButton, Card as AntCard, Checkbox as AntCheckbox, Col as AntCol, Divider as AntDivider, + Empty as AntEmpty, Flex as AntFlex, Form as AntForm, Input as AntInput, Layout as AntLayout, + List as AntList, Menu as AntMenu, Radio as AntRadio, Row as AntRow, + Skeleton as AntSkeleton, Space as AntSpace, Switch as AntSwitch, Tag as AntTag, @@ -41,6 +46,7 @@ export type { BreadcrumbItemType as AntBreadcrumbItemType, BreadcrumbProps as AntBreadcrumbProps, } from "antd/lib/breadcrumb/Breadcrumb"; +export type { ListItemProps as AntListItemProps } from "antd/lib/list"; export type { BaseOptionType as AntBaseOptionType, DefaultOptionType as AntDefaultOptionType, diff --git a/clients/package-lock.json b/clients/package-lock.json index 62d4f02527..b3a00b04f1 100644 --- a/clients/package-lock.json +++ b/clients/package-lock.json @@ -18,6 +18,7 @@ "admin-ui": { "dependencies": { "@ant-design/cssinjs": "^1.21.0", + "@date-fns/tz": "^1.2.0", "@fontsource/inter": "^4.5.15", "@monaco-editor/react": "^4.6.0", "@reduxjs/toolkit": "^1.9.3", @@ -28,8 +29,8 @@ "cytoscape": "^3.30.0", "cytoscape-klay": "^3.1.4", "d3-hierarchy": "^3.1.2", - "date-fns": "^2.29.3", - "date-fns-tz": "^2.0.0", + "date-fns": "^4.1.0", + "date-fns-tz": "^3.2.0", "eslint-plugin-tailwindcss": "^3.17.4", "fides-js": "^0.0.1", "fidesui": "*", @@ -3058,6 +3059,12 @@ "ms": "^2.1.1" } }, + "node_modules/@date-fns/tz": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@date-fns/tz/-/tz-1.2.0.tgz", + "integrity": "sha512-LBrd7MiJZ9McsOgxqWX7AaxrDjcFVjWH/tIKJd7pnR7McaslGYOP1QmmiBXdJH/H/yLCT+rcQ7FaPBUxRGUtrg==", + "license": "MIT" + }, "node_modules/@emotion/babel-plugin": { "version": "11.11.0", "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz", @@ -10328,26 +10335,22 @@ } }, "node_modules/date-fns": { - "version": "2.30.0", - "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", - "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", - "dependencies": { - "@babel/runtime": "^7.21.0" - }, - "engines": { - "node": ">=0.11" - }, + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-4.1.0.tgz", + "integrity": "sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==", + "license": "MIT", "funding": { - "type": "opencollective", - "url": "https://opencollective.com/date-fns" + "type": "github", + "url": "https://github.com/sponsors/kossnocorp" } }, "node_modules/date-fns-tz": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-2.0.1.tgz", - "integrity": "sha512-fJCG3Pwx8HUoLhkepdsP7Z5RsucUi+ZBOxyM5d0ZZ6c4SdYustq0VMmOu6Wf7bli+yS/Jwp91TOCqn9jMcVrUA==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-3.2.0.tgz", + "integrity": "sha512-sg8HqoTEulcbbbVXeg84u5UnlsQa8GS5QXMqjjYIhS4abEVVKIUwe0/l/UhrZdKaL/W5eWZNlbTeEIiOXTcsBQ==", + "license": "MIT", "peerDependencies": { - "date-fns": "2.x" + "date-fns": "^3.0.0 || ^4.0.0" } }, "node_modules/dayjs": { From a583252764b5507df81740882eba24a4a46ca898 Mon Sep 17 00:00:00 2001 From: Jason Gill <jason.gill@ethyca.com> Date: Thu, 9 Jan 2025 12:02:51 -0700 Subject: [PATCH 3/9] Action Center: View discovered System Aggregate Results (#5653) --- .../admin-ui/cypress/e2e/action-center.cy.ts | 58 +++++++++- .../aggregate-results.json | 0 .../system-aggregate-results.json | 84 +++++++++++++++ clients/admin-ui/cypress/support/stubs.ts | 9 +- .../admin-ui/src/features/common/api.slice.ts | 1 - .../features/common/custom-fields/Layout.tsx | 16 --- .../features/common/custom-fields/index.ts | 1 - ...nitorPage.tsx => DisabledMonitorsPage.tsx} | 10 +- ...itorResult.tsx => EmptyMonitorsResult.tsx} | 2 +- .../action-center/MonitorResult.tsx | 15 +-- .../action-center/action-center.slice.ts | 48 +++++++++ .../action-center/actionCenter.slice.tsx | 24 ----- .../useDiscoveredSystemAggregateColumns.tsx | 75 +++++++++++++ .../tables/DiscoveredSystemAggregateTable.tsx | 101 ++++++++++++++++++ .../DiscoveredSystemAggregateActionsCell.tsx | 14 +++ .../DiscoveredSystemAggregateStatusCell.tsx | 33 ++++++ .../action-center/types.ts | 24 +++-- .../ConnectionTypeLogo.tsx | 9 +- .../{types.ts => types.d.ts} | 11 -- .../action-center/[monitorId]/index.tsx | 26 ++++- .../data-discovery/action-center/index.tsx | 49 +++++---- .../src/types/common/PaginationQueryParams.ts | 8 ++ 22 files changed, 519 insertions(+), 99 deletions(-) rename clients/admin-ui/cypress/fixtures/detection-discovery/{results => activity-center}/aggregate-results.json (100%) create mode 100644 clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json delete mode 100644 clients/admin-ui/src/features/common/custom-fields/Layout.tsx rename clients/admin-ui/src/features/data-discovery-and-detection/action-center/{DisabledMonitorPage.tsx => DisabledMonitorsPage.tsx} (68%) rename clients/admin-ui/src/features/data-discovery-and-detection/action-center/{EmptyMonitorResult.tsx => EmptyMonitorsResult.tsx} (92%) create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts delete mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateActionsCell.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateStatusCell.tsx rename clients/admin-ui/src/features/datastore-connections/{types.ts => types.d.ts} (91%) diff --git a/clients/admin-ui/cypress/e2e/action-center.cy.ts b/clients/admin-ui/cypress/e2e/action-center.cy.ts index b3541de4a7..1f816ae3b3 100644 --- a/clients/admin-ui/cypress/e2e/action-center.cy.ts +++ b/clients/admin-ui/cypress/e2e/action-center.cy.ts @@ -46,7 +46,7 @@ describe("Action center", () => { }); }); - describe("Action center monitor results", () => { + describe("Action center monitor aggregate results", () => { const webMonitorKey = "my_web_monitor_2"; const integrationMonitorKey = "My_New_BQ_Monitor"; beforeEach(() => { @@ -116,4 +116,60 @@ describe("Action center", () => { // TODO: mock pagination and also test skeleton loading state }); }); + + describe("Action center system aggregate results", () => { + const webMonitorKey = "my_web_monitor_1"; + beforeEach(() => { + cy.visit(`${ACTION_CENTER_ROUTE}/${webMonitorKey}`); + }); + it("should display a breadcrumb", () => { + cy.getByTestId("page-breadcrumb").within(() => { + cy.get("a.ant-breadcrumb-link") + .should("contain", "All activity") + .should("have.attr", "href", ACTION_CENTER_ROUTE); + cy.contains("my_web_monitor_1").should("exist"); + }); + }); + it("should render the aggregated system results in a table", () => { + cy.wait("@getSystemAggregateResults"); + cy.getByTestId("column-system_name").should("exist"); + cy.getByTestId("column-total_updates").should("exist"); + cy.getByTestId("column-data_use").should("exist"); + cy.getByTestId("column-locations").should("exist"); + cy.getByTestId("column-domains").should("exist"); + cy.getByTestId("column-actions").should("exist"); + cy.getByTestId("search-bar").should("exist"); + cy.getByTestId("pagination-btn").should("exist"); + cy.getByTestId("row-0-col-system_name").within(() => { + cy.getByTestId("change-icon").should("exist"); // new result + cy.contains("Uncategorized assets").should("exist"); + }); + // data use column should be empty for uncategorized assets + cy.getByTestId("row-0-col-data_use").children().should("have.length", 0); + cy.getByTestId("row-1-col-system_name").within(() => { + cy.getByTestId("change-icon").should("not.exist"); // existing result + cy.contains("Google Tag Manager").should("exist"); + }); + // TODO: data use column should not be empty for other assets + // cy.getByTestId("row-1-col-data_use").children().should("not.have.length", 0); + + // multiple locations + cy.getByTestId("row-2-col-locations").should("contain", "2 locations"); + // single location + cy.getByTestId("row-3-col-locations").should("contain", "USA"); + + // multiple domains + cy.getByTestId("row-0-col-domains").should("contain", "29 domains"); + // single domain + cy.getByTestId("row-3-col-domains").should( + "contain", + "analytics.google.com", + ); + }); + // it("should navigate to table view on row click", () => { + // cy.getByTestId("row-1").click(); + // cy.url().should("contain", "fds.1046"); + // cy.getByTestId("page-breadcrumb").should("contain", "fds.1046"); + // }); + }); }); diff --git a/clients/admin-ui/cypress/fixtures/detection-discovery/results/aggregate-results.json b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/aggregate-results.json similarity index 100% rename from clients/admin-ui/cypress/fixtures/detection-discovery/results/aggregate-results.json rename to clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/aggregate-results.json diff --git a/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json new file mode 100644 index 0000000000..3bcc4b2328 --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json @@ -0,0 +1,84 @@ +{ + "items": [ + { + "id": null, + "name": null, + "system_key": null, + "vendor_id": null, + "total_updates": 108, + "locations": ["USA"], + "domains": [ + "alb.reddit.com", + "api.hubapi.com", + "app.revenuehero.io", + ".ethyca.com", + "ethyca.com", + "ethyca.fides-cdn.ethyca.com", + "forms.hscollectedforms.net", + "forms.hubspot.com", + "forms-na1.hsforms.com", + "googleads.g.doubleclick.net", + ".hsadspixel.net", + ".hsforms.com", + ".hs-scripts.com", + ".hubspot.com", + "js.hsadspixel.net", + "js.hs-analytics.net", + "js.hs-banner.com", + "js.hscollectedforms.net", + "js.hs-scripts.com", + "kit.fontawesome.com", + ".linkedin.com", + "pixel-config.reddit.com", + "px.ads.linkedin.com", + "snap.licdn.com", + "stats.g.doubleclick.net", + "track.hubspot.com", + "www.clickcease.com", + ".www.linkedin.com", + "www.redditstatic.com" + ] + }, + { + "id": "system_key-72649f03-7a30-4758-9772-e74fca3b6788", + "name": "Google Tag Manager", + "system_key": "system_key-72649f03-7a30-4758-9772-e74fca3b6788", + "vendor_id": "fds.1046", + "total_updates": 10, + "locations": ["USA"], + "domains": [ + "td.doubleclick.net", + "www.google.com", + "www.googletagmanager.com" + ] + }, + { + "id": "system_key-652c8984-ade7-470b-bce4-7e184621be9d", + "name": "Hubspot", + "system_key": "system_key-652c8984-ade7-470b-bce4-7e184621be9d", + "vendor_id": "fds.1053", + "total_updates": 6, + "locations": ["USA", "Canada"], + "domains": [ + "forms.hsforms.com", + ".hs-analytics.net", + ".hs-banner.com", + ".hsforms.net", + "js.hsforms.net" + ] + }, + { + "id": "fds.1047", + "name": "Google Analytics", + "system_key": null, + "vendor_id": "fds.1047", + "total_updates": 1, + "locations": ["USA"], + "domains": ["analytics.google.com"] + } + ], + "total": 4, + "page": 1, + "size": 25, + "pages": 1 +} diff --git a/clients/admin-ui/cypress/support/stubs.ts b/clients/admin-ui/cypress/support/stubs.ts index 2bc49dd917..6c3109f1e9 100644 --- a/clients/admin-ui/cypress/support/stubs.ts +++ b/clients/admin-ui/cypress/support/stubs.ts @@ -512,6 +512,13 @@ export const stubActionCenter = () => { }, }).as("getTranslationConfig"); cy.intercept("GET", "/api/v1/plus/discovery-monitor/aggregate-results*", { - fixture: "detection-discovery/results/aggregate-results", + fixture: "detection-discovery/activity-center/aggregate-results", }).as("getMonitorResults"); + cy.intercept( + "GET", + "/api/v1//plus/discovery-monitor/system-aggregate-results*", + { + fixture: "detection-discovery/activity-center/system-aggregate-results", + }, + ).as("getSystemAggregateResults"); }; diff --git a/clients/admin-ui/src/features/common/api.slice.ts b/clients/admin-ui/src/features/common/api.slice.ts index 971ff2a766..4a6c5a9f1e 100644 --- a/clients/admin-ui/src/features/common/api.slice.ts +++ b/clients/admin-ui/src/features/common/api.slice.ts @@ -41,7 +41,6 @@ export const baseApi = createApi({ "Languages", "Locations", "Messaging Templates", - "Monitor Summary", "Dictionary", "System Vendors", "Latest Scan", diff --git a/clients/admin-ui/src/features/common/custom-fields/Layout.tsx b/clients/admin-ui/src/features/common/custom-fields/Layout.tsx deleted file mode 100644 index 6e3cb005f9..0000000000 --- a/clients/admin-ui/src/features/common/custom-fields/Layout.tsx +++ /dev/null @@ -1,16 +0,0 @@ -import { StackProps, VStack } from "fidesui"; -import * as React from "react"; - -const Layout = ({ children, ...props }: StackProps) => ( - <VStack - alignItems="stretch" - flexShrink={0} - gap="18px" - overflow="auto" - {...props} - > - {children} - </VStack> -); - -export { Layout }; diff --git a/clients/admin-ui/src/features/common/custom-fields/index.ts b/clients/admin-ui/src/features/common/custom-fields/index.ts index 90342f6b0e..c1f980165d 100644 --- a/clients/admin-ui/src/features/common/custom-fields/index.ts +++ b/clients/admin-ui/src/features/common/custom-fields/index.ts @@ -2,5 +2,4 @@ export * from "./constants"; export * from "./CustomFieldsList"; export * from "./helpers"; export * from "./hooks"; -export * from "./Layout"; export * from "./types"; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorsPage.tsx similarity index 68% rename from clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx rename to clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorsPage.tsx index 0cac2e4d62..cb80d6c6ba 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorsPage.tsx @@ -2,15 +2,15 @@ import { AntAlert as Alert, AntFlex as Flex, Spinner } from "fidesui"; import Layout from "~/features/common/Layout"; -interface DisabledMonitorPageProps { +interface DisabledMonitorsPageProps { isConfigLoading: boolean; } -const DISABLED_MONITOR_MESSAGE = "Action center is currently disabled."; +const DISABLED_MONITORS_MESSAGE = "Action center is currently disabled."; -export const DisabledMonitorPage = ({ +export const DisabledMonitorsPage = ({ isConfigLoading, -}: DisabledMonitorPageProps) => ( +}: DisabledMonitorsPageProps) => ( <Layout title="Action center" mainProps={{ className: "h-full" }}> <Flex justify="center" align="center" className="h-full"> {isConfigLoading ? ( @@ -18,7 +18,7 @@ export const DisabledMonitorPage = ({ ) : ( <Alert message="Coming soon..." - description={DISABLED_MONITOR_MESSAGE} + description={DISABLED_MONITORS_MESSAGE} type="info" showIcon /> diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorsResult.tsx similarity index 92% rename from clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx rename to clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorsResult.tsx index f878a958b8..dfd82237fb 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorsResult.tsx @@ -3,7 +3,7 @@ import NextLink from "next/link"; import { INTEGRATION_MANAGEMENT_ROUTE } from "~/features/common/nav/v2/routes"; -export const EmptyMonitorResult = () => ( +export const EmptyMonitorsResult = () => ( <Empty image={Empty.PRESENTED_IMAGE_SIMPLE} description="All caught up! Set up an integration monitor to track your infrastructure in greater detail." diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx index 24c8c49ac4..3614217ffe 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx @@ -10,16 +10,17 @@ import { Icons, } from "fidesui"; import NextLink from "next/link"; +import { useEffect, useState } from "react"; import { ACTION_CENTER_ROUTE } from "~/features/common/nav/v2/routes"; import { formatDate, getWebsiteIconUrl } from "~/features/common/utils"; -import { MonitorSummary } from "./types"; +import { MonitorAggregatedResults } from "./types"; const { Text } = Typography; interface MonitorResultProps extends ListItemProps { - monitorSummary: MonitorSummary; + monitorSummary: MonitorAggregatedResults; showSkeleton?: boolean; } @@ -28,9 +29,7 @@ export const MonitorResult = ({ showSkeleton, ...props }: MonitorResultProps) => { - if (!monitorSummary) { - return null; - } + const [iconUrl, setIconUrl] = useState<string | undefined>(undefined); const { name, @@ -54,7 +53,11 @@ export const MonitorResult = ({ }) : undefined; - const iconUrl = property ? getWebsiteIconUrl(property) : undefined; + useEffect(() => { + if (property) { + setIconUrl(getWebsiteIconUrl(property)); + } + }, [property]); return ( <List.Item data-testid={`monitor-result-${key}`} {...props}> diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts new file mode 100644 index 0000000000..5e25656721 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts @@ -0,0 +1,48 @@ +import { baseApi } from "~/features/common/api.slice"; +import { PaginationQueryParams } from "~/types/common/PaginationQueryParams"; + +import { + MonitorSummaryPaginatedResponse, + MonitorSystemAggregatePaginatedResponse, +} from "./types"; + +const actionCenterApi = baseApi.injectEndpoints({ + endpoints: (build) => ({ + getAggregateMonitorResults: build.query< + MonitorSummaryPaginatedResponse, + { + search?: string; + } & PaginationQueryParams + >({ + query: ({ page = 1, size = 20, search }) => ({ + url: `/plus/discovery-monitor/aggregate-results`, + params: { page, size, search, diff_status: "addition" }, + }), + providesTags: ["Discovery Monitor Results"], + }), + getDiscoveredSystemAggregate: build.query< + MonitorSystemAggregatePaginatedResponse, + { + key: string; + search?: string; + } & PaginationQueryParams + >({ + query: ({ key, page = 1, size = 20, search }) => ({ + url: `/plus/discovery-monitor/system-aggregate-results`, + params: { + monitor_config_id: key, + page, + size, + search, + diff_status: "addition", + }, + }), + providesTags: ["Discovery Monitor Results"], + }), + }), +}); + +export const { + useGetAggregateMonitorResultsQuery, + useGetDiscoveredSystemAggregateQuery, +} = actionCenterApi; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx deleted file mode 100644 index 6d217a0c4b..0000000000 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx +++ /dev/null @@ -1,24 +0,0 @@ -import { baseApi } from "~/features/common/api.slice"; - -import { MonitorSummaryPaginatedResponse } from "./types"; - -const actionCenterApi = baseApi.injectEndpoints({ - endpoints: (build) => ({ - getMonitorSummary: build.query< - MonitorSummaryPaginatedResponse, - { - pageIndex?: number; - pageSize?: number; - search?: string; - } - >({ - query: ({ pageIndex = 1, pageSize = 20, search }) => ({ - url: `/plus/discovery-monitor/aggregate-results`, - params: { page: pageIndex, size: pageSize, search }, - }), - providesTags: ["Monitor Summary"], - }), - }), -}); - -export const { useGetMonitorSummaryQuery } = actionCenterApi; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx new file mode 100644 index 0000000000..747ab1ef6c --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx @@ -0,0 +1,75 @@ +import { createColumnHelper } from "@tanstack/react-table"; + +import { DefaultCell } from "~/features/common/table/v2"; + +import { DiscoveredSystemActionsCell } from "../tables/cells/DiscoveredSystemAggregateActionsCell"; +import { DiscoveredSystemStatusCell } from "../tables/cells/DiscoveredSystemAggregateStatusCell"; +import { MonitorSystemAggregate } from "../types"; + +export const useDiscoveredSystemAggregateColumns = () => { + const columnHelper = createColumnHelper<MonitorSystemAggregate>(); + + const columns = [ + columnHelper.accessor((row) => row.name, { + id: "system_name", + cell: (props) => ( + <DiscoveredSystemStatusCell system={props.row.original} /> + ), + header: "System", + meta: { + width: "auto", + }, + }), + columnHelper.accessor((row) => row.total_updates, { + id: "total_updates", + cell: (props) => <DefaultCell value={props.getValue()} />, + header: "Assets", + size: 80, + }), + columnHelper.display({ + id: "data_use", + header: "Categories of consent", + meta: { + width: "auto", + }, + }), + columnHelper.accessor((row) => row.locations, { + id: "locations", + cell: (props) => ( + <DefaultCell + value={ + props.getValue().length > 1 + ? `${props.getValue().length} locations` + : props.getValue()[0] + } + /> + ), + header: "Locations", + }), + columnHelper.accessor((row) => row.domains, { + id: "domains", + cell: (props) => ( + <DefaultCell + value={ + props.getValue().length > 1 + ? `${props.getValue().length} domains` + : props.getValue()[0] + } + /> + ), + header: "Domains", + }), + columnHelper.display({ + id: "actions", + cell: (props) => ( + <DiscoveredSystemActionsCell system={props.row.original} /> + ), + header: "Actions", + meta: { + width: "auto", + }, + }), + ]; + + return { columns }; +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx new file mode 100644 index 0000000000..01d4b8c4d5 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx @@ -0,0 +1,101 @@ +import { getCoreRowModel, useReactTable } from "@tanstack/react-table"; +import { Box, Flex } from "fidesui"; +import { useEffect, useState } from "react"; + +import { + FidesTableV2, + PaginationBar, + TableActionBar, + TableSkeletonLoader, + useServerSidePagination, +} from "~/features/common/table/v2"; +import { useGetDiscoveredSystemAggregateQuery } from "~/features/data-discovery-and-detection/action-center/action-center.slice"; + +import { SearchInput } from "../../SearchInput"; +import { useDiscoveredSystemAggregateColumns } from "../hooks/useDiscoveredSystemAggregateColumns"; + +interface DiscoveredSystemAggregateTableProps { + monitorId: string; +} + +export const DiscoveredSystemAggregateTable = ({ + monitorId, +}: DiscoveredSystemAggregateTableProps) => { + const { + PAGE_SIZES, + pageSize, + setPageSize, + onPreviousPageClick, + isPreviousPageDisabled, + onNextPageClick, + isNextPageDisabled, + startRange, + endRange, + pageIndex, + setTotalPages, + resetPageIndexToDefault, + } = useServerSidePagination(); + const [searchQuery, setSearchQuery] = useState(""); + + useEffect(() => { + resetPageIndexToDefault(); + }, [monitorId, searchQuery, resetPageIndexToDefault]); + + const { data, isLoading, isFetching } = useGetDiscoveredSystemAggregateQuery({ + key: monitorId, + page: pageIndex, + size: pageSize, + search: searchQuery, + }); + + useEffect(() => { + if (data) { + setTotalPages(data.pages || 1); + } + }, [data, setTotalPages]); + + const { columns } = useDiscoveredSystemAggregateColumns(); + + const tableInstance = useReactTable({ + getCoreRowModel: getCoreRowModel(), + columns, + manualPagination: true, + data: data?.items || [], + columnResizeMode: "onChange", + }); + + if (isLoading) { + return <TableSkeletonLoader rowHeight={36} numRows={36} />; + } + + return ( + <> + <TableActionBar> + <Flex + direction="row" + alignItems="center" + justifyContent="space-between" + width="full" + > + <Flex gap={6} align="center"> + <Box flexShrink={0}> + <SearchInput value={searchQuery} onChange={setSearchQuery} /> + </Box> + </Flex> + </Flex> + </TableActionBar> + <FidesTableV2 tableInstance={tableInstance} /> + <PaginationBar + totalRows={data?.items.length || 0} + pageSizes={PAGE_SIZES} + setPageSize={setPageSize} + onPreviousPageClick={onPreviousPageClick} + isPreviousPageDisabled={isPreviousPageDisabled || isFetching} + onNextPageClick={onNextPageClick} + isNextPageDisabled={isNextPageDisabled || isFetching} + startRange={startRange} + endRange={endRange} + /> + </> + ); +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateActionsCell.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateActionsCell.tsx new file mode 100644 index 0000000000..d00127b884 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateActionsCell.tsx @@ -0,0 +1,14 @@ +import { AntFlex as Flex } from "fidesui"; + +import { MonitorSystemAggregate } from "../../types"; + +interface DiscoveredSystemActionsCellProps { + system: MonitorSystemAggregate; +} + +export const DiscoveredSystemActionsCell = ({ + system, +}: DiscoveredSystemActionsCellProps) => { + console.log(system); + return <Flex> </Flex>; +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateStatusCell.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateStatusCell.tsx new file mode 100644 index 0000000000..3c13f34290 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateStatusCell.tsx @@ -0,0 +1,33 @@ +import { Flex, Text, Tooltip } from "fidesui"; + +import { STATUS_INDICATOR_MAP } from "~/features/data-discovery-and-detection/statusIndicators"; + +import { MonitorSystemAggregate } from "../../types"; + +interface DiscoveredSystemStatusCellProps { + system: MonitorSystemAggregate; +} + +export const DiscoveredSystemStatusCell = ({ + system, +}: DiscoveredSystemStatusCellProps) => { + return ( + <Flex alignItems="center" height="100%"> + {!system?.system_key && ( + <Tooltip label="New system"> + {/* icon has to be wrapped in a span for the tooltip to work */} + <span>{STATUS_INDICATOR_MAP.Change}</span> + </Tooltip> + )} + <Text + fontSize="xs" + fontWeight="semibold" + lineHeight={4} + overflow="hidden" + textOverflow="ellipsis" + > + {system?.name || "Uncategorized assets"} + </Text> + </Flex> + ); +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts index e33d824d58..f2933bca51 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts @@ -1,7 +1,9 @@ +import { PaginatedResponse } from "~/types/common/PaginationQueryParams"; + // TODO: [HJ-334] remove these in favor of autogenerated types from the API -export interface MonitorSummary { +export interface MonitorAggregatedResults { updates: Record<string, number>; - property?: string; + property?: string; // this is a guess, it doesn't exist yet in the API last_monitored: string | number; key: string; name: string; @@ -9,9 +11,17 @@ export interface MonitorSummary { warning?: boolean | string; } -export interface MonitorSummaryPaginatedResponse { - items: MonitorSummary[]; - page: number; - size: number; - total: number; +export interface MonitorSummaryPaginatedResponse + extends PaginatedResponse<MonitorAggregatedResults> {} + +export interface MonitorSystemAggregate { + name: string; + system_key: string | null; // null when the system is not a known system + vendor_id: string; + total_updates: 0; + locations: string[]; + domains: string[]; } + +export interface MonitorSystemAggregatePaginatedResponse + extends PaginatedResponse<MonitorSystemAggregate> {} diff --git a/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx b/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx index fc51a7ae17..86b15a4645 100644 --- a/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx +++ b/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx @@ -12,12 +12,19 @@ import { CONNECTOR_LOGOS_PATH, FALLBACK_CONNECTOR_LOGOS_PATH, } from "./constants"; -import { isConnectionSystemTypeMap, isDatastoreConnection } from "./types"; type ConnectionTypeLogoProps = { data: string | ConnectionConfigurationResponse | ConnectionSystemTypeMap; }; +const isDatastoreConnection = ( + obj: any, +): obj is ConnectionConfigurationResponse => + (obj as ConnectionConfigurationResponse).connection_type !== undefined; + +const isConnectionSystemTypeMap = (obj: any): obj is ConnectionSystemTypeMap => + (obj as ConnectionSystemTypeMap).encoded_icon !== undefined; + const ConnectionTypeLogo = ({ data, ...props diff --git a/clients/admin-ui/src/features/datastore-connections/types.ts b/clients/admin-ui/src/features/datastore-connections/types.d.ts similarity index 91% rename from clients/admin-ui/src/features/datastore-connections/types.ts rename to clients/admin-ui/src/features/datastore-connections/types.d.ts index e4a171038f..7d1e20d841 100644 --- a/clients/admin-ui/src/features/datastore-connections/types.ts +++ b/clients/admin-ui/src/features/datastore-connections/types.d.ts @@ -1,6 +1,5 @@ import { ConnectionConfigurationResponse, - ConnectionSystemTypeMap, ConnectionType, DatasetConfigCtlDataset, SystemType, @@ -128,16 +127,6 @@ export type DatastoreConnectionResponse = { ]; }; -export const isDatastoreConnection = ( - obj: any, -): obj is ConnectionConfigurationResponse => - (obj as ConnectionConfigurationResponse).connection_type !== undefined; - -export const isConnectionSystemTypeMap = ( - obj: any, -): obj is ConnectionSystemTypeMap => - (obj as ConnectionSystemTypeMap).encoded_icon !== undefined; - export type DatastoreConnectionParams = { search: string; connection_type?: string[]; diff --git a/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx b/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx index 6f07a74600..5f96ddfa1a 100644 --- a/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx +++ b/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx @@ -1,5 +1,27 @@ -const MonitorResultSystems = () => { - return <div>Monitor Result Systems FPO</div>; +import { NextPage } from "next"; +import { useRouter } from "next/router"; + +import FixedLayout from "~/features/common/FixedLayout"; +import { ACTION_CENTER_ROUTE } from "~/features/common/nav/v2/routes"; +import PageHeader from "~/features/common/PageHeader"; +import { DiscoveredSystemAggregateTable } from "~/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable"; + +const MonitorResultSystems: NextPage = () => { + const router = useRouter(); + const monitorId = decodeURIComponent(router.query.monitorId as string); + + return ( + <FixedLayout title="Action center - Assets by system"> + <PageHeader + heading="Action center" + breadcrumbItems={[ + { title: "All activity", href: ACTION_CENTER_ROUTE }, + { title: monitorId }, + ]} + /> + <DiscoveredSystemAggregateTable monitorId={monitorId} /> + </FixedLayout> + ); }; export default MonitorResultSystems; diff --git a/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx b/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx index 6edb23321c..598a357f4d 100644 --- a/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx +++ b/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx @@ -15,11 +15,11 @@ import { PaginationBar, useServerSidePagination, } from "~/features/common/table/v2"; -import { useGetMonitorSummaryQuery } from "~/features/data-discovery-and-detection/action-center/actionCenter.slice"; -import { DisabledMonitorPage } from "~/features/data-discovery-and-detection/action-center/DisabledMonitorPage"; -import { EmptyMonitorResult } from "~/features/data-discovery-and-detection/action-center/EmptyMonitorResult"; +import { useGetAggregateMonitorResultsQuery } from "~/features/data-discovery-and-detection/action-center/action-center.slice"; +import { DisabledMonitorsPage } from "~/features/data-discovery-and-detection/action-center/DisabledMonitorsPage"; +import { EmptyMonitorsResult } from "~/features/data-discovery-and-detection/action-center/EmptyMonitorsResult"; import { MonitorResult } from "~/features/data-discovery-and-detection/action-center/MonitorResult"; -import { MonitorSummary } from "~/features/data-discovery-and-detection/action-center/types"; +import { MonitorAggregatedResults } from "~/features/data-discovery-and-detection/action-center/types"; import { SearchInput } from "~/features/data-discovery-and-detection/SearchInput"; import { useGetConfigurationSettingsQuery } from "~/features/privacy-requests"; @@ -51,14 +51,15 @@ const ActionCenterPage = () => { resetPageIndexToDefault(); }, [searchQuery, resetPageIndexToDefault]); - const { data, isError, isLoading, isFetching } = useGetMonitorSummaryQuery( - { - pageIndex, - pageSize, - search: searchQuery, - }, - { skip: isConfigLoading || !webMonitorEnabled }, - ); + const { data, isError, isLoading, isFetching } = + useGetAggregateMonitorResultsQuery( + { + page: pageIndex, + size: pageSize, + search: searchQuery, + }, + { skip: isConfigLoading || !webMonitorEnabled }, + ); useEffect(() => { if (isError && !!toast && webMonitorEnabled) { @@ -123,7 +124,7 @@ const ActionCenterPage = () => { ); if (!webMonitorEnabled) { - return <DisabledMonitorPage isConfigLoading={isConfigLoading} />; + return <DisabledMonitorsPage isConfigLoading={isConfigLoading} />; } return ( @@ -141,16 +142,20 @@ const ActionCenterPage = () => { loading={isLoading} dataSource={results || loadingResults} locale={{ - emptyText: <EmptyMonitorResult />, + emptyText: <EmptyMonitorsResult />, + }} + renderItem={(summary: MonitorAggregatedResults) => { + return ( + !!summary && ( + <MonitorResult + showSkeleton={isFetching} + key={summary.key} + monitorSummary={summary} + actions={getWebsiteMonitorActions(summary.key)} // TODO: when monitor type becomes available, use it to determine actions. Defaulting to website monitor actions for now. + /> + ) + ); }} - renderItem={(summary: MonitorSummary) => ( - <MonitorResult - showSkeleton={isFetching} - key={summary.key} - monitorSummary={summary} - actions={getWebsiteMonitorActions(summary.key)} // TODO: when monitor type becomes available, use it to determine actions. Defaulting to website monitor actions for now. - /> - )} /> {!!results && !!data?.total && data.total > pageSize && ( diff --git a/clients/admin-ui/src/types/common/PaginationQueryParams.ts b/clients/admin-ui/src/types/common/PaginationQueryParams.ts index bfdeda1475..0600daee7f 100644 --- a/clients/admin-ui/src/types/common/PaginationQueryParams.ts +++ b/clients/admin-ui/src/types/common/PaginationQueryParams.ts @@ -2,3 +2,11 @@ export interface PaginationQueryParams { page: number; size: number; } + +export interface PaginatedResponse<T> { + items: T[]; + page: number; + size: number; + total: number; + pages: number; +} From 1de84b5837eb68474c3325abbcf59219dfea69d4 Mon Sep 17 00:00:00 2001 From: Andres Torres <andres.torres.marroquin@gmail.com> Date: Fri, 10 Jan 2025 08:17:38 -0600 Subject: [PATCH 4/9] HJ-352 - Fixes some fields that are not present on the CSV and they shouldn't be present on the web datamap report (#5645) Co-authored-by: Jason Gill <jason.gill@ethyca.com> --- .../custom-reports/custom-report.json | 5 +- .../cypress/fixtures/datamap/datamap.json | 18 ------ .../fixtures/datamap/empty_datamap.json | 2 - .../cypress/fixtures/datamap/minimal.json | 60 ------------------- .../src/features/datamap/datamap.slice.ts | 7 +-- .../datamap/reporting/DatamapReportTable.tsx | 28 +++++---- .../reporting/DatamapReportTableColumns.tsx | 24 +------- .../features/datamap/reporting/constants.tsx | 15 ++--- .../src/types/api/models/DatamapReport.ts | 3 - .../src/components/types/api/models/System.ts | 4 -- .../types/api/models/DatamapReport.ts | 3 - 11 files changed, 26 insertions(+), 143 deletions(-) diff --git a/clients/admin-ui/cypress/fixtures/custom-reports/custom-report.json b/clients/admin-ui/cypress/fixtures/custom-reports/custom-report.json index b2e2238323..9a3ae9bc4e 100644 --- a/clients/admin-ui/cypress/fixtures/custom-reports/custom-report.json +++ b/clients/admin-ui/cypress/fixtures/custom-reports/custom-report.json @@ -24,7 +24,7 @@ "legal_address", "cookie_refresh", "data_security_practices", - "DATA_SHARED_WITH_THIRD_PARTIES", + "data_shared_with_third_parties", "data_stewards", "declaration_name", "does_international_transfers", @@ -40,7 +40,6 @@ "legal_basis_for_profiling", "legal_basis_for_transfers", "legitimate_interest_disclosure_url", - "link_to_processor_contract", "processes_personal_data", "reason_for_exemption", "requires_data_protection_assessments", @@ -48,8 +47,6 @@ "retention_period", "shared_categories", "special_category_legal_basis", - "system_dependencies", - "third_country_safeguards", "third_parties", "system_undeclared_data_categories", "data_use_undeclared_data_categories", diff --git a/clients/admin-ui/cypress/fixtures/datamap/datamap.json b/clients/admin-ui/cypress/fixtures/datamap/datamap.json index fa73ee31ca..f4607dfe6d 100644 --- a/clients/admin-ui/cypress/fixtures/datamap/datamap.json +++ b/clients/admin-ui/cypress/fixtures/datamap/datamap.json @@ -6,9 +6,7 @@ "system.privacy_declaration.data_use.name": "Purpose of Processing", "system.privacy_declaration.data_subjects.name": "Categories of Individuals", "unioned_data_categories": "Categories of Personal Data (Fides Taxonomy)", - "system.link_to_processor_contract": "Link to Contract with Processor", "third_country_combined": "Third Country Transfers", - "system.third_country_safeguards": "Safeguards for Exceptional Transfers of Personal Data", "organization.link_to_security_policy": "General Description of Security Measures", "system.privacy_declaration.data_subjects.rights_available": "Rights available to individuals", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "Existence of automated decision-making, including profiling (if applicable)", @@ -24,9 +22,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.contact", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -42,9 +38,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.device.cookie_id", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -60,9 +54,7 @@ "system.privacy_declaration.data_use.name": "Advertising, Marketing or Promotion", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.device.cookie_id", - "system.link_to_processor_contract": "", "third_country_combined": "N/A", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -78,9 +70,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.contact.email", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -96,9 +86,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "system.operations", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -114,9 +102,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.unique_id", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -132,9 +118,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.contact.address.state", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -150,9 +134,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.name", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", diff --git a/clients/admin-ui/cypress/fixtures/datamap/empty_datamap.json b/clients/admin-ui/cypress/fixtures/datamap/empty_datamap.json index db645b12be..47ab0e62cc 100644 --- a/clients/admin-ui/cypress/fixtures/datamap/empty_datamap.json +++ b/clients/admin-ui/cypress/fixtures/datamap/empty_datamap.json @@ -6,9 +6,7 @@ "system.privacy_declaration.data_use.name": "Purpose of Processing", "system.privacy_declaration.data_subjects.name": "Categories of Individuals", "unioned_data_categories": "Categories of Personal Data (Fides Taxonomy)", - "system.link_to_processor_contract": "Link to Contract with Processor", "third_country_combined": "Third Country Transfers", - "system.third_country_safeguards": "Safeguards for Exceptional Transfers of Personal Data", "organization.link_to_security_policy": "General Description of Security Measures", "system.privacy_declaration.data_subjects.rights_available": "Rights available to individuals", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "Existence of automated decision-making, including profiling (if applicable)", diff --git a/clients/admin-ui/cypress/fixtures/datamap/minimal.json b/clients/admin-ui/cypress/fixtures/datamap/minimal.json index 2b3c178ba5..d0eea25ece 100644 --- a/clients/admin-ui/cypress/fixtures/datamap/minimal.json +++ b/clients/admin-ui/cypress/fixtures/datamap/minimal.json @@ -33,7 +33,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -42,13 +41,11 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [ "user.contact.email", "user.device.cookie_id" ], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -88,7 +85,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -97,10 +93,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -140,7 +134,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -149,10 +142,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -192,7 +183,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -201,10 +191,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -244,7 +232,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -253,10 +240,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -296,7 +281,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -305,10 +289,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -348,7 +330,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -357,10 +338,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -400,7 +379,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -409,10 +387,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -452,7 +428,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -461,10 +436,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -504,7 +477,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -513,10 +485,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -556,7 +526,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -565,10 +534,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -608,7 +575,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -617,10 +583,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -677,7 +641,6 @@ "legal_basis_for_transfers": [], "legal_name": "1plusX AG", "legitimate_interest_disclosure_url": "https://www.1plusx.com/privacy-policy/", - "link_to_processor_contract": null, "privacy_policy": "https://www.1plusx.com/privacy-policy/", "processes_personal_data": true, "reason_for_exemption": null, @@ -686,9 +649,7 @@ "retention_period": "90", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1plusX", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": true, @@ -745,7 +706,6 @@ "legal_basis_for_transfers": [], "legal_name": "1plusX AG", "legitimate_interest_disclosure_url": "https://www.1plusx.com/privacy-policy/", - "link_to_processor_contract": null, "privacy_policy": "https://www.1plusx.com/privacy-policy/", "processes_personal_data": true, "reason_for_exemption": null, @@ -754,9 +714,7 @@ "retention_period": "90", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1plusX", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": true, @@ -813,7 +771,6 @@ "legal_basis_for_transfers": [], "legal_name": "1plusX AG", "legitimate_interest_disclosure_url": "https://www.1plusx.com/privacy-policy/", - "link_to_processor_contract": null, "privacy_policy": "https://www.1plusx.com/privacy-policy/", "processes_personal_data": true, "reason_for_exemption": null, @@ -822,9 +779,7 @@ "retention_period": "90", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1plusX", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": true, @@ -881,7 +836,6 @@ "legal_basis_for_transfers": [], "legal_name": "1plusX AG", "legitimate_interest_disclosure_url": "https://www.1plusx.com/privacy-policy/", - "link_to_processor_contract": null, "privacy_policy": "https://www.1plusx.com/privacy-policy/", "processes_personal_data": true, "reason_for_exemption": null, @@ -890,9 +844,7 @@ "retention_period": "90", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1plusX", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": true, @@ -949,7 +901,6 @@ "legal_basis_for_transfers": [], "legal_name": "1plusX AG", "legitimate_interest_disclosure_url": "https://www.1plusx.com/privacy-policy/", - "link_to_processor_contract": null, "privacy_policy": "https://www.1plusx.com/privacy-policy/", "processes_personal_data": true, "reason_for_exemption": null, @@ -958,9 +909,7 @@ "retention_period": "90", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1plusX", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": true, @@ -1017,7 +966,6 @@ "legal_basis_for_transfers": [], "legal_name": "1plusX AG", "legitimate_interest_disclosure_url": "https://www.1plusx.com/privacy-policy/", - "link_to_processor_contract": null, "privacy_policy": "https://www.1plusx.com/privacy-policy/", "processes_personal_data": true, "reason_for_exemption": null, @@ -1026,9 +974,7 @@ "retention_period": "90", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1plusX", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": true, @@ -1067,7 +1013,6 @@ "legal_basis_for_transfers": [], "legal_name": "1trn", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://1trn.com/privacy.php", "processes_personal_data": true, "reason_for_exemption": null, @@ -1076,9 +1021,7 @@ "retention_period": null, "shared_categories": null, "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1trn", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": false, "uses_non_cookie_access": false, @@ -1117,7 +1060,6 @@ "legal_basis_for_transfers": [], "legal_name": "", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": null, "processes_personal_data": true, "reason_for_exemption": null, @@ -1126,9 +1068,7 @@ "retention_period": "", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "test", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": false, "uses_non_cookie_access": false, diff --git a/clients/admin-ui/src/features/datamap/datamap.slice.ts b/clients/admin-ui/src/features/datamap/datamap.slice.ts index a383763127..72a1721c10 100644 --- a/clients/admin-ui/src/features/datamap/datamap.slice.ts +++ b/clients/admin-ui/src/features/datamap/datamap.slice.ts @@ -46,12 +46,7 @@ const DEFAULT_ACTIVE_COLUMNS = [ SYSTEM_DESCRIPTION, ]; -const DEPRECATED_COLUMNS = [ - "third_country_combined", - "system.third_country_safeguards", - "dataset.fides_key", - "system.link_to_processor_contract", -]; +const DEPRECATED_COLUMNS = ["third_country_combined", "dataset.fides_key"]; // API endpoints const datamapApi = baseApi.injectEndpoints({ diff --git a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx index d621bb1bf0..7960003dda 100644 --- a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx @@ -173,7 +173,7 @@ export const DatamapReportTable = () => { const [ exportMinimalDatamapReport, - { isLoading: isExportingReport, isSuccess: isExportReportSuccess }, + { isLoading: isExportingReport, isError: isExportReportError }, ] = useExportMinimalDatamapReportMutation(); const { data, totalRows } = useMemo(() => { @@ -208,15 +208,17 @@ export const DatamapReportTable = () => { const columns = useMemo( () => - getDatamapReportColumns({ - onSelectRow: (row) => setSelectedSystemId(row.fides_key), - getDataUseDisplayName, - getDataCategoryDisplayName, - getDataSubjectDisplayName, - datamapReport, - customFields, - isRenaming: isRenamingColumns, - }), + datamapReport + ? getDatamapReportColumns({ + onSelectRow: (row) => setSelectedSystemId(row.fides_key), + getDataUseDisplayName, + getDataCategoryDisplayName, + getDataSubjectDisplayName, + datamapReport, + customFields, + isRenaming: isRenamingColumns, + }) + : [], [ getDataUseDisplayName, getDataSubjectDisplayName, @@ -274,7 +276,7 @@ export const DatamapReportTable = () => { }, }, }).then(() => { - if (isExportReportSuccess) { + if (!isExportReportError) { onExportReportClose(); } }); @@ -303,7 +305,7 @@ export const DatamapReportTable = () => { }); useEffect(() => { - if (groupBy && !!tableInstance) { + if (groupBy && !!tableInstance && !!datamapReport) { if (tableInstance.getState().columnOrder.length === 0) { const tableColumnIds = tableInstance.getAllColumns().map((c) => c.id); setColumnOrder(getColumnOrder(groupBy, tableColumnIds)); @@ -314,7 +316,7 @@ export const DatamapReportTable = () => { } } // eslint-disable-next-line react-hooks/exhaustive-deps - }, [groupBy, tableInstance]); + }, [groupBy, tableInstance, datamapReport]); useEffect(() => { // changing the groupBy should wait until the data is loaded to update the grouping diff --git a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx index b8d2ffda86..0325c26b88 100644 --- a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx @@ -241,6 +241,9 @@ export const getDatamapReportColumns = ({ columnHelper.accessor((row) => row.data_shared_with_third_parties, { id: COLUMN_IDS.DATA_SHARED_WITH_THIRD_PARTIES, }), + columnHelper.accessor((row) => row.processes_special_category_data, { + id: COLUMN_IDS.PROCESSES_SPECIAL_CATEGORY_DATA, + }), columnHelper.accessor((row) => row.data_stewards, { id: COLUMN_IDS.DATA_STEWARDS, cell: (props) => ( @@ -364,9 +367,6 @@ export const getDatamapReportColumns = ({ columnHelper.accessor((row) => row.legitimate_interest_disclosure_url, { id: COLUMN_IDS.LEGITIMATE_INTEREST_DISCLOSURE_URL, }), - columnHelper.accessor((row) => row.link_to_processor_contract, { - id: COLUMN_IDS.LINK_TO_PROCESSOR_CONTRACT, - }), columnHelper.accessor((row) => row.processes_personal_data, { id: COLUMN_IDS.PROCESSES_PERSONAL_DATA, }), @@ -414,24 +414,6 @@ export const getDatamapReportColumns = ({ columnHelper.accessor((row) => row.special_category_legal_basis, { id: COLUMN_IDS.SPECIAL_CATEGORY_LEGAL_BASIS, }), - columnHelper.accessor((row) => row.system_dependencies, { - id: COLUMN_IDS.SYSTEM_DEPENDENCIES, - cell: (props) => ( - <GroupCountBadgeCell - suffix="dependencies" - ignoreZero - value={props.getValue()} - {...props} - /> - ), - meta: { - showHeaderMenu: !isRenaming, - width: "auto", - }, - }), - columnHelper.accessor((row) => row.third_country_safeguards, { - id: COLUMN_IDS.THIRD_COUNTRY_SAFEGUARDS, - }), columnHelper.accessor((row) => row.third_parties, { id: COLUMN_IDS.THIRD_PARTIES, }), diff --git a/clients/admin-ui/src/features/datamap/reporting/constants.tsx b/clients/admin-ui/src/features/datamap/reporting/constants.tsx index 8ab825e1f4..e471370b60 100644 --- a/clients/admin-ui/src/features/datamap/reporting/constants.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/constants.tsx @@ -27,7 +27,7 @@ export enum COLUMN_IDS { LEGAL_ADDRESS = "legal_address", COOKIE_REFRESH = "cookie_refresh", DATA_SECURITY_PRACTICES = "data_security_practices", - DATA_SHARED_WITH_THIRD_PARTIES = "DATA_SHARED_WITH_THIRD_PARTIES", + DATA_SHARED_WITH_THIRD_PARTIES = "data_shared_with_third_parties", DATA_STEWARDS = "data_stewards", DECLARATION_NAME = "declaration_name", DOES_INTERNATIONAL_TRANSFERS = "does_international_transfers", @@ -43,7 +43,6 @@ export enum COLUMN_IDS { LEGAL_BASIS_FOR_PROFILING = "legal_basis_for_profiling", LEGAL_BASIS_FOR_TRANSFERS = "legal_basis_for_transfers", LEGITIMATE_INTEREST_DISCLOSURE_URL = "legitimate_interest_disclosure_url", - LINK_TO_PROCESSOR_CONTRACT = "link_to_processor_contract", PROCESSES_PERSONAL_DATA = "processes_personal_data", REASON_FOR_EXEMPTION = "reason_for_exemption", REQUIRES_DATA_PROTECTION_ASSESSMENTS = "requires_data_protection_assessments", @@ -51,8 +50,6 @@ export enum COLUMN_IDS { RETENTION_PERIOD = "retention_period", SHARED_CATEGORIES = "shared_categories", SPECIAL_CATEGORY_LEGAL_BASIS = "special_category_legal_basis", - SYSTEM_DEPENDENCIES = "system_dependencies", - THIRD_COUNTRY_SAFEGUARDS = "third_country_safeguards", THIRD_PARTIES = "third_parties", COOKIES = "cookies", USES_COOKIES = "uses_cookies", @@ -60,6 +57,7 @@ export enum COLUMN_IDS { USES_PROFILING = "uses_profiling", SYSTEM_UNDECLARED_DATA_CATEGORIES = "system_undeclared_data_categories", DATA_USE_UNDECLARED_DATA_CATEGORIES = "data_use_undeclared_data_categories", + PROCESSES_SPECIAL_CATEGORY_DATA = "processes_special_category_data", } export const DEFAULT_COLUMN_NAMES: Record<COLUMN_IDS, string> = { @@ -81,7 +79,7 @@ export const DEFAULT_COLUMN_NAMES: Record<COLUMN_IDS, string> = { [COLUMN_IDS.DECLARATION_NAME]: "Declaration name", [COLUMN_IDS.DOES_INTERNATIONAL_TRANSFERS]: "Does international transfers", [COLUMN_IDS.DPA_LOCATION]: "DPA location", - [COLUMN_IDS.DESTINATIONS]: "Destinations", + [COLUMN_IDS.DESTINATIONS]: "Destination", [COLUMN_IDS.EXEMPT_FROM_PRIVACY_REGULATIONS]: "Exempt from privacy regulations", [COLUMN_IDS.FEATURES]: "Features", @@ -89,13 +87,12 @@ export const DEFAULT_COLUMN_NAMES: Record<COLUMN_IDS, string> = { [COLUMN_IDS.FLEXIBLE_LEGAL_BASIS_FOR_PROCESSING]: "Flexible legal basis for processing", [COLUMN_IDS.IMPACT_ASSESSMENT_LOCATION]: "Impact assessment location", - [COLUMN_IDS.SOURCES]: "Sources", + [COLUMN_IDS.SOURCES]: "Source", [COLUMN_IDS.JOINT_CONTROLLER_INFO]: "Joint controller info", [COLUMN_IDS.LEGAL_BASIS_FOR_PROFILING]: "Legal basis for profiling", [COLUMN_IDS.LEGAL_BASIS_FOR_TRANSFERS]: "Legal basis for transfers", [COLUMN_IDS.LEGITIMATE_INTEREST_DISCLOSURE_URL]: "Legitimate interest disclosure URL", - [COLUMN_IDS.LINK_TO_PROCESSOR_CONTRACT]: "Link to processor contract", [COLUMN_IDS.PROCESSES_PERSONAL_DATA]: "Processes personal data", [COLUMN_IDS.REASON_FOR_EXEMPTION]: "Reason for exemption", [COLUMN_IDS.REQUIRES_DATA_PROTECTION_ASSESSMENTS]: @@ -104,8 +101,6 @@ export const DEFAULT_COLUMN_NAMES: Record<COLUMN_IDS, string> = { [COLUMN_IDS.RETENTION_PERIOD]: "Retention period", [COLUMN_IDS.SHARED_CATEGORIES]: "Shared categories", [COLUMN_IDS.SPECIAL_CATEGORY_LEGAL_BASIS]: "Special category legal basis", - [COLUMN_IDS.SYSTEM_DEPENDENCIES]: "System dependencies", - [COLUMN_IDS.THIRD_COUNTRY_SAFEGUARDS]: "Third country safeguards", [COLUMN_IDS.THIRD_PARTIES]: "Third parties", [COLUMN_IDS.SYSTEM_UNDECLARED_DATA_CATEGORIES]: "System undeclared data categories", @@ -115,4 +110,6 @@ export const DEFAULT_COLUMN_NAMES: Record<COLUMN_IDS, string> = { [COLUMN_IDS.USES_COOKIES]: "Uses cookies", [COLUMN_IDS.USES_NON_COOKIE_ACCESS]: "Uses non-cookie access", [COLUMN_IDS.USES_PROFILING]: "Uses profiling", + [COLUMN_IDS.PROCESSES_SPECIAL_CATEGORY_DATA]: + "Processes special category data", }; diff --git a/clients/admin-ui/src/types/api/models/DatamapReport.ts b/clients/admin-ui/src/types/api/models/DatamapReport.ts index 5283343ba5..b3f8cffc13 100644 --- a/clients/admin-ui/src/types/api/models/DatamapReport.ts +++ b/clients/admin-ui/src/types/api/models/DatamapReport.ts @@ -33,7 +33,6 @@ export type DatamapReport = { legal_basis_for_transfers?: Array<string> | null; legal_name?: string | null; legitimate_interest_disclosure_url?: string | null; - link_to_processor_contract?: string | null; privacy_policy?: string | null; processes_personal_data: boolean; reason_for_exemption?: string | null; @@ -42,9 +41,7 @@ export type DatamapReport = { retention_period?: string | null; shared_categories?: Array<string> | null; special_category_legal_basis?: string | null; - system_dependencies?: string | null; system_name: string; - third_country_safeguards?: string | null; third_parties?: string | null; uses_cookies: boolean; uses_non_cookie_access: boolean; diff --git a/clients/fidesui/src/components/types/api/models/System.ts b/clients/fidesui/src/components/types/api/models/System.ts index b17d8a75b8..ebc3a86179 100644 --- a/clients/fidesui/src/components/types/api/models/System.ts +++ b/clients/fidesui/src/components/types/api/models/System.ts @@ -79,10 +79,6 @@ export type System = { * */ privacy_declarations: Array<PrivacyDeclaration>; - /** - * A list of fides keys to model dependencies. - */ - system_dependencies?: Array<string>; /** * * The contact details information model. diff --git a/clients/privacy-center/types/api/models/DatamapReport.ts b/clients/privacy-center/types/api/models/DatamapReport.ts index 5283343ba5..b3f8cffc13 100644 --- a/clients/privacy-center/types/api/models/DatamapReport.ts +++ b/clients/privacy-center/types/api/models/DatamapReport.ts @@ -33,7 +33,6 @@ export type DatamapReport = { legal_basis_for_transfers?: Array<string> | null; legal_name?: string | null; legitimate_interest_disclosure_url?: string | null; - link_to_processor_contract?: string | null; privacy_policy?: string | null; processes_personal_data: boolean; reason_for_exemption?: string | null; @@ -42,9 +41,7 @@ export type DatamapReport = { retention_period?: string | null; shared_categories?: Array<string> | null; special_category_legal_basis?: string | null; - system_dependencies?: string | null; system_name: string; - third_country_safeguards?: string | null; third_parties?: string | null; uses_cookies: boolean; uses_non_cookie_access: boolean; From 5557f79305b7be60ad6294b8960a011dc97e9e12 Mon Sep 17 00:00:00 2001 From: Lucano Vera <lucanovera@live.com.ar> Date: Mon, 13 Jan 2025 11:27:10 -0300 Subject: [PATCH 5/9] Fix cypress e2e pipeline (#5659) --- .github/workflows/cypress_e2e.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/cypress_e2e.yml b/.github/workflows/cypress_e2e.yml index 952e4fc46d..7da0f97836 100644 --- a/.github/workflows/cypress_e2e.yml +++ b/.github/workflows/cypress_e2e.yml @@ -26,6 +26,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Setup Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install Nox run: pip install nox>=2022 From 73188c53f9479cd6dbd4f3637a4fc03faec9d3cf Mon Sep 17 00:00:00 2001 From: Lucano Vera <lucanovera@live.com.ar> Date: Mon, 13 Jan 2025 12:06:42 -0300 Subject: [PATCH 6/9] LA-203 Update-CMP-Privacy-Center-powered-by-URL-to-https-ethyca.com (#5656) --- CHANGELOG.md | 3 +++ clients/fides-js/src/components/BrandLink.tsx | 2 +- clients/fides-js/src/components/EthycaLogo.tsx | 2 ++ clients/privacy-center/components/BrandLink.tsx | 10 +++++++++- .../privacy-center/cypress/e2e/consent-banner.cy.ts | 2 +- .../public/fides-js-components-demo.html | 1 + 6 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1862215a98..074f760587 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - Added Action Center MVP behind new feature flag [#5622](https://github.com/ethyca/fides/pull/5622) - Added cache-clearing methods to the `DBCache` model to allow deleting cache entries [#5629](https://github.com/ethyca/fides/pull/5629) +### Changed +- Updated brand link url [#5656](https://github.com/ethyca/fides/pull/5656) + ### Fixed - Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) - Fixed column ordering issue in the Data Map report [#5649](https://github.com/ethyca/fides/pull/5649) diff --git a/clients/fides-js/src/components/BrandLink.tsx b/clients/fides-js/src/components/BrandLink.tsx index dd84bb79ef..a22a207743 100644 --- a/clients/fides-js/src/components/BrandLink.tsx +++ b/clients/fides-js/src/components/BrandLink.tsx @@ -5,7 +5,7 @@ import EthycaLogo from "./EthycaLogo"; const BrandLink = () => ( <div className="fides-brand"> <a - href="https://fid.es/powered" + href="https://ethyca.com/" target="_blank" rel="noopener noreferrer" className="fides-brand-link" diff --git a/clients/fides-js/src/components/EthycaLogo.tsx b/clients/fides-js/src/components/EthycaLogo.tsx index fcec225fd0..64b62ff936 100644 --- a/clients/fides-js/src/components/EthycaLogo.tsx +++ b/clients/fides-js/src/components/EthycaLogo.tsx @@ -7,6 +7,8 @@ const EthycaLogo = () => ( height="20" fill="currentColor" className="ethyca-logo" + role="img" + aria-label="Ethyca" > <path d="M11.5602 0H8.70311V7.9999H11.5602V0ZM23.0026 12.0001H20.1455V20H23.0026V12.0001ZM14.1258 20H17.5799L12.1315 11.9774V17.0631L14.1258 20ZM8.13179 17.0626V11.9769L2.68588 19.9995H6.14003L8.13129 17.0626H8.13179ZM23.5744 2.93688V8.02263L29.0198 0H25.5682L23.5739 2.93688H23.5744ZM19.5742 2.93688L17.5799 0H14.1258L19.5742 8.02263V2.93688ZM19.5717 8.57121H12.132V11.4283H19.5717V8.57121ZM31 8.57121H23.5603V11.4283H31V8.57121ZM8.13179 8.57121H0.691589V11.4283H8.13179V8.57121Z" /> </svg> diff --git a/clients/privacy-center/components/BrandLink.tsx b/clients/privacy-center/components/BrandLink.tsx index 15976ceb06..d614c0afd6 100644 --- a/clients/privacy-center/components/BrandLink.tsx +++ b/clients/privacy-center/components/BrandLink.tsx @@ -22,9 +22,17 @@ const BrandLink = ({ right={right} textDecoration="none" _hover={{ textDecoration: "none" }} + href="https://ethyca.com/" {...props} > - Powered by <EthycaLogo color="minos.500" h="20px" w="31px" /> + Powered by{" "} + <EthycaLogo + color="minos.500" + h="20px" + w="31px" + role="img" + aria-label="Ethyca" + /> </Link> ); }; diff --git a/clients/privacy-center/cypress/e2e/consent-banner.cy.ts b/clients/privacy-center/cypress/e2e/consent-banner.cy.ts index eeb1bf582d..ce2eae5e1e 100644 --- a/clients/privacy-center/cypress/e2e/consent-banner.cy.ts +++ b/clients/privacy-center/cypress/e2e/consent-banner.cy.ts @@ -333,7 +333,7 @@ describe("Consent overlay", () => { cy.get("a.fides-brand-link").should( "have.attr", "href", - "https://fid.es/powered", + "https://ethyca.com/", ); }); }); diff --git a/clients/privacy-center/public/fides-js-components-demo.html b/clients/privacy-center/public/fides-js-components-demo.html index b60af186d4..57ce779cd7 100644 --- a/clients/privacy-center/public/fides-js-components-demo.html +++ b/clients/privacy-center/public/fides-js-components-demo.html @@ -207,6 +207,7 @@ privacyCenterUrl: "http://localhost:3001", fidesApiUrl: "http://localhost:8080/api/v1", fidesPrimaryColor: "#008000", + showFidesBrandLink: true, }, }; if (init !== "false") { From 7ea4ad58d49dfcd525344fdf1de86078797febf5 Mon Sep 17 00:00:00 2001 From: jpople <jeremy@ethyca.com> Date: Mon, 13 Jan 2025 10:05:18 -0600 Subject: [PATCH 7/9] Render 'Reclassify' button in overflow menu when applicable (#5655) --- CHANGELOG.md | 1 + .../cypress/e2e/discovery-detection.cy.ts | 3 +- .../hooks/useDiscoveryResultColumns.tsx | 10 +- .../tables/cells/DiscoveryItemActionsCell.tsx | 124 ++++++++++++------ 4 files changed, 98 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 074f760587..76114c602c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ### Changed - Updated brand link url [#5656](https://github.com/ethyca/fides/pull/5656) +- Changed "Reclassify" D&D button to show in an overflow menu when row actions are overcrowded [#5655](https://github.com/ethyca/fides/pull/5655) ### Fixed - Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) diff --git a/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts b/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts index 08931756b8..b6831c64dd 100644 --- a/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts +++ b/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts @@ -327,7 +327,8 @@ describe("discovery and detection", () => { cy.getByTestId( "row-my_bigquery_monitor.prj-bigquery-418515.test_dataset_1", ).within(() => { - cy.getByTestId("action-Reclassify").click(); + cy.getByTestId("actions-overflow-btn").click(); + cy.getByTestId("action-reclassify").click({ force: true }); cy.wait("@confirmResource"); }); }); diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDiscoveryResultColumns.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDiscoveryResultColumns.tsx index 6944f66bcd..c37b67bc72 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDiscoveryResultColumns.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDiscoveryResultColumns.tsx @@ -74,7 +74,9 @@ const useDiscoveryResultColumns = ({ <DefaultCell value="--" /> ), header: "Actions", - size: 180, + meta: { + width: "auto", + }, }), ]; return { columns }; @@ -137,6 +139,9 @@ const useDiscoveryResultColumns = ({ <DiscoveryItemActionsCell resource={props.row.original} /> ), header: "Actions", + meta: { + width: "auto", + }, }), ]; return { columns }; @@ -194,6 +199,9 @@ const useDiscoveryResultColumns = ({ <DiscoveryItemActionsCell resource={props.row.original} /> ), header: "Actions", + meta: { + width: "auto", + }, }), ]; return { columns }; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/tables/cells/DiscoveryItemActionsCell.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/tables/cells/DiscoveryItemActionsCell.tsx index af87194d63..7d66da1103 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/tables/cells/DiscoveryItemActionsCell.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/tables/cells/DiscoveryItemActionsCell.tsx @@ -1,4 +1,16 @@ -import { CheckIcon, HStack, RepeatIcon, ViewOffIcon } from "fidesui"; +import { + AntButton as Button, + CheckIcon, + HStack, + Menu, + MenuButton, + MenuItem, + MenuList, + MoreIcon, + RepeatIcon, + Spacer, + ViewOffIcon, +} from "fidesui"; import { useAlert } from "~/features/common/hooks"; import { DiscoveryMonitorItem } from "~/features/data-discovery-and-detection/types/DiscoveryMonitorItem"; @@ -54,21 +66,50 @@ const DiscoveryItemActionsCell = ({ resource }: DiscoveryItemActionsProps) => { const showMuteAction = itemHasClassificationChanges || childItemsHaveClassificationChanges; + // if promote and mute are both shown, show "Reclassify" in an overflow menu + // to avoid having too many buttons in the cell + const showReclassifyInOverflow = showPromoteAction && showMuteAction; + + const handlePromote = async () => { + await promoteResourceMutation({ + staged_resource_urn: resource.urn, + }); + successAlert( + `These changes have been added to a Fides dataset. To view, navigate to "Manage datasets".`, + `Table changes confirmed`, + ); + }; + + const handleMute = async () => { + await muteResourceMutation({ + staged_resource_urn: resource.urn, + }); + successAlert( + `Ignored changes will not be added to a Fides dataset.`, + `${resource.name || "Changes"} ignored`, + ); + }; + + const handleReclassify = async () => { + await confirmResourceMutation({ + staged_resource_urn: resource.urn, + monitor_config_id: resource.monitor_config_id!, + start_classification: true, + classify_monitored_resources: true, + }); + successAlert( + `Reclassification of ${getResourceName(resource) || "the resource"} has begun. The results may take some time to appear in the “Data discovery“ tab.`, + `Reclassification started`, + ); + }; + return ( - <HStack onClick={(e) => e.stopPropagation()}> + <HStack onClick={(e) => e.stopPropagation()} gap={2}> {showPromoteAction && ( <ActionButton title="Confirm" icon={<CheckIcon />} - onClick={async () => { - await promoteResourceMutation({ - staged_resource_urn: resource.urn, - }); - successAlert( - `These changes have been added to a Fides dataset. To view, navigate to "Manage datasets".`, - `Table changes confirmed`, - ); - }} + onClick={handlePromote} disabled={anyActionIsLoading} loading={promoteIsLoading} /> @@ -77,37 +118,44 @@ const DiscoveryItemActionsCell = ({ resource }: DiscoveryItemActionsProps) => { <ActionButton title="Ignore" icon={<ViewOffIcon />} - onClick={async () => { - await muteResourceMutation({ - staged_resource_urn: resource.urn, - }); - successAlert( - `Ignored changes will not be added to a Fides dataset.`, - `${resource.name || "Changes"} ignored`, - ); - }} + onClick={handleMute} disabled={anyActionIsLoading} loading={muteIsLoading} /> )} - <ActionButton - title="Reclassify" - icon={<RepeatIcon />} - onClick={async () => { - await confirmResourceMutation({ - staged_resource_urn: resource.urn, - monitor_config_id: resource.monitor_config_id!, - start_classification: true, - classify_monitored_resources: true, - }); - successAlert( - `Reclassification of ${getResourceName(resource) || "the resource"} has begun. The results may take some time to appear in the “Data discovery“ tab.`, - `Reclassification started`, - ); - }} - disabled={anyActionIsLoading} - loading={confirmIsLoading} - /> + {!showReclassifyInOverflow && ( + <ActionButton + title="Reclassify" + icon={<RepeatIcon />} + onClick={handleReclassify} + disabled={anyActionIsLoading} + loading={confirmIsLoading} + /> + )} + <Spacer /> + {showReclassifyInOverflow && ( + <Menu> + <MenuButton + as={Button} + size="small" + // TS expects Chakra's type prop (HTML type) but we want to assign the Ant type + // @ts-ignore + type="text" + icon={<MoreIcon transform="rotate(90deg)" />} + className="w-6 gap-0" + data-testid="actions-overflow-btn" + /> + <MenuList> + <MenuItem + onClick={handleReclassify} + icon={<RepeatIcon />} + data-testid="action-reclassify" + > + Reclassify + </MenuItem> + </MenuList> + </Menu> + )} </HStack> ); }; From 83cb612e4bc7f1fe424cab8a0a54c024331dfe53 Mon Sep 17 00:00:00 2001 From: Dave Quinlan <83430497+daveqnet@users.noreply.github.com> Date: Mon, 13 Jan 2025 17:08:18 +0000 Subject: [PATCH 8/9] chore: remove patch version pin from redis image tag (#5660) --- docker-compose.child-env.yml | 2 +- docker-compose.yml | 2 +- docker/docker-compose.minimal-config.yml | 2 +- src/fides/data/sample_project/docker-compose.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.child-env.yml b/docker-compose.child-env.yml index d56871f799..108524ac6e 100644 --- a/docker-compose.child-env.yml +++ b/docker-compose.child-env.yml @@ -64,7 +64,7 @@ services: - node.labels.fides.app-db-data == true redis-child: - image: "redis:6.2.5-alpine" + image: "redis:6.2-alpine" command: redis-server --requirepass redispassword expose: - 6379 diff --git a/docker-compose.yml b/docker-compose.yml index 6ba4ae2da4..2917b0fa11 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -136,7 +136,7 @@ services: - /fides/src/fides.egg-info redis: - image: "redis:6.2.5-alpine" + image: "redis:6.2-alpine" # AUTH option #1: no authentication at all # command: redis-server # AUTH option #2: require password diff --git a/docker/docker-compose.minimal-config.yml b/docker/docker-compose.minimal-config.yml index 15e49ea9e6..b06c1e2a7c 100644 --- a/docker/docker-compose.minimal-config.yml +++ b/docker/docker-compose.minimal-config.yml @@ -52,7 +52,7 @@ services: - node.labels.fides.app-db-data == true redis: - image: "redis:6.2.5-alpine" + image: "redis:6.2-alpine" command: redis-server --requirepass redispassword expose: - 6379 diff --git a/src/fides/data/sample_project/docker-compose.yml b/src/fides/data/sample_project/docker-compose.yml index 66de719665..bcbfd5239b 100644 --- a/src/fides/data/sample_project/docker-compose.yml +++ b/src/fides/data/sample_project/docker-compose.yml @@ -103,7 +103,7 @@ services: redis: container_name: fides-redis - image: redis:6.2.5-alpine + image: redis:6.2-alpine command: redis-server --requirepass redispassword ports: - "7379:6379" From de209feb31ce2619c9db608d41fa5d58877589cd Mon Sep 17 00:00:00 2001 From: Adrian Galvan <adrian@ethyca.com> Date: Mon, 13 Jan 2025 10:02:13 -0800 Subject: [PATCH 9/9] Splitting up query config files and tests (#5591) --- CHANGELOG.md | 1 + .../bigquery_enterprise_test_dataset.yml | 552 +++-------- .../dataset/bigquery_example_test_dataset.yml | 18 - .../api/service/connectors/base_connector.py | 11 + .../service/connectors/bigquery_connector.py | 5 + .../service/connectors/postgres_connector.py | 5 + .../query_configs/bigquery_query_config.py | 28 +- .../connectors/query_configs/query_config.py | 69 +- .../query_configs/snowflake_query_config.py | 9 +- .../service/connectors/scylla_connector.py | 5 + .../service/connectors/scylla_query_config.py | 26 +- src/fides/api/task/graph_task.py | 13 +- ...s_example_custom_request_field_dataset.yml | 2 +- tests/fixtures/application_fixtures.py | 53 ++ tests/fixtures/postgres_fixtures.py | 30 + .../integration_tests/test_mariadb_task.py | 101 ++ .../ops/integration_tests/test_mssql_task.py | 101 ++ .../ops/integration_tests/test_mysql_task.py | 101 ++ .../integration_tests/test_scylladb_task.py | 190 ++++ tests/ops/integration_tests/test_sql_task.py | 885 ++---------------- .../integration_tests/test_timescale_task.py | 294 ++++++ .../connectors/test_bigquery_connector.py | 4 +- .../connectors/test_bigquery_queryconfig.py | 4 +- .../connectors/test_dynamodb_query_config.py | 129 +++ .../connectors/test_mongo_query_config.py | 283 ++++++ .../service/connectors/test_query_config.py | 576 ++++-------- .../connectors/test_scylladb_query_config.py | 47 + .../connectors/test_snowflake_query_config.py | 4 +- .../example_datasets/multiple_identities.yml | 2 - ...le_identities_with_external_dependency.yml | 2 - .../example_datasets/no_identities.yml | 2 - .../example_datasets/single_identity.yml | 2 - ...ngle_identity_with_internal_dependency.yml | 2 - ...est_bigquery_enterprise_privacy_request.py | 22 +- .../test_postgres_privacy_requests.py | 30 +- tests/ops/test_helpers/dataset_utils.py | 30 +- 36 files changed, 1914 insertions(+), 1724 deletions(-) create mode 100644 tests/ops/integration_tests/test_mariadb_task.py create mode 100644 tests/ops/integration_tests/test_mssql_task.py create mode 100644 tests/ops/integration_tests/test_mysql_task.py create mode 100644 tests/ops/integration_tests/test_scylladb_task.py create mode 100644 tests/ops/integration_tests/test_timescale_task.py create mode 100644 tests/ops/service/connectors/test_dynamodb_query_config.py create mode 100644 tests/ops/service/connectors/test_mongo_query_config.py create mode 100644 tests/ops/service/connectors/test_scylladb_query_config.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 76114c602c..9eb9994d6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ### Changed - Updated brand link url [#5656](https://github.com/ethyca/fides/pull/5656) - Changed "Reclassify" D&D button to show in an overflow menu when row actions are overcrowded [#5655](https://github.com/ethyca/fides/pull/5655) +- Removed primary key requirements for BigQuery and Postgres erasures [#5591](https://github.com/ethyca/fides/pull/5591) ### Fixed - Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) diff --git a/data/dataset/bigquery_enterprise_test_dataset.yml b/data/dataset/bigquery_enterprise_test_dataset.yml index 59d27e68a2..64668192d0 100644 --- a/data/dataset/bigquery_enterprise_test_dataset.yml +++ b/data/dataset/bigquery_enterprise_test_dataset.yml @@ -1,405 +1,149 @@ dataset: - - fides_key: enterprise_dsr_testing - organization_fides_key: default_organization - tags: null - name: Bigquery Enterprise Test Dataset - description: BigQuery dataset containing real data - meta: null - data_categories: null - fides_meta: - resource_id: enterprise_dsr_testing.prj-sandbox-55855.enterprise_dsr_testing - after: null - namespace: - dataset_id: enterprise_dsr_testing - project_id: prj-sandbox-55855 - collections: - - name: comments - description: null - data_categories: null - fields: - - name: creation_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: id - description: null - data_categories: - - system.operations - fides_meta: - references: null - identity: null - primary_key: true - data_type: integer - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - - name: post_id - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: score - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: text - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: user_display_name - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: user_id - description: null - data_categories: - - user.contact - fides_meta: - references: - - dataset: enterprise_dsr_testing - field: users.id - direction: from - identity: null - primary_key: null - data_type: null - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - fides_meta: null - - name: post_history - description: null - data_categories: null - fields: - - name: comment - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: creation_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: id - description: null - data_categories: - - system.operations - fides_meta: - references: null - identity: null - primary_key: true - data_type: integer - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - - name: post_history_type_id - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: post_id - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: revision_guid - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: text - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: user_id - description: null - data_categories: - - system.operations - fides_meta: - references: - - dataset: enterprise_dsr_testing - field: users.id - direction: from - identity: null - primary_key: null - data_type: null - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - fides_meta: null - - name: stackoverflow_posts - description: null - data_categories: null - fields: - - name: accepted_answer_id - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: answer_count - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: body - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: comment_count - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: community_owned_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: creation_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: favorite_count - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: id - description: null - data_categories: - - system.operations - fides_meta: - references: null - identity: null - primary_key: true - data_type: integer - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - - name: last_activity_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: last_edit_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: last_editor_display_name - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: last_editor_user_id - description: null - data_categories: - - system.operations - fides_meta: - references: - - dataset: enterprise_dsr_testing - field: users.id - direction: from - identity: null - primary_key: null - data_type: null - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - - name: owner_display_name - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: owner_user_id - description: null - data_categories: - - system.operations - fides_meta: - references: - - dataset: enterprise_dsr_testing - field: users.id - direction: from - identity: null - primary_key: null - data_type: integer - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - - name: parent_id - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: post_type_id - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: score - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: tags - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: title - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: view_count - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - fides_meta: null - - name: users - description: null - data_categories: null - fields: - - name: about_me - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: age - description: null - data_categories: - - user - fides_meta: null - fields: null - - name: creation_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: display_name - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: down_votes - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: id - description: null - data_categories: - - user.contact - fides_meta: - references: null - identity: stackoverflow_user_id - primary_key: true - data_type: integer - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - - name: last_access_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: location - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: profile_image_url - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: reputation - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: up_votes - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: views - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: website_url - description: null - data_categories: - - user - fides_meta: null - fields: null - fides_meta: - after: null - erase_after: - - enterprise_dsr_testing.comments - skip_processing: false - masking_strategy_override: null - partitioning: null + - fides_key: enterprise_dsr_testing + organization_fides_key: default_organization + name: Bigquery Enterprise Test Dataset + description: BigQuery dataset containing real data + fides_meta: + resource_id: enterprise_dsr_testing.prj-sandbox-55855.enterprise_dsr_testing + namespace: + dataset_id: enterprise_dsr_testing + project_id: prj-sandbox-55855 + collections: + - name: comments + fields: + - name: creation_date + data_categories: [system.operations] + - name: id + data_categories: [system.operations] + fides_meta: + data_type: integer + - name: post_id + data_categories: [system.operations] + - name: score + data_categories: [system.operations] + - name: text + data_categories: [user.contact] + - name: user_display_name + data_categories: [user.contact] + - name: user_id + data_categories: [user.contact] + fides_meta: + references: + - dataset: enterprise_dsr_testing + field: users.id + direction: from + - name: post_history + fields: + - name: comment + data_categories: [user.contact] + - name: creation_date + data_categories: [system.operations] + - name: id + data_categories: [system.operations] + fides_meta: + data_type: integer + - name: post_history_type_id + data_categories: [system.operations] + - name: post_id + data_categories: [system.operations] + - name: revision_guid + data_categories: [system.operations] + - name: text + data_categories: [user.contact] + - name: user_id + data_categories: [system.operations] + fides_meta: + references: + - dataset: enterprise_dsr_testing + field: users.id + direction: from + - name: stackoverflow_posts + fields: + - name: accepted_answer_id + data_categories: [system.operations] + - name: answer_count + data_categories: [system.operations] + - name: body + data_categories: [user.contact] + - name: comment_count + data_categories: [system.operations] + - name: community_owned_date + data_categories: [system.operations] + - name: creation_date + data_categories: [system.operations] + - name: favorite_count + data_categories: [system.operations] + - name: id + data_categories: [system.operations] + fides_meta: + data_type: integer + - name: last_activity_date + data_categories: [system.operations] + - name: last_edit_date + data_categories: [system.operations] + - name: last_editor_display_name + data_categories: [system.operations] + - name: last_editor_user_id + data_categories: [system.operations] + fides_meta: + references: + - dataset: enterprise_dsr_testing + field: users.id + direction: from + - name: owner_display_name + data_categories: [user.contact] + - name: owner_user_id + data_categories: [system.operations] + fides_meta: + references: + - dataset: enterprise_dsr_testing + field: users.id + direction: from + data_type: integer + - name: parent_id + data_categories: [system.operations] + - name: post_type_id + data_categories: [system.operations] + - name: score + data_categories: [system.operations] + - name: tags + data_categories: [system.operations] + - name: title + data_categories: [user.contact] + - name: view_count + data_categories: [system.operations] + - name: users + fields: + - name: about_me + data_categories: [user.contact] + - name: age + data_categories: [user] + - name: creation_date + data_categories: [system.operations] + - name: display_name + data_categories: [user.contact] + - name: down_votes + data_categories: [system.operations] + - name: id + data_categories: [user.contact] + fides_meta: + identity: stackoverflow_user_id + data_type: integer + - name: last_access_date + data_categories: [system.operations] + - name: location + data_categories: [user.contact] + - name: profile_image_url + data_categories: [user.contact] + - name: reputation + data_categories: [system.operations] + - name: up_votes + data_categories: [system.operations] + - name: views + data_categories: [system.operations] + - name: website_url + data_categories: [user] + fides_meta: + erase_after: + - enterprise_dsr_testing.comments + skip_processing: false diff --git a/data/dataset/bigquery_example_test_dataset.yml b/data/dataset/bigquery_example_test_dataset.yml index 11fdac1aba..c4ea16cb44 100644 --- a/data/dataset/bigquery_example_test_dataset.yml +++ b/data/dataset/bigquery_example_test_dataset.yml @@ -13,8 +13,6 @@ dataset: data_categories: [user.contact.address.street] - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: state data_categories: [user.contact.address.state] - name: street @@ -53,8 +51,6 @@ dataset: data_type: string - name: id data_categories: [user.unique_id] - fides_meta: - primary_key: True - name: name data_categories: [user.name] fides_meta: @@ -80,8 +76,6 @@ dataset: data_type: string - name: id data_categories: [user.unique_id] - fides_meta: - primary_key: True - name: name data_categories: [user.name] fides_meta: @@ -98,8 +92,6 @@ dataset: direction: from - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: time data_categories: [user.sensor] @@ -114,8 +106,6 @@ dataset: direction: from - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: shipping_address_id data_categories: [system.operations] fides_meta: @@ -166,8 +156,6 @@ dataset: direction: from - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: name data_categories: [user.financial] - name: preferred @@ -177,8 +165,6 @@ dataset: fields: - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: name data_categories: [system.operations] - name: price @@ -193,8 +179,6 @@ dataset: data_type: string - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: month data_categories: [system.operations] - name: name @@ -227,8 +211,6 @@ dataset: direction: from - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: opened data_categories: [system.operations] diff --git a/src/fides/api/service/connectors/base_connector.py b/src/fides/api/service/connectors/base_connector.py index ca3439f523..e1f735df1c 100644 --- a/src/fides/api/service/connectors/base_connector.py +++ b/src/fides/api/service/connectors/base_connector.py @@ -132,3 +132,14 @@ def execute_standalone_retrieval_query( raise NotImplementedError( "execute_standalone_retrieval_query must be implemented in a concrete subclass" ) + + @property + def requires_primary_keys(self) -> bool: + """ + Indicates if datasets linked to this connector require primary keys for erasures. + Defaults to True. + """ + + # Defaulting to true for now so we can keep the default behavior and + # incrementally determine the need for primary keys across all connectors + return True diff --git a/src/fides/api/service/connectors/bigquery_connector.py b/src/fides/api/service/connectors/bigquery_connector.py index 8b51f90842..ae6fe4b909 100644 --- a/src/fides/api/service/connectors/bigquery_connector.py +++ b/src/fides/api/service/connectors/bigquery_connector.py @@ -33,6 +33,11 @@ class BigQueryConnector(SQLConnector): secrets_schema = BigQuerySchema + @property + def requires_primary_keys(self) -> bool: + """BigQuery does not have the concept of primary keys so they're not required for erasures.""" + return False + # Overrides BaseConnector.build_uri def build_uri(self) -> str: """Build URI of format""" diff --git a/src/fides/api/service/connectors/postgres_connector.py b/src/fides/api/service/connectors/postgres_connector.py index 5354d4ec13..2abafc01c8 100644 --- a/src/fides/api/service/connectors/postgres_connector.py +++ b/src/fides/api/service/connectors/postgres_connector.py @@ -19,6 +19,11 @@ class PostgreSQLConnector(SQLConnector): secrets_schema = PostgreSQLSchema + @property + def requires_primary_keys(self) -> bool: + """Postgres allows arbitrary columns in the WHERE clause for updates so primary keys are not required.""" + return False + def build_uri(self) -> str: """Build URI of format postgresql://[user[:password]@][netloc][:port][/dbname]""" config = self.secrets_schema(**self.configuration.secrets or {}) diff --git a/src/fides/api/service/connectors/query_configs/bigquery_query_config.py b/src/fides/api/service/connectors/query_configs/bigquery_query_config.py index 681e2b9c60..6060ff5822 100644 --- a/src/fides/api/service/connectors/query_configs/bigquery_query_config.py +++ b/src/fides/api/service/connectors/query_configs/bigquery_query_config.py @@ -123,15 +123,15 @@ def generate_update( TODO: DRY up this method and `generate_delete` a bit """ update_value_map: Dict[str, Any] = self.update_value_map(row, policy, request) - non_empty_primary_keys: Dict[str, Field] = filter_nonempty_values( + non_empty_reference_field_keys: Dict[str, Field] = filter_nonempty_values( { fpath.string_path: fld.cast(row[fpath.string_path]) - for fpath, fld in self.primary_key_field_paths.items() + for fpath, fld in self.reference_field_paths.items() if fpath.string_path in row } ) - valid = len(non_empty_primary_keys) > 0 and update_value_map + valid = len(non_empty_reference_field_keys) > 0 and update_value_map if not valid: logger.warning( "There is not enough data to generate a valid update statement for {}", @@ -140,8 +140,8 @@ def generate_update( return [] table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True) - pk_clauses: List[ColumnElement] = [ - getattr(table.c, k) == v for k, v in non_empty_primary_keys.items() + where_clauses: List[ColumnElement] = [ + getattr(table.c, k) == v for k, v in non_empty_reference_field_keys.items() ] if self.partitioning: @@ -153,13 +153,13 @@ def generate_update( for partition_clause in partition_clauses: partitioned_queries.append( table.update() - .where(*(pk_clauses + [text(partition_clause)])) + .where(*(where_clauses + [text(partition_clause)])) .values(**update_value_map) ) return partitioned_queries - return [table.update().where(*pk_clauses).values(**update_value_map)] + return [table.update().where(*where_clauses).values(**update_value_map)] def generate_delete(self, row: Row, client: Engine) -> List[Delete]: """Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement. @@ -172,15 +172,15 @@ def generate_delete(self, row: Row, client: Engine) -> List[Delete]: TODO: DRY up this method and `generate_update` a bit """ - non_empty_primary_keys: Dict[str, Field] = filter_nonempty_values( + non_empty_reference_field_keys: Dict[str, Field] = filter_nonempty_values( { fpath.string_path: fld.cast(row[fpath.string_path]) - for fpath, fld in self.primary_key_field_paths.items() + for fpath, fld in self.reference_field_paths.items() if fpath.string_path in row } ) - valid = len(non_empty_primary_keys) > 0 + valid = len(non_empty_reference_field_keys) > 0 if not valid: logger.warning( "There is not enough data to generate a valid DELETE statement for {}", @@ -189,8 +189,8 @@ def generate_delete(self, row: Row, client: Engine) -> List[Delete]: return [] table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True) - pk_clauses: List[ColumnElement] = [ - getattr(table.c, k) == v for k, v in non_empty_primary_keys.items() + where_clauses: List[ColumnElement] = [ + getattr(table.c, k) == v for k, v in non_empty_reference_field_keys.items() ] if self.partitioning: @@ -202,9 +202,9 @@ def generate_delete(self, row: Row, client: Engine) -> List[Delete]: for partition_clause in partition_clauses: partitioned_queries.append( - table.delete().where(*(pk_clauses + [text(partition_clause)])) + table.delete().where(*(where_clauses + [text(partition_clause)])) ) return partitioned_queries - return [table.delete().where(*pk_clauses)] + return [table.delete().where(*where_clauses)] diff --git a/src/fides/api/service/connectors/query_configs/query_config.py b/src/fides/api/service/connectors/query_configs/query_config.py index 6e868964af..9f5ddb0251 100644 --- a/src/fides/api/service/connectors/query_configs/query_config.py +++ b/src/fides/api/service/connectors/query_configs/query_config.py @@ -100,6 +100,15 @@ def primary_key_field_paths(self) -> Dict[FieldPath, Field]: if field.primary_key } + @property + def reference_field_paths(self) -> Dict[FieldPath, Field]: + """Mapping of FieldPaths to Fields that have incoming identity or dataset references""" + return { + field_path: field + for field_path, field in self.field_map().items() + if field_path in {edge.f2.field_path for edge in self.node.incoming_edges} + } + def query_sources(self) -> Dict[str, List[CollectionAddress]]: """Display the input collection(s) for each query key for display purposes. @@ -412,14 +421,16 @@ def generate_query_without_tuples( # pylint: disable=R0914 def get_update_stmt( self, update_clauses: List[str], - pk_clauses: List[str], + where_clauses: List[str], ) -> str: """Returns a SQL UPDATE statement to fit SQL syntax.""" - return f"UPDATE {self.node.address.collection} SET {', '.join(update_clauses)} WHERE {' AND '.join(pk_clauses)}" + return f"UPDATE {self.node.address.collection} SET {', '.join(update_clauses)} WHERE {' AND '.join(where_clauses)}" @abstractmethod def get_update_clauses( - self, update_value_map: Dict[str, Any], non_empty_primary_keys: Dict[str, Field] + self, + update_value_map: Dict[str, Any], + where_clause_fields: Dict[str, Field], ) -> List[str]: """Returns a list of update clauses for the update statement.""" @@ -428,7 +439,7 @@ def format_query_stmt(self, query_str: str, update_value_map: Dict[str, Any]) -> """Returns a formatted update statement in the appropriate dialect.""" @abstractmethod - def format_key_map_for_update_stmt(self, fields: List[str]) -> List[str]: + def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]: """Adds the appropriate formatting for update statements in this datastore.""" def generate_update_stmt( @@ -436,7 +447,8 @@ def generate_update_stmt( ) -> Optional[T]: """Returns an update statement in generic SQL-ish dialect.""" update_value_map: Dict[str, Any] = self.update_value_map(row, policy, request) - non_empty_primary_keys: Dict[str, Field] = filter_nonempty_values( + + non_empty_primary_key_fields: Dict[str, Field] = filter_nonempty_values( { fpath.string_path: fld.cast(row[fpath.string_path]) for fpath, fld in self.primary_key_field_paths.items() @@ -444,17 +456,30 @@ def generate_update_stmt( } ) + non_empty_reference_fields: Dict[str, Field] = filter_nonempty_values( + { + fpath.string_path: fld.cast(row[fpath.string_path]) + for fpath, fld in self.reference_field_paths.items() + if fpath.string_path in row + } + ) + + # Create parameter mappings with masked_ prefix for SET values + param_map = { + **{f"masked_{k}": v for k, v in update_value_map.items()}, + **non_empty_primary_key_fields, + **non_empty_reference_fields, + } + update_clauses = self.get_update_clauses( - update_value_map, non_empty_primary_keys + {k: f"masked_{k}" for k in update_value_map}, + non_empty_primary_key_fields or non_empty_reference_fields, ) - pk_clauses = self.format_key_map_for_update_stmt( - list(non_empty_primary_keys.keys()) + where_clauses = self.format_key_map_for_update_stmt( + {k: k for k in non_empty_primary_key_fields or non_empty_reference_fields} ) - for k, v in non_empty_primary_keys.items(): - update_value_map[k] = v - - valid = len(pk_clauses) > 0 and len(update_clauses) > 0 + valid = len(where_clauses) > 0 and len(update_clauses) > 0 if not valid: logger.warning( "There is not enough data to generate a valid update statement for {}", @@ -462,12 +487,9 @@ def generate_update_stmt( ) return None - query_str = self.get_update_stmt( - update_clauses, - pk_clauses, - ) - logger.info("query = {}, params = {}", Pii(query_str), Pii(update_value_map)) - return self.format_query_stmt(query_str, update_value_map) + query_str = self.get_update_stmt(update_clauses, where_clauses) + logger.info("query = {}, params = {}", Pii(query_str), Pii(param_map)) + return self.format_query_stmt(query_str, param_map) class SQLQueryConfig(SQLLikeQueryConfig[Executable]): @@ -538,16 +560,17 @@ def generate_query( ) return None - def format_key_map_for_update_stmt(self, fields: List[str]) -> List[str]: + def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]: """Adds the appropriate formatting for update statements in this datastore.""" - fields.sort() - return [f"{k} = :{k}" for k in fields] + return [f"{k} = :{v}" for k, v in sorted(param_map.items())] def get_update_clauses( - self, update_value_map: Dict[str, Any], non_empty_primary_keys: Dict[str, Field] + self, + update_value_map: Dict[str, Any], + where_clause_fields: Dict[str, Field], ) -> List[str]: """Returns a list of update clauses for the update statement.""" - return self.format_key_map_for_update_stmt(list(update_value_map.keys())) + return self.format_key_map_for_update_stmt(update_value_map) def format_query_stmt( self, query_str: str, update_value_map: Dict[str, Any] diff --git a/src/fides/api/service/connectors/query_configs/snowflake_query_config.py b/src/fides/api/service/connectors/query_configs/snowflake_query_config.py index 574e1ea1b1..ec640191d8 100644 --- a/src/fides/api/service/connectors/query_configs/snowflake_query_config.py +++ b/src/fides/api/service/connectors/query_configs/snowflake_query_config.py @@ -59,15 +59,14 @@ def get_formatted_query_string( """Returns a query string with double quotation mark formatting as required by Snowflake syntax.""" return f'SELECT {field_list} FROM {self._generate_table_name()} WHERE ({" OR ".join(clauses)})' - def format_key_map_for_update_stmt(self, fields: List[str]) -> List[str]: + def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]: """Adds the appropriate formatting for update statements in this datastore.""" - fields.sort() - return [f'"{k}" = :{k}' for k in fields] + return [f'"{k}" = :{v}' for k, v in sorted(param_map.items())] def get_update_stmt( self, update_clauses: List[str], - pk_clauses: List[str], + where_clauses: List[str], ) -> str: """Returns a parameterized update statement in Snowflake dialect.""" - return f'UPDATE {self._generate_table_name()} SET {", ".join(update_clauses)} WHERE {" AND ".join(pk_clauses)}' + return f'UPDATE {self._generate_table_name()} SET {", ".join(update_clauses)} WHERE {" AND ".join(where_clauses)}' diff --git a/src/fides/api/service/connectors/scylla_connector.py b/src/fides/api/service/connectors/scylla_connector.py index 43a821930c..ff17674b88 100644 --- a/src/fides/api/service/connectors/scylla_connector.py +++ b/src/fides/api/service/connectors/scylla_connector.py @@ -28,6 +28,11 @@ class ScyllaConnectorMissingKeyspace(Exception): class ScyllaConnector(BaseConnector[Cluster]): """Scylla Connector""" + @property + def requires_primary_keys(self) -> bool: + """ScyllaDB requires primary keys for erasures.""" + return True + def build_uri(self) -> str: """ Builds URI - Not yet implemented diff --git a/src/fides/api/service/connectors/scylla_query_config.py b/src/fides/api/service/connectors/scylla_query_config.py index 2a72270a40..1fa52d573d 100644 --- a/src/fides/api/service/connectors/scylla_query_config.py +++ b/src/fides/api/service/connectors/scylla_query_config.py @@ -70,21 +70,27 @@ def generate_query( ) -> Optional[ScyllaDBStatement]: return self.generate_query_without_tuples(input_data, policy) - def format_key_map_for_update_stmt(self, fields: List[str]) -> List[str]: + def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]: """Adds the appropriate formatting for update statements in this datastore.""" - fields.sort() - return [f"{k} = %({k})s" for k in fields] + return [f"{k} = %({v})s" for k, v in sorted(param_map.items())] def get_update_clauses( - self, update_value_map: Dict[str, Any], non_empty_primary_keys: Dict[str, Field] + self, + update_value_map: Dict[str, Any], + where_clause_fields: Dict[str, Field], ) -> List[str]: - """Returns a list of update clauses for the update statement.""" + """Returns a list of update clauses for the update statement. + + Omits primary key fields from updates since ScyllaDB prohibits + updating primary key fields. + """ + return self.format_key_map_for_update_stmt( - [ - key - for key in update_value_map.keys() - if key not in non_empty_primary_keys - ] + { + key: value + for key, value in update_value_map.items() + if key not in where_clause_fields + } ) def format_query_data_name(self, query_data_name: str) -> str: diff --git a/src/fides/api/task/graph_task.py b/src/fides/api/task/graph_task.py index 6b78b57297..145094ea25 100644 --- a/src/fides/api/task/graph_task.py +++ b/src/fides/api/task/graph_task.py @@ -603,12 +603,19 @@ def erasure_request( *erasure_prereqs: int, # TODO Remove when we stop support for DSR 2.0. DSR 3.0 enforces with downstream_tasks. ) -> int: """Run erasure request""" + # if there is no primary key specified in the graph node configuration # note this in the execution log and perform no erasures on this node - if not self.execution_node.collection.contains_field(lambda f: f.primary_key): + if ( + self.connector.requires_primary_keys + and not self.execution_node.collection.contains_field( + lambda f: f.primary_key + ) + ): logger.warning( - "No erasures on {} as there is no primary_key defined.", + 'Skipping erasures on "{}" as the "{}" connector requires a primary key to be defined in one of the collection fields, but none was found.', self.execution_node.address, + self.connector.configuration.connection_type, ) if self.request_task.id: # For DSR 3.0, largely for testing. DSR 3.0 uses Request Task status @@ -617,7 +624,7 @@ def erasure_request( # TODO Remove when we stop support for DSR 2.0 self.resources.cache_erasure(self.key.value, 0) self.update_status( - "No values were erased since no primary key was defined for this collection", + "No values were erased since no primary key was defined in any of the fields for this collection", None, ActionType.erasure, ExecutionLogStatus.complete, diff --git a/src/fides/data/sample_project/sample_resources/postgres_example_custom_request_field_dataset.yml b/src/fides/data/sample_project/sample_resources/postgres_example_custom_request_field_dataset.yml index 96b58645d4..0171258b1d 100644 --- a/src/fides/data/sample_project/sample_resources/postgres_example_custom_request_field_dataset.yml +++ b/src/fides/data/sample_project/sample_resources/postgres_example_custom_request_field_dataset.yml @@ -2,7 +2,7 @@ dataset: - fides_key: postgres_example_custom_request_field_dataset data_categories: [] description: Postgres example dataset with a custom request field - name: Postgrex Example Custom Request Field Dataset + name: Postgres Example Custom Request Field Dataset collections: - name: dynamic_email_address_config fields: diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 8356c42111..d030919aed 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -864,6 +864,59 @@ def erasure_policy( "rule_id": erasure_rule.id, }, ) + + yield erasure_policy + try: + rule_target.delete(db) + except ObjectDeletedError: + pass + try: + erasure_rule.delete(db) + except ObjectDeletedError: + pass + try: + erasure_policy.delete(db) + except ObjectDeletedError: + pass + + +@pytest.fixture(scope="function") +def erasure_policy_address_city( + db: Session, + oauth_client: ClientDetail, +) -> Generator: + erasure_policy = Policy.create( + db=db, + data={ + "name": "example erasure policy", + "key": "example_erasure_policy", + "client_id": oauth_client.id, + }, + ) + + erasure_rule = Rule.create( + db=db, + data={ + "action_type": ActionType.erasure.value, + "client_id": oauth_client.id, + "name": "Erasure Rule", + "policy_id": erasure_policy.id, + "masking_strategy": { + "strategy": "null_rewrite", + "configuration": {}, + }, + }, + ) + + rule_target = RuleTarget.create( + db=db, + data={ + "client_id": oauth_client.id, + "data_category": DataCategory("user.contact.address.city").value, + "rule_id": erasure_rule.id, + }, + ) + yield erasure_policy try: rule_target.delete(db) diff --git a/tests/fixtures/postgres_fixtures.py b/tests/fixtures/postgres_fixtures.py index 5e2aaec047..6db641fd1d 100644 --- a/tests/fixtures/postgres_fixtures.py +++ b/tests/fixtures/postgres_fixtures.py @@ -2,6 +2,7 @@ from uuid import uuid4 import pytest +from fideslang.models import Dataset as FideslangDataset from sqlalchemy.orm import Session from sqlalchemy.orm.exc import ObjectDeletedError from sqlalchemy_utils.functions import drop_database @@ -23,6 +24,7 @@ from fides.api.models.sql_models import System from fides.api.service.connectors import PostgreSQLConnector from fides.config import CONFIG +from tests.ops.test_helpers.dataset_utils import remove_primary_keys from tests.ops.test_helpers.db_utils import seed_postgres_data from .application_fixtures import integration_secrets @@ -111,6 +113,34 @@ def postgres_example_test_dataset_config_read_access( ctl_dataset.delete(db=db) +@pytest.fixture +def postgres_example_test_dataset_config_read_access_without_primary_keys( + read_connection_config: ConnectionConfig, + db: Session, + example_datasets: List[Dict], +) -> Generator: + postgres_dataset = example_datasets[0] + fides_key = postgres_dataset["fides_key"] + + dataset = FideslangDataset(**postgres_dataset) + updated_dataset = remove_primary_keys(dataset) + ctl_dataset = CtlDataset.create_from_dataset_dict( + db, updated_dataset.model_dump(mode="json") + ) + + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": read_connection_config.id, + "fides_key": fides_key, + "ctl_dataset_id": ctl_dataset.id, + }, + ) + yield dataset + dataset.delete(db=db) + ctl_dataset.delete(db=db) + + @pytest.fixture def postgres_example_test_dataset_config_skipped_login_collection( read_connection_config: ConnectionConfig, diff --git a/tests/ops/integration_tests/test_mariadb_task.py b/tests/ops/integration_tests/test_mariadb_task.py new file mode 100644 index 0000000000..3951a2830a --- /dev/null +++ b/tests/ops/integration_tests/test_mariadb_task.py @@ -0,0 +1,101 @@ +import pytest + +from fides.api.models.privacy_request import ExecutionLog + +from ...conftest import access_runner_tester +from ..graph.graph_test_util import assert_rows_match, records_matching_fields +from ..task.traversal_data import integration_db_graph + + +@pytest.mark.integration_mariadb +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], +) +async def test_mariadb_access_request_task( + db, + policy, + connection_config_mariadb, + mariadb_integration_db, + dsr_version, + request, + privacy_request, +) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + v = access_runner_tester( + privacy_request, + policy, + integration_db_graph("my_maria_db_1"), + [connection_config_mariadb], + {"email": "customer-1@example.com"}, + db, + ) + + assert_rows_match( + v["my_maria_db_1:address"], + min_size=2, + keys=["id", "street", "city", "state", "zip"], + ) + assert_rows_match( + v["my_maria_db_1:orders"], + min_size=3, + keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], + ) + assert_rows_match( + v["my_maria_db_1:payment_card"], + min_size=2, + keys=["id", "name", "ccn", "customer_id", "billing_address_id"], + ) + assert_rows_match( + v["my_maria_db_1:customer"], + min_size=1, + keys=["id", "name", "email", "address_id"], + ) + + # links + assert v["my_maria_db_1:customer"][0]["email"] == "customer-1@example.com" + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, dataset_name="my_maria_db_1", collection_name="customer" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_maria_db_1", collection_name="address" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_maria_db_1", collection_name="orders" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, + dataset_name="my_maria_db_1", + collection_name="payment_card", + ) + ) + > 0 + ) diff --git a/tests/ops/integration_tests/test_mssql_task.py b/tests/ops/integration_tests/test_mssql_task.py new file mode 100644 index 0000000000..6bc23eeda0 --- /dev/null +++ b/tests/ops/integration_tests/test_mssql_task.py @@ -0,0 +1,101 @@ +import pytest + +from fides.api.models.privacy_request import ExecutionLog + +from ...conftest import access_runner_tester +from ..graph.graph_test_util import assert_rows_match, records_matching_fields +from ..task.traversal_data import integration_db_graph + + +@pytest.mark.integration_mssql +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], +) +async def test_mssql_access_request_task( + db, + policy, + connection_config_mssql, + mssql_integration_db, + privacy_request, + dsr_version, + request, +) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + v = access_runner_tester( + privacy_request, + policy, + integration_db_graph("my_mssql_db_1"), + [connection_config_mssql], + {"email": "customer-1@example.com"}, + db, + ) + + assert_rows_match( + v["my_mssql_db_1:address"], + min_size=2, + keys=["id", "street", "city", "state", "zip"], + ) + assert_rows_match( + v["my_mssql_db_1:orders"], + min_size=3, + keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], + ) + assert_rows_match( + v["my_mssql_db_1:payment_card"], + min_size=2, + keys=["id", "name", "ccn", "customer_id", "billing_address_id"], + ) + assert_rows_match( + v["my_mssql_db_1:customer"], + min_size=1, + keys=["id", "name", "email", "address_id"], + ) + + # links + assert v["my_mssql_db_1:customer"][0]["email"] == "customer-1@example.com" + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mssql_db_1", collection_name="customer" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mssql_db_1", collection_name="address" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mssql_db_1", collection_name="orders" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, + dataset_name="my_mssql_db_1", + collection_name="payment_card", + ) + ) + > 0 + ) diff --git a/tests/ops/integration_tests/test_mysql_task.py b/tests/ops/integration_tests/test_mysql_task.py new file mode 100644 index 0000000000..40551dd4d9 --- /dev/null +++ b/tests/ops/integration_tests/test_mysql_task.py @@ -0,0 +1,101 @@ +import pytest + +from fides.api.models.privacy_request import ExecutionLog + +from ...conftest import access_runner_tester +from ..graph.graph_test_util import assert_rows_match, records_matching_fields +from ..task.traversal_data import integration_db_graph + + +@pytest.mark.integration +@pytest.mark.integration_mysql +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], +) +async def test_mysql_access_request_task( + db, + policy, + connection_config_mysql, + mysql_integration_db, + privacy_request, + dsr_version, + request, +) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + v = access_runner_tester( + privacy_request, + policy, + integration_db_graph("my_mysql_db_1"), + [connection_config_mysql], + {"email": "customer-1@example.com"}, + db, + ) + + assert_rows_match( + v["my_mysql_db_1:address"], + min_size=2, + keys=["id", "street", "city", "state", "zip"], + ) + assert_rows_match( + v["my_mysql_db_1:orders"], + min_size=3, + keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], + ) + assert_rows_match( + v["my_mysql_db_1:payment_card"], + min_size=2, + keys=["id", "name", "ccn", "customer_id", "billing_address_id"], + ) + assert_rows_match( + v["my_mysql_db_1:customer"], + min_size=1, + keys=["id", "name", "email", "address_id"], + ) + + # links + assert v["my_mysql_db_1:customer"][0]["email"] == "customer-1@example.com" + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mysql_db_1", collection_name="customer" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mysql_db_1", collection_name="address" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mysql_db_1", collection_name="orders" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, + dataset_name="my_mysql_db_1", + collection_name="payment_card", + ) + ) + > 0 + ) diff --git a/tests/ops/integration_tests/test_scylladb_task.py b/tests/ops/integration_tests/test_scylladb_task.py new file mode 100644 index 0000000000..8ced1317ad --- /dev/null +++ b/tests/ops/integration_tests/test_scylladb_task.py @@ -0,0 +1,190 @@ +import pytest +from sqlalchemy.orm import Session + +from fides.api.models.privacy_request import ExecutionLogStatus, PrivacyRequest +from fides.api.service.connectors.scylla_connector import ScyllaConnectorMissingKeyspace +from fides.api.task.graph_task import get_cached_data_for_erasures + +from ...conftest import access_runner_tester, erasure_runner_tester +from ..graph.graph_test_util import assert_rows_match, erasure_policy +from ..task.traversal_data import integration_scylladb_graph + + +@pytest.mark.integration +@pytest.mark.integration_scylladb +@pytest.mark.asyncio +class TestScyllaDSRs: + @pytest.mark.parametrize( + "dsr_version", + ["use_dsr_2_0"], + ) + async def test_scylladb_access_request_task_no_keyspace_dsr2( + self, + db: Session, + policy, + integration_scylladb_config, + scylladb_integration_no_keyspace, + privacy_request, + dsr_version, + request, + ) -> None: + request.getfixturevalue(dsr_version) + + with pytest.raises(ScyllaConnectorMissingKeyspace) as err: + v = access_runner_tester( + privacy_request, + policy, + integration_scylladb_graph("scylla_example"), + [integration_scylladb_config], + {"email": "customer-1@example.com"}, + db, + ) + + assert ( + "No keyspace provided in the ScyllaDB configuration for connector scylla_example" + in str(err.value) + ) + + @pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0"], + ) + async def test_scylladb_access_request_task_no_keyspace_dsr3( + self, + db, + policy, + integration_scylladb_config, + scylladb_integration_no_keyspace, + privacy_request: PrivacyRequest, + dsr_version, + request, + ) -> None: + request.getfixturevalue(dsr_version) + v = access_runner_tester( + privacy_request, + policy, + integration_scylladb_graph("scylla_example"), + [integration_scylladb_config], + {"email": "customer-1@example.com"}, + db, + ) + + assert v == {} + assert ( + privacy_request.access_tasks.count() == 6 + ) # There's 4 tables plus the root and terminal "dummy" tasks + + # Root task should be completed + assert privacy_request.access_tasks.first().collection_name == "__ROOT__" + assert ( + privacy_request.access_tasks.first().status == ExecutionLogStatus.complete + ) + + # All other tasks should be error + for access_task in privacy_request.access_tasks.offset(1): + assert access_task.status == ExecutionLogStatus.error + + @pytest.mark.parametrize( + "dsr_version", + ["use_dsr_2_0", "use_dsr_3_0"], + ) + async def test_scylladb_access_request_task( + self, + db, + policy, + integration_scylladb_config_with_keyspace, + scylla_reset_db, + scylladb_integration_with_keyspace, + privacy_request, + dsr_version, + request, + ) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + results = access_runner_tester( + privacy_request, + policy, + integration_scylladb_graph("scylla_example_with_keyspace"), + [integration_scylladb_config_with_keyspace], + {"email": "customer-1@example.com"}, + db, + ) + + assert_rows_match( + results["scylla_example_with_keyspace:users"], + min_size=1, + keys=[ + "age", + "alternative_contacts", + "do_not_contact", + "email", + "name", + "last_contacted", + "logins", + "states_lived", + ], + ) + assert_rows_match( + results["scylla_example_with_keyspace:user_activity"], + min_size=3, + keys=["timestamp", "user_agent", "activity_type"], + ) + assert_rows_match( + results["scylla_example_with_keyspace:payment_methods"], + min_size=2, + keys=["card_number", "expiration_date"], + ) + assert_rows_match( + results["scylla_example_with_keyspace:orders"], + min_size=2, + keys=["order_amount", "order_date", "order_description"], + ) + + @pytest.mark.parametrize( + "dsr_version", + ["use_dsr_2_0", "use_dsr_3_0"], + ) + async def test_scylladb_erasure_task( + self, + db, + integration_scylladb_config_with_keyspace, + scylladb_integration_with_keyspace, + scylla_reset_db, + privacy_request, + dsr_version, + request, + ): + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + seed_email = "customer-1@example.com" + + policy = erasure_policy( + db, "user.name", "user.behavior", "user.device", "user.payment" + ) + privacy_request.policy_id = policy.id + privacy_request.save(db) + + graph = integration_scylladb_graph("scylla_example_with_keyspace") + access_runner_tester( + privacy_request, + policy, + integration_scylladb_graph("scylla_example_with_keyspace"), + [integration_scylladb_config_with_keyspace], + {"email": seed_email}, + db, + ) + results = erasure_runner_tester( + privacy_request, + policy, + graph, + [integration_scylladb_config_with_keyspace], + {"email": seed_email}, + get_cached_data_for_erasures(privacy_request.id), + db, + ) + assert results == { + "scylla_example_with_keyspace:user_activity": 3, + "scylla_example_with_keyspace:users": 1, + "scylla_example_with_keyspace:payment_methods": 2, + "scylla_example_with_keyspace:orders": 2, + } diff --git a/tests/ops/integration_tests/test_sql_task.py b/tests/ops/integration_tests/test_sql_task.py index 298d77229a..bbac30df82 100644 --- a/tests/ops/integration_tests/test_sql_task.py +++ b/tests/ops/integration_tests/test_sql_task.py @@ -6,30 +6,16 @@ import pytest from fideslang import Dataset from sqlalchemy import text -from sqlalchemy.orm import Session - -from fides.api.graph.config import ( - Collection, - CollectionAddress, - FieldAddress, - GraphDataset, - ScalarField, -) + +from fides.api.graph.config import Collection, FieldAddress, GraphDataset, ScalarField from fides.api.graph.data_type import DataType, StringTypeConverter from fides.api.graph.graph import DatasetGraph, Edge, Node from fides.api.graph.traversal import TraversalNode from fides.api.models.connectionconfig import ConnectionConfig from fides.api.models.datasetconfig import convert_dataset_to_graph from fides.api.models.policy import ActionType, Policy, Rule, RuleTarget -from fides.api.models.privacy_request import ( - ExecutionLog, - ExecutionLogStatus, - PrivacyRequest, - PrivacyRequestStatus, - RequestTask, -) +from fides.api.models.privacy_request import ExecutionLog, RequestTask from fides.api.service.connectors import get_connector -from fides.api.service.connectors.scylla_connector import ScyllaConnectorMissingKeyspace from fides.api.task.filter_results import filter_data_categories from fides.api.task.graph_task import get_cached_data_for_erasures from fides.config import CONFIG @@ -42,12 +28,7 @@ field, records_matching_fields, ) -from ..task.traversal_data import ( - integration_db_graph, - integration_scylladb_graph, - postgres_db_graph_dataset, - str_converter, -) +from ..task.traversal_data import integration_db_graph, postgres_db_graph_dataset @pytest.mark.integration_postgres @@ -57,7 +38,7 @@ "dsr_version", ["use_dsr_3_0", "use_dsr_2_0"], ) -async def test_sql_erasure_ignores_collections_without_pk( +async def test_sql_erasure_does_not_ignore_collections_without_pk( db, postgres_inserts, integration_postgres_config, @@ -116,7 +97,7 @@ async def test_sql_erasure_ignores_collections_without_pk( .all() ) logs = [log.__dict__ for log in logs] - # since address has no primary_key=True field, it's erasure is skipped + # erasure is not skipped since primary_key is not required assert ( len( records_matching_fields( @@ -126,13 +107,13 @@ async def test_sql_erasure_ignores_collections_without_pk( message="No values were erased since no primary key was defined for this collection", ) ) - == 1 + == 0 ) assert v == { "postgres_example:customer": 1, "postgres_example:payment_card": 0, "postgres_example:orders": 0, - "postgres_example:address": 0, + "postgres_example:address": 2, } @@ -456,516 +437,55 @@ async def test_postgres_privacy_requests_against_non_default_schema( db, ) - # Confirm data retrieved from backup_schema, not public schema. This data only exists in the backup_schema. - assert access_results == { - f"{database_name}:address": [ - { - "id": 7, - "street": "Test Street", - "city": "Test Town", - "state": "TX", - "zip": "79843", - } - ], - f"{database_name}:payment_card": [], - f"{database_name}:orders": [], - f"{database_name}:customer": [ - { - "id": 1, - "name": "Johanna Customer", - "email": "customer-500@example.com", - "address_id": 7, - } - ], - } - - erasure_results = erasure_runner_tester( - privacy_request_with_erasure_policy, - erasure_policy, - graph, - [postgres_connection_config_with_schema], - {"email": customer_email}, - get_cached_data_for_erasures(privacy_request_with_erasure_policy.id), - db, - ) - - # Confirm record masked in non-default schema - assert erasure_results == { - f"{database_name}:customer": 1, - f"{database_name}:payment_card": 0, - f"{database_name}:orders": 0, - f"{database_name}:address": 0, - }, "Only one record on customer table has targeted data category" - customer_records = postgres_integration_db.execute( - text("select * from backup_schema.customer where id = 1;") - ) - johanna_record = [c for c in customer_records][0] - assert johanna_record.email == customer_email # Not masked - assert johanna_record.name is None # Masked by erasure request - - -@pytest.mark.integration_mssql -@pytest.mark.integration -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_mssql_access_request_task( - db, - policy, - connection_config_mssql, - mssql_integration_db, - privacy_request, - dsr_version, - request, -) -> None: - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - v = access_runner_tester( - privacy_request, - policy, - integration_db_graph("my_mssql_db_1"), - [connection_config_mssql], - {"email": "customer-1@example.com"}, - db, - ) - - assert_rows_match( - v["my_mssql_db_1:address"], - min_size=2, - keys=["id", "street", "city", "state", "zip"], - ) - assert_rows_match( - v["my_mssql_db_1:orders"], - min_size=3, - keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], - ) - assert_rows_match( - v["my_mssql_db_1:payment_card"], - min_size=2, - keys=["id", "name", "ccn", "customer_id", "billing_address_id"], - ) - assert_rows_match( - v["my_mssql_db_1:customer"], - min_size=1, - keys=["id", "name", "email", "address_id"], - ) - - # links - assert v["my_mssql_db_1:customer"][0]["email"] == "customer-1@example.com" - - logs = ( - ExecutionLog.query(db=db) - .filter(ExecutionLog.privacy_request_id == privacy_request.id) - .all() - ) - - logs = [log.__dict__ for log in logs] - assert ( - len( - records_matching_fields( - logs, dataset_name="my_mssql_db_1", collection_name="customer" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, dataset_name="my_mssql_db_1", collection_name="address" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, dataset_name="my_mssql_db_1", collection_name="orders" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, - dataset_name="my_mssql_db_1", - collection_name="payment_card", - ) - ) - > 0 - ) - - -@pytest.mark.integration -@pytest.mark.integration_mysql -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_mysql_access_request_task( - db, - policy, - connection_config_mysql, - mysql_integration_db, - privacy_request, - dsr_version, - request, -) -> None: - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - v = access_runner_tester( - privacy_request, - policy, - integration_db_graph("my_mysql_db_1"), - [connection_config_mysql], - {"email": "customer-1@example.com"}, - db, - ) - - assert_rows_match( - v["my_mysql_db_1:address"], - min_size=2, - keys=["id", "street", "city", "state", "zip"], - ) - assert_rows_match( - v["my_mysql_db_1:orders"], - min_size=3, - keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], - ) - assert_rows_match( - v["my_mysql_db_1:payment_card"], - min_size=2, - keys=["id", "name", "ccn", "customer_id", "billing_address_id"], - ) - assert_rows_match( - v["my_mysql_db_1:customer"], - min_size=1, - keys=["id", "name", "email", "address_id"], - ) - - # links - assert v["my_mysql_db_1:customer"][0]["email"] == "customer-1@example.com" - - logs = ( - ExecutionLog.query(db=db) - .filter(ExecutionLog.privacy_request_id == privacy_request.id) - .all() - ) - - logs = [log.__dict__ for log in logs] - assert ( - len( - records_matching_fields( - logs, dataset_name="my_mysql_db_1", collection_name="customer" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, dataset_name="my_mysql_db_1", collection_name="address" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, dataset_name="my_mysql_db_1", collection_name="orders" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, - dataset_name="my_mysql_db_1", - collection_name="payment_card", - ) - ) - > 0 - ) - - -@pytest.mark.integration_mariadb -@pytest.mark.integration -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_mariadb_access_request_task( - db, - policy, - connection_config_mariadb, - mariadb_integration_db, - dsr_version, - request, - privacy_request, -) -> None: - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - v = access_runner_tester( - privacy_request, - policy, - integration_db_graph("my_maria_db_1"), - [connection_config_mariadb], - {"email": "customer-1@example.com"}, - db, - ) - - assert_rows_match( - v["my_maria_db_1:address"], - min_size=2, - keys=["id", "street", "city", "state", "zip"], - ) - assert_rows_match( - v["my_maria_db_1:orders"], - min_size=3, - keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], - ) - assert_rows_match( - v["my_maria_db_1:payment_card"], - min_size=2, - keys=["id", "name", "ccn", "customer_id", "billing_address_id"], - ) - assert_rows_match( - v["my_maria_db_1:customer"], - min_size=1, - keys=["id", "name", "email", "address_id"], - ) - - # links - assert v["my_maria_db_1:customer"][0]["email"] == "customer-1@example.com" - - logs = ( - ExecutionLog.query(db=db) - .filter(ExecutionLog.privacy_request_id == privacy_request.id) - .all() - ) - - logs = [log.__dict__ for log in logs] - assert ( - len( - records_matching_fields( - logs, dataset_name="my_maria_db_1", collection_name="customer" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, dataset_name="my_maria_db_1", collection_name="address" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, dataset_name="my_maria_db_1", collection_name="orders" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, - dataset_name="my_maria_db_1", - collection_name="payment_card", - ) - ) - > 0 - ) - - -@pytest.mark.integration -@pytest.mark.integration_scylladb -@pytest.mark.asyncio -class TestScyllaDSRs: - @pytest.mark.parametrize( - "dsr_version", - ["use_dsr_2_0"], - ) - async def test_scylladb_access_request_task_no_keyspace_dsr2( - self, - db: Session, - policy, - integration_scylladb_config, - scylladb_integration_no_keyspace, - privacy_request, - dsr_version, - request, - ) -> None: - request.getfixturevalue(dsr_version) - - with pytest.raises(ScyllaConnectorMissingKeyspace) as err: - v = access_runner_tester( - privacy_request, - policy, - integration_scylladb_graph("scylla_example"), - [integration_scylladb_config], - {"email": "customer-1@example.com"}, - db, - ) - - assert ( - "No keyspace provided in the ScyllaDB configuration for connector scylla_example" - in str(err.value) - ) - - @pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0"], - ) - async def test_scylladb_access_request_task_no_keyspace_dsr3( - self, - db, - policy, - integration_scylladb_config, - scylladb_integration_no_keyspace, - privacy_request: PrivacyRequest, - dsr_version, - request, - ) -> None: - request.getfixturevalue(dsr_version) - v = access_runner_tester( - privacy_request, - policy, - integration_scylladb_graph("scylla_example"), - [integration_scylladb_config], - {"email": "customer-1@example.com"}, - db, - ) - - assert v == {} - assert ( - privacy_request.access_tasks.count() == 6 - ) # There's 4 tables plus the root and terminal "dummy" tasks - - # Root task should be completed - assert privacy_request.access_tasks.first().collection_name == "__ROOT__" - assert ( - privacy_request.access_tasks.first().status == ExecutionLogStatus.complete - ) - - # All other tasks should be error - for access_task in privacy_request.access_tasks.offset(1): - assert access_task.status == ExecutionLogStatus.error - - @pytest.mark.parametrize( - "dsr_version", - ["use_dsr_2_0", "use_dsr_3_0"], - ) - async def test_scylladb_access_request_task( - self, - db, - policy, - integration_scylladb_config_with_keyspace, - scylla_reset_db, - scylladb_integration_with_keyspace, - privacy_request, - dsr_version, - request, - ) -> None: - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - results = access_runner_tester( - privacy_request, - policy, - integration_scylladb_graph("scylla_example_with_keyspace"), - [integration_scylladb_config_with_keyspace], - {"email": "customer-1@example.com"}, - db, - ) - - assert_rows_match( - results["scylla_example_with_keyspace:users"], - min_size=1, - keys=[ - "age", - "alternative_contacts", - "do_not_contact", - "email", - "name", - "last_contacted", - "logins", - "states_lived", - ], - ) - assert_rows_match( - results["scylla_example_with_keyspace:user_activity"], - min_size=3, - keys=["timestamp", "user_agent", "activity_type"], - ) - assert_rows_match( - results["scylla_example_with_keyspace:payment_methods"], - min_size=2, - keys=["card_number", "expiration_date"], - ) - assert_rows_match( - results["scylla_example_with_keyspace:orders"], - min_size=2, - keys=["order_amount", "order_date", "order_description"], - ) - - @pytest.mark.parametrize( - "dsr_version", - ["use_dsr_2_0", "use_dsr_3_0"], - ) - async def test_scylladb_erasure_task( - self, - db, - integration_scylladb_config_with_keyspace, - scylladb_integration_with_keyspace, - scylla_reset_db, - privacy_request, - dsr_version, - request, - ): - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - seed_email = "customer-1@example.com" + # Confirm data retrieved from backup_schema, not public schema. This data only exists in the backup_schema. + assert access_results == { + f"{database_name}:address": [ + { + "id": 7, + "street": "Test Street", + "city": "Test Town", + "state": "TX", + "zip": "79843", + } + ], + f"{database_name}:payment_card": [], + f"{database_name}:orders": [], + f"{database_name}:customer": [ + { + "id": 1, + "name": "Johanna Customer", + "email": "customer-500@example.com", + "address_id": 7, + } + ], + } - policy = erasure_policy( - db, "user.name", "user.behavior", "user.device", "user.payment" - ) - privacy_request.policy_id = policy.id - privacy_request.save(db) + erasure_results = erasure_runner_tester( + privacy_request_with_erasure_policy, + erasure_policy, + graph, + [postgres_connection_config_with_schema], + {"email": customer_email}, + get_cached_data_for_erasures(privacy_request_with_erasure_policy.id), + db, + ) - graph = integration_scylladb_graph("scylla_example_with_keyspace") - access_runner_tester( - privacy_request, - policy, - integration_scylladb_graph("scylla_example_with_keyspace"), - [integration_scylladb_config_with_keyspace], - {"email": seed_email}, - db, - ) - results = erasure_runner_tester( - privacy_request, - policy, - graph, - [integration_scylladb_config_with_keyspace], - {"email": seed_email}, - get_cached_data_for_erasures(privacy_request.id), - db, - ) - assert results == { - "scylla_example_with_keyspace:user_activity": 3, - "scylla_example_with_keyspace:users": 1, - "scylla_example_with_keyspace:payment_methods": 2, - "scylla_example_with_keyspace:orders": 2, - } + # Confirm record masked in non-default schema + assert erasure_results == { + f"{database_name}:customer": 1, + f"{database_name}:payment_card": 0, + f"{database_name}:orders": 0, + f"{database_name}:address": 0, + }, "Only one record on customer table has targeted data category" + customer_records = postgres_integration_db.execute( + text("select * from backup_schema.customer where id = 1;") + ) + johanna_record = [c for c in customer_records][0] + assert johanna_record.email == customer_email # Not masked + assert johanna_record.name is None # Masked by erasure request +@pytest.mark.integration_postgres @pytest.mark.integration @pytest.mark.asyncio @pytest.mark.parametrize( @@ -1565,18 +1085,17 @@ async def test_retry_erasure( execution_logs = db.query(ExecutionLog).filter_by( privacy_request_id=privacy_request.id, action_type=ActionType.erasure ) - assert 40 == execution_logs.count() + assert 44 == execution_logs.count() - # These nodes were able to complete because they didn't have a PK - nothing to erase visit_logs = execution_logs.filter_by(collection_name="visit") - assert {"in_processing", "complete"} == { + assert ["in_processing", "retrying", "retrying", "error"] == [ el.status.value for el in visit_logs - } + ] order_item_logs = execution_logs.filter_by(collection_name="order_item") - assert {"in_processing", "complete"} == { + assert ["in_processing", "retrying", "retrying", "error"] == [ el.status.value for el in order_item_logs - } + ] # Address log mask data couldn't run, attempted to retry twice per configuration address_logs = execution_logs.filter_by(collection_name="address").order_by( ExecutionLog.created_at @@ -1585,297 +1104,19 @@ async def test_retry_erasure( el.status.value for el in address_logs ] - # Downstream request tasks were marked as error. Some tasks completed because there is no PK - # on their collection and we can't erase - assert {rt.status.value for rt in privacy_request.erasure_tasks} == { + # Downstream request tasks (other than __ROOT__) were marked as error. + assert [rt.status.value for rt in privacy_request.erasure_tasks] == [ "complete", "error", "error", "error", - "complete", "error", "error", "error", "error", "error", - "complete", "error", "error", - } - - -@pytest.mark.integration_timescale -@pytest.mark.integration -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_timescale_access_request_task( - db, - policy, - timescale_connection_config, - timescale_integration_db, - privacy_request, - dsr_version, - request, -) -> None: - database_name = "my_timescale_db_1" - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - v = access_runner_tester( - privacy_request, - policy, - integration_db_graph(database_name), - [timescale_connection_config], - {"email": "customer-1@example.com"}, - db, - ) - - assert_rows_match( - v[f"{database_name}:address"], - min_size=2, - keys=["id", "street", "city", "state", "zip"], - ) - assert_rows_match( - v[f"{database_name}:orders"], - min_size=3, - keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], - ) - assert_rows_match( - v[f"{database_name}:payment_card"], - min_size=2, - keys=["id", "name", "ccn", "customer_id", "billing_address_id"], - ) - assert_rows_match( - v[f"{database_name}:customer"], - min_size=1, - keys=["id", "name", "email", "address_id"], - ) - - # links - assert v[f"{database_name}:customer"][0]["email"] == "customer-1@example.com" - - logs = ( - ExecutionLog.query(db=db) - .filter(ExecutionLog.privacy_request_id == privacy_request.id) - .all() - ) - - logs = [log.__dict__ for log in logs] - - assert ( - len( - records_matching_fields( - logs, dataset_name=database_name, collection_name="customer" - ) - ) - > 0 - ) - - assert ( - len( - records_matching_fields( - logs, dataset_name=database_name, collection_name="address" - ) - ) - > 0 - ) - - assert ( - len( - records_matching_fields( - logs, dataset_name=database_name, collection_name="orders" - ) - ) - > 0 - ) - - assert ( - len( - records_matching_fields( - logs, - dataset_name=database_name, - collection_name="payment_card", - ) - ) - > 0 - ) - - -@pytest.mark.integration_timescale -@pytest.mark.integration -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_timescale_erasure_request_task( - db, - erasure_policy, - timescale_connection_config, - timescale_integration_db, - privacy_request_with_erasure_policy, - dsr_version, - request, -) -> None: - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - rule = erasure_policy.rules[0] - target = rule.targets[0] - target.data_category = "user" - target.save(db) - - database_name = "my_timescale_db_1" - - dataset = postgres_db_graph_dataset(database_name, timescale_connection_config.key) - - # Set some data categories on fields that will be targeted by the policy above - field([dataset], database_name, "customer", "name").data_categories = ["user.name"] - field([dataset], database_name, "address", "street").data_categories = ["user"] - field([dataset], database_name, "payment_card", "ccn").data_categories = ["user"] - - graph = DatasetGraph(dataset) - - v = access_runner_tester( - privacy_request_with_erasure_policy, - erasure_policy, - graph, - [timescale_connection_config], - {"email": "customer-1@example.com"}, - db, - ) - - v = erasure_runner_tester( - privacy_request_with_erasure_policy, - erasure_policy, - graph, - [timescale_connection_config], - {"email": "customer-1@example.com"}, - get_cached_data_for_erasures(privacy_request_with_erasure_policy.id), - db, - ) - assert v == { - f"{database_name}:customer": 1, - f"{database_name}:orders": 0, - f"{database_name}:payment_card": 2, - f"{database_name}:address": 2, - }, "No erasure on orders table - no data categories targeted" - - # Verify masking in appropriate tables - address_cursor = timescale_integration_db.execute( - text("select * from address where id in (1, 2)") - ) - for address in address_cursor: - assert address.street is None # Masked due to matching data category - assert address.state is not None - assert address.city is not None - assert address.zip is not None - - customer_cursor = timescale_integration_db.execute( - text("select * from customer where id = 1") - ) - customer = [customer for customer in customer_cursor][0] - assert customer.name is None # Masked due to matching data category - assert customer.email == "customer-1@example.com" - assert customer.address_id is not None - - payment_card_cursor = timescale_integration_db.execute( - text("select * from payment_card where id in ('pay_aaa-aaa', 'pay_bbb-bbb')") - ) - payment_cards = [card for card in payment_card_cursor] - assert all( - [card.ccn is None for card in payment_cards] - ) # Masked due to matching data category - assert not any([card.name is None for card in payment_cards]) is None - - -@pytest.mark.integration_timescale -@pytest.mark.integration -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_timescale_query_and_mask_hypertable( - db, - erasure_policy, - timescale_connection_config, - timescale_integration_db, - privacy_request_with_erasure_policy, - dsr_version, - request, -) -> None: - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - database_name = "my_timescale_db_1" - - dataset = postgres_db_graph_dataset(database_name, timescale_connection_config.key) - # For this test, add a new collection to our standard dataset corresponding to the - # "onsite_personnel" timescale hypertable - onsite_personnel_collection = Collection( - name="onsite_personnel", - fields=[ - ScalarField( - name="responsible", data_type_converter=str_converter, identity="email" - ), - ScalarField( - name="time", data_type_converter=str_converter, primary_key=True - ), - ], - ) - - dataset.collections.append(onsite_personnel_collection) - graph = DatasetGraph(dataset) - rule = erasure_policy.rules[0] - target = rule.targets[0] - target.data_category = "user" - target.save(db) - # Update data category on responsible field - field( - [dataset], database_name, "onsite_personnel", "responsible" - ).data_categories = ["user.contact.email"] - - access_results = access_runner_tester( - privacy_request_with_erasure_policy, - erasure_policy, - graph, - [timescale_connection_config], - {"email": "employee-1@example.com"}, - db, - ) - - # Demonstrate hypertable can be queried - assert access_results[f"{database_name}:onsite_personnel"] == [ - {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 1, 9, 0)}, - {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 2, 9, 0)}, - {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 3, 9, 0)}, - {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 5, 9, 0)}, - ] - - # Run an erasure on the hypertable targeting the responsible field - v = erasure_runner_tester( - privacy_request_with_erasure_policy, - erasure_policy, - graph, - [timescale_connection_config], - {"email": "employee-1@example.com"}, - get_cached_data_for_erasures(privacy_request_with_erasure_policy.id), - db, - ) - - assert v == { - f"{database_name}:customer": 0, - f"{database_name}:orders": 0, - f"{database_name}:payment_card": 0, - f"{database_name}:address": 0, - f"{database_name}:onsite_personnel": 4, - }, "onsite_personnel.responsible was the only targeted data category" - - personnel_records = timescale_integration_db.execute( - text("select * from onsite_personnel") - ) - for record in personnel_records: - assert ( - record.responsible != "employee-1@example.com" - ) # These emails have all been masked + "error", + "error", + ] diff --git a/tests/ops/integration_tests/test_timescale_task.py b/tests/ops/integration_tests/test_timescale_task.py new file mode 100644 index 0000000000..97af65ce65 --- /dev/null +++ b/tests/ops/integration_tests/test_timescale_task.py @@ -0,0 +1,294 @@ +from datetime import datetime + +import pytest +from sqlalchemy import text + +from fides.api.graph.config import Collection, ScalarField +from fides.api.graph.graph import DatasetGraph +from fides.api.models.privacy_request import ExecutionLog +from fides.api.task.graph_task import get_cached_data_for_erasures + +from ...conftest import access_runner_tester, erasure_runner_tester +from ..graph.graph_test_util import assert_rows_match, field, records_matching_fields +from ..task.traversal_data import ( + integration_db_graph, + postgres_db_graph_dataset, + str_converter, +) + + +@pytest.mark.integration_timescale +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], +) +async def test_timescale_access_request_task( + db, + policy, + timescale_connection_config, + timescale_integration_db, + privacy_request, + dsr_version, + request, +) -> None: + database_name = "my_timescale_db_1" + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + v = access_runner_tester( + privacy_request, + policy, + integration_db_graph(database_name), + [timescale_connection_config], + {"email": "customer-1@example.com"}, + db, + ) + + assert_rows_match( + v[f"{database_name}:address"], + min_size=2, + keys=["id", "street", "city", "state", "zip"], + ) + assert_rows_match( + v[f"{database_name}:orders"], + min_size=3, + keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], + ) + assert_rows_match( + v[f"{database_name}:payment_card"], + min_size=2, + keys=["id", "name", "ccn", "customer_id", "billing_address_id"], + ) + assert_rows_match( + v[f"{database_name}:customer"], + min_size=1, + keys=["id", "name", "email", "address_id"], + ) + + # links + assert v[f"{database_name}:customer"][0]["email"] == "customer-1@example.com" + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + + assert ( + len( + records_matching_fields( + logs, dataset_name=database_name, collection_name="customer" + ) + ) + > 0 + ) + + assert ( + len( + records_matching_fields( + logs, dataset_name=database_name, collection_name="address" + ) + ) + > 0 + ) + + assert ( + len( + records_matching_fields( + logs, dataset_name=database_name, collection_name="orders" + ) + ) + > 0 + ) + + assert ( + len( + records_matching_fields( + logs, + dataset_name=database_name, + collection_name="payment_card", + ) + ) + > 0 + ) + + +@pytest.mark.integration_timescale +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], +) +async def test_timescale_erasure_request_task( + db, + erasure_policy, + timescale_connection_config, + timescale_integration_db, + privacy_request_with_erasure_policy, + dsr_version, + request, +) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + rule = erasure_policy.rules[0] + target = rule.targets[0] + target.data_category = "user" + target.save(db) + + database_name = "my_timescale_db_1" + + dataset = postgres_db_graph_dataset(database_name, timescale_connection_config.key) + + # Set some data categories on fields that will be targeted by the policy above + field([dataset], database_name, "customer", "name").data_categories = ["user.name"] + field([dataset], database_name, "address", "street").data_categories = ["user"] + field([dataset], database_name, "payment_card", "ccn").data_categories = ["user"] + + graph = DatasetGraph(dataset) + + v = access_runner_tester( + privacy_request_with_erasure_policy, + erasure_policy, + graph, + [timescale_connection_config], + {"email": "customer-1@example.com"}, + db, + ) + + v = erasure_runner_tester( + privacy_request_with_erasure_policy, + erasure_policy, + graph, + [timescale_connection_config], + {"email": "customer-1@example.com"}, + get_cached_data_for_erasures(privacy_request_with_erasure_policy.id), + db, + ) + assert v == { + f"{database_name}:customer": 1, + f"{database_name}:orders": 0, + f"{database_name}:payment_card": 2, + f"{database_name}:address": 2, + }, "No erasure on orders table - no data categories targeted" + + # Verify masking in appropriate tables + address_cursor = timescale_integration_db.execute( + text("select * from address where id in (1, 2)") + ) + for address in address_cursor: + assert address.street is None # Masked due to matching data category + assert address.state is not None + assert address.city is not None + assert address.zip is not None + + customer_cursor = timescale_integration_db.execute( + text("select * from customer where id = 1") + ) + customer = [customer for customer in customer_cursor][0] + assert customer.name is None # Masked due to matching data category + assert customer.email == "customer-1@example.com" + assert customer.address_id is not None + + payment_card_cursor = timescale_integration_db.execute( + text("select * from payment_card where id in ('pay_aaa-aaa', 'pay_bbb-bbb')") + ) + payment_cards = [card for card in payment_card_cursor] + assert all( + [card.ccn is None for card in payment_cards] + ) # Masked due to matching data category + assert not any([card.name is None for card in payment_cards]) is None + + +@pytest.mark.integration_timescale +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], +) +async def test_timescale_query_and_mask_hypertable( + db, + erasure_policy, + timescale_connection_config, + timescale_integration_db, + privacy_request_with_erasure_policy, + dsr_version, + request, +) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + database_name = "my_timescale_db_1" + + dataset = postgres_db_graph_dataset(database_name, timescale_connection_config.key) + # For this test, add a new collection to our standard dataset corresponding to the + # "onsite_personnel" timescale hypertable + onsite_personnel_collection = Collection( + name="onsite_personnel", + fields=[ + ScalarField( + name="responsible", data_type_converter=str_converter, identity="email" + ), + ScalarField( + name="time", data_type_converter=str_converter, primary_key=True + ), + ], + ) + + dataset.collections.append(onsite_personnel_collection) + graph = DatasetGraph(dataset) + rule = erasure_policy.rules[0] + target = rule.targets[0] + target.data_category = "user" + target.save(db) + # Update data category on responsible field + field( + [dataset], database_name, "onsite_personnel", "responsible" + ).data_categories = ["user.contact.email"] + + access_results = access_runner_tester( + privacy_request_with_erasure_policy, + erasure_policy, + graph, + [timescale_connection_config], + {"email": "employee-1@example.com"}, + db, + ) + + # Demonstrate hypertable can be queried + assert access_results[f"{database_name}:onsite_personnel"] == [ + {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 1, 9, 0)}, + {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 2, 9, 0)}, + {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 3, 9, 0)}, + {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 5, 9, 0)}, + ] + + # Run an erasure on the hypertable targeting the responsible field + v = erasure_runner_tester( + privacy_request_with_erasure_policy, + erasure_policy, + graph, + [timescale_connection_config], + {"email": "employee-1@example.com"}, + get_cached_data_for_erasures(privacy_request_with_erasure_policy.id), + db, + ) + + assert v == { + f"{database_name}:customer": 0, + f"{database_name}:orders": 0, + f"{database_name}:payment_card": 0, + f"{database_name}:address": 0, + f"{database_name}:onsite_personnel": 4, + }, "onsite_personnel.responsible was the only targeted data category" + + personnel_records = timescale_integration_db.execute( + text("select * from onsite_personnel") + ) + for record in personnel_records: + assert ( + record.responsible != "employee-1@example.com" + ) # These emails have all been masked diff --git a/tests/ops/service/connectors/test_bigquery_connector.py b/tests/ops/service/connectors/test_bigquery_connector.py index a9524777fe..2e7bc3b075 100644 --- a/tests/ops/service/connectors/test_bigquery_connector.py +++ b/tests/ops/service/connectors/test_bigquery_connector.py @@ -129,7 +129,7 @@ def test_generate_update_partitioned_table( assert len(updates) == 2 assert ( str(updates[0]) - == "UPDATE `silken-precinct-284918.fidesopstest.customer` SET `name`=%(name:STRING)s WHERE `silken-precinct-284918.fidesopstest.customer`.`id` = %(id_1:INT64)s AND `created` > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1000 DAY) AND `created` <= CURRENT_TIMESTAMP()" + == "UPDATE `silken-precinct-284918.fidesopstest.customer` SET `name`=%(name:STRING)s WHERE `silken-precinct-284918.fidesopstest.customer`.`email` = %(email_1:STRING)s AND `created` > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1000 DAY) AND `created` <= CURRENT_TIMESTAMP()" ) def test_generate_delete_partitioned_table( @@ -158,7 +158,7 @@ def test_generate_delete_partitioned_table( assert len(deletes) == 2 assert ( str(deletes[0]) - == "DELETE FROM `silken-precinct-284918.fidesopstest.customer` WHERE `silken-precinct-284918.fidesopstest.customer`.`id` = %(id_1:INT64)s AND `created` > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1000 DAY) AND `created` <= CURRENT_TIMESTAMP()" + == "DELETE FROM `silken-precinct-284918.fidesopstest.customer` WHERE `silken-precinct-284918.fidesopstest.customer`.`email` = %(email_1:STRING)s AND `created` > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1000 DAY) AND `created` <= CURRENT_TIMESTAMP()" ) def test_retrieve_partitioned_data( diff --git a/tests/ops/service/connectors/test_bigquery_queryconfig.py b/tests/ops/service/connectors/test_bigquery_queryconfig.py index 06c51c5105..24a16517b6 100644 --- a/tests/ops/service/connectors/test_bigquery_queryconfig.py +++ b/tests/ops/service/connectors/test_bigquery_queryconfig.py @@ -196,7 +196,7 @@ def test_generate_delete_stmt( ) stmts = set(str(stmt) for stmt in delete_stmts) expected_stmts = { - "DELETE FROM `employee` WHERE `employee`.`id` = %(id_1:STRING)s" + "DELETE FROM `employee` WHERE `employee`.`address_id` = %(address_id_1:STRING)s AND `employee`.`email` = %(email_1:STRING)s" } assert stmts == expected_stmts @@ -289,6 +289,6 @@ def test_generate_namespaced_delete_stmt( ) stmts = set(str(stmt) for stmt in delete_stmts) expected_stmts = { - "DELETE FROM `silken-precinct-284918.fidesopstest.employee` WHERE `silken-precinct-284918.fidesopstest.employee`.`id` = %(id_1:STRING)s" + "DELETE FROM `silken-precinct-284918.fidesopstest.employee` WHERE `silken-precinct-284918.fidesopstest.employee`.`address_id` = %(address_id_1:STRING)s AND `silken-precinct-284918.fidesopstest.employee`.`email` = %(email_1:STRING)s" } assert stmts == expected_stmts diff --git a/tests/ops/service/connectors/test_dynamodb_query_config.py b/tests/ops/service/connectors/test_dynamodb_query_config.py new file mode 100644 index 0000000000..4591ae9385 --- /dev/null +++ b/tests/ops/service/connectors/test_dynamodb_query_config.py @@ -0,0 +1,129 @@ +from datetime import datetime, timezone + +import pytest +from boto3.dynamodb.types import TypeDeserializer +from fideslang.models import Dataset + +from fides.api.graph.config import CollectionAddress +from fides.api.graph.graph import DatasetGraph +from fides.api.graph.traversal import Traversal +from fides.api.models.datasetconfig import convert_dataset_to_graph +from fides.api.models.privacy_request import PrivacyRequest +from fides.api.service.connectors.query_configs.dynamodb_query_config import ( + DynamoDBQueryConfig, +) + +privacy_request = PrivacyRequest(id="234544") + + +class TestDynamoDBQueryConfig: + @pytest.fixture(scope="function") + def identity(self): + identity = {"email": "customer-test_uuid@example.com"} + return identity + + @pytest.fixture(scope="function") + def dataset_graph(self, integration_dynamodb_config, example_datasets): + dataset = Dataset(**example_datasets[11]) + dataset_graph = convert_dataset_to_graph( + dataset, integration_dynamodb_config.key + ) + + return DatasetGraph(*[dataset_graph]) + + @pytest.fixture(scope="function") + def traversal(self, identity, dataset_graph): + dynamo_traversal = Traversal(dataset_graph, identity) + return dynamo_traversal + + @pytest.fixture(scope="function") + def customer_node(self, traversal): + return traversal.traversal_node_dict[ + CollectionAddress("dynamodb_example_test_dataset", "customer") + ].to_mock_execution_node() + + @pytest.fixture(scope="function") + def customer_identifier_node(self, traversal): + return traversal.traversal_node_dict[ + CollectionAddress("dynamodb_example_test_dataset", "customer_identifier") + ].to_mock_execution_node() + + @pytest.fixture(scope="function") + def customer_row(self): + row = { + "customer_email": {"S": "customer-1@example.com"}, + "name": {"S": "John Customer"}, + "address_id": {"L": [{"S": "1"}, {"S": "2"}]}, + "personal_info": {"M": {"gender": {"S": "male"}, "age": {"S": "99"}}}, + "id": {"S": "1"}, + } + return row + + @pytest.fixture(scope="function") + def deserialized_customer_row(self, customer_row): + deserialized_customer_row = {} + deserializer = TypeDeserializer() + for key, value in customer_row.items(): + deserialized_customer_row[key] = deserializer.deserialize(value) + return deserialized_customer_row + + @pytest.fixture(scope="function") + def customer_identifier_row(self): + row = { + "customer_id": {"S": "customer-1@example.com"}, + "email": {"S": "customer-1@example.com"}, + "name": {"S": "Customer 1"}, + "created": {"S": datetime.now(timezone.utc).isoformat()}, + } + return row + + @pytest.fixture(scope="function") + def deserialized_customer_identifier_row(self, customer_identifier_row): + deserialized_customer_identifier_row = {} + deserializer = TypeDeserializer() + for key, value in customer_identifier_row.items(): + deserialized_customer_identifier_row[key] = deserializer.deserialize(value) + return deserialized_customer_identifier_row + + def test_get_query_param_formatting_single_key( + self, + resources_dict, + customer_node, + ) -> None: + input_data = { + "fidesops_grouped_inputs": [], + "email": ["customer-test_uuid@example.com"], + } + attribute_definitions = [{"AttributeName": "email", "AttributeType": "S"}] + query_config = DynamoDBQueryConfig(customer_node, attribute_definitions) + item = query_config.generate_query( + input_data=input_data, policy=resources_dict["policy"] + ) + assert item["ExpressionAttributeValues"] == { + ":value": {"S": "customer-test_uuid@example.com"} + } + assert item["KeyConditionExpression"] == "email = :value" + + def test_put_query_param_formatting_single_key( + self, + erasure_policy, + customer_node, + deserialized_customer_row, + ) -> None: + input_data = { + "fidesops_grouped_inputs": [], + "email": ["customer-test_uuid@example.com"], + } + attribute_definitions = [{"AttributeName": "email", "AttributeType": "S"}] + query_config = DynamoDBQueryConfig(customer_node, attribute_definitions) + update_item = query_config.generate_update_stmt( + deserialized_customer_row, erasure_policy, privacy_request + ) + + assert update_item == { + "customer_email": {"S": "customer-1@example.com"}, + "name": {"NULL": True}, + "address_id": {"L": [{"S": "1"}, {"S": "2"}]}, + "personal_info": {"M": {"gender": {"S": "male"}, "age": {"S": "99"}}}, + "id": {"S": "1"}, + } diff --git a/tests/ops/service/connectors/test_mongo_query_config.py b/tests/ops/service/connectors/test_mongo_query_config.py new file mode 100644 index 0000000000..c0f6079df1 --- /dev/null +++ b/tests/ops/service/connectors/test_mongo_query_config.py @@ -0,0 +1,283 @@ +import pytest +from fideslang.models import Dataset + +from fides.api.graph.config import ( + CollectionAddress, + FieldAddress, + FieldPath, + ObjectField, + ScalarField, +) +from fides.api.graph.graph import DatasetGraph, Edge +from fides.api.graph.traversal import Traversal +from fides.api.models.datasetconfig import convert_dataset_to_graph +from fides.api.models.privacy_request import PrivacyRequest +from fides.api.schemas.masking.masking_configuration import HashMaskingConfiguration +from fides.api.schemas.masking.masking_secrets import MaskingSecretCache, SecretType +from fides.api.service.connectors.query_configs.mongodb_query_config import ( + MongoQueryConfig, +) +from fides.api.service.masking.strategy.masking_strategy_hash import HashMaskingStrategy +from fides.api.util.data_category import DataCategory + +from ...task.traversal_data import combined_mongo_postgresql_graph +from ...test_helpers.cache_secrets_helper import cache_secret + +privacy_request = PrivacyRequest(id="234544") + + +class TestMongoQueryConfig: + @pytest.fixture(scope="function") + def combined_traversal(self, connection_config, integration_mongodb_config): + mongo_dataset, postgres_dataset = combined_mongo_postgresql_graph( + connection_config, integration_mongodb_config + ) + combined_dataset_graph = DatasetGraph(mongo_dataset, postgres_dataset) + combined_traversal = Traversal( + combined_dataset_graph, + {"email": "customer-1@examplecom"}, + ) + return combined_traversal + + @pytest.fixture(scope="function") + def customer_details_node(self, combined_traversal): + return combined_traversal.traversal_node_dict[ + CollectionAddress("mongo_test", "customer_details") + ].to_mock_execution_node() + + @pytest.fixture(scope="function") + def customer_feedback_node(self, combined_traversal): + return combined_traversal.traversal_node_dict[ + CollectionAddress("mongo_test", "customer_feedback") + ].to_mock_execution_node() + + def test_field_map_nested(self, customer_details_node): + config = MongoQueryConfig(customer_details_node) + + field_map = config.field_map() + assert isinstance(field_map[FieldPath("workplace_info")], ObjectField) + assert isinstance( + field_map[FieldPath("workplace_info", "employer")], ScalarField + ) + + def test_primary_key_field_paths(self, customer_details_node): + config = MongoQueryConfig(customer_details_node) + assert list(config.primary_key_field_paths.keys()) == [FieldPath("_id")] + assert isinstance(config.primary_key_field_paths[FieldPath("_id")], ScalarField) + + def test_nested_query_field_paths( + self, customer_details_node, customer_feedback_node + ): + assert customer_details_node.query_field_paths == { + FieldPath("customer_id"), + } + + assert customer_feedback_node.query_field_paths == { + FieldPath("customer_information", "email") + } + + def test_nested_typed_filtered_values(self, customer_feedback_node): + """Identity data is located on a nested object""" + input_data = { + "customer_information.email": ["test@example.com"], + "ignore": ["abcde"], + } + assert customer_feedback_node.typed_filtered_values(input_data) == { + "customer_information.email": ["test@example.com"] + } + + def test_generate_query( + self, + policy, + example_datasets, + integration_mongodb_config, + connection_config, + ): + dataset_postgres = Dataset(**example_datasets[0]) + graph = convert_dataset_to_graph(dataset_postgres, connection_config.key) + dataset_mongo = Dataset(**example_datasets[1]) + mongo_graph = convert_dataset_to_graph( + dataset_mongo, integration_mongodb_config.key + ) + dataset_graph = DatasetGraph(*[graph, mongo_graph]) + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) + # Edge created from Root to nested customer_information.email field + assert ( + Edge( + FieldAddress("__ROOT__", "__ROOT__", "email"), + FieldAddress( + "mongo_test", "customer_feedback", "customer_information", "email" + ), + ) + in traversal.edges + ) + + # Test query on nested field + customer_feedback = traversal.traversal_node_dict[ + CollectionAddress("mongo_test", "customer_feedback") + ].to_mock_execution_node() + config = MongoQueryConfig(customer_feedback) + input_data = {"customer_information.email": ["customer-1@example.com"]} + # Tuple of query, projection - Searching for documents with nested + # customer_information.email = customer-1@example.com + assert config.generate_query(input_data, policy) == ( + {"customer_information.email": "customer-1@example.com"}, + {"_id": 1, "customer_information": 1, "date": 1, "message": 1, "rating": 1}, + ) + + # Test query nested data + customer_details = traversal.traversal_node_dict[ + CollectionAddress("mongo_test", "customer_details") + ].to_mock_execution_node() + config = MongoQueryConfig(customer_details) + input_data = {"customer_id": [1]} + # Tuple of query, projection - Projection is specifying fields at the top-level. Nested data will + # be filtered later. + assert config.generate_query(input_data, policy) == ( + {"customer_id": 1}, + { + "_id": 1, + "birthday": 1, + "comments": 1, + "customer_id": 1, + "customer_uuid": 1, + "emergency_contacts": 1, + "children": 1, + "gender": 1, + "travel_identifiers": 1, + "workplace_info": 1, + }, + ) + + def test_generate_update_stmt_multiple_fields( + self, + erasure_policy, + example_datasets, + integration_mongodb_config, + connection_config, + ): + dataset_postgres = Dataset(**example_datasets[0]) + graph = convert_dataset_to_graph(dataset_postgres, connection_config.key) + dataset_mongo = Dataset(**example_datasets[1]) + mongo_graph = convert_dataset_to_graph( + dataset_mongo, integration_mongodb_config.key + ) + dataset_graph = DatasetGraph(*[graph, mongo_graph]) + + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) + customer_details = traversal.traversal_node_dict[ + CollectionAddress("mongo_test", "customer_details") + ].to_mock_execution_node() + config = MongoQueryConfig(customer_details) + row = { + "birthday": "1988-01-10", + "gender": "male", + "customer_id": 1, + "_id": 1, + "workplace_info": { + "position": "Chief Strategist", + "direct_reports": ["Robbie Margo", "Sully Hunter"], + }, + "emergency_contacts": [{"name": "June Customer", "phone": "444-444-4444"}], + "children": ["Christopher Customer", "Courtney Customer"], + } + + # Make target more broad + rule = erasure_policy.rules[0] + target = rule.targets[0] + target.data_category = DataCategory("user").value + + mongo_statement = config.generate_update_stmt( + row, erasure_policy, privacy_request + ) + + expected_result_0 = {"_id": 1} + expected_result_1 = { + "$set": { + "birthday": None, + "children.0": None, + "children.1": None, + "customer_id": None, + "emergency_contacts.0.name": None, + "workplace_info.direct_reports.0": None, # Both direct reports are masked. + "workplace_info.direct_reports.1": None, + "emergency_contacts.0.phone": None, + "gender": None, + "workplace_info.position": None, + } + } + + print(mongo_statement[1]) + print(expected_result_1) + assert mongo_statement[0] == expected_result_0 + assert mongo_statement[1] == expected_result_1 + + def test_generate_update_stmt_multiple_rules( + self, + erasure_policy_two_rules, + example_datasets, + integration_mongodb_config, + connection_config, + ): + dataset_postgres = Dataset(**example_datasets[0]) + graph = convert_dataset_to_graph(dataset_postgres, connection_config.key) + dataset_mongo = Dataset(**example_datasets[1]) + mongo_graph = convert_dataset_to_graph( + dataset_mongo, integration_mongodb_config.key + ) + dataset_graph = DatasetGraph(*[graph, mongo_graph]) + + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) + + customer_details = traversal.traversal_node_dict[ + CollectionAddress("mongo_test", "customer_details") + ].to_mock_execution_node() + + config = MongoQueryConfig(customer_details) + row = { + "birthday": "1988-01-10", + "gender": "male", + "customer_id": 1, + "_id": 1, + "workplace_info": { + "position": "Chief Strategist", + "direct_reports": ["Robbie Margo", "Sully Hunter"], + }, + "emergency_contacts": [{"name": "June Customer", "phone": "444-444-4444"}], + "children": ["Christopher Customer", "Courtney Customer"], + } + + rule = erasure_policy_two_rules.rules[0] + rule.masking_strategy = { + "strategy": "hash", + "configuration": {"algorithm": "SHA-512"}, + } + target = rule.targets[0] + target.data_category = DataCategory("user.demographic.date_of_birth").value + + rule_two = erasure_policy_two_rules.rules[1] + rule_two.masking_strategy = { + "strategy": "random_string_rewrite", + "configuration": {"length": 30}, + } + target = rule_two.targets[0] + target.data_category = DataCategory("user.demographic.gender").value + # cache secrets for hash strategy + secret = MaskingSecretCache[str]( + secret="adobo", + masking_strategy=HashMaskingStrategy.name, + secret_type=SecretType.salt, + ) + cache_secret(secret, privacy_request.id) + + mongo_statement = config.generate_update_stmt( + row, erasure_policy_two_rules, privacy_request + ) + assert mongo_statement[0] == {"_id": 1} + assert len(mongo_statement[1]["$set"]["gender"]) == 30 + assert ( + mongo_statement[1]["$set"]["birthday"] + == HashMaskingStrategy(HashMaskingConfiguration(algorithm="SHA-512")).mask( + ["1988-01-10"], request_id=privacy_request.id + )[0] + ) diff --git a/tests/ops/service/connectors/test_query_config.py b/tests/ops/service/connectors/test_query_config.py index 01d7b9dbd2..eac650d587 100644 --- a/tests/ops/service/connectors/test_query_config.py +++ b/tests/ops/service/connectors/test_query_config.py @@ -1,43 +1,29 @@ -from datetime import datetime, timezone from typing import Any, Dict, Set from unittest import mock import pytest -from boto3.dynamodb.types import TypeDeserializer from fideslang.models import Dataset from fides.api.common_exceptions import MissingNamespaceSchemaException -from fides.api.graph.config import ( - CollectionAddress, - FieldAddress, - FieldPath, - ObjectField, - ScalarField, -) +from fides.api.graph.config import CollectionAddress, FieldPath from fides.api.graph.execution import ExecutionNode -from fides.api.graph.graph import DatasetGraph, Edge +from fides.api.graph.graph import DatasetGraph from fides.api.graph.traversal import Traversal, TraversalNode from fides.api.models.datasetconfig import convert_dataset_to_graph from fides.api.models.privacy_request import PrivacyRequest from fides.api.schemas.masking.masking_configuration import HashMaskingConfiguration from fides.api.schemas.masking.masking_secrets import MaskingSecretCache, SecretType from fides.api.schemas.namespace_meta.namespace_meta import NamespaceMeta -from fides.api.service.connectors.query_configs.dynamodb_query_config import ( - DynamoDBQueryConfig, -) -from fides.api.service.connectors.query_configs.mongodb_query_config import ( - MongoQueryConfig, -) from fides.api.service.connectors.query_configs.query_config import ( QueryConfig, SQLQueryConfig, ) -from fides.api.service.connectors.scylla_query_config import ScyllaDBQueryConfig from fides.api.service.masking.strategy.masking_strategy_hash import HashMaskingStrategy from fides.api.util.data_category import DataCategory from tests.fixtures.application_fixtures import load_dataset +from tests.ops.test_helpers.dataset_utils import remove_primary_keys -from ...task.traversal_data import combined_mongo_postgresql_graph, integration_db_graph +from ...task.traversal_data import integration_db_graph from ...test_helpers.cache_secrets_helper import cache_secret, clear_cache_secrets # customers -> address, order @@ -286,9 +272,47 @@ def test_generate_update_stmt_one_field( "id": 1, } text_clause = config.generate_update_stmt(row, erasure_policy, privacy_request) - assert text_clause.text == """UPDATE customer SET name = :name WHERE id = :id""" - assert text_clause._bindparams["name"].key == "name" - assert text_clause._bindparams["name"].value is None # Null masking strategy + assert ( + text_clause.text + == """UPDATE customer SET name = :masked_name WHERE id = :id""" + ) + assert text_clause._bindparams["masked_name"].key == "masked_name" + assert ( + text_clause._bindparams["masked_name"].value is None + ) # Null masking strategy + + def test_generate_update_stmt_one_field_inbound_reference( + self, erasure_policy_address_city, example_datasets, connection_config + ): + dataset = Dataset(**example_datasets[0]) + graph = convert_dataset_to_graph(dataset, connection_config.key) + dataset_graph = DatasetGraph(*[graph]) + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) + + address_node = traversal.traversal_node_dict[ + CollectionAddress("postgres_example_test_dataset", "address") + ].to_mock_execution_node() + + config = SQLQueryConfig(address_node) + row = { + "id": 1, + "house": "123", + "street": "Main St", + "city": "San Francisco", + "state": "CA", + "zip": "94105", + } + text_clause = config.generate_update_stmt( + row, erasure_policy_address_city, privacy_request + ) + assert ( + text_clause.text + == """UPDATE address SET city = :masked_city WHERE id = :id""" + ) + assert text_clause._bindparams["masked_city"].key == "masked_city" + assert ( + text_clause._bindparams["masked_city"].value is None + ) # Null masking strategy def test_generate_update_stmt_length_truncation( self, @@ -316,11 +340,14 @@ def test_generate_update_stmt_length_truncation( text_clause = config.generate_update_stmt( row, erasure_policy_string_rewrite_long, privacy_request ) - assert text_clause.text == """UPDATE customer SET name = :name WHERE id = :id""" - assert text_clause._bindparams["name"].key == "name" + assert ( + text_clause.text + == """UPDATE customer SET name = :masked_name WHERE id = :id""" + ) + assert text_clause._bindparams["masked_name"].key == "masked_name" # length truncation on name field assert ( - text_clause._bindparams["name"].value + text_clause._bindparams["masked_name"].value == "some rewrite value that is very long and" ) @@ -365,22 +392,23 @@ def test_generate_update_stmt_multiple_fields_same_rule( text_clause = config.generate_update_stmt(row, erasure_policy, privacy_request) assert ( text_clause.text - == "UPDATE customer SET email = :email, name = :name WHERE id = :id" + == "UPDATE customer SET email = :masked_email, name = :masked_name WHERE id = :id" ) - assert text_clause._bindparams["name"].key == "name" + assert text_clause._bindparams["masked_name"].key == "masked_name" # since length is set to 40 in dataset.yml, we expect only first 40 chars of masked val assert ( - text_clause._bindparams["name"].value + text_clause._bindparams["masked_name"].value == HashMaskingStrategy(HashMaskingConfiguration(algorithm="SHA-512")).mask( ["John Customer"], request_id=privacy_request.id )[0][0:40] ) assert ( - text_clause._bindparams["email"].value + text_clause._bindparams["masked_email"].value == HashMaskingStrategy(HashMaskingConfiguration(algorithm="SHA-512")).mask( ["customer-1@example.com"], request_id=privacy_request.id )[0] ) + assert text_clause._bindparams["id"].value == 1 clear_cache_secrets(privacy_request.id) def test_generate_update_stmts_from_multiple_rules( @@ -409,251 +437,145 @@ def test_generate_update_stmts_from_multiple_rules( assert ( text_clause.text - == "UPDATE customer SET email = :email, name = :name WHERE id = :id" + == "UPDATE customer SET email = :masked_email, name = :masked_name WHERE id = :id" ) # Two different masking strategies used for name and email - assert text_clause._bindparams["name"].value is None # Null masking strategy assert ( - text_clause._bindparams["email"].value == "*****" + text_clause._bindparams["masked_name"].value is None + ) # Null masking strategy + assert ( + text_clause._bindparams["masked_email"].value == "*****" ) # String rewrite masking strategy + def test_generate_update_stmt_one_field_without_primary_keys( + self, erasure_policy, example_datasets, connection_config + ): + dataset = remove_primary_keys(Dataset(**example_datasets[0])) + graph = convert_dataset_to_graph(dataset, connection_config.key) + dataset_graph = DatasetGraph(*[graph]) + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) -class TestMongoQueryConfig: - @pytest.fixture(scope="function") - def combined_traversal(self, connection_config, integration_mongodb_config): - mongo_dataset, postgres_dataset = combined_mongo_postgresql_graph( - connection_config, integration_mongodb_config - ) - combined_dataset_graph = DatasetGraph(mongo_dataset, postgres_dataset) - combined_traversal = Traversal( - combined_dataset_graph, - {"email": "customer-1@examplecom"}, - ) - return combined_traversal - - @pytest.fixture(scope="function") - def customer_details_node(self, combined_traversal): - return combined_traversal.traversal_node_dict[ - CollectionAddress("mongo_test", "customer_details") - ].to_mock_execution_node() - - @pytest.fixture(scope="function") - def customer_feedback_node(self, combined_traversal): - return combined_traversal.traversal_node_dict[ - CollectionAddress("mongo_test", "customer_feedback") + customer_node = traversal.traversal_node_dict[ + CollectionAddress("postgres_example_test_dataset", "customer") ].to_mock_execution_node() - def test_field_map_nested(self, customer_details_node): - config = MongoQueryConfig(customer_details_node) - - field_map = config.field_map() - assert isinstance(field_map[FieldPath("workplace_info")], ObjectField) - assert isinstance( - field_map[FieldPath("workplace_info", "employer")], ScalarField + config = SQLQueryConfig(customer_node) + row = { + "email": "customer-1@example.com", + "name": "John Customer", + "address_id": 1, + "id": 1, + } + text_clause = config.generate_update_stmt(row, erasure_policy, privacy_request) + assert ( + text_clause.text + == """UPDATE customer SET name = :masked_name WHERE email = :email""" ) + assert text_clause._bindparams["masked_name"].key == "masked_name" + assert ( + text_clause._bindparams["masked_name"].value is None + ) # Null masking strategy - def test_primary_key_field_paths(self, customer_details_node): - config = MongoQueryConfig(customer_details_node) - assert list(config.primary_key_field_paths.keys()) == [FieldPath("_id")] - assert isinstance(config.primary_key_field_paths[FieldPath("_id")], ScalarField) - - def test_nested_query_field_paths( - self, customer_details_node, customer_feedback_node + def test_generate_update_stmt_one_field_inbound_reference_without_primary_keys( + self, erasure_policy_address_city, example_datasets, connection_config ): - assert customer_details_node.query_field_paths == { - FieldPath("customer_id"), - } + dataset = remove_primary_keys(Dataset(**example_datasets[0])) + graph = convert_dataset_to_graph(dataset, connection_config.key) + dataset_graph = DatasetGraph(*[graph]) + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) - assert customer_feedback_node.query_field_paths == { - FieldPath("customer_information", "email") - } + address_node = traversal.traversal_node_dict[ + CollectionAddress("postgres_example_test_dataset", "address") + ].to_mock_execution_node() - def test_nested_typed_filtered_values(self, customer_feedback_node): - """Identity data is located on a nested object""" - input_data = { - "customer_information.email": ["test@example.com"], - "ignore": ["abcde"], - } - assert customer_feedback_node.typed_filtered_values(input_data) == { - "customer_information.email": ["test@example.com"] + config = SQLQueryConfig(address_node) + row = { + "id": 1, + "house": "123", + "street": "Main St", + "city": "San Francisco", + "state": "CA", + "zip": "94105", } - - def test_generate_query( - self, - policy, - example_datasets, - integration_mongodb_config, - connection_config, - ): - dataset_postgres = Dataset(**example_datasets[0]) - graph = convert_dataset_to_graph(dataset_postgres, connection_config.key) - dataset_mongo = Dataset(**example_datasets[1]) - mongo_graph = convert_dataset_to_graph( - dataset_mongo, integration_mongodb_config.key + text_clause = config.generate_update_stmt( + row, erasure_policy_address_city, privacy_request ) - dataset_graph = DatasetGraph(*[graph, mongo_graph]) - traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) - # Edge created from Root to nested customer_information.email field assert ( - Edge( - FieldAddress("__ROOT__", "__ROOT__", "email"), - FieldAddress( - "mongo_test", "customer_feedback", "customer_information", "email" - ), - ) - in traversal.edges - ) - - # Test query on nested field - customer_feedback = traversal.traversal_node_dict[ - CollectionAddress("mongo_test", "customer_feedback") - ].to_mock_execution_node() - config = MongoQueryConfig(customer_feedback) - input_data = {"customer_information.email": ["customer-1@example.com"]} - # Tuple of query, projection - Searching for documents with nested - # customer_information.email = customer-1@example.com - assert config.generate_query(input_data, policy) == ( - {"customer_information.email": "customer-1@example.com"}, - {"_id": 1, "customer_information": 1, "date": 1, "message": 1, "rating": 1}, - ) - - # Test query nested data - customer_details = traversal.traversal_node_dict[ - CollectionAddress("mongo_test", "customer_details") - ].to_mock_execution_node() - config = MongoQueryConfig(customer_details) - input_data = {"customer_id": [1]} - # Tuple of query, projection - Projection is specifying fields at the top-level. Nested data will - # be filtered later. - assert config.generate_query(input_data, policy) == ( - {"customer_id": 1}, - { - "_id": 1, - "birthday": 1, - "comments": 1, - "customer_id": 1, - "customer_uuid": 1, - "emergency_contacts": 1, - "children": 1, - "gender": 1, - "travel_identifiers": 1, - "workplace_info": 1, - }, + text_clause.text + == """UPDATE address SET city = :masked_city WHERE id = :id""" ) + assert text_clause._bindparams["masked_city"].key == "masked_city" + assert ( + text_clause._bindparams["masked_city"].value is None + ) # Null masking strategy - def test_generate_update_stmt_multiple_fields( + def test_generate_update_stmt_length_truncation_without_primary_keys( self, - erasure_policy, + erasure_policy_string_rewrite_long, example_datasets, - integration_mongodb_config, connection_config, ): - dataset_postgres = Dataset(**example_datasets[0]) - graph = convert_dataset_to_graph(dataset_postgres, connection_config.key) - dataset_mongo = Dataset(**example_datasets[1]) - mongo_graph = convert_dataset_to_graph( - dataset_mongo, integration_mongodb_config.key - ) - dataset_graph = DatasetGraph(*[graph, mongo_graph]) - + dataset = remove_primary_keys(Dataset(**example_datasets[0])) + graph = convert_dataset_to_graph(dataset, connection_config.key) + dataset_graph = DatasetGraph(*[graph]) traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) - customer_details = traversal.traversal_node_dict[ - CollectionAddress("mongo_test", "customer_details") + + customer_node = traversal.traversal_node_dict[ + CollectionAddress("postgres_example_test_dataset", "customer") ].to_mock_execution_node() - config = MongoQueryConfig(customer_details) + + config = SQLQueryConfig(customer_node) row = { - "birthday": "1988-01-10", - "gender": "male", - "customer_id": 1, - "_id": 1, - "workplace_info": { - "position": "Chief Strategist", - "direct_reports": ["Robbie Margo", "Sully Hunter"], - }, - "emergency_contacts": [{"name": "June Customer", "phone": "444-444-4444"}], - "children": ["Christopher Customer", "Courtney Customer"], + "email": "customer-1@example.com", + "name": "John Customer", + "address_id": 1, + "id": 1, } - # Make target more broad - rule = erasure_policy.rules[0] - target = rule.targets[0] - target.data_category = DataCategory("user").value - - mongo_statement = config.generate_update_stmt( - row, erasure_policy, privacy_request + text_clause = config.generate_update_stmt( + row, erasure_policy_string_rewrite_long, privacy_request ) - - expected_result_0 = {"_id": 1} - expected_result_1 = { - "$set": { - "birthday": None, - "children.0": None, - "children.1": None, - "customer_id": None, - "emergency_contacts.0.name": None, - "workplace_info.direct_reports.0": None, # Both direct reports are masked. - "workplace_info.direct_reports.1": None, - "emergency_contacts.0.phone": None, - "gender": None, - "workplace_info.position": None, - } - } - - print(mongo_statement[1]) - print(expected_result_1) - assert mongo_statement[0] == expected_result_0 - assert mongo_statement[1] == expected_result_1 - - def test_generate_update_stmt_multiple_rules( - self, - erasure_policy_two_rules, - example_datasets, - integration_mongodb_config, - connection_config, - ): - dataset_postgres = Dataset(**example_datasets[0]) - graph = convert_dataset_to_graph(dataset_postgres, connection_config.key) - dataset_mongo = Dataset(**example_datasets[1]) - mongo_graph = convert_dataset_to_graph( - dataset_mongo, integration_mongodb_config.key + assert ( + text_clause.text + == """UPDATE customer SET name = :masked_name WHERE email = :email""" + ) + assert text_clause._bindparams["masked_name"].key == "masked_name" + # length truncation on name field + assert ( + text_clause._bindparams["masked_name"].value + == "some rewrite value that is very long and" ) - dataset_graph = DatasetGraph(*[graph, mongo_graph]) + def test_generate_update_stmt_multiple_fields_same_rule_without_primary_keys( + self, erasure_policy, example_datasets, connection_config + ): + dataset = remove_primary_keys(Dataset(**example_datasets[0])) + graph = convert_dataset_to_graph(dataset, connection_config.key) + dataset_graph = DatasetGraph(*[graph]) traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) - customer_details = traversal.traversal_node_dict[ - CollectionAddress("mongo_test", "customer_details") + customer_node = traversal.traversal_node_dict[ + CollectionAddress("postgres_example_test_dataset", "customer") ].to_mock_execution_node() - config = MongoQueryConfig(customer_details) + config = SQLQueryConfig(customer_node) row = { - "birthday": "1988-01-10", - "gender": "male", - "customer_id": 1, - "_id": 1, - "workplace_info": { - "position": "Chief Strategist", - "direct_reports": ["Robbie Margo", "Sully Hunter"], - }, - "emergency_contacts": [{"name": "June Customer", "phone": "444-444-4444"}], - "children": ["Christopher Customer", "Courtney Customer"], + "email": "customer-1@example.com", + "name": "John Customer", + "address_id": 1, + "id": 1, } - rule = erasure_policy_two_rules.rules[0] + # Make target more broad + rule = erasure_policy.rules[0] + target = rule.targets[0] + target.data_category = DataCategory("user").value + + # Update rule masking strategy rule.masking_strategy = { "strategy": "hash", "configuration": {"algorithm": "SHA-512"}, } - target = rule.targets[0] - target.data_category = DataCategory("user.demographic.date_of_birth").value - - rule_two = erasure_policy_two_rules.rules[1] - rule_two.masking_strategy = { - "strategy": "random_string_rewrite", - "configuration": {"length": 30}, - } - target = rule_two.targets[0] - target.data_category = DataCategory("user.demographic.gender").value # cache secrets for hash strategy secret = MaskingSecretCache[str]( secret="adobo", @@ -662,169 +584,63 @@ def test_generate_update_stmt_multiple_rules( ) cache_secret(secret, privacy_request.id) - mongo_statement = config.generate_update_stmt( - row, erasure_policy_two_rules, privacy_request + text_clause = config.generate_update_stmt(row, erasure_policy, privacy_request) + assert ( + text_clause.text + == "UPDATE customer SET email = :masked_email, name = :masked_name WHERE email = :email" ) - assert mongo_statement[0] == {"_id": 1} - assert len(mongo_statement[1]["$set"]["gender"]) == 30 + assert text_clause._bindparams["masked_name"].key == "masked_name" + # since length is set to 40 in dataset.yml, we expect only first 40 chars of masked val assert ( - mongo_statement[1]["$set"]["birthday"] + text_clause._bindparams["masked_name"].value == HashMaskingStrategy(HashMaskingConfiguration(algorithm="SHA-512")).mask( - ["1988-01-10"], request_id=privacy_request.id - )[0] + ["John Customer"], request_id=privacy_request.id + )[0][0:40] ) - - -class TestDynamoDBQueryConfig: - @pytest.fixture(scope="function") - def identity(self): - identity = {"email": "customer-test_uuid@example.com"} - return identity - - @pytest.fixture(scope="function") - def dataset_graph(self, integration_dynamodb_config, example_datasets): - dataset = Dataset(**example_datasets[11]) - dataset_graph = convert_dataset_to_graph( - dataset, integration_dynamodb_config.key + assert ( + text_clause._bindparams["masked_email"].value + == HashMaskingStrategy(HashMaskingConfiguration(algorithm="SHA-512")).mask( + ["customer-1@example.com"], request_id=privacy_request.id + )[0] ) + assert text_clause._bindparams["email"].value == "customer-1@example.com" + clear_cache_secrets(privacy_request.id) - return DatasetGraph(*[dataset_graph]) - - @pytest.fixture(scope="function") - def traversal(self, identity, dataset_graph): - dynamo_traversal = Traversal(dataset_graph, identity) - return dynamo_traversal - - @pytest.fixture(scope="function") - def customer_node(self, traversal): - return traversal.traversal_node_dict[ - CollectionAddress("dynamodb_example_test_dataset", "customer") - ].to_mock_execution_node() - - @pytest.fixture(scope="function") - def customer_identifier_node(self, traversal): - return traversal.traversal_node_dict[ - CollectionAddress("dynamodb_example_test_dataset", "customer_identifier") - ].to_mock_execution_node() - - @pytest.fixture(scope="function") - def customer_row(self): - row = { - "customer_email": {"S": "customer-1@example.com"}, - "name": {"S": "John Customer"}, - "address_id": {"L": [{"S": "1"}, {"S": "2"}]}, - "personal_info": {"M": {"gender": {"S": "male"}, "age": {"S": "99"}}}, - "id": {"S": "1"}, - } - return row - - @pytest.fixture(scope="function") - def deserialized_customer_row(self, customer_row): - deserialized_customer_row = {} - deserializer = TypeDeserializer() - for key, value in customer_row.items(): - deserialized_customer_row[key] = deserializer.deserialize(value) - return deserialized_customer_row - - @pytest.fixture(scope="function") - def customer_identifier_row(self): + def test_generate_update_stmts_from_multiple_rules_without_primary_keys( + self, erasure_policy_two_rules, example_datasets, connection_config + ): + dataset = remove_primary_keys(Dataset(**example_datasets[0])) + graph = convert_dataset_to_graph(dataset, connection_config.key) + dataset_graph = DatasetGraph(*[graph]) + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) row = { - "customer_id": {"S": "customer-1@example.com"}, - "email": {"S": "customer-1@example.com"}, - "name": {"S": "Customer 1"}, - "created": {"S": datetime.now(timezone.utc).isoformat()}, - } - return row - - @pytest.fixture(scope="function") - def deserialized_customer_identifier_row(self, customer_identifier_row): - deserialized_customer_identifier_row = {} - deserializer = TypeDeserializer() - for key, value in customer_identifier_row.items(): - deserialized_customer_identifier_row[key] = deserializer.deserialize(value) - return deserialized_customer_identifier_row - - def test_get_query_param_formatting_single_key( - self, - resources_dict, - customer_node, - ) -> None: - input_data = { - "fidesops_grouped_inputs": [], - "email": ["customer-test_uuid@example.com"], - } - attribute_definitions = [{"AttributeName": "email", "AttributeType": "S"}] - query_config = DynamoDBQueryConfig(customer_node, attribute_definitions) - item = query_config.generate_query( - input_data=input_data, policy=resources_dict["policy"] - ) - assert item["ExpressionAttributeValues"] == { - ":value": {"S": "customer-test_uuid@example.com"} - } - assert item["KeyConditionExpression"] == "email = :value" - - def test_put_query_param_formatting_single_key( - self, - erasure_policy, - customer_node, - deserialized_customer_row, - ) -> None: - input_data = { - "fidesops_grouped_inputs": [], - "email": ["customer-test_uuid@example.com"], - } - attribute_definitions = [{"AttributeName": "email", "AttributeType": "S"}] - query_config = DynamoDBQueryConfig(customer_node, attribute_definitions) - update_item = query_config.generate_update_stmt( - deserialized_customer_row, erasure_policy, privacy_request - ) - - assert update_item == { - "customer_email": {"S": "customer-1@example.com"}, - "name": {"NULL": True}, - "address_id": {"L": [{"S": "1"}, {"S": "2"}]}, - "personal_info": {"M": {"gender": {"S": "male"}, "age": {"S": "99"}}}, - "id": {"S": "1"}, + "email": "customer-1@example.com", + "name": "John Customer", + "address_id": 1, + "id": 1, } - -class TestScyllaDBQueryConfig: - @pytest.fixture(scope="function") - def complete_execution_node( - self, example_datasets, integration_scylladb_config_with_keyspace - ): - dataset = Dataset(**example_datasets[15]) - graph = convert_dataset_to_graph( - dataset, integration_scylladb_config_with_keyspace.key - ) - dataset_graph = DatasetGraph(*[graph]) - identity = {"email": "customer-1@example.com"} - scylla_traversal = Traversal(dataset_graph, identity) - return scylla_traversal.traversal_node_dict[ - CollectionAddress("scylladb_example_test_dataset", "users") + customer_node = traversal.traversal_node_dict[ + CollectionAddress("postgres_example_test_dataset", "customer") ].to_mock_execution_node() - def test_dry_run_query_no_data(self, scylladb_execution_node): - query_config = ScyllaDBQueryConfig(scylladb_execution_node) - dry_run_query = query_config.dry_run_query() - assert dry_run_query is None + config = SQLQueryConfig(customer_node) - def test_dry_run_query_with_data(self, complete_execution_node): - query_config = ScyllaDBQueryConfig(complete_execution_node) - dry_run_query = query_config.dry_run_query() - assert ( - dry_run_query - == "SELECT age, alternative_contacts, ascii_data, big_int_data, do_not_contact, double_data, duration, email, float_data, last_contacted, logins, name, states_lived, timestamp, user_id, uuid FROM users WHERE email = ? ALLOW FILTERING;" + text_clause = config.generate_update_stmt( + row, erasure_policy_two_rules, privacy_request ) - def test_query_to_str(self, complete_execution_node): - query_config = ScyllaDBQueryConfig(complete_execution_node) - statement = ( - "SELECT name FROM users WHERE email = %(email)s", - {"email": "test@example.com"}, + assert ( + text_clause.text + == "UPDATE customer SET email = :masked_email, name = :masked_name WHERE email = :email" ) - query_to_str = query_config.query_to_str(statement, {}) - assert query_to_str == "SELECT name FROM users WHERE email = 'test@example.com'" + # Two different masking strategies used for name and email + assert ( + text_clause._bindparams["masked_name"].value is None + ) # Null masking strategy + assert ( + text_clause._bindparams["masked_email"].value == "*****" + ) # String rewrite masking strategy class TestSQLLikeQueryConfig: diff --git a/tests/ops/service/connectors/test_scylladb_query_config.py b/tests/ops/service/connectors/test_scylladb_query_config.py new file mode 100644 index 0000000000..3cbc6f493f --- /dev/null +++ b/tests/ops/service/connectors/test_scylladb_query_config.py @@ -0,0 +1,47 @@ +import pytest +from fideslang.models import Dataset + +from fides.api.graph.config import CollectionAddress +from fides.api.graph.graph import DatasetGraph +from fides.api.graph.traversal import Traversal +from fides.api.models.datasetconfig import convert_dataset_to_graph +from fides.api.service.connectors.scylla_query_config import ScyllaDBQueryConfig + + +class TestScyllaDBQueryConfig: + @pytest.fixture(scope="function") + def complete_execution_node( + self, example_datasets, integration_scylladb_config_with_keyspace + ): + dataset = Dataset(**example_datasets[15]) + graph = convert_dataset_to_graph( + dataset, integration_scylladb_config_with_keyspace.key + ) + dataset_graph = DatasetGraph(*[graph]) + identity = {"email": "customer-1@example.com"} + scylla_traversal = Traversal(dataset_graph, identity) + return scylla_traversal.traversal_node_dict[ + CollectionAddress("scylladb_example_test_dataset", "users") + ].to_mock_execution_node() + + def test_dry_run_query_no_data(self, scylladb_execution_node): + query_config = ScyllaDBQueryConfig(scylladb_execution_node) + dry_run_query = query_config.dry_run_query() + assert dry_run_query is None + + def test_dry_run_query_with_data(self, complete_execution_node): + query_config = ScyllaDBQueryConfig(complete_execution_node) + dry_run_query = query_config.dry_run_query() + assert ( + dry_run_query + == "SELECT age, alternative_contacts, ascii_data, big_int_data, do_not_contact, double_data, duration, email, float_data, last_contacted, logins, name, states_lived, timestamp, user_id, uuid FROM users WHERE email = ? ALLOW FILTERING;" + ) + + def test_query_to_str(self, complete_execution_node): + query_config = ScyllaDBQueryConfig(complete_execution_node) + statement = ( + "SELECT name FROM users WHERE email = %(email)s", + {"email": "test@example.com"}, + ) + query_to_str = query_config.query_to_str(statement, {}) + assert query_to_str == "SELECT name FROM users WHERE email = 'test@example.com'" diff --git a/tests/ops/service/connectors/test_snowflake_query_config.py b/tests/ops/service/connectors/test_snowflake_query_config.py index 5521a1a88a..4f4b23b8c4 100644 --- a/tests/ops/service/connectors/test_snowflake_query_config.py +++ b/tests/ops/service/connectors/test_snowflake_query_config.py @@ -150,7 +150,7 @@ def test_generate_update_stmt( ) assert ( str(update_stmt) - == 'UPDATE "address" SET "city" = :city, "house" = :house, "state" = :state, "street" = :street, "zip" = :zip WHERE "id" = :id' + == 'UPDATE "address" SET "city" = :masked_city, "house" = :masked_house, "state" = :masked_state, "street" = :masked_street, "zip" = :masked_zip WHERE "id" = :id' ) def test_generate_namespaced_update_stmt( @@ -191,5 +191,5 @@ def test_generate_namespaced_update_stmt( ) assert ( str(update_stmt) - == 'UPDATE "FIDESOPS_TEST"."TEST"."address" SET "city" = :city, "house" = :house, "state" = :state, "street" = :street, "zip" = :zip WHERE "id" = :id' + == 'UPDATE "FIDESOPS_TEST"."TEST"."address" SET "city" = :masked_city, "house" = :masked_house, "state" = :masked_state, "street" = :masked_street, "zip" = :masked_zip WHERE "id" = :id' ) diff --git a/tests/ops/service/dataset/example_datasets/multiple_identities.yml b/tests/ops/service/dataset/example_datasets/multiple_identities.yml index 053afb3ced..dd76dbfa6d 100644 --- a/tests/ops/service/dataset/example_datasets/multiple_identities.yml +++ b/tests/ops/service/dataset/example_datasets/multiple_identities.yml @@ -16,8 +16,6 @@ dataset: data_type: string - name: id data_categories: [user.unique_id] - fides_meta: - primary_key: True - name: name data_categories: [user.name] fides_meta: diff --git a/tests/ops/service/dataset/example_datasets/multiple_identities_with_external_dependency.yml b/tests/ops/service/dataset/example_datasets/multiple_identities_with_external_dependency.yml index fdfcd32bfc..db9e227a74 100644 --- a/tests/ops/service/dataset/example_datasets/multiple_identities_with_external_dependency.yml +++ b/tests/ops/service/dataset/example_datasets/multiple_identities_with_external_dependency.yml @@ -32,7 +32,5 @@ dataset: direction: from - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: shipping_address_id data_categories: [system.operations] diff --git a/tests/ops/service/dataset/example_datasets/no_identities.yml b/tests/ops/service/dataset/example_datasets/no_identities.yml index fac879de99..82b56f9c65 100644 --- a/tests/ops/service/dataset/example_datasets/no_identities.yml +++ b/tests/ops/service/dataset/example_datasets/no_identities.yml @@ -13,8 +13,6 @@ dataset: data_categories: [user.contact.email] - name: id data_categories: [user.unique_id] - fides_meta: - primary_key: True - name: name data_categories: [user.name] fides_meta: diff --git a/tests/ops/service/dataset/example_datasets/single_identity.yml b/tests/ops/service/dataset/example_datasets/single_identity.yml index 19cdc7df3e..ce1506886d 100644 --- a/tests/ops/service/dataset/example_datasets/single_identity.yml +++ b/tests/ops/service/dataset/example_datasets/single_identity.yml @@ -16,8 +16,6 @@ dataset: data_type: string - name: id data_categories: [user.unique_id] - fides_meta: - primary_key: True - name: name data_categories: [user.name] fides_meta: diff --git a/tests/ops/service/dataset/example_datasets/single_identity_with_internal_dependency.yml b/tests/ops/service/dataset/example_datasets/single_identity_with_internal_dependency.yml index 708aefbaf0..af73f8bcb8 100644 --- a/tests/ops/service/dataset/example_datasets/single_identity_with_internal_dependency.yml +++ b/tests/ops/service/dataset/example_datasets/single_identity_with_internal_dependency.yml @@ -16,8 +16,6 @@ dataset: data_type: string - name: id data_categories: [user.unique_id] - fides_meta: - primary_key: True - name: name data_categories: [user.name] fides_meta: diff --git a/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py b/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py index 8fb7e29729..5a133c031f 100644 --- a/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py +++ b/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py @@ -1,27 +1,9 @@ -import time -from datetime import datetime, timezone -from typing import Any, Dict, List, Set from unittest import mock -from unittest.mock import ANY, Mock, call -from uuid import uuid4 -import pydash import pytest from fides.api.models.audit_log import AuditLog, AuditLogAction -from fides.api.models.privacy_request import ( - ActionType, - CheckpointActionRequired, - ExecutionLog, - ExecutionLogStatus, - PolicyPreWebhook, - PrivacyRequest, - PrivacyRequestStatus, -) -from fides.api.schemas.masking.masking_configuration import MaskingConfiguration -from fides.api.schemas.masking.masking_secrets import MaskingSecretCache -from fides.api.schemas.policy import Rule -from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy +from fides.api.models.privacy_request import ExecutionLog from tests.ops.service.privacy_request.test_request_runner_service import ( get_privacy_request_results, ) @@ -54,7 +36,7 @@ def test_create_and_process_access_request_bigquery_enterprise( customer_email = "customer-1@example.com" user_id = ( - 1754 # this is a real (not generated) user id in the Stackoverflow dataset + 1754 # this is a real (not generated) user id in the Stack Overflow dataset ) data = { "requested_at": "2024-08-30T16:09:37.359Z", diff --git a/tests/ops/service/privacy_request/test_postgres_privacy_requests.py b/tests/ops/service/privacy_request/test_postgres_privacy_requests.py index 2959efd463..3f5ec661f5 100644 --- a/tests/ops/service/privacy_request/test_postgres_privacy_requests.py +++ b/tests/ops/service/privacy_request/test_postgres_privacy_requests.py @@ -160,9 +160,16 @@ def test_upload_access_results_has_data_use_map( "dsr_version", ["use_dsr_3_0", "use_dsr_2_0"], ) +@pytest.mark.parametrize( + "dataset_config", + [ + "postgres_example_test_dataset_config_read_access", + "postgres_example_test_dataset_config_read_access_without_primary_keys", + ], +) def test_create_and_process_access_request_postgres( trigger_webhook_mock, - postgres_example_test_dataset_config_read_access, + dataset_config, postgres_integration_db, db, cache, @@ -174,6 +181,7 @@ def test_create_and_process_access_request_postgres( run_privacy_request_task, ): request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + request.getfixturevalue(dataset_config) customer_email = "customer-1@example.com" data = { @@ -196,7 +204,7 @@ def test_create_and_process_access_request_postgres( assert results[key] is not None assert results[key] != {} - result_key_prefix = f"postgres_example_test_dataset:" + result_key_prefix = "postgres_example_test_dataset:" customer_key = result_key_prefix + "customer" assert results[customer_key][0]["email"] == customer_email @@ -278,14 +286,14 @@ def test_create_and_process_access_request_with_custom_identities_postgres( assert results[key] is not None assert results[key] != {} - result_key_prefix = f"postgres_example_test_dataset:" + result_key_prefix = "postgres_example_test_dataset:" customer_key = result_key_prefix + "customer" assert results[customer_key][0]["email"] == customer_email visit_key = result_key_prefix + "visit" assert results[visit_key][0]["email"] == customer_email - loyalty_key = f"postgres_example_test_extended_dataset:loyalty" + loyalty_key = "postgres_example_test_extended_dataset:loyalty" assert results[loyalty_key][0]["id"] == loyalty_id log_id = pr.execution_logs[0].id @@ -355,7 +363,7 @@ def test_create_and_process_access_request_with_valid_skipped_collection( assert "login" not in results.keys() - result_key_prefix = f"postgres_example_test_dataset:" + result_key_prefix = "postgres_example_test_dataset:" customer_key = result_key_prefix + "customer" assert results[customer_key][0]["email"] == customer_email @@ -712,9 +720,16 @@ def test_create_and_process_erasure_request_with_table_joins( "dsr_version", ["use_dsr_3_0", "use_dsr_2_0"], ) +@pytest.mark.parametrize( + "dataset_config", + [ + "postgres_example_test_dataset_config_read_access", + "postgres_example_test_dataset_config_read_access_without_primary_keys", + ], +) def test_create_and_process_erasure_request_read_access( postgres_integration_db, - postgres_example_test_dataset_config_read_access, + dataset_config, db, cache, erasure_policy, @@ -723,6 +738,7 @@ def test_create_and_process_erasure_request_read_access( run_privacy_request_task, ): request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + request.getfixturevalue(dataset_config) customer_email = "customer-2@example.com" customer_id = 2 @@ -739,7 +755,7 @@ def test_create_and_process_erasure_request_read_access( data, ) errored_execution_logs = pr.execution_logs.filter_by(status="error") - assert errored_execution_logs.count() == 9 + assert errored_execution_logs.count() == 11 assert ( errored_execution_logs[0].message == "No values were erased since this connection " diff --git a/tests/ops/test_helpers/dataset_utils.py b/tests/ops/test_helpers/dataset_utils.py index e60efb9892..d51e1f47ff 100644 --- a/tests/ops/test_helpers/dataset_utils.py +++ b/tests/ops/test_helpers/dataset_utils.py @@ -13,7 +13,11 @@ ) from fides.api.graph.data_type import DataType, get_data_type, to_data_type_string from fides.api.models.connectionconfig import ConnectionConfig -from fides.api.models.datasetconfig import DatasetConfig, convert_dataset_to_graph +from fides.api.models.datasetconfig import ( + DatasetConfig, + DatasetField, + convert_dataset_to_graph, +) from fides.api.util.collection_util import Row SAAS_DATASET_DIRECTORY = "data/saas/dataset/" @@ -231,3 +235,27 @@ def get_simple_fields(fields: Iterable[Field]) -> List[Dict[str, Any]]: object["fields"] = get_simple_fields(field.fields.values()) object_list.append(object) return object_list + + +def remove_primary_keys(dataset: Dataset) -> Dataset: + """Returns a copy of the dataset with primary key fields removed from fides_meta.""" + dataset_copy = dataset.model_copy(deep=True) + + for collection in dataset_copy.collections: + for field in collection.fields: + if field.fides_meta: + if field.fides_meta.primary_key: + field.fides_meta.primary_key = None + if field.fields: + _remove_nested_primary_keys(field.fields) + + return dataset_copy + + +def _remove_nested_primary_keys(fields: List[DatasetField]) -> None: + """Helper function to recursively remove primary keys from nested fields.""" + for field in fields: + if field.fides_meta and field.fides_meta.primary_key: + field.fides_meta.primary_key = None + if field.fields: + _remove_nested_primary_keys(field.fields)