From c0151cc10a60a849c6ecf6f817aae5bbcd4377ec Mon Sep 17 00:00:00 2001 From: kirk Date: Wed, 14 Aug 2024 08:56:12 -0400 Subject: [PATCH 1/9] get_urn_list method --- src/fides/api/models/detection_discovery.py | 11 ++++++++++- tests/ops/models/test_detection_discovery.py | 5 +++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/fides/api/models/detection_discovery.py b/src/fides/api/models/detection_discovery.py index c70cd7789d..cf65436c47 100644 --- a/src/fides/api/models/detection_discovery.py +++ b/src/fides/api/models/detection_discovery.py @@ -4,7 +4,7 @@ from enum import Enum from typing import Any, Dict, Iterable, Optional, Type -from sqlalchemy import ARRAY, Boolean, Column, DateTime, ForeignKey, String +from sqlalchemy import ARRAY, Boolean, Column, DateTime, ForeignKey, String, select from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.orm import Session, relationship @@ -280,6 +280,15 @@ class StagedResource(Base): def get_urn(cls, db: Session, urn: str) -> Optional[StagedResource]: """Utility to retrieve the staged resource with the given URN""" return cls.get_by(db=db, field="urn", value=urn) + + @classmethod + def get_urn_list(cls, db: Session, urns: Iterable[str]) -> Iterable[StagedResource]: + """ + Utility to retrieve all staged resources with the given URNs + """ + + results = db.execute(select(StagedResource).where(StagedResource.urn.in_(urns))) + return results.scalars().all() def add_child_diff_status(self, diff_status: DiffStatus) -> None: """Increments the specified child diff status""" diff --git a/tests/ops/models/test_detection_discovery.py b/tests/ops/models/test_detection_discovery.py index 7052b3924a..188a5a163a 100644 --- a/tests/ops/models/test_detection_discovery.py +++ b/tests/ops/models/test_detection_discovery.py @@ -60,6 +60,11 @@ def create_staged_resource(self, db: Session): }, ) return resource + + def test_get_urn_list(self, db: Session, create_staged_resource) -> None: + urn_list = [create_staged_resource.urn] + from_db = StagedResource.get_urn_list(db, urn_list) + assert len(from_db) == len(urn_list) def test_create_staged_resource(self, db: Session, create_staged_resource) -> None: """ From 4b03fd1ad274a261ca655def7b6c760f62127055 Mon Sep 17 00:00:00 2001 From: kirk Date: Wed, 14 Aug 2024 09:02:41 -0400 Subject: [PATCH 2/9] changelog update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd60d0381c..9d28e26102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The types of changes are: - Adds user_read scope to approver role so that they can update their own password [#5178](https://github.com/ethyca/fides/pull/5178) - Added PATCH endpoint for partially updating connection secrets [#5172](https://github.com/ethyca/fides/pull/5172) - Add success toast on confirming classification in data discovery tables [#5182](https://github.com/ethyca/fides/pull/5182) +- Add function to return list of StagedResource objs according to list of URNs [#5192](https://github.com/ethyca/fides/pull/5192) ### Fixed - Fixed the OAuth2 configuration for the Snap integration [#5158](https://github.com/ethyca/fides/pull/5158) From 6fdbbadbdb26f7acceac455d736f5a945620051b Mon Sep 17 00:00:00 2001 From: kirk Date: Wed, 14 Aug 2024 09:09:50 -0400 Subject: [PATCH 3/9] formatting and test fix --- CHANGELOG.md | 2 +- src/fides/api/models/detection_discovery.py | 5 ++--- tests/ops/models/test_detection_discovery.py | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d28e26102..44e2f3f403 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ The types of changes are: ### Added - Added support for mapping a system's integration's consentable items to privacy notices [#5156](https://github.com/ethyca/fides/pull/5156) -- Added support for SSO Login with multiple providers (Fides Plus feature) [#5134](https://github.com/ethyca/fides/pull/5134) +- Added support for SSO Login with multiple wproviders (Fides Plus feature) [#5134](https://github.com/ethyca/fides/pull/5134) - Adds user_read scope to approver role so that they can update their own password [#5178](https://github.com/ethyca/fides/pull/5178) - Added PATCH endpoint for partially updating connection secrets [#5172](https://github.com/ethyca/fides/pull/5172) - Add success toast on confirming classification in data discovery tables [#5182](https://github.com/ethyca/fides/pull/5182) diff --git a/src/fides/api/models/detection_discovery.py b/src/fides/api/models/detection_discovery.py index cf65436c47..b929677b25 100644 --- a/src/fides/api/models/detection_discovery.py +++ b/src/fides/api/models/detection_discovery.py @@ -280,14 +280,13 @@ class StagedResource(Base): def get_urn(cls, db: Session, urn: str) -> Optional[StagedResource]: """Utility to retrieve the staged resource with the given URN""" return cls.get_by(db=db, field="urn", value=urn) - + @classmethod def get_urn_list(cls, db: Session, urns: Iterable[str]) -> Iterable[StagedResource]: """ Utility to retrieve all staged resources with the given URNs """ - - results = db.execute(select(StagedResource).where(StagedResource.urn.in_(urns))) + results = db.execute(select(StagedResource).where(StagedResource.urn.in_(urns))) # type: ignore return results.scalars().all() def add_child_diff_status(self, diff_status: DiffStatus) -> None: diff --git a/tests/ops/models/test_detection_discovery.py b/tests/ops/models/test_detection_discovery.py index 188a5a163a..b059ac9b61 100644 --- a/tests/ops/models/test_detection_discovery.py +++ b/tests/ops/models/test_detection_discovery.py @@ -60,11 +60,12 @@ def create_staged_resource(self, db: Session): }, ) return resource - + def test_get_urn_list(self, db: Session, create_staged_resource) -> None: urn_list = [create_staged_resource.urn] from_db = StagedResource.get_urn_list(db, urn_list) assert len(from_db) == len(urn_list) + assert from_db[0].urn == urn_list[0] def test_create_staged_resource(self, db: Session, create_staged_resource) -> None: """ From b94b1ba34adc2739236e1550058e91ed463c64df Mon Sep 17 00:00:00 2001 From: kirk Date: Wed, 14 Aug 2024 09:10:18 -0400 Subject: [PATCH 4/9] revert --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44e2f3f403..9d28e26102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ The types of changes are: ### Added - Added support for mapping a system's integration's consentable items to privacy notices [#5156](https://github.com/ethyca/fides/pull/5156) -- Added support for SSO Login with multiple wproviders (Fides Plus feature) [#5134](https://github.com/ethyca/fides/pull/5134) +- Added support for SSO Login with multiple providers (Fides Plus feature) [#5134](https://github.com/ethyca/fides/pull/5134) - Adds user_read scope to approver role so that they can update their own password [#5178](https://github.com/ethyca/fides/pull/5178) - Added PATCH endpoint for partially updating connection secrets [#5172](https://github.com/ethyca/fides/pull/5172) - Add success toast on confirming classification in data discovery tables [#5182](https://github.com/ethyca/fides/pull/5182) From c41a8f3942b33cbd001a3031cb577c118e198b9d Mon Sep 17 00:00:00 2001 From: kirk Date: Wed, 14 Aug 2024 10:15:52 -0400 Subject: [PATCH 5/9] async urn getters in OSS too --- src/fides/api/models/detection_discovery.py | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/fides/api/models/detection_discovery.py b/src/fides/api/models/detection_discovery.py index b929677b25..c9f3426a8c 100644 --- a/src/fides/api/models/detection_discovery.py +++ b/src/fides/api/models/detection_discovery.py @@ -8,6 +8,7 @@ from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.orm import Session, relationship +from sqlalchemy.ext.asyncio import AsyncSession from fides.api.db.base_class import Base, FidesBase from fides.api.models.connectionconfig import ConnectionConfig @@ -289,6 +290,30 @@ def get_urn_list(cls, db: Session, urns: Iterable[str]) -> Iterable[StagedResour results = db.execute(select(StagedResource).where(StagedResource.urn.in_(urns))) # type: ignore return results.scalars().all() + @classmethod + async def get_urn_async( + cls, db: AsyncSession, urn: str + ) -> Optional[StagedResource]: + """ + Utility to retrieve the staged resource with the given URN using an async session + """ + results = await db.execute( + select(StagedResource).where(StagedResource.urn == urn) # type: ignore + ) + return results.scalars().first() + + @classmethod + async def get_urn_list_async( + cls, db: AsyncSession, urns: Iterable[str] + ) -> List[StagedResourceDb]: + """ + Utility to retrieve all staged resources with the given URNs using an async session + """ + results = await db.execute( + select(StagedResource).where(StagedResource.urn.in_(urns)) # type: ignore + ) + return results.scalars().all() + def add_child_diff_status(self, diff_status: DiffStatus) -> None: """Increments the specified child diff status""" self.child_diff_statuses[diff_status.value] = ( From bd05a9f3010186777e6b9026d4c29e95f6f9bb5e Mon Sep 17 00:00:00 2001 From: kirk Date: Wed, 14 Aug 2024 10:17:35 -0400 Subject: [PATCH 6/9] naming --- src/fides/api/models/detection_discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fides/api/models/detection_discovery.py b/src/fides/api/models/detection_discovery.py index c9f3426a8c..428dd1386a 100644 --- a/src/fides/api/models/detection_discovery.py +++ b/src/fides/api/models/detection_discovery.py @@ -305,7 +305,7 @@ async def get_urn_async( @classmethod async def get_urn_list_async( cls, db: AsyncSession, urns: Iterable[str] - ) -> List[StagedResourceDb]: + ) -> List[StagedResource]: """ Utility to retrieve all staged resources with the given URNs using an async session """ From 7717dbe17732c1f196cea3c295e93a6ba56771ce Mon Sep 17 00:00:00 2001 From: kirk Date: Wed, 14 Aug 2024 10:26:58 -0400 Subject: [PATCH 7/9] formatting --- src/fides/api/models/detection_discovery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fides/api/models/detection_discovery.py b/src/fides/api/models/detection_discovery.py index 428dd1386a..cf0a7d89c0 100644 --- a/src/fides/api/models/detection_discovery.py +++ b/src/fides/api/models/detection_discovery.py @@ -2,13 +2,13 @@ from datetime import datetime from enum import Enum -from typing import Any, Dict, Iterable, Optional, Type +from typing import Any, Dict, Iterable, List, Optional, Type from sqlalchemy import ARRAY, Boolean, Column, DateTime, ForeignKey, String, select from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.orm import Session, relationship -from sqlalchemy.ext.asyncio import AsyncSession from fides.api.db.base_class import Base, FidesBase from fides.api.models.connectionconfig import ConnectionConfig From 9330a4a51510cc4e7785bd31d63ec9b75b0649dc Mon Sep 17 00:00:00 2001 From: kirk Date: Wed, 14 Aug 2024 13:06:02 -0400 Subject: [PATCH 8/9] async getters and tests --- tests/conftest.py | 1 - tests/ops/models/test_detection_discovery.py | 22 +++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 287504a336..85488c0185 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -120,7 +120,6 @@ async def async_session(test_client): @pytest.mark.asyncio async def async_session_temp(test_client): assert CONFIG.test_mode - assert requests.post == test_client.post create_citext_extension(sync_engine) diff --git a/tests/ops/models/test_detection_discovery.py b/tests/ops/models/test_detection_discovery.py index b059ac9b61..b88db19ac4 100644 --- a/tests/ops/models/test_detection_discovery.py +++ b/tests/ops/models/test_detection_discovery.py @@ -1,8 +1,10 @@ from datetime import datetime, timezone +from typing import List import pytest from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified +from sqlalchemy.ext.asyncio import AsyncSession from fides.api.models.connectionconfig import ConnectionConfig from fides.api.models.detection_discovery import ( @@ -61,12 +63,30 @@ def create_staged_resource(self, db: Session): ) return resource - def test_get_urn_list(self, db: Session, create_staged_resource) -> None: + def test_get_urn(self, db: Session, create_staged_resource) -> None: urn_list = [create_staged_resource.urn] from_db = StagedResource.get_urn_list(db, urn_list) assert len(from_db) == len(urn_list) assert from_db[0].urn == urn_list[0] + # single urn + from_db_single = StagedResource.get_urn(db, urn_list[0]) + assert from_db_single.urn == urn_list[0] + + async def test_get_urn_async( + self, async_session_temp: AsyncSession, create_staged_resource + ) -> None: + urn_list: List[str] = [str(create_staged_resource.urn)] + + from_db_single = await StagedResource.get_urn_async( + async_session_temp, urn_list[0] + ) + assert from_db_single.urn == urn_list[0] + + from_db = await StagedResource.get_urn_list_async(async_session_temp, urn_list) + assert len(from_db) == len(urn_list) + assert from_db[0].urn == urn_list[0] + def test_create_staged_resource(self, db: Session, create_staged_resource) -> None: """ Creation fixture creates the resource, this tests that it was created successfully From 69fde365af92e9557367b7940633ba6a3e626a76 Mon Sep 17 00:00:00 2001 From: kirk Date: Wed, 14 Aug 2024 13:06:43 -0400 Subject: [PATCH 9/9] formatting --- src/fides/api/models/detection_discovery.py | 8 ++++---- tests/ops/models/test_detection_discovery.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/fides/api/models/detection_discovery.py b/src/fides/api/models/detection_discovery.py index cf0a7d89c0..f6bdcd9eb7 100644 --- a/src/fides/api/models/detection_discovery.py +++ b/src/fides/api/models/detection_discovery.py @@ -304,13 +304,13 @@ async def get_urn_async( @classmethod async def get_urn_list_async( - cls, db: AsyncSession, urns: Iterable[str] - ) -> List[StagedResource]: + cls, db: AsyncSession, urns: List[str] + ) -> Optional[List[StagedResource]]: """ - Utility to retrieve all staged resources with the given URNs using an async session + Utility to retrieve the staged resource with the given URN using an async session """ results = await db.execute( - select(StagedResource).where(StagedResource.urn.in_(urns)) # type: ignore + select(StagedResource).where(StagedResource.urn.in_(urns)) ) return results.scalars().all() diff --git a/tests/ops/models/test_detection_discovery.py b/tests/ops/models/test_detection_discovery.py index b88db19ac4..6e41349d6b 100644 --- a/tests/ops/models/test_detection_discovery.py +++ b/tests/ops/models/test_detection_discovery.py @@ -2,9 +2,9 @@ from typing import List import pytest +from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified -from sqlalchemy.ext.asyncio import AsyncSession from fides.api.models.connectionconfig import ConnectionConfig from fides.api.models.detection_discovery import (