diff --git a/CHANGELOG.md b/CHANGELOG.md index 4db5f64c7a..ca8124523e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The types of changes are: - New page in the Cookie House sample app to demonstrate the use of embedding the FidesJS SDK on the page [#5564](https://github.com/ethyca/fides/pull/5564) - Added event based communication example to the Cookie House sample app [#5597](https://github.com/ethyca/fides/pull/5597) - Added new erasure tests for BigQuery Enterprise [#5554](https://github.com/ethyca/fides/pull/5554) +- Added new `has_next` parameter for the `link` pagination strategy [#5596](https://github.com/ethyca/fides/pull/5596) ### Changed - Adjusted Ant's Select component colors and icon [#5594](https://github.com/ethyca/fides/pull/5594) diff --git a/src/fides/api/schemas/saas/strategy_configuration.py b/src/fides/api/schemas/saas/strategy_configuration.py index 368441dcb8..8135d17698 100644 --- a/src/fides/api/schemas/saas/strategy_configuration.py +++ b/src/fides/api/schemas/saas/strategy_configuration.py @@ -63,6 +63,7 @@ class LinkPaginationConfiguration(StrategyConfiguration): source: LinkSource rel: Optional[str] = None path: Optional[str] = None + has_next: Optional[str] = None @model_validator(mode="before") @classmethod diff --git a/src/fides/api/service/pagination/pagination_strategy_link.py b/src/fides/api/service/pagination/pagination_strategy_link.py index 7d706298be..501855f203 100644 --- a/src/fides/api/service/pagination/pagination_strategy_link.py +++ b/src/fides/api/service/pagination/pagination_strategy_link.py @@ -23,6 +23,7 @@ def __init__(self, configuration: LinkPaginationConfiguration): self.source = configuration.source self.rel = configuration.rel self.path = configuration.path + self.has_next = configuration.has_next def get_next_request( self, @@ -40,6 +41,12 @@ def get_next_request( if not response_data: return None + if self.has_next: + has_next = pydash.get(response.json(), self.has_next) + logger.info(f"The {self.has_next} field has a value of {has_next}") + if str(has_next).lower() != "true": + return None + # read the next_link from the correct location based on the source value next_link = None if self.source == LinkSource.headers.value: diff --git a/tests/ops/service/pagination/test_pagination_strategy_link.py b/tests/ops/service/pagination/test_pagination_strategy_link.py index 46e753dfcc..764d98abf5 100644 --- a/tests/ops/service/pagination/test_pagination_strategy_link.py +++ b/tests/ops/service/pagination/test_pagination_strategy_link.py @@ -49,6 +49,42 @@ def response_with_empty_string_link(): return response +@pytest.fixture(scope="function") +def response_with_has_next_conditional_true(): + response = Response() + response._content = bytes( + json.dumps( + { + "customers": [{"id": 1}, {"id": 2}, {"id": 3}], + "links": { + "next": "https://domain.com/customers?page=def", + "hasNext": True, + }, + } + ), + "utf-8", + ) + return response + + +@pytest.fixture(scope="function") +def response_with_has_next_conditional_false(): + response = Response() + response._content = bytes( + json.dumps( + { + "customers": [{"id": 1}, {"id": 2}, {"id": 3}], + "links": { + "next": "https://domain.com/customers?page=abc", + "hasNext": False, + }, + } + ), + "utf-8", + ) + return response + + def test_link_in_headers(response_with_header_link): config = LinkPaginationConfiguration(source="headers", rel="next") request_params: SaaSRequestParams = SaaSRequestParams( @@ -132,6 +168,50 @@ def test_link_in_body_empty_string(response_with_empty_string_link): assert next_request is None +## TODO: Tests for when the link exists but there is a conditional boolean that checks if there is a next page +def test_link_in_body_with_conditional_boolean_true( + response_with_has_next_conditional_true, +): + config = LinkPaginationConfiguration( + source="body", path="links.next", has_next="links.hasNext" + ) + request_params: SaaSRequestParams = SaaSRequestParams( + method=HTTPMethod.GET, + path="/customers", + query_params={"page": "abc"}, + ) + + paginator = LinkPaginationStrategy(config) + next_request: Optional[SaaSRequestParams] = paginator.get_next_request( + request_params, {}, response_with_has_next_conditional_true, "customers" + ) + + assert next_request == SaaSRequestParams( + method=HTTPMethod.GET, + path="/customers", + query_params={"page": "def"}, + ) + + +def test_link_in_body_with_conditional_boolean_false( + response_with_has_next_conditional_false, +): + config = LinkPaginationConfiguration( + source="body", path="links.next", has_next="links.hasNext" + ) + request_params: SaaSRequestParams = SaaSRequestParams( + method=HTTPMethod.GET, + path="/customers", + query_params={"page": "abc"}, + ) + + paginator = LinkPaginationStrategy(config) + next_request: Optional[SaaSRequestParams] = paginator.get_next_request( + request_params, {}, response_with_has_next_conditional_false, "customers" + ) + assert next_request is None + + def test_wrong_source(): with pytest.raises(ValueError) as exc: LinkPaginationConfiguration(source="somewhere", path="links.next")