From 3c7919e38cc043d8cc4ffc282a3a8cb9531f88e4 Mon Sep 17 00:00:00 2001 From: kang Date: Sat, 2 Nov 2024 23:17:35 +0800 Subject: [PATCH] Fix pagination issue in `JSONResponseCursorPaginator` with empty string cursor value Adjusts the `update_state()` method to set `_next_reference` to None when the cursor value extracted from the JSON response is an empty string, preventing unintended pagination requests. Fixed #2012 --- dlt/sources/helpers/rest_client/paginators.py | 2 +- tests/sources/helpers/rest_client/test_paginators.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dlt/sources/helpers/rest_client/paginators.py b/dlt/sources/helpers/rest_client/paginators.py index 82b97e253b..a4290e0b9b 100644 --- a/dlt/sources/helpers/rest_client/paginators.py +++ b/dlt/sources/helpers/rest_client/paginators.py @@ -651,7 +651,7 @@ def __init__( def update_state(self, response: Response, data: Optional[List[Any]] = None) -> None: """Extracts the cursor value from the JSON response.""" values = jsonpath.find_values(self.cursor_path, response.json()) - self._next_reference = values[0] if values else None + self._next_reference = values[0] if values and values[0] else None def update_request(self, request: Request) -> None: """Updates the request with the cursor query parameter.""" diff --git a/tests/sources/helpers/rest_client/test_paginators.py b/tests/sources/helpers/rest_client/test_paginators.py index 49a6275536..85276a263f 100644 --- a/tests/sources/helpers/rest_client/test_paginators.py +++ b/tests/sources/helpers/rest_client/test_paginators.py @@ -530,6 +530,12 @@ def test_update_state(self): assert paginator._next_reference == "cursor-2" assert paginator.has_next_page is True + def test_update_state_when_cursor_path_is_empty_string(self): + paginator = JSONResponseCursorPaginator(cursor_path="next_cursor") + response = Mock(Response, json=lambda: {"next_cursor": "", "results": []}) + paginator.update_state(response) + assert paginator.has_next_page is False + def test_update_request(self): paginator = JSONResponseCursorPaginator(cursor_path="next_cursor") paginator._next_reference = "cursor-2"