@pcrespov fix list check and improve implementation of pagination

iterator

clients/python/src/osparc/_utils.py

@@ -1,7 +1,7 @@
 import asyncio
 import hashlib
 from pathlib import Path
-from typing import AsyncGenerator, Callable, Optional, Tuple, TypeVar, Union
+from typing import AsyncGenerator, Callable, Optional, Tuple, TypeVar, Union, cast, List
 from collections.abc import Iterator
 import httpx
 from osparc_client import (
@@ -44,39 +44,42 @@ def __init__(
     ):
         self._first_page_callback: Callable[[], Page] = first_page_callback
         self._api_client: ApiClient = api_client
-        self._next_page_url: Optional[str] = None
         self._client: httpx.Client = httpx.Client(
             auth=auth, base_url=base_url, follow_redirects=True
         )
         self._page: Optional[Page] = None
-        self._page_item_counter = 0
+        self._items_iterator: Optional[Iterator] = None
 
     def __del__(self):
         self._client.close()
 
     def __next__(self):
         if self._page is None:
             self._page = self._first_page_callback()
-        if self._page_item_counter > (len(self._page.items) - 1):
+            self._items_iterator = iter(cast(List, self._page.items))
+        assert self._items_iterator is not None  # nosec
+        try:
+            return next(self._items_iterator)
+        except StopIteration as exc:
             next_page_url = self._page.links.next
             if next_page_url is None:
                 self._page = None
-                self._page_item_counter = 0
-                raise StopIteration
+                self._items_iterator = None
+                raise StopIteration from exc
             response = self._client.get(next_page_url)
             self._page = self._api_client._ApiClient__deserialize(
                 response.json(), type(self._page)
             )
-            self._page_item_counter = 0
-        next_item = self._page.items[self._page_item_counter]
-        self._page_item_counter += 1
-        return next_item
+            assert self._page is not None  # nosec
+            self._items_iterator = iter(cast(List, self._page.items))
+            return next(self._items_iterator)
 
     def __len__(self) -> int:
         """Number of elements which the iterator can produce"""
-        page: Page = self._first_page_callback()
-        assert isinstance(page.total, int)
-        return page.total
+        if self._page is not None:
+            return cast(int, self._page.total)
+        self._page = self._first_page_callback()
+        return cast(int, self._page.total)
 
 
 async def file_chunk_generator(

clients/python/test/test_osparc/test_basic.py

@@ -2,7 +2,7 @@
 import os
 import subprocess
 from pathlib import Path
-from typing import Any, Dict, List, Set
+from typing import Any, Dict, List, Set, cast
 
 import osparc
 import osparc._settings
@@ -14,6 +14,7 @@
 from urllib.parse import urlparse
 from osparc._utils import PaginationIterator
 from functools import partial
+from copy import deepcopy
 
 _CLIENTS_PYTHON_DIR: Path = Path(__file__).parent.parent.parent
 
@@ -106,17 +107,19 @@ def test_pagination_iterator(
     )
 
     def _sideeffect(all_items: List, request: httpx.Request):
-        all_items += page_file.items
-        if len(all_items) >= page_file.total:
-            all_items = all_items[: page_file.total]
+        n_remaining_items = cast(int, page_file.total) - len(all_items)
+        assert n_remaining_items >= 0
+        if len(page_file.items) >= n_remaining_items:
+            page_file.items = page_file.items[:n_remaining_items]
             page_file.links.next = None
+        all_items += page_file.items
         return httpx.Response(status_code=200, json=page_file.to_dict())
 
     with respx.mock(
         base_url=_base_url,
         assert_all_called=True,
     ) as respx_mock:
-        server_items: List[osparc.File] = page_file.items
+        server_items: List[osparc.File] = deepcopy(page_file.items)
         respx_mock.get(urlparse(page_file.links.next).path).mock(
             side_effect=partial(_sideeffect, server_items)
         )
@@ -126,7 +129,7 @@ def _sideeffect(all_items: List, request: httpx.Request):
         )
         client_items = [item for item in pagination_iterator]
         assert len(server_items) > 0
-        assert all(si == ci for si, ci in zip(server_items, client_items))
+        assert server_items == client_items
 
         first_client_item = next(pagination_iterator)
         assert first_client_item == server_items[0]