✨ Pagination with ordering (#1)

* ✨ add `order_by` and `descending` options to query / scan and fetch_all methods Signed-off-by: ff137 <[email protected]> * ✨ add `order_by` and `descending` options to PaginatedQuerySchema Signed-off-by: ff137 <[email protected]> * ✨ modify `get_limit_offset` to `get_paginated_query_params` Signed-off-by: ff137 <[email protected]> * ✨ add ordering to InMemoryStorage scan and fetch_all methods Signed-off-by: ff137 <[email protected]> * 🚧 test in-progress aries-askar PR: openwallet-foundation/askar#291 Signed-off-by: ff137 <[email protected]> * ⬆️ Update lock file Signed-off-by: ff137 <[email protected]> * 🎨 fix ruff warning Signed-off-by: ff137 <[email protected]> * ✅ fix assertions Signed-off-by: ff137 <[email protected]> * 🚧 test aries-askar with TestPyPI package Signed-off-by: ff137 <[email protected]> * 🚧 test latest askar testpypi package Signed-off-by: ff137 <[email protected]> * 🎨 Update order_by description and default value. Include in schema Signed-off-by: ff137 <[email protected]> * ⬆️ Update aries-askar test pypi package to pre-orjson feat release Signed-off-by: ff137 <[email protected]> --------- Signed-off-by: ff137 <[email protected]>
didx-xyz · Aug 29, 2024 · 6cf965e · 6cf965e
1 parent 8e539b5
commit 6cf965e
Show file tree

Hide file tree

Showing 16 changed files with 608 additions and 411 deletions.
diff --git a/aries_cloudagent/messaging/models/base_record.py b/aries_cloudagent/messaging/models/base_record.py
@@ -293,6 +293,8 @@ async def query(
         *,
         limit: Optional[int] = None,
         offset: Optional[int] = None,
+        order_by: Optional[str] = None,
+        descending: bool = False,
         post_filter_positive: dict = None,
         post_filter_negative: dict = None,
         alt: bool = False,
@@ -304,6 +306,8 @@ async def query(
             tag_filter: An optional dictionary of tag filter clauses
             limit: The maximum number of records to retrieve
             offset: The offset to start retrieving records from
+            order_by: An optional field by which to order the records.
+            descending: Whether to order the records in descending order.
             post_filter_positive: Additional value filters to apply matching positively
             post_filter_negative: Additional value filters to apply matching negatively
             alt: set to match any (positive=True) value or miss all (positive=False)
@@ -327,11 +331,15 @@ async def query(
                 tag_query=tag_query,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
             )
         else:
             rows = await storage.find_all_records(
                 type_filter=cls.RECORD_TYPE,
                 tag_query=tag_query,
+                order_by=order_by,
+                descending=descending,
             )
 
         num_results_post_filter = 0  # used if applying pagination post-filter

diff --git a/aries_cloudagent/messaging/models/paginated_query.py b/aries_cloudagent/messaging/models/paginated_query.py
@@ -4,6 +4,7 @@
 
 from aiohttp.web import BaseRequest
 from marshmallow import fields
+from marshmallow.validate import OneOf
 
 from ...messaging.models.openapi import OpenAPISchema
 from ...storage.base import DEFAULT_PAGE_SIZE, MAXIMUM_PAGE_SIZE
@@ -31,18 +32,40 @@ class PaginatedQuerySchema(OpenAPISchema):
         metadata={"description": "Offset for pagination", "example": 0},
         error_messages={"validator_failed": "Value must be 0 or greater"},
     )
+    order_by = fields.Str(
+        required=False,
+        load_default="id",
+        validate=OneOf(["id"]),  # only one possible column supported in askar
+        metadata={
+            "description": (
+                'The column to order results by. Only "id" is currently supported.'
+            )
+        },
+        error_messages={"validator_failed": '`order_by` only supports column "id"'},
+    )
+    descending = fields.Bool(
+        required=False,
+        load_default=False,
+        metadata={"description": "Order results in descending order if true"},
+    )
 
 
-def get_limit_offset(request: BaseRequest) -> Tuple[int, int]:
-    """Read the limit and offset query parameters from a request as ints, with defaults.
+def get_paginated_query_params(request: BaseRequest) -> Tuple[int, int, str, bool]:
+    """Read the limit, offset, order_by, and descending query parameters from a request.
 
     Args:
-        request: aiohttp request object
+        request: aiohttp request object.
 
     Returns:
-        A tuple of the limit and offset values
+        A tuple containing:
+        - limit (int): The number of results to return, defaulting to DEFAULT_PAGE_SIZE.
+        - offset (int): The offset for pagination, defaulting to 0.
+        - order_by (str): The field by which to order results, defaulting to "id".
+        - descending (bool): Order results in descending order; defaults to False.
     """
 
     limit = int(request.query.get("limit", DEFAULT_PAGE_SIZE))
     offset = int(request.query.get("offset", 0))
-    return limit, offset
+    order_by = request.query.get("order_by", "id")
+    descending = bool(request.query.get("descending", False))
+    return limit, offset, order_by, descending
diff --git a/aries_cloudagent/messaging/models/tests/test_base_record.py b/aries_cloudagent/messaging/models/tests/test_base_record.py
@@ -169,6 +169,8 @@ async def test_query(self):
         result = await BaseRecordImpl.query(session, tag_filter)
         mock_storage.find_all_records.assert_awaited_once_with(
             type_filter=BaseRecordImpl.RECORD_TYPE,
+            order_by=None,
+            descending=False,
             tag_query=tag_filter,
         )
         assert result and isinstance(result[0], BaseRecordImpl)
@@ -221,6 +223,8 @@ async def test_query_post_filter(self):
         mock_storage.find_all_records.assert_awaited_once_with(
             type_filter=ARecordImpl.RECORD_TYPE,
             tag_query=tag_filter,
+            order_by=None,
+            descending=False,
         )
         assert result and isinstance(result[0], ARecordImpl)
         assert result[0]._id == record_id
@@ -344,6 +348,8 @@ async def test_query_with_limit(self):
             tag_query=tag_filter,
             limit=10,
             offset=0,
+            order_by=None,
+            descending=False,
         )
         assert result and isinstance(result[0], ARecordImpl)
         assert result[0]._id == record_id
@@ -374,6 +380,8 @@ async def test_query_with_offset(self):
             tag_query=tag_filter,
             limit=DEFAULT_PAGE_SIZE,
             offset=10,
+            order_by=None,
+            descending=False,
         )
         assert result and isinstance(result[0], ARecordImpl)
         assert result[0]._id == record_id
@@ -404,6 +412,8 @@ async def test_query_with_limit_and_offset(self):
             tag_query=tag_filter,
             limit=10,
             offset=5,
+            order_by=None,
+            descending=False,
         )
         assert result and isinstance(result[0], ARecordImpl)
         assert result[0]._id == record_id
@@ -436,7 +446,10 @@ async def test_query_with_limit_and_offset_and_post_filter(self):
             post_filter_positive={"a": "one"},
         )
         mock_storage.find_all_records.assert_awaited_once_with(
-            type_filter=ARecordImpl.RECORD_TYPE, tag_query=tag_filter
+            type_filter=ARecordImpl.RECORD_TYPE,
+            tag_query=tag_filter,
+            order_by=None,
+            descending=False,
         )
         assert len(result) == 10
         assert result and isinstance(result[0], ARecordImpl)

diff --git a/aries_cloudagent/multitenant/admin/routes.py b/aries_cloudagent/multitenant/admin/routes.py
@@ -16,7 +16,10 @@
 from ...core.profile import ProfileManagerProvider
 from ...messaging.models.base import BaseModelError
 from ...messaging.models.openapi import OpenAPISchema
-from ...messaging.models.paginated_query import PaginatedQuerySchema, get_limit_offset
+from ...messaging.models.paginated_query import (
+    PaginatedQuerySchema,
+    get_paginated_query_params,
+)
 from ...messaging.valid import UUID4_EXAMPLE, JSONWebToken
 from ...multitenant.base import BaseMultitenantManager
 from ...storage.error import StorageError, StorageNotFoundError
@@ -382,7 +385,7 @@ async def wallets_list(request: web.BaseRequest):
     if wallet_name:
         query["wallet_name"] = wallet_name
 
-    limit, offset = get_limit_offset(request)
+    limit, offset, order_by, descending = get_paginated_query_params(request)
 
     try:
         async with profile.session() as session:
@@ -391,6 +394,8 @@ async def wallets_list(request: web.BaseRequest):
                 tag_filter=query,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
             )
         results = [format_wallet_record(record) for record in records]
         results.sort(key=lambda w: w["created_at"])

diff --git a/aries_cloudagent/protocols/connections/v1_0/routes.py b/aries_cloudagent/protocols/connections/v1_0/routes.py
@@ -19,7 +19,10 @@
 from ....connections.models.conn_record import ConnRecord, ConnRecordSchema
 from ....messaging.models.base import BaseModelError
 from ....messaging.models.openapi import OpenAPISchema
-from ....messaging.models.paginated_query import PaginatedQuerySchema, get_limit_offset
+from ....messaging.models.paginated_query import (
+    PaginatedQuerySchema,
+    get_paginated_query_params,
+)
 from ....messaging.valid import (
     ENDPOINT_EXAMPLE,
     ENDPOINT_VALIDATE,
@@ -469,7 +472,7 @@ async def connections_list(request: web.BaseRequest):
     if request.query.get("connection_protocol"):
         post_filter["connection_protocol"] = request.query["connection_protocol"]
 
-    limit, offset = get_limit_offset(request)
+    limit, offset, order_by, descending = get_paginated_query_params(request)
 
     profile = context.profile
     try:
@@ -479,6 +482,8 @@ async def connections_list(request: web.BaseRequest):
                 tag_filter,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
                 post_filter_positive=post_filter,
                 alt=True,
             )

diff --git a/aries_cloudagent/protocols/connections/v1_0/tests/test_routes.py b/aries_cloudagent/protocols/connections/v1_0/tests/test_routes.py
@@ -103,6 +103,8 @@ async def test_connections_list(self):
                     },
                     limit=100,
                     offset=0,
+                    order_by="id",
+                    descending=False,
                     post_filter_positive={
                         "their_role": list(ConnRecord.Role.REQUESTER.value),
                         "connection_protocol": "connections/1.0",

diff --git a/aries_cloudagent/protocols/issue_credential/v1_0/routes.py b/aries_cloudagent/protocols/issue_credential/v1_0/routes.py
@@ -22,7 +22,10 @@
 from ....messaging.credential_definitions.util import CRED_DEF_TAGS
 from ....messaging.models.base import BaseModelError
 from ....messaging.models.openapi import OpenAPISchema
-from ....messaging.models.paginated_query import PaginatedQuerySchema, get_limit_offset
+from ....messaging.models.paginated_query import (
+    PaginatedQuerySchema,
+    get_paginated_query_params,
+)
 from ....messaging.valid import (
     INDY_CRED_DEF_ID_EXAMPLE,
     INDY_CRED_DEF_ID_VALIDATE,
@@ -404,7 +407,7 @@ async def credential_exchange_list(request: web.BaseRequest):
         if request.query.get(k, "") != ""
     }
 
-    limit, offset = get_limit_offset(request)
+    limit, offset, order_by, descending = get_paginated_query_params(request)
 
     try:
         async with context.profile.session() as session:
@@ -413,6 +416,8 @@ async def credential_exchange_list(request: web.BaseRequest):
                 tag_filter=tag_filter,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
                 post_filter_positive=post_filter,
             )
         results = [record.serialize() for record in records]

diff --git a/aries_cloudagent/protocols/issue_credential/v2_0/routes.py b/aries_cloudagent/protocols/issue_credential/v2_0/routes.py
@@ -26,7 +26,10 @@
 from ....messaging.decorators.attach_decorator import AttachDecorator
 from ....messaging.models.base import BaseModelError
 from ....messaging.models.openapi import OpenAPISchema
-from ....messaging.models.paginated_query import PaginatedQuerySchema, get_limit_offset
+from ....messaging.models.paginated_query import (
+    PaginatedQuerySchema,
+    get_paginated_query_params,
+)
 from ....messaging.valid import (
     INDY_CRED_DEF_ID_EXAMPLE,
     INDY_CRED_DEF_ID_VALIDATE,
@@ -568,7 +571,7 @@ async def credential_exchange_list(request: web.BaseRequest):
         if request.query.get(k, "") != ""
     }
 
-    limit, offset = get_limit_offset(request)
+    limit, offset, order_by, descending = get_paginated_query_params(request)
 
     try:
         async with profile.session() as session:
@@ -577,6 +580,8 @@ async def credential_exchange_list(request: web.BaseRequest):
                 tag_filter=tag_filter,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
                 post_filter_positive=post_filter,
             )
 

diff --git a/aries_cloudagent/protocols/present_proof/v1_0/routes.py b/aries_cloudagent/protocols/present_proof/v1_0/routes.py
@@ -25,7 +25,10 @@
 from ....messaging.decorators.attach_decorator import AttachDecorator
 from ....messaging.models.base import BaseModelError
 from ....messaging.models.openapi import OpenAPISchema
-from ....messaging.models.paginated_query import PaginatedQuerySchema, get_limit_offset
+from ....messaging.models.paginated_query import (
+    PaginatedQuerySchema,
+    get_paginated_query_params,
+)
 from ....messaging.valid import (
     INDY_EXTRA_WQL_EXAMPLE,
     INDY_EXTRA_WQL_VALIDATE,
@@ -309,7 +312,7 @@ async def presentation_exchange_list(request: web.BaseRequest):
         if request.query.get(k, "") != ""
     }
 
-    limit, offset = get_limit_offset(request)
+    limit, offset, order_by, descending = get_paginated_query_params(request)
 
     try:
         async with context.profile.session() as session:
@@ -318,6 +321,8 @@ async def presentation_exchange_list(request: web.BaseRequest):
                 tag_filter=tag_filter,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
                 post_filter_positive=post_filter,
             )
         results = [record.serialize() for record in records]

diff --git a/aries_cloudagent/protocols/present_proof/v2_0/routes.py b/aries_cloudagent/protocols/present_proof/v2_0/routes.py
@@ -26,7 +26,10 @@
 from ....messaging.decorators.attach_decorator import AttachDecorator
 from ....messaging.models.base import BaseModelError
 from ....messaging.models.openapi import OpenAPISchema
-from ....messaging.models.paginated_query import PaginatedQuerySchema, get_limit_offset
+from ....messaging.models.paginated_query import (
+    PaginatedQuerySchema,
+    get_paginated_query_params,
+)
 from ....messaging.valid import (
     INDY_EXTRA_WQL_EXAMPLE,
     INDY_EXTRA_WQL_VALIDATE,
@@ -448,7 +451,7 @@ async def present_proof_list(request: web.BaseRequest):
         if request.query.get(k, "") != ""
     }
 
-    limit, offset = get_limit_offset(request)
+    limit, offset, order_by, descending = get_paginated_query_params(request)
 
     try:
         async with profile.session() as session:
@@ -457,6 +460,8 @@ async def present_proof_list(request: web.BaseRequest):
                 tag_filter=tag_filter,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
                 post_filter_positive=post_filter,
             )
         results = [record.serialize() for record in records]

diff --git a/aries_cloudagent/storage/askar.py b/aries_cloudagent/storage/askar.py
@@ -174,6 +174,8 @@ async def find_paginated_records(
         tag_query: Mapping = None,
         limit: int = DEFAULT_PAGE_SIZE,
         offset: int = 0,
+        order_by: Optional[str] = None,
+        descending: bool = False,
     ) -> Sequence[StorageRecord]:
         """Retrieve a page of records matching a particular type filter and tag query.
 
@@ -182,6 +184,11 @@ async def find_paginated_records(
             tag_query: An optional dictionary of tag filter clauses
             limit: The maximum number of records to retrieve
             offset: The offset to start retrieving records from
+            order_by: An optional field by which to order the records.
+            descending: Whether to order the records in descending order.
+
+        Returns:
+            A sequence of StorageRecord matching the filter and query parameters.
         """
         results = []
 
@@ -190,6 +197,8 @@ async def find_paginated_records(
             tag_filter=tag_query,
             limit=limit,
             offset=offset,
+            order_by=order_by,
+            descending=descending,
             profile=self._session.profile.settings.get("wallet.askar_profile"),
         ):
             results += (
@@ -206,13 +215,19 @@ async def find_all_records(
         self,
         type_filter: str,
         tag_query: Mapping = None,
+        order_by: Optional[str] = None,
+        descending: bool = False,
         options: Mapping = None,
     ):
         """Retrieve all records matching a particular type filter and tag query."""
         for_update = bool(options and options.get("forUpdate"))
         results = []
         for row in await self._session.handle.fetch_all(
-            type_filter, tag_query, for_update=for_update
+            category=type_filter,
+            tag_filter=tag_query,
+            order_by=order_by,
+            descending=descending,
+            for_update=for_update,
         ):
             results.append(
                 StorageRecord(