From 11be96298576ac7e33b8dad6deb5b3e83aa3e881 Mon Sep 17 00:00:00 2001 From: Ben Brown Date: Wed, 28 Aug 2024 14:54:41 +0100 Subject: [PATCH 1/2] fix: Retry individual calls to the /missing_objects endpoint Split out the individual API call into its own function so retry attempts continue from the position of the loop in the calling function, rather than looping all over again. --- flat-manager-client | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/flat-manager-client b/flat-manager-client index 58f3c30..0be8fdf 100755 --- a/flat-manager-client +++ b/flat-manager-client @@ -241,24 +241,28 @@ def chunks(iterable, n): retry=TENACITY_RETRY_EXCEPTIONS, reraise=True, ) +async def _missing_chunk(session, build_url, chunk, headers): + wanted_json = json.dumps({"wanted": chunk}).encode("utf-8") + data = gzip.compress(wanted_json) + + resp = await session.get(build_url + "/missing_objects", data=data, headers=headers) + async with resp: + if resp.status != 200: + raise ApiError(resp, await resp.text()) + + return await resp.json() + + async def missing_objects(session, build_url, token, wanted): missing = [] for chunk in chunks(wanted, 2000): - wanted_json = json.dumps({"wanted": chunk}).encode("utf-8") - data = gzip.compress(wanted_json) headers = { "Authorization": "Bearer " + token, "Content-Encoding": "gzip", "Content-Type": "application/json", } - resp = await session.get( - build_url + "/missing_objects", data=data, headers=headers - ) - async with resp: - if resp.status != 200: - raise ApiError(resp, await resp.text()) - data = await resp.json() - missing.extend(data["missing"]) + data = await _missing_chunk(session, build_url, chunk, headers) + missing.extend(data["missing"]) return missing From 4daed7b7a8a87a3d477a86275b028b98bbdd8ba1 Mon Sep 17 00:00:00 2001 From: Ben Brown Date: Wed, 28 Aug 2024 14:59:35 +0100 Subject: [PATCH 2/2] fix: Raise ServerApiError in _missing_chunk() to trigger retries --- flat-manager-client | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flat-manager-client b/flat-manager-client index 0be8fdf..f7b3ec6 100755 --- a/flat-manager-client +++ b/flat-manager-client @@ -247,6 +247,8 @@ async def _missing_chunk(session, build_url, chunk, headers): resp = await session.get(build_url + "/missing_objects", data=data, headers=headers) async with resp: + if resp.status >= 500: + raise ServerApiError(resp, await resp.text()) if resp.status != 200: raise ApiError(resp, await resp.text())