From 99c4decd7c55e55045b1d9dbcc2f3181b554a26d Mon Sep 17 00:00:00 2001 From: Ben Brown Date: Wed, 28 Aug 2024 14:54:41 +0100 Subject: [PATCH] fix: Retry individual calls to the /missing_objects endpoint Split out the individual API call into its own function so retry attempts continue from the position of the loop in the calling function, rather than looping all over again. --- flat-manager-client | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/flat-manager-client b/flat-manager-client index bf3da59..530cfe4 100755 --- a/flat-manager-client +++ b/flat-manager-client @@ -220,22 +220,28 @@ def chunks(l, n): retry=TENACITY_RETRY_EXCEPTIONS, reraise=True, ) +async def _missing_chunk(session, build_url, chunk, headers): + wanted_json = json.dumps({"wanted": chunk}).encode("utf-8") + data = gzip.compress(wanted_json) + + resp = await session.get(build_url + "/missing_objects", data=data, headers=headers) + async with resp: + if resp.status != 200: + raise ApiError(resp, await resp.text()) + + return await resp.json() + + async def missing_objects(session, build_url, token, wanted): missing=[] for chunk in chunks(wanted, 2000): - wanted_json=json.dumps({'wanted': chunk}).encode('utf-8') - data=gzip.compress(wanted_json) headers = { 'Authorization': 'Bearer ' + token, 'Content-Encoding': 'gzip', 'Content-Type': 'application/json' } - resp = await session.get(build_url + "/missing_objects", data=data, headers=headers) - async with resp: - if resp.status != 200: - raise ApiError(resp, await resp.text()) - data = await resp.json() - missing.extend(data["missing"]) + data = await _missing_chunk(session, build_url, chunk, headers) + missing.extend(data["missing"]) return missing @retry(