Catch connection errors when fetching data during pull-through caching

closes #1499

CHANGES/1499.bugfix

@@ -0,0 +1 @@
+Made the pull-through caching machinery resilient to connection errors.

pulp_container/app/exceptions.py

@@ -1,4 +1,16 @@
-from rest_framework.exceptions import NotFound, ParseError
+from rest_framework.exceptions import APIException, NotFound, ParseError
+
+
+class BadGateway(APIException):
+    status_code = 502
+    default_detail = "Invalid response received from the upstream."
+    default_code = "BAD_GATEWAY"
+
+
+class GatewayTimeout(APIException):
+    status_code = 504
+    default_detail = "Response from the upstream timed out."
+    default_code = "GATEWAY_TIMEOUT"
 
 
 class RepositoryNotFound(NotFound):

pulp_container/app/registry.py

@@ -8,7 +8,7 @@
 from urllib.parse import urljoin
 
 from aiohttp import web
-from aiohttp.client_exceptions import ClientResponseError
+from aiohttp.client_exceptions import ClientResponseError, ClientConnectionError
 from aiohttp.web_exceptions import HTTPTooManyRequests
 from django_guid import set_guid
 from django_guid.utils import generate_guid
@@ -21,6 +21,7 @@
 from pulpcore.plugin.models import RemoteArtifact, Content, ContentArtifact
 from pulpcore.plugin.content import ArtifactResponse
 from pulpcore.plugin.tasking import dispatch
+from pulpcore.plugin.exceptions import TimeoutException
 
 from pulp_container.app.cache import RegistryContentCache
 from pulp_container.app.models import ContainerDistribution, Tag, Blob, Manifest, BlobManifest
@@ -157,7 +158,7 @@ async def get_tag(self, request):
                 response = await downloader.run(
                     extra_data={"headers": V2_ACCEPT_HEADERS, "http_method": "head"}
                 )
-            except ClientResponseError:
+            except (ClientResponseError, ClientConnectionError, TimeoutException):
                 # the manifest is not available on the remote anymore
                 # but the old one is still stored in the database
                 pass

pulp_container/app/registry_api.py

@@ -12,7 +12,7 @@
 import hashlib
 import re
 
-from aiohttp.client_exceptions import ClientResponseError
+from aiohttp.client_exceptions import ClientResponseError, ClientConnectionError
 from itertools import chain
 from urllib.parse import urljoin, urlparse, urlunparse, parse_qs, urlencode
 from tempfile import NamedTemporaryFile
@@ -29,6 +29,8 @@
 from pulpcore.plugin.files import PulpTemporaryUploadedFile
 from pulpcore.plugin.tasking import add_and_remove, dispatch
 from pulpcore.plugin.util import get_objects_for_user, get_url
+from pulpcore.plugin.exceptions import TimeoutException
+
 from rest_framework.exceptions import (
     AuthenticationFailed,
     NotAuthenticated,
@@ -55,6 +57,8 @@
     RegistryApiCache,
 )
 from pulp_container.app.exceptions import (
+    BadGateway,
+    GatewayTimeout,
     InvalidRequest,
     RepositoryNotFound,
     RepositoryInvalid,
@@ -1117,9 +1121,13 @@ def fetch_manifest(self, remote, pk):
                 # the client could request the manifest outside the docker hub pull limit;
                 # it is necessary to pass this information back to the client
                 raise Throttled()
-            else:
-                # TODO: do not mask out relevant errors, like HTTP 502
+            elif response_error.status == 404:
                 raise ManifestNotFound(reference=pk)
+            else:
+                raise BadGateway(detail=response_error.message)
+        except (ClientConnectionError, TimeoutException):
+            # The remote server is not available at the moment
+            raise GatewayTimeout()
         else:
             digest = response.headers.get("docker-content-digest")
             return models.Manifest.objects.filter(digest=digest).first()

requirements.txt

@@ -1,3 +1,3 @@
 jsonschema>=4.4,<4.23
-pulpcore>=3.43.0,<3.55
+pulpcore>=3.46.0,<3.55
 pyjwt[crypto]>=2.4,<2.9