diff --git a/CHANGES/1751.deprecation b/CHANGES/1751.deprecation new file mode 100644 index 000000000..0aa486def --- /dev/null +++ b/CHANGES/1751.deprecation @@ -0,0 +1 @@ +Deprecated the Manifest `is_flatpak` and `is_bootable` fields in favor of the new `type` field. diff --git a/CHANGES/1751.feature b/CHANGES/1751.feature new file mode 100644 index 000000000..4d39daf8b --- /dev/null +++ b/CHANGES/1751.feature @@ -0,0 +1,2 @@ +Introduced the `type` field on the Manifests endpoint to enable easier differentiation of image +types. diff --git a/pulp_container/app/management/commands/container-handle-image-data.py b/pulp_container/app/management/commands/container-handle-image-data.py index a8635f1e7..53c118933 100644 --- a/pulp_container/app/management/commands/container-handle-image-data.py +++ b/pulp_container/app/management/commands/container-handle-image-data.py @@ -39,10 +39,14 @@ class Command(BaseCommand): def handle(self, *args, **options): manifests_updated_count = 0 - manifests_v1 = Manifest.objects.filter(data__isnull=True, media_type=MEDIA_TYPE.MANIFEST_V1) + manifests_v1 = Manifest.objects.filter( + Q(media_type=MEDIA_TYPE.MANIFEST_V1), Q(data__isnull=True) | Q(type__isnull=True) + ) manifests_updated_count += self.update_manifests(manifests_v1) - manifests_v2 = Manifest.objects.filter(Q(data__isnull=True) | Q(annotations={}, labels={})) + manifests_v2 = Manifest.objects.filter( + Q(data__isnull=True) | Q(annotations={}, labels={}) | Q(type__isnull=True) + ) manifests_v2 = manifests_v2.exclude( media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI, MEDIA_TYPE.MANIFEST_V1] ) @@ -68,6 +72,15 @@ def handle(self, *args, **options): def update_manifests(self, manifests_qs): manifests_updated_count = 0 manifests_to_update = [] + fields_to_update = [ + "annotations", + "labels", + "is_bootable", + "is_flatpak", + "data", + "type", + ] + for manifest in manifests_qs.iterator(): # suppress non-existing/already migrated artifacts and corrupted JSON files with suppress(ObjectDoesNotExist, JSONDecodeError): @@ -76,7 +89,6 @@ def update_manifests(self, manifests_qs): manifests_to_update.append(manifest) if len(manifests_to_update) > 1000: - fields_to_update = ["annotations", "labels", "is_bootable", "is_flatpak", "data"] manifests_qs.model.objects.bulk_update( manifests_to_update, fields_to_update, @@ -85,7 +97,6 @@ def update_manifests(self, manifests_qs): manifests_to_update.clear() if manifests_to_update: - fields_to_update = ["annotations", "labels", "is_bootable", "is_flatpak", "data"] manifests_qs.model.objects.bulk_update( manifests_to_update, fields_to_update, @@ -100,11 +111,12 @@ def init_manifest(self, manifest): manifest_data, raw_bytes_data = get_content_data(manifest_artifact) manifest.data = raw_bytes_data.decode("utf-8") - if not (manifest.annotations or manifest.labels): + if not (manifest.annotations or manifest.labels or manifest.type): manifest.init_metadata(manifest_data) manifest._artifacts.clear() - return True + elif not manifest.type: + return manifest.init_image_nature() return False diff --git a/pulp_container/app/migrations/0042_add_manifest_nature_field.py b/pulp_container/app/migrations/0042_add_manifest_nature_field.py new file mode 100644 index 000000000..73b3c304c --- /dev/null +++ b/pulp_container/app/migrations/0042_add_manifest_nature_field.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.16 on 2024-10-21 19:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('container', '0041_add_pull_through_pull_permissions'), + ] + + operations = [ + migrations.AddField( + model_name='manifest', + name='type', + field=models.CharField(null=True), + ), + ] diff --git a/pulp_container/app/models.py b/pulp_container/app/models.py index b96701c92..7c67f5bfb 100644 --- a/pulp_container/app/models.py +++ b/pulp_container/app/models.py @@ -32,7 +32,13 @@ from . import downloaders from pulp_container.app.utils import get_content_data -from pulp_container.constants import MEDIA_TYPE, SIGNATURE_TYPE +from pulp_container.constants import ( + COSIGN_MEDIA_TYPES, + MANIFEST_MEDIA_TYPES, + MANIFEST_TYPE, + MEDIA_TYPE, + SIGNATURE_TYPE, +) logger = getLogger(__name__) @@ -72,6 +78,7 @@ class Manifest(Content): digest (models.TextField): The manifest digest. schema_version (models.IntegerField): The manifest schema version. media_type (models.TextField): The manifest media type. + type (models.TextField): The manifest's type (flatpak, bootable, signature, etc.). data (models.TextField): The manifest's data in text format. annotations (models.JSONField): Metadata stored inside the image manifest. labels (models.JSONField): Metadata stored inside the image configuration. @@ -99,12 +106,15 @@ class Manifest(Content): digest = models.TextField(db_index=True) schema_version = models.IntegerField() media_type = models.TextField(choices=MANIFEST_CHOICES) + type = models.CharField(null=True) data = models.TextField(null=True) annotations = models.JSONField(default=dict) labels = models.JSONField(default=dict) + # DEPRECATED: this field is deprecated and will be removed in a future release. is_bootable = models.BooleanField(default=False) + # DEPRECATED: this field is deprecated and will be removed in a future release. is_flatpak = models.BooleanField(default=False) blobs = models.ManyToManyField(Blob, through="BlobManifest") @@ -154,40 +164,93 @@ def init_image_nature(self): return self.init_manifest_nature() def init_manifest_list_nature(self): + updated_type = False + if not self.type: + self.type = MANIFEST_TYPE.UNKNOWN + updated_type = True + for manifest in self.listed_manifests.all(): # it suffices just to have a single manifest of a specific nature; # there is no case where the manifest is both bootable and flatpak-based - if manifest.is_bootable: + if manifest.type == MANIFEST_TYPE.BOOTABLE: + self.type = MANIFEST_TYPE.BOOTABLE self.is_bootable = True return True - elif manifest.is_flatpak: + elif manifest.type == MANIFEST_TYPE.FLATPAK: + self.type = MANIFEST_TYPE.FLATPAK self.is_flatpak = True return True - return False + return updated_type def init_manifest_nature(self): if self.is_bootable_image(): + # DEPRECATED: is_bootable is deprecated and will be removed in a future release. self.is_bootable = True + self.type = MANIFEST_TYPE.BOOTABLE return True elif self.is_flatpak_image(): + # DEPRECATED: is_flatpak is deprecated and will be removed in a future release. self.is_flatpak = True + self.type = MANIFEST_TYPE.FLATPAK return True - else: - return False + elif self.is_helm_chart(): + self.type = MANIFEST_TYPE.HELM + return True + elif media_type := self.is_cosign(): + self.type = self.get_cosign_type(media_type) + return True + elif self.is_manifest_image(): + self.type = MANIFEST_TYPE.IMAGE + return True + + return False def is_bootable_image(self): - if ( + return ( self.annotations.get("containers.bootc") == "1" or self.labels.get("containers.bootc") == "1" - ): - return True - else: - return False + ) def is_flatpak_image(self): return True if self.labels.get("org.flatpak.ref") else False + def is_manifest_image(self): + return self.media_type in MANIFEST_MEDIA_TYPES.IMAGE + + @property + def json_manifest(self): + return json.loads(self.data) + + def is_cosign(self): + try: + # layers is not a mandatory field + layers = self.json_manifest["layers"] + except KeyError: + return False + + for layer in layers: + if layer["mediaType"] in COSIGN_MEDIA_TYPES: + return layer["mediaType"] + return False + + def get_cosign_type(self, media_type): + if media_type in MEDIA_TYPE.COSIGN_SBOM: + return MANIFEST_TYPE.COSIGN_SBOM + + cosign_types_mapping = { + MEDIA_TYPE.COSIGN_BLOB: MANIFEST_TYPE.COSIGN_SIGNATURE, + MEDIA_TYPE.COSIGN_ATTESTATION: MANIFEST_TYPE.COSIGN_ATTESTATION, + MEDIA_TYPE.COSIGN_ATTESTATION_BUNDLE: MANIFEST_TYPE.COSIGN_ATTESTATION_BUNDLE, + } + return cosign_types_mapping.get(media_type) + + def is_helm_chart(self): + try: + return self.json_manifest["config"]["mediaType"] == MEDIA_TYPE.CONFIG_BLOB_HELM + except KeyError: + return False + class Meta: default_related_name = "%(app_label)s_%(model_name)s" unique_together = ("digest",) diff --git a/pulp_container/app/registry_api.py b/pulp_container/app/registry_api.py index 83c95e78e..0d8606fce 100644 --- a/pulp_container/app/registry_api.py +++ b/pulp_container/app/registry_api.py @@ -90,6 +90,7 @@ ) from pulp_container.constants import ( EMPTY_BLOB, + MANIFEST_TYPE, SIGNATURE_API_EXTENSION_VERSION, SIGNATURE_HEADER, SIGNATURE_PAYLOAD_MAX_SIZE, @@ -1213,6 +1214,11 @@ def put(self, request, path, pk=None): ManifestInvalid(digest=manifest_digest) manifest_list = self._init_manifest(manifest_digest, media_type, raw_text_data) + manifest_list.type = MANIFEST_TYPE.UNKNOWN + for manifest in found_manifests: + if manifest.type in [MANIFEST_TYPE.BOOTABLE, MANIFEST_TYPE.FLATPAK]: + manifest_list.type = manifest.type + break manifest_list = self._save_manifest(manifest_list) manifests_to_list = [] @@ -1235,6 +1241,9 @@ def put(self, request, path, pk=None): ) manifest = manifest_list + # DEPRECATED: is_bootable and is_flatpak are deprecated and will be removed in a + # future release. Keeping this block for now to avoid introducing a bug or + # a regression. # once relations for listed manifests are established, it is # possible to initialize the nature of the manifest list if manifest.init_manifest_list_nature(): diff --git a/pulp_container/app/serializers.py b/pulp_container/app/serializers.py index 1287e49fe..3203dd538 100644 --- a/pulp_container/app/serializers.py +++ b/pulp_container/app/serializers.py @@ -64,6 +64,11 @@ class ManifestSerializer(NoArtifactContentSerializer): digest = serializers.CharField(help_text="sha256 of the Manifest file") schema_version = serializers.IntegerField(help_text="Manifest schema version") media_type = serializers.CharField(help_text="Manifest media type of the file") + type = serializers.CharField( + help_text="Manifest type (flatpak, bootable, signature, etc.).", + required=False, + default=None, + ) listed_manifests = DetailRelatedField( many=True, help_text="Manifests that are referenced by this Manifest List", @@ -93,15 +98,23 @@ class ManifestSerializer(NoArtifactContentSerializer): help_text=_("Property describing metadata stored inside the image configuration"), ) + # DEPRECATED: this field is deprecated and will be removed in a future release. is_bootable = serializers.BooleanField( required=False, default=False, - help_text=_("A boolean determining whether users can boot from an image or not."), + help_text=_( + "A boolean determining whether users can boot from an image or not." + "[deprecated] check type field instead" + ), ) + # DEPRECATED: this field is deprecated and will be removed in a future release. is_flatpak = serializers.BooleanField( required=False, default=False, - help_text=_("A boolean determining whether the image bundles a Flatpak application"), + help_text=_( + "A boolean determining whether the image bundles a Flatpak application." + "[deprecated] check type field instead" + ), ) class Meta: @@ -116,6 +129,7 @@ class Meta: "labels", "is_bootable", "is_flatpak", + "type", ) model = models.Manifest diff --git a/pulp_container/app/tasks/builder.py b/pulp_container/app/tasks/builder.py index 80d87b339..0739242aa 100644 --- a/pulp_container/app/tasks/builder.py +++ b/pulp_container/app/tasks/builder.py @@ -83,6 +83,7 @@ def add_image_from_directory_to_repository(path, repository, tag): with repository.new_version() as new_repo_version: manifest_json = json.loads(manifest_text_data) + manifest.init_metadata(manifest_json) config_blob = get_or_create_blob(manifest_json["config"], manifest, path) manifest.config_blob = config_blob diff --git a/pulp_container/app/tasks/sync_stages.py b/pulp_container/app/tasks/sync_stages.py index 15ade2596..436eed0c2 100644 --- a/pulp_container/app/tasks/sync_stages.py +++ b/pulp_container/app/tasks/sync_stages.py @@ -12,6 +12,7 @@ from pulpcore.plugin.stages import DeclarativeArtifact, DeclarativeContent, Stage, ContentSaver from pulp_container.constants import ( + MANIFEST_TYPE, MEDIA_TYPE, SIGNATURE_API_EXTENSION_VERSION, SIGNATURE_HEADER, @@ -287,9 +288,21 @@ async def resolve_flush(self): self.manifest_dcs.clear() for manifest_list_dc in self.manifest_list_dcs: + manifest_list_type = MANIFEST_TYPE.UNKNOWN for listed_manifest in manifest_list_dc.extra_data["listed_manifests"]: # Just await here. They will be associated in the post_save hook. await listed_manifest["manifest_dc"].resolution() + + # if manifestlist type is not defined as "unknown" we already checked its nature, + # in this case, there is no need to verify the other manifests from list + if manifest_list_type != MANIFEST_TYPE.UNKNOWN and listed_manifest[ + "manifest_dc" + ].content.type in [ + MANIFEST_TYPE.BOOTABLE, + MANIFEST_TYPE.FLATPAK, + ]: + manifest_list_type = listed_manifest["manifest_dc"].content.type + manifest_list_dc.content.type = manifest_list_type await self.put(manifest_list_dc) self.manifest_list_dcs.clear() @@ -392,6 +405,7 @@ def create_manifest(self, manifest_data, raw_text_data, media_type, digest=None) annotations=manifest_data.get("annotations", {}), ) + manifest.init_manifest_nature() manifest_dc = DeclarativeContent(content=manifest) return manifest_dc @@ -644,6 +658,8 @@ def _post_save(self, batch): if manifest_list_manifests: ManifestListManifest.objects.bulk_create(manifest_list_manifests, ignore_conflicts=True) + # DEPRECATED: is_bootable/is_flatpak are deprecated and will be removed in a future release. + # keeping this block for now to avoid introducing a bug or a regression # after creating the relation between listed manifests and manifest lists, # it is possible to initialize the nature of the corresponding manifest lists for ml in manifest_lists: diff --git a/pulp_container/constants.py b/pulp_container/constants.py index 8d6463481..3479aa1cb 100644 --- a/pulp_container/constants.py +++ b/pulp_container/constants.py @@ -19,6 +19,10 @@ FOREIGN_BLOB_OCI_TAR_GZIP="application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", FOREIGN_BLOB_OCI_TAR_ZSTD="application/vnd.oci.image.layer.nondistributable.v1.tar+zstd", OCI_EMPTY_JSON="application/vnd.oci.empty.v1+json", + CONFIG_BLOB_HELM="application/vnd.cncf.helm.config.v1+json", + COSIGN_BLOB="application/vnd.dev.cosign.simplesigning.v1+json", + COSIGN_ATTESTATION="application/vnd.dsse.envelope.v1+json", + COSIGN_ATTESTATION_BUNDLE="application/vnd.dev.sigstore.bundle.v0.3+json", ) V2_ACCEPT_HEADERS = { @@ -71,3 +75,34 @@ SIGNATURE_PAYLOAD_MAX_SIZE = 4 * MEGABYTE SIGNATURE_API_EXTENSION_VERSION = 2 + +MANIFEST_TYPE = SimpleNamespace( + IMAGE="image", + BOOTABLE="bootable", + FLATPAK="flatpak", + HELM="helm", + COSIGN_SIGNATURE="cosign_signature", + COSIGN_ATTESTATION="cosign_attestation", + COSIGN_ATTESTATION_BUNDLE="cosign_attestation_bundle", + COSIGN_SBOM="cosign_sbom", + UNKNOWN="unknown", +) + +# COSIGN SPEC +# note: SBOM attachments are deprecated and support will be removed in a Cosign release soon +COSIGN_SBOM_FORMATS = ["application/vnd.cyclonedx", "text/spdx", "application/vnd.syft+json"] +COSIGN_SBOM_FORMATS_SUFFIXES = ["xml", "json"] +COSIGN_SBOM_FORMATS_WITH_SUFFIXES = [ + f"{sbom_formats}+{sbom_suffixes}" + for sbom_formats in COSIGN_SBOM_FORMATS + if sbom_formats != "application/vnd.syft+json" # syft is a JSON only format + for sbom_suffixes in COSIGN_SBOM_FORMATS_SUFFIXES +] + +MEDIA_TYPE.COSIGN_SBOM = COSIGN_SBOM_FORMATS + COSIGN_SBOM_FORMATS_WITH_SUFFIXES +COSIGN_MEDIA_TYPES = [ + *MEDIA_TYPE.COSIGN_SBOM, + MEDIA_TYPE.COSIGN_BLOB, + MEDIA_TYPE.COSIGN_ATTESTATION, + MEDIA_TYPE.COSIGN_ATTESTATION_BUNDLE, +] diff --git a/pulp_container/tests/functional/api/test_build_images.py b/pulp_container/tests/functional/api/test_build_images.py index 00574123d..3980246b0 100644 --- a/pulp_container/tests/functional/api/test_build_images.py +++ b/pulp_container/tests/functional/api/test_build_images.py @@ -7,6 +7,7 @@ from pulp_smash.pulp3.bindings import monitor_task from pulpcore.client.pulp_container import ApiException, ContainerContainerDistribution +from pulp_container.constants import MANIFEST_TYPE @pytest.fixture @@ -62,6 +63,7 @@ def _build_image(repository, containerfile=None, containerfile_name=None, build_ def test_build_image_with_uploaded_containerfile( build_image, + check_manifest_fields, containerfile_name, container_distribution_api, container_repo, @@ -85,6 +87,9 @@ def test_build_image_with_uploaded_containerfile( local_registry.pull(distribution.base_path) image = local_registry.inspect(distribution.base_path) assert image[0]["Config"]["Cmd"] == ["cat", "/tmp/inside-image.txt"] + assert check_manifest_fields( + manifest_filters={"digest": image[0]["Digest"]}, fields={"type": MANIFEST_TYPE.IMAGE} + ) def test_build_image_from_repo_version_with_anon_user( diff --git a/pulp_container/tests/functional/api/test_pull_through_cache.py b/pulp_container/tests/functional/api/test_pull_through_cache.py index f4f6ac01d..3eb170e50 100644 --- a/pulp_container/tests/functional/api/test_pull_through_cache.py +++ b/pulp_container/tests/functional/api/test_pull_through_cache.py @@ -7,6 +7,7 @@ from subprocess import CalledProcessError from uuid import uuid4 +from pulp_container.constants import MANIFEST_TYPE from pulp_container.tests.functional.constants import ( REGISTRY_V2, PULP_HELLO_WORLD_REPO, @@ -38,6 +39,7 @@ def _add_pull_through_entities_to_cleanup(path): def pull_and_verify( anonymous_user, add_pull_through_entities_to_cleanup, + check_manifest_fields, container_pull_through_distribution_api, container_distribution_api, container_repository_api, @@ -59,6 +61,12 @@ def _pull_and_verify(images, pull_through_distribution): local_registry.pull(local_image_path) local_image = local_registry.inspect(local_image_path) + # 1.1. check pulp manifest model fields + assert check_manifest_fields( + manifest_filters={"digest": local_image[0]["Digest"]}, + fields={"type": MANIFEST_TYPE.IMAGE}, + ) + path, tag = local_image_path.split(":") tags_to_verify.append(tag) diff --git a/pulp_container/tests/functional/api/test_push_content.py b/pulp_container/tests/functional/api/test_push_content.py index baf659f62..07898ba34 100644 --- a/pulp_container/tests/functional/api/test_push_content.py +++ b/pulp_container/tests/functional/api/test_push_content.py @@ -17,7 +17,7 @@ PulpTestCase, ) -from pulp_container.constants import MEDIA_TYPE +from pulp_container.constants import MEDIA_TYPE, MANIFEST_TYPE from pulp_container.tests.functional.api import rbac_base from pulp_container.tests.functional.constants import REGISTRY_V2_REPO_PULP @@ -39,6 +39,7 @@ def test_push_using_registry_client_admin( add_to_cleanup, registry_client, local_registry, + check_manifest_fields, container_namespace_api, ): """Test push with official registry client and logged in as admin.""" @@ -48,6 +49,14 @@ def test_push_using_registry_client_admin( registry_client.pull(image_path) local_registry.tag_and_push(image_path, local_url) local_registry.pull(local_url) + + # check pulp manifest model fields + local_image = local_registry.inspect(local_url) + assert check_manifest_fields( + manifest_filters={"digest": local_image[0]["Digest"]}, + fields={"type": MANIFEST_TYPE.IMAGE}, + ) + # ensure that same content can be pushed twice without permission errors local_registry.tag_and_push(image_path, local_url) diff --git a/pulp_container/tests/functional/api/test_sync.py b/pulp_container/tests/functional/api/test_sync.py index bfb088fdd..d2d534fe7 100644 --- a/pulp_container/tests/functional/api/test_sync.py +++ b/pulp_container/tests/functional/api/test_sync.py @@ -3,9 +3,11 @@ import pytest from pulpcore.tests.functional.utils import PulpTaskError -from pulp_container.tests.functional.constants import PULP_FIXTURE_1, PULP_LABELED_FIXTURE - +from pulp_container.constants import MEDIA_TYPE, MANIFEST_TYPE from pulp_container.tests.functional.constants import ( + PULP_FIXTURE_1, + PULP_LABELED_FIXTURE, + PULP_HELLO_WORLD_LINUX_AMD64_DIGEST, REGISTRY_V2_FEED_URL, ) @@ -39,13 +41,29 @@ def _synced_container_repository_factory( @pytest.mark.parallel -def test_basic_sync(container_repo, container_remote, container_repository_api, container_sync): - container_sync(container_repo, container_remote) +def test_basic_sync( + check_manifest_fields, + container_repo, + container_remote, + container_repository_api, + container_sync, +): + repo_version = container_sync(container_repo, container_remote).created_resources[0] repository = container_repository_api.read(container_repo.pulp_href) assert "versions/1/" in repository.latest_version_href latest_version_href = repository.latest_version_href + + assert check_manifest_fields( + manifest_filters={ + "repository_version": repo_version, + "media_type": [MEDIA_TYPE.MANIFEST_V2], + "digest": PULP_HELLO_WORLD_LINUX_AMD64_DIGEST, + }, + fields={"type": MANIFEST_TYPE.IMAGE}, + ) + container_sync( container_repo, container_remote ) # We expect that this second sync doesn't create a new repo version diff --git a/pulp_container/tests/functional/conftest.py b/pulp_container/tests/functional/conftest.py index 4152eedeb..bcae14793 100644 --- a/pulp_container/tests/functional/conftest.py +++ b/pulp_container/tests/functional/conftest.py @@ -461,3 +461,16 @@ def _pull_through_distribution(includes=None, excludes=None, private=False): return distribution return _pull_through_distribution + + +@pytest.fixture +def check_manifest_fields(container_manifest_api): + def _check_manifest_fields(**kwargs): + manifest = container_manifest_api.list(**kwargs["manifest_filters"]) + manifest = manifest.to_dict()["results"][0] + for key in kwargs["fields"]: + if manifest[key] != kwargs["fields"][key]: + return False + return True + + return _check_manifest_fields