Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add manifest arch, os, and compressed layers size fields #1782

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES/1767.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added `architecture`, `os`, and `compressed_layers_size` fields to Manifest.
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import json
from json.decoder import JSONDecodeError

from gettext import gettext as _
Expand Down Expand Up @@ -39,10 +40,22 @@ class Command(BaseCommand):
def handle(self, *args, **options):
manifests_updated_count = 0

manifests_v1 = Manifest.objects.filter(data__isnull=True, media_type=MEDIA_TYPE.MANIFEST_V1)
manifests_v1 = Manifest.objects.filter(
Q(media_type=MEDIA_TYPE.MANIFEST_V1),
Q(data__isnull=True)
| Q(architecture__isnull=True)
| Q(os__isnull=True)
| Q(compressed_layers_size__isnull=True),
)
manifests_updated_count += self.update_manifests(manifests_v1)

manifests_v2 = Manifest.objects.filter(Q(data__isnull=True) | Q(annotations={}, labels={}))
manifests_v2 = Manifest.objects.filter(
Q(data__isnull=True)
| Q(annotations={}, labels={})
| Q(architecture__isnull=True)
| Q(os__isnull=True)
| Q(compressed_layers_size__isnull=True)
)
manifests_v2 = manifests_v2.exclude(
media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI, MEDIA_TYPE.MANIFEST_V1]
)
Expand All @@ -68,6 +81,17 @@ def handle(self, *args, **options):
def update_manifests(self, manifests_qs):
manifests_updated_count = 0
manifests_to_update = []
fields_to_update = [
"annotations",
"labels",
"is_bootable",
"is_flatpak",
"data",
"os",
"architecture",
"compressed_layers_size",
]

for manifest in manifests_qs.iterator():
# suppress non-existing/already migrated artifacts and corrupted JSON files
with suppress(ObjectDoesNotExist, JSONDecodeError):
Expand All @@ -76,7 +100,6 @@ def update_manifests(self, manifests_qs):
manifests_to_update.append(manifest)

if len(manifests_to_update) > 1000:
fields_to_update = ["annotations", "labels", "is_bootable", "is_flatpak", "data"]
manifests_qs.model.objects.bulk_update(
manifests_to_update,
fields_to_update,
Expand All @@ -85,7 +108,6 @@ def update_manifests(self, manifests_qs):
manifests_to_update.clear()

if manifests_to_update:
fields_to_update = ["annotations", "labels", "is_bootable", "is_flatpak", "data"]
manifests_qs.model.objects.bulk_update(
manifests_to_update,
fields_to_update,
Expand All @@ -103,8 +125,23 @@ def init_manifest(self, manifest):
if not (manifest.annotations or manifest.labels):
manifest.init_metadata(manifest_data)

if self.needs_os_arch_size_update(manifest):
self.init_manifest_os_arch_size(manifest)
manifest._artifacts.clear()
return True

elif self.needs_os_arch_size_update(manifest):
self.init_manifest_os_arch_size(manifest)
return True

return False

def needs_os_arch_size_update(self, manifest):
return manifest.media_type not in [MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI] and not (
manifest.architecture or manifest.os or manifest.compressed_layers_size
)

def init_manifest_os_arch_size(self, manifest):
manifest_data = json.loads(manifest.data)
manifest.init_architecture_and_os(manifest_data)
manifest.init_compressed_layers_size(manifest_data)
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# Generated by Django 4.2.16 on 2024-10-02 12:04
import warnings

from django.db import migrations, models

def print_warning_for_updating_manifest_fields(apps, schema_editor):
warnings.warn(
"Run 'pulpcore-manager container-handle-image-data' to update the manifests' "
"os, architecture, and compressed_layers_size fields."
)

class Migration(migrations.Migration):

dependencies = [
('container', '0041_add_pull_through_pull_permissions'),
]

operations = [
migrations.AddField(
model_name='manifest',
name='architecture',
field=models.TextField(null=True),
),
migrations.AddField(
model_name='manifest',
name='compressed_layers_size',
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name='manifest',
name='os',
field=models.TextField(null=True),
),
migrations.AlterField(
model_name='manifestlistmanifest',
name='architecture',
field=models.TextField(blank=True, default=''),
),
migrations.AlterField(
model_name='manifestlistmanifest',
name='os',
field=models.TextField(blank=True, default=''),
),
migrations.RunPython(
print_warning_for_updating_manifest_fields,
reverse_code=migrations.RunPython.noop,
elidable=True,
),
]
39 changes: 37 additions & 2 deletions pulp_container/app/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,11 @@ class Manifest(Content):
labels (models.JSONField): Metadata stored inside the image configuration.
is_bootable (models.BooleanField): Indicates whether the image is bootable or not.
is_flatpak (models.BooleanField): Indicates whether the image is a flatpak package or not.
architecture (models.TextField): CPU architecture for which the binaries in the image are
designed to run.
os (models.TextField): Operating System which the image is built to run on.
compressed_layers_size (models.IntegerField): Sum of the sizes, in bytes, of all compressed
layers.

Relations:
blobs (models.ManyToManyField): Many-to-many relationship with Blob.
Expand All @@ -103,6 +108,9 @@ class Manifest(Content):

annotations = models.JSONField(default=dict)
labels = models.JSONField(default=dict)
architecture = models.TextField(null=True)
os = models.TextField(null=True)
compressed_layers_size = models.IntegerField(null=True)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how about compressed_image_size


is_bootable = models.BooleanField(default=False)
is_flatpak = models.BooleanField(default=False)
Expand Down Expand Up @@ -176,6 +184,32 @@ def init_manifest_nature(self):
else:
return False

def init_architecture_and_os(self, manifest_data):
# schema1 has the architecture/os definition in the Manifest (not in the ConfigBlob)
if manifest_data.get("architecture", None) or manifest_data.get("os", None):
self.architecture = manifest_data.get("architecture", None)
self.os = manifest_data.get("os", None)
return

manifest_config = manifest_data.get("config", None)
config_blob_sha256 = manifest_config.get("digest", None)
blob_artifact = Artifact.objects.get(sha256=config_blob_sha256.removeprefix("sha256:"))
config_blob, _ = get_content_data(blob_artifact)
self.architecture = config_blob.get("architecture", None)
self.os = config_blob.get("os", None)

def init_compressed_layers_size(self, manifest_data):
# manifestv2 schema1 has only blobSum definition for each layer
if manifest_data.get("fsLayers", None):
self.compressed_layers_size = 0
return

layers = manifest_data.get("layers")
compressed_size = 0
for layer in layers:
compressed_size += layer.get("size")
self.compressed_layers_size = compressed_size

def is_bootable_image(self):
if (
self.annotations.get("containers.bootc") == "1"
Expand Down Expand Up @@ -222,8 +256,9 @@ class ManifestListManifest(models.Model):
manifest_list (models.ForeignKey): Many-to-one relationship with ManifestList.
"""

architecture = models.TextField()
os = models.TextField()
# in oci-index spec, platform is an optional field
architecture = models.TextField(default="", blank=True)
os = models.TextField(default="", blank=True)
os_version = models.TextField(default="", blank=True)
os_features = models.TextField(default="", blank=True)
features = models.TextField(default="", blank=True)
Expand Down
2 changes: 2 additions & 0 deletions pulp_container/app/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -454,10 +454,12 @@ async def init_pending_content(self, digest, manifest_data, media_type, raw_text
config_blob=config_blob,
data=raw_text_data,
)
await sync_to_async(manifest.init_architecture_and_os)(manifest_data)

# skip if media_type of schema1
if media_type in (MEDIA_TYPE.MANIFEST_V2, MEDIA_TYPE.MANIFEST_OCI):
await sync_to_async(manifest.init_metadata)(manifest_data=manifest_data)
await sync_to_async(manifest.init_compressed_layers_size)(manifest_data=manifest_data)

try:
await manifest.asave()
Expand Down
22 changes: 13 additions & 9 deletions pulp_container/app/registry_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -1203,7 +1203,7 @@ def put(self, request, path, pk=None):
if is_manifest_list:
manifests = {}
for manifest in content_data.get("manifests"):
manifests[manifest["digest"]] = manifest["platform"]
manifests[manifest["digest"]] = manifest.get("platform", None)

digests = set(manifests.keys())

Expand All @@ -1217,17 +1217,17 @@ def put(self, request, path, pk=None):

manifests_to_list = []
for manifest in found_manifests:
platform = manifests[manifest.digest]
manifest_to_list = models.ManifestListManifest(
manifest_list=manifest,
image_manifest=manifest_list,
architecture=platform["architecture"],
os=platform["os"],
features=platform.get("features", ""),
variant=platform.get("variant", ""),
os_version=platform.get("os.version", ""),
os_features=platform.get("os.features", ""),
)
if platform := manifests[manifest.digest]:
manifest_to_list.architecture = (platform["architecture"],)
manifest_to_list.os = (platform["os"],)
manifest_to_list.features = (platform.get("features", ""),)
manifest_to_list.variant = (platform.get("variant", ""),)
manifest_to_list.os_version = (platform.get("os.version", ""),)
manifest_to_list.os_features = (platform.get("os.features", ""),)
manifests_to_list.append(manifest_to_list)

models.ManifestListManifest.objects.bulk_create(
Expand Down Expand Up @@ -1295,6 +1295,7 @@ def put(self, request, path, pk=None):
config_blob = found_config_blobs.first()
manifest = self._init_manifest(manifest_digest, media_type, raw_text_data, config_blob)
manifest.init_metadata(manifest_data=content_data)
manifest.init_compressed_layers_size(manifest_data=content_data)

manifest = self._save_manifest(manifest)

Expand Down Expand Up @@ -1351,13 +1352,16 @@ def put(self, request, path, pk=None):
return ManifestResponse(manifest, path, request, status=201)

def _init_manifest(self, manifest_digest, media_type, raw_text_data, config_blob=None):
return models.Manifest(
manifest = models.Manifest(
digest=manifest_digest,
schema_version=2,
media_type=media_type,
config_blob=config_blob,
data=raw_text_data,
)
if config_blob:
manifest.init_architecture_and_os(json.loads(raw_text_data))
return manifest

def _save_manifest(self, manifest):
try:
Expand Down
18 changes: 18 additions & 0 deletions pulp_container/app/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,21 @@ class ManifestSerializer(NoArtifactContentSerializer):
default=False,
help_text=_("A boolean determining whether the image bundles a Flatpak application"),
)
architecture = serializers.CharField(
help_text="The CPU architecture which the binaries in this image are built to run on.",
required=False,
default=None,
)
os = serializers.CharField(
help_text="The name of the operating system which the image is built to run on.",
required=False,
default=None,
)
compressed_layers_size = serializers.IntegerField(
help_text="Specifies the sum of the sizes, in bytes, of all compressed layers",
required=False,
default=None,
)

class Meta:
fields = NoArtifactContentSerializer.Meta.fields + (
Expand All @@ -116,6 +131,9 @@ class Meta:
"labels",
"is_bootable",
"is_flatpak",
"architecture",
"os",
"compressed_layers_size",
)
model = models.Manifest

Expand Down
6 changes: 5 additions & 1 deletion pulp_container/app/tasks/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,15 @@ def add_image_from_directory_to_repository(path, repository, tag):

config_blob = get_or_create_blob(manifest_json["config"], manifest, path)
manifest.config_blob = config_blob
manifest.save()
manifest.init_architecture_and_os(manifest_json)

pks_to_add = []
compressed_size = 0
for layer in manifest_json["layers"]:
compressed_size += layer.get("size")
pks_to_add.append(get_or_create_blob(layer, manifest, path).pk)
manifest.compressed_layers_size = compressed_size
manifest.save()

pks_to_add.extend([manifest.pk, tag.pk, config_blob.pk])
new_repo_version.add_content(Content.objects.filter(pk__in=pks_to_add))
Expand Down
Loading