From 0e87500cc2ece6bebbbbe2a313681dbcac31dd58 Mon Sep 17 00:00:00 2001 From: git-hyagi <45576767+git-hyagi@users.noreply.github.com> Date: Tue, 25 Jun 2024 16:54:03 -0300 Subject: [PATCH] Add repository_version param as a building context closes: #479 --- CHANGES/479.feature | 1 + CHANGES/479.removal | 1 + .../app/global_access_conditions.py | 18 ++- pulp_container/app/serializers.py | 36 ++---- pulp_container/app/tasks/builder.py | 36 +++--- pulp_container/app/viewsets.py | 6 +- .../tests/functional/api/test_build_images.py | 112 +++++++++++++++--- staging_docs/admin/guides/build-image.md | 16 ++- 8 files changed, 157 insertions(+), 69 deletions(-) create mode 100644 CHANGES/479.feature create mode 100644 CHANGES/479.removal diff --git a/CHANGES/479.feature b/CHANGES/479.feature new file mode 100644 index 000000000..612867f99 --- /dev/null +++ b/CHANGES/479.feature @@ -0,0 +1 @@ +Added support for using the file `repository_version` as the build context for a Containerfile. diff --git a/CHANGES/479.removal b/CHANGES/479.removal new file mode 100644 index 000000000..8f7dec12a --- /dev/null +++ b/CHANGES/479.removal @@ -0,0 +1 @@ +Removed support for using raw artifacts as the build context for a Containerfile. diff --git a/pulp_container/app/global_access_conditions.py b/pulp_container/app/global_access_conditions.py index abca8c772..9c1e7a042 100644 --- a/pulp_container/app/global_access_conditions.py +++ b/pulp_container/app/global_access_conditions.py @@ -1,10 +1,12 @@ from logging import getLogger -from pulpcore.plugin.models import Repository +from pulpcore.plugin.models import Repository, RepositoryVersion +from pulpcore.plugin.util import get_objects_for_user from pulpcore.plugin.viewsets import RepositoryVersionViewSet from pulp_container.app import models from pulp_container.app.viewsets import ContainerDistributionViewSet +from pulp_container.app import serializers _logger = getLogger(__name__) @@ -118,3 +120,17 @@ def has_distribution_perms(request, view, action, permission): return any( (request.user.has_perm(permission, distribution.cast()) for distribution in distributions) ) + + +def has_file_repo_perms(request, view, action, permission): + serializer = serializers.OCIBuildImageSerializer( + data=request.data, context={"request": request} + ) + serializer.is_valid(raise_exception=True) + repo_version = serializer.validated_data.get("repo_version", None) + if not repo_version: + return True + + repo_version_qs = RepositoryVersion.objects.filter(pk=repo_version) + file_repositories = get_objects_for_user(request.user, permission, repo_version_qs) + return repo_version_qs == file_repositories diff --git a/pulp_container/app/serializers.py b/pulp_container/app/serializers.py index 3cac522bd..2aa0baf27 100644 --- a/pulp_container/app/serializers.py +++ b/pulp_container/app/serializers.py @@ -1,5 +1,4 @@ from gettext import gettext as _ -import os import re from django.core.validators import URLValidator @@ -750,11 +749,10 @@ class OCIBuildImageSerializer(ValidateFieldsMixin, serializers.Serializer): tag = serializers.CharField( required=False, default="latest", help_text="A tag name for the new image being built." ) - artifacts = serializers.JSONField( + repo_version = RepositoryVersionRelatedField( required=False, - help_text="A JSON string where each key is an artifact href and the value is it's " - "relative path (name) inside the /pulp_working_directory of the build container " - "executing the Containerfile.", + help_text=_("RepositoryVersion to be used as the build context for container images."), + allow_null=True, ) def __init__(self, *args, **kwargs): @@ -778,28 +776,10 @@ def validate(self, data): raise serializers.ValidationError( _("'containerfile' or 'containerfile_artifact' must " "be specified.") ) - artifacts = {} - if "artifacts" in data: - for url, relative_path in data["artifacts"].items(): - if os.path.isabs(relative_path): - raise serializers.ValidationError( - _("Relative path cannot start with '/'. " "{0}").format(relative_path) - ) - artifactfield = RelatedField( - view_name="artifacts-detail", - queryset=Artifact.objects.all(), - source="*", - initial=url, - ) - try: - artifact = artifactfield.run_validation(data=url) - artifact.touch() - artifacts[str(artifact.pk)] = relative_path - except serializers.ValidationError as e: - # Append the URL of missing Artifact to the error message - e.detail[0] = "%s %s" % (e.detail[0], url) - raise e - data["artifacts"] = artifacts + + if "repo_version" in data: + data["repo_version"] = data["repo_version"].pk + return data class Meta: @@ -808,7 +788,7 @@ class Meta: "containerfile", "repository", "tag", - "artifacts", + "repo_version", ) diff --git a/pulp_container/app/tasks/builder.py b/pulp_container/app/tasks/builder.py index 58daa3796..25b286c9c 100644 --- a/pulp_container/app/tasks/builder.py +++ b/pulp_container/app/tasks/builder.py @@ -14,7 +14,7 @@ ) from pulp_container.constants import MEDIA_TYPE from pulp_container.app.utils import calculate_digest -from pulpcore.plugin.models import Artifact, ContentArtifact, Content +from pulpcore.plugin.models import Artifact, ContentArtifact, Content, RepositoryVersion def get_or_create_blob(layer_json, manifest, path): @@ -96,7 +96,7 @@ def add_image_from_directory_to_repository(path, repository, tag): def build_image_from_containerfile( - containerfile_pk=None, artifacts=None, repository_pk=None, tag=None + containerfile_pk=None, repository_pk=None, tag=None, repo_version=None ): """ Builds an OCI container image from a Containerfile. @@ -106,11 +106,10 @@ def build_image_from_containerfile( Args: containerfile_pk (str): The pk of an Artifact that contains the Containerfile - artifacts (dict): A dictionary where each key is an artifact PK and the value is it's - relative path (name) inside the /pulp_working_directory of the build - container executing the Containerfile. repository_pk (str): The pk of a Repository to add the OCI container image tag (str): Tag name for the new image in the repository + repo_version: The pk of a RepositoryVersion with the artifacts used in the build context + of the Containerfile. Returns: A class:`pulpcore.plugin.models.RepositoryVersion` that contains the new OCI container @@ -124,16 +123,16 @@ def build_image_from_containerfile( working_directory = os.path.abspath(working_directory) context_path = os.path.join(working_directory, "context") os.makedirs(context_path, exist_ok=True) - for key, val in artifacts.items(): - artifact = Artifact.objects.get(pk=key) - dest_path = os.path.join(context_path, val) - dirs = os.path.split(dest_path)[0] - if dirs: - os.makedirs(dirs, exist_ok=True) - with open(dest_path, "wb") as dest: - shutil.copyfileobj(artifact.file, dest) - containerfile_path = os.path.join(working_directory, "Containerfile") + if repo_version: + file_repository = RepositoryVersion.objects.get(pk=repo_version) + content_artifacts = ContentArtifact.objects.filter( + content__in=file_repository.content + ).order_by("-content__pulp_created") + for content in content_artifacts.select_related("artifact").iterator(): + _copy_file_from_artifact(context_path, content.relative_path, content.artifact.file) + + containerfile_path = os.path.join(working_directory, "Containerfile") with open(containerfile_path, "wb") as dest: shutil.copyfileobj(containerfile.file, dest) @@ -166,3 +165,12 @@ def build_image_from_containerfile( repository_version = add_image_from_directory_to_repository(image_dir, repository, tag) return repository_version + + +def _copy_file_from_artifact(context_path, relative_path, artifact): + dest_path = os.path.join(context_path, relative_path) + dirs = os.path.split(dest_path)[0] + if dirs: + os.makedirs(dirs, exist_ok=True) + with open(dest_path, "wb") as dest: + shutil.copyfileobj(artifact.file, dest) diff --git a/pulp_container/app/viewsets.py b/pulp_container/app/viewsets.py index e42be79f9..f9b87de7d 100644 --- a/pulp_container/app/viewsets.py +++ b/pulp_container/app/viewsets.py @@ -696,6 +696,7 @@ class ContainerRepositoryViewSet( "condition": [ "has_model_or_obj_perms:container.build_image_containerrepository", "has_model_or_obj_perms:container.view_containerrepository", + "has_file_repo_perms:file.view_filerepository", ], }, { @@ -946,8 +947,7 @@ def build_image(self, request, pk): containerfile.touch() tag = serializer.validated_data["tag"] - artifacts = serializer.validated_data["artifacts"] - Artifact.objects.filter(pk__in=artifacts.keys()).touch() + repo_version = serializer.validated_data.get("repo_version", None) result = dispatch( tasks.build_image_from_containerfile, @@ -956,7 +956,7 @@ def build_image(self, request, pk): "containerfile_pk": str(containerfile.pk), "tag": tag, "repository_pk": str(repository.pk), - "artifacts": artifacts, + "repo_version": repo_version, }, ) return OperationPostponedResponse(result, request) diff --git a/pulp_container/tests/functional/api/test_build_images.py b/pulp_container/tests/functional/api/test_build_images.py index 5569dfa31..2892b56b7 100644 --- a/pulp_container/tests/functional/api/test_build_images.py +++ b/pulp_container/tests/functional/api/test_build_images.py @@ -9,6 +9,7 @@ from pulp_smash.pulp3.bindings import monitor_task from pulpcore.client.pulp_container import ( + ApiException, ContainerContainerDistribution, ContainerContainerRepository, ) @@ -19,7 +20,7 @@ def containerfile_name(): """A fixture for a basic container file used for building images.""" with NamedTemporaryFile() as containerfile: containerfile.write( - b"""FROM busybox:latest + b"""FROM quay.io/quay/busybox:latest # Copy a file using COPY statement. Use the relative path specified in the 'artifacts' parameter. COPY foo/bar/example.txt /tmp/inside-image.txt # Print the content of the file when the container starts @@ -29,35 +30,110 @@ def containerfile_name(): yield containerfile.name -def test_build_image( - pulpcore_bindings, +@pytest.fixture +def create_file_and_container_repos_with_sample_data( container_repository_api, - container_distribution_api, + file_bindings, + file_repository_factory, gen_object_with_cleanup, - containerfile_name, - local_registry, + tmp_path_factory, ): - """Test if a user can build an OCI image.""" - with NamedTemporaryFile() as text_file: - text_file.write(b"some text") - text_file.flush() - artifact = gen_object_with_cleanup(pulpcore_bindings.ArtifactsApi, text_file.name) - - repository = gen_object_with_cleanup( + container_repo = gen_object_with_cleanup( container_repository_api, ContainerContainerRepository(**gen_repo()) ) - artifacts = '{{"{}": "foo/bar/example.txt"}}'.format(artifact.pulp_href) - build_response = container_repository_api.build_image( - repository.pulp_href, containerfile=containerfile_name, artifacts=artifacts + filename = tmp_path_factory.mktemp("fixtures") / "example.txt" + filename.write_bytes(b"test content") + file_repo = file_repository_factory(autopublish=True) + upload_task = file_bindings.ContentFilesApi.create( + relative_path="foo/bar/example.txt", file=filename, repository=file_repo.pulp_href + ).task + monitor_task(upload_task) + + return container_repo, file_repo + + +@pytest.fixture +def build_image(container_repository_api): + def _build_image(repository, containerfile, repo_version=None): + return container_repository_api.build_image( + container_container_repository_href=repository, + containerfile=containerfile, + repo_version=repo_version, + ) + + return _build_image + + +def test_build_image( + build_image, + containerfile_name, + container_distribution_api, + create_file_and_container_repos_with_sample_data, + delete_orphans_pre, + gen_object_with_cleanup, + local_registry, +): + """Test build an OCI image from a file repository_version.""" + container_repo, file_repo = create_file_and_container_repos_with_sample_data + build_image( + container_repo.pulp_href, + containerfile_name, + repo_version=f"{file_repo.pulp_href}versions/1/", ) - monitor_task(build_response.task) distribution = gen_object_with_cleanup( container_distribution_api, - ContainerContainerDistribution(**gen_distribution(repository=repository.pulp_href)), + ContainerContainerDistribution(**gen_distribution(repository=container_repo.pulp_href)), ) local_registry.pull(distribution.base_path) image = local_registry.inspect(distribution.base_path) assert image[0]["Config"]["Cmd"] == ["cat", "/tmp/inside-image.txt"] + + +def test_build_image_from_repo_version_with_anon_user( + build_image, + containerfile_name, + create_file_and_container_repos_with_sample_data, + delete_orphans_pre, + gen_user, +): + """Test if a user without permission to file repo can build an OCI image.""" + user_helpless = gen_user( + model_roles=[ + "container.containerdistribution_collaborator", + "container.containerrepository_content_manager", + ] + ) + container_repo, file_repo = create_file_and_container_repos_with_sample_data + with user_helpless, pytest.raises(ApiException): + build_image( + container_repo.pulp_href, + containerfile_name, + repo_version=f"{file_repo.pulp_href}versions/1/", + ) + + +def test_build_image_from_repo_version_with_creator_user( + build_image, + containerfile_name, + create_file_and_container_repos_with_sample_data, + delete_orphans_pre, + gen_user, +): + """Test if a user (with the expected permissions) can build an OCI image.""" + user = gen_user( + model_roles=[ + "container.containerdistribution_collaborator", + "container.containerrepository_content_manager", + "file.filerepository_viewer", + ] + ) + container_repo, file_repo = create_file_and_container_repos_with_sample_data + with user: + build_image( + container_repo.pulp_href, + containerfile_name, + repo_version=f"{file_repo.pulp_href}versions/1/", + ) diff --git a/staging_docs/admin/guides/build-image.md b/staging_docs/admin/guides/build-image.md index b71f69e9c..2b0bb597f 100644 --- a/staging_docs/admin/guides/build-image.md +++ b/staging_docs/admin/guides/build-image.md @@ -41,12 +41,18 @@ CMD ["cat", "/inside-image.txt"]' >> Containerfile ## Build an OCI image ```bash -TASK_HREF=$(http --form POST :$REPO_HREF'build_image/' containerfile@./Containerfile \ -artifacts="{\"$ARTIFACT_HREF\": \"foo/bar/example.txt\"}" | jq -r '.task') +ARTIFACT_SHA256=$(http :$ARTIFACT_HREF | jq -r '.sha256') +pulp file repository create --name bar + +REPO_VERSION=$(pulp file content create --relative-path foo/bar/example.txt \ +--sha256 $ARTIFACT_SHA256 --repository bar | jq .pulp_href) + +TASK_HREF=$(http --form POST :$REPO_HREF'build_image/' "containerfile@./Containerfile" \ +repo_version=$REPO_VERSION | jq -r '.task') ``` + !!! warning - Non-staff users, lacking read access to the `artifacts` endpoint, may encounter restricted - functionality as they are prohibited from listing artifacts uploaded to Pulp and utilizing - them within the build process. + File repositories synced with on-demand remotes will not automatically pull the missing artifacts. + Trying to build using a file that is not yet pulled will fail.