From 0faf9eb84e9b66d9bc7fcdfcfde6fe225d1ea0f1 Mon Sep 17 00:00:00 2001 From: Martin Basti Date: Thu, 7 Nov 2024 19:28:10 +0100 Subject: [PATCH] cachi2: update koji_import plugin Update koji_import to process cachi2 results. Signed-off-by: Martin Basti --- atomic_reactor/plugins/koji_import.py | 118 ++++++++++++++++++-------- tests/plugins/test_koji_import.py | 62 ++++++++++++-- 2 files changed, 139 insertions(+), 41 deletions(-) diff --git a/atomic_reactor/plugins/koji_import.py b/atomic_reactor/plugins/koji_import.py index 9dd190d95..f84d1cdce 100644 --- a/atomic_reactor/plugins/koji_import.py +++ b/atomic_reactor/plugins/koji_import.py @@ -36,6 +36,7 @@ from atomic_reactor.plugins.fetch_sources import PLUGIN_FETCH_SOURCES_KEY from atomic_reactor.constants import ( + PLUGIN_CACHI2_POSTPROCESS, PLUGIN_EXPORT_OPERATOR_MANIFESTS_KEY, PLUGIN_KOJI_IMPORT_PLUGIN_KEY, PLUGIN_KOJI_IMPORT_SOURCE_CONTAINER_PLUGIN_KEY, @@ -303,41 +304,76 @@ def set_pnc_build_metadata(self, extra): if pnc_build_metadata: extra['image']['pnc'] = pnc_build_metadata - def set_remote_sources_metadata(self, extra): - remote_source_result = self.workflow.data.plugins_results.get( - PLUGIN_RESOLVE_REMOTE_SOURCE - ) - if remote_source_result: - if self.workflow.conf.allow_multiple_remote_sources: - remote_sources_image_metadata = [ - {"name": remote_source["name"], "url": remote_source["url"].rstrip('/download')} - for remote_source in remote_source_result - ] - extra["image"]["remote_sources"] = remote_sources_image_metadata - - remote_sources_typeinfo_metadata = [ - { - "name": remote_source["name"], - "url": remote_source["url"].rstrip('/download'), - "archives": [ - remote_source["remote_source_json"]["filename"], - remote_source["remote_source_tarball"]["filename"], - remote_source["remote_source_json_env"]["filename"], - remote_source["remote_source_json_config"]["filename"], - ], - } - for remote_source in remote_source_result - ] - else: - extra["image"]["remote_source_url"] = remote_source_result[0]["url"] - remote_sources_typeinfo_metadata = { - "remote_source_url": remote_source_result[0]["url"] - } + def set_remote_sources_metadata_cachito(self, remote_source_result, extra): + if self.workflow.conf.allow_multiple_remote_sources: + remote_sources_image_metadata = [ + {"name": remote_source["name"], "url": remote_source["url"].rstrip('/download')} + for remote_source in remote_source_result + ] + extra["image"]["remote_sources"] = remote_sources_image_metadata - remote_source_typeinfo = { - KOJI_BTYPE_REMOTE_SOURCES: remote_sources_typeinfo_metadata, + remote_sources_typeinfo_metadata = [ + { + "name": remote_source["name"], + "url": remote_source["url"].rstrip('/download'), + "archives": [ + remote_source["remote_source_json"]["filename"], + remote_source["remote_source_tarball"]["filename"], + remote_source["remote_source_json_env"]["filename"], + remote_source["remote_source_json_config"]["filename"], + ], + } + for remote_source in remote_source_result + ] + else: + extra["image"]["remote_source_url"] = remote_source_result[0]["url"] + remote_sources_typeinfo_metadata = { + "remote_source_url": remote_source_result[0]["url"] } - extra.setdefault("typeinfo", {}).update(remote_source_typeinfo) + + remote_source_typeinfo = { + KOJI_BTYPE_REMOTE_SOURCES: remote_sources_typeinfo_metadata, + } + extra.setdefault("typeinfo", {}).update(remote_source_typeinfo) + + def set_remote_sources_metadata_cachi2(self, remote_source_result, extra): + remote_sources_typeinfo_metadata = [] + if self.workflow.conf.allow_multiple_remote_sources: + remote_sources_image_metadata = [ + {"name": remote_source["name"]} + for remote_source in remote_source_result + ] + extra["image"]["remote_sources"] = remote_sources_image_metadata + + remote_sources_typeinfo_metadata = [ + { + "name": remote_source["name"], + "archives": [ + remote_source["remote_source_json"]["filename"], + remote_source["remote_source_tarball"]["filename"], + remote_source["remote_source_json_env"]["filename"], + ], + } + for remote_source in remote_source_result + ] + + remote_source_typeinfo = { + KOJI_BTYPE_REMOTE_SOURCES: remote_sources_typeinfo_metadata, + } + extra.setdefault("typeinfo", {}).update(remote_source_typeinfo) + + def set_remote_sources_metadata(self, extra): + func_map = { + PLUGIN_RESOLVE_REMOTE_SOURCE: self.set_remote_sources_metadata_cachito, + PLUGIN_CACHI2_POSTPROCESS: self.set_remote_sources_metadata_cachi2, + } + for plugin_name, func in func_map.items(): + remote_source_result = self.workflow.data.plugins_results.get( + plugin_name + ) + if remote_source_result: + func(remote_source_result, extra) + break def set_remote_source_file_metadata(self, extra): maven_url_sources_metadata_results = self.workflow.data.plugins_results.get( @@ -613,9 +649,21 @@ def _filesystem_koji_task_id(self) -> Optional[int]: def _collect_remote_sources(self) -> Iterable[ArtifactOutputInfo]: wf_data = self.workflow.data + + remote_source_keys = [ + "remote_source_json", "remote_source_json_env", "remote_source_json_config", + ] + # a list of metadata describing the remote sources. plugin_results: List[Dict[str, Any]] plugin_results = wf_data.plugins_results.get(PLUGIN_RESOLVE_REMOTE_SOURCE) or [] + if not plugin_results: + # Cachi2 + plugin_results = wf_data.plugins_results.get(PLUGIN_CACHI2_POSTPROCESS) or [] + remote_source_keys = [ + "remote_source_json", "remote_source_json_env", + ] + tmpdir = tempfile.mkdtemp() for remote_source in plugin_results: @@ -624,9 +672,7 @@ def _collect_remote_sources(self) -> Iterable[ArtifactOutputInfo]: dest_filename = remote_source_tarball['filename'] yield local_filename, dest_filename, KOJI_BTYPE_REMOTE_SOURCES, None - for source_key in ( - "remote_source_json", "remote_source_json_env", "remote_source_json_config", - ): + for source_key in remote_source_keys: data_json = remote_source[source_key] data_json_filename = data_json['filename'] file_path = os.path.join(tmpdir, data_json_filename) diff --git a/tests/plugins/test_koji_import.py b/tests/plugins/test_koji_import.py index c5cb5f00c..d44a61202 100644 --- a/tests/plugins/test_koji_import.py +++ b/tests/plugins/test_koji_import.py @@ -7,6 +7,7 @@ """ from collections import namedtuple +from enum import Enum import json from pathlib import Path from typing import Any, Dict @@ -37,6 +38,7 @@ from atomic_reactor.source import GitSource, PathSource from atomic_reactor.constants import (IMAGE_TYPE_DOCKER_ARCHIVE, KOJI_BTYPE_OPERATOR_MANIFESTS, PLUGIN_ADD_FILESYSTEM_KEY, + PLUGIN_CACHI2_POSTPROCESS, PLUGIN_EXPORT_OPERATOR_MANIFESTS_KEY, PLUGIN_MAVEN_URL_SOURCES_METADATA_KEY, PLUGIN_GROUP_MANIFESTS_KEY, PLUGIN_KOJI_PARENT_KEY, @@ -185,6 +187,12 @@ def taskFinished(self, task_id): REGISTRY = 'docker.example.com' +class RemoteSourceKind(Enum): + NONE = 1 + CACHITO = 2 + CACHI2 = 3 + + def fake_subprocess_output(cmd): if cmd.startswith('/bin/rpm'): return FAKE_RPM_OUTPUT @@ -268,7 +276,8 @@ def mock_environment(workflow: DockerBuildWorkflow, source_dir: Path, has_op_appregistry_manifests=False, has_op_bundle_manifests=False, push_operator_manifests_enabled=False, source_build=False, - has_remote_source=False, has_remote_source_file=False, + has_remote_source: RemoteSourceKind = RemoteSourceKind.NONE, + has_remote_source_file=False, has_pnc_build_metadata=False, scratch=False): if session is None: session = MockedClientSession('') @@ -524,7 +533,7 @@ def custom_get(method, url, headers, **kwargs): results = workflow.data.plugins_results results[PLUGIN_EXPORT_OPERATOR_MANIFESTS_KEY] = str(archive_file) - if has_remote_source: + if has_remote_source == RemoteSourceKind.CACHITO: source_path = build_dir_path / REMOTE_SOURCE_TARBALL_FILENAME source_path.write_text('dummy file', 'utf-8') remote_source_result = [ @@ -550,6 +559,28 @@ def custom_get(method, url, headers, **kwargs): } ] workflow.data.plugins_results[PLUGIN_RESOLVE_REMOTE_SOURCE] = remote_source_result + elif has_remote_source == RemoteSourceKind.CACHI2: + source_path = build_dir_path / REMOTE_SOURCE_TARBALL_FILENAME + source_path.write_text('dummy file', 'utf-8') + remote_source_result = [ + { + "name": None, + "url": "https://cachito.com/api/v1/requests/21048/download", + "remote_source_json": { + "filename": REMOTE_SOURCE_JSON_FILENAME, + "json": {"stub": "data"}, + }, + "remote_source_json_env": { + "filename": REMOTE_SOURCE_JSON_ENV_FILENAME, + "json": {"var": {"stub": "data"}}, + }, + "remote_source_tarball": { + "filename": REMOTE_SOURCE_TARBALL_FILENAME, + "path": str(source_path), + }, + } + ] + workflow.data.plugins_results[PLUGIN_CACHI2_POSTPROCESS] = remote_source_result else: workflow.data.plugins_results[PLUGIN_RESOLVE_REMOTE_SOURCE] = None @@ -1977,7 +2008,7 @@ def test_operators_bundle_metadata_csv_modifications( assert extra['image']['operator_manifests'] == expected - @pytest.mark.parametrize('has_remote_source', [True, False]) + @pytest.mark.parametrize('has_remote_source', list(RemoteSourceKind)) @pytest.mark.parametrize('allow_multiple_remote_sources', [True, False]) def test_remote_sources(self, workflow, source_dir, has_remote_source, allow_multiple_remote_sources): @@ -2000,7 +2031,7 @@ def test_remote_sources(self, workflow, source_dir, assert isinstance(extra, dict) # https://github.com/PyCQA/pylint/issues/2186 # pylint: disable=W1655 - if has_remote_source: + if has_remote_source == RemoteSourceKind.CACHITO: if allow_multiple_remote_sources: assert extra['image']['remote_sources'] == [ { @@ -2031,7 +2062,28 @@ def test_remote_sources(self, workflow, source_dir, } assert REMOTE_SOURCE_TARBALL_FILENAME in session.uploaded_files.keys() assert REMOTE_SOURCE_JSON_FILENAME in session.uploaded_files.keys() - + elif has_remote_source == RemoteSourceKind.CACHI2: + if allow_multiple_remote_sources: + assert extra['image']['remote_sources'] == [ + { + 'name': None, + } + ] + assert 'remote-sources' in extra['typeinfo'] + assert extra['typeinfo']['remote-sources'] == [ + { + 'name': None, + 'archives': [ + 'remote-source.json', 'remote-source.tar.gz', + 'remote-source.env.json' + ], + } + ] + assert REMOTE_SOURCE_TARBALL_FILENAME in session.uploaded_files.keys() + assert REMOTE_SOURCE_JSON_FILENAME in session.uploaded_files.keys() + else: + assert REMOTE_SOURCE_TARBALL_FILENAME in session.uploaded_files.keys() + assert REMOTE_SOURCE_JSON_FILENAME in session.uploaded_files.keys() else: assert 'remote_source_url' not in extra['image'] assert REMOTE_SOURCE_TARBALL_FILENAME not in session.uploaded_files.keys()