Skip to content

Commit

Permalink
cachi2: update koji_import plugin
Browse files Browse the repository at this point in the history 
Update koji_import to process cachi2 results.

Signed-off-by: Martin Basti <[email protected]>
  • Loading branch information
@MartinBasti
MartinBasti committed Nov 12, 2024
1 parent 5f5e2ff commit 0faf9eb
Show file tree
Hide file tree
Showing 2 changed files with 139 additions and 41 deletions.
118 changes: 82 additions & 36 deletions atomic_reactor/plugins/koji_import.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
from atomic_reactor.plugins.fetch_sources import PLUGIN_FETCH_SOURCES_KEY

from atomic_reactor.constants import (
PLUGIN_CACHI2_POSTPROCESS,
PLUGIN_EXPORT_OPERATOR_MANIFESTS_KEY,
PLUGIN_KOJI_IMPORT_PLUGIN_KEY,
PLUGIN_KOJI_IMPORT_SOURCE_CONTAINER_PLUGIN_KEY,
Expand Down Expand Up @@ -303,41 +304,76 @@ def set_pnc_build_metadata(self, extra):
if pnc_build_metadata:
extra['image']['pnc'] = pnc_build_metadata

def set_remote_sources_metadata(self, extra):
remote_source_result = self.workflow.data.plugins_results.get(
PLUGIN_RESOLVE_REMOTE_SOURCE
)
if remote_source_result:
if self.workflow.conf.allow_multiple_remote_sources:
remote_sources_image_metadata = [
{"name": remote_source["name"], "url": remote_source["url"].rstrip('/download')}
for remote_source in remote_source_result
]
extra["image"]["remote_sources"] = remote_sources_image_metadata

remote_sources_typeinfo_metadata = [
{
"name": remote_source["name"],
"url": remote_source["url"].rstrip('/download'),
"archives": [
remote_source["remote_source_json"]["filename"],
remote_source["remote_source_tarball"]["filename"],
remote_source["remote_source_json_env"]["filename"],
remote_source["remote_source_json_config"]["filename"],
],
}
for remote_source in remote_source_result
]
else:
extra["image"]["remote_source_url"] = remote_source_result[0]["url"]
remote_sources_typeinfo_metadata = {
"remote_source_url": remote_source_result[0]["url"]
}
def set_remote_sources_metadata_cachito(self, remote_source_result, extra):
if self.workflow.conf.allow_multiple_remote_sources:
remote_sources_image_metadata = [
{"name": remote_source["name"], "url": remote_source["url"].rstrip('/download')}
for remote_source in remote_source_result
]
extra["image"]["remote_sources"] = remote_sources_image_metadata

remote_source_typeinfo = {
KOJI_BTYPE_REMOTE_SOURCES: remote_sources_typeinfo_metadata,
remote_sources_typeinfo_metadata = [
{
"name": remote_source["name"],
"url": remote_source["url"].rstrip('/download'),
"archives": [
remote_source["remote_source_json"]["filename"],
remote_source["remote_source_tarball"]["filename"],
remote_source["remote_source_json_env"]["filename"],
remote_source["remote_source_json_config"]["filename"],
],
}
for remote_source in remote_source_result
]
else:
extra["image"]["remote_source_url"] = remote_source_result[0]["url"]
remote_sources_typeinfo_metadata = {
"remote_source_url": remote_source_result[0]["url"]
}
extra.setdefault("typeinfo", {}).update(remote_source_typeinfo)

remote_source_typeinfo = {
KOJI_BTYPE_REMOTE_SOURCES: remote_sources_typeinfo_metadata,
}
extra.setdefault("typeinfo", {}).update(remote_source_typeinfo)

def set_remote_sources_metadata_cachi2(self, remote_source_result, extra):
remote_sources_typeinfo_metadata = []
if self.workflow.conf.allow_multiple_remote_sources:
remote_sources_image_metadata = [
{"name": remote_source["name"]}
for remote_source in remote_source_result
]
extra["image"]["remote_sources"] = remote_sources_image_metadata

remote_sources_typeinfo_metadata = [
{
"name": remote_source["name"],
"archives": [
remote_source["remote_source_json"]["filename"],
remote_source["remote_source_tarball"]["filename"],
remote_source["remote_source_json_env"]["filename"],
],
}
for remote_source in remote_source_result
]

remote_source_typeinfo = {
KOJI_BTYPE_REMOTE_SOURCES: remote_sources_typeinfo_metadata,
}
extra.setdefault("typeinfo", {}).update(remote_source_typeinfo)

def set_remote_sources_metadata(self, extra):
func_map = {
PLUGIN_RESOLVE_REMOTE_SOURCE: self.set_remote_sources_metadata_cachito,
PLUGIN_CACHI2_POSTPROCESS: self.set_remote_sources_metadata_cachi2,
}
for plugin_name, func in func_map.items():
remote_source_result = self.workflow.data.plugins_results.get(
plugin_name
)
if remote_source_result:
func(remote_source_result, extra)
break

def set_remote_source_file_metadata(self, extra):
maven_url_sources_metadata_results = self.workflow.data.plugins_results.get(
Expand Down Expand Up @@ -613,9 +649,21 @@ def _filesystem_koji_task_id(self) -> Optional[int]:

def _collect_remote_sources(self) -> Iterable[ArtifactOutputInfo]:
wf_data = self.workflow.data

remote_source_keys = [
"remote_source_json", "remote_source_json_env", "remote_source_json_config",
]

# a list of metadata describing the remote sources.
plugin_results: List[Dict[str, Any]]
plugin_results = wf_data.plugins_results.get(PLUGIN_RESOLVE_REMOTE_SOURCE) or []
if not plugin_results:
# Cachi2
plugin_results = wf_data.plugins_results.get(PLUGIN_CACHI2_POSTPROCESS) or []
remote_source_keys = [
"remote_source_json", "remote_source_json_env",
]

tmpdir = tempfile.mkdtemp()

for remote_source in plugin_results:
Expand All @@ -624,9 +672,7 @@ def _collect_remote_sources(self) -> Iterable[ArtifactOutputInfo]:
dest_filename = remote_source_tarball['filename']
yield local_filename, dest_filename, KOJI_BTYPE_REMOTE_SOURCES, None

for source_key in (
"remote_source_json", "remote_source_json_env", "remote_source_json_config",
):
for source_key in remote_source_keys:
data_json = remote_source[source_key]
data_json_filename = data_json['filename']
file_path = os.path.join(tmpdir, data_json_filename)
Expand Down
62 changes: 57 additions & 5 deletions tests/plugins/test_koji_import.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
"""

from collections import namedtuple
from enum import Enum
import json
from pathlib import Path
from typing import Any, Dict
Expand Down Expand Up @@ -37,6 +38,7 @@
from atomic_reactor.source import GitSource, PathSource
from atomic_reactor.constants import (IMAGE_TYPE_DOCKER_ARCHIVE, KOJI_BTYPE_OPERATOR_MANIFESTS,
PLUGIN_ADD_FILESYSTEM_KEY,
PLUGIN_CACHI2_POSTPROCESS,
PLUGIN_EXPORT_OPERATOR_MANIFESTS_KEY,
PLUGIN_MAVEN_URL_SOURCES_METADATA_KEY,
PLUGIN_GROUP_MANIFESTS_KEY, PLUGIN_KOJI_PARENT_KEY,
Expand Down Expand Up @@ -185,6 +187,12 @@ def taskFinished(self, task_id):
REGISTRY = 'docker.example.com'


class RemoteSourceKind(Enum):
NONE = 1
CACHITO = 2
CACHI2 = 3


def fake_subprocess_output(cmd):
if cmd.startswith('/bin/rpm'):
return FAKE_RPM_OUTPUT
Expand Down Expand Up @@ -268,7 +276,8 @@ def mock_environment(workflow: DockerBuildWorkflow, source_dir: Path,
has_op_appregistry_manifests=False,
has_op_bundle_manifests=False,
push_operator_manifests_enabled=False, source_build=False,
has_remote_source=False, has_remote_source_file=False,
has_remote_source: RemoteSourceKind = RemoteSourceKind.NONE,
has_remote_source_file=False,
has_pnc_build_metadata=False, scratch=False):
if session is None:
session = MockedClientSession('')
Expand Down Expand Up @@ -524,7 +533,7 @@ def custom_get(method, url, headers, **kwargs):
results = workflow.data.plugins_results
results[PLUGIN_EXPORT_OPERATOR_MANIFESTS_KEY] = str(archive_file)

if has_remote_source:
if has_remote_source == RemoteSourceKind.CACHITO:
source_path = build_dir_path / REMOTE_SOURCE_TARBALL_FILENAME
source_path.write_text('dummy file', 'utf-8')
remote_source_result = [
Expand All @@ -550,6 +559,28 @@ def custom_get(method, url, headers, **kwargs):
}
]
workflow.data.plugins_results[PLUGIN_RESOLVE_REMOTE_SOURCE] = remote_source_result
elif has_remote_source == RemoteSourceKind.CACHI2:
source_path = build_dir_path / REMOTE_SOURCE_TARBALL_FILENAME
source_path.write_text('dummy file', 'utf-8')
remote_source_result = [
{
"name": None,
"url": "https://cachito.com/api/v1/requests/21048/download",
"remote_source_json": {
"filename": REMOTE_SOURCE_JSON_FILENAME,
"json": {"stub": "data"},
},
"remote_source_json_env": {
"filename": REMOTE_SOURCE_JSON_ENV_FILENAME,
"json": {"var": {"stub": "data"}},
},
"remote_source_tarball": {
"filename": REMOTE_SOURCE_TARBALL_FILENAME,
"path": str(source_path),
},
}
]
workflow.data.plugins_results[PLUGIN_CACHI2_POSTPROCESS] = remote_source_result
else:
workflow.data.plugins_results[PLUGIN_RESOLVE_REMOTE_SOURCE] = None

Expand Down Expand Up @@ -1977,7 +2008,7 @@ def test_operators_bundle_metadata_csv_modifications(

assert extra['image']['operator_manifests'] == expected

@pytest.mark.parametrize('has_remote_source', [True, False])
@pytest.mark.parametrize('has_remote_source', list(RemoteSourceKind))
@pytest.mark.parametrize('allow_multiple_remote_sources', [True, False])
def test_remote_sources(self, workflow, source_dir,
has_remote_source, allow_multiple_remote_sources):
Expand All @@ -2000,7 +2031,7 @@ def test_remote_sources(self, workflow, source_dir,
assert isinstance(extra, dict)
# https://github.com/PyCQA/pylint/issues/2186
# pylint: disable=W1655
if has_remote_source:
if has_remote_source == RemoteSourceKind.CACHITO:
if allow_multiple_remote_sources:
assert extra['image']['remote_sources'] == [
{
Expand Down Expand Up @@ -2031,7 +2062,28 @@ def test_remote_sources(self, workflow, source_dir,
}
assert REMOTE_SOURCE_TARBALL_FILENAME in session.uploaded_files.keys()
assert REMOTE_SOURCE_JSON_FILENAME in session.uploaded_files.keys()

elif has_remote_source == RemoteSourceKind.CACHI2:
if allow_multiple_remote_sources:
assert extra['image']['remote_sources'] == [
{
'name': None,
}
]
assert 'remote-sources' in extra['typeinfo']
assert extra['typeinfo']['remote-sources'] == [
{
'name': None,
'archives': [
'remote-source.json', 'remote-source.tar.gz',
'remote-source.env.json'
],
}
]
assert REMOTE_SOURCE_TARBALL_FILENAME in session.uploaded_files.keys()
assert REMOTE_SOURCE_JSON_FILENAME in session.uploaded_files.keys()
else:
assert REMOTE_SOURCE_TARBALL_FILENAME in session.uploaded_files.keys()
assert REMOTE_SOURCE_JSON_FILENAME in session.uploaded_files.keys()
else:
assert 'remote_source_url' not in extra['image']
assert REMOTE_SOURCE_TARBALL_FILENAME not in session.uploaded_files.keys()
Expand Down

0 comments on commit 0faf9eb

Please sign in to comment.