diff --git a/CHANGELOG.md b/CHANGELOG.md index 20be121..74d3a54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -n/a +- Accepts content set labels and repo IDs for content calculation ## [0.1.19] - 2019-06-25 diff --git a/tests/test_cli.py b/tests/test_cli.py index 5bec3ee..404a324 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -58,7 +58,8 @@ def test_default_config_source(mock_ubipopulate): args = ['--pulp-hostname', 'foo.pulp.com', '--user', 'foo', '--pass', 'foo'] main(args) mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, [], None, - False, 4, None) + False, 4, None, content_sets=None, + repo_ids=None) @mock.patch('ubipop.UbiPopulate') @@ -67,7 +68,8 @@ def test_custom_config_source(mock_ubipopulate): 'custom/conf/dir'] main(args) mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, [], - 'custom/conf/dir', False, 4, None) + 'custom/conf/dir', False, 4, None, content_sets=None, + repo_ids=None) @mock.patch('ubipop.UbiPopulate') @@ -76,7 +78,8 @@ def test_crt(mock_ubipopulate): 'custom/conf/dir'] main(args) mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('/cert.cert', ), False, [], - 'custom/conf/dir', False, 4, None) + 'custom/conf/dir', False, 4, None, content_sets=None, + repo_ids=None) @mock.patch('ubipop.UbiPopulate') @@ -85,7 +88,28 @@ def test_specified_filenames(mock_ubipopulate): 'custom/conf/dir', 'f1', 'f2'] main(args) mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, ['f1', 'f2'], - 'custom/conf/dir', False, 4, None) + 'custom/conf/dir', False, 4, None, content_sets=None, + repo_ids=None) + + +@mock.patch('ubipop.UbiPopulate') +def test_specified_content_sets(mock_ubipopulate): + args = ['--pulp-hostname', 'foo.pulp.com', '--user', 'foo', '--pass', 'foo', '--content-sets', + 'test_repo1-rpms', ] + main(args) + mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, [], None, + False, 4, None, content_sets=['test_repo1-rpms', ], + repo_ids=None) + + +@mock.patch('ubipop.UbiPopulate') +def test_specified_repo_ids(mock_ubipopulate): + args = ['--pulp-hostname', 'foo.pulp.com', '--user', 'foo', '--pass', 'foo', '--repo-ids', + 'test_repo1', ] + main(args) + mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, [], None, + False, 4, None, content_sets=None, + repo_ids=['test_repo1', ]) @mock.patch('ubipop.UbiPopulate') @@ -94,7 +118,8 @@ def test_dry_run(mock_ubipopulate): 'custom/conf/dir', 'f1', 'f2', '--dry-run'] main(args) mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), True, ['f1', 'f2'], - 'custom/conf/dir', False, 4, None) + 'custom/conf/dir', False, 4, None, content_sets=None, + repo_ids=None) @mock.patch('ubipop.UbiPopulate') @@ -103,7 +128,8 @@ def test_custom_workers_number(mock_ubipopulate): 'custom/conf/dir', 'f1', 'f2', '--workers', '42'] main(args) mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, ['f1', 'f2'], - 'custom/conf/dir', False, 42, None) + 'custom/conf/dir', False, 42, None, content_sets=None, + repo_ids=None) @mock.patch('ubipop.UbiPopulate') @@ -112,7 +138,8 @@ def test_insecure(mock_ubipopulate): 'custom/conf/dir', 'f1', 'f2', '--workers', '42', '--insecure'] main(args) mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, ['f1', 'f2'], - 'custom/conf/dir', True, 42, None) + 'custom/conf/dir', True, 42, None, content_sets=None, + repo_ids=None) @mock.patch('ubipop.UbiPopulate') @@ -121,4 +148,5 @@ def test_output_file(mock_ubipopulate): '--output-repos', '/foo/out/repos.txt'] main(args) mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, [], None, - False, 4, '/foo/out/repos.txt') + False, 4, '/foo/out/repos.txt', content_sets=None, + repo_ids=None) diff --git a/tests/test_pulp.py b/tests/test_pulp.py index b4cfb05..d688fc7 100644 --- a/tests/test_pulp.py +++ b/tests/test_pulp.py @@ -39,6 +39,7 @@ def fixture_search_repo_response(): "id": "test_repo", "notes": { "arch": "x86_64", + "content_set": "test_repo-source-rpms", "platform_full_version": "7", }, "distributors": [{ @@ -51,7 +52,11 @@ def fixture_search_repo_response(): @pytest.fixture(name='mock_repo') def fixture_mock_repo(): yield Repo( - "test_repo", "x86_64", "7", [ + repo_id="test_repo", + arch="x86_64", + content_set="test_repo-source-rpms", + platform_full_version="7", + dist_ids_type_ids=[ ("dist_id_1", "dist_type_id_1"), ("dist_id_2", "dist_type_id_2"), ], @@ -167,12 +172,26 @@ def fixture_mock_unassociate(requests_mock, mock_repo, mock_response_for_async_r def test_search_repo_by_cs(mock_pulp, mock_search_repos): # pylint: disable=unused-argument - repos = mock_pulp.search_repo_by_cs("foo") + repos = mock_pulp.search_repo_by_cs("test_repo-source-rpms") assert len(repos) == 1 repo = repos[0] assert repo.repo_id == "test_repo" assert repo.arch == "x86_64" + assert repo.content_set == "test_repo-source-rpms" + assert repo.platform_full_version == "7" + assert repo.distributors_ids_type_ids_tuples[0] == ("dist_id", "d_type_id") + + +def test_search_repo_by_id(mock_pulp, mock_search_repos): + # pylint: disable=unused-argument + repos = mock_pulp.search_repo_by_id("test_repo") + + assert len(repos) == 1 + repo = repos[0] + assert repo.repo_id == "test_repo" + assert repo.arch == "x86_64" + assert repo.content_set == "test_repo-source-rpms" assert repo.platform_full_version == "7" assert repo.distributors_ids_type_ids_tuples[0] == ("dist_id", "d_type_id") @@ -286,6 +305,7 @@ def make_mock_response(status, text=None): [ # test everything is retryable (True, 500, None, "search_repo_by_cs", ("",), '{}', pulp_client.HTTP_TOTAL_RETRIES), + (True, 500, None, "search_repo_by_id", ("",), '{}', pulp_client.HTTP_TOTAL_RETRIES), (True, 500, None, "search_rpms", (MagicMock(repo_id='fake_id'),), '[]', pulp_client.HTTP_TOTAL_RETRIES), (True, 500, None, "search_modules", diff --git a/tests/test_ubipop.py b/tests/test_ubipop.py index cb2a75a..30ca0f6 100644 --- a/tests/test_ubipop.py +++ b/tests/test_ubipop.py @@ -64,6 +64,7 @@ def get_test_repo(**kwargs): return Repo( kwargs.get('repo_id'), kwargs.get('arch'), + kwargs.get('content_set'), kwargs.get('platform_full_version'), kwargs.get('distributors_ids_type_ids'), ) @@ -654,6 +655,26 @@ def test_ubipopulate_load_ubiconfig(mocked_ubiconfig_load): assert ubipop.ubiconfig_list[0] == "test" +def test_load_ubiconfig_by_content_set_labels(): + """Ensure correct config is returned when given a content set label""" + ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False, ubiconfig_dir_or_url=TEST_DATA_DIR, + content_sets=["rhel-7-server-rpms", ]) + assert len(ubipop.ubiconfig_list) == 1 + assert ubipop.ubiconfig_list[0].content_sets.rpm.output == "ubi-7-server-rpms" + + +@patch("ubipop._pulp_client.Pulp.search_repo_by_id") +def test_load_ubiconfig_by_repo_ids(mocked_search_repo_by_id): + """Ensure correct config is returned when given a repo ID""" + mocked_search_repo_by_id.return_value = [ + get_test_repo(repo_id="rhel-7-server", content_set="rhel-7-server-rpms"), + ] + ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False, ubiconfig_dir_or_url=TEST_DATA_DIR, + repo_ids=["rhel-7-server", ]) + assert len(ubipop.ubiconfig_list) == 1 + assert ubipop.ubiconfig_list[0].content_sets.rpm.output == "ubi-7-server-rpms" + + @pytest.fixture(name='mocked_ubiconfig_load_all') def fixture_mocked_ubiconfig_load_all(): with patch('ubiconfig.get_loader') as get_loader: diff --git a/tests/test_utils.py b/tests/test_utils.py index ca83ab0..796fa0b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -16,15 +16,15 @@ def test_raise_not_implemented_pulp_action(): units = ["unit1", "unit2"] - repo = Repo("test", "1", "2", None) + repo = Repo("test", "test-rpms", "1", "2", None) action = PulpAction(units, repo) pytest.raises(NotImplementedError, action.get_action, None) def test_raise_not_implemented_associate_action(): units = ["unit1", "unit2"] - repo = Repo("test", "1", "2", None) - src_repo = Repo("test", "1", "2", None) + repo = Repo("test", "test-rpms", "1", "2", None) + src_repo = Repo("test", "test-rpms", "1", "2", None) action = AssociateAction(units, repo, src_repo) pytest.raises(NotImplementedError, action.get_action, None) @@ -36,8 +36,8 @@ def test_raise_not_implemented_associate_action(): ]) def test_get_action_associate(klass, method): units = ["unit1", "unit2"] - dst_repo = Repo("test_dst", "1", "2", None) - src_repo = Repo("test_src", "1", "2", None) + dst_repo = Repo("test_dst", "test_dst-rpms", "1", "2", None) + src_repo = Repo("test_src", "test_src-rpms", "1", "2", None) action = klass(units, dst_repo, src_repo) associate_action, src_repo_current, dst_repo_current, current_units = \ action.get_action(MagicMock()) @@ -55,7 +55,7 @@ def test_get_action_associate(klass, method): ]) def test_get_action_unassociate(klass, method): units = ["unit1", "unit2"] - dst_repo = Repo("test_dst", "1", "2", None) + dst_repo = Repo("test_dst", "test_dst-rpms", "1", "2", None) action = klass(units, dst_repo) associate_action, dst_repo_current, current_units = action.get_action(MagicMock()) diff --git a/ubipop/__init__.py b/ubipop/__init__.py index 12bde8c..404728f 100644 --- a/ubipop/__init__.py +++ b/ubipop/__init__.py @@ -80,25 +80,57 @@ def _ensure_repos_existence(self): class UbiPopulate(object): def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=None, - ubiconfig_dir_or_url=None, insecure=False, workers_count=4, - output_repos=None): + ubiconfig_dir_or_url=None, insecure=False, workers_count=4, output_repos=None, + **kwargs): - self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, - ubiconfig_dir_or_url) self.pulp = Pulp(pulp_hostname, pulp_auth, insecure) self.dry_run = dry_run self.output_repos = output_repos self._executor = Executors.thread_pool(max_workers=workers_count).with_retry() + self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url, + content_sets=kwargs.get('content_sets', None), + repo_ids=kwargs.get('repo_ids', None)) - def _load_ubiconfig(self, filenames, ubiconfig_dir_or_url): + def _load_ubiconfig(self, filenames, ubiconfig_dir_or_url, content_sets=None, repo_ids=None): loader = ubiconfig.get_loader(ubiconfig_dir_or_url) ubi_conf_list = [] + if filenames: ubi_conf_list.extend(loader.load(filename) for filename in filenames) else: ubi_conf_list.extend(loader.load_all()) - return ubi_conf_list + return self._filter_ubi_conf_list(ubi_conf_list, content_sets, repo_ids) + + def _filter_ubi_conf_list(self, config_list, content_sets, repo_ids): + """ + Reduces the list of UBI configurations to only those matching + provided content sets and/or repo IDs. + """ + + filtered_conf_list = [] + + content_sets = content_sets or [] + repo_ids = repo_ids or [] + + if not content_sets and not repo_ids: + return config_list + + fts = [self._executor.submit(self.pulp.search_repo_by_id, r) for r in repo_ids] + for repo_list in [ft.result() for ft in fts]: + content_sets.extend(repo.content_set for repo in repo_list) + + for conf in config_list: + for label in [ + conf.content_sets.rpm.input, conf.content_sets.rpm.output, + conf.content_sets.srpm.input, conf.content_sets.srpm.output, + conf.content_sets.debuginfo.input, conf.content_sets.debuginfo.output, + ]: + if label in content_sets: + filtered_conf_list.append(conf) + break + + return filtered_conf_list def populate_ubi_repos(self): out_repos = set() diff --git a/ubipop/_pulp_client.py b/ubipop/_pulp_client.py index 45ca5c1..4233707 100644 --- a/ubipop/_pulp_client.py +++ b/ubipop/_pulp_client.py @@ -79,22 +79,40 @@ def do_request(self, req_type, url, data=None): return ret - def search_repo_by_cs(self, content_set): + def _search_repo(self, criteria): url = "repositories/search/" - payload = {"criteria": {"filters": {"notes.content_set": content_set}}, - "distributors": True} - - ret = self.do_request("post", url, payload) + ret = self.do_request("post", url, criteria) ret.raise_for_status() repos = [] for item in ret.json(): notes = item['notes'] - repos.append(Repo(item['id'], notes['arch'], notes['platform_full_version'], - [(distributor['id'], distributor['distributor_type_id']) - for distributor in item['distributors']])) + dist_info = [(dist['id'], dist['distributor_type_id']) for dist in item['distributors']] + repos.append(Repo( + repo_id=item['id'], + arch=notes['arch'], + content_set=notes['content_set'], + platform_full_version=notes['platform_full_version'], + dist_ids_type_ids=dist_info, + )) return repos + def search_repo_by_cs(self, content_set): + """Returns a list of Repo objects with matching content set labels""" + + criteria = {"criteria": {"filters": {"notes.content_set": content_set}}, + "distributors": True} + + return self._search_repo(criteria) + + def search_repo_by_id(self, repo_id): + """Returns a list of Repo objects with matching repo IDs""" + + criteria = {"criteria": {"filters": {"id": repo_id}}, + "distributors": True} + + return self._search_repo(criteria) + def search_rpms(self, repo, name=None, arch=None, name_globbing=False, filename=None): url = "repositories/{REPO_ID}/search/units/".format(REPO_ID=repo.repo_id) criteria = {"type_ids": ["rpm", "srpm"]} @@ -288,11 +306,12 @@ def publish_repo(self, repo): class Repo(object): - def __init__(self, repo_id, arch, platform_full_version, distributors_ids_type_ids): + def __init__(self, repo_id, arch, content_set, platform_full_version, dist_ids_type_ids): self.repo_id = repo_id self.arch = arch + self.content_set = content_set self.platform_full_version = platform_full_version - self.distributors_ids_type_ids_tuples = distributors_ids_type_ids + self.distributors_ids_type_ids_tuples = dist_ids_type_ids class Package(object): diff --git a/ubipop/cli.py b/ubipop/cli.py index 2c6648f..8a37bf3 100644 --- a/ubipop/cli.py +++ b/ubipop/cli.py @@ -15,7 +15,11 @@ def parse_args(args): parser = argparse.ArgumentParser() parser.add_argument('input', action="store", nargs='*', help="path to ubi config file") - parser.add_argument('--conf-src', action="store", + parser.add_argument('--content-sets', action="store", nargs='+', type=str, required=False, + help="content set labels from which to source ubi config") + parser.add_argument('--repo-ids', action="store", nargs='+', type=str, required=False, + help="repo IDs from which to source ubi config") + parser.add_argument('--conf-src', action="store", required=False, help="source of ubi config, directory or url") parser.add_argument('--dry-run', action='store_true', default=False, help="if True, print pulp actions") @@ -59,10 +63,9 @@ def main(args): opts, auth = parse_args(args) - ubipop.UbiPopulate(opts.pulp_hostname, auth, opts.dry_run, opts.input, - opts.conf_src, opts.insecure, opts.workers, - opts.output_repos)\ - .populate_ubi_repos() + ubipop.UbiPopulate(opts.pulp_hostname, auth, opts.dry_run, opts.input, opts.conf_src, + opts.insecure, opts.workers, opts.output_repos, + content_sets=opts.content_sets, repo_ids=opts.repo_ids).populate_ubi_repos() def entry_point():