Enable population by content set(s) or repo ID(s) (release-engineerin…

…g#126)

UbiPopulate now accepts specific content set lables and/or repository
IDs for content calculation.

Fixes release-engineering#121

CHANGELOG.md

@@ -6,7 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-n/a
+- Accepts content set labels and repo IDs for content calculation
 
 ## [0.1.19] - 2019-06-25
 

tests/test_cli.py

@@ -58,7 +58,8 @@ def test_default_config_source(mock_ubipopulate):
     args = ['--pulp-hostname', 'foo.pulp.com', '--user', 'foo', '--pass', 'foo']
     main(args)
     mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, [], None,
-                                             False, 4, None)
+                                             False, 4, None, content_sets=None,
+                                             repo_ids=None)
 
 
 @mock.patch('ubipop.UbiPopulate')
@@ -67,7 +68,8 @@ def test_custom_config_source(mock_ubipopulate):
             'custom/conf/dir']
     main(args)
     mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, [],
-                                             'custom/conf/dir', False, 4, None)
+                                             'custom/conf/dir', False, 4, None, content_sets=None,
+                                             repo_ids=None)
 
 
 @mock.patch('ubipop.UbiPopulate')
@@ -76,7 +78,8 @@ def test_crt(mock_ubipopulate):
             'custom/conf/dir']
     main(args)
     mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('/cert.cert', ), False, [],
-                                             'custom/conf/dir', False, 4, None)
+                                             'custom/conf/dir', False, 4, None, content_sets=None,
+                                             repo_ids=None)
 
 
 @mock.patch('ubipop.UbiPopulate')
@@ -85,7 +88,28 @@ def test_specified_filenames(mock_ubipopulate):
             'custom/conf/dir', 'f1', 'f2']
     main(args)
     mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, ['f1', 'f2'],
-                                             'custom/conf/dir', False, 4, None)
+                                             'custom/conf/dir', False, 4, None, content_sets=None,
+                                             repo_ids=None)
+
+
+@mock.patch('ubipop.UbiPopulate')
+def test_specified_content_sets(mock_ubipopulate):
+    args = ['--pulp-hostname', 'foo.pulp.com', '--user', 'foo', '--pass', 'foo', '--content-sets',
+            'test_repo1-rpms', ]
+    main(args)
+    mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, [], None,
+                                             False, 4, None, content_sets=['test_repo1-rpms', ],
+                                             repo_ids=None)
+
+
+@mock.patch('ubipop.UbiPopulate')
+def test_specified_repo_ids(mock_ubipopulate):
+    args = ['--pulp-hostname', 'foo.pulp.com', '--user', 'foo', '--pass', 'foo', '--repo-ids',
+            'test_repo1', ]
+    main(args)
+    mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, [], None,
+                                             False, 4, None, content_sets=None,
+                                             repo_ids=['test_repo1', ])
 
 
 @mock.patch('ubipop.UbiPopulate')
@@ -94,7 +118,8 @@ def test_dry_run(mock_ubipopulate):
             'custom/conf/dir', 'f1', 'f2', '--dry-run']
     main(args)
     mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), True, ['f1', 'f2'],
-                                             'custom/conf/dir', False, 4, None)
+                                             'custom/conf/dir', False, 4, None, content_sets=None,
+                                             repo_ids=None)
 
 
 @mock.patch('ubipop.UbiPopulate')
@@ -103,7 +128,8 @@ def test_custom_workers_number(mock_ubipopulate):
             'custom/conf/dir', 'f1', 'f2', '--workers', '42']
     main(args)
     mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, ['f1', 'f2'],
-                                             'custom/conf/dir', False, 42, None)
+                                             'custom/conf/dir', False, 42, None, content_sets=None,
+                                             repo_ids=None)
 
 
 @mock.patch('ubipop.UbiPopulate')
@@ -112,7 +138,8 @@ def test_insecure(mock_ubipopulate):
             'custom/conf/dir', 'f1', 'f2', '--workers', '42', '--insecure']
     main(args)
     mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, ['f1', 'f2'],
-                                             'custom/conf/dir', True, 42, None)
+                                             'custom/conf/dir', True, 42, None, content_sets=None,
+                                             repo_ids=None)
 
 
 @mock.patch('ubipop.UbiPopulate')
@@ -121,4 +148,5 @@ def test_output_file(mock_ubipopulate):
             '--output-repos', '/foo/out/repos.txt']
     main(args)
     mock_ubipopulate.assert_called_once_with('foo.pulp.com', ('foo', 'foo'), False, [], None,
-                                             False, 4, '/foo/out/repos.txt')
+                                             False, 4, '/foo/out/repos.txt', content_sets=None,
+                                             repo_ids=None)

tests/test_pulp.py

@@ -39,6 +39,7 @@ def fixture_search_repo_response():
         "id": "test_repo",
         "notes": {
             "arch": "x86_64",
+            "content_set": "test_repo-source-rpms",
             "platform_full_version": "7",
         },
         "distributors": [{
@@ -51,7 +52,11 @@ def fixture_search_repo_response():
 @pytest.fixture(name='mock_repo')
 def fixture_mock_repo():
     yield Repo(
-        "test_repo", "x86_64", "7", [
+        repo_id="test_repo",
+        arch="x86_64",
+        content_set="test_repo-source-rpms",
+        platform_full_version="7",
+        dist_ids_type_ids=[
             ("dist_id_1", "dist_type_id_1"),
             ("dist_id_2", "dist_type_id_2"),
         ],
@@ -167,12 +172,26 @@ def fixture_mock_unassociate(requests_mock, mock_repo, mock_response_for_async_r
 
 def test_search_repo_by_cs(mock_pulp, mock_search_repos):
     # pylint: disable=unused-argument
-    repos = mock_pulp.search_repo_by_cs("foo")
+    repos = mock_pulp.search_repo_by_cs("test_repo-source-rpms")
 
     assert len(repos) == 1
     repo = repos[0]
     assert repo.repo_id == "test_repo"
     assert repo.arch == "x86_64"
+    assert repo.content_set == "test_repo-source-rpms"
+    assert repo.platform_full_version == "7"
+    assert repo.distributors_ids_type_ids_tuples[0] == ("dist_id", "d_type_id")
+
+
+def test_search_repo_by_id(mock_pulp, mock_search_repos):
+    # pylint: disable=unused-argument
+    repos = mock_pulp.search_repo_by_id("test_repo")
+
+    assert len(repos) == 1
+    repo = repos[0]
+    assert repo.repo_id == "test_repo"
+    assert repo.arch == "x86_64"
+    assert repo.content_set == "test_repo-source-rpms"
     assert repo.platform_full_version == "7"
     assert repo.distributors_ids_type_ids_tuples[0] == ("dist_id", "d_type_id")
 
@@ -286,6 +305,7 @@ def make_mock_response(status, text=None):
     [
         # test everything is retryable
         (True, 500, None, "search_repo_by_cs", ("",), '{}', pulp_client.HTTP_TOTAL_RETRIES),
+        (True, 500, None, "search_repo_by_id", ("",), '{}', pulp_client.HTTP_TOTAL_RETRIES),
         (True, 500, None, "search_rpms",
          (MagicMock(repo_id='fake_id'),), '[]', pulp_client.HTTP_TOTAL_RETRIES),
         (True, 500, None, "search_modules",

tests/test_ubipop.py

@@ -64,6 +64,7 @@ def get_test_repo(**kwargs):
     return Repo(
         kwargs.get('repo_id'),
         kwargs.get('arch'),
+        kwargs.get('content_set'),
         kwargs.get('platform_full_version'),
         kwargs.get('distributors_ids_type_ids'),
     )
@@ -654,6 +655,26 @@ def test_ubipopulate_load_ubiconfig(mocked_ubiconfig_load):
     assert ubipop.ubiconfig_list[0] == "test"
 
 
+def test_load_ubiconfig_by_content_set_labels():
+    """Ensure correct config is returned when given a content set label"""
+    ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False, ubiconfig_dir_or_url=TEST_DATA_DIR,
+                         content_sets=["rhel-7-server-rpms", ])
+    assert len(ubipop.ubiconfig_list) == 1
+    assert ubipop.ubiconfig_list[0].content_sets.rpm.output == "ubi-7-server-rpms"
+
+
+@patch("ubipop._pulp_client.Pulp.search_repo_by_id")
+def test_load_ubiconfig_by_repo_ids(mocked_search_repo_by_id):
+    """Ensure correct config is returned when given a repo ID"""
+    mocked_search_repo_by_id.return_value = [
+        get_test_repo(repo_id="rhel-7-server", content_set="rhel-7-server-rpms"),
+    ]
+    ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False, ubiconfig_dir_or_url=TEST_DATA_DIR,
+                         repo_ids=["rhel-7-server", ])
+    assert len(ubipop.ubiconfig_list) == 1
+    assert ubipop.ubiconfig_list[0].content_sets.rpm.output == "ubi-7-server-rpms"
+
+
 @pytest.fixture(name='mocked_ubiconfig_load_all')
 def fixture_mocked_ubiconfig_load_all():
     with patch('ubiconfig.get_loader') as get_loader:

tests/test_utils.py

@@ -16,15 +16,15 @@
 
 def test_raise_not_implemented_pulp_action():
     units = ["unit1", "unit2"]
-    repo = Repo("test", "1", "2", None)
+    repo = Repo("test", "test-rpms", "1", "2", None)
     action = PulpAction(units, repo)
     pytest.raises(NotImplementedError, action.get_action, None)
 
 
 def test_raise_not_implemented_associate_action():
     units = ["unit1", "unit2"]
-    repo = Repo("test", "1", "2", None)
-    src_repo = Repo("test", "1", "2", None)
+    repo = Repo("test", "test-rpms", "1", "2", None)
+    src_repo = Repo("test", "test-rpms", "1", "2", None)
     action = AssociateAction(units, repo, src_repo)
     pytest.raises(NotImplementedError, action.get_action, None)
 
@@ -36,8 +36,8 @@ def test_raise_not_implemented_associate_action():
 ])
 def test_get_action_associate(klass, method):
     units = ["unit1", "unit2"]
-    dst_repo = Repo("test_dst", "1", "2", None)
-    src_repo = Repo("test_src", "1", "2", None)
+    dst_repo = Repo("test_dst", "test_dst-rpms", "1", "2", None)
+    src_repo = Repo("test_src", "test_src-rpms", "1", "2", None)
     action = klass(units, dst_repo, src_repo)
     associate_action, src_repo_current, dst_repo_current, current_units = \
         action.get_action(MagicMock())
@@ -55,7 +55,7 @@ def test_get_action_associate(klass, method):
 ])
 def test_get_action_unassociate(klass, method):
     units = ["unit1", "unit2"]
-    dst_repo = Repo("test_dst", "1", "2", None)
+    dst_repo = Repo("test_dst", "test_dst-rpms", "1", "2", None)
     action = klass(units, dst_repo)
     associate_action, dst_repo_current, current_units = action.get_action(MagicMock())
 

ubipop/__init__.py

@@ -80,25 +80,57 @@ def _ensure_repos_existence(self):
 
 class UbiPopulate(object):
     def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=None,
-                 ubiconfig_dir_or_url=None, insecure=False, workers_count=4,
-                 output_repos=None):
+                 ubiconfig_dir_or_url=None, insecure=False, workers_count=4, output_repos=None,
+                 **kwargs):
 
-        self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list,
-                                                   ubiconfig_dir_or_url)
         self.pulp = Pulp(pulp_hostname, pulp_auth, insecure)
         self.dry_run = dry_run
         self.output_repos = output_repos
         self._executor = Executors.thread_pool(max_workers=workers_count).with_retry()
+        self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url,
+                                                   content_sets=kwargs.get('content_sets', None),
+                                                   repo_ids=kwargs.get('repo_ids', None))
 
-    def _load_ubiconfig(self, filenames, ubiconfig_dir_or_url):
+    def _load_ubiconfig(self, filenames, ubiconfig_dir_or_url, content_sets=None, repo_ids=None):
         loader = ubiconfig.get_loader(ubiconfig_dir_or_url)
         ubi_conf_list = []
+
         if filenames:
             ubi_conf_list.extend(loader.load(filename) for filename in filenames)
         else:
             ubi_conf_list.extend(loader.load_all())
 
-        return ubi_conf_list
+        return self._filter_ubi_conf_list(ubi_conf_list, content_sets, repo_ids)
+
+    def _filter_ubi_conf_list(self, config_list, content_sets, repo_ids):
+        """
+        Reduces the list of UBI configurations to only those matching
+        provided content sets and/or repo IDs.
+        """
+
+        filtered_conf_list = []
+
+        content_sets = content_sets or []
+        repo_ids = repo_ids or []
+
+        if not content_sets and not repo_ids:
+            return config_list
+
+        fts = [self._executor.submit(self.pulp.search_repo_by_id, r) for r in repo_ids]
+        for repo_list in [ft.result() for ft in fts]:
+            content_sets.extend(repo.content_set for repo in repo_list)
+
+        for conf in config_list:
+            for label in [
+                conf.content_sets.rpm.input, conf.content_sets.rpm.output,
+                conf.content_sets.srpm.input, conf.content_sets.srpm.output,
+                conf.content_sets.debuginfo.input, conf.content_sets.debuginfo.output,
+            ]:
+                if label in content_sets:
+                    filtered_conf_list.append(conf)
+                    break
+
+        return filtered_conf_list
 
     def populate_ubi_repos(self):
         out_repos = set()

ubipop/_pulp_client.py

@@ -79,22 +79,40 @@ def do_request(self, req_type, url, data=None):
 
         return ret
 
-    def search_repo_by_cs(self, content_set):
+    def _search_repo(self, criteria):
         url = "repositories/search/"
-        payload = {"criteria": {"filters": {"notes.content_set": content_set}},
-                   "distributors": True}
-
-        ret = self.do_request("post", url, payload)
+        ret = self.do_request("post", url, criteria)
         ret.raise_for_status()
         repos = []
         for item in ret.json():
             notes = item['notes']
-            repos.append(Repo(item['id'], notes['arch'], notes['platform_full_version'],
-                              [(distributor['id'], distributor['distributor_type_id'])
-                               for distributor in item['distributors']]))
+            dist_info = [(dist['id'], dist['distributor_type_id']) for dist in item['distributors']]
+            repos.append(Repo(
+                repo_id=item['id'],
+                arch=notes['arch'],
+                content_set=notes['content_set'],
+                platform_full_version=notes['platform_full_version'],
+                dist_ids_type_ids=dist_info,
+            ))
 
         return repos
 
+    def search_repo_by_cs(self, content_set):
+        """Returns a list of Repo objects with matching content set labels"""
+
+        criteria = {"criteria": {"filters": {"notes.content_set": content_set}},
+                    "distributors": True}
+
+        return self._search_repo(criteria)
+
+    def search_repo_by_id(self, repo_id):
+        """Returns a list of Repo objects with matching repo IDs"""
+
+        criteria = {"criteria": {"filters": {"id": repo_id}},
+                    "distributors": True}
+
+        return self._search_repo(criteria)
+
     def search_rpms(self, repo, name=None, arch=None, name_globbing=False, filename=None):
         url = "repositories/{REPO_ID}/search/units/".format(REPO_ID=repo.repo_id)
         criteria = {"type_ids": ["rpm", "srpm"]}
@@ -288,11 +306,12 @@ def publish_repo(self, repo):
 
 
 class Repo(object):
-    def __init__(self, repo_id, arch, platform_full_version, distributors_ids_type_ids):
+    def __init__(self, repo_id, arch, content_set, platform_full_version, dist_ids_type_ids):
         self.repo_id = repo_id
         self.arch = arch
+        self.content_set = content_set
         self.platform_full_version = platform_full_version
-        self.distributors_ids_type_ids_tuples = distributors_ids_type_ids
+        self.distributors_ids_type_ids_tuples = dist_ids_type_ids
 
 
 class Package(object):

ubipop/cli.py

@@ -15,7 +15,11 @@ def parse_args(args):
     parser = argparse.ArgumentParser()
     parser.add_argument('input', action="store", nargs='*',
                         help="path to ubi config file")
-    parser.add_argument('--conf-src', action="store",
+    parser.add_argument('--content-sets', action="store", nargs='+', type=str, required=False,
+                        help="content set labels from which to source ubi config")
+    parser.add_argument('--repo-ids', action="store", nargs='+', type=str, required=False,
+                        help="repo IDs from which to source ubi config")
+    parser.add_argument('--conf-src', action="store", required=False,
                         help="source of ubi config, directory or url")
     parser.add_argument('--dry-run', action='store_true', default=False,
                         help="if True, print pulp actions")
@@ -59,10 +63,9 @@ def main(args):
 
     opts, auth = parse_args(args)
 
-    ubipop.UbiPopulate(opts.pulp_hostname, auth, opts.dry_run, opts.input,
-                       opts.conf_src, opts.insecure, opts.workers,
-                       opts.output_repos)\
-        .populate_ubi_repos()
+    ubipop.UbiPopulate(opts.pulp_hostname, auth, opts.dry_run, opts.input, opts.conf_src,
+                       opts.insecure, opts.workers, opts.output_repos,
+                       content_sets=opts.content_sets, repo_ids=opts.repo_ids).populate_ubi_repos()
 
 
 def entry_point():