Skip to content

Commit

Permalink
conan-io#282 Add CONAN_REMOVE_OUTDATE_PACKAGES option
Browse files Browse the repository at this point in the history
- The new option will run conan remove <ref> -r <remote> -f --outdated
  after to upload a package

Signed-off-by: Uilian Ries <[email protected]>
  • Loading branch information
uilianries committed Mar 15, 2019
1 parent d4dcf59 commit 2a95fc6
Show file tree
Hide file tree
Showing 7 changed files with 160 additions and 3 deletions.
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1032,6 +1032,7 @@ Using **CONAN_CLANG_VERSIONS** env variable in Travis ci or Appveyor:
- "missing": Build only missing packages.
- "outdated": Build only missing or if the available package is not built with the current recipe. Useful to upload new configurations, e.j packages for a new compiler without
rebuild all packages.
- **remove_outdated_packages**: Remove all outdated packages from remote after to upload a package. Default [False]
- **test_folder**: Custom test folder consumed by Conan create, e.j .conan/test_package
- **config_url**: Conan config URL be installed before to build e.j https://github.com/bincrafters/conan-config.git
Expand Down Expand Up @@ -1167,6 +1168,7 @@ This is especially useful for CI integration.
- **CONAN_CONFIG_URL**: Conan config URL be installed before to build e.j https://github.com/bincrafters/conan-config.git
- **CONAN_BASE_PROFILE**: Apply options, settings, etc. to this profile instead of `default`.
- **CONAN_IGNORE_SKIP_CI**: Ignore `[skip ci]` in commit message.
- **CONAN_REMOVE_OUTDATED_PACKAGES**: Remove all outdated packages from remote after to upload a package. Default [False]
- **CPT_TEST_FOLDER**: Custom test_package path, e.j .conan/test_package
Expand Down
40 changes: 40 additions & 0 deletions cpt/eraser.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# -*- coding: utf-8 -*-

class Eraser(object):
""" Helper to connect on remove and remove outdated packages
"""

def __init__(self, conan_api, remote_manager, auth_manager, printer, remove):
""" Initialize Eraser instance
:param conan_api: Conan API instance
:param remote_manager: Remote manager instance
:param auth_manager: Authication manager to access the remote
:param printer: CPT output
:param remove: True if should remove outdated packages from remote. Otherwise, False.
"""
self.conan_api = conan_api
self.remote_manager = remote_manager
self.auth_manager = auth_manager
self.printer = printer
self.remove = remove

def remove_outdated_packages(self, reference):
""" Remove outdated packages from remote
:param reference: Package reference e.g. foo/0.1.0@user/channel
"""
if not self.remote_manager or not self.remote_manager.upload_remote_name:
self.printer.print_message("Remove outdated skipped, no remote available")
return
remote_name = self.remote_manager.upload_remote_name

if not self.auth_manager or not self.auth_manager.credentials_ready(remote_name):
self.printer.print_message("Remove outdated skipped, credentials for remote '%s' not available" % remote_name)
return

if self.remove:
self.printer.print_message("Removing outdated packages for '%s'" % str(reference))
self.auth_manager.login(remote_name)
self.conan_api.remove(pattern=str(reference),
force=True,
remote_name=remote_name,
outdated=True)
9 changes: 9 additions & 0 deletions cpt/packager.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from cpt.tools import get_bool_from_env
from cpt.tools import split_colon_env
from cpt.uploader import Uploader
from cpt.eraser import Eraser


def load_cf_class(path, conan_api):
Expand Down Expand Up @@ -95,6 +96,7 @@ def __init__(self, username=None, channel=None, runner=None,
docker_conan_home=None,
pip_install=None,
build_policy=None,
remove_outdated_packages=False,
always_update_conan_in_docker=False,
conan_api=None,
client_cache=None,
Expand Down Expand Up @@ -130,6 +132,11 @@ def __init__(self, username=None, channel=None, runner=None,
default_username=self.username,
skip_check_credentials=self.skip_check_credentials)

self._remove_outdated_packages = remove_outdated_packages or \
get_bool_from_env("CONAN_REMOVE_OUTDATED_PACKAGES")
self.eraser = Eraser(self.conan_api, self.remotes_manager, self.auth_manager, self.printer,
self._remove_outdated_packages)

# Upload related variables
self.upload_retry = upload_retry or os.getenv("CONAN_UPLOAD_RETRY", 3)

Expand Down Expand Up @@ -518,6 +525,7 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None):
profile_abs_path = save_profile_to_tmp(profile_text)
r = CreateRunner(profile_abs_path, build.reference, self.conan_api,
self.uploader,
eraser=self.eraser,
exclude_vcvars_precommand=self.exclude_vcvars_precommand,
build_policy=self.build_policy,
runner=self.runner,
Expand All @@ -539,6 +547,7 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None):
docker_image_skip_pull=self._docker_image_skip_pull,
build_policy=self.build_policy,
always_update_conan_in_docker=self._update_conan_in_docker,
remove_outdated_packages=self._remove_outdated_packages,
upload=self._upload_enabled(),
upload_retry=self.upload_retry,
runner=self.runner,
Expand Down
6 changes: 5 additions & 1 deletion cpt/run_in_docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from cpt.remotes import RemotesManager
from cpt.runner import CreateRunner, unscape_env
from cpt.uploader import Uploader
from cpt.eraser import Eraser


def run():
Expand All @@ -29,6 +30,9 @@ def run():
test_folder = unscape_env(os.getenv("CPT_TEST_FOLDER"))
reference = ConanFileReference.loads(os.getenv("CONAN_REFERENCE"))

remove_outdated_packages = unscape_env(os.getenv("CPT_REMOVE_OUTDATED_PACKAGES"))
eraser = Eraser(conan_api, remotes_manager, auth_manager, printer, remove_outdated_packages)

profile_text = unscape_env(os.getenv("CPT_PROFILE"))
abs_profile_path = save_profile_to_tmp(profile_text)
base_profile_text = unscape_env(os.getenv("CPT_BASE_PROFILE"))
Expand All @@ -39,7 +43,7 @@ def run():
base_profile_text)

upload = os.getenv("CPT_UPLOAD_ENABLED")
runner = CreateRunner(abs_profile_path, reference, conan_api, uploader,
runner = CreateRunner(abs_profile_path, reference, conan_api, uploader, eraser=eraser,
build_policy=build_policy, printer=printer, upload=upload,
test_folder=test_folder, config_url=config_url)
runner.run()
Expand Down
10 changes: 8 additions & 2 deletions cpt/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,14 @@

class CreateRunner(object):

def __init__(self, profile_abs_path, reference, conan_api, uploader,
def __init__(self, profile_abs_path, reference, conan_api, uploader, eraser,
exclude_vcvars_precommand=False, build_policy=None, runner=None,
cwd=None, printer=None, upload=False, test_folder=None, config_url=None):

self.printer = printer or Printer()
self._cwd = cwd or os.getcwd()
self._uploader = uploader
self._eraser = eraser
self._upload = upload
self._conan_api = conan_api
self._profile_abs_path = profile_abs_path
Expand Down Expand Up @@ -104,11 +105,13 @@ def run(self):
self.printer.print_rule()
return
for installed in r['installed']:
if installed["recipe"]["id"] == str(self._reference):
str_ref = str(self._reference)
if installed["recipe"]["id"] == str_ref:
package_id = installed['packages'][0]['id']
if installed['packages'][0]["built"]:
self._uploader.upload_packages(self._reference,
self._upload, package_id)
self._eraser.remove_outdated_packages(str_ref)
else:
self.printer.print_message("Skipping upload for %s, "
"it hasn't been built" % package_id)
Expand All @@ -121,6 +124,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference
docker_image_skip_update=False, build_policy=None,
docker_image_skip_pull=False,
always_update_conan_in_docker=False,
remove_outdated_packages=False,
upload=False, upload_retry=None,
runner=None,
docker_shell="", docker_conan_home="",
Expand All @@ -137,6 +141,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference
self._build_policy = build_policy
self._docker_image = docker_image
self._always_update_conan_in_docker = always_update_conan_in_docker
self._remove_outdated_packages = remove_outdated_packages
self._docker_image_skip_update = docker_image_skip_update
self._docker_image_skip_pull = docker_image_skip_pull
self._sudo_docker_command = sudo_docker_command or ""
Expand Down Expand Up @@ -256,6 +261,7 @@ def get_env_vars(self):
ret["CONAN_TEMP_TEST_FOLDER"] = "1" # test package folder to a temp one
ret["CPT_UPLOAD_ENABLED"] = self._upload
ret["CPT_UPLOAD_RETRY"] = self._upload_retry
ret["CPT_REMOVE_OUTDATED_PACKAGES"] = self._remove_outdated_packages
ret["CPT_BUILD_POLICY"] = escape_env(self._build_policy)
ret["CPT_TEST_FOLDER"] = escape_env(self._test_folder)
ret["CPT_CONFIG_URL"] = escape_env(self._config_url)
Expand Down
66 changes: 66 additions & 0 deletions cpt/test/test_client/erase_checks_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# -*- coding: utf-8 -*-
import unittest

from conans.client.tools import environment_append
from conans.test.utils.tools import TestClient, TestServer

from cpt.test.test_client.tools import get_patched_multipackager


class EraseTest(unittest.TestCase):

old_conanfile = """from conans import ConanFile
class Pkg(ConanFile):
name = "lib"
version = "1.0"
options = {"shared": [True, False]}
default_options = "shared=False"
def build(self):
self.output.warn("OLD")
"""

new_conanfile = """from conans import ConanFile
class Pkg(ConanFile):
name = "lib"
version = "1.0"
options = {"shared": [True, False], "foo": [True, False]}
default_options = "shared=False", "foo=True"
def build(self):
self.output.warn("NEW")
"""

def test_remove_updated_packages_env_var(self):
ts = TestServer(users={"user": "password"})
tc = TestClient(servers={"default": ts}, users={"default": [("user", "password")]})
tc.save({"conanfile.py": self.old_conanfile})
with environment_append({"CONAN_UPLOAD": ts.fake_url, "CONAN_LOGIN_USERNAME": "user",
"CONAN_PASSWORD": "password", "CONAN_USERNAME": "user",
"CONAN_REMOVE_OUTDATED_PACKAGES": "1"}):
mulitpackager = get_patched_multipackager(tc, build_policy="missing",
exclude_vcvars_precommand=True)
mulitpackager.add({}, {"shared": True})
mulitpackager.add({}, {"shared": False})
mulitpackager.run()
self.assertIn("Uploading package 1/2", tc.out)
self.assertIn("Uploading package 2/2", tc.out)
self.assertIn("OLD", tc.out)
self.assertIn("Removing outdated packages for 'lib/1.0@user/testing'", tc.out)

def test_remove_updated_packages_params(self):
ts = TestServer(users={"user": "password"})
tc = TestClient(servers={"default": ts}, users={"default": [("user", "password")]})
tc.save({"conanfile.py": self.old_conanfile})
with environment_append({"CONAN_UPLOAD": ts.fake_url, "CONAN_LOGIN_USERNAME": "user",
"CONAN_PASSWORD": "password", "CONAN_USERNAME": "user"}):
mulitpackager = get_patched_multipackager(tc, build_policy="missing",
exclude_vcvars_precommand=True,
remove_outdated_packages=True)
mulitpackager.add({}, {"shared": True})
mulitpackager.add({}, {"shared": False})
mulitpackager.run()
self.assertIn("Uploading package 1/2", tc.out)
self.assertIn("Uploading package 2/2", tc.out)
self.assertIn("OLD", tc.out)
self.assertIn("Removing outdated packages for 'lib/1.0@user/testing'", tc.out)
30 changes: 30 additions & 0 deletions cpt/test/unit/eraser_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# -*- coding: utf-8 -*-
import unittest

from collections import namedtuple
from conans.test.utils.tools import TestBufferConanOutput
from cpt.eraser import Eraser
from cpt.printer import Printer
from cpt.test.unit.packager_test import MockConanAPI


class AuthTest(unittest.TestCase):

def setUp(self):
self.conan_api = MockConanAPI()
self.output = TestBufferConanOutput()
self.printer = Printer(self.output.write)

def test_invalid_remote(self):
eraser = Eraser(self.conan_api, None, None, self.printer, True)
eraser.remove_outdated_packages("foo/0.1.0@user/channel")
self.assertIn("Remove outdated skipped, no remote available", self.output)
self.assertFalse(self.conan_api.calls)

def test_invalid_authentication(self):
FakeRemoteManager = namedtuple("FakeRemoteManager", "upload_remote_name")
remote_manager = FakeRemoteManager(upload_remote_name="default")
eraser = Eraser(self.conan_api, remote_manager, None, self.printer, True)
eraser.remove_outdated_packages("foo/0.1.0@user/channel")
self.assertIn("Remove outdated skipped, credentials for remote 'default' not available", self.output)
self.assertFalse(self.conan_api.calls)

0 comments on commit 2a95fc6

Please sign in to comment.