From 7eff1ff01baacbe4ec1056340eea5a49ed3e82a2 Mon Sep 17 00:00:00 2001 From: Uilian Ries Date: Thu, 15 Apr 2021 15:40:27 -0300 Subject: [PATCH] Sync release 0.35.1 into Develop (#559) Co-authored-by: Artalus Co-authored-by: Sascha Bannier --- README.md | 2 + cpt/__init__.py | 2 +- cpt/packager.py | 13 ++- cpt/run_in_docker.py | 15 ++- cpt/runner.py | 20 +++- cpt/test/integration/docker_test.py | 112 ++++++++++++++++----- cpt/test/test_client/upload_checks_test.py | 68 ++++++++++++- cpt/tools.py | 5 + cpt/uploader.py | 10 +- 9 files changed, 206 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 81cb6799..e6c2485a 100644 --- a/README.md +++ b/README.md @@ -1099,6 +1099,7 @@ Using **CONAN_CLANG_VERSIONS** env variable in Travis ci or Appveyor: - **upload_only_when_tag**: Will try to upload only if the branch is a tag. Default [False] - **upload_only_recipe**: If defined, will try to upload **only** the recipes. The built packages will **not** be uploaded. Default [False] - **upload_dependencies**: Will try to upload dependencies to your remote. Default [False] +- **upload_force**: Will try to force uploaded all packages. Default [True] - **build_types**: List containing specific build types. Default ["Release", "Debug"] - **cppstds**: List containing specific cpp standards. Default None - **skip_check_credentials**: Conan will skip checking the user credentials before building the packages. And if no user/remote is specified, will try to upload with the @@ -1219,6 +1220,7 @@ This is especially useful for CI integration. - **CONAN_UPLOAD_ONLY_WHEN_TAG**: If defined, will try to upload the packages only when the current branch is a tag. - **CONAN_UPLOAD_ONLY_RECIPE**: If defined, will try to upload **only** the recipes. The built packages will **not** be uploaded. - **CONAN_UPLOAD_DEPENDENCIES**: If defined, will try to upload the listed package dependencies to your remote. +- **CONAN_UPLOAD_FORCE**: If defined, will try to force upload all packages. Default is `True`. - **CONAN_SKIP_CHECK_CREDENTIALS**: Conan will skip checking the user credentials before building the packages. And if no user/remote is specified, will try to upload with the already stored credentiales in the local cache. Default [False] diff --git a/cpt/__init__.py b/cpt/__init__.py index a047d1c6..b31de231 100644 --- a/cpt/__init__.py +++ b/cpt/__init__.py @@ -1,5 +1,5 @@ -__version__ = '0.36.0-dev' +__version__ = '0.35.1' def get_client_version(): diff --git a/cpt/packager.py b/cpt/packager.py index e65460d8..dd7d1102 100644 --- a/cpt/packager.py +++ b/cpt/packager.py @@ -21,9 +21,10 @@ from cpt.profiles import get_profiles, save_profile_to_tmp from cpt.remotes import RemotesManager from cpt.runner import CreateRunner, DockerCreateRunner -from cpt.tools import get_bool_from_env +from cpt.tools import get_bool_from_env, get_custom_bool_from_env from cpt.tools import split_colon_env from cpt.uploader import Uploader +from cpt.config import ConfigManager def load_cf_class(path, conan_api): @@ -52,12 +53,12 @@ def load_cf_class(path, conan_api): conan_api.create_app() remotes = conan_api.app.cache.registry.load_remotes() conan_api.app.python_requires.enable_remotes(remotes=remotes) - conan_api.app.pyreq_loader.enable_remotes(remotes=remotes) if client_version < Version("1.20.0"): return conan_api.app.loader.load_class(path) elif client_version < Version("1.21.0"): return conan_api.app.loader.load_basic(path) else: + conan_api.app.pyreq_loader.enable_remotes(remotes=remotes) return conan_api.app.loader.load_named(path, None, None, None, None) @@ -114,6 +115,7 @@ def __init__(self, username=None, channel=None, runner=None, upload_only_when_stable=None, upload_only_when_tag=None, upload_only_recipe=None, + upload_force=None, build_types=None, cppstds=None, skip_check_credentials=False, @@ -184,10 +186,12 @@ def __init__(self, username=None, channel=None, runner=None, self.upload_only_when_tag = get_bool_from_env("CONAN_UPLOAD_ONLY_WHEN_TAG") self.upload_only_recipe = upload_only_recipe or get_bool_from_env("CONAN_UPLOAD_ONLY_RECIPE") + self.upload_force = upload_force if upload_force is not None \ + else get_custom_bool_from_env("CONAN_UPLOAD_FORCE", True) self.remotes_manager.add_remotes_to_conan() self.uploader = Uploader(self.conan_api, self.remotes_manager, self.auth_manager, - self.printer, self.upload_retry) + self.printer, self.upload_retry, self.upload_force) self._builds = [] self._named_builds = {} @@ -648,6 +652,8 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None): self.printer.print_message("Using specified default " "base profile: %s" % base_profile_name) self.printer.print_message("**************************************************") + if self.config_url: + ConfigManager(self.conan_api, self.printer).install(url=self.config_url, args=self.config_args) profile_text, base_profile_text = get_profiles(self.client_cache, build, base_profile_name) @@ -687,6 +693,7 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None): upload=self._upload_enabled(), upload_retry=self.upload_retry, upload_only_recipe=self.upload_only_recipe, + upload_force=self.upload_force, runner=self.runner, docker_shell=self.docker_shell, docker_conan_home=self.docker_conan_home, diff --git a/cpt/run_in_docker.py b/cpt/run_in_docker.py index 8db97878..b20f6021 100644 --- a/cpt/run_in_docker.py +++ b/cpt/run_in_docker.py @@ -3,6 +3,7 @@ from conans import tools from conans.client.conan_api import Conan from conans.model.ref import ConanFileReference +from conans.model.version import Version from cpt.auth import AuthManager from cpt.printer import Printer @@ -10,11 +11,18 @@ from cpt.remotes import RemotesManager from cpt.runner import CreateRunner, unscape_env from cpt.uploader import Uploader +from cpt import get_client_version def run(): - # Get all from environ - conan_api, client_cache, _ = Conan.factory() + conan_version = get_client_version() + if conan_version < Version("1.18.0"): + conan_api, client_cache, _ = Conan.factory() + else: + conan_api, _, _ = Conan.factory() + conan_api.create_app() + client_cache = conan_api.app.cache + printer = Printer() remotes_manager = RemotesManager(conan_api, printer) @@ -24,7 +32,8 @@ def run(): upload_retry = os.getenv("CPT_UPLOAD_RETRY") upload_only_recipe = os.getenv("CPT_UPLOAD_ONLY_RECIPE") - uploader = Uploader(conan_api, remotes_manager, auth_manager, printer, upload_retry) + upload_force = os.getenv("CPT_UPLOAD_FORCE") + uploader = Uploader(conan_api, remotes_manager, auth_manager, printer, upload_retry, upload_force) build_policy = unscape_env(os.getenv("CPT_BUILD_POLICY")) test_folder = unscape_env(os.getenv("CPT_TEST_FOLDER")) reference = ConanFileReference.loads(os.getenv("CONAN_REFERENCE")) diff --git a/cpt/runner.py b/cpt/runner.py index 65c10997..8fc4984d 100644 --- a/cpt/runner.py +++ b/cpt/runner.py @@ -2,6 +2,7 @@ import sys import subprocess import re +import time from collections import namedtuple from conans import tools @@ -170,6 +171,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference docker_image_skip_pull=False, always_update_conan_in_docker=False, upload=False, upload_retry=None, upload_only_recipe=None, + upload_force=None, runner=None, docker_shell="", docker_conan_home="", docker_platform_param="", docker_run_options="", @@ -192,6 +194,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference self._upload = upload self._upload_retry = upload_retry self._upload_only_recipe = upload_only_recipe + self._upload_force = upload_force self._reference = reference self._conan_pip_package = conan_pip_package self._build_policy = build_policy @@ -325,9 +328,15 @@ def run(self, pull_image=True, docker_entry_script=None): def pull_image(self): with self.printer.foldable_output("docker pull"): - ret = self._runner("%s docker pull %s" % (self._sudo_docker_command, self._docker_image)) - if ret != 0: - raise Exception("Error pulling the image: %s" % self._docker_image) + for retry in range(1, 4): + ret = self._runner("%s docker pull %s" % (self._sudo_docker_command, self._docker_image)) + if ret == 0: + break + elif retry == 3: + raise Exception("Error pulling the image: %s" % self._docker_image) + self.printer.print_message("Could not pull docker image '{}'. Retry ({})" + .format(self._docker_image, retry)) + time.sleep(3) def get_env_vars(self): ret = {key: value for key, value in os.environ.items() if key.startswith("CONAN_") and @@ -338,11 +347,12 @@ def get_env_vars(self): ret["CPT_BASE_PROFILE"] = escape_env(self._base_profile_text) ret["CPT_BASE_PROFILE_NAME"] = escape_env(self._base_profile_name) - ret["CONAN_USERNAME"] = escape_env(self._reference.user) + ret["CONAN_USERNAME"] = escape_env(self._reference.user or ret.get("CONAN_USERNAME")) ret["CONAN_TEMP_TEST_FOLDER"] = "1" # test package folder to a temp one ret["CPT_UPLOAD_ENABLED"] = self._upload ret["CPT_UPLOAD_RETRY"] = self._upload_retry ret["CPT_UPLOAD_ONLY_RECIPE"] = self._upload_only_recipe + ret["CPT_UPLOAD_FORCE"] = self._upload_force ret["CPT_BUILD_POLICY"] = escape_env(self._build_policy) ret["CPT_TEST_FOLDER"] = escape_env(self._test_folder) ret["CPT_CONFIG_URL"] = escape_env(self._config_url) @@ -367,7 +377,7 @@ def unscape_env(text): def escape_env(text): if not text: return text - return text.replace("\n", "@@").replace('"', '||') + return text.replace("\r", "").replace("\n", "@@").replace('"', '||') class PrintRunner(object): diff --git a/cpt/test/integration/docker_test.py b/cpt/test/integration/docker_test.py index 683bee20..184e2cca 100644 --- a/cpt/test/integration/docker_test.py +++ b/cpt/test/integration/docker_test.py @@ -3,6 +3,7 @@ import time import textwrap + from conans import tools from conans.model.ref import ConanFileReference from conans.model.version import Version @@ -34,13 +35,13 @@ def test_docker(self): client_version = get_client_version() ci_manager = MockCIManager() unique_ref = "zlib/%s" % str(time.time()) - conanfile = """from conans import ConanFile -import os - -class Pkg(ConanFile): - settings = "os", "compiler", "build_type", "arch" + conanfile = textwrap.dedent(""" + from conans import ConanFile + import os -""" + class Pkg(ConanFile): + settings = "os", "compiler", "build_type", "arch" + """) self.save_conanfile(conanfile) with tools.environment_append({"CONAN_DOCKER_RUN_OPTIONS": "--network=host -v{}:/tmp/cpt".format(self.root_project_folder), @@ -53,7 +54,7 @@ class Pkg(ConanFile): "CONAN_PASSWORD": "demo"}): self.packager = ConanMultiPackager(channel="mychannel", - gcc_versions=["6"], + gcc_versions=["8"], archs=["x86", "x86_64"], build_types=["Release"], reference=unique_ref, @@ -96,7 +97,7 @@ class Pkg(ConanFile): "CONAN_DOCKER_IMAGE_SKIP_UPDATE": "TRUE", "CONAN_UPLOAD_ONLY_WHEN_STABLE": "1"}): self.packager = ConanMultiPackager(channel="mychannel", - gcc_versions=["6"], + gcc_versions=["8"], archs=["x86", "x86_64"], build_types=["Release"], reference=unique_ref, @@ -116,16 +117,17 @@ class Pkg(ConanFile): @unittest.skipUnless(is_linux_and_have_docker(), "Requires Linux and Docker") def test_docker_run_options(self): - conanfile = """from conans import ConanFile -import os + conanfile = textwrap.dedent(""" + from conans import ConanFile + import os -class Pkg(ConanFile): - settings = "os", "compiler", "build_type", "arch" - requires = "zlib/1.2.11@conan/stable" + class Pkg(ConanFile): + settings = "os", "compiler", "build_type", "arch" + requires = "zlib/1.2.11@conan/stable" - def build(self): - pass -""" + def build(self): + pass + """) self.save_conanfile(conanfile) # Validate by Environemnt Variable with tools.environment_append({"CONAN_DOCKER_ENTRY_SCRIPT": "pip install -U /tmp/cpt", @@ -204,16 +206,17 @@ def test_docker_run_android(self): @unittest.skipUnless(is_linux_and_have_docker(), "Requires Linux and Docker") def test_docker_custom_pip_command(self): - conanfile = """from conans import ConanFile - import os + conanfile = textwrap.dedent(""" + from conans import ConanFile + import os - class Pkg(ConanFile): - settings = "os", "compiler", "build_type", "arch" - requires = "zlib/1.2.11@conan/stable" + class Pkg(ConanFile): + settings = "os", "compiler", "build_type", "arch" + requires = "zlib/1.2.11@conan/stable" - def build(self): - pass - """ + def build(self): + pass + """) self.save_conanfile(conanfile) with tools.environment_append({"CONAN_DOCKER_ENTRY_SCRIPT": "pip install -U /tmp/cpt", "CONAN_USERNAME": "bar", @@ -236,6 +239,38 @@ def build(self): self.assertIn("Error updating the image", str(raised.exception)) self.assertIn("foobar install conan_package_tools", str(raised.exception)) + @unittest.skipUnless(is_linux_and_have_docker(), "Requires Linux and Docker") + def test_docker_base_profile(self): + conanfile = textwrap.dedent(""" + from conans import ConanFile + + class Pkg(ConanFile): + + def build(self): + pass + """) + + self.save_conanfile(conanfile) + with tools.environment_append({"CONAN_DOCKER_RUN_OPTIONS": "--network=host -v{}:/tmp/cpt".format(self.root_project_folder), + "CONAN_DOCKER_ENTRY_SCRIPT": "pip install -U /tmp/cpt", + "CONAN_DOCKER_IMAGE": "conanio/gcc8", + "CONAN_USE_DOCKER": "1", + "CONAN_REFERENCE": "foo/0.0.1@bar/testing", + "CONAN_DOCKER_IMAGE_SKIP_UPDATE": "TRUE", + "CONAN_FORCE_SELINUX": "TRUE", + "CONAN_DOCKER_USE_SUDO": "FALSE", + "CONAN_DOCKER_SHELL": "/bin/bash -c", + }): + self.packager = ConanMultiPackager(gcc_versions=["8"], + archs=["x86_64"], + build_types=["Release"], + config_url="https://github.com/bincrafters/bincrafters-config.git", + out=self.output.write) + self.packager.add({}) + self.packager.run(base_profile_name="linux-gcc8-amd64") + self.assertIn('Using specified default base profile: linux-gcc8-amd64', self.output) + self.assertIn('-e CPT_BASE_PROFILE_NAME="linux-gcc8-amd64"', self.output) + @unittest.skipUnless(is_linux_and_have_docker(), "Requires Linux and Docker") def test_docker_hidden_password(self): conanfile = textwrap.dedent(""" @@ -267,3 +302,32 @@ def build(self): self.packager.run() self.assertIn('-e CONAN_LOGIN_USERNAME="xxxxxxxx"', self.output) self.assertIn('-e CONAN_PASSWORD="xxxxxxxx"', self.output) + + @unittest.skipUnless(is_linux_and_have_docker(), "Requires Linux and Docker") + def test_docker_underscore_user_channel(self): + conanfile = textwrap.dedent(""" + from conans import ConanFile + + class Pkg(ConanFile): + def build(self): + pass + """) + + self.save_conanfile(conanfile) + with tools.environment_append({"CONAN_USERNAME": "_", + "CONAN_CHANNEL": "_", + "CONAN_DOCKER_IMAGE": "conanio/gcc8", + "CONAN_REFERENCE": "foo/0.0.1", + "CONAN_DOCKER_IMAGE_SKIP_UPDATE": "TRUE", + "CONAN_FORCE_SELINUX": "TRUE", + "CONAN_DOCKER_USE_SUDO": "FALSE", + "CONAN_DOCKER_SHELL": "/bin/bash -c", + }): + self.packager = ConanMultiPackager(gcc_versions=["8"], + archs=["x86_64"], + build_types=["Release"], + out=self.output.write) + self.packager.add({}) + self.packager.run() + self.assertIn('-e CONAN_USERNAME="_"', self.output) + self.assertIn('-e CONAN_CHANNEL="_"', self.output) diff --git a/cpt/test/test_client/upload_checks_test.py b/cpt/test/test_client/upload_checks_test.py index 496f5bf9..6f299241 100644 --- a/cpt/test/test_client/upload_checks_test.py +++ b/cpt/test/test_client/upload_checks_test.py @@ -1,8 +1,12 @@ import unittest import os import zipfile +import textwrap from conans.client.tools import environment_append +from conans.model.manifest import FileTreeManifest +from conans.model.ref import PackageReference + from cpt.test.utils.tools import TestClient, TestServer from cpt.test.unit.utils import MockCIManager @@ -144,7 +148,7 @@ def test_upload_only_recipe_env_var(self): self.assertIn(" Uploading packages for 'lib/1.0@user/mychannel'", tc.out) self.assertIn("Uploading lib/1.0@user/mychannel to remote", tc.out) - self.assertIn("Recipe is up to date, upload skipped", tc.out) + self.assertNotIn("Recipe is up to date, upload skipped", tc.out) self.assertIn("Uploading package 1/2", tc.out) self.assertIn("Uploading package 2/2", tc.out) @@ -183,7 +187,7 @@ def test_upload_only_recipe_params(self): self.assertIn(" Uploading packages for 'lib/1.0@user/mychannel'", tc.out) self.assertIn("Uploading lib/1.0@user/mychannel to remote", tc.out) - self.assertIn("Recipe is up to date, upload skipped", tc.out) + self.assertNotIn("Recipe is up to date, upload skipped", tc.out) self.assertIn("Uploading package 1/2", tc.out) self.assertIn("Uploading package 2/2", tc.out) @@ -222,6 +226,66 @@ def test_upload_partial_reference(self): self.assertIn("Uploading packages for 'lib/1.0@'", tc.out) self.assertIn("lib/1.0: WARN: HALLO", tc.out) + def test_forced_upload(self): + NO_SETTINGS_PACKAGE_ID = "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9" + conanfile = textwrap.dedent(""" + from conans import ConanFile + class Pkg(ConanFile): + + def configure(self): + self.output.warn("Mens sana in corpore sano") + """) + ts = TestServer(users={"foo": "password"}) + tc = TestClient(servers={"foo_server": ts}, users={"foo_server": [("foo", "password")]}) + + tc.save({"conanfile.py": conanfile}) + tc.run("create . lib/1.0@foo/stable") + tc.run("upload lib/1.0@foo/stable --all -r foo_server") + pref = PackageReference.loads("lib/1.0@foo/stable#{}:{}".format(0, NO_SETTINGS_PACKAGE_ID)) + path = os.path.join(ts.server_store.package_revisions_root(pref), "0") + manifest = FileTreeManifest.load(path) + manifest.time += 1000 + manifest.save(path) + + tc.save({"conanfile.py": conanfile.replace("warn", "info")}) + tc.run("create . lib/1.0@foo/stable") + + # Force is True, package must be uploaded all times + with environment_append({"CONAN_UPLOAD": ts.fake_url, "CONAN_LOGIN_USERNAME": "foo", + "CONAN_PASSWORD": "password", "CONAN_USERNAME": "foo", + "CONAN_UPLOAD_FORCE": "True"}): + mulitpackager = get_patched_multipackager(tc, exclude_vcvars_precommand=True) + mulitpackager.add_common_builds(reference="lib/1.0@foo/stable", + shared_option_name=False) + mulitpackager.run() + self.assertIn("Uploading packages for 'lib/1.0@foo/stable'", tc.out) + self.assertNotIn("Recipe is up to date, upload skipped", tc.out) + self.assertNotIn("Package is up to date, upload skipped", tc.out) + + with environment_append({"CONAN_UPLOAD": ts.fake_url, "CONAN_LOGIN_USERNAME": "foo", + "CONAN_PASSWORD": "password", "CONAN_USERNAME": "foo", + "CONAN_UPLOAD_FORCE": "FALSE"}): + mulitpackager = get_patched_multipackager(tc, exclude_vcvars_precommand=True, + upload_force=True) + mulitpackager.add_common_builds(reference="lib/1.0@foo/stable", + shared_option_name=False) + mulitpackager.run() + self.assertIn("Uploading packages for 'lib/1.0@foo/stable'", tc.out) + self.assertNotIn("Package is up to date, upload skipped", tc.out) + self.assertNotIn("Recipe is up to date, upload skipped", tc.out) + + # Force is False. Must not upload any package + with environment_append({"CONAN_UPLOAD": ts.fake_url, "CONAN_LOGIN_USERNAME": "foo", + "CONAN_PASSWORD": "password", "CONAN_USERNAME": "foo", + "CONAN_UPLOAD_FORCE": "FALSE"}): + mulitpackager = get_patched_multipackager(tc, exclude_vcvars_precommand=True, + upload_force=False) + mulitpackager.add_common_builds(reference="lib/1.0@foo/stable", + shared_option_name=False) + mulitpackager.run() + self.assertIn("Uploading packages for 'lib/1.0@foo/stable'", tc.out) + self.assertIn("Recipe is up to date, upload skipped", tc.out) + self.assertIn("Package is up to date, upload skipped", tc.out) class UploadDependenciesTest(unittest.TestCase): diff --git a/cpt/tools.py b/cpt/tools.py index 149f706c..edddcb1d 100644 --- a/cpt/tools.py +++ b/cpt/tools.py @@ -6,6 +6,11 @@ def get_bool_from_env(var_name): return str(val).lower() not in ("0", "none", "false") +def get_custom_bool_from_env(var_name, default=None): + val = os.getenv(var_name, default) + return str(val).lower() in ('true', 'on', 'ok', 'y', 'yes', '1') + + def split_colon_env(varname): if os.getenv(varname) is None: return None diff --git a/cpt/uploader.py b/cpt/uploader.py index 18bae96b..cec060a4 100644 --- a/cpt/uploader.py +++ b/cpt/uploader.py @@ -1,16 +1,18 @@ from conans.model.version import Version +from conans.client.cmd.uploader import UPLOAD_POLICY_FORCE from cpt import get_client_version class Uploader(object): - def __init__(self, conan_api, remote_manager, auth_manager, printer, upload_retry): + def __init__(self, conan_api, remote_manager, auth_manager, printer, upload_retry, force): self.conan_api = conan_api self.remote_manager = remote_manager self.auth_manager = auth_manager self.printer = printer self._upload_retry = upload_retry + self._force = force if not self._upload_retry: self._upload_retry = 0 @@ -39,17 +41,19 @@ def _upload_artifacts(self, reference, upload, package_id=None): self.conan_api.upload(str(reference), package=package_id, remote=remote_name, - force=True, + force=self._force, retry=int(self._upload_retry)) elif client_version < Version("1.8.0"): self.conan_api.upload(str(reference), package=package_id, remote_name=remote_name, - force=True, + force=self._force, retry=int(self._upload_retry)) else: all_packages = package_id != None + policy = UPLOAD_POLICY_FORCE if self._force else None self.conan_api.upload(str(reference), all_packages=all_packages, remote_name=remote_name, + policy=policy, retry=int(self._upload_retry))