From 52106c22df2028de0f1346fff3a9d40e37b671c3 Mon Sep 17 00:00:00 2001 From: Marty Falatic Date: Wed, 16 Jan 2019 15:55:23 -0800 Subject: [PATCH] Implement 'Links' functionality; add dry run option This allows PyUp to retrieve and display links for a given package using the new `package_metadata` PyUp API. This provides greater parity with the Github Enterprise PyUp. Also added a dry run option to make live testing easier to accomplish --- pyup/bot.py | 7 +++--- pyup/cli.py | 9 +++++--- pyup/providers/github.py | 29 +++++++++++++++++++++++- pyup/providers/gitlab.py | 27 ++++++++++++++++++++-- pyup/requirements.py | 20 ++++++++++++++++ pyup/templates/_links.md | 10 ++++++++ pyup/templates/sequential_update_body.md | 8 +++++++ 7 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 pyup/templates/_links.md diff --git a/pyup/bot.py b/pyup/bot.py index 1e896f5..64e0ab9 100644 --- a/pyup/bot.py +++ b/pyup/bot.py @@ -13,9 +13,9 @@ class Bot(object): def __init__(self, repo, user_token, bot_token=None, provider=GithubProvider, bundle=RequirementsBundle, config=Config, - integration=False, provider_url=None): + integration=False, provider_url=None, dryrun=False): self.req_bundle = bundle() - self.provider = provider(self.req_bundle, integration, provider_url) + self.provider = provider(bundle=self.req_bundle, intergration=integration, url=provider_url, dryrun=dryrun) self.user_token = user_token self.bot_token = bot_token self.fetched_files = [] @@ -31,6 +31,7 @@ def __init__(self, repo, user_token, bot_token=None, self._fetched_prs = False self.integration = integration + self.dryrun = dryrun @property def user_repo(self): @@ -184,7 +185,7 @@ def apply_updates(self, initial, scheduled): # some scheduled updates don't have commits in them. This happens if a package is # outdated, but the config file is blocking the update (insecure, no updates). # check if this is the case here. - if not updates: + if not updates or self.dryrun: continue if self.config.pr_prefix: diff --git a/pyup/cli.py b/pyup/cli.py index 4431a7b..c7ae463 100644 --- a/pyup/cli.py +++ b/pyup/cli.py @@ -23,8 +23,10 @@ @click.option('--branch', help='Set the branch the bot should use', default='master') @click.option('--initial', help='Set this to bundle all PRs into a large one', default=False, is_flag=True) +@click.option('--dryrun', help='Dry run, creates no new artifacts', + default=False, is_flag=True) @click.option('--log', help='Set the log level', default="ERROR") -def main(repo, user_token, bot_token, key, provider, provider_url, branch, initial, log): +def main(repo, user_token, bot_token, key, provider, provider_url, dryrun, branch, initial, log): logging.basicConfig(level=getattr(logging, log.upper(), None)) settings.configure(key=key) @@ -42,6 +44,7 @@ def main(repo, user_token, bot_token, key, provider, provider_url, branch, initi bot_token=bot_token, provider=ProviderClass, provider_url=provider_url, + dryrun=dryrun, ) bot.update(branch=branch, initial=initial) @@ -54,9 +57,9 @@ def main(repo, user_token, bot_token, key, provider, provider_url, branch, initi class CLIBot(Bot): def __init__(self, repo, user_token, bot_token=None, - provider=GithubProvider, bundle=RequirementsBundle, provider_url=None): + provider=GithubProvider, bundle=RequirementsBundle, provider_url=None, dryrun=False): bundle = CLIBundle - super(CLIBot, self).__init__(repo, user_token, bot_token, provider, bundle, provider_url=provider_url) + super(CLIBot, self).__init__(repo, user_token, bot_token, provider, bundle, provider_url=provider_url, dryrun=dryrun) def iter_updates(self, initial, scheduled): diff --git a/pyup/providers/github.py b/pyup/providers/github.py index 9520b46..30e06cd 100644 --- a/pyup/providers/github.py +++ b/pyup/providers/github.py @@ -13,9 +13,12 @@ class Provider(object): - def __init__(self, bundle, integration=False, url=None): + name = 'github' + + def __init__(self, bundle, integration=False, url=None, dryrun=False): self.bundle = bundle self.integration = integration + self.dryrun = dryrun self.url = url @classmethod @@ -88,6 +91,9 @@ def get_file(self, repo, path, branch): def create_and_commit_file(self, repo, path, branch, content, commit_message, committer): # integrations don't support committer data being set. Add this as extra kwarg # if we're not dealing with an integration token + if self.dryrun: + return + extra_kwargs = {} if not self.integration: extra_kwargs["committer"] = self.get_committer_data(committer) @@ -111,6 +117,9 @@ def get_requirement_file(self, repo, path, branch): return None def create_branch(self, repo, base_branch, new_branch): + if self.dryrun: + return + try: ref = repo.get_git_ref("/".join(["heads", base_branch])) repo.create_git_ref(ref="refs/heads/" + new_branch, sha=ref.object.sha) @@ -143,6 +152,9 @@ def delete_branch(self, repo, branch, prefix): :param repo: github.Repository :param branch: string name of the branch to delete """ + if self.dryrun: + return + # extra safeguard to make sure we are handling a bot branch here assert branch.startswith(prefix) ref = repo.get_git_ref("/".join(["heads", branch])) @@ -156,6 +168,9 @@ def create_commit(self, path, branch, commit_message, content, sha, repo, commit # integrations don't support committer data being set. Add this as extra kwarg # if we're not dealing with an integration token + if self.dryrun: + return + extra_kwargs = {} if not self.integration: extra_kwargs["committer"] = self.get_committer_data(committer) @@ -207,6 +222,9 @@ def get_pull_request_committer(self, repo, pull_request): return [] def close_pull_request(self, bot_repo, user_repo, pull_request, comment, prefix): + if self.dryrun: + return True + try: pull_request = bot_repo.get_pull(pull_request.number) pull_request.create_issue_comment(comment) @@ -219,6 +237,9 @@ def close_pull_request(self, bot_repo, user_repo, pull_request, comment, prefix) return False def create_pull_request(self, repo, title, body, base_branch, new_branch, pr_label, assignees, **kwargs): + if self.dryrun: + return + try: if len(body) >= 65536: logger.warning("PR body exceeds maximum length of 65536 chars, reducing") @@ -254,6 +275,9 @@ def create_pull_request(self, repo, title, body, base_branch, new_branch, pr_lab "Unable to create pull request on {repo}".format(repo=repo)) def get_or_create_label(self, repo, name): + if self.dryrun: + return '' + try: label = repo.get_label(name=name) except UnknownObjectException: @@ -267,6 +291,9 @@ def get_or_create_label(self, repo, name): return label def create_issue(self, repo, title, body): + if self.dryrun: + return + try: return repo.create_issue( title=title, diff --git a/pyup/providers/gitlab.py b/pyup/providers/gitlab.py index bfb3468..86f16f0 100644 --- a/pyup/providers/gitlab.py +++ b/pyup/providers/gitlab.py @@ -20,9 +20,10 @@ class Committer(object): def __init__(self, login): self.login = login - def __init__(self, bundle, intergration=False, url=None): + def __init__(self, bundle, intergration=False, url=None, dryrun=False): self.bundle = bundle self.url = url + self.dryrun = dryrun if intergration: raise NotImplementedError( 'Gitlab provider does not support integration mode') @@ -83,8 +84,10 @@ def get_file(self, repo, path, branch): return contentfile.decode().decode("utf-8"), contentfile def create_and_commit_file(self, repo, path, branch, content, commit_message, committer): - # TODO: committer + if self.dryrun: + return + return repo.files.create({ 'file_path': path, 'branch': branch, @@ -102,6 +105,9 @@ def get_requirement_file(self, repo, path, branch): return None def create_branch(self, repo, base_branch, new_branch): + if self.dryrun: + return + try: repo.branches.create({"branch": new_branch, "ref": base_branch}) @@ -134,6 +140,9 @@ def delete_branch(self, repo, branch, prefix): :param repo: github.Repository :param branch: string name of the branch to delete """ + if self.dryrun: + return + # make sure that the name of the branch begins with pyup. assert branch.startswith(prefix) obj = repo.branches.get(branch) @@ -141,6 +150,8 @@ def delete_branch(self, repo, branch, prefix): def create_commit(self, path, branch, commit_message, content, sha, repo, committer): # TODO: committer + if self.dryrun: + return f = repo.files.get(file_path=path, ref=branch) # Gitlab supports a plaintext encoding, which is when the encoding @@ -158,6 +169,9 @@ def get_pull_request_committer(self, repo, pull_request): ] def close_pull_request(self, bot_repo, user_repo, pull_request, comment, prefix): + if self.dryrun: + return True + mr = user_repo.mergerequests.get(pull_request.number) mr.state_event = 'close' mr.save() @@ -168,11 +182,17 @@ def close_pull_request(self, bot_repo, user_repo, pull_request, comment, prefix) self.delete_branch(user_repo, source_branch, prefix) def _merge_merge_request(self, mr, config): + if self.dryrun: + return + mr.merge(should_remove_source_branch=config.gitlab.should_remove_source_branch, merge_when_pipeline_succeeds=True) def create_pull_request(self, repo, title, body, base_branch, new_branch, pr_label, assignees, config): # TODO: Check permissions + if self.dryrun: + return + try: if len(body) >= 65536: logger.warning("PR body exceeds maximum length of 65536 chars, reducing") @@ -223,6 +243,9 @@ def create_pull_request(self, repo, title, body, base_branch, new_branch, pr_lab ) def create_issue(self, repo, title, body): + if self.dryrun: + return + return repo.issues.create({ 'title': title, 'description': body diff --git a/pyup/requirements.py b/pyup/requirements.py index e8ff2eb..0e7a41a 100644 --- a/pyup/requirements.py +++ b/pyup/requirements.py @@ -231,6 +231,7 @@ def __init__(self, name, specs, line, lineno, extras, file_type): self._is_insecure = None self._changelog = None + self._package_metadata = None def __eq__(self, other): return ( @@ -441,6 +442,25 @@ def changelog(self): self._changelog[version] = log return self._changelog + @property + def package_metadata(self): + if self._package_metadata is None: + self._package_metadata = OrderedDict() + if settings.api_key: + r = requests.get( + "https://pyup.io/api/v1/package_metadata/{}/".format(self.key), + headers={"X-Api-Key": settings.api_key} + ) + if r.status_code == 403: + raise InvalidKeyError + if r.status_code == 200: + data = r.json() + if data and 'links' in data: + self._package_metadata = OrderedDict( + (source, link) for source, link in data['links'] + ) + return self._package_metadata + @property def is_outdated(self): if self.version and self.latest_version_within_specs: diff --git a/pyup/templates/_links.md b/pyup/templates/_links.md new file mode 100644 index 0000000..20c500c --- /dev/null +++ b/pyup/templates/_links.md @@ -0,0 +1,10 @@ +
+ Links + {% for source, link in package_metadata.items() %} + {% if link %} + - {{ source }}: {{ link }} + {% endif %} + {% endfor %} +
+ + diff --git a/pyup/templates/sequential_update_body.md b/pyup/templates/sequential_update_body.md index 092439e..c8e1a64 100644 --- a/pyup/templates/sequential_update_body.md +++ b/pyup/templates/sequential_update_body.md @@ -12,4 +12,12 @@ This PR pins [{{ requirement.full_name }}](https://pypi.org/project/{{ requireme *The bot wasn't able to find a changelog for this release. [Got an idea?](https://github.com/pyupio/changelogs/issues/new)* {% endif %} +{% if requirement.package_metadata %} +{% with package_metadata=requirement.package_metadata %} +{% include "_links.md" %} +{% endwith %} +{% elif api_key %} +*The bot wasn't able to find the links for this package. [Got an idea?](https://github.com/pyupio/changelogs/issues/new)* +{% endif %} + {% include "_api_key.md" %}