Implement 'Links' functionality; add dry run option

This allows PyUp to retrieve and display links for a given package using
the new `package_metadata` PyUp API.
This provides greater parity with the Github Enterprise PyUp.

Also added a dry run option to make live testing easier to accomplish

pyup/bot.py

@@ -13,9 +13,9 @@
 class Bot(object):
     def __init__(self, repo, user_token, bot_token=None,
                  provider=GithubProvider, bundle=RequirementsBundle, config=Config,
-                 integration=False, provider_url=None):
+                 integration=False, provider_url=None, dryrun=False):
         self.req_bundle = bundle()
-        self.provider = provider(self.req_bundle, integration, provider_url)
+        self.provider = provider(bundle=self.req_bundle, intergration=integration, url=provider_url, dryrun=dryrun)
         self.user_token = user_token
         self.bot_token = bot_token
         self.fetched_files = []
@@ -31,6 +31,7 @@ def __init__(self, repo, user_token, bot_token=None,
         self._fetched_prs = False
 
         self.integration = integration
+        self.dryrun = dryrun
 
     @property
     def user_repo(self):
@@ -184,7 +185,7 @@ def apply_updates(self, initial, scheduled):
             # some scheduled updates don't have commits in them. This happens if a package is
             # outdated, but the config file is blocking the update (insecure, no updates).
             # check if this is the case here.
-            if not updates:
+            if not updates or self.dryrun:
                 continue
 
             if self.config.pr_prefix:

pyup/cli.py

@@ -23,8 +23,10 @@
 @click.option('--branch', help='Set the branch the bot should use', default='master')
 @click.option('--initial', help='Set this to bundle all PRs into a large one',
               default=False, is_flag=True)
+@click.option('--dryrun', help='Dry run, creates no new artifacts',
+              default=False, is_flag=True)
 @click.option('--log', help='Set the log level', default="ERROR")
-def main(repo, user_token, bot_token, key, provider, provider_url, branch, initial, log):
+def main(repo, user_token, bot_token, key, provider, provider_url, dryrun, branch, initial, log):
     logging.basicConfig(level=getattr(logging, log.upper(), None))
 
     settings.configure(key=key)
@@ -42,6 +44,7 @@ def main(repo, user_token, bot_token, key, provider, provider_url, branch, initi
         bot_token=bot_token,
         provider=ProviderClass,
         provider_url=provider_url,
+        dryrun=dryrun,
     )
 
     bot.update(branch=branch, initial=initial)
@@ -54,9 +57,9 @@ def main(repo, user_token, bot_token, key, provider, provider_url, branch, initi
 class CLIBot(Bot):
 
     def __init__(self, repo, user_token, bot_token=None,
-                 provider=GithubProvider, bundle=RequirementsBundle, provider_url=None):
+                 provider=GithubProvider, bundle=RequirementsBundle, provider_url=None, dryrun=False):
         bundle = CLIBundle
-        super(CLIBot, self).__init__(repo, user_token, bot_token, provider, bundle, provider_url=provider_url)
+        super(CLIBot, self).__init__(repo, user_token, bot_token, provider, bundle, provider_url=provider_url, dryrun=dryrun)
 
     def iter_updates(self, initial, scheduled):
 

pyup/providers/github.py

@@ -13,9 +13,12 @@
 
 
 class Provider(object):
-    def __init__(self, bundle, integration=False, url=None):
+    name = 'github'
+
+    def __init__(self, bundle, integration=False, url=None, dryrun=False):
         self.bundle = bundle
         self.integration = integration
+        self.dryrun = dryrun
         self.url = url
 
     @classmethod
@@ -88,6 +91,9 @@ def get_file(self, repo, path, branch):
     def create_and_commit_file(self, repo, path, branch, content, commit_message, committer):
         # integrations don't support committer data being set. Add this as extra kwarg
         # if we're not dealing with an integration token
+        if self.dryrun:
+            return
+
         extra_kwargs = {}
         if not self.integration:
             extra_kwargs["committer"] = self.get_committer_data(committer)
@@ -111,6 +117,9 @@ def get_requirement_file(self, repo, path, branch):
         return None
 
     def create_branch(self, repo, base_branch, new_branch):
+        if self.dryrun:
+            return
+
         try:
             ref = repo.get_git_ref("/".join(["heads", base_branch]))
             repo.create_git_ref(ref="refs/heads/" + new_branch, sha=ref.object.sha)
@@ -143,6 +152,9 @@ def delete_branch(self, repo, branch, prefix):
         :param repo: github.Repository
         :param branch: string name of the branch to delete
         """
+        if self.dryrun:
+            return
+
         # extra safeguard to make sure we are handling a bot branch here
         assert branch.startswith(prefix)
         ref = repo.get_git_ref("/".join(["heads", branch]))
@@ -156,6 +168,9 @@ def create_commit(self, path, branch, commit_message, content, sha, repo, commit
 
         # integrations don't support committer data being set. Add this as extra kwarg
         # if we're not dealing with an integration token
+        if self.dryrun:
+            return
+
         extra_kwargs = {}
         if not self.integration:
             extra_kwargs["committer"] = self.get_committer_data(committer)
@@ -207,6 +222,9 @@ def get_pull_request_committer(self, repo, pull_request):
             return []
 
     def close_pull_request(self, bot_repo, user_repo, pull_request, comment, prefix):
+        if self.dryrun:
+            return True
+
         try:
             pull_request = bot_repo.get_pull(pull_request.number)
             pull_request.create_issue_comment(comment)
@@ -219,6 +237,9 @@ def close_pull_request(self, bot_repo, user_repo, pull_request, comment, prefix)
             return False
 
     def create_pull_request(self, repo, title, body, base_branch, new_branch, pr_label, assignees, **kwargs):
+        if self.dryrun:
+            return
+
         try:
             if len(body) >= 65536:
                 logger.warning("PR body exceeds maximum length of 65536 chars, reducing")
@@ -254,6 +275,9 @@ def create_pull_request(self, repo, title, body, base_branch, new_branch, pr_lab
                 "Unable to create pull request on {repo}".format(repo=repo))
 
     def get_or_create_label(self, repo, name):
+        if self.dryrun:
+            return ''
+
         try:
             label = repo.get_label(name=name)
         except UnknownObjectException:
@@ -267,6 +291,9 @@ def get_or_create_label(self, repo, name):
         return label
 
     def create_issue(self, repo, title, body):
+        if self.dryrun:
+            return
+
         try:
             return repo.create_issue(
                 title=title,

pyup/providers/gitlab.py

@@ -20,9 +20,10 @@ class Committer(object):
         def __init__(self, login):
             self.login = login
 
-    def __init__(self, bundle, intergration=False, url=None):
+    def __init__(self, bundle, intergration=False, url=None, dryrun=False):
         self.bundle = bundle
         self.url = url
+        self.dryrun = dryrun
         if intergration:
             raise NotImplementedError(
                 'Gitlab provider does not support integration mode')
@@ -83,8 +84,10 @@ def get_file(self, repo, path, branch):
             return contentfile.decode().decode("utf-8"), contentfile
 
     def create_and_commit_file(self, repo, path, branch, content, commit_message, committer):
-
         # TODO: committer
+        if self.dryrun:
+            return
+
         return repo.files.create({
             'file_path': path,
             'branch': branch,
@@ -102,6 +105,9 @@ def get_requirement_file(self, repo, path, branch):
         return None
 
     def create_branch(self, repo, base_branch, new_branch):
+        if self.dryrun:
+            return
+
         try:
             repo.branches.create({"branch": new_branch,
                                   "ref": base_branch})
@@ -134,13 +140,18 @@ def delete_branch(self, repo, branch, prefix):
         :param repo: github.Repository
         :param branch: string name of the branch to delete
         """
+        if self.dryrun:
+            return
+
         # make sure that the name of the branch begins with pyup.
         assert branch.startswith(prefix)
         obj = repo.branches.get(branch)
         obj.delete()
 
     def create_commit(self, path, branch, commit_message, content, sha, repo, committer):
         # TODO: committer
+        if self.dryrun:
+            return
 
         f = repo.files.get(file_path=path, ref=branch)
         # Gitlab supports a plaintext encoding, which is when the encoding
@@ -158,6 +169,9 @@ def get_pull_request_committer(self, repo, pull_request):
         ]
 
     def close_pull_request(self, bot_repo, user_repo, pull_request, comment, prefix):
+        if self.dryrun:
+            return True
+
         mr = user_repo.mergerequests.get(pull_request.number)
         mr.state_event = 'close'
         mr.save()
@@ -168,11 +182,17 @@ def close_pull_request(self, bot_repo, user_repo, pull_request, comment, prefix)
         self.delete_branch(user_repo, source_branch, prefix)
 
     def _merge_merge_request(self, mr, config):
+        if self.dryrun:
+            return
+
         mr.merge(should_remove_source_branch=config.gitlab.should_remove_source_branch,
                  merge_when_pipeline_succeeds=True)
 
     def create_pull_request(self, repo, title, body, base_branch, new_branch, pr_label, assignees, config):
         # TODO: Check permissions
+        if self.dryrun:
+            return
+
         try:
             if len(body) >= 65536:
                 logger.warning("PR body exceeds maximum length of 65536 chars, reducing")
@@ -223,6 +243,9 @@ def create_pull_request(self, repo, title, body, base_branch, new_branch, pr_lab
                         )
 
     def create_issue(self, repo, title, body):
+        if self.dryrun:
+            return
+
         return repo.issues.create({
             'title': title,
             'description': body

pyup/requirements.py

@@ -231,6 +231,7 @@ def __init__(self, name, specs, line, lineno, extras, file_type):
 
         self._is_insecure = None
         self._changelog = None
+        self._package_metadata = None
 
     def __eq__(self, other):
         return (
@@ -441,6 +442,25 @@ def changelog(self):
                                 self._changelog[version] = log
         return self._changelog
 
+    @property
+    def package_metadata(self):
+        if self._package_metadata is None:
+            self._package_metadata = OrderedDict()
+            if settings.api_key:
+                r = requests.get(
+                    "https://pyup.io/api/v1/package_metadata/{}/".format(self.key),
+                    headers={"X-Api-Key": settings.api_key}
+                )
+                if r.status_code == 403:
+                    raise InvalidKeyError
+                if r.status_code == 200:
+                    data = r.json()
+                    if data and 'links' in data:
+                        self._package_metadata = OrderedDict(
+                            (source, link) for source, link in data['links']
+                        )
+        return self._package_metadata
+
     @property
     def is_outdated(self):
         if self.version and self.latest_version_within_specs:

pyup/templates/_links.md

@@ -0,0 +1,10 @@
+<details>
+  <summary>Links</summary>
+  {% for source, link in package_metadata.items() %}
+  {% if link %}
+   - {{ source }}: {{ link }}
+   {% endif %}
+  {% endfor %}
+</details>
+
+

pyup/templates/sequential_update_body.md

@@ -12,4 +12,12 @@ This PR pins [{{ requirement.full_name }}](https://pypi.org/project/{{ requireme
 *The bot wasn't able to find a changelog for this release. [Got an idea?](https://github.com/pyupio/changelogs/issues/new)*
 {% endif %}
 
+{% if requirement.package_metadata %}
+{% with package_metadata=requirement.package_metadata %}
+{% include "_links.md" %}
+{% endwith %}
+{% elif api_key %}
+*The bot wasn't able to find the links for this package. [Got an idea?](https://github.com/pyupio/changelogs/issues/new)*
+{% endif %}
+
 {% include "_api_key.md" %}