Merge pull request #769 from googlefonts/push-100plus-files

push: Get all files in a pr

Lib/gftools/gfgithub.py

@@ -3,98 +3,110 @@
 import requests
 import typing
 import urllib
+import time
 
 
-GITHUB_GRAPHQL_API = 'https://api.github.com/graphql'
-GITHUB_V3_REST_API = 'https://api.github.com'
+GITHUB_GRAPHQL_API = "https://api.github.com/graphql"
+GITHUB_V3_REST_API = "https://api.github.com"
 
 
 class GitHubClient:
     def __init__(self, repo_owner, repo_name):
-        if not 'GH_TOKEN' in os.environ:
+        if not "GH_TOKEN" in os.environ:
             raise Exception("GH_TOKEN environment variable not set")
-        self.gh_token = os.environ['GH_TOKEN']
+        self.gh_token = os.environ["GH_TOKEN"]
         self.repo_owner = repo_owner
         self.repo_name = repo_name
-    
+
     def _post(self, url, payload: typing.Dict):
-        headers = {'Authorization': f'bearer {self.gh_token}'}
+        headers = {"Authorization": f"bearer {self.gh_token}"}
         response = requests.post(url, json=payload, headers=headers)
         if response.status_code == requests.codes.unprocessable:
             # has a helpful response.json with an 'errors' key.
             pass
         else:
             response.raise_for_status()
         json = response.json()
-        if 'errors' in json:
-            errors = pprint.pformat(json['errors'], indent=2)
-            raise Exception(f'GitHub POST query failed to url {url}:\n {errors}')
+        if "errors" in json:
+            errors = pprint.pformat(json["errors"], indent=2)
+            raise Exception(f"GitHub POST query failed to url {url}:\n {errors}")
         return json
-    
+
     def _get(self, url):
-        headers = {'Authorization': f'bearer {self.gh_token}'}
+        headers = {"Authorization": f"bearer {self.gh_token}"}
         response = requests.get(url, headers=headers)
         response.raise_for_status()
         json = response.json()
-        if 'errors' in json:
-            errors = pprint.pformat(json['errors'], indent=2)
-            raise Exception(f'GitHub REST query failed:\n {errors}')
+        if "errors" in json:
+            errors = pprint.pformat(json["errors"], indent=2)
+            raise Exception(f"GitHub REST query failed:\n {errors}")
         return json
-        
+
     def _run_graphql(self, query, variables):
-        payload = {'query': query, 'variables': variables}
+        payload = {"query": query, "variables": variables}
         return self._post(GITHUB_GRAPHQL_API, payload)
-    
+
     def rest_url(self, path, **kwargs):
-        base_url = f'{GITHUB_V3_REST_API}/repos/{self.repo_owner}/{self.repo_name}/{path}'
+        base_url = (
+            f"{GITHUB_V3_REST_API}/repos/{self.repo_owner}/{self.repo_name}/{path}"
+        )
         if kwargs:
-            base_url += '?' + '&'.join(f'{k}={urllib.parse.quote(v)}' for k, v in kwargs.items())
+            base_url += "?" + "&".join(
+                f"{k}={urllib.parse.quote(v)}" for k, v in kwargs.items()
+            )
         return base_url
 
     def get_blob(self, file_sha):
-        url = self.rest_url(f'git/blobs/{file_sha}')
+        url = self.rest_url(f"git/blobs/{file_sha}")
         headers = {
-            'Accept': 'application/vnd.github.v3.raw',
-            'Authorization': f'bearer {self.gh_token}'
+            "Accept": "application/vnd.github.v3.raw",
+            "Authorization": f"bearer {self.gh_token}",
         }
         response = requests.get(url, headers=headers)
         response.raise_for_status()
         return response
-    
+
     def open_prs(self, pr_head: str, pr_base_branch: str) -> typing.List:
-        return self._get(self.rest_url("pulls", state="open", head=pr_head, base=pr_base_branch))
-
+        return self._get(
+            self.rest_url("pulls", state="open", head=pr_head, base=pr_base_branch)
+        )
+
     def create_pr(self, title: str, body: str, head: str, base: str):
         return self._post(
             self.rest_url("pulls"),
             {
-                'title': title,
-                'body': body,
-                'head': head,
-                'base': base,
-                'maintainer_can_modify': True
-            }
+                "title": title,
+                "body": body,
+                "head": head,
+                "base": base,
+                "maintainer_can_modify": True,
+            },
         )
-    
+
     def create_issue_comment(self, issue_number: int, body: str):
         return self._post(
-            self.rest_url(f'issues/{issue_number}/comments'),
-            {
-                'body': body
-            }
-        )
-
-    def create_issue(self, title: str, body: str):
-        return self._post(
-            self.rest_url("issues"),
-            {
-                'title': title,
-                'body': body
-            }
+            self.rest_url(f"issues/{issue_number}/comments"), {"body": body}
         )
 
+    def create_issue(self, title: str, body: str):
+        return self._post(self.rest_url("issues"), {"title": title, "body": body})
 
-
-
-
-
+    def pr_files(self, pr_number: int, sleep=4):
+        res = []
+        cur_page = 1
+        url = self.rest_url(
+            f"pulls/{pr_number}/files", per_page="100", page=str(cur_page)
+        )
+        request = self._get(url)
+        while request:
+            res += request
+            cur_page += 1
+            url = self.rest_url(
+                f"pulls/{pr_number}/files", per_page="100", page=str(cur_page)
+            )
+            request = self._get(url)
+            # sleep so we don't hit api rate limits. We should get at least 1k
+            # requests per hour so sleeping for 4 secs by default means we
+            # shouldn't hit any issues.
+            time.sleep(sleep)
+        return res

Lib/gftools/push/trafficjam.py

@@ -130,7 +130,9 @@ def from_string(string: str):  # type: ignore[misc]
           content {
             ... on PullRequest {
               id
+              number
               files(first: 100) {
+                totalCount
                 nodes {
                   path
                 }
@@ -463,6 +465,19 @@ def from_traffic_jam(cls):
         # sort items by pr number
         board_items.sort(key=lambda k: k["content"]["url"])
 
+        # get files for prs which have more than 100 changed files
+        for item in board_items:
+            changed_files = item["content"]["files"]["totalCount"]
+            if changed_files <= 100:
+                continue
+            pr_number = item['content']['number']
+            pr_url = item["content"]["url"]
+            log.warn(
+                f"{pr_url} has {changed_files} changed files. Attempting to fetch them."
+            )
+            files = g.pr_files(pr_number)
+            item["content"]["files"]["nodes"] = [{"path": f["filename"]} for f in files]
+
         results = cls()
         for item in board_items:
             status = item.get("status", {}).get("name", None)

Lib/gftools/scripts/manage_traffic_jam.py

@@ -69,6 +69,13 @@ def user_input(self, item: PushItem):
             "Bump pushlist: [y/n], block: [b] skip pr: [s], inspect: [i], quit: [q]?: "
         )
 
+        if "*" in user_input:
+            item.bump_pushlist()
+            for sub_item in self.push_items:
+                if sub_item.url != item.url:
+                    continue
+                sub_item.push_list = item.push_list
+            self.skip_pr = item.url
         if "y" in user_input:
             item.bump_pushlist()
         if "b" in user_input:

tests/test_gfgithub.py

@@ -0,0 +1,16 @@
+from gftools.gfgithub import GitHubClient
+import pytest
+
+
+@pytest.mark.parametrize(
+    "pr_number,file_count",
+    [
+        (6874, 1),
+        (6779, 3),
+        (2987, 178),
+        (6787, 568),
+    ]
+)
+def test_pr_files(pr_number, file_count):
+    client = GitHubClient("google", "fonts")
+    assert len(client.pr_files(pr_number)) == file_count