improve prow-test-dumper to crosscheck settings

Improve prow-test-dumper.py not only to dump test settings from prow
config.yaml, to actually crosscheck versus GitHub branch protections.
This highlights any mismatches we might have, accounting for both
missing but also extra checks.

Also, set defaults to metal3-io file locations so --job-dir and
--config do not need to be input in our use case.

GITHUB_TOKEN with maintainer read rights must be exported in env, or
supplied with --token to do this. If token is not found, old behavior
of just dumping the config is done, but no crosschecking. Using token
with invalid/missing rights error out.

Signed-off-by: Tuomo Tanskanen <[email protected]>

hack/prow_test_dumper.py

@@ -1,15 +1,17 @@
 #!/usr/bin/env python3
 """
-Dump required tests out of prow config in human (and machine) readable format
+Dump required tests out of prow config in human (and machine) readable format.
+Crosscheck versus GitHub branch protection settings and print discrepancies.
 """
 
 from argparse import Namespace, ArgumentParser
 from collections import defaultdict
+import os
+import sys
 from typing import DefaultDict, List
 
-# pip3 install PyYAML
+import requests
 import yaml
-import os
 
 
 def get_external_branchprotection(data: dict) -> DefaultDict[str, List[str]]:
@@ -44,31 +46,89 @@ def get_prow_branchprotection(data: dict) -> DefaultDict[str, List[str]]:
     return protected_repos
 
 
+def get_github_branch_protection(owner: str, repo: str, branch: str, token: str) -> List:
+    """
+    Check branch protection settings for a specific branch in a GitHub repository.
+    """
+    # this is a hack required due input data format
+    if branch == "ALL BRANCHES":
+        branch = get_github_default_branch(owner=owner, repo=repo, token=token)
+
+    headers = {"Authorization": f"token {token}", "Accept": "application/vnd.github.v3+json"}
+    url = f"https://api.github.com/repos/{owner}/{repo}/branches/{branch}/protection"
+
+    try:
+        response = requests.get(url, headers=headers, timeout=30)
+        response.raise_for_status()
+
+        if response.status_code == 200:
+            settings = response.json()
+            if "required_status_checks" in settings:
+                status_checks = settings["required_status_checks"]
+                if status_checks:
+                    return status_checks.get("contexts", [])
+    except requests.exceptions.RequestException as ex:
+        sys.exit(f"Error: {str(ex)}")
+    return []
+
+
+def get_github_default_branch(owner: str, repo: str, token: str) -> str:
+    """
+    Get default branch for repo
+    """
+    headers = {"Authorization": f"token {token}", "Accept": "application/vnd.github.v3+json"}
+    url = f"https://api.github.com/repos/{owner}/{repo}"
+
+    try:
+        response = requests.get(url, headers=headers, timeout=30)
+        response.raise_for_status()
+        return response.json()["default_branch"]
+    except requests.exceptions.RequestException as e:
+        sys.exit(f"Error: {str(e)}")
+
+
 def parse_args() -> Namespace:
     """parse arguments"""
-    parser = ArgumentParser(description="Dump required tests out of Prow config.yaml and job-config")
+    parser = ArgumentParser(
+        description="Dump required tests out of Prow config.yaml and job-config"
+    )
     parser.add_argument(
-        "file",
+        "--config",
         type=str,
+        default="prow/config/config.yaml",
         help="""
         Prow's config.yaml (if you define presubmit separately, see --job-dir)
         """,
     )
     parser.add_argument(
         "--job-dir",
+        type=str,
+        default="prow/config/jobs/",
         help="""
-        folder containing additional job definition yaml files. This will be search recursively for yaml files.
+        folder containing additional job definition yaml files. This will be
+        searched recursively for YAML files.
         """,
     )
+    parser.add_argument(
+        "--token",
+        type=str,
+        help="""
+        GitHub API token. Need to have maintainer's read access to all repos.
+        Can also be defined via GITHUB_TOKEN environment variable. Leave empty
+        if you do not have enough permissions to skip GitHub API calls.
+        """,
+    )
+
     args = parser.parse_args()
     return args
 
 
 def main():
     """print out required jobs"""
     args = parse_args()
+    github_token = os.getenv("GITHUB_TOKEN") or args.token
 
-    with open(args.file, "r", encoding="utf-8") as fh:
+    with open(args.config, "r", encoding="utf-8") as fh:
         parsed_data = yaml.safe_load(fh)
 
     # protection can come from two places: branch-protection and presubmits
@@ -78,6 +138,7 @@ def main():
     pre_submit = get_prow_branchprotection(presubmits)
     required.update(pre_submit)
 
+    # parse job-dir as well, as we have split Prow config into per branch jobs
     if args.job_dir:
         for root, _, filenames in os.walk(args.job_dir):
             for filename in filenames:
@@ -89,10 +150,25 @@ def main():
                     pre_submit = get_prow_branchprotection(presubmits)
                     required.update(pre_submit)
 
-    for repo, checks in sorted(required.items()):
-        print(f"{repo}:")
+    # print out the results and crosscheck Github settings
+    for target, checks in sorted(required.items()):
+        gh_checks = []
+        if github_token:
+            owner, repo, branch = target.split("/")
+            gh_checks = get_github_branch_protection(
+                owner=owner, repo=repo, branch=branch, token=github_token
+            )
+        print(f"{target}:")
+
         for check in checks:
-            print(f"- {check}")
+            if not github_token or check in gh_checks:
+                print(f"- {check}")
+            else:
+                print(f"- {check} (MISSING FROM GITHUB!)")
+
+        for check in gh_checks:
+            if check not in checks:
+                print(f"- {check} (EXTRA IN GH, OR MISSING FROM PROW CONFIG!)")
         print()