upload circleci wheels from github

avoids partial release situations

.circleci/config.yml

@@ -44,20 +44,6 @@ jobs:
       - store_artifacts:
           path: wheelhouse/
 
-      - when:
-          condition:
-            matches:
-              pattern: ".+"
-              value: "<< pipeline.git.tag >>"
-          steps:
-            - run:
-                environment:
-                  TWINE_NONINTERACTIVE: "1"
-                  TWINE_USERNAME: __token__
-                command: |
-                  python3 -m pip install twine
-                  python3 -m twine upload --skip-existing wheelhouse/*
-
 workflows:
   wheels: # This is the name of the workflow, feel free to change it to better match your workflow.
     # Inside the workflow, you define the jobs you want to run.

.github/workflows/test.yml

@@ -8,12 +8,14 @@ on:
     paths-ignore:
       - "docs/**"
       - "tools/**"
+      - ".circleci/**"
       - ".github/workflows/*"
       - "!.github/workflows/test.yml"
   pull_request:
     paths-ignore:
       - "docs/**"
       - "tools/**"
+      - ".circleci/**"
       - ".github/workflows/*"
       - "!.github/workflows/test.yml"
 

.github/workflows/wheels.yml

@@ -22,6 +22,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
+  PYTHONUNBUFFERED: "1"
   TWINE_NONINTERACTIVE: "1"
 #  CIBW_PRERELEASE_PYTHONS: "1"
 
@@ -177,6 +178,38 @@ jobs:
           path: "wheelhouse/*"
           if-no-files-found: error
 
+  circle-wheels:
+    if: github.repository_owner == 'zeromq'
+    runs-on: ubuntu-22.04
+    needs:
+      # not strictly required, but avoids wasting cpu time waiting
+      - wheels
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+
+      - name: install dependencies
+        run: |
+          pip install --upgrade setuptools pip wheel
+          pip install requests
+
+      - name: fetch wheels from circleci
+        run: python3 tools/circle_wheels.py
+        env:
+          CIRCLECI_TOKEN: ${{ secrets.CIRCLECI_TOKEN }}
+          PR_HEAD_SHA: ${{ github.event.pull_request && github.event.pull_request.head.sha }}
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-linux-arm
+          path: "wheelhouse/*"
+          if-no-files-found: error
+
   github-release:
     permissions:
       contents: write
@@ -212,6 +245,7 @@ jobs:
     needs:
       - sdist
       - wheel
+      - circle-wheels
     steps:
       - uses: actions/download-artifact@v4
         with:
@@ -229,6 +263,7 @@ jobs:
     needs:
       - sdist
       - wheel
+      - circle-wheels
     steps:
       - uses: actions/download-artifact@v4
         with:

tools/circle_wheels.py

@@ -0,0 +1,105 @@
+import os
+import sys
+import time
+from pathlib import Path
+
+import requests
+
+s = requests.Session()
+if os.getenv("CIRCLECI_TOKEN"):
+    # get credentials
+    # not _required_
+    s.headers["Circle-Token"] = os.environ["CIRCLECI_TOKEN"]
+
+slug = "gh/zeromq/pyzmq"
+
+
+def get(url):
+    """Make an API request"""
+    print(f"Getting {url}")
+    r = s.get(url)
+    r.raise_for_status()
+    return r.json()
+
+
+def get_pipeline(sha):
+    print(f"Getting pipeline for {sha}")
+    pipelines = get(f"https://circleci.com/api/v2/project/{slug}/pipeline")
+    for pipeline in pipelines["items"]:
+        print(
+            pipeline['number'], pipeline['vcs']['revision'], pipeline['vcs'].get('tag')
+        )
+        if pipeline['vcs']['revision'] == sha:
+            return pipeline
+    print(f"No pipeline found for {sha}")
+    return None
+
+
+def get_workflows(pipeline):
+    print(f"Getting workflows for pipeline {pipeline['number']}")
+    return get(f"https://circleci.com/api/v2/pipeline/{pipeline['id']}/workflow")[
+        "items"
+    ]
+
+
+def get_jobs(workflow):
+    print(f"Getting jobs for for workflow {workflow['name']}")
+    return get(f"https://circleci.com/api/v2/workflow/{workflow['id']}/job")["items"]
+
+
+def download_artifact(artifact):
+    print(f"Downloading {artifact['path']}")
+    p = Path(artifact['path'])
+    p.parent.mkdir(exist_ok=True)
+    with p.open("wb") as f:
+        r = s.get(artifact["url"], stream=True)
+        for chunk in r.iter_content(65536):
+            f.write(chunk)
+
+
+def download_artifacts(job):
+    print(f"Downloading artifacts for {job['job_number']}")
+    for artifact in get(
+        f"https://circleci.com/api/v2/project/{slug}/{job['job_number']}/artifacts"
+    )["items"]:
+        download_artifact(artifact)
+
+
+def main():
+    # circleci tracks the PR head,
+    # but github only reports the PR merge commit
+    sha = os.getenv("PR_HEAD_SHA")
+    if not sha:
+        sha = os.environ["GITHUB_SHA"]
+
+    for _ in range(10):
+        pipeline = get_pipeline(sha)
+        if pipeline is None:
+            # wait and try again
+            time.sleep(10)
+        else:
+            break
+    workflows = get_workflows(pipeline)
+    while not all(w["stopped_at"] for w in workflows):
+        for w in workflows:
+            print(
+                f"Workflow {pipeline['number']}/{w['name']}: {w['status']} started at {w['started_at']}"
+            )
+        time.sleep(15)
+        workflows = get_workflows(pipeline)
+
+    for workflow in workflows:
+        if workflow["status"] != "success":
+            sys.exit(
+                f"workflow {workflow['name']} did not succeed: {workflow['status']}"
+            )
+
+    jobs = []
+    for workflow in workflows:
+        jobs.extend(get_jobs(workflow))
+    for job in jobs:
+        download_artifacts(job)
+
+
+if __name__ == "__main__":
+    main()