diff --git a/.circleci/config.yml b/.circleci/config.yml index 3a5a6d338..29779d031 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,20 +44,6 @@ jobs: - store_artifacts: path: wheelhouse/ - - when: - condition: - matches: - pattern: ".+" - value: "<< pipeline.git.tag >>" - steps: - - run: - environment: - TWINE_NONINTERACTIVE: "1" - TWINE_USERNAME: __token__ - command: | - python3 -m pip install twine - python3 -m twine upload --skip-existing wheelhouse/* - workflows: wheels: # This is the name of the workflow, feel free to change it to better match your workflow. # Inside the workflow, you define the jobs you want to run. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 217a9324d..66d1a3970 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,12 +8,14 @@ on: paths-ignore: - "docs/**" - "tools/**" + - ".circleci/**" - ".github/workflows/*" - "!.github/workflows/test.yml" pull_request: paths-ignore: - "docs/**" - "tools/**" + - ".circleci/**" - ".github/workflows/*" - "!.github/workflows/test.yml" diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 71cafcc21..39c8d9bd4 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -22,6 +22,7 @@ concurrency: cancel-in-progress: true env: + PYTHONUNBUFFERED: "1" TWINE_NONINTERACTIVE: "1" # CIBW_PRERELEASE_PYTHONS: "1" @@ -177,6 +178,38 @@ jobs: path: "wheelhouse/*" if-no-files-found: error + circle-wheels: + if: github.repository_owner == 'zeromq' + runs-on: ubuntu-22.04 + needs: + # not strictly required, but avoids wasting cpu time waiting + - wheels + steps: + - uses: actions/checkout@v4 + + - name: setup python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + + - name: install dependencies + run: | + pip install --upgrade setuptools pip wheel + pip install requests + + - name: fetch wheels from circleci + run: python3 tools/circle_wheels.py + env: + CIRCLECI_TOKEN: ${{ secrets.CIRCLECI_TOKEN }} + PR_HEAD_SHA: ${{ github.event.pull_request && github.event.pull_request.head.sha }} + + - uses: actions/upload-artifact@v4 + with: + name: wheels-linux-arm + path: "wheelhouse/*" + if-no-files-found: error + github-release: permissions: contents: write @@ -212,6 +245,7 @@ jobs: needs: - sdist - wheel + - circle-wheels steps: - uses: actions/download-artifact@v4 with: @@ -229,6 +263,7 @@ jobs: needs: - sdist - wheel + - circle-wheels steps: - uses: actions/download-artifact@v4 with: diff --git a/tools/circle_wheels.py b/tools/circle_wheels.py new file mode 100644 index 000000000..f8692dd20 --- /dev/null +++ b/tools/circle_wheels.py @@ -0,0 +1,105 @@ +import os +import sys +import time +from pathlib import Path + +import requests + +s = requests.Session() +if os.getenv("CIRCLECI_TOKEN"): + # get credentials + # not _required_ + s.headers["Circle-Token"] = os.environ["CIRCLECI_TOKEN"] + +slug = "gh/zeromq/pyzmq" + + +def get(url): + """Make an API request""" + print(f"Getting {url}") + r = s.get(url) + r.raise_for_status() + return r.json() + + +def get_pipeline(sha): + print(f"Getting pipeline for {sha}") + pipelines = get(f"https://circleci.com/api/v2/project/{slug}/pipeline") + for pipeline in pipelines["items"]: + print( + pipeline['number'], pipeline['vcs']['revision'], pipeline['vcs'].get('tag') + ) + if pipeline['vcs']['revision'] == sha: + return pipeline + print(f"No pipeline found for {sha}") + return None + + +def get_workflows(pipeline): + print(f"Getting workflows for pipeline {pipeline['number']}") + return get(f"https://circleci.com/api/v2/pipeline/{pipeline['id']}/workflow")[ + "items" + ] + + +def get_jobs(workflow): + print(f"Getting jobs for for workflow {workflow['name']}") + return get(f"https://circleci.com/api/v2/workflow/{workflow['id']}/job")["items"] + + +def download_artifact(artifact): + print(f"Downloading {artifact['path']}") + p = Path(artifact['path']) + p.parent.mkdir(exist_ok=True) + with p.open("wb") as f: + r = s.get(artifact["url"], stream=True) + for chunk in r.iter_content(65536): + f.write(chunk) + + +def download_artifacts(job): + print(f"Downloading artifacts for {job['job_number']}") + for artifact in get( + f"https://circleci.com/api/v2/project/{slug}/{job['job_number']}/artifacts" + )["items"]: + download_artifact(artifact) + + +def main(): + # circleci tracks the PR head, + # but github only reports the PR merge commit + sha = os.getenv("PR_HEAD_SHA") + if not sha: + sha = os.environ["GITHUB_SHA"] + + for _ in range(10): + pipeline = get_pipeline(sha) + if pipeline is None: + # wait and try again + time.sleep(10) + else: + break + workflows = get_workflows(pipeline) + while not all(w["stopped_at"] for w in workflows): + for w in workflows: + print( + f"Workflow {pipeline['number']}/{w['name']}: {w['status']} started at {w['started_at']}" + ) + time.sleep(15) + workflows = get_workflows(pipeline) + + for workflow in workflows: + if workflow["status"] != "success": + sys.exit( + f"workflow {workflow['name']} did not succeed: {workflow['status']}" + ) + + jobs = [] + for workflow in workflows: + jobs.extend(get_jobs(workflow)) + for job in jobs: + download_artifacts(job) + + +if __name__ == "__main__": + main()