From 7afa31a95449af1c5bfd86e448d1b67e04878528 Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 6 Aug 2024 14:24:22 -0700 Subject: [PATCH 1/5] Add a workflow which builds wheels and (only on release) publishes to PyPI --- .github/workflows/wheels.yml | 100 +++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 .github/workflows/wheels.yml diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000000..b8e36aa971 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,100 @@ +name: Build Wheels & Publish to PyPI + +on: + pull_request: + workflow_dispatch: + release: + types: [published] + +jobs: + build_sdist: + name: Build sdist + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: install python dependencies + run: pip install build + + - name: build sdist + run: | + python -m build --sdist -o wheelhouse + + - name: List and check sdist + run: | + ls -lh wheelhouse/ + twine check wheelhouse/* + + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: wheels + path: ./wheelhouse/*.tar.gz + + build_wheels: + name: > + build ${{ matrix.python-version }} on ${{ matrix.platform || matrix.os }} + ${{ (matrix.arch) || '' }} + strategy: + fail-fast: false + matrix: + os: [ubuntu] + python-version: ['cp39', 'cp310'] + + runs-on: ${{ format('{0}-latest', matrix.os) }} + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Build ${{ matrix.platform || matrix.os }} binaries + uses: pypa/cibuildwheel@v2.20.0 + env: + CIBW_BUILD: '${{ matrix.python-version }}-*' + # Linux wheels are built in manylinux containers + CIBW_BUILD_VERBOSITY: 1 + + - name: List and check wheels + run: | + pip install twine pkginfo>=1.10.0 + ${{ matrix.ls || 'ls -lh' }} wheelhouse/ + twine check wheelhouse/* + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels + path: ./wheelhouse/*.whl + + upload_to_pypi: + name: Upload to PyPI + runs-on: ubuntu-latest + if: (github.event_name == 'release' && startsWith(github.ref, 'refs/tags')) || (github.event_name == 'workflow_dispatch') + needs: [build_wheels, build_sdist] + environment: + name: pypi + url: https://pypi.org/p/tfx + permissions: + id-token: write + steps: + - name: Retrieve wheels and sdist + uses: actions/download-artifact@v4 + + - name: List the build artifacts + run: | + ls -lAs wheels/ + + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@release/v1.9 + with: + packages_dir: wheels/ From ba2971fcebac00e105ab136bdf3acd499a13cc5c Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 6 Aug 2024 16:09:43 -0700 Subject: [PATCH 2/5] Avoid using cibuildwheel; more major changes to build system needed for that --- .github/workflows/wheels.yml | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b8e36aa971..4ddca8509b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -57,12 +57,26 @@ jobs: with: python-version: '3.10' - - name: Build ${{ matrix.platform || matrix.os }} binaries - uses: pypa/cibuildwheel@v2.20.0 - env: - CIBW_BUILD: '${{ matrix.python-version }}-*' - # Linux wheels are built in manylinux containers - CIBW_BUILD_VERBOSITY: 1 + - name: Install python build dependencies + run: | + pip install wheel + + - uses: bazel-contrib/setup-bazel@0.8.5 + name: Set up Bazel + with: + # Avoid downloading Bazel every time. + bazelisk-cache: true + # Store build cache per workflow. + disk-cache: ${{ github.workflow }} + # Share repository cache between workflows. + repository-cache: true + + - name: Build wheels + run: | + package_build/initialize.sh && + python package_build/tfx/setup.py bdist_wheel && + python package_build/ml-pipelines-sdk/setup.py bdist_wheel + mv dist/*.whl wheelhouse/ - name: List and check wheels run: | From a5acb7572162f67ec7b5e6f892b6d33149900381 Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 6 Aug 2024 16:40:37 -0700 Subject: [PATCH 3/5] Print bazel info and version; ignore bazel build files --- .github/workflows/wheels.yml | 18 ++++++++++++++---- .gitignore | 3 +++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4ddca8509b..2311a8ee71 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -6,6 +6,9 @@ on: release: types: [published] +env: + USE_BAZEL_VERSION: "7.2.1" + jobs: build_sdist: name: Build sdist @@ -45,7 +48,8 @@ jobs: fail-fast: false matrix: os: [ubuntu] - python-version: ['cp39', 'cp310'] + python-version: ['cp310'] + # python-version: ['cp39', 'cp310'] runs-on: ${{ format('{0}-latest', matrix.os) }} steps: @@ -67,14 +71,20 @@ jobs: # Avoid downloading Bazel every time. bazelisk-cache: true # Store build cache per workflow. - disk-cache: ${{ github.workflow }} + disk-cache: ${{ github.workflow }}-${{ hashFiles('.github/workflows/wheels.yml') }} # Share repository cache between workflows. repository-cache: true + - name: Verify bazel installation + run: | + which bazel + bazel info + bazel version + - name: Build wheels run: | - package_build/initialize.sh && - python package_build/tfx/setup.py bdist_wheel && + package_build/initialize.sh + python package_build/tfx/setup.py bdist_wheel python package_build/ml-pipelines-sdk/setup.py bdist_wheel mv dist/*.whl wheelhouse/ diff --git a/.gitignore b/.gitignore index 7e2b2e42e8..e39a63bb11 100644 --- a/.gitignore +++ b/.gitignore @@ -141,3 +141,6 @@ bazel-* **/*_pb2.py **/*_pb2_grpc.py # LINT.ThenChange(.dockerignore) + +MODULE.bazel +MODULE.bazel.lock From d8dd9561be9dc7e051e33d2c211cc8fc8c3ef7f8 Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 6 Aug 2024 16:44:24 -0700 Subject: [PATCH 4/5] Add twine build dependency --- .github/workflows/wheels.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2311a8ee71..8beb876e2c 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -23,7 +23,7 @@ jobs: python-version: '3.10' - name: install python dependencies - run: pip install build + run: pip install build twine - name: build sdist run: | @@ -48,8 +48,7 @@ jobs: fail-fast: false matrix: os: [ubuntu] - python-version: ['cp310'] - # python-version: ['cp39', 'cp310'] + python-version: ['cp39', 'cp310'] runs-on: ${{ format('{0}-latest', matrix.os) }} steps: @@ -86,6 +85,7 @@ jobs: package_build/initialize.sh python package_build/tfx/setup.py bdist_wheel python package_build/ml-pipelines-sdk/setup.py bdist_wheel + mkdir wheelhouse mv dist/*.whl wheelhouse/ - name: List and check wheels From 10d32c0bb789daaa698f1282ebb19c052f2561d4 Mon Sep 17 00:00:00 2001 From: pdmurray Date: Wed, 14 Aug 2024 09:32:35 -0700 Subject: [PATCH 5/5] Fix upload/download-artifact v4 breaking changes --- .github/workflows/wheels.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 8beb876e2c..cd271a390f 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -37,7 +37,7 @@ jobs: - name: Upload sdist uses: actions/upload-artifact@v4 with: - name: wheels + name: sdist path: ./wheelhouse/*.tar.gz build_wheels: @@ -97,7 +97,7 @@ jobs: - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-${{ matrix.python-version }}-${{ matrix.os }} path: ./wheelhouse/*.whl upload_to_pypi: @@ -113,6 +113,8 @@ jobs: steps: - name: Retrieve wheels and sdist uses: actions/download-artifact@v4 + merge-multiple: true + path: wheels/ - name: List the build artifacts run: |