diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index e998ccbd8741..03f46ddab6b0 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -57,10 +57,9 @@ jobs: outputs: image-tag: ${{ github.event.pull_request.head.sha || github.sha }} python-versions: ${{ steps.selective-checks.outputs.python-versions }} - upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} - all-python-versions-list-as-string: >- - ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} + python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} + upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} run-tests: ${{ steps.selective-checks.outputs.run-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} @@ -81,7 +80,6 @@ jobs: github.event.pull_request.head.sha || github.sha }}" - if: github.repository == 'apache/airflow' steps: - name: "Cleanup repo" shell: bash @@ -168,9 +166,7 @@ jobs: needs: [build-info] uses: ./.github/workflows/ci-image-build.yml # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow" - if: | - needs.build-info.outputs.ci-image-build == 'true' && - github.event.pull_request.head.repo.full_name != 'apache/airflow' + if: needs.build-info.outputs.ci-image-build == 'true' with: runs-on: '["ubuntu-22.04"]' do-build: ${{ needs.build-info.outputs.ci-image-build }} @@ -178,7 +174,6 @@ jobs: pull-request-target: "true" is-committer-build: ${{ needs.build-info.outputs.is-committer-build }} push-image: "true" - upload-constraints: "true" use-uv: "true" image-tag: ${{ needs.build-info.outputs.image-tag }} platform: "linux/amd64" @@ -188,18 +183,26 @@ jobs: upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} + generate-constraints: + name: "Generate constraints" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/generate-constraints.yml + with: + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + image-tag: ${{ needs.build-info.outputs.image-tag }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + build-prod-images: name: Build PROD images permissions: contents: read packages: write secrets: inherit - needs: [build-info, build-ci-images] + needs: [build-info, generate-constraints] uses: ./.github/workflows/prod-image-build.yml # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow" - if: | - needs.build-info.outputs.prod-image-build == 'true' && - github.event.pull_request.head.repo.full_name != 'apache/airflow' + if: needs.build-info.outputs.prod-image-build == 'true' with: runs-on: '["ubuntu-22.04"]' build-type: "Regular" @@ -211,7 +214,6 @@ jobs: push-image: "true" use-uv: "true" image-tag: ${{ needs.build-info.outputs.image-tag }} - ci-image-tag: ${{ needs.build-info.outputs.image-tag }} platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index d28a34daf5ac..75a611ba55ab 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -56,10 +56,6 @@ on: # yamllint disable-line rule:truthy required: false default: "true" type: string - upload-constraints: - description: "Whether to upload constraints artifacts" - default: "false" - type: string debian-version: description: "Base Debian distribution to use for the build (bookworm/bullseye)" type: string @@ -216,17 +212,3 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh if: always() && inputs.do-build == 'true' && inputs.platform == 'linux/arm64' - - name: "Source constraints: ${{ matrix.python-version }}" - shell: bash - run: > - breeze release-management generate-constraints --python "${{ matrix.python-version }}" - --airflow-constraints-mode constraints-source-providers --answer yes - if: inputs.do-build == 'true' && inputs.upload-constraints == 'true' - - name: "Upload constraint artifacts" - uses: actions/upload-artifact@v4 - with: - name: source-constraints-${{ matrix.python-version }} - path: ./files/constraints-*/constraints-source-providers-*.txt - retention-days: 7 - if-no-files-found: error - if: inputs.do-build == 'true' && inputs.upload-constraints == 'true' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cea684bd2fb3..8b86ce8a6eb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,7 +188,6 @@ jobs: image-tag: ${{ needs.build-info.outputs.image-tag }} platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} - upload-constraints: "true" branch: ${{ needs.build-info.outputs.default-branch }} use-uv: "true" upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} @@ -254,43 +253,38 @@ jobs: include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} - static-checks-mypy-and-constraints-generation: - name: "Static checks, mypy and constraints generation" + + generate-constraints: + name: "Generate constraints" needs: [build-info, wait-for-ci-images] - uses: ./.github/workflows/static-checks-mypy-and-constraints-generation.yml + uses: ./.github/workflows/generate-constraints.yml + if: needs.build-info.outputs.ci-image-build == 'true' + with: + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + image-tag: ${{ needs.build-info.outputs.image-tag }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + + static-checks-mypy-docs: + name: "Static checks, mypy, docs" + needs: [build-info, wait-for-ci-images] + uses: ./.github/workflows/static-checks-mypy-docs.yml with: runs-on: ${{ needs.build-info.outputs.runs-on }} image-tag: ${{ needs.build-info.outputs.image-tag }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} branch: ${{ needs.build-info.outputs.default-branch }} + canary-run: ${{ needs.build-info.outputs.canary-run }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} + docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }} latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} ci-image-build: ${{ needs.build-info.outputs.ci-image-build }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - - docs: - name: "Docs" - uses: ./.github/workflows/docs.yml - needs: [build-info, wait-for-ci-images] - permissions: - contents: read - packages: read - secrets: inherit - if: > - needs.build-info.outputs.docs-build == 'true' && - needs.build-info.outputs.latest-versions-only != 'true' - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - image-tag: ${{ needs.build-info.outputs.image-tag }} - default-branch: ${{ needs.build-info.outputs.default-branch }} - docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }} - default-python-version: ${{ needs.build-info.outputs.default-python-version }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - canary-run: ${{ needs.build-info.outputs.canary-run }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} providers: name: "Provider checks" @@ -475,7 +469,7 @@ jobs: name: > ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }} PROD images in-workflow - needs: [build-info, build-ci-images] + needs: [build-info, build-ci-images, generate-constraints] uses: ./.github/workflows/prod-image-build.yml permissions: contents: read @@ -491,7 +485,6 @@ jobs: do-build: ${{ needs.build-info.outputs.in-workflow-build }} upload-package-artifact: "true" image-tag: ${{ needs.build-info.outputs.image-tag }} - ci-image-tag: ${{ needs.build-info.outputs.image-tag }} platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} @@ -549,7 +542,7 @@ jobs: additional-prod-image-tests: name: "Additional PROD image tests" - needs: [build-info, wait-for-prod-images, static-checks-mypy-and-constraints-generation] + needs: [build-info, wait-for-prod-images, generate-constraints] uses: ./.github/workflows/additional-prod-image-tests.yml with: runs-on: ${{ needs.build-info.outputs.runs-on }} @@ -591,16 +584,16 @@ jobs: secrets: inherit needs: - build-info - - docs + - generate-constraints - wait-for-ci-images - wait-for-prod-images + - static-checks-mypy-docs - tests-sqlite - tests-mysql - tests-postgres - tests-non-db - tests-special - tests-integration - - static-checks-mypy-and-constraints-generation uses: ./.github/workflows/finalize-tests.yml with: runs-on: ${{ needs.build-info.outputs.runs-on }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index f5fdcfa4c4af..000000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,130 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: Documentation -on: # yamllint disable-line rule:truthy - workflow_call: - inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' - type: string - image-tag: - description: "Tag to set for the image" - required: true - type: string - default-branch: - description: "Default branch of the repository" - required: true - type: string - docs-list-as-string: - description: "Stringified list of docs to build (space separated)" - required: true - type: string - default-python-version: - description: "Which version of python should be used by default" - required: true - type: string - include-success-outputs: - description: "Whether to include success outputs" - required: true - type: string - canary-run: - description: "Whether this is a canary run (true/false)" - required: true - type: string -jobs: - build-docs: - timeout-minutes: 60 - name: "Build documentation" - runs-on: ${{ fromJSON(inputs.runs-on) }} - strategy: - fail-fast: false - matrix: - flag: ["--docs-only", "--spellcheck-only"] - env: - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_USERNAME: ${{ github.actor }} - IMAGE_TAG: "${{ inputs.image-tag }}" - INCLUDE_NOT_READY_PROVIDERS: "true" - INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}" - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - VERBOSE: "true" - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}" - uses: ./.github/actions/prepare_breeze_and_image - - uses: actions/cache@v4 - id: cache-doc-inventories - with: - path: ./docs/_inventory_cache/ - key: docs-inventory-${{ hashFiles('pyproject.toml;') }} - restore-keys: | - docs-inventory-${{ hashFiles('pyproject.toml;') }} - docs-inventory- - - name: "Building docs with ${{ matrix.flag }} flag" - run: > - breeze build-docs ${{ inputs.docs-list-as-string }} ${{ matrix.flag }} - - name: "Clone airflow-site" - run: > - git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site && - echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV" - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Publish docs" - run: > - breeze release-management publish-docs --override-versioned --run-in-parallel - ${{ inputs.docs-list-as-string }} - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Generate back references for providers" - run: breeze release-management add-back-references all-providers - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Generate back references for apache-airflow" - run: breeze release-management add-back-references apache-airflow - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Generate back references for docker-stack" - run: breeze release-management add-back-references docker-stack - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Generate back references for helm-chart" - run: breeze release-management add-back-references helm-chart - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - if: > - inputs.canary-run == 'true' && - inputs.default-branch == 'main' && - matrix.flag == '--docs-only' - with: - aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-central-1 - - name: "Upload documentation to AWS S3" - if: > - inputs.canary-run == 'true' && - inputs.default-branch == 'main' && - matrix.flag == '--docs-only' - run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml new file mode 100644 index 000000000000..f7930a150440 --- /dev/null +++ b/.github/workflows/generate-constraints.yml @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Generate constraints +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + python-versions-list-as-string: + description: "Stringified array of all Python versions to test - separated by spaces." + required: true + type: string + image-tag: + description: "Tag to set for the image" + required: true + type: string + chicken-egg-providers: + description: "Space-separated list of providers that should be installed from context files" + required: true + type: string + debug-resources: + description: "Whether to run in debug mode (true/false)" + required: true + type: string +jobs: + generate-constraints: + permissions: + contents: read + timeout-minutes: 70 + name: Generate constraints ${{ inputs.python-versions-list-as-string }} + runs-on: ['ubuntu-22.04'] + env: + DEBUG_RESOURCES: ${{ inputs.debug-resources }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + INCLUDE_SUCCESS_OUTPUTS: "true" + IMAGE_TAG: ${{ inputs.image-tag }} + PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }} + VERBOSE: "true" + VERSION_SUFFIX_FOR_PYPI: "dev0" + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: Login to ghcr.io + run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: "\ + Pull CI images \ + ${{ inputs.python-versions-list-as-string }}:\ + ${{ inputs.image-tag }}" + run: breeze ci-image pull --run-in-parallel --tag-as-latest + - name: " + Verify CI images \ + ${{ inputs.python-versions-list-as-string }}:\ + ${{ inputs.image-tag }}" + run: breeze ci-image verify --run-in-parallel + - name: "Source constraints" + shell: bash + run: > + breeze release-management generate-constraints --run-in-parallel + --airflow-constraints-mode constraints-source-providers --answer yes + - name: "No providers constraints" + shell: bash + timeout-minutes: 25 + run: > + breeze release-management generate-constraints --run-in-parallel + --airflow-constraints-mode constraints-no-providers --answer yes + - name: "Prepare chicken-eggs provider packages" + # In case of provider packages which use latest dev0 version of providers, we should prepare them + # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency + # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 + shell: bash + run: > + breeze release-management prepare-provider-packages --include-not-ready-providers + --package-format wheel --version-suffix-for-pypi dev0 + ${{ inputs.chicken-egg-providers }} + if: inputs.chicken-egg-providers != '' + - name: "PyPI constraints" + shell: bash + timeout-minutes: 25 + run: > + breeze release-management generate-constraints --run-in-parallel + --airflow-constraints-mode constraints --answer yes + --chicken-egg-providers "${{ inputs.chicken-egg-providers }}" + - name: "Dependency upgrade summary" + shell: bash + run: | + for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do + echo "Summarizing Python $PYTHON_VERSION" + cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true + done + - name: "Upload constraint artifacts" + uses: actions/upload-artifact@v4 + with: + name: constraints + path: ./files/constraints-*/constraints-*.txt + retention-days: 7 + if-no-files-found: error diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 4180516983ce..ad9c35a445af 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -79,10 +79,6 @@ on: # yamllint disable-line rule:truthy description: "Tag to set for the image" required: true type: string - ci-image-tag: - description: "Tag to use for the CI image used during the build" - required: true - type: string python-versions: description: "JSON-formatted array of Python versions to build images from" required: true @@ -295,39 +291,16 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" name: prod-packages path: ./docker-context-files if: inputs.do-build == 'true' - - name: "Download constraints from the CI build" + - name: "Download constraints" uses: actions/download-artifact@v4 with: - name: source-constraints-${{ matrix.python-version }} + name: constraints path: ./docker-context-files - if: inputs.do-build == 'true' && inputs.build-provider-packages == 'true' + if: inputs.do-build == 'true' - name: Login to ghcr.io shell: bash run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin if: inputs.do-build == 'true' - - name: Pull CI images ${{ matrix.python-version }}:${{ inputs.ci-image-tag }} - run: > - breeze ci-image pull --tag-as-latest --image-tag "${{ inputs.ci-image-tag }}" - --python "${{ matrix.python-version }}" - if: inputs.do-build == 'true' && inputs.build-provider-packages != 'true' - - name: "PyPI constraints" - timeout-minutes: 25 - run: > - breeze release-management generate-constraints --python "${{ matrix.python-version }}" - --airflow-constraints-mode constraints --answer yes - --chicken-egg-providers "${{ inputs.chicken-egg-providers }}" - env: - # always use UV for constraints generation even if we build `pip` image - USE_UV: "true" - if: inputs.do-build == 'true' && inputs.build-provider-packages != 'true' - # This cleanup is needed as we will run out of disk space if we keep both CI and PROD images - # on public runners. - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - if: inputs.do-build == 'true' && inputs.build-provider-packages != 'true' - - name: "Copy constraints to docker-context-files" - run: cp -vr --no-preserve=mode,ownership ./files/constraints-* ./docker-context-files - if: inputs.do-build == 'true' && inputs.build-provider-packages != 'true' - name: "Build PROD images w/ source providers ${{ matrix.python-version }}:${{ inputs.image-tag }}" shell: bash run: > diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index 7fc010f36f2e..48469af793d8 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -62,7 +62,6 @@ jobs: build-type: "Bullseye" upload-package-artifact: "false" image-tag: bullseye-${{ inputs.image-tag }} - ci-image-tag: ${{ inputs.image-tag }} debian-version: "bullseye" python-versions: ${{ inputs.python-versions }} default-python-version: ${{ inputs.default-python-version }} @@ -83,7 +82,6 @@ jobs: build-type: "MySQL Client" upload-package-artifact: "false" image-tag: mysql-${{ inputs.image-tag }} - ci-image-tag: ${{ inputs.image-tag }} install-mysql-client-type: "mysql" python-versions: ${{ inputs.python-versions }} default-python-version: ${{ inputs.default-python-version }} @@ -104,7 +102,6 @@ jobs: build-type: "pip" upload-package-artifact: "false" image-tag: mysql-${{ inputs.image-tag }} - ci-image-tag: ${{ inputs.image-tag }} install-mysql-client-type: "mysql" python-versions: ${{ inputs.python-versions }} default-python-version: ${{ inputs.default-python-version }} diff --git a/.github/workflows/static-checks-mypy-and-constraints-generation.yml b/.github/workflows/static-checks-mypy-docs.yml similarity index 64% rename from .github/workflows/static-checks-mypy-and-constraints-generation.yml rename to .github/workflows/static-checks-mypy-docs.yml index b0e0c69bbccf..60cc4b4b1cfb 100644 --- a/.github/workflows/static-checks-mypy-and-constraints-generation.yml +++ b/.github/workflows/static-checks-mypy-docs.yml @@ -16,7 +16,7 @@ # under the License. # --- -name: Static checks, mypy and constraints generation +name: Static checks, mypy, docs on: # yamllint disable-line rule:truthy workflow_call: inputs: @@ -37,10 +37,18 @@ on: # yamllint disable-line rule:truthy description: "Branch used to run the CI jobs in (main/v2_*_test)." required: true type: string + canary-run: + description: "Whether this is a canary run (true/false)" + required: true + type: string default-python-version: description: "Which version of python should be used by default" required: true type: string + docs-list-as-string: + description: "Stringified list of docs to build (space separated)" + required: true + type: string upgrade-to-newer-dependencies: description: "Whether to upgrade to newer dependencies (true/false)" required: true @@ -65,88 +73,15 @@ on: # yamllint disable-line rule:truthy description: "Whether to skip pre-commits (true/false)" required: true type: string + include-success-outputs: + description: "Whether to include success outputs" + required: true + type: string debug-resources: description: "Whether to debug resources (true/false)" required: true type: string jobs: - generate-constraints: - permissions: - contents: read - timeout-minutes: 70 - name: Generate constraints ${{ inputs.python-versions-list-as-string }} - runs-on: ['ubuntu-22.04'] - env: - DEBUG_RESOURCES: ${{ inputs.debug-resources }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_USERNAME: ${{ github.actor }} - INCLUDE_SUCCESS_OUTPUTS: "true" - PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }} - VERBOSE: "true" - VERSION_SUFFIX_FOR_PYPI: "dev0" - if: inputs.ci-image-build == 'true' - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: Login to ghcr.io - run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: Pull CI images ${{ inputs.python-versions-list-as-string }}:${{ inputs.image-tag }} - run: breeze ci-image pull --run-in-parallel --tag-as-latest - - name: Verify CI images ${{ inputs.python-versions-list-as-string }}:${{ inputs.image-tag }} - run: breeze ci-image verify --run-in-parallel - - name: "Source constraints" - shell: bash - run: > - breeze release-management generate-constraints --run-in-parallel - --airflow-constraints-mode constraints-source-providers --answer yes - - name: "No providers constraints" - shell: bash - timeout-minutes: 25 - run: > - breeze release-management generate-constraints --run-in-parallel - --airflow-constraints-mode constraints-no-providers --answer yes - - name: "Prepare chicken-eggs provider packages" - # In case of provider packages which use latest dev0 version of providers, we should prepare them - # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency - # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 - shell: bash - run: > - breeze release-management prepare-provider-packages --include-not-ready-providers - --package-format wheel --version-suffix-for-pypi dev0 - ${{ inputs.chicken-egg-providers }} - if: inputs.chicken-egg-providers != '' - - name: "PyPI constraints" - shell: bash - timeout-minutes: 25 - run: > - breeze release-management generate-constraints --run-in-parallel - --airflow-constraints-mode constraints --answer yes - --chicken-egg-providers "${{ inputs.chicken-egg-providers }}" - - name: "Dependency upgrade summary" - shell: bash - run: | - for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do - echo "Summarizing Python $PYTHON_VERSION" - cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true - done - - name: "Upload constraint artifacts" - uses: actions/upload-artifact@v4 - with: - name: constraints - path: ./files/constraints-*/constraints-*.txt - retention-days: 7 - if-no-files-found: error - static-checks: timeout-minutes: 45 name: "Static checks" @@ -229,6 +164,85 @@ jobs: VERBOSE: "false" COLUMNS: "250" SKIP_GROUP_OUTPUT: "true" - DEFAULT_BRANCH: ${{ inputs.default-branch }} + DEFAULT_BRANCH: ${{ inputs.branch }} RUFF_FORMAT: "github" INCLUDE_MYPY_VOLUME: "false" + + build-docs: + timeout-minutes: 60 + name: "Build documentation" + runs-on: ${{ fromJSON(inputs.runs-on) }} + strategy: + fail-fast: false + matrix: + flag: ["--docs-only", "--spellcheck-only"] + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + IMAGE_TAG: "${{ inputs.image-tag }}" + INCLUDE_NOT_READY_PROVIDERS: "true" + INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}" + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + VERBOSE: "true" + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}" + uses: ./.github/actions/prepare_breeze_and_image + - uses: actions/cache@v4 + id: cache-doc-inventories + with: + path: ./docs/_inventory_cache/ + key: docs-inventory-${{ hashFiles('pyproject.toml;') }} + restore-keys: | + docs-inventory-${{ hashFiles('pyproject.toml;') }} + docs-inventory- + - name: "Building docs with ${{ matrix.flag }} flag" + run: > + breeze build-docs ${{ inputs.docs-list-as-string }} ${{ matrix.flag }} + - name: "Clone airflow-site" + run: > + git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site && + echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV" + if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + - name: "Publish docs" + run: > + breeze release-management publish-docs --override-versioned --run-in-parallel + ${{ inputs.docs-list-as-string }} + if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + - name: "Generate back references for providers" + run: breeze release-management add-back-references all-providers + if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + - name: "Generate back references for apache-airflow" + run: breeze release-management add-back-references apache-airflow + if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + - name: "Generate back references for docker-stack" + run: breeze release-management add-back-references docker-stack + if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + - name: "Generate back references for helm-chart" + run: breeze release-management add-back-references helm-chart + if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + if: > + inputs.canary-run == 'true' && + inputs.branch == 'main' && + matrix.flag == '--docs-only' + with: + aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} + aws-region: eu-central-1 + - name: "Upload documentation to AWS S3" + if: > + inputs.canary-run == 'true' && + inputs.branch == 'main' && + matrix.flag == '--docs-only' + run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs diff --git a/hatch_build.py b/hatch_build.py index ac4b54b185ec..0f4f5ec48c37 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations + import itertools import json import logging