Better fix for constraint generation dependency for PROD build

When we build PROD images in CI, we need constraints, to know which
version of the depencies to use. Those constraints are build by
earlier steps in the build and uploaded as artifacts so that they
can be re-used by PROD image build. However depending on the type
of PROD build (in main or in v2-*) different constraints are used:
for main build we use source constraints because we are installing
provider packages build from main sources, in v2* builds we are using
PyPI constraints, because we are installing provider packages from
PyPI. This made the dependency tree a bit complicated because PROD
builds would have to wait for constraints generation rather than
use quickly prepared source constraints and in apache#38533 we introduced
an extra step in PROD build for v2* branches to additionally generate
constraint in-the job that builds PROD images.

This however has some consequences, such as potentially incosistencies
between constraints used during other parts of the build, because PROD
images can be potentially built much later than the generated
constraints for the rest of the build. Also it introduced the need
for the PROD builds to pull CI images to generate the constraints
which adds an overhead in the process and makes PROD builds in v2
branches few minutes longer than they could be.

This PR approaches it differently. Since now with `uv` constraints
generation is way faster (few minutes, we could easily make PROD builds
use always the same constraint artifacts generated in single job and
pay a little delay penalty of starting the PROD builds but get it back
at PROD build time (because constraints will be generated only once).

Also that removes (back) the need of PROD builds to pull the CI images
and removes the need to generate source constraints at CI image build
time - which will decrease the build time a little.

This PR makes necessary changes to make it happens:

* removes source constraints generation in CI images
* extracting generate-constraints job to be separate workflow and
  merging "docs" instead (we are hitting the limit of 20 workflows)
* add dependency from all PROD build to wait for generate-constraints
* remove CI image pulling and related steps from PROD image build
* switches PROD to download and use the full set of constraints
  generated by `generate-constraints` job

.github/workflows/build-images.yml

@@ -57,10 +57,9 @@ jobs:
     outputs:
       image-tag: ${{ github.event.pull_request.head.sha || github.sha }}
       python-versions: ${{ steps.selective-checks.outputs.python-versions }}
-      upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
-      all-python-versions-list-as-string: >-
-        ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
+      python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
       default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
+      upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
       run-tests: ${{ steps.selective-checks.outputs.run-tests }}
       run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
       ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
@@ -178,7 +177,6 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      upload-constraints: "true"
       use-uv: "true"
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: "linux/amd64"
@@ -188,13 +186,23 @@ jobs:
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}
 
+  generate-constraints:
+    name: "Generate constraints"
+    needs: [build-info, build-ci-images]
+    uses: ./.github/workflows/generate-constraints.yml
+    with:
+      python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+      image-tag: ${{ needs.build-info.outputs.image-tag }}
+      chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+      debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+
   build-prod-images:
     name: Build PROD images
     permissions:
       contents: read
       packages: write
     secrets: inherit
-    needs: [build-info, build-ci-images]
+    needs: [build-info, generate-constraints]
     uses: ./.github/workflows/prod-image-build.yml
     # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow"
     if: |
@@ -211,7 +219,6 @@ jobs:
       push-image: "true"
       use-uv: "true"
       image-tag: ${{ needs.build-info.outputs.image-tag }}
-      ci-image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}

.github/workflows/ci-image-build.yml

@@ -56,10 +56,6 @@ on:  # yamllint disable-line rule:truthy
         required: false
         default: "true"
         type: string
-      upload-constraints:
-        description: "Whether to upload constraints artifacts"
-        default: "false"
-        type: string
       debian-version:
         description: "Base Debian distribution to use for the build (bookworm/bullseye)"
         type: string
@@ -216,17 +212,3 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
       - name: "Stop ARM instance"
         run: ./scripts/ci/images/ci_stop_arm_instance.sh
         if: always() && inputs.do-build == 'true' && inputs.platform == 'linux/arm64'
-      - name: "Source constraints: ${{ matrix.python-version }}"
-        shell: bash
-        run: >
-          breeze release-management generate-constraints --python "${{ matrix.python-version }}"
-          --airflow-constraints-mode constraints-source-providers --answer yes
-        if: inputs.do-build == 'true' && inputs.upload-constraints == 'true'
-      - name: "Upload constraint artifacts"
-        uses: actions/upload-artifact@v4
-        with:
-          name: source-constraints-${{ matrix.python-version }}
-          path: ./files/constraints-*/constraints-source-providers-*.txt
-          retention-days: 7
-          if-no-files-found: error
-        if: inputs.do-build == 'true' && inputs.upload-constraints == 'true'

.github/workflows/ci.yml

@@ -188,7 +188,6 @@ jobs:
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
-      upload-constraints: "true"
       branch: ${{ needs.build-info.outputs.default-branch }}
       use-uv: "true"
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
@@ -254,43 +253,38 @@ jobs:
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
 
-  static-checks-mypy-and-constraints-generation:
-    name: "Static checks, mypy and constraints generation"
+
+  generate-constraints:
+    name: "Generate constraints"
     needs: [build-info, wait-for-ci-images]
-    uses: ./.github/workflows/static-checks-mypy-and-constraints-generation.yml
+    uses: ./.github/workflows/generate-constraints.yml
+    if: needs.build-info.outputs.ci-image-build == 'true'
+    with:
+      python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+      image-tag: ${{ needs.build-info.outputs.image-tag }}
+      chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+      debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+
+  static-checks-mypy-docs:
+    name: "Static checks, mypy, docs"
+    needs: [build-info, wait-for-ci-images]
+    uses: ./.github/workflows/static-checks-mypy-docs.yml
     with:
       runs-on: ${{ needs.build-info.outputs.runs-on }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
       branch: ${{ needs.build-info.outputs.default-branch }}
+      canary-run: ${{ needs.build-info.outputs.canary-run }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }}
       latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }}
       basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
       ci-image-build: ${{ needs.build-info.outputs.ci-image-build }}
-      debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-
-  docs:
-    name: "Docs"
-    uses: ./.github/workflows/docs.yml
-    needs: [build-info, wait-for-ci-images]
-    permissions:
-      contents: read
-      packages: read
-    secrets: inherit
-    if: >
-      needs.build-info.outputs.docs-build == 'true' &&
-      needs.build-info.outputs.latest-versions-only != 'true'
-    with:
-      runs-on: ${{ needs.build-info.outputs.runs-on }}
-      image-tag: ${{ needs.build-info.outputs.image-tag }}
-      default-branch: ${{ needs.build-info.outputs.default-branch }}
-      docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }}
-      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
-      canary-run: ${{ needs.build-info.outputs.canary-run }}
+      debug-resources: ${{ needs.build-info.outputs.debug-resources }}
 
   providers:
     name: "Provider checks"
@@ -475,7 +469,7 @@ jobs:
     name: >
       ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }}
       PROD images in-workflow
-    needs: [build-info, build-ci-images]
+    needs: [build-info, build-ci-images, generate-constraints]
     uses: ./.github/workflows/prod-image-build.yml
     permissions:
       contents: read
@@ -491,7 +485,6 @@ jobs:
       do-build: ${{ needs.build-info.outputs.in-workflow-build }}
       upload-package-artifact: "true"
       image-tag: ${{ needs.build-info.outputs.image-tag }}
-      ci-image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
@@ -549,7 +542,7 @@ jobs:
 
   additional-prod-image-tests:
     name: "Additional PROD image tests"
-    needs: [build-info, wait-for-prod-images, static-checks-mypy-and-constraints-generation]
+    needs: [build-info, wait-for-prod-images, generate-constraints]
     uses: ./.github/workflows/additional-prod-image-tests.yml
     with:
       runs-on: ${{ needs.build-info.outputs.runs-on }}
@@ -591,16 +584,16 @@ jobs:
     secrets: inherit
     needs:
       - build-info
-      - docs
+      - generate-constraints
       - wait-for-ci-images
       - wait-for-prod-images
+      - static-checks-mypy-docs
       - tests-sqlite
       - tests-mysql
       - tests-postgres
       - tests-non-db
       - tests-special
       - tests-integration
-      - static-checks-mypy-and-constraints-generation
     uses: ./.github/workflows/finalize-tests.yml
     with:
       runs-on: ${{ needs.build-info.outputs.runs-on }}