feat: ProverJobProcessor & circuit prover (#3287) #1965
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and release Stage | |
| on: | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name != 'push' || github.ref != 'refs/heads/main' }} | |
| jobs: | |
| changed_files: | |
| runs-on: ubuntu-latest | |
| name: Test changed-files | |
| outputs: | |
| core: ${{ steps.changed-files-yaml.outputs.core_any_changed }} | |
| prover: ${{ steps.changed-files-yaml.outputs.prover_any_changed }} | |
| all: ${{ steps.changed-files-yaml.outputs.all_any_changed }} | |
| steps: | |
| - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 | |
| with: | |
| fetch-depth: 2 | |
| - name: Get all test, doc and src files that have changed | |
| id: changed-files-yaml | |
| uses: tj-actions/changed-files@fea790cb660e33aef4bdf07304e28fedd77dfa13 # v39 | |
| with: | |
| files_yaml: | | |
| # TODO: make it more granular, as already implemented in CI workflow | |
| # We don't want to be rebuilding and redeploying all the Docker images when eg. only document have changed | |
| prover: | |
| - prover/** | |
| - core/lib/** | |
| - '!core/lib/zksync_core/**' | |
| core: | |
| - core/** | |
| all: | |
| - '!core/**' | |
| - '!prover/**' | |
| setup: | |
| name: Setup | |
| runs-on: [matterlabs-deployer-stage] | |
| outputs: | |
| image_tag_suffix: ${{ steps.generate-tag-suffix.outputs.image_tag_suffix }} | |
| prover_fri_gpu_key_id: ${{ steps.extract-prover-fri-setup-key-ids.outputs.gpu_short_commit_sha }} | |
| steps: | |
| - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 | |
| - name: Generate image tag suffix | |
| id: generate-tag-suffix | |
| run: | | |
| sha=$(git rev-parse --short HEAD) | |
| ts=$(date +%s%N | cut -b1-13) | |
| echo "image_tag_suffix=${sha}-${ts}" >> $GITHUB_OUTPUT | |
| - name: Generate outputs with Prover FRI setup data keys IDs | |
| id: extract-prover-fri-setup-key-ids | |
| run: | | |
| ./prover/extract-setup-data-keys.sh >> $GITHUB_OUTPUT | |
| build-push-core-images: | |
| name: Build and push images | |
| needs: [setup, changed_files] | |
| uses: ./.github/workflows/new-build-core-template.yml | |
| if: needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.all == 'true' | |
| with: | |
| image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} | |
| action: "push" | |
| secrets: | |
| DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| build-push-tee-prover-images: | |
| name: Build and push images | |
| needs: [setup, changed_files] | |
| uses: ./.github/workflows/build-tee-prover-template.yml | |
| if: needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.all == 'true' | |
| with: | |
| image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} | |
| secrets: | |
| DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} | |
| build-push-contract-verifier: | |
| name: Build and push images | |
| needs: [setup, changed_files] | |
| uses: ./.github/workflows/new-build-contract-verifier-template.yml | |
| if: needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.all == 'true' | |
| with: | |
| image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} | |
| action: "push" | |
| secrets: | |
| DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| build-push-prover-images: | |
| name: Build and push images | |
| needs: [setup, changed_files] | |
| uses: ./.github/workflows/new-build-prover-template.yml | |
| if: needs.changed_files.outputs.prover == 'true' || needs.changed_files.outputs.all == 'true' | |
| with: | |
| image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} | |
| ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }} | |
| CUDA_ARCH: "60;70;75;80;89" | |
| action: "push" | |
| secrets: | |
| DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| build-push-witness-generator-image-avx512: | |
| name: Build and push prover images with avx512 instructions | |
| needs: [setup, changed_files] | |
| uses: ./.github/workflows/new-build-witness-generator-template.yml | |
| if: needs.changed_files.outputs.prover == 'true' || needs.changed_files.outputs.all == 'true' | |
| with: | |
| image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512 | |
| ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }} | |
| CUDA_ARCH: "60;70;75;80;89" | |
| WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl " | |
| action: "push" | |
| secrets: | |
| DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| build-gar-prover-fri-gpu-and-circuit-prover-gpu-gar: | |
| name: Build GAR prover FRI GPU | |
| needs: [setup, build-push-prover-images] | |
| uses: ./.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml | |
| if: needs.changed_files.outputs.prover == 'true' || needs.changed_files.outputs.all == 'true' | |
| with: | |
| setup_keys_id: ${{ needs.setup.outputs.prover_fri_gpu_key_id }} | |
| image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} | |
| protocol_version: ${{ needs.build-push-prover-images.outputs.protocol_version }} |