Skip to content

Commit

Permalink
Merge branch 'main' into popzxc-auto-download-compilers
Browse files Browse the repository at this point in the history
  • Loading branch information
popzxc authored Nov 15, 2024
2 parents ca3e5a2 + 90b537b commit 109b699
Show file tree
Hide file tree
Showing 383 changed files with 3,820 additions and 2,042 deletions.
20 changes: 18 additions & 2 deletions .github/workflows/ci-core-reusable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -357,10 +357,17 @@ jobs:
run: |
ci_run zkstack dev test build
- name: Build tested binaries
run: |
ci_run zkstack server build
ci_run zkstack external-node build
ci_run zkstack contract-verifier build
- name: Initialize Contract verifier
run: |
ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
ci_run zkstack contract-verifier run --chain era &> ${{ env.SERVER_LOGS_DIR }}/contract-verifier-rollup.log &
ci_run zkstack contract-verifier wait --chain era --verbose
- name: Run servers
run: |
Expand All @@ -375,10 +382,14 @@ jobs:
--components=api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,vm_playground,da_dispatcher,consensus \
&> ${{ env.SERVER_LOGS_DIR }}/consensus.log &
ci_run sleep 5
ci_run zkstack server wait --ignore-prerequisites --verbose --chain era
ci_run zkstack server wait --ignore-prerequisites --verbose --chain validium
ci_run zkstack server wait --ignore-prerequisites --verbose --chain custom_token
ci_run zkstack server wait --ignore-prerequisites --verbose --chain consensus
- name: Setup attester committee for the consensus chain
- name: Set up attester committee for the consensus chain
run: |
ci_run zkstack consensus wait-for-registry --ignore-prerequisites --verbose --chain consensus
ci_run zkstack consensus set-attester-committee --chain consensus --from-genesis &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log
- name: Run integration tests
Expand Down Expand Up @@ -417,6 +428,11 @@ jobs:
ci_run zkstack external-node run --ignore-prerequisites --chain validium &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/validium.log &
ci_run zkstack external-node run --ignore-prerequisites --chain custom_token &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/custom_token.log &
ci_run zkstack external-node run --ignore-prerequisites --chain consensus --enable-consensus &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/consensus.log &
ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain era
ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain validium
ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain custom_token
ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain consensus
- name: Run integration tests en
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ jobs:
outputs:
core: ${{ steps.changed-files.outputs.core_any_changed }}
prover: ${{ steps.changed-files.outputs.prover_any_changed }}
zkstack_cli: ${{ steps.changed-files.outputs.zkstack_cli_any_changed }}
docs: ${{ steps.changed-files.outputs.docs_any_changed }}
all: ${{ steps.changed-files.outputs.all_any_changed }}
steps:
Expand Down Expand Up @@ -178,6 +177,7 @@ jobs:
image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512
action: "build"
WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl"
ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
secrets:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
Expand Down
67 changes: 67 additions & 0 deletions .github/workflows/deploy-core-docs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
name: Deploy core docs

on:
push:
branches:
- "main"
tags:
- "core-v*.*.*"
paths:
- 'docs/**'
- '.github/workflows/deploy-core-docs.yml'
pull_request:
paths:
- 'docs/**'
- '.github/workflows/deploy-core-docs.yml'
workflow_dispatch:
inputs:
ref:
description: "Branch, tag or commit to deploy the core docs. If empty, use the ref that triggered the workflow."
required: false
default: ""
version:
type: string
description: "Version of the documentation to deploy"
required: false
default: "latest"

# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
concurrency:
group: "pages"
cancel-in-progress: false

jobs:

deploy-core-docs:
runs-on: ubuntu-latest
permissions:
contents: write
env:
DOCS_DIR: 'docs'
PROJECT: 'core'
ENABLE_TESTS: false
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ inputs.ref || '' }}

- name: Extract version from tag
if: startsWith(github.ref, 'refs/tags/')
id: extract-version
shell: 'bash -ex {0}'
run: |
TAG="${{ github.ref_name }}"
VERSION="${TAG#*-}"
echo "version=${VERSION}" >> "${GITHUB_OUTPUT}"
- name: Deploy core docs
uses: matter-labs/deploy-mdbooks@73f638643d1be948d1002fe5433747f4a3e37a29 # v1
with:
version: ${{ inputs.version || steps.extract-version.outputs.version || github.ref_name }}
docs-dir: ${{ env.DOCS_DIR }}
github-token: ${{ secrets.GITHUB_TOKEN }}
enable-tests: ${{ env.ENABLE_TESTS }}
project: ${{ env.PROJECT }}
deploy: ${{ github.event_name != 'pull_request' }}
67 changes: 67 additions & 0 deletions .github/workflows/deploy-prover-docs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
name: Deploy prover docs

on:
push:
branches:
- "main"
tags:
- "prover-v*.*.*"
paths:
- 'prover/docs/**'
- '.github/workflows/deploy-prover-docs.yml'
pull_request:
paths:
- 'prover/docs/**'
- '.github/workflows/deploy-prover-docs.yml'
workflow_dispatch:
inputs:
ref:
description: "Branch, tag or commit to deploy the prover docs. If empty, use the ref that triggered the workflow."
required: false
default: ""
version:
type: string
description: "Version of the documentation to deploy"
required: false
default: "latest"

# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
concurrency:
group: "pages"
cancel-in-progress: false

jobs:

deploy-prover-docs:
runs-on: ubuntu-latest
permissions:
contents: write
env:
DOCS_DIR: 'prover/docs'
PROJECT: 'prover'
ENABLE_TESTS: false
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
ref: ${{ inputs.ref || '' }}

- name: Extract version from tag
if: startsWith(github.ref, 'refs/tags/')
id: extract-version
shell: 'bash -ex {0}'
run: |
TAG="${{ github.ref_name }}"
VERSION="${TAG#*-}"
echo "version=${VERSION}" >> "${GITHUB_OUTPUT}"
- name: Deploy prover docs
uses: matter-labs/deploy-mdbooks@73f638643d1be948d1002fe5433747f4a3e37a29 # v1
with:
version: ${{ inputs.version || steps.extract-version.outputs.version || github.ref_name }}
docs-dir: ${{ env.DOCS_DIR }}
github-token: ${{ secrets.GITHUB_TOKEN }}
enable-tests: ${{ env.ENABLE_TESTS }}
project: ${{ env.PROJECT }}
deploy: ${{ github.event_name != 'pull_request' }}
26 changes: 10 additions & 16 deletions .github/workflows/new-build-contract-verifier-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -212,31 +212,25 @@ jobs:
uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
with:
context: .
push: ${{ inputs.action == 'push' }}
load: true
platforms: ${{ matrix.platforms }}
file: docker/${{ matrix.components }}/Dockerfile
build-args: |
SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage
SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com
SCCACHE_GCS_RW_MODE=READ_WRITE
RUSTC_WRAPPER=sccache
tags: |
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:latest
matterlabs/${{ matrix.components }}:latest
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:latest2.0
matterlabs/${{ matrix.components }}:latest2.0
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA }}
matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA }}
matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}
matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
- name: Push docker image
if: ${{ inputs.action == 'push' }}
run: |
docker push us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
docker push matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
create_manifest:
name: Create release manifest
runs-on: matterlabs-ci-runner
Expand Down
31 changes: 11 additions & 20 deletions .github/workflows/new-build-core-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -219,35 +219,28 @@ jobs:
docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
gcloud auth configure-docker us-docker.pkg.dev -q
- name: Build and push
- name: Build docker image
uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
with:
context: .
push: ${{ inputs.action == 'push' }}
load: true
platforms: ${{ matrix.platforms }}
file: docker/${{ matrix.components }}/Dockerfile
build-args: |
SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage
SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com
SCCACHE_GCS_RW_MODE=READ_WRITE
RUSTC_WRAPPER=sccache
tags: |
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:latest
matterlabs/${{ matrix.components }}:latest
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:latest2.0
matterlabs/${{ matrix.components }}:latest2.0
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA }}
matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA }}
matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}
matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}
tags: |
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
- name: Push docker image
if: ${{ inputs.action == 'push' }}
run: |
docker push us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
docker push matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
create_manifest:
name: Create release manifest
runs-on: matterlabs-ci-runner
Expand All @@ -269,13 +262,11 @@ jobs:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4

- name: login to Docker registries
shell: bash
run: |
docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
gcloud auth configure-docker us-docker.pkg.dev -q
- name: Create Docker manifest
shell: bash
run: |
docker_repositories=("matterlabs/${{ matrix.component.name }}" "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component.name }}")
platforms=${{ matrix.component.platform }}
Expand Down
22 changes: 21 additions & 1 deletion .github/workflows/new-build-prover-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ jobs:
- prover-job-monitor
- proof-fri-gpu-compressor
- prover-autoscaler
- circuit-prover-gpu
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
Expand Down Expand Up @@ -152,17 +153,29 @@ jobs:
uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
with:
context: .
push: ${{ inputs.action == 'push' }}
load: true
build-args: |
CUDA_ARCH=${{ inputs.CUDA_ARCH }}
SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage
SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com
SCCACHE_GCS_RW_MODE=READ_WRITE
RUSTC_WRAPPER=sccache
PROTOCOL_VERSION=${{ env.PROTOCOL_VERSION }}
ERA_BELLMAN_CUDA_RELEASE=${{ inputs.ERA_BELLMAN_CUDA_RELEASE }}
file: docker/${{ matrix.components }}/Dockerfile
tags: |
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ env.IMAGE_TAG_SHA_TS }}
matterlabs/${{ matrix.components }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ env.IMAGE_TAG_SHA_TS }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
- name: Push docker image
if: ${{ inputs.action == 'push' }}
run: |
docker push us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ env.IMAGE_TAG_SHA_TS }}
docker push matterlabs/${{ matrix.components }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ env.IMAGE_TAG_SHA_TS }}
docker push us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
docker push matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
copy-images:
name: Copy images between docker registries
Expand All @@ -189,10 +202,17 @@ jobs:
docker buildx imagetools create \
--tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ inputs.image_tag_suffix }} \
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ inputs.image_tag_suffix }}
docker buildx imagetools create \
--tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} \
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }}
- name: Login and push to Europe GAR
run: |
gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
docker buildx imagetools create \
--tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ inputs.image_tag_suffix }} \
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ inputs.image_tag_suffix }}
docker buildx imagetools create \
--tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} \
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }}
9 changes: 9 additions & 0 deletions .github/workflows/new-build-witness-generator-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ on:
description: "DOCKERHUB_TOKEN"
required: true
inputs:
ERA_BELLMAN_CUDA_RELEASE:
description: "ERA_BELLMAN_CUDA_RELEASE"
type: string
required: true
image_tag_suffix:
description: "Optional suffix to override tag name generation"
type: string
Expand Down Expand Up @@ -127,7 +131,12 @@ jobs:
SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com
SCCACHE_GCS_RW_MODE=READ_WRITE
RUSTC_WRAPPER=sccache
PROTOCOL_VERSION=${{ env.PROTOCOL_VERSION }}
ERA_BELLMAN_CUDA_RELEASE=${{ inputs.ERA_BELLMAN_CUDA_RELEASE }}
RUST_FLAGS=${{ inputs.WITNESS_GENERATOR_RUST_FLAGS }}
file: docker/${{ matrix.components }}/Dockerfile
tags: |
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ env.IMAGE_TAG_SHA_TS }}
matterlabs/${{ matrix.components }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ env.IMAGE_TAG_SHA_TS }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
Loading

0 comments on commit 109b699

Please sign in to comment.