refactor: script refinements and regression check in CI (#52)

* chore: lint code * fix: fund the claim tx manager address * chore: update tool check script * chore: create keystores in a different script * chore: lint * fix: typo * fix: typos * fix: another typo * chore: same * fix: typo * chore: reorganize templates * chore: lint * chore: clean up * fix: typo * chore: nit * fix: keystore output dir * chore: more clean up * chore: clean up * chore: add timestamps to run-contract-setup * chore: clean up * chore: nit * chore: add more logs * fix: issue * test: more attemps and higher frequency * fix: deploy script * fix: retries * fix: typos * fix: typo * fix: typo * fix: typo * ci: check that batches are being verified in ci * fix: typo * chore: faster l1 account funding * fix: install foundry and use timeout env var * feat: make cast calls in parallel * fix: fund * fix: missed a cast cmd * fix: private key when funding * fix: account funding * test: make calls in parallel * test: another one * chore: clean up * Revert "fix: typo" This reverts commit 977051d. * Reapply "fix: typo" This reverts commit e088f4f.
0xPolygon · Apr 12, 2024 · d5bb810 · d5bb810
1 parent f1f29d1
commit d5bb810
Show file tree

Hide file tree

Showing 21 changed files with 443 additions and 287 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -10,6 +10,9 @@ concurrency:
   group: deploy-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  CHECK_VERIFIED_BATCHES_TIMEOUT_SECONDS: "300" # 5 minutes
+
 jobs:
   # Deploy the CDK environment in one step, with the gas token feature enabled.
   monolithic_cdk:
@@ -26,16 +29,44 @@ jobs:
           kurtosis analytics disable
 
       - name: Install yq
-        run: |
-          pip3 install yq
+        run: pip3 install yq
 
-      # Deploy components
+      - name: Install foundry
+        uses: foundry-rs/foundry-toolchain@v1
+
+      # Deploy components.
       - name: Enable gas token feature
         run: yq -Y --in-place '.zkevm_use_gas_token_contract = true' params.yml
 
       - name: Deploy Kurtosis CDK package
         run: kurtosis run --enclave cdk-v1 --args-file params.yml --image-download always .
 
+      # Check that batches are being verified.
+      - name: Check that batches are being verified
+        run: |
+          timeout_seconds="${CHECK_VERIFIED_BATCHES_TIMEOUT_SECONDS:-300}"
+          start_time=$(date +%s)
+          end_time=$((start_time + timeout_seconds))
+
+          export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
+          while true; do
+              current_time=$(date +%s)
+              if (( current_time > end_time )); then
+                  echo "[$(date '+%Y-%m-%d %H:%M:%S')] ❌ Exiting... Timeout reached. No batches were verified."
+                  exit 1
+              fi
+
+              verified_batches=$(cast rpc zkevm_verifiedBatchNumber | sed 's/"//g')
+              echo "[$(date '+%Y-%m-%d %H:%M:%S')] Verified Batches: $verified_batches"
+
+              if (( verified_batches > 0 )); then
+                  echo "[$(date '+%Y-%m-%d %H:%M:%S')] ✅ Exiting... At least one batch was verified."
+                  exit 0
+              fi
+
+              sleep 10
+          done
+
   # Deploy the CDK environment incrementally, stage by stage.
   incremental_cdk:
     runs-on: ubuntu-latest
@@ -51,8 +82,10 @@ jobs:
           kurtosis analytics disable
 
       - name: Install yq
-        run: |
-          pip3 install yq
+        run: pip3 install yq
+
+      - name: Install foundry
+        uses: foundry-rs/foundry-toolchain@v1
 
       # Deploy components.
       - name: Disable All Deployment Steps
@@ -106,3 +139,29 @@ jobs:
           yq -Y --in-place '.deploy_observability = true' params.yml
           kurtosis run --enclave cdk-v1 --args-file params.yml .
           yq -Y --in-place '.deploy_observability = false' params.yml # reset
+
+      # Check that batches are being verified.
+      - name: Check that batches are being verified
+        run: |
+          timeout_seconds="${CHECK_VERIFIED_BATCHES_TIMEOUT_SECONDS:-300}"
+          start_time=$(date +%s)
+          end_time=$((start_time + timeout_seconds))
+
+          export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
+          while true; do
+              current_time=$(date +%s)
+              if (( current_time > end_time )); then
+                  echo "[$(date '+%Y-%m-%d %H:%M:%S')] ❌ Exiting... Timeout reached. No batches were verified."
+                  exit 1
+              fi
+
+              verified_batches=$(cast rpc zkevm_verifiedBatchNumber | sed 's/"//g')
+              echo "[$(date '+%Y-%m-%d %H:%M:%S')] Verified Batches: $verified_batches"
+
+              if (( verified_batches > 0 )); then
+                  echo "[$(date '+%Y-%m-%d %H:%M:%S')] ✅ Exiting... At least one batch was verified."
+                  exit 0
+              fi
+
+              sleep 10
+          done
diff --git a/README.org b/README.org
@@ -161,7 +161,7 @@ should be populated already with the CDK fields like:
 - ~L1Config.polygonZkEVMAddress~
 
 If you're unsure how to populate these fields please check out how
-it's done within [[./templates/run-contract-setup.sh][run-constract-setup.sh]]. When you have the genesis
+it's done within [[./templates/contract-deploy/run-contract-setup.sh][run-constract-setup.sh]]. When you have the genesis
 file ready, drop it into [[./templates/permissionless-node/genesis.json]].
 
 In addition to the genesis setup, we'll also need to tweak a parameter

diff --git a/cdk_bridge_infra.star b/cdk_bridge_infra.star
@@ -59,7 +59,9 @@ def run(plan, args):
 
 
 def create_bridge_config_artifact(plan, args, contract_setup_addresses):
-    bridge_config_template = read_file(src="./templates/bridge-config.toml")
+    bridge_config_template = read_file(
+        src="./templates/bridge-infra/bridge-config.toml"
+    )
     return plan.render_templates(
         name="bridge-config-artifact",
         config={
@@ -87,7 +89,9 @@ def create_bridge_config_artifact(plan, args, contract_setup_addresses):
 
 
 def create_agglayer_config_artifact(plan, args, contract_setup_addresses):
-    agglayer_config_template = read_file(src="./templates/agglayer-config.toml")
+    agglayer_config_template = read_file(
+        src="./templates/bridge-infra/agglayer-config.toml"
+    )
     return plan.render_templates(
         name="agglayer-config-artifact",
         config={

diff --git a/cdk_central_environment.star b/cdk_central_environment.star
@@ -32,7 +32,16 @@ def run(plan, args):
     # Create the zkevm node config.
     node_config_template = read_file(src="./templates/trusted-node/node-config.toml")
     node_config_artifact = plan.render_templates(
-        config={"node-config.toml": struct(template=node_config_template, data=args)},
+        config={
+            "node-config.toml": struct(
+                template=node_config_template,
+                data=args
+                | {
+                    "is_cdk_validium": args["zkevm_rollup_consensus"]
+                    == "PolygonValidiumEtrog",
+                },
+            )
+        },
         name="trusted-node-config",
     )
 
@@ -90,7 +99,7 @@ def get_keystores_artifacts(plan, args):
 
 
 def create_dac_config_artifact(plan, args):
-    dac_config_template = read_file(src="./templates/dac-config.toml")
+    dac_config_template = read_file(src="./templates/trusted-node/dac-config.toml")
     contract_setup_addresses = service_package.get_contract_setup_addresses(plan, args)
     return plan.render_templates(
         name="dac-config-artifact",

diff --git a/deploy_zkevm_contracts.star b/deploy_zkevm_contracts.star
@@ -1,6 +1,8 @@
 def run(plan, args):
     # Create deploy parameters
-    deploy_parameters_template = read_file(src="./templates/deploy_parameters.json")
+    deploy_parameters_template = read_file(
+        src="./templates/contract-deploy/deploy_parameters.json"
+    )
     deploy_parameters_artifact = plan.render_templates(
         name="deploy-parameters-artifact",
         config={
@@ -12,7 +14,7 @@ def run(plan, args):
 
     # Create rollup paramaters
     create_rollup_parameters_template = read_file(
-        src="./templates/create_rollup_parameters.json"
+        src="./templates/contract-deploy/create_rollup_parameters.json"
     )
     create_rollup_parameters_artifact = plan.render_templates(
         name="create-rollup-parameters-artifact",
@@ -25,7 +27,7 @@ def run(plan, args):
 
     # Create contract deployment script
     contract_deployment_script_template = read_file(
-        src="./templates/run-contract-setup.sh"
+        src="./templates/contract-deploy/run-contract-setup.sh"
     )
     contract_deployment_script_artifact = plan.render_templates(
         name="contract-deployment-script-artifact",
@@ -36,12 +38,26 @@ def run(plan, args):
         },
     )
 
+    # Create keystores script
+    create_keystores_script_template = read_file(
+        src="./templates/contract-deploy/create-keystores.sh"
+    )
+    create_keystores_script_artifact = plan.render_templates(
+        name="create-keystores-script-artifact",
+        config={
+            "create-keystores.sh": struct(
+                template=create_keystores_script_template, data=args
+            )
+        },
+    )
+
     # Create helper service to deploy contracts
+    contracts_service_name = "contracts" + args["deployment_suffix"]
     zkevm_contracts_image = "{}:fork{}".format(
         args["zkevm_contracts_image"], args["zkevm_rollup_fork_id"]
     )
     plan.add_service(
-        name="contracts" + args["deployment_suffix"],
+        name=contracts_service_name,
         config=ServiceConfig(
             image=zkevm_contracts_image,
             files={
@@ -51,6 +67,7 @@ def run(plan, args):
                         deploy_parameters_artifact,
                         create_rollup_parameters_artifact,
                         contract_deployment_script_artifact,
+                        create_keystores_script_artifact,
                     ]
                 ),
             },
@@ -60,20 +77,38 @@ def run(plan, args):
 
     # TODO: Check if the contracts were already initialized.. I'm leaving this here for now, but it's not useful!!
     contract_init_stat = plan.exec(
-        service_name="contracts" + args["deployment_suffix"],
+        description="Checking if contracts are already initialized",
+        service_name=contracts_service_name,
         acceptable_codes=[0, 1],
         recipe=ExecRecipe(command=["stat", "/opt/zkevm/.init-complete.lock"]),
     )
 
-    # Deploy contracts
+    # Deploy contracts.
     plan.exec(
-        service_name="contracts" + args["deployment_suffix"],
+        description="Deploying zkevm contracts on L1",
+        service_name=contracts_service_name,
         recipe=ExecRecipe(
-            command=["chmod", "a+x", "/opt/contract-deploy/run-contract-setup.sh"]
+            command=[
+                "/bin/sh",
+                "-c",
+                "chmod +x {0} && {0}".format(
+                    "/opt/contract-deploy/run-contract-setup.sh"
+                ),
+            ]
         ),
     )
-    plan.print("Running zkEVM contract deployment. This might take some time...")
+
+    # Create keystores.
     plan.exec(
-        service_name="contracts" + args["deployment_suffix"],
-        recipe=ExecRecipe(command=["/opt/contract-deploy/run-contract-setup.sh"]),
+        description="Creating keystores for zkevm-node/cdk-validium components",
+        service_name=contracts_service_name,
+        recipe=ExecRecipe(
+            command=[
+                "/bin/sh",
+                "-c",
+                "chmod +x {0} && {0}".format(
+                    "/opt/contract-deploy/create-keystores.sh"
+                ),
+            ]
+        ),
     )
diff --git a/docs/gas-token.org b/docs/gas-token.org
@@ -20,7 +20,7 @@ edit the [[../params.yml][params.yml]] file to set ~zkevm_use_gas_token_contract
 setup:
 
 1. During contract deployment, an ERC20 token will be deployed on L1.
-2. The [[../templates/create_rollup_parameters.json][rollup creation parameters]] will be modified to set the
+2. The [[../templates/contract-deploy/create_rollup_parameters.json][rollup creation parameters]] will be modified to set the
    ~gasTokenAddress~ to the address of the L1 ERC20.
 
 After setting the ~zkevm_use_gas_token_contract~, you should be all

diff --git a/docs/zkevm-contracts.Dockerfile b/docs/zkevm-contracts.Dockerfile
@@ -1,7 +1,7 @@
 FROM golang:1.21 AS polycli-builder
 ARG POLYCLI_VERSION
 WORKDIR /opt/polygon-cli
-RUN git clone --branch $POLYCLI_VERSION https://github.com/maticnetwork/polygon-cli.git . \
+RUN git clone --branch ${POLYCLI_VERSION} https://github.com/maticnetwork/polygon-cli.git . \
   && CGO_ENABLED=0 go build -o polycli main.go
 
 

diff --git a/lib/service.star b/lib/service.star
@@ -13,7 +13,9 @@ def get_contract_setup_addresses(plan, args):
         },
     )
     result = plan.exec(
-        service_name="contracts" + args["deployment_suffix"], recipe=exec_recipe
+        description="Getting contract setup addresses",
+        service_name="contracts" + args["deployment_suffix"],
+        recipe=exec_recipe,
     )
     return get_exec_recipe_result(result)
 
@@ -27,18 +29,3 @@ def get_exec_recipe_result(result):
             new_key = key[len(key_prefix) :]
             result_dict[new_key] = value
     return result_dict
-
-
-# Run jq command on a file in a service.
-def _run_jq_on_service(plan, service_name, filename, jq_command):
-    plan.print("Extracting contract addresses and ports...")
-    exec_recipe = ExecRecipe(
-        command=[
-            "/bin/sh",
-            "-c",
-            "cat {}".format(filename),
-        ],
-        extract={"extracted_value": "fromjson | {}".format(jq_command)},
-    )
-    result = plan.exec(service_name=service_name, recipe=exec_recipe)
-    return result["extract.extracted_value"]
diff --git a/lib/zkevm_prover.star b/lib/zkevm_prover.star
@@ -13,7 +13,10 @@ def start_executor(plan, args, config_artifact):
 
 
 def _start_service(plan, type, args, config_artifact):
-    cpu_arch_result = plan.run_sh(run="uname -m | tr -d '\n'")
+    cpu_arch_result = plan.run_sh(
+        description="Determining CPU system architecture",
+        run="uname -m | tr -d '\n'",
+    )
     cpu_arch = cpu_arch_result.output
 
     return plan.add_service(

diff --git a/main.star b/main.star
@@ -10,19 +10,6 @@ observability_package = import_module("./observability.star")
 def run(plan, args):
     plan.print("Deploying CDK environment...")
 
-    # Determine system architecture
-    cpu_arch_result = plan.run_sh(
-        run="uname -m | tr -d '\n'", description="Determining CPU system architecture"
-    )
-    cpu_arch = cpu_arch_result.output
-    plan.print("Running on {} architecture".format(cpu_arch))
-    if not "cpu_arch" in args:
-        args["cpu_arch"] = cpu_arch
-
-    args["is_cdk_validium"] = False
-    if args["zkevm_rollup_consensus"] == "PolygonValidiumEtrog":
-        args["is_cdk_validium"] = True
-
     # Deploy a local L1.
     if args["deploy_l1"]:
         plan.print("Deploying a local L1")

diff --git a/observability.star b/observability.star
@@ -9,7 +9,9 @@ bridge_package = import_module("./cdk_bridge_infra.star")
 
 def start_panoptichain(plan, args):
     # Create the panoptichain config.
-    panoptichain_config_template = read_file(src="./templates/panoptichain-config.yml")
+    panoptichain_config_template = read_file(
+        src="./templates/observability/panoptichain-config.yml"
+    )
     contract_setup_addresses = service_package.get_contract_setup_addresses(plan, args)
     panoptichain_config_artifact = plan.render_templates(
         name="panoptichain-config",