draft for benchmarking

.github/workflows/cron_jerigon_zero_testing.yml

@@ -6,7 +6,7 @@ on:
   #   # Run every Sunday at 12:00 AM (UTC)
   #   - cron: "0 0 * * 0"
   push:
-    branches: [develop, main]
+    branches: [develop]
   pull_request:
     branches:
       - "**"
@@ -48,39 +48,44 @@ jobs:
       - name: Run erigon network
         run: |
           cd ..
-          tar xf "$(pwd)/zk_evm/test-data/erigon-data.tar.gz"
+          tar xf "$(pwd)/zk_evm/test_data/erigon-data.tar.gz"
           docker pull ghcr.io/0xpolygonzero/erigon:feat-zero
           docker run -d -p 8545:8545 -v $(pwd):/data \
             ghcr.io/0xpolygonzero/erigon:feat-zero \
             --datadir=/data/erigon/execution-data --http.api=eth,erigon,engine,web3,net,debug,trace,txpool,admin \
             --http.vhosts=* --ws --http --http.addr=0.0.0.0 --http.corsdomain=* --http.port=8545 \
             --metrics --metrics.addr=0.0.0.0 --metrics.port=9001 --db.size.limit=3000MB
+          export ETH_RPC_URL="http://localhost:8545"
 
-      # TODO - Decide the number of blocks we want to test with
+      # TODO - Decide the number of blocks we want to test with. Currently choosing any 1 random blocks
       - name: Regression test with zero tracer in real mode
         run: |
-          export ETH_RPC_URL="http://localhost:8545"
           rm -rf proofs/* circuits/* ./proofs.json test.out verify.out leader.out
-          random_numbers=($(shuf -i 1-500 -n 10))
+          random_numbers=($(shuf -i 1-500 -n 1))
           for number in "${random_numbers[@]}"; do
-            echo $number
             hex_number="0x$(echo "obase=16; $number" | bc)"
-            echo $hex_number
-            OUTPUT_TO_TERMINAL=true RUN_VERIFICATION=true ./scripts/prove_rpc.sh $hex_number $hex_number $ETH_RPC_URL jerigon true 3000 100
+            OUTPUT_TO_TERMINAL=true RUN_VERIFICATION=true ./scripts/prove_rpc.sh $hex_number $hex_number $ETH_RPC_URL jerigon true 3000 100 test_only
           done
 
-      # - name: Download previous results for becnhmarking
-      #   uses: dawidd6/action-download-artifact@v6
-      #   with:
-      #     workflow: cron_jerigon_zero_testing.yml
-      #     workflow_conclusion: success
-      #     name: jerigon_zero_testing_benchmark
-      #     path: ./
-      #     if_no_artifact_found: ignore
+      - name: Download previous results
+        uses: dawidd6/action-download-artifact@v6
+        with:
+          workflow: cron_jerigon_zero_testing.yml
+          workflow_conclusion: success
+          name: jerigon_zero_benchmark
+          path: ./
+          if_no_artifact_found: ignore
+
+      # TODO - Put more stats in the output file
+      - name: Run the benchmark script
+        run: |
+          ./scripts/jerigon_zero_benchmark.sh
 
-      # - name: Benchmarking test with zero tracer in real mode
-      #   run: |
-      #     export ETH_RPC_URL="http://localhost:8545"
-      #     rm -rf proofs/* circuits/* ./proofs.json test.out verify.out leader.out
-      #     echo "Running the benchmarking script..."
-      #     ./scripts/jerigon_zero_benchmark.sh | tee benchmark_output.log
+      - name: Upload new results
+        uses: actions/upload-artifact@v4
+        with:
+          name: jerigon_zero_benchmark
+          path: |
+            ./jerigon_zero_output.log
+          retention-days: 90
+          overwrite: true

scripts/jerigon_zero_benchmark.sh

@@ -2,6 +2,9 @@
 # ------------------------------------------------------------------------------
 set -exo pipefail
 
+# Args:
+# 1 --> Output file
+
 # We're going to set the parallelism in line with the total cpu count
 if [[ "$OSTYPE" == "darwin"* ]]; then
     num_procs=$(sysctl -n hw.physicalcpu)
@@ -16,7 +19,9 @@ PROOF_OUTPUT_DIR="${REPO_ROOT}/proofs"
 BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}"
 echo "Block batch size: $BLOCK_BATCH_SIZE"
 
-OUTPUT_LOG="${REPO_ROOT}/output.log"
+OUTPUT_LOG="jerigon_zero_output.log"
+BLOCK_OUTPUT_LOG="jerigon_zero_block_output.log"
+
 PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json"
 TEST_OUT_PATH="${REPO_ROOT}/test.out"
 
@@ -28,30 +33,34 @@ export RUST_MIN_STACK=33554432
 export RUST_BACKTRACE=full
 export RUST_LOG=info
 
-INPUT_FILE=$1
+echo "$(date +"%Y-%m-%d %H:%M:%S")" &>> $OUTPUT_LOG
+
+blocks=(1 2)
 
-if [[ $INPUT_FILE == "" ]]; then
-    echo "Please provide witness json input file, e.g. artifacts/witness_b19240705.json"
+for block in "${blocks[@]}"; do
+  echo "Processing block: $block"
+  # TODO - Maybe store output_$block.json permanently
+  ./target/release/rpc --rpc-url $ETH_RPC_URL fetch --start-block $block --end-block $block > "output_${block}.json"
+  start_time=$(date +%s%N)
+  perf stat -e cycles ./target/release/leader --test-only --runtime in-memory --load-strategy monolithic --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < "output_${block}.json" &> $BLOCK_OUTPUT_LOG
+  end_time=$(date +%s%N)
+
+  # TODO - Handle case when block proving was unsuccessful
+  set +o pipefail
+  cat $BLOCK_OUTPUT_LOG | grep "Successfully wrote to disk proof file " | awk '{print $NF}' | tee $PROOFS_FILE_LIST
+  if [ ! -s "$PROOFS_FILE_LIST" ]; then
+    # Some error occurred, display the logs and exit.
+    cat $OUTPUT_LOG
+    echo "Proof list not generated, some error happened. For more details check the log file $OUTPUT_LOG"
     exit 1
-fi
+  fi
 
-start_time=$(date +%s%N)
-perf stat -e cycles "${REPO_ROOT}/target/release/leader" --runtime in-memory --load-strategy monolithic --block-batch-size $BLOCK_BATCH_SIZE \
- --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE &> $OUTPUT_LOG
-end_time=$(date +%s%N)
-
-set +o pipefail
-cat $OUTPUT_LOG | grep "Successfully wrote to disk proof file " | awk '{print $NF}' | tee $PROOFS_FILE_LIST
-if [ ! -s "$PROOFS_FILE_LIST" ]; then
-  # Some error occurred, display the logs and exit.
-  cat $OUTPUT_LOG
-  echo "Proof list not generated, some error happened. For more details check the log file $OUTPUT_LOG"
-  exit 1
-fi
+  duration_sec=$(echo "scale=3; ($end_time - $start_time) / 1000000000" | bc -l)
 
-duration_ns=$((end_time - start_time))
-duration_sec=$(echo "$duration_ns / 1000000000" | bc -l)
+  echo "Success!"
+  echo "Proving duration for block" $block ":" $duration_sec "seconds" &>> $OUTPUT_LOG
+done
 
-echo "Success!"
-echo "Proving duration:" $duration_sec " seconds"
-echo "Note, this duration is inclusive of circuit handling and overall process initialization";
+echo "" &>> $OUTPUT_LOG
+echo "" &>> $OUTPUT_LOG
+echo "" &>> $OUTPUT_LOG