Merge remote-tracking branch 'origin/master'

.github/workflows/benchmark.yml

@@ -1,4 +1,4 @@
-name: Schedule based benchmark of pallet-contracts and pallet-evm
+name: Benchmark of pallet-contracts and pallet-evm
 
 on:
   schedule:
@@ -10,8 +10,9 @@ env:
   MOONBEAM_DIR: moonbeam_release
   MOONBEAM_BIN: moonbeam_release/*/target/release/moonbeam
   MOONBEAM_VERSION: version
-  BENCHMARK_DIR: results
-  TEST_PARAMS: --instance-count 1 --call-count 1000
+  BENCHMARK_DIR: stats
+  TEST_PARAMS: --instance-count 1 --call-count 2000
+  BENCHMARK_URI: https://raw.githubusercontent.com/paritytech/smart-bench/gh-pages
 
 jobs:
   build_dev_moonbeam:
@@ -69,9 +70,9 @@ jobs:
     strategy:
       matrix:
         type: [ink-wasm, sol-wasm, evm]
-        contract: [erc20]
+        contract: [erc20, flipper, triangle-number, storage-read, storage-write]
     env:
-      BENCHMARK_FILE: benchmark-${{ matrix.type }}-${{ matrix.contract }}.csv
+      BENCHMARK_FILE: benchmark_${{ matrix.type }}_${{ matrix.contract }}.csv
     needs: build_dev_moonbeam
     runs-on: ubuntu-latest
     steps:
@@ -160,7 +161,7 @@ jobs:
           STATS=$(./run.sh -- ${{ matrix.type }} ${{ matrix.contract }} ${TEST_PARAMS})
           BLOCKS=$(echo ${STATS} | grep -o 'Total Blocks: [0-9]*' | awk '{print $3}')
           EXTRINSICS=$(echo ${STATS} | grep -o 'Total Extrinsics: [0-9]*' | awk '{print $3}')
-          TPS=$(echo ${STATS} | grep -o 'TPS: [0-9]*' | awk '{print $2}')
+          TPS=$(echo ${STATS} | grep -o 'sTPS: [0-9]*' | awk '{print $2}')
           echo "Blocks: ${BLOCKS}"
           echo "Extrinsics: ${EXTRINSICS}"
           echo "TPS: ${TPS}"
@@ -203,8 +204,6 @@ jobs:
   collect:
     runs-on: ubuntu-latest
     needs: [smart_contract_benchmark]
-    env:
-      BENCHMARK_FILE: benchmark-results.csv
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -216,18 +215,41 @@ jobs:
 
       - name: Merge CSV
         run: |
-          cat ${{ env.BENCHMARK_DIR }}/*/*.csv >> ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }}
+          for file in ${{ env.BENCHMARK_DIR }}/*/*.csv; do
+            # Extract contract name
+            contract_name=$(basename "$file" | sed 's/^.*_\(.*\)\.csv$/\1/')
+            benchmark_file=bench_${contract_name}.csv
+            if [ ! -f ${{ env.BENCHMARK_DIR }}/${benchmark_file} ]; then
+              curl -L -o ${{ env.BENCHMARK_DIR }}/${benchmark_file}  ${{ env.BENCHMARK_URI }}/${benchmark_file} || exit 1
+            fi
+            cat $file >> ${{ env.BENCHMARK_DIR }}/${benchmark_file}
+          done
 
       - name: Generate graph
         run: |
-          cd stats
-          ./get_graph.sh --panel-id=2 --csv-data=../${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} --output=../${{ env.BENCHMARK_DIR }}/tps.png
+          for file in ${{ env.BENCHMARK_DIR }}/bench_*.csv; do
+            contract_name=$(basename "$file" | sed 's/^.*_\(.*\)\.csv$/\1/')
+            pushd stats
+            ./get_graph.sh --panel-id=2 --csv-data=../${file} --output=../${{ env.BENCHMARK_DIR }}/stps_${contract_name}.png
+            popd
+          done
 
       - name: Commit benchmark stats
         run: |
           CURRENT_DATE=$(date +"%Y%m%d")
+          # Set git config
           git config user.email "[email protected]"
           git config user.name "paritytech-ci"
-          git add ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} ${{ env.BENCHMARK_DIR }}/tps.png
-          git commit -m "Add benchmark results on ${CURRENT_DATE}"
-          git push
+          git fetch origin gh-pages
+          # saving stats
+          mkdir /tmp/stats
+          mv ${{ env.BENCHMARK_DIR }}/bench_*.csv /tmp/stats
+          mv ${{ env.BENCHMARK_DIR }}/stps_*.png /tmp/stats
+          git checkout gh-pages
+          mv /tmp/stats/* .
+          # Upload files
+          git add *.csv *.png --force
+          git status
+          git commit -m "Updated stats in ${CURRENT_DATE} and pushed to gh-pages"
+          git push origin gh-pages --force
+          rm -rf .git/ ./*

.github/workflows/ci.yaml

@@ -74,5 +74,5 @@ jobs:
         run: |
           git clone --verbose --depth 1 https://github.com/paritytech/ink.git
           mv Cargo.toml _Cargo.toml
-          ./scripts/ci/build-contract.sh ./ink/integration-tests/erc20
+          ./scripts/ci/build-contract.sh ./ink/integration-tests/public/erc20
           mv _Cargo.toml Cargo.toml

README.md

@@ -117,3 +117,20 @@ Before running tests, smart-bench needs to be build using `cargo build` command.
 Integration tests requires two types of nodes to be installed and available on `PATH`.
 - [`moonbeam`](https://github.com/PureStake/moonbeam/) with enabled [`dev RPC`](https://github.com/paritytech/substrate-contracts-node/blob/539cf0271090f406cb3337e4d97680a6a63bcd2f/node/src/rpc.rs#L60) for Solidity/EVM contracts
 - [`substrate-contracts-node`](https://github.com/paritytech/substrate-contracts-node/) for Ink! and Solang (Solidity/Wasm) contracts
+
+### Benchmarks
+
+## Erc20
+![Erc20](https://github.com/paritytech/smart-bench/blob/gh-pages/stps_erc20.png?raw=true)
+
+## Flipper
+![Flipper](https://github.com/paritytech/smart-bench/blob/gh-pages/stps_flipper.png?raw=true)
+
+## Storage Read
+![Storage Read](https://github.com/paritytech/smart-bench/blob/gh-pages/stps_storage-read.png?raw=true)
+
+## Storage Write
+![Storage Write](https://github.com/paritytech/smart-bench/blob/gh-pages/stps_storage-write.png?raw=true)
+
+## Triangle Number
+![Triangle Number](https://github.com/paritytech/smart-bench/blob/gh-pages/stps_triangle-number.png?raw=true)

src/evm/runner.rs

@@ -3,7 +3,9 @@ use std::collections::HashSet;
 use super::xts::{
     api::{
         self,
+        ethereum::calls::types::Transact,
         ethereum::events::Executed,
+        runtime_types::ethereum::transaction::{TransactionAction, TransactionV2},
         runtime_types::evm_core::error::{ExitReason, ExitSucceed},
     },
     MoonbeamApi,
@@ -25,7 +27,7 @@ pub struct MoonbeamRunner {
     pub api: MoonbeamApi,
     signer: SecretKey,
     address: Address,
-    calls: Vec<(String, Vec<Call>)>,
+    calls: Vec<(String, Vec<RunnerCall>)>,
     pub call_signers: Option<Vec<SecretKey>>,
 }
 
@@ -95,7 +97,7 @@ impl MoonbeamRunner {
                 .estimate_gas(self.address, Some(contract), &data)
                 .await
                 .note("Error estimating gas")?;
-            calls.push(Call {
+            calls.push(RunnerCall {
                 name: name.to_string(),
                 contract,
                 data,
@@ -208,22 +210,44 @@ impl MoonbeamRunner {
     ///
     /// for given block, ethereum transaction hash can be retrieved
     /// from events of type ethereum.Executed
-    async fn get_eth_hashes_from_events_in_block(
+    async fn get_block_details(
         client: OnlineClient<DefaultConfig>,
         block_hash: sp_core::H256,
-    ) -> color_eyre::Result<Vec<sp_core::H256>> {
-        let events = client.events().at(block_hash).await?;
+    ) -> color_eyre::Result<(u64, Vec<sp_core::H256>)> {
+        let block = client.blocks().at(block_hash).await?;
         let mut tx_hashes = Vec::new();
-        for event in events.iter() {
-            let event = event?;
-            if let Some(Executed {
-                transaction_hash, ..
-            }) = event.as_event::<Executed>()?
-            {
-                tx_hashes.push(transaction_hash);
+        let extrinsics_details = block
+            .extrinsics()
+            .await?
+            .iter()
+            .collect::<Result<Vec<_>, _>>()?;
+
+        for extrinsic_detail in extrinsics_details {
+            if let Some(Transact { transaction }) = extrinsic_detail.as_extrinsic::<Transact>()? {
+                if let TransactionV2::Legacy(tx) = transaction {
+                    if let TransactionAction::Call(_) = tx.action {
+                        let events = extrinsic_detail.events().await?;
+                        for event in events.iter() {
+                            let event = event?;
+                            if let Some(Executed {
+                                transaction_hash, ..
+                            }) = event.as_event::<Executed>()?
+                            {
+                                tx_hashes.push(transaction_hash);
+                            }
+                        }
+                    }
+                }
             }
         }
-        Ok(tx_hashes)
+        let storage_timestamp_storage_addr = api::storage().timestamp().now();
+        let time_stamp = client
+            .storage()
+            .at(block_hash)
+            .fetch(&storage_timestamp_storage_addr)
+            .await?
+            .unwrap();
+        Ok((time_stamp, tx_hashes))
     }
 
     /// Call each contract instance `call_count` times. Wait for all txs to be included in a block
@@ -292,14 +316,14 @@ impl MoonbeamRunner {
 
         let wait_for_txs = crate::collect_block_stats(block_stats, remaining_hashes, |hash| {
             let client = self.api.client.clone();
-            Self::get_eth_hashes_from_events_in_block(client, hash)
+            Self::get_block_details(client, hash)
         });
 
         Ok(wait_for_txs)
     }
 }
 
-struct Call {
+struct RunnerCall {
     name: String,
     contract: Address,
     data: Vec<u8>,

src/integration_tests.rs

@@ -26,7 +26,7 @@ fn is_match(stdout: &str, pattern: &str) -> bool {
 }
 
 const SMART_BENCH_STATS_PATTERN: &str = r"[0-9]+: PoV Size=[0-9]+KiB\([0-9]+%\) Weight RefTime=[0-9]+ms\([0-9]+%\) Weight ProofSize=[0-9]+KiB\([0-9]+%\) Witness=[0-9]+KiB Block=[0-9]+KiB NumExtrinsics=[0-9]+";
-const SMART_BENCH_LAST_LINE_PATTERN: &str = r"TPS: \d+(\.\d+)?";
+const SMART_BENCH_LAST_LINE_PATTERN: &str = r"sTPS: \d+(\.\d+)?";
 const CONTRACTS_NODE_WASM: &str = "substrate-contracts-node";
 const CONTRACTS_NODE_EVM: &str = "moonbeam";
 
@@ -164,7 +164,7 @@ async fn test_ink_contract_success() {
         .arg("ink-wasm")
         .arg("flipper")
         .args(["--instance-count", "1"])
-        .args(["--call-count", "1"])
+        .args(["--call-count", "10"])
         .args(["--url", "ws://localhost:9944"])
         .arg("--single-signer")
         .timeout(std::time::Duration::from_secs(5))
@@ -210,7 +210,7 @@ async fn test_solidity_wasm_contract_success() {
         .arg("sol-wasm")
         .arg("flipper")
         .args(["--instance-count", "1"])
-        .args(["--call-count", "1"])
+        .args(["--call-count", "10"])
         .args(["--url", "ws://localhost:9944"])
         .arg("--single-signer")
         .timeout(std::time::Duration::from_secs(5))
@@ -257,7 +257,7 @@ async fn test_solidity_evm_contract_success() {
         .arg("evm")
         .arg("flipper")
         .args(["--instance-count", "1"])
-        .args(["--call-count", "1"])
+        .args(["--call-count", "10"])
         .args(["--url", "ws://localhost:9944"])
         .arg("--single-signer")
         .timeout(std::time::Duration::from_secs(5))

src/stats.rs

@@ -4,20 +4,22 @@ use std::task::Poll;
 use std::collections::HashSet;
 use std::sync::{Arc, Mutex};
 pub struct BlockInfo {
+    // block time stamp
+    pub time_stamp: u64,
     pub stats: blockstats::BlockStats,
     // list of hashes to look for
-    pub hashes: Vec<sp_core::H256>,
+    pub contract_call_hashes: Vec<sp_core::H256>,
 }
 
 /// Subscribes to block stats. Completes once *all* hashes in `remaining_hashes` have been received.
 pub fn collect_block_stats<F, Fut>(
     block_stats: impl TryStream<Ok = blockstats::BlockStats, Error = subxt::Error> + Unpin,
     remaining_hashes: HashSet<sp_core::H256>,
-    get_hashes_in_block: F,
+    get_block_details: F,
 ) -> impl TryStream<Ok = BlockInfo, Error = color_eyre::Report>
 where
+    Fut: Future<Output = color_eyre::Result<(u64, Vec<sp_core::H256>)>>,
     F: Fn(sp_core::H256) -> Fut + Copy,
-    Fut: Future<Output = color_eyre::Result<Vec<sp_core::H256>>>,
 {
     let block_stats_arc = Arc::new(Mutex::new(block_stats));
     let remaining_hashes_arc = Arc::new(Mutex::new(remaining_hashes));
@@ -39,38 +41,89 @@ where
         async move {
             let stats = block_stats.lock().unwrap().try_next().await?.unwrap();
             tracing::debug!("{stats:?}");
-            let hashes = get_hashes_in_block(stats.hash).await?;
+            let (time_stamp, hashes) = get_block_details(stats.hash).await?;
             let mut remaining_hashes = remaining_hashes.lock().unwrap();
             for xt in &hashes {
                 remaining_hashes.remove(xt);
             }
-            Ok(BlockInfo { hashes, stats })
+            Ok(BlockInfo {
+                time_stamp,
+                contract_call_hashes: hashes,
+                stats,
+            })
         }
     })
 }
 
-///  Print the block info stats to the console
+/// This function prints statistics to the standard output.
+
+/// The TPS calculation is based on the following assumptions about smart-bench:
+/// - smart-bench instantiates smart contracts on the chain and waits for the completion of these transactions.
+/// - Starting from some future block (after creation), smart-bench uploads transactions related to contract calls to the node.
+/// - Sending contract call transactions to the node is continuous once started and is not mixed with any other type of transactions.
+/// - Smart-bench finishes benchmarking at the block that contains the last contract call from the set.
+
+/// TPS calculation is exclusively concerned with contract calls, disregarding any system or contract-creating transactions.
+
+/// TPS calculation excludes the last block of the benchmark, as its full utilization is not guaranteed. In other words, only blocks in the middle will consist entirely of contract calls.
 pub async fn print_block_info(
     block_info: impl TryStream<Ok = BlockInfo, Error = color_eyre::Report>,
 ) -> color_eyre::Result<()> {
-    let mut total_extrinsics = 0u64;
-    let mut total_blocks = 0u64;
+    let mut call_extrinsics_per_block: Vec<u64> = Vec::new();
+    let mut call_block_expected = false;
+    let mut time_stamp = None;
+    let mut time_diff = None;
     println!();
     block_info
         .try_for_each(|block| {
             println!("{}", block.stats);
-            total_extrinsics += block.stats.num_extrinsics;
-            total_blocks += 1;
+            let contract_calls_count = block.contract_call_hashes.len() as u64;
+            // Skip blocks at the beggining until we see first call related transaction
+            // Once first call is seen, we expect all further blocks to contain calls until all calls are covered
+            if !call_block_expected && contract_calls_count > 0 {
+                call_block_expected = true;
+            }
+            if call_block_expected {
+                call_extrinsics_per_block.push(contract_calls_count);
+            }
+
+            if time_diff.is_none() {
+                if let Some(ts) = time_stamp {
+                    time_diff = Some((block.time_stamp - ts) as f64 / 1000.0)
+                } else {
+                    time_stamp = Some(block.time_stamp)
+                }
+            }
             future::ready(Ok(()))
         })
         .await?;
+
+    // Skip the last block as it's not stressed to its full capabilities, 
+    // since there is a very low chance of hitting that exact amount of transactions 
+    // (it will contain as many transactions as there are left to execute).
+    let call_extrinsics_per_block =
+        &call_extrinsics_per_block[0..call_extrinsics_per_block.len() - 1];
+
+    let tps_blocks = call_extrinsics_per_block.len();
+    let tps_total_extrinsics = call_extrinsics_per_block.iter().sum::<u64>();
     println!("\nSummary:");
-    println!("Total Blocks: {total_blocks}");
-    println!("Total Extrinsics: {total_extrinsics}");
-    println!("TPS - Transaction execution time per second, assuming a 0.5-second execution time per block");
-    println!(
-        "TPS: {}",
-        total_extrinsics as f64 / (total_blocks as f64 * 0.5)
-    );
+    println!("Total Blocks: {tps_blocks}");
+    println!("Total Extrinsics: {tps_total_extrinsics}");
+    let diff = time_diff.unwrap_or_else(|| {
+        // default block build time
+        let default = 12.0;
+        println!("Warning: Could not calculate block build time, assuming {default}");
+        default
+    });
+    println!("Block Build Time: {diff}");
+    if tps_blocks > 0 {
+        println!("sTPS - Standard Transaction Per Second");
+        println!(
+            "sTPS: {:.2}",
+            tps_total_extrinsics as f64 / (tps_blocks as f64 * diff)
+        );
+    } else {
+        println!("sTPS - Error - not enough data to calculate sTPS, consider increasing --call-count value")
+    }
     Ok(())
 }