From 7d55405a5c307c2bfcb0d6983a3af2e158e81113 Mon Sep 17 00:00:00 2001 From: William Smith Date: Thu, 31 Oct 2024 19:12:33 -0700 Subject: [PATCH 1/6] [Traffic Control] Improved error metrics --- crates/sui-core/src/authority_server.rs | 3 ++- crates/sui-core/src/traffic_controller/metrics.rs | 12 ++++++++++-- crates/sui-core/src/traffic_controller/mod.rs | 7 +++++++ crates/sui-core/src/traffic_controller/policies.rs | 6 ++++++ crates/sui-json-rpc/src/axum_router.rs | 3 ++- 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/crates/sui-core/src/authority_server.rs b/crates/sui-core/src/authority_server.rs index 0e9aeb30ee1ef..1f17262963824 100644 --- a/crates/sui-core/src/authority_server.rs +++ b/crates/sui-core/src/authority_server.rs @@ -1296,9 +1296,10 @@ impl ValidatorService { traffic_controller.tally(TrafficTally { direct: client, through_fullnode: None, - error_weight: error.map(normalize).unwrap_or(Weight::zero()), + error_weight: error.clone().map(normalize).unwrap_or(Weight::zero()), spam_weight, timestamp: SystemTime::now(), + error_type: error.map(|e| e.to_string()), }) } unwrapped_response diff --git a/crates/sui-core/src/traffic_controller/metrics.rs b/crates/sui-core/src/traffic_controller/metrics.rs index d7f9e80601880..9e05c95271033 100644 --- a/crates/sui-core/src/traffic_controller/metrics.rs +++ b/crates/sui-core/src/traffic_controller/metrics.rs @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 use prometheus::{ - register_int_counter_with_registry, register_int_gauge_with_registry, IntCounter, IntGauge, - Registry, + register_int_counter_vec_with_registry, register_int_counter_with_registry, + register_int_gauge_with_registry, IntCounter, IntCounterVec, IntGauge, Registry, }; #[derive(Clone)] @@ -18,6 +18,7 @@ pub struct TrafficControllerMetrics { pub num_dry_run_blocked_requests: IntCounter, pub tally_handled: IntCounter, pub error_tally_handled: IntCounter, + pub tally_errors: IntCounterVec, pub deadmans_switch_enabled: IntGauge, pub highest_direct_spam_rate: IntGauge, pub highest_proxied_spam_rate: IntGauge, @@ -90,6 +91,13 @@ impl TrafficControllerMetrics { registry ) .unwrap(), + tally_errors: register_int_counter_vec_with_registry!( + "traffic_control_tally_errors", + "Number of tally errors, grouped by error type", + &["error_type"], + registry + ) + .unwrap(), deadmans_switch_enabled: register_int_gauge_with_registry!( "deadmans_switch_enabled", "If 1, the deadman's switch is enabled and all traffic control diff --git a/crates/sui-core/src/traffic_controller/mod.rs b/crates/sui-core/src/traffic_controller/mod.rs index cacec53b5cc7c..0085934e44802 100644 --- a/crates/sui-core/src/traffic_controller/mod.rs +++ b/crates/sui-core/src/traffic_controller/mod.rs @@ -430,6 +430,10 @@ async fn handle_error_tally( } let resp = policy.handle_tally(tally.clone()); metrics.error_tally_handled.inc(); + metrics + .tally_errors + .with_label_values(&[tally.error_type.as_deref().unwrap_or("unknown")]) + .inc(); if let Some(fw_config) = fw_config { if fw_config.delegate_error_blocking && !mem_drainfile_present { let client = nodefw_client @@ -509,6 +513,7 @@ async fn handle_policy_response( { // Only increment the metric if the client was not already blocked debug!("Blocking client: {:?}", client); + metrics.requests_blocked_at_protocol.inc(); metrics.connection_ip_blocklist_len.inc(); } } @@ -523,6 +528,7 @@ async fn handle_policy_response( { // Only increment the metric if the client was not already blocked debug!("Blocking proxied client: {:?}", client); + metrics.requests_blocked_at_protocol.inc(); metrics.proxy_ip_blocklist_len.inc(); } } @@ -744,6 +750,7 @@ impl TrafficSim { client, // TODO add proxy IP for testing None, + None, // TODO add weight adjustments Weight::one(), Weight::one(), diff --git a/crates/sui-core/src/traffic_controller/policies.rs b/crates/sui-core/src/traffic_controller/policies.rs index 458af6862fa45..2044809ab0704 100644 --- a/crates/sui-core/src/traffic_controller/policies.rs +++ b/crates/sui-core/src/traffic_controller/policies.rs @@ -223,6 +223,7 @@ pub struct TrafficTally { pub direct: Option, pub through_fullnode: Option, pub error_weight: Weight, + pub error_type: Option, pub spam_weight: Weight, pub timestamp: SystemTime, } @@ -231,6 +232,7 @@ impl TrafficTally { pub fn new( direct: Option, through_fullnode: Option, + error_type: Option, error_weight: Weight, spam_weight: Weight, ) -> Self { @@ -238,6 +240,7 @@ impl TrafficTally { direct, through_fullnode, error_weight, + error_type, spam_weight, timestamp: SystemTime::now(), } @@ -516,6 +519,7 @@ mod tests { direct: Some(IpAddr::V4(Ipv4Addr::new(8, 7, 6, 5))), through_fullnode: Some(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4))), error_weight: Weight::zero(), + error_type: None, spam_weight: Weight::one(), timestamp: SystemTime::now(), }; @@ -523,6 +527,7 @@ mod tests { direct: Some(IpAddr::V4(Ipv4Addr::new(8, 7, 6, 5))), through_fullnode: Some(IpAddr::V4(Ipv4Addr::new(4, 3, 2, 1))), error_weight: Weight::zero(), + error_type: None, spam_weight: Weight::one(), timestamp: SystemTime::now(), }; @@ -530,6 +535,7 @@ mod tests { direct: Some(IpAddr::V4(Ipv4Addr::new(8, 7, 6, 5))), through_fullnode: Some(IpAddr::V4(Ipv4Addr::new(5, 6, 7, 8))), error_weight: Weight::zero(), + error_type: None, spam_weight: Weight::one(), timestamp: SystemTime::now(), }; diff --git a/crates/sui-json-rpc/src/axum_router.rs b/crates/sui-json-rpc/src/axum_router.rs index aa2e1e4edcadb..e1e0eaf2ecc13 100644 --- a/crates/sui-json-rpc/src/axum_router.rs +++ b/crates/sui-json-rpc/src/axum_router.rs @@ -252,7 +252,7 @@ fn handle_traffic_resp( traffic_controller.tally(TrafficTally { direct: client, through_fullnode: None, - error_weight: error.map(normalize).unwrap_or(Weight::zero()), + error_weight: error.clone().map(normalize).unwrap_or(Weight::zero()), // For now, count everything as spam with equal weight // on the rpc node side, including gas-charging endpoints // such as `sui_executeTransactionBlock`, as this can enable @@ -262,6 +262,7 @@ fn handle_traffic_resp( // to provide a weight distribution based on the method being called. spam_weight: Weight::one(), timestamp: SystemTime::now(), + error_type: error.map(|e| e.to_string()), }); } From 176490794e1e9d7aae4ef19e393e63809d4a5474 Mon Sep 17 00:00:00 2001 From: William Smith Date: Fri, 1 Nov 2024 10:50:30 -0700 Subject: [PATCH 2/6] aggregate error info --- crates/sui-core/src/authority_server.rs | 7 +++++-- .../src/traffic_controller/metrics.rs | 6 +++--- crates/sui-core/src/traffic_controller/mod.rs | 20 +++++++++++-------- .../src/traffic_controller/policies.rs | 18 ++++++----------- crates/sui-json-rpc/src/axum_router.rs | 7 +++++-- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/crates/sui-core/src/authority_server.rs b/crates/sui-core/src/authority_server.rs index 1f17262963824..60cb049986bc1 100644 --- a/crates/sui-core/src/authority_server.rs +++ b/crates/sui-core/src/authority_server.rs @@ -1296,10 +1296,13 @@ impl ValidatorService { traffic_controller.tally(TrafficTally { direct: client, through_fullnode: None, - error_weight: error.clone().map(normalize).unwrap_or(Weight::zero()), + error_info: error.map(|e| { + let error_type = String::from(e.clone().as_ref()); + let error_weight = normalize(e); + (error_weight, error_type) + }), spam_weight, timestamp: SystemTime::now(), - error_type: error.map(|e| e.to_string()), }) } unwrapped_response diff --git a/crates/sui-core/src/traffic_controller/metrics.rs b/crates/sui-core/src/traffic_controller/metrics.rs index 9e05c95271033..6077a7bb8098a 100644 --- a/crates/sui-core/src/traffic_controller/metrics.rs +++ b/crates/sui-core/src/traffic_controller/metrics.rs @@ -18,7 +18,7 @@ pub struct TrafficControllerMetrics { pub num_dry_run_blocked_requests: IntCounter, pub tally_handled: IntCounter, pub error_tally_handled: IntCounter, - pub tally_errors: IntCounterVec, + pub tally_error_types: IntCounterVec, pub deadmans_switch_enabled: IntGauge, pub highest_direct_spam_rate: IntGauge, pub highest_proxied_spam_rate: IntGauge, @@ -91,8 +91,8 @@ impl TrafficControllerMetrics { registry ) .unwrap(), - tally_errors: register_int_counter_vec_with_registry!( - "traffic_control_tally_errors", + tally_error_types: register_int_counter_vec_with_registry!( + "traffic_control_tally_error_types", "Number of tally errors, grouped by error type", &["error_type"], registry diff --git a/crates/sui-core/src/traffic_controller/mod.rs b/crates/sui-core/src/traffic_controller/mod.rs index 0085934e44802..701c862e5e5ec 100644 --- a/crates/sui-core/src/traffic_controller/mod.rs +++ b/crates/sui-core/src/traffic_controller/mod.rs @@ -425,15 +425,20 @@ async fn handle_error_tally( metrics: Arc, mem_drainfile_present: bool, ) -> Result<(), reqwest::Error> { - if !tally.error_weight.is_sampled() { + let error_weight = if let Some((error_weight, error_type)) = tally.clone().error_info { + metrics + .tally_error_types + .with_label_values(&[error_type.as_str()]) + .inc(); + error_weight + } else { + return Ok(()); + }; + if !error_weight.is_sampled() { return Ok(()); } - let resp = policy.handle_tally(tally.clone()); + let resp = policy.handle_tally(tally); metrics.error_tally_handled.inc(); - metrics - .tally_errors - .with_label_values(&[tally.error_type.as_deref().unwrap_or("unknown")]) - .inc(); if let Some(fw_config) = fw_config { if fw_config.delegate_error_blocking && !mem_drainfile_present { let client = nodefw_client @@ -750,9 +755,8 @@ impl TrafficSim { client, // TODO add proxy IP for testing None, - None, // TODO add weight adjustments - Weight::one(), + None, Weight::one(), )); } else { diff --git a/crates/sui-core/src/traffic_controller/policies.rs b/crates/sui-core/src/traffic_controller/policies.rs index 2044809ab0704..ba246fcc6f4a4 100644 --- a/crates/sui-core/src/traffic_controller/policies.rs +++ b/crates/sui-core/src/traffic_controller/policies.rs @@ -222,8 +222,7 @@ impl TrafficSketch { pub struct TrafficTally { pub direct: Option, pub through_fullnode: Option, - pub error_weight: Weight, - pub error_type: Option, + pub error_info: Option<(Weight, String)>, pub spam_weight: Weight, pub timestamp: SystemTime, } @@ -232,15 +231,13 @@ impl TrafficTally { pub fn new( direct: Option, through_fullnode: Option, - error_type: Option, - error_weight: Weight, + error_info: Option<(Weight, String)>, spam_weight: Weight, ) -> Self { Self { direct, through_fullnode, - error_weight, - error_type, + error_info, spam_weight, timestamp: SystemTime::now(), } @@ -518,24 +515,21 @@ mod tests { let alice = TrafficTally { direct: Some(IpAddr::V4(Ipv4Addr::new(8, 7, 6, 5))), through_fullnode: Some(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4))), - error_weight: Weight::zero(), - error_type: None, + error_info: None, spam_weight: Weight::one(), timestamp: SystemTime::now(), }; let bob = TrafficTally { direct: Some(IpAddr::V4(Ipv4Addr::new(8, 7, 6, 5))), through_fullnode: Some(IpAddr::V4(Ipv4Addr::new(4, 3, 2, 1))), - error_weight: Weight::zero(), - error_type: None, + error_info: None, spam_weight: Weight::one(), timestamp: SystemTime::now(), }; let charlie = TrafficTally { direct: Some(IpAddr::V4(Ipv4Addr::new(8, 7, 6, 5))), through_fullnode: Some(IpAddr::V4(Ipv4Addr::new(5, 6, 7, 8))), - error_weight: Weight::zero(), - error_type: None, + error_info: None, spam_weight: Weight::one(), timestamp: SystemTime::now(), }; diff --git a/crates/sui-json-rpc/src/axum_router.rs b/crates/sui-json-rpc/src/axum_router.rs index e1e0eaf2ecc13..47ef61987938f 100644 --- a/crates/sui-json-rpc/src/axum_router.rs +++ b/crates/sui-json-rpc/src/axum_router.rs @@ -252,7 +252,11 @@ fn handle_traffic_resp( traffic_controller.tally(TrafficTally { direct: client, through_fullnode: None, - error_weight: error.clone().map(normalize).unwrap_or(Weight::zero()), + error_info: error.map(|e| { + let error_type = e.to_string(); + let error_weight = normalize(e); + (error_weight, error_type) + }), // For now, count everything as spam with equal weight // on the rpc node side, including gas-charging endpoints // such as `sui_executeTransactionBlock`, as this can enable @@ -262,7 +266,6 @@ fn handle_traffic_resp( // to provide a weight distribution based on the method being called. spam_weight: Weight::one(), timestamp: SystemTime::now(), - error_type: error.map(|e| e.to_string()), }); } From 77a25b68f6efc3b42ff60e49a346a8edc5d6515a Mon Sep 17 00:00:00 2001 From: William Smith Date: Sat, 2 Nov 2024 19:32:01 -0700 Subject: [PATCH 3/6] add test logging for debug run -- REVERT LATER --- crates/sui-core/src/traffic_controller/mod.rs | 20 ++++++++++++------- .../src/traffic_controller/policies.rs | 9 +++++++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/crates/sui-core/src/traffic_controller/mod.rs b/crates/sui-core/src/traffic_controller/mod.rs index 701c862e5e5ec..b11c89f51f996 100644 --- a/crates/sui-core/src/traffic_controller/mod.rs +++ b/crates/sui-core/src/traffic_controller/mod.rs @@ -284,6 +284,10 @@ async fn run_clear_blocklists_loop(blocklists: Blocklists, metrics: Arc, mem_drainfile_present: bool, ) -> Result<(), reqwest::Error> { - let error_weight = if let Some((error_weight, error_type)) = tally.clone().error_info { - metrics - .tally_error_types - .with_label_values(&[error_type.as_str()]) - .inc(); - error_weight - } else { + let Some((error_weight, error_type)) = tally.clone().error_info else { return Ok(()); }; if !error_weight.is_sampled() { return Ok(()); } + error!( + "TESTNG -- handling error_type {:?} from client {:?}", + error_type, tally.direct, + ); + metrics + .tally_error_types + .with_label_values(&[error_type.as_str()]) + .inc(); let resp = policy.handle_tally(tally); metrics.error_tally_handled.inc(); if let Some(fw_config) = fw_config { diff --git a/crates/sui-core/src/traffic_controller/policies.rs b/crates/sui-core/src/traffic_controller/policies.rs index ba246fcc6f4a4..40ce2ae2d14bc 100644 --- a/crates/sui-core/src/traffic_controller/policies.rs +++ b/crates/sui-core/src/traffic_controller/policies.rs @@ -14,7 +14,7 @@ use std::hash::Hash; use std::time::Duration; use std::time::{Instant, SystemTime}; use sui_types::traffic_control::{FreqThresholdConfig, PolicyConfig, PolicyType, Weight}; -use tracing::info; +use tracing::{error, info}; const HIGHEST_RATES_CAPACITY: usize = 20; @@ -360,7 +360,12 @@ impl FreqThresholdPolicy { let block_client = if let Some(source) = tally.direct { let key = SketchKey(source, ClientType::Direct); self.sketch.increment_count(&key); - if self.sketch.get_request_rate(&key) >= self.client_threshold as f64 { + let req_rate = self.sketch.get_request_rate(&key); + error!( + "TESTNG -- req_rate: {:?}, client_threshold: {:?}, client: {:?}", + req_rate, self.client_threshold, source, + ); + if req_rate >= self.client_threshold as f64 { Some(source) } else { None From f8cd99d5354ac4c1a70e7f564d4ec8fe4c46d395 Mon Sep 17 00:00:00 2001 From: William Smith Date: Sun, 3 Nov 2024 10:04:29 -0800 Subject: [PATCH 4/6] Make testing debug statements into trace statements --- crates/sui-core/src/traffic_controller/mod.rs | 12 +++++------- crates/sui-core/src/traffic_controller/policies.rs | 10 ++++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/sui-core/src/traffic_controller/mod.rs b/crates/sui-core/src/traffic_controller/mod.rs index b11c89f51f996..f759e1d58ee75 100644 --- a/crates/sui-core/src/traffic_controller/mod.rs +++ b/crates/sui-core/src/traffic_controller/mod.rs @@ -284,10 +284,7 @@ async fn run_clear_blocklists_loop(blocklists: Blocklists, metrics: Arc= self.client_threshold as f64 { Some(source) From 1c210394a6da3ee71da38497acb856ff74e95d01 Mon Sep 17 00:00:00 2001 From: William Smith Date: Mon, 4 Nov 2024 16:16:47 -0800 Subject: [PATCH 5/6] Enforce invariant that the x-forwarded-for header length is not less than the associated config setting on the node --- crates/sui-core/src/authority_server.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/sui-core/src/authority_server.rs b/crates/sui-core/src/authority_server.rs index 60cb049986bc1..6c8bdc956c391 100644 --- a/crates/sui-core/src/authority_server.rs +++ b/crates/sui-core/src/authority_server.rs @@ -1225,6 +1225,19 @@ impl ValidatorService { return None; } let contents_len = header_contents.len(); + // Network topology should not be very dynamic, therefore if it changes and the above + // invariant is violated, we should fail loudly so that the node config can be updated. + assert!( + contents_len >= *num_hops, + "x-forwarded-for header value of {:?} contains {} values, but {} hops were specified. \ + Expected at least {} values. Please correctly set the `x-forwarded-for` value under \ + `client-id-source` in the node config.", + header_contents, + contents_len, + num_hops, + contents_len, + ); + let contents_len = header_contents.len(); let Some(client_ip) = header_contents.get(contents_len - num_hops) else { error!( From f514b2f1f3955230f808c11e2c8e604f6b74579b Mon Sep 17 00:00:00 2001 From: William Smith Date: Tue, 5 Nov 2024 09:54:46 -0800 Subject: [PATCH 6/6] address review comment --- crates/sui-core/src/authority_server.rs | 39 ++++++++++++------- crates/sui-core/src/traffic_controller/mod.rs | 9 ++--- .../tests/traffic_control_tests.rs | 34 +++++++++------- 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/crates/sui-core/src/authority_server.rs b/crates/sui-core/src/authority_server.rs index 6c8bdc956c391..af4d80ae342d4 100644 --- a/crates/sui-core/src/authority_server.rs +++ b/crates/sui-core/src/authority_server.rs @@ -198,6 +198,7 @@ pub struct ValidatorServiceMetrics { forwarded_header_parse_error: IntCounter, forwarded_header_invalid: IntCounter, forwarded_header_not_included: IntCounter, + client_id_source_config_mismatch: IntCounter, } impl ValidatorServiceMetrics { @@ -329,6 +330,12 @@ impl ValidatorServiceMetrics { registry, ) .unwrap(), + client_id_source_config_mismatch: register_int_counter_with_registry!( + "validator_service_client_id_source_config_mismatch", + "Number of times detected that client id source config doesn't agree with x-forwarded-for header", + registry, + ) + .unwrap(), } } @@ -1225,19 +1232,19 @@ impl ValidatorService { return None; } let contents_len = header_contents.len(); - // Network topology should not be very dynamic, therefore if it changes and the above - // invariant is violated, we should fail loudly so that the node config can be updated. - assert!( - contents_len >= *num_hops, - "x-forwarded-for header value of {:?} contains {} values, but {} hops were specified. \ - Expected at least {} values. Please correctly set the `x-forwarded-for` value under \ - `client-id-source` in the node config.", - header_contents, - contents_len, - num_hops, - contents_len, - ); - let contents_len = header_contents.len(); + if contents_len < *num_hops { + error!( + "x-forwarded-for header value of {:?} contains {} values, but {} hops were specified. \ + Expected at least {} values. Please correctly set the `x-forwarded-for` value under \ + `client-id-source` in the node config.", + header_contents, + contents_len, + num_hops, + contents_len, + ); + self.metrics.client_id_source_config_mismatch.inc(); + return None; + } let Some(client_ip) = header_contents.get(contents_len - num_hops) else { error!( @@ -1337,8 +1344,10 @@ fn make_tonic_request_for_testing(message: T) -> tonic::Request { // TODO: refine error matching here fn normalize(err: SuiError) -> Weight { match err { - SuiError::UserInputError { .. } - | SuiError::InvalidSignature { .. } + SuiError::UserInputError { + error: UserInputError::IncorrectUserSignature { .. }, + } => Weight::one(), + SuiError::InvalidSignature { .. } | SuiError::SignerSignatureAbsent { .. } | SuiError::SignerSignatureNumberMismatch { .. } | SuiError::IncorrectSigner { .. } diff --git a/crates/sui-core/src/traffic_controller/mod.rs b/crates/sui-core/src/traffic_controller/mod.rs index f759e1d58ee75..8d1dbbe0a8c31 100644 --- a/crates/sui-core/src/traffic_controller/mod.rs +++ b/crates/sui-core/src/traffic_controller/mod.rs @@ -284,7 +284,6 @@ async fn run_clear_blocklists_loop(blocklists: Blocklists, metrics: Arc Result<(), anyhow::Er .with_policy_config(Some(policy_config)) .build(); let committee = network_config.committee_with_network(); - let _test_cluster = TestClusterBuilder::new() + let test_cluster = TestClusterBuilder::new() .set_network_config(network_config) .build() .await; @@ -235,12 +237,13 @@ async fn test_validator_traffic_control_error_blocked() -> Result<(), anyhow::Er ); let (_, auth_client) = local_clients.first_key_value().unwrap(); - // transaction signed using user wallet from a different chain/genesis, - // therefore we should fail with UserInputError - let other_cluster = TestClusterBuilder::new().build().await; - - let mut txns = batch_make_transfer_transactions(&other_cluster.wallet, n as usize).await; - let tx = txns.swap_remove(0); + let mut txns = batch_make_transfer_transactions(&test_cluster.wallet, n as usize).await; + let mut tx = txns.swap_remove(0); + let signatures = tx.tx_signatures_mut_for_testing(); + signatures.pop(); + signatures.push(GenericSignature::Signature( + sui_types::crypto::Signature::Ed25519SuiSignature(Ed25519SuiSignature::default()), + )); // it should take no more than 4 requests to be added to the blocklist for _ in 0..n { @@ -251,7 +254,7 @@ async fn test_validator_traffic_control_error_blocked() -> Result<(), anyhow::Er } } } - panic!("Expected spam policy to trigger within {n} requests"); + panic!("Expected error policy to trigger within {n} requests"); } #[tokio::test] @@ -406,7 +409,7 @@ async fn test_validator_traffic_control_error_delegated() -> Result<(), anyhow:: .with_firewall_config(Some(firewall_config)) .build(); let committee = network_config.committee_with_network(); - let _test_cluster = TestClusterBuilder::new() + let test_cluster = TestClusterBuilder::new() .set_network_config(network_config) .build() .await; @@ -416,12 +419,13 @@ async fn test_validator_traffic_control_error_delegated() -> Result<(), anyhow:: ); let (_, auth_client) = local_clients.first_key_value().unwrap(); - // transaction signed using user wallet from a different chain/genesis, - // therefore we should fail with UserInputError - let other_cluster = TestClusterBuilder::new().build().await; - - let mut txns = batch_make_transfer_transactions(&other_cluster.wallet, n as usize).await; - let tx = txns.swap_remove(0); + let mut txns = batch_make_transfer_transactions(&test_cluster.wallet, n as usize).await; + let mut tx = txns.swap_remove(0); + let signatures = tx.tx_signatures_mut_for_testing(); + signatures.pop(); + signatures.push(GenericSignature::Signature( + sui_types::crypto::Signature::Ed25519SuiSignature(Ed25519SuiSignature::default()), + )); // start test firewall server let mut server = NodeFwTestServer::new();