From e9d68b2b833f53b0414728b456e644d66b83e3fc Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Mon, 25 Mar 2024 21:00:17 +0200 Subject: [PATCH] Optimize and refactor lookup_txos() - Avoid unnecessary copying of prev outpoints - When looking for both mempool and on-chain txos, accumulate the set of outpoints that remain to be looked up to avoid re-checking for them later again in the found set - Refactored lookup_txos() to use lookup_txo() internally rather than the other way around, which was less efficient - Lookup txos in mempool first, then on-chain - ChainQuery::lookup_txos() now returns a Result instead of panicking when outpoints are missing - Removed ChainQuery::lookup_avail_txos() and allow_missing, which are no longer neceesary --- src/bin/tx-fingerprint-stats.rs | 4 ++-- src/new_index/mempool.rs | 37 ++++++++++++++------------------- src/new_index/query.rs | 2 +- src/new_index/schema.rs | 29 +++++++++----------------- src/rest.rs | 2 +- 5 files changed, 30 insertions(+), 44 deletions(-) diff --git a/src/bin/tx-fingerprint-stats.rs b/src/bin/tx-fingerprint-stats.rs index 8e4c35e30..afe980f8c 100644 --- a/src/bin/tx-fingerprint-stats.rs +++ b/src/bin/tx-fingerprint-stats.rs @@ -83,12 +83,12 @@ fn main() { //info!("{:?},{:?}", txid, blockid); let prevouts = chain.lookup_txos( - &tx.input + tx.input .iter() .filter(|txin| has_prevout(txin)) .map(|txin| txin.previous_output) .collect(), - ); + ).unwrap(); let total_out: u64 = tx.output.iter().map(|out| out.value.to_sat()).sum(); let small_out = tx diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index 179829fd2..94ba7a41d 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -1,5 +1,5 @@ use arraydeque::{ArrayDeque, Wrapping}; -use itertools::Itertools; +use itertools::{Either, Itertools}; #[cfg(not(feature = "liquid"))] use bitcoin::consensus::encode::serialize; @@ -310,7 +310,7 @@ impl Mempool { self.txstore.insert(txid, tx); } // Phase 2: index history and spend edges (can fail if some txos cannot be found) - let txos = match self.lookup_txos(&self.get_prevouts(&txids)) { + let txos = match self.lookup_txos(self.get_prevouts(&txids)) { Ok(txos) => txos, Err(err) => { warn!("lookup txouts failed: {}", err); @@ -397,34 +397,29 @@ impl Mempool { } } - pub fn lookup_txo(&self, outpoint: &OutPoint) -> Result { - let mut outpoints = BTreeSet::new(); - outpoints.insert(*outpoint); - Ok(self.lookup_txos(&outpoints)?.remove(outpoint).unwrap()) + fn lookup_txo(&self, outpoint: &OutPoint) -> Option { + self.txstore + .get(&outpoint.txid) + .and_then(|tx| tx.output.get(outpoint.vout as usize).cloned()) } - pub fn lookup_txos(&self, outpoints: &BTreeSet) -> Result> { + pub fn lookup_txos(&self, outpoints: BTreeSet) -> Result> { let _timer = self .latency .with_label_values(&["lookup_txos"]) .start_timer(); - let confirmed_txos = self.chain.lookup_avail_txos(outpoints); + // Get the txos available in the mempool, skipping over (and collecting) missing ones + let (mut txos, remain_outpoints): (HashMap<_, _>, _) = outpoints + .into_iter() + .partition_map(|outpoint| match self.lookup_txo(&outpoint) { + Some(txout) => Either::Left((outpoint, txout)), + None => Either::Right(outpoint), + }); - let mempool_txos = outpoints - .iter() - .filter(|outpoint| !confirmed_txos.contains_key(outpoint)) - .map(|outpoint| { - self.txstore - .get(&outpoint.txid) - .and_then(|tx| tx.output.get(outpoint.vout as usize).cloned()) - .map(|txout| (*outpoint, txout)) - .chain_err(|| format!("missing outpoint {:?}", outpoint)) - }) - .collect::>>()?; + // Get the remaining txos from the chain (fails if any are missing) + txos.extend(self.chain.lookup_txos(remain_outpoints)?); - let mut txos = confirmed_txos; - txos.extend(mempool_txos); Ok(txos) } diff --git a/src/new_index/query.rs b/src/new_index/query.rs index 1e621ac0d..60d7510ab 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -118,7 +118,7 @@ impl Query { .or_else(|| self.mempool().lookup_raw_txn(txid)) } - pub fn lookup_txos(&self, outpoints: &BTreeSet) -> HashMap { + pub fn lookup_txos(&self, outpoints: BTreeSet) -> HashMap { // the mempool lookup_txos() internally looks up confirmed txos as well self.mempool() .lookup_txos(outpoints) diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 95346cdef..d738050a1 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -327,7 +327,7 @@ impl Indexer { fn index(&self, blocks: &[BlockEntry]) { let previous_txos_map = { let _timer = self.start_timer("index_lookup"); - lookup_txos(&self.store.txstore_db, &get_previous_txos(blocks), false) + lookup_txos(&self.store.txstore_db, get_previous_txos(blocks)).unwrap() }; let rows = { let _timer = self.start_timer("index_process"); @@ -859,14 +859,9 @@ impl ChainQuery { lookup_txo(&self.store.txstore_db, outpoint) } - pub fn lookup_txos(&self, outpoints: &BTreeSet) -> HashMap { + pub fn lookup_txos(&self, outpoints: BTreeSet) -> Result> { let _timer = self.start_timer("lookup_txos"); - lookup_txos(&self.store.txstore_db, outpoints, false) - } - - pub fn lookup_avail_txos(&self, outpoints: &BTreeSet) -> HashMap { - let _timer = self.start_timer("lookup_available_txos"); - lookup_txos(&self.store.txstore_db, outpoints, true) + lookup_txos(&self.store.txstore_db, outpoints) } pub fn lookup_spend(&self, outpoint: &OutPoint) -> Option { @@ -1033,21 +1028,17 @@ fn get_previous_txos(block_entries: &[BlockEntry]) -> BTreeSet { .collect() } -fn lookup_txos( - txstore_db: &DB, - outpoints: &BTreeSet, - allow_missing: bool, -) -> HashMap { - let mut remain_outpoints = outpoints.iter(); +fn lookup_txos(txstore_db: &DB, outpoints: BTreeSet) -> Result> { + let keys = outpoints.iter().map(TxOutRow::key).collect::>(); + let mut remain_outpoints = outpoints.into_iter(); txstore_db - .multi_get(outpoints.iter().map(TxOutRow::key)) + .multi_get(keys) .into_iter() - .filter_map(|res| { + .map(|res| { let outpoint = remain_outpoints.next().unwrap(); match res.unwrap() { - Some(txo) => Some((*outpoint, deserialize(&txo).expect("failed to parse TxOut"))), - None if allow_missing => None, - None => panic!("missing txo {}", outpoint), + Some(txo) => Ok((outpoint, deserialize(&txo).expect("failed to parse TxOut"))), + None => Err(format!("missing txo {}", outpoint).into()), } }) .collect() diff --git a/src/rest.rs b/src/rest.rs index 336c43f4a..43eab5c60 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -468,7 +468,7 @@ fn prepare_txs( }) .collect(); - let prevouts = query.lookup_txos(&outpoints); + let prevouts = query.lookup_txos(outpoints); txs.into_iter() .map(|(tx, blockid)| TransactionValue::new(tx, blockid, &prevouts, config))