[POC/WIP] Add optional Bip352 silentpayments index

Cargo.toml

@@ -22,9 +22,9 @@ spec = "internal/config_specification.toml"
 
 [dependencies]
 anyhow = "1.0"
-bitcoin = { version = "0.32.2", features = ["serde", "rand-std"] }
-bitcoin_slices = { version = "0.8", features = ["bitcoin", "sha2"] }
-bitcoincore-rpc = { version = "0.19.0" }
+bitcoin = { git = "https://github.com/jlest01/rust-bitcoin.git", branch = "bip352-silentpayments-module-v0", features = ["serde", "rand-std"] }
+bitcoin_slices = { git = "https://github.com/jlest01/bitcoin_slices.git", branch = "bip352-silentpayments-module-v0", features = ["bitcoin", "sha2"] }
+bitcoincore-rpc = {  git = "https://github.com/jlest01/rust-bitcoincore-rpc.git", branch = "bip352-silentpayments-module-v0" }
 configure_me = "0.4"
 crossbeam-channel = "0.5"
 dirs-next = "2.0"

internal/config_specification.toml

@@ -145,3 +145,7 @@ doc = "Logging filters, overriding `RUST_LOG` environment variable (see https://
 name = "signet_magic"
 type = "String"
 doc = "network magic for custom signet network in hex format, as found in Bitcoin Core logs (signet only)"
+
+[[switch]]
+name = "silent_payments_index"
+doc = "Enable silent payments index."

rust-toolchain.toml

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "1.63.0"
+channel = "1.70.0"
 components = [ "rustfmt" ]

src/config.rs

@@ -146,6 +146,7 @@ pub struct Config {
     pub disable_electrum_rpc: bool,
     pub server_banner: String,
     pub signet_magic: Magic,
+    pub silent_payments_index: bool,
 }
 
 pub struct SensitiveAuth(pub Auth);
@@ -368,6 +369,7 @@ impl Config {
             disable_electrum_rpc: config.disable_electrum_rpc,
             server_banner: config.server_banner,
             signet_magic: magic,
+            silent_payments_index: config.silent_payments_index,
         };
         eprintln!(
             "Starting electrs {} on {} {} with {:?}",

src/db.rs

@@ -9,10 +9,12 @@ use crate::types::{HashPrefix, SerializedHashPrefixRow, SerializedHeaderRow};
 #[derive(Default)]
 pub(crate) struct WriteBatch {
     pub(crate) tip_row: [u8; 32],
+    pub(crate) sp_tip_row: [u8; 32],
     pub(crate) header_rows: Vec<SerializedHeaderRow>,
     pub(crate) funding_rows: Vec<SerializedHashPrefixRow>,
     pub(crate) spending_rows: Vec<SerializedHashPrefixRow>,
     pub(crate) txid_rows: Vec<SerializedHashPrefixRow>,
+    pub(crate) tweak_rows: Vec<Vec<u8>>,
 }
 
 impl WriteBatch {
@@ -21,6 +23,7 @@ impl WriteBatch {
         self.funding_rows.sort_unstable();
         self.spending_rows.sort_unstable();
         self.txid_rows.sort_unstable();
+        self.tweak_rows.sort_unstable();
     }
 }
 
@@ -35,11 +38,13 @@ const HEADERS_CF: &str = "headers";
 const TXID_CF: &str = "txid";
 const FUNDING_CF: &str = "funding";
 const SPENDING_CF: &str = "spending";
+const TWEAK_CF: &str = "tweak";
 
-const COLUMN_FAMILIES: &[&str] = &[CONFIG_CF, HEADERS_CF, TXID_CF, FUNDING_CF, SPENDING_CF];
+const COLUMN_FAMILIES: &[&str] = &[CONFIG_CF, HEADERS_CF, TXID_CF, FUNDING_CF, SPENDING_CF, TWEAK_CF];
 
 const CONFIG_KEY: &str = "C";
 const TIP_KEY: &[u8] = b"T";
+const SP_KEY: &[u8] = b"SP";
 
 // Taken from https://github.com/facebook/rocksdb/blob/master/include/rocksdb/db.h#L654-L689
 const DB_PROPERTIES: &[&str] = &[
@@ -218,6 +223,10 @@ impl DBStore {
         self.db.cf_handle(HEADERS_CF).expect("missing HEADERS_CF")
     }
 
+    fn tweak_cf(&self) -> &rocksdb::ColumnFamily {
+        self.db.cf_handle(TWEAK_CF).expect("missing TWEAK_CF")
+    }
+
     pub(crate) fn iter_funding(
         &self,
         prefix: HashPrefix,
@@ -264,12 +273,28 @@ impl DBStore {
         self.iter_cf(self.headers_cf(), opts, None)
     }
 
+    pub(crate) fn read_tweaks(&self, height: u64) -> Vec<(Box<[u8]>, Box<[u8]>)> {
+        let mut opts = rocksdb::ReadOptions::default();
+        opts.set_iterate_lower_bound(height.to_be_bytes());
+        opts.fill_cache(false);
+        self.db
+            .iterator_cf_opt(self.tweak_cf(), opts, rocksdb::IteratorMode::Start)
+            .map(|row| row.expect("tweak iterator failed"))
+            .collect()
+    }
+
     pub(crate) fn get_tip(&self) -> Option<Vec<u8>> {
         self.db
             .get_cf(self.headers_cf(), TIP_KEY)
             .expect("get_tip failed")
     }
 
+    pub(crate) fn last_sp(&self) -> Option<Vec<u8>> {
+        self.db
+            .get_cf(self.headers_cf(), SP_KEY)
+            .expect("last_sp failed")
+    }
+
     pub(crate) fn write(&self, batch: &WriteBatch) {
         let mut db_batch = rocksdb::WriteBatch::default();
         for key in &batch.funding_rows {
@@ -293,6 +318,23 @@ impl DBStore {
         self.db.write_opt(db_batch, &opts).unwrap();
     }
 
+    pub(crate) fn write_sp(&self, batch: &WriteBatch) {
+        let mut db_batch = rocksdb::WriteBatch::default();
+
+        for key in &batch.tweak_rows {
+            if key.len() > 8 {
+                db_batch.put_cf(self.tweak_cf(), &key[..8], &key[8..]);
+            }
+        }
+        db_batch.put_cf(self.headers_cf(), SP_KEY, batch.sp_tip_row);
+
+        let mut opts = rocksdb::WriteOptions::new();
+        let bulk_import = self.bulk_import.load(Ordering::Relaxed);
+        opts.set_sync(!bulk_import);
+        opts.disable_wal(bulk_import);
+        self.db.write_opt(db_batch, &opts).unwrap();
+    }
+
     pub(crate) fn flush(&self) {
         debug!("flushing DB column families");
         let mut config = self.get_config().unwrap_or_default();

src/electrum.rs

@@ -235,6 +235,10 @@ impl Rpc {
         Ok(json!({"count": count, "hex": String::from_iter(hex_headers), "max": max_count}))
     }
 
+    fn sp_tweaks(&self, (start_height,): (usize,)) -> Result<Value> {
+        Ok(json!(self.tracker.get_tweaks(start_height)?))
+    }
+
     fn estimate_fee(&self, (nblocks,): (u16,)) -> Result<Value> {
         Ok(self
             .daemon
@@ -529,7 +533,18 @@ impl Rpc {
             Err(response) => return response, // params parsing may fail - the response contains request id
         };
         self.rpc_duration.observe_duration(&call.method, || {
-            if self.tracker.status().is_err() {
+            let is_sp_indexing = self.tracker.silent_payments_index && self.tracker.sp_status().is_err();
+
+            if is_sp_indexing {
+                match &call.params {
+                    Params::SpTweaks(_) => {
+                        return error_msg(&call.id, RpcError::UnavailableIndex)
+                    }
+                    _ => (),
+                };
+            }
+
+            if is_sp_indexing || self.tracker.status().is_err() {
                 // Allow only a few RPC (for sync status notification) not requiring index DB being compacted.
                 match &call.params {
                     Params::BlockHeader(_)
@@ -543,6 +558,7 @@ impl Rpc {
                 Params::Banner => Ok(json!(self.banner)),
                 Params::BlockHeader(args) => self.block_header(*args),
                 Params::BlockHeaders(args) => self.block_headers(*args),
+                Params::SpTweaks(args) => self.sp_tweaks(*args),
                 Params::Donation => Ok(Value::Null),
                 Params::EstimateFee(args) => self.estimate_fee(*args),
                 Params::Features => self.features(),
@@ -572,6 +588,7 @@ enum Params {
     Banner,
     BlockHeader((usize,)),
     BlockHeaders((usize, usize)),
+    SpTweaks((usize,)),
     TransactionBroadcast((String,)),
     Donation,
     EstimateFee((u16,)),
@@ -597,6 +614,7 @@ impl Params {
         Ok(match method {
             "blockchain.block.header" => Params::BlockHeader(convert(params)?),
             "blockchain.block.headers" => Params::BlockHeaders(convert(params)?),
+            "blockchain.block.tweaks" => Params::SpTweaks(convert(params)?),
             "blockchain.estimatefee" => Params::EstimateFee(convert(params)?),
             "blockchain.headers.subscribe" => Params::HeadersSubscribe,
             "blockchain.relayfee" => Params::RelayFee,