From: 志宇 Date: Thu, 22 May 2025 07:14:18 +0000 (+1000) Subject: feat(chain)!: Persist spks derived from `KeychainTxOutIndex` X-Git-Tag: core-0.6.0~1^2~7 X-Git-Url: http://internal-gitweb-vhost/script/%22https:/database/sparse_chain/enum.InsertTxError.html?a=commitdiff_plain;h=d299daea4278f9c63fcf57ae7b7079c0b6f7ebc4;p=bdk feat(chain)!: Persist spks derived from `KeychainTxOutIndex` --- diff --git a/crates/bitcoind_rpc/examples/filter_iter.rs b/crates/bitcoind_rpc/examples/filter_iter.rs index 21300e70..b80542bc 100644 --- a/crates/bitcoind_rpc/examples/filter_iter.rs +++ b/crates/bitcoind_rpc/examples/filter_iter.rs @@ -33,8 +33,9 @@ fn main() -> anyhow::Result<()> { let (mut chain, _) = LocalChain::from_genesis_hash(genesis_block(NETWORK).block_hash()); let mut graph = IndexedTxGraph::>::new({ let mut index = KeychainTxOutIndex::default(); - index.insert_descriptor("external", descriptor.clone())?; - index.insert_descriptor("internal", change_descriptor.clone())?; + // TODO: Properly deal with these changesets. + let _ = index.insert_descriptor("external", descriptor.clone())?; + let _ = index.insert_descriptor("internal", change_descriptor.clone())?; index }); diff --git a/crates/chain/benches/canonicalization.rs b/crates/chain/benches/canonicalization.rs index d425ecc6..4cb26c6b 100644 --- a/crates/chain/benches/canonicalization.rs +++ b/crates/chain/benches/canonicalization.rs @@ -83,7 +83,7 @@ fn setup(f: F) -> (KeychainTxGraph, Lo let (desc, _) = >::parse_descriptor(&Secp256k1::new(), DESC).unwrap(); let mut index = KeychainTxOutIndex::new(10); - index.insert_descriptor((), desc).unwrap(); + let _ = index.insert_descriptor((), desc).unwrap(); let mut tx_graph = KeychainTxGraph::new(index); f(&mut tx_graph, &chain); diff --git a/crates/chain/src/indexer/keychain_txout.rs b/crates/chain/src/indexer/keychain_txout.rs index 11d234e6..0fa11ad8 100644 --- a/crates/chain/src/indexer/keychain_txout.rs +++ b/crates/chain/src/indexer/keychain_txout.rs @@ -2,6 +2,7 @@ //! indexes [`TxOut`]s with them. use crate::{ + alloc::boxed::Box, collections::*, miniscript::{Descriptor, DescriptorPublicKey}, spk_client::{FullScanRequestBuilder, SyncRequestBuilder}, @@ -10,7 +11,9 @@ use crate::{ DescriptorExt, DescriptorId, Indexed, Indexer, KeychainIndexed, SpkIterator, }; use alloc::{borrow::ToOwned, vec::Vec}; -use bitcoin::{Amount, OutPoint, ScriptBuf, SignedAmount, Transaction, TxOut, Txid}; +use bitcoin::{ + key::Secp256k1, Amount, OutPoint, ScriptBuf, SignedAmount, Transaction, TxOut, Txid, +}; use core::{ fmt::Debug, ops::{Bound, RangeBounds}, @@ -128,6 +131,8 @@ pub struct KeychainTxOutIndex { descriptors: HashMap>, last_revealed: HashMap, lookahead: u32, + + spk_cache: BTreeMap>, } impl Default for KeychainTxOutIndex { @@ -155,7 +160,12 @@ impl Indexer for KeychainTxOutIndex { if self.last_revealed.get(did) < Some(&index) { self.last_revealed.insert(*did, index); changeset.last_revealed.insert(*did, index); - self.replenish_inner_index(*did, &keychain, self.lookahead); + self.replenish_inner_index( + *did, + &keychain, + self.lookahead, + changeset.spk_cache.entry(*did).or_default(), + ); } } changeset @@ -173,6 +183,16 @@ impl Indexer for KeychainTxOutIndex { fn initial_changeset(&self) -> Self::ChangeSet { ChangeSet { last_revealed: self.last_revealed.clone().into_iter().collect(), + spk_cache: self + .spk_cache + .iter() + .map(|(desc, spks)| { + ( + *desc, + spks.iter().map(|(i, spk)| (*i, spk.clone())).collect(), + ) + }) + .collect(), } } @@ -204,6 +224,7 @@ impl KeychainTxOutIndex { descriptor_id_to_keychain: Default::default(), last_revealed: Default::default(), lookahead, + spk_cache: Default::default(), } } @@ -365,7 +386,9 @@ impl KeychainTxOutIndex { &mut self, keychain: K, descriptor: Descriptor, - ) -> Result> { + ) -> Result<(bool, ChangeSet), InsertDescriptorError> { + let mut changeset = ChangeSet::default(); + let did = descriptor.descriptor_id(); if !self.keychain_to_descriptor_id.contains_key(&keychain) && !self.descriptor_id_to_keychain.contains_key(&did) @@ -373,15 +396,20 @@ impl KeychainTxOutIndex { self.descriptors.insert(did, descriptor.clone()); self.keychain_to_descriptor_id.insert(keychain.clone(), did); self.descriptor_id_to_keychain.insert(did, keychain.clone()); - self.replenish_inner_index(did, &keychain, self.lookahead); - return Ok(true); + self.replenish_inner_index( + did, + &keychain, + self.lookahead, + changeset.spk_cache.entry(did).or_default(), + ); + return Ok((true, changeset)); } if let Some(existing_desc_id) = self.keychain_to_descriptor_id.get(&keychain) { let descriptor = self.descriptors.get(existing_desc_id).expect("invariant"); if *existing_desc_id != did { return Err(InsertDescriptorError::KeychainAlreadyAssigned { - existing_assignment: descriptor.clone(), + existing_assignment: Box::new(descriptor.clone()), keychain, }); } @@ -393,12 +421,12 @@ impl KeychainTxOutIndex { if *existing_keychain != keychain { return Err(InsertDescriptorError::DescriptorAlreadyAssigned { existing_assignment: existing_keychain.clone(), - descriptor, + descriptor: Box::new(descriptor), }); } } - Ok(false) + Ok((false, changeset)) } /// Gets the descriptor associated with the keychain. Returns `None` if the keychain doesn't @@ -420,47 +448,100 @@ impl KeychainTxOutIndex { /// Store lookahead scripts until `target_index` (inclusive). /// /// This does not change the global `lookahead` setting. - pub fn lookahead_to_target(&mut self, keychain: K, target_index: u32) { + pub fn lookahead_to_target( + &mut self, + keychain: K, + target_index: u32, + derived_spks: &mut impl Extend>, + ) { if let Some((next_index, _)) = self.next_index(keychain.clone()) { let temp_lookahead = (target_index + 1) .checked_sub(next_index) .filter(|&index| index > 0); if let Some(temp_lookahead) = temp_lookahead { - self.replenish_inner_index_keychain(keychain, temp_lookahead); + self.replenish_inner_index_keychain(keychain, temp_lookahead, derived_spks); } } } - fn replenish_inner_index_did(&mut self, did: DescriptorId, lookahead: u32) { + fn replenish_inner_index_did( + &mut self, + did: DescriptorId, + lookahead: u32, + derived_spks: &mut impl Extend>, + ) { if let Some(keychain) = self.descriptor_id_to_keychain.get(&did).cloned() { - self.replenish_inner_index(did, &keychain, lookahead); + self.replenish_inner_index(did, &keychain, lookahead, derived_spks); } } - fn replenish_inner_index_keychain(&mut self, keychain: K, lookahead: u32) { + fn replenish_inner_index_keychain( + &mut self, + keychain: K, + lookahead: u32, + derived_spks: &mut impl Extend>, + ) { if let Some(did) = self.keychain_to_descriptor_id.get(&keychain) { - self.replenish_inner_index(*did, &keychain, lookahead); + self.replenish_inner_index(*did, &keychain, lookahead, derived_spks); } } /// Syncs the state of the inner spk index after changes to a keychain - fn replenish_inner_index(&mut self, did: DescriptorId, keychain: &K, lookahead: u32) { + fn replenish_inner_index( + &mut self, + did: DescriptorId, + keychain: &K, + lookahead: u32, + derived_spks: &mut impl Extend>, + ) { let descriptor = self.descriptors.get(&did).expect("invariant"); - let next_store_index = self + + let mut next_index = self .inner .all_spks() .range(&(keychain.clone(), u32::MIN)..=&(keychain.clone(), u32::MAX)) .last() .map_or(0, |((_, index), _)| *index + 1); - let next_reveal_index = self.last_revealed.get(&did).map_or(0, |v| *v + 1); - for (new_index, new_spk) in - SpkIterator::new_with_range(descriptor, next_store_index..next_reveal_index + lookahead) - { + + // Exclusive: index to stop at. + let stop_index = if descriptor.has_wildcard() { + let next_reveal_index = self.last_revealed.get(&did).map_or(0, |v| *v + 1); + (next_reveal_index + lookahead).min(BIP32_MAX_INDEX) + } else { + 1 + }; + + let cached_spk_iter = core::iter::from_fn({ + let secp = Secp256k1::verification_only(); + let _desc = &descriptor; + let spk_cache = self.spk_cache.entry(did).or_default(); + let _i = &mut next_index; + move || -> Option> { + if *_i >= stop_index { + return None; + } + let spk_i = *_i; + *_i = spk_i.saturating_add(1); + + if let Some(spk) = spk_cache.get(_i) { + return Some((spk_i, spk.clone())); + } + let spk = _desc + .derived_descriptor(&secp, spk_i) + .expect("The descriptor cannot have hardened derivation") + .script_pubkey(); + derived_spks.extend(core::iter::once((spk_i, spk.clone()))); + spk_cache.insert(spk_i, spk.clone()); + Some((spk_i, spk.clone())) + } + }); + + for (new_index, new_spk) in cached_spk_iter { let _inserted = self .inner .insert_spk((keychain.clone(), new_index), new_spk); - debug_assert!(_inserted, "replenish lookahead: must not have existing spk: keychain={:?}, lookahead={}, next_store_index={}, next_reveal_index={}", keychain, lookahead, next_store_index, next_reveal_index); + debug_assert!(_inserted, "replenish lookahead: must not have existing spk: keychain={:?}, lookahead={}, next_index={}", keychain, lookahead, next_index); } } @@ -693,7 +774,12 @@ impl KeychainTxOutIndex { let did = self.keychain_to_descriptor_id.get(&keychain)?; self.last_revealed.insert(*did, next_index); changeset.last_revealed.insert(*did, next_index); - self.replenish_inner_index(*did, &keychain, self.lookahead); + self.replenish_inner_index( + *did, + &keychain, + self.lookahead, + changeset.spk_cache.entry(*did).or_default(), + ); } let script = self .inner @@ -779,10 +865,13 @@ impl KeychainTxOutIndex { /// Applies the `ChangeSet` to the [`KeychainTxOutIndex`] pub fn apply_changeset(&mut self, changeset: ChangeSet) { - for (&desc_id, &index) in &changeset.last_revealed { - let v = self.last_revealed.entry(desc_id).or_default(); + for (did, index) in changeset.last_revealed { + let v = self.last_revealed.entry(did).or_default(); *v = index.max(*v); - self.replenish_inner_index_did(desc_id, self.lookahead); + self.replenish_inner_index_did(did, self.lookahead, &mut Vec::new()); + } + for (did, spks) in changeset.spk_cache { + self.spk_cache.entry(did).or_default().extend(spks); } } } @@ -793,7 +882,7 @@ pub enum InsertDescriptorError { /// The descriptor has already been assigned to a keychain so you can't assign it to another DescriptorAlreadyAssigned { /// The descriptor you have attempted to reassign - descriptor: Descriptor, + descriptor: Box>, /// The keychain that the descriptor is already assigned to existing_assignment: K, }, @@ -802,7 +891,7 @@ pub enum InsertDescriptorError { /// The keychain that you have attempted to reassign keychain: K, /// The descriptor that the keychain is already assigned to - existing_assignment: Descriptor, + existing_assignment: Box>, }, } @@ -852,6 +941,10 @@ impl std::error::Error for InsertDescriptorError {} pub struct ChangeSet { /// Contains for each descriptor_id the last revealed index of derivation pub last_revealed: BTreeMap, + + /// Spk cache. + #[cfg_attr(feature = "serde", serde(default))] + pub spk_cache: BTreeMap>, } impl Merge for ChangeSet { @@ -872,11 +965,15 @@ impl Merge for ChangeSet { } } } + + for (did, spks) in other.spk_cache { + self.spk_cache.entry(did).or_default().extend(spks); + } } /// Returns whether the changeset are empty. fn is_empty(&self) -> bool { - self.last_revealed.is_empty() + self.last_revealed.is_empty() && self.spk_cache.is_empty() } } diff --git a/crates/chain/src/rusqlite_impl.rs b/crates/chain/src/rusqlite_impl.rs index 3bc105d0..ca13115b 100644 --- a/crates/chain/src/rusqlite_impl.rs +++ b/crates/chain/src/rusqlite_impl.rs @@ -521,6 +521,8 @@ impl keychain_txout::ChangeSet { pub const SCHEMA_NAME: &'static str = "bdk_keychaintxout"; /// Name for table that stores last revealed indices per descriptor id. pub const LAST_REVEALED_TABLE_NAME: &'static str = "bdk_descriptor_last_revealed"; + /// Name for table that stores derived spks. + pub const DERIVED_SPKS_TABLE_NAME: &'static str = "bdk_descriptor_derived_spks"; /// Get v0 of sqlite [keychain_txout::ChangeSet] schema pub fn schema_v0() -> String { @@ -533,10 +535,28 @@ impl keychain_txout::ChangeSet { ) } + /// Get v1 of sqlite [keychain_txout::ChangeSet] schema + pub fn schema_v1() -> String { + format!( + "CREATE TABLE {} ( \ + descriptor_id TEXT NOT NULL REFERENCES {}, \ + spk_index INTEGER NOT NULL, \ + spk BLOB NOT NULL, \ + PRIMARY KEY (descriptor_id, index) \ + ) STRICT", + Self::DERIVED_SPKS_TABLE_NAME, + Self::LAST_REVEALED_TABLE_NAME, + ) + } + /// Initialize sqlite tables for persisting /// [`KeychainTxOutIndex`](keychain_txout::KeychainTxOutIndex). pub fn init_sqlite_tables(db_tx: &rusqlite::Transaction) -> rusqlite::Result<()> { - migrate_schema(db_tx, Self::SCHEMA_NAME, &[&Self::schema_v0()]) + migrate_schema( + db_tx, + Self::SCHEMA_NAME, + &[&Self::schema_v0(), &Self::schema_v1()], + ) } /// Construct [`KeychainTxOutIndex`](keychain_txout::KeychainTxOutIndex) from sqlite database @@ -561,6 +581,26 @@ impl keychain_txout::ChangeSet { changeset.last_revealed.insert(descriptor_id, last_revealed); } + let mut statement = db_tx.prepare(&format!( + "SELECT descriptor_id, spk_index, spk FROM {}", + Self::DERIVED_SPKS_TABLE_NAME + ))?; + let row_iter = statement.query_map([], |row| { + Ok(( + row.get::<_, Impl>("descriptor_id")?, + row.get::<_, u32>("spk_index")?, + row.get::<_, Impl>("spk")?, + )) + })?; + for row in row_iter { + let (Impl(descriptor_id), spk_index, Impl(spk)) = row?; + changeset + .spk_cache + .entry(descriptor_id) + .or_default() + .insert(spk_index, spk); + } + Ok(changeset) } @@ -579,6 +619,20 @@ impl keychain_txout::ChangeSet { })?; } + let mut statement = db_tx.prepare_cached(&format!( + "REPLACE INTO {}(descriptor_id, spk_index, spk) VALUES(:descriptor_id, :spk_index, :spk)", + Self::DERIVED_SPKS_TABLE_NAME, + ))?; + for (&descriptor_id, spks) in &self.spk_cache { + for (&spk_index, spk) in spks { + statement.execute(named_params! { + ":descriptor_id": Impl(descriptor_id), + ":spk_index": spk_index, + ":spk": Impl(spk.clone()), + })?; + } + } + Ok(()) } } diff --git a/crates/chain/tests/test_indexed_tx_graph.rs b/crates/chain/tests/test_indexed_tx_graph.rs index 4846841c..abc2c823 100644 --- a/crates/chain/tests/test_indexed_tx_graph.rs +++ b/crates/chain/tests/test_indexed_tx_graph.rs @@ -10,6 +10,7 @@ use bdk_chain::{ indexer::keychain_txout::KeychainTxOutIndex, local_chain::LocalChain, tx_graph, Balance, CanonicalizationParams, ChainPosition, ConfirmationBlockTime, DescriptorExt, + SpkIterator, }; use bdk_testenv::{ block_id, hash, @@ -32,14 +33,27 @@ fn insert_relevant_txs() { .expect("must be valid"); let spk_0 = descriptor.at_derivation_index(0).unwrap().script_pubkey(); let spk_1 = descriptor.at_derivation_index(9).unwrap().script_pubkey(); + let lookahead = 10; let mut graph = IndexedTxGraph::>::new( - KeychainTxOutIndex::new(10), + KeychainTxOutIndex::new(lookahead), ); - let _ = graph + let (is_inserted, changeset) = graph .index .insert_descriptor((), descriptor.clone()) .unwrap(); + assert!(is_inserted); + assert_eq!( + changeset, + keychain_txout::ChangeSet { + spk_cache: [( + descriptor.descriptor_id(), + SpkIterator::new_with_range(&descriptor, 0..lookahead).collect() + )] + .into(), + ..Default::default() + } + ); let tx_a = Transaction { output: vec![ @@ -80,6 +94,15 @@ fn insert_relevant_txs() { }, indexer: keychain_txout::ChangeSet { last_revealed: [(descriptor.descriptor_id(), 9_u32)].into(), + spk_cache: [(descriptor.descriptor_id(), { + let index_after_spk_1 = 9 /* index of spk_1 */ + 1; + SpkIterator::new_with_range( + &descriptor, + index_after_spk_1..index_after_spk_1 + lookahead, + ) + .collect() + })] + .into(), }, }; @@ -93,6 +116,12 @@ fn insert_relevant_txs() { tx_graph: changeset.tx_graph, indexer: keychain_txout::ChangeSet { last_revealed: changeset.indexer.last_revealed, + spk_cache: [( + descriptor.descriptor_id(), + SpkIterator::new_with_range(&descriptor, 0..=9 /* index of spk_1*/ + lookahead) + .collect(), + )] + .into(), }, }; @@ -144,14 +173,20 @@ fn test_list_owned_txouts() { KeychainTxOutIndex::new(10), ); - assert!(graph - .index - .insert_descriptor("keychain_1".into(), desc_1) - .unwrap()); - assert!(graph - .index - .insert_descriptor("keychain_2".into(), desc_2) - .unwrap()); + assert!( + graph + .index + .insert_descriptor("keychain_1".into(), desc_1) + .unwrap() + .0 + ); + assert!( + graph + .index + .insert_descriptor("keychain_2".into(), desc_2) + .unwrap() + .0 + ); // Get trusted and untrusted addresses diff --git a/crates/chain/tests/test_keychain_txout_index.rs b/crates/chain/tests/test_keychain_txout_index.rs index 8b299b89..27baeacc 100644 --- a/crates/chain/tests/test_keychain_txout_index.rs +++ b/crates/chain/tests/test_keychain_txout_index.rs @@ -3,7 +3,7 @@ use bdk_chain::{ collections::BTreeMap, indexer::keychain_txout::{ChangeSet, KeychainTxOutIndex}, - DescriptorExt, DescriptorId, Indexer, Merge, + DescriptorExt, DescriptorId, Indexer, Merge, SpkIterator, }; use bdk_testenv::{ hash, @@ -81,9 +81,11 @@ fn merge_changesets_check_last_revealed() { let mut lhs = ChangeSet { last_revealed: lhs_di, + ..Default::default() }; let rhs = ChangeSet { last_revealed: rhs_di, + ..Default::default() }; lhs.merge(rhs); @@ -99,10 +101,14 @@ fn merge_changesets_check_last_revealed() { #[test] fn test_set_all_derivation_indices() { + let lookahead = 0; let external_descriptor = parse_descriptor(DESCRIPTORS[0]); let internal_descriptor = parse_descriptor(DESCRIPTORS[1]); - let mut txout_index = - init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 0); + let mut txout_index = init_txout_index( + external_descriptor.clone(), + internal_descriptor.clone(), + lookahead, + ); let derive_to: BTreeMap<_, _> = [(TestKeychain::External, 12), (TestKeychain::Internal, 24)].into(); let last_revealed: BTreeMap<_, _> = [ @@ -110,10 +116,22 @@ fn test_set_all_derivation_indices() { (internal_descriptor.descriptor_id(), 24), ] .into(); + let spk_cache: BTreeMap> = [ + ( + external_descriptor.descriptor_id(), + SpkIterator::new_with_range(&external_descriptor, 0..=12).collect(), + ), + ( + internal_descriptor.descriptor_id(), + SpkIterator::new_with_range(&internal_descriptor, 0..=24).collect(), + ), + ] + .into(); assert_eq!( txout_index.reveal_to_target_multi(&derive_to), ChangeSet { - last_revealed: last_revealed.clone() + last_revealed: last_revealed.clone(), + spk_cache: spk_cache.clone(), } ); assert_eq!(txout_index.last_revealed_indices(), derive_to); @@ -589,7 +607,7 @@ fn lookahead_to_target() { } None => target, }; - index.lookahead_to_target(keychain.clone(), target); + index.lookahead_to_target(keychain.clone(), target, &mut Vec::new()); let keys: Vec<_> = (0..) .take_while(|&i| index.spk_at_index(keychain.clone(), i).is_some()) .collect(); @@ -606,14 +624,16 @@ fn applying_changesets_one_by_one_vs_aggregate_must_have_same_result() { let changesets: &[ChangeSet] = &[ ChangeSet { last_revealed: [(desc.descriptor_id(), 10)].into(), + ..Default::default() }, ChangeSet { last_revealed: [(desc.descriptor_id(), 12)].into(), + ..Default::default() }, ]; let mut indexer_a = KeychainTxOutIndex::::new(0); - indexer_a + let _ = indexer_a .insert_descriptor(TestKeychain::External, desc.clone()) .expect("must insert keychain"); for changeset in changesets { @@ -621,7 +641,7 @@ fn applying_changesets_one_by_one_vs_aggregate_must_have_same_result() { } let mut indexer_b = KeychainTxOutIndex::::new(0); - indexer_b + let _ = indexer_b .insert_descriptor(TestKeychain::External, desc.clone()) .expect("must insert keychain"); let aggregate_changesets = changesets diff --git a/examples/example_cli/src/lib.rs b/examples/example_cli/src/lib.rs index d432d12b..1fb1abc3 100644 --- a/examples/example_cli/src/lib.rs +++ b/examples/example_cli/src/lib.rs @@ -836,10 +836,13 @@ pub fn init_or_load( // insert descriptors and apply loaded changeset let mut index = KeychainTxOutIndex::default(); if let Some(desc) = changeset.descriptor { - index.insert_descriptor(Keychain::External, desc)?; + // TODO: We should apply changeset to the graph before inserting descriptors. + // TODO: Then persist the new _changesets returned to db. + let (_, _changeset) = index.insert_descriptor(Keychain::External, desc)?; } if let Some(change_desc) = changeset.change_descriptor { - index.insert_descriptor(Keychain::Internal, change_desc)?; + let (_, _changeset) = + index.insert_descriptor(Keychain::Internal, change_desc)?; } let mut graph = KeychainTxGraph::new(index); graph.apply_changeset(indexed_tx_graph::ChangeSet {