diff --git a/.vscode/settings.json b/.vscode/settings.json index e083a622..f01e03c0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,7 @@ { "cSpell.words": [ "analyser", + "Appendable", "Banksy", "bdfs", "Fulfillable", diff --git a/src/factor_instances_provider/agnostic_paths/derivation_preset.rs b/src/factor_instances_provider/agnostic_paths/derivation_preset.rs index 5e04a035..20dab4ba 100644 --- a/src/factor_instances_provider/agnostic_paths/derivation_preset.rs +++ b/src/factor_instances_provider/agnostic_paths/derivation_preset.rs @@ -2,26 +2,30 @@ use crate::prelude::*; /// Derivation Presets are Network agnostic and Index agnostic /// "templates" for DerivationPaths. -#[derive(Clone, Debug, Copy, Hash, PartialEq, Eq, enum_iterator::Sequence)] +#[derive(Clone, Copy, Hash, PartialEq, Eq, enum_iterator::Sequence, derive_more::Debug)] pub enum DerivationPreset { /// Used to form DerivationPaths used to derive FactorInstances /// for "veci": Virtual Entity Creating (Factor)Instance for accounts. /// `(EntityKind::Account, KeySpace::Unsecurified, KeyKind::TransactionSigning)` + #[debug("A-VECI")] AccountVeci, /// Used to form DerivationPaths used to derive FactorInstances /// for "mfa" to securify accounts. /// `(EntityKind::Account, KeySpace::Securified, KeyKind::TransactionSigning)` + #[debug("A-MFA")] AccountMfa, /// Used to form DerivationPaths used to derive FactorInstances /// for "veci": Virtual Entity Creating (Factor)Instance for personas. /// `(EntityKind::Identity, KeySpace::Unsecurified, KeyKind::TransactionSigning)` + #[debug("I-VECI")] IdentityVeci, /// Used to form DerivationPaths used to derive FactorInstances /// for "mfa" to securify personas. /// `(EntityKind::Identity, KeySpace::Securified, KeyKind::TransactionSigning)` + #[debug("I-MFA")] IdentityMfa, } impl DerivationPreset { diff --git a/src/factor_instances_provider/agnostic_paths/index_agnostic_path.rs b/src/factor_instances_provider/agnostic_paths/index_agnostic_path.rs index 5f025212..f4338923 100644 --- a/src/factor_instances_provider/agnostic_paths/index_agnostic_path.rs +++ b/src/factor_instances_provider/agnostic_paths/index_agnostic_path.rs @@ -1,7 +1,9 @@ use crate::prelude::*; /// A DerivationPath which is not indexed. On a specific network. -#[derive(Clone, Debug, Copy, Hash, PartialEq, Eq)] +#[derive(Clone, Copy, Hash, PartialEq, Eq, derive_more::Debug, derive_more::Display)] +#[display("{}/{}/{}/?{}", network_id, entity_kind, key_kind, key_space.indicator())] +#[debug("{:?}/{:?}/{:?}/?{}", network_id, entity_kind, key_kind, key_space.indicator())] pub struct IndexAgnosticPath { pub network_id: NetworkID, pub entity_kind: CAP26EntityKind, @@ -9,16 +11,32 @@ pub struct IndexAgnosticPath { pub key_space: KeySpace, } -impl From<(NetworkID, DerivationPreset)> for IndexAgnosticPath { - fn from((network_id, agnostic_path): (NetworkID, DerivationPreset)) -> Self { +impl IndexAgnosticPath { + pub fn new( + network_id: NetworkID, + entity_kind: CAP26EntityKind, + key_kind: CAP26KeyKind, + key_space: KeySpace, + ) -> Self { Self { network_id, - entity_kind: agnostic_path.entity_kind(), - key_kind: agnostic_path.key_kind(), - key_space: agnostic_path.key_space(), + entity_kind, + key_kind, + key_space, } } } + +impl From<(NetworkID, DerivationPreset)> for IndexAgnosticPath { + fn from((network_id, agnostic_path): (NetworkID, DerivationPreset)) -> Self { + Self::new( + network_id, + agnostic_path.entity_kind(), + agnostic_path.key_kind(), + agnostic_path.key_space(), + ) + } +} impl TryFrom for DerivationPreset { type Error = CommonError; fn try_from(value: IndexAgnosticPath) -> Result { @@ -44,26 +62,13 @@ impl TryFrom for DerivationPreset { } } -#[derive(Clone, Copy, Hash, PartialEq, Eq)] +#[derive(Clone, Copy, Hash, PartialEq, Eq, derive_more::Debug)] +#[debug("🎯: {:?} #{}", self.derivation_preset, self.quantity)] pub struct QuantifiedDerivationPresets { pub derivation_preset: DerivationPreset, pub quantity: usize, } -/// For `DerivationPreset` we keep track of -/// the quantity of instances that are cached and -#[derive(Clone, Debug, Hash, PartialEq, Eq)] -pub struct QuantifiedToCacheToUseDerivationPresets { - pub derivation_preset: DerivationPreset, - pub quantity: QuantityToCacheToUseDirectly, -} - -#[derive(Clone, Hash, PartialEq, Eq)] -pub struct QuantifiedToCacheToUseIndexAgnosticPath { - pub agnostic_path: IndexAgnosticPath, - pub quantity: QuantityToCacheToUseDirectly, -} - impl From<(IndexAgnosticPath, HDPathComponent)> for DerivationPath { fn from((path, index): (IndexAgnosticPath, HDPathComponent)) -> Self { assert_eq!(index.key_space(), path.key_space); diff --git a/src/factor_instances_provider/agnostic_paths/quantities.rs b/src/factor_instances_provider/agnostic_paths/quantities.rs index a1b9161f..a5866235 100644 --- a/src/factor_instances_provider/agnostic_paths/quantities.rs +++ b/src/factor_instances_provider/agnostic_paths/quantities.rs @@ -1,57 +1,3 @@ use crate::prelude::*; pub const CACHE_FILLING_QUANTITY: usize = 30; - -#[derive(Clone, Debug, Hash, PartialEq, Eq)] -pub enum QuantityToCacheToUseDirectly { - OnlyCacheFilling { - /// `CACHE_FILLING_QUANTITY` - `FOUND_IN_CACHE` - fill_cache: usize, - /// We peeked into the cache and found FactorInstance with the max index, which we must used - /// when we calculate the next index path, we are gonna do `max(max_from_profile, max_from_cache)` - /// where `max_from_cache` is this `max_index` field. - instance_with_max_index: Option, - }, - - /// We will derive `remaining + extra_to_fill_cache` more instances - /// - /// If: - /// CACHE_FILLING_QUANTITY: 30 - /// FOUND_IN_CACHE: 12 - /// REQUESTED: 14 - /// - /// Then `remaining` below will be `2` and `extra_to_fill_cache` will be - /// `CACHE_FILLING_QUANTITY` (`30`) since all `FOUND_IN_CACHE` instances - /// will be used and method `total_quantity_to_derive` below will return - /// `2 + 30 = 32` - ToCacheToUseDirectly { - /// Remaining quantity to satisfy the request, `originally_requested - from_cache_instances.len()` - /// Used later to split the newly derived instances into two groups, to cache and to use directly, - /// can be zero. - remaining: usize, - - /// Typically `CACHE_FILLING_QUANTITY` (always?) - extra_to_fill_cache: usize, - }, -} - -impl QuantityToCacheToUseDirectly { - pub fn max_index(&self) -> Option { - match self { - Self::OnlyCacheFilling { - fill_cache: _, - instance_with_max_index, - } => instance_with_max_index.clone(), - Self::ToCacheToUseDirectly { .. } => None, - } - } - pub fn total_quantity_to_derive(&self) -> usize { - match self { - Self::OnlyCacheFilling { fill_cache, .. } => *fill_cache, - Self::ToCacheToUseDirectly { - remaining, - extra_to_fill_cache, - } => *remaining + *extra_to_fill_cache, - } - } -} diff --git a/src/factor_instances_provider/next_index_assigner/mod.rs b/src/factor_instances_provider/next_index_assigner/mod.rs index b1c1888e..a687d9d3 100644 --- a/src/factor_instances_provider/next_index_assigner/mod.rs +++ b/src/factor_instances_provider/next_index_assigner/mod.rs @@ -1,11 +1,11 @@ mod next_derivation_entity_index_assigner; +mod next_derivation_entity_index_cache_analyzing_assigner; mod next_derivation_entity_index_profile_analyzing_assigner; mod next_derivation_entity_index_with_ephemeral_offsets; mod next_derivation_entity_index_with_ephemeral_offsets_for_factor_source; -mod offset_from_cache; pub use next_derivation_entity_index_assigner::*; +pub use next_derivation_entity_index_cache_analyzing_assigner::*; pub use next_derivation_entity_index_profile_analyzing_assigner::*; pub use next_derivation_entity_index_with_ephemeral_offsets::*; pub use next_derivation_entity_index_with_ephemeral_offsets_for_factor_source::*; -pub use offset_from_cache::*; diff --git a/src/factor_instances_provider/next_index_assigner/next_derivation_entity_index_assigner.rs b/src/factor_instances_provider/next_index_assigner/next_derivation_entity_index_assigner.rs index e5bdd11f..8424a440 100644 --- a/src/factor_instances_provider/next_index_assigner/next_derivation_entity_index_assigner.rs +++ b/src/factor_instances_provider/next_index_assigner/next_derivation_entity_index_assigner.rs @@ -5,7 +5,7 @@ use crate::prelude::*; /// some NetworkID. /// /// This assigner works with the: -/// * cache (indirectly, via the `OffsetFromCache` parameter on `next` [should probably clean up]) +/// * cache /// * profile /// * local offsets /// @@ -17,9 +17,8 @@ use crate::prelude::*; /// &mut self, /// fs_id: FactorSourceIDFromHash, /// path: IndexAgnosticPath, -/// cache_offset: OffsetFromCache, /// ) -> Result { -/// let next_from_cache = offset_from_cache.next(fs_id, path).unwrap_or(0); +/// let next_from_cache = self.cache_analyzing.next(fs_id, path).unwrap_or(0); /// let next_from_profile = self.profile_analyzing.next(fs_id, path).unwrap_or(0); /// /// let max_index = std::cmp::max(next_from_profile, next_from_cache); @@ -31,17 +30,25 @@ pub struct NextDerivationEntityIndexAssigner { #[allow(dead_code)] network_id: NetworkID, profile_analyzing: NextDerivationEntityIndexProfileAnalyzingAssigner, + cache_analyzing: NextDerivationEntityIndexCacheAnalyzingAssigner, ephemeral_offsets: NextDerivationEntityIndexWithEphemeralOffsets, } impl NextDerivationEntityIndexAssigner { - pub fn new(network_id: NetworkID, profile: Option) -> Self { + pub fn new( + network_id: NetworkID, + profile: Option, + cache: FactorInstancesCache, + ) -> Self { let profile_analyzing = NextDerivationEntityIndexProfileAnalyzingAssigner::new(network_id, profile); + let cache_analyzing = NextDerivationEntityIndexCacheAnalyzingAssigner::new(cache); + let ephemeral_offsets = NextDerivationEntityIndexWithEphemeralOffsets::default(); Self { network_id, profile_analyzing, - ephemeral_offsets: NextDerivationEntityIndexWithEphemeralOffsets::default(), + cache_analyzing, + ephemeral_offsets, } } @@ -49,40 +56,29 @@ impl NextDerivationEntityIndexAssigner { &self, factor_source_id: FactorSourceIDFromHash, index_agnostic_path: IndexAgnosticPath, - cache_offset: OffsetFromCache, ) -> Result { let default_index = HDPathComponent::new_with_key_space_and_base_index( index_agnostic_path.key_space, U30::new(0).unwrap(), ); - // Must update local offset based on values found in cache. - // Imagine we are securifying 3 accounts with a single FactorSource - // `L` to keep things simple, profile already contains 28 securified - // accounts controlled by `L`, with the highest entity index is `27^` - // We look for keys in the cache for `L` and we find 2, with entity - // indices `[28^, 29^]`, so we need to derive 2 (+CACHE_FILLING_QUANTITY) - // more keys. The next index assigner will correctly use a profile based offset - // of 28^ for `L`, since it found the max value `28^` in Profile controlled by `L`. - // If we would use `next` now, the index would be `next = max + 1`, and - // `max = offset_from_profile + ephemeral_offset` = `28^ + 0^` = 28^. - // Which is wrong! Since the cache contains `28^` and `29^`, we should - // derive `2 (+CACHE_FILLING_QUANTITY)` starting at `30^`. - let maybe_next_from_cache = cache_offset.next(factor_source_id, index_agnostic_path)?; + let maybe_next_from_cache = self + .cache_analyzing + .next(factor_source_id, index_agnostic_path)?; let next_from_cache = maybe_next_from_cache.unwrap_or(default_index); - let local = self + let ephemeral = self .ephemeral_offsets .reserve(factor_source_id, index_agnostic_path); let maybe_next_from_profile = self .profile_analyzing - .next(index_agnostic_path, factor_source_id)?; + .next(factor_source_id, index_agnostic_path)?; let next_from_profile = maybe_next_from_profile.unwrap_or(default_index); let max_index = std::cmp::max(next_from_profile, next_from_cache); - max_index.add_n(local) + max_index.add_n(ephemeral) } } diff --git a/src/factor_instances_provider/next_index_assigner/next_derivation_entity_index_cache_analyzing_assigner.rs b/src/factor_instances_provider/next_index_assigner/next_derivation_entity_index_cache_analyzing_assigner.rs new file mode 100644 index 00000000..9efb9f5e --- /dev/null +++ b/src/factor_instances_provider/next_index_assigner/next_derivation_entity_index_cache_analyzing_assigner.rs @@ -0,0 +1,22 @@ +use crate::prelude::*; + +pub struct NextDerivationEntityIndexCacheAnalyzingAssigner { + cache: FactorInstancesCache, +} +impl NextDerivationEntityIndexCacheAnalyzingAssigner { + pub fn new(cache: FactorInstancesCache) -> Self { + Self { cache } + } + + pub fn next( + &self, + factor_source_id: FactorSourceIDFromHash, + index_agnostic_path: IndexAgnosticPath, + ) -> Result> { + let max = self + .cache + .max_index_for(factor_source_id, index_agnostic_path); + let Some(max) = max else { return Ok(None) }; + max.add_one().map(Some) + } +} diff --git a/src/factor_instances_provider/next_index_assigner/next_derivation_entity_index_profile_analyzing_assigner.rs b/src/factor_instances_provider/next_index_assigner/next_derivation_entity_index_profile_analyzing_assigner.rs index 3ca01595..9a688e69 100644 --- a/src/factor_instances_provider/next_index_assigner/next_derivation_entity_index_profile_analyzing_assigner.rs +++ b/src/factor_instances_provider/next_index_assigner/next_derivation_entity_index_profile_analyzing_assigner.rs @@ -158,23 +158,23 @@ impl NextDerivationEntityIndexProfileAnalyzingAssigner { /// for more details. pub fn next( &self, - agnostic_path: IndexAgnosticPath, factor_source_id: FactorSourceIDFromHash, + agnostic_path: IndexAgnosticPath, ) -> Result> { if agnostic_path.network_id != self.network_id { return Err(CommonError::NetworkDiscrepancy); } let derivation_preset = DerivationPreset::try_from(agnostic_path)?; - let last = match derivation_preset { + let max = match derivation_preset { DerivationPreset::AccountVeci => self.max_account_veci(factor_source_id), DerivationPreset::AccountMfa => self.max_account_mfa(factor_source_id), DerivationPreset::IdentityVeci => self.max_identity_veci(factor_source_id), DerivationPreset::IdentityMfa => self.max_identity_mfa(factor_source_id), }; - let Some(last) = last else { return Ok(None) }; - last.add_one().map(Some) + let Some(max) = max else { return Ok(None) }; + max.add_one().map(Some) } } @@ -189,8 +189,8 @@ mod tests { let sut = Sut::new(NetworkID::Mainnet, None); assert_eq!( sut.next( + FactorSourceIDFromHash::fs0(), DerivationPreset::AccountVeci.index_agnostic_path_on_network(NetworkID::Stokenet), - FactorSourceIDFromHash::fs0() ), Err(CommonError::NetworkDiscrepancy) ); @@ -206,8 +206,8 @@ mod tests { ); let next = sut .next( - preset.index_agnostic_path_on_network(network_id), FactorSourceIDFromHash::fs0(), + preset.index_agnostic_path_on_network(network_id), ) .unwrap(); @@ -227,8 +227,8 @@ mod tests { ); let next = sut .next( - preset.index_agnostic_path_on_network(network_id), FactorSourceIDFromHash::fs1(), // <-- UNUSED + preset.index_agnostic_path_on_network(network_id), ) .unwrap(); @@ -245,8 +245,8 @@ mod tests { ); let next = sut .next( - preset.index_agnostic_path_on_network(network_id), FactorSourceIDFromHash::fs0(), + preset.index_agnostic_path_on_network(network_id), ) .unwrap(); @@ -270,8 +270,8 @@ mod tests { ); let next = sut .next( - preset.index_agnostic_path_on_network(network_id), FactorSourceIDFromHash::fs10(), + preset.index_agnostic_path_on_network(network_id), ) .unwrap(); @@ -299,7 +299,7 @@ mod tests { type F = FactorSourceIDFromHash; for fid in [F::fs2(), F::fs6(), F::fs7(), F::fs8(), F::fs9()] { let next = sut - .next(preset.index_agnostic_path_on_network(network_id), fid) + .next(fid, preset.index_agnostic_path_on_network(network_id)) .unwrap(); assert_eq!(next, Some(HDPathComponent::securifying_base_index(8))) @@ -317,7 +317,7 @@ mod tests { type F = FactorSourceIDFromHash; for fid in [F::fs2(), F::fs6(), F::fs7(), F::fs8(), F::fs9()] { let next = sut - .next(preset.index_agnostic_path_on_network(network_id), fid) + .next(fid, preset.index_agnostic_path_on_network(network_id)) .unwrap(); assert_eq!(next, Some(HDPathComponent::securifying_base_index(8))) @@ -341,8 +341,8 @@ mod tests { ); let next = sut .next( - preset.index_agnostic_path_on_network(network_id), FactorSourceIDFromHash::fs1(), + preset.index_agnostic_path_on_network(network_id), ) .unwrap(); @@ -388,8 +388,8 @@ mod tests { ); let next = sut .next( - DerivationPreset::AccountVeci.index_agnostic_path_on_network(network_id), fsid, + DerivationPreset::AccountVeci.index_agnostic_path_on_network(network_id), ) .unwrap(); diff --git a/src/factor_instances_provider/next_index_assigner/offset_from_cache.rs b/src/factor_instances_provider/next_index_assigner/offset_from_cache.rs deleted file mode 100644 index c77ec3b3..00000000 --- a/src/factor_instances_provider/next_index_assigner/offset_from_cache.rs +++ /dev/null @@ -1,48 +0,0 @@ -use crate::prelude::*; - -pub enum OffsetFromCache { - /// Finding max amongst already loaded (and removed) from cache, saved - /// locally - FindMaxInRemoved { - pf_found_in_cache: IndexMap, - }, - /// Known max by having peeked into the cache earlier. - KnownMax { - instance: HierarchicalDeterministicFactorInstance, - }, -} - -impl OffsetFromCache { - pub fn next( - &self, - factor_source_id: FactorSourceIDFromHash, - index_agnostic_path: IndexAgnosticPath, - ) -> Result> { - let Some(m) = self._max(factor_source_id, index_agnostic_path) else { - return Ok(None); - }; - m.add_one().map(Some) - } - - fn _max( - &self, - factor_source_id: FactorSourceIDFromHash, - index_agnostic_path: IndexAgnosticPath, - ) -> Option { - match self { - Self::FindMaxInRemoved { pf_found_in_cache } => pf_found_in_cache - .get(&factor_source_id) - .cloned() - .unwrap_or_default() - .into_iter() - .filter(|f| f.agnostic_path() == index_agnostic_path) - .map(|f| f.derivation_path().index) - .max(), - Self::KnownMax { instance } => { - assert_eq!(instance.factor_source_id(), factor_source_id); - assert_eq!(instance.agnostic_path(), index_agnostic_path); - Some(instance.derivation_path().index) - } - } - } -} diff --git a/src/factor_instances_provider/provider/factor_instances_cache.rs b/src/factor_instances_provider/provider/factor_instances_cache.rs index 4140e5b1..65e46cd9 100644 --- a/src/factor_instances_provider/provider/factor_instances_cache.rs +++ b/src/factor_instances_provider/provider/factor_instances_cache.rs @@ -1,4 +1,7 @@ -use std::ops::Add; +use std::{ + borrow::Borrow, + ops::{Add, Index}, +}; use crate::prelude::*; @@ -96,7 +99,7 @@ use crate::prelude::*; #[derive(Debug, Default, Clone)] pub struct FactorInstancesCache { /// PER FactorSource PER IndexAgnosticPath FactorInstances (matching that IndexAgnosticPath) - pub values: IndexMap>, + map: IndexMap>, } impl FactorInstancesCache { @@ -111,24 +114,25 @@ impl FactorInstancesCache { /// in the future. pub fn insert_for_factor( &mut self, - factor_source_id: FactorSourceIDFromHash, - instances: FactorInstances, + factor_source_id: &FactorSourceIDFromHash, + instances: &FactorInstances, ) -> Result { - let instances = instances.into_iter().collect_vec(); + let instances = instances.clone().into_iter().collect_vec(); let instances_by_agnostic_path = instances .into_iter() .into_group_map_by(|f| f.agnostic_path()) .into_iter() .map(|(k, v)| { - if v.iter().any(|f| f.factor_source_id != factor_source_id) { + if v.iter().any(|f| f.factor_source_id != *factor_source_id) { return Err(CommonError::FactorSourceDiscrepancy); } Ok((k, FactorInstances::from_iter(v))) }) .collect::>>()?; let mut skipped_an_index_resulting_in_non_contiguousness = false; - if let Some(existing_for_factor) = self.values.get_mut(&factor_source_id) { + + if let Some(existing_for_factor) = self.map.get_mut(factor_source_id) { for (agnostic_path, instances) in instances_by_agnostic_path { let instances = instances.factor_instances(); @@ -159,8 +163,8 @@ impl FactorInstancesCache { } } } else { - self.values - .insert(factor_source_id, instances_by_agnostic_path); + self.map + .insert(*factor_source_id, instances_by_agnostic_path); } Ok(skipped_an_index_resulting_in_non_contiguousness) @@ -169,7 +173,7 @@ impl FactorInstancesCache { /// Inserts all instance in `per_factor`. pub fn insert_all( &mut self, - per_factor: IndexMap, + per_factor: &IndexMap, ) -> Result<()> { for (factor_source_id, instances) in per_factor { _ = self.insert_for_factor(factor_source_id, instances)?; @@ -177,108 +181,238 @@ impl FactorInstancesCache { Ok(()) } - /// Reads out the instance of `factor_source_id` without mutating the cache. - pub fn peek_all_instances_of_factor_source( + pub fn max_index_for( &self, factor_source_id: FactorSourceIDFromHash, - ) -> Option> { - self.values.get(&factor_source_id).cloned() + agnostic_path: IndexAgnosticPath, + ) -> Option { + let for_factor = self.map.get(&factor_source_id)?; + let instances = for_factor.get(&agnostic_path)?; + instances + .factor_instances() + .last() + .map(|fi| fi.derivation_entity_index()) } - #[cfg(test)] - pub fn total_number_of_factor_instances(&self) -> usize { - self.values - .values() - .map(|x| { - x.values() - .map(|y| y.len()) - .reduce(Add::add) - .unwrap_or_default() - }) - .reduce(Add::add) - .unwrap_or_default() + pub fn get_poly_factor( + &self, + factor_source_ids: &IndexSet, + index_agnostic_path: &IndexAgnosticPath, + ) -> Result> { + let mut pf = IndexMap::new(); + for factor_source_id in factor_source_ids { + let Some(instances) = self.get_mono_factor(factor_source_id, index_agnostic_path) + else { + continue; + }; + pf.insert(*factor_source_id, instances); + } + Ok(pf) + } + + pub fn get_poly_factor_with_quantities( + &self, + factor_source_ids: &IndexSet, + originally_requested_quantified_derivation_preset: &QuantifiedDerivationPresets, + network_id: NetworkID, + ) -> Result { + let originally_requested_derivation_preset = + originally_requested_quantified_derivation_preset.derivation_preset; + let requested_qty = originally_requested_quantified_derivation_preset.quantity; + let mut is_qty_satisfied_for_all_factor_sources = true; + + let mut pf_pdp_qty_to_derive = + IndexMap::>::new(); + + let mut pf_instances = IndexMap::::new(); + + for factor_source_id in factor_source_ids { + let mut pdp_qty_to_derive = IndexMap::::new(); + + for derivation_preset in DerivationPreset::all() { + let index_agnostic_path = + derivation_preset.index_agnostic_path_on_network(network_id); + let instances = self + .get_mono_factor(factor_source_id, &index_agnostic_path) + .unwrap_or_default(); + if derivation_preset == originally_requested_derivation_preset { + let is_qty_satisfied_for_factor = instances.len() >= requested_qty; + if !is_qty_satisfied_for_factor { + let to_derive = CACHE_FILLING_QUANTITY + requested_qty - instances.len(); + pdp_qty_to_derive.insert(derivation_preset, to_derive); + + pf_instances.insert(*factor_source_id, instances); + } else { + let instances_enough_to_satisfy = + &instances.factor_instances().into_iter().collect_vec() + [..requested_qty]; + pf_instances.insert( + *factor_source_id, + FactorInstances::from_iter(instances_enough_to_satisfy.iter().cloned()), + ); + } + is_qty_satisfied_for_all_factor_sources = + is_qty_satisfied_for_all_factor_sources && is_qty_satisfied_for_factor; + } else if instances.len() < CACHE_FILLING_QUANTITY { + let to_derive = CACHE_FILLING_QUANTITY - instances.len(); + pdp_qty_to_derive.insert(derivation_preset, to_derive); + } + } + pf_pdp_qty_to_derive.insert(*factor_source_id, pdp_qty_to_derive); + } + + Ok(if is_qty_satisfied_for_all_factor_sources { + CachedInstancesWithQuantitiesOutcome::Satisfied(pf_instances) + } else { + CachedInstancesWithQuantitiesOutcome::NotSatisfied { + partial_instances: pf_instances, + pf_pdp_qty_to_derive, + } + }) } } -/// The outcome of reading factor instances from the cache, for a requested -/// quantity. -pub enum QuantityOutcome { - /// No `FactorInstances` was found for the request `(FactorSourceID, IndexAgnosticPath, Quantity)` - Empty, - - /// Only some `FactorInstances` was found for the request `(FactorSourceID, IndexAgnosticPath, Quantity)`, - /// being less than the requested quantity - Partial { - /// (NonEmpty) Instances found in cache, which is fewer than `originally_requested` - instances: FactorInstances, - /// Remaining quantity to satisfy the request, `originally_requested - instances.len()` - remaining: usize, +#[derive(enum_as_inner::EnumAsInner)] +pub enum CachedInstancesWithQuantitiesOutcome { + Satisfied(IndexMap), + NotSatisfied { + partial_instances: IndexMap, + pf_pdp_qty_to_derive: IndexMap>, }, +} - /// The cache contained enough `FactorInstances` for the request `(FactorSourceID, IndexAgnosticPath, Quantity)`, - /// either the exact same amount, or with "spare" ones. - Full { - /// (NonEmpty) Instances found in cache, which has the same length as `originally_requested` - instances: FactorInstances, - }, +impl CachedInstancesWithQuantitiesOutcome { + pub fn satisfied(&self) -> Option> { + self.as_satisfied().cloned() + } + pub fn quantities_to_derive( + &self, + ) -> Result>> { + match &self { + CachedInstancesWithQuantitiesOutcome::Satisfied(_) => panic!("programmer error"), + CachedInstancesWithQuantitiesOutcome::NotSatisfied { + pf_pdp_qty_to_derive, + .. + } => Ok(pf_pdp_qty_to_derive.clone()), + } + } + + pub fn partially_satisfied(self) -> Result> { + match self { + CachedInstancesWithQuantitiesOutcome::Satisfied(_) => panic!("programmer error"), + CachedInstancesWithQuantitiesOutcome::NotSatisfied { + partial_instances, .. + } => Ok(partial_instances), + } + } } impl FactorInstancesCache { - /// Removes all FactorInstances matching (FactorSourceID, IndexAgnosticPath), - /// and returns them - if any. - fn __remove( - &mut self, + pub fn get_mono_factor( + &self, factor_source_id: &FactorSourceIDFromHash, index_agnostic_path: &IndexAgnosticPath, - ) -> FactorInstances { - if let Some(cached_for_factor) = self.values.get_mut(factor_source_id) { - if let Some(found_cached) = cached_for_factor.shift_remove(index_agnostic_path) { - return found_cached; + ) -> Option { + let for_factor = self.map.get(factor_source_id)?; + let instances = for_factor.get(index_agnostic_path)?; + Some(instances.clone()) + } + + pub fn delete(&mut self, pf_instances: &IndexMap) { + for (factor_source_id, instances_to_delete) in pf_instances { + if instances_to_delete.is_empty() { + continue; + } + let existing_for_factor = self + .map + .get_mut(factor_source_id) + .expect("expected to delete factors"); + + let instances_to_delete_by_path = instances_to_delete.factor_instances() + .into_iter() + .into_group_map_by(|f| { + f.agnostic_path() + }) + .into_iter() + .collect::>>(); + + for (index_agnostic_path, instances_to_delete) in instances_to_delete_by_path { + let instances_to_delete = + IndexSet::::from_iter( + instances_to_delete.into_iter(), + ); + + let existing_for_path = existing_for_factor + .get(&index_agnostic_path) + .expect("expected to delete") + .factor_instances(); + + if !existing_for_path.is_superset(&instances_to_delete) { + panic!("Programmer error! Some of the factors to delete were not in cache!"); + } + let to_keep = existing_for_path + .symmetric_difference(&instances_to_delete) + .cloned() + .collect::(); + + // replace + existing_for_factor.insert(index_agnostic_path, to_keep); } } - FactorInstances::default() + + self.prune(); } - /// Mutates the cache, removing `quantity` many FactorInstances for `factor_source_id` - /// for `index_agnostic_path` and return the outcome of this, this result in any - /// of these outcomes: - /// * Empty - /// * Partial - /// * Full - pub fn remove( - &mut self, - factor_source_id: &FactorSourceIDFromHash, - index_agnostic_path: &IndexAgnosticPath, - quantity: usize, - ) -> QuantityOutcome { - let instances = self.__remove(factor_source_id, index_agnostic_path); - if instances.is_empty() { - return QuantityOutcome::Empty; - } - let len = instances.len(); - if len == quantity { - return QuantityOutcome::Full { instances }; - } - if len < quantity { - return QuantityOutcome::Partial { - instances, - remaining: quantity - len, - }; - } - assert!(len > quantity); - // Split the read instances at `quantity` many. - let instances = instances.factor_instances().into_iter().collect_vec(); - let (to_use, to_put_back) = instances.split_at(quantity); - let to_put_back = FactorInstances::from_iter(to_put_back.iter().cloned()); - - // put back the ones exceeding requested quantity - if let Some(cached_for_factor) = self.values.get_mut(factor_source_id) { - cached_for_factor.insert(*index_agnostic_path, to_put_back); + /// "Prunes" the cache from empty collections + fn prune(&mut self) { + let ids = self.factor_source_ids(); + for factor_source_id in ids.iter() { + let inner_map = self.map.get_mut(factor_source_id).unwrap(); + if inner_map.is_empty() { + // empty map, prune it! + self.map.shift_remove(factor_source_id); + continue; + } + // see if pruning of instances inside of values `inner_map` is needed + let inner_ids = inner_map + .keys() + .cloned() + .collect::>(); + for inner_id in inner_ids.iter() { + if inner_map.get(inner_id).unwrap().is_empty() { + // FactorInstances empty, prune it! + inner_map.shift_remove(inner_id); + } + } } + } - QuantityOutcome::Full { - instances: FactorInstances::from_iter(to_use.iter().cloned()), - } + fn factor_source_ids(&self) -> IndexSet { + self.map.keys().cloned().collect() + } + pub fn insert(&mut self, pf_instances: &IndexMap) { + self.insert_all(pf_instances).expect("works") + } + + /// Reads out the instance of `factor_source_id` without mutating the cache. + pub fn peek_all_instances_of_factor_source( + &self, + factor_source_id: FactorSourceIDFromHash, + ) -> Option> { + self.map.get(&factor_source_id).cloned() + } + + pub fn total_number_of_factor_instances(&self) -> usize { + self.map + .values() + .map(|x| { + x.values() + .map(|y| y.len()) + .reduce(Add::add) + .unwrap_or_default() + }) + .reduce(Add::add) + .unwrap_or_default() } } @@ -288,7 +422,7 @@ impl FactorInstancesCache { /// each DerivationPreset pub fn is_full(&self, network_id: NetworkID, factor_source_id: FactorSourceIDFromHash) -> bool { let count: usize = self - .values + .map .get(&factor_source_id) .and_then(|c| { c.values() @@ -314,6 +448,8 @@ impl FactorInstancesCache { #[cfg(test)] mod tests { + use std::fs; + use super::*; type Sut = FactorInstancesCache; @@ -328,7 +464,7 @@ mod tests { fsid, ); assert!(!sut - .insert_for_factor(fsid, FactorInstances::from_iter([fi0])) + .insert_for_factor(&fsid, &FactorInstances::from_iter([fi0])) .unwrap()); let fi2 = HierarchicalDeterministicFactorInstance::mainnet_tx( CAP26EntityKind::Account, @@ -336,7 +472,7 @@ mod tests { fsid, ); assert!(sut - .insert_for_factor(fsid, FactorInstances::from_iter([fi2])) + .insert_for_factor(&fsid, &FactorInstances::from_iter([fi2])) .unwrap(),); } @@ -350,12 +486,62 @@ mod tests { ); assert!(sut .insert_for_factor( - FactorSourceIDFromHash::fs1(), - FactorInstances::from_iter([fi0]) + &FactorSourceIDFromHash::fs1(), + &FactorInstances::from_iter([fi0]) ) .is_err()); } + #[test] + fn delete() { + let mut sut = Sut::default(); + + let factor_source_ids = HDFactorSource::all() + .into_iter() + .map(|f| f.factor_source_id()) + .collect::>(); + + let n = 30; + let mut to_delete = IndexMap::::new(); + let mut to_remain = IndexMap::::new(); + for factor_source_id in factor_source_ids.clone() { + let fsid = factor_source_id; + let instances = (0..n) + .map(|i| { + let fi = HierarchicalDeterministicFactorInstance::mainnet_tx( + CAP26EntityKind::Account, + HDPathComponent::unsecurified_hardening_base_index(i), + fsid, + ); + if i < 10 { + to_delete.append_or_insert_to(&fsid, IndexSet::just(fi.clone())); + } else { + to_remain.append_or_insert_to(&fsid, IndexSet::just(fi.clone())); + } + fi + }) + .collect::>(); + + sut.insert_for_factor(&fsid, &FactorInstances::from(instances)) + .unwrap(); + } + + sut.delete(&to_delete); + assert_eq!( + sut.get_poly_factor( + &factor_source_ids, + &IndexAgnosticPath::new( + NetworkID::Mainnet, + CAP26EntityKind::Account, + CAP26KeyKind::TransactionSigning, + KeySpace::Unsecurified + ) + ) + .unwrap(), + to_remain + ); + } + #[test] fn throws_if_same_is_added() { let mut sut = Sut::default(); @@ -371,11 +557,11 @@ mod tests { fsid, ); assert!(!sut - .insert_for_factor(fsid, FactorInstances::from_iter([fi0.clone(), fi1])) + .insert_for_factor(&fsid, &FactorInstances::from_iter([fi0.clone(), fi1])) .unwrap()); assert_eq!( - sut.insert_for_factor(fsid, FactorInstances::from_iter([fi0.clone()])) + sut.insert_for_factor(&fsid, &FactorInstances::from_iter([fi0.clone()])) .err() .unwrap(), CommonError::CacheAlreadyContainsFactorInstance { diff --git a/src/factor_instances_provider/provider/factor_instances_provider.rs b/src/factor_instances_provider/provider/factor_instances_provider.rs index 9dde33e6..bf096aa8 100644 --- a/src/factor_instances_provider/provider/factor_instances_provider.rs +++ b/src/factor_instances_provider/provider/factor_instances_provider.rs @@ -2,7 +2,7 @@ use std::sync::{Arc, RwLock}; use itertools::cloned; -use crate::prelude::*; +use crate::{factor_instances_provider::next_index_assigner, prelude::*}; /// A coordinator between a cache, an optional profile and the KeysCollector. /// @@ -34,39 +34,34 @@ impl FactorInstancesProvider { network_id: NetworkID, // typically mainnet interactors: Arc, ) -> Result { - // This is hacky! We are using `account_veci` as agnostic_path, we could - // have used any other value... we are not going to use any instances directly - // at all, why we specify `0` here, we piggyback on the rest of the logic - // to derive more... We should most definitely switch to `DerivationTemplate` enum - let quantity_of_instances_to_use_directly = IndexMap::kv( - factor_source.factor_source_id(), - QuantifiedDerivationPresets { - quantity: 0, // HACKY - derivation_preset: DerivationPreset::AccountVeci, // HACKY - }, - ); - - let outcome = Self::with( + let derived = Self::derive_more( + IndexSet::just(factor_source.clone()), + IndexMap::kv( + factor_source.factor_source_id(), + DerivationPreset::all() + .into_iter() + .map(|dp| (dp, CACHE_FILLING_QUANTITY)) + .collect::>(), + ), network_id, + profile, cache, - IndexSet::just(factor_source.clone()), - quantity_of_instances_to_use_directly, - &NextDerivationEntityIndexAssigner::new(network_id, profile), interactors, ) .await?; + cache.insert(&derived); - let outcome = outcome - .per_factor + let derived = derived .get(&factor_source.factor_source_id()) - .cloned() - .expect("Expected to have instances for the (new) factor source"); - - assert!( - outcome.to_use_directly.is_empty(), - "Programmer error, expected to return an empty list of instances to use directly" + .unwrap() + .clone(); + let outcome = InternalFactorInstancesProviderOutcomeForFactor::new( + factor_source.factor_source_id(), + derived.clone(), + FactorInstances::default(), + FactorInstances::default(), + derived, ); - Ok(outcome.into()) } @@ -146,16 +141,13 @@ impl FactorInstancesProvider { ) -> Result { let outcome = Self::with( network_id, - cache, IndexSet::just(factor_source.clone()), - IndexMap::kv( - factor_source.factor_source_id(), - QuantifiedDerivationPresets { - quantity: 1, - derivation_preset: DerivationPreset::veci_entity_kind(entity_kind), - }, - ), - &NextDerivationEntityIndexAssigner::new(network_id, profile), + QuantifiedDerivationPresets { + quantity: 1, + derivation_preset: DerivationPreset::veci_entity_kind(entity_kind), + }, + profile, + cache, interactors, ) .await?; @@ -269,21 +261,13 @@ impl FactorInstancesProvider { let outcome = Self::with( network_id, + factor_sources_to_use, + QuantifiedDerivationPresets { + quantity: addresses_of_entities.len(), + derivation_preset, + }, + profile, cache, - factor_sources, - factor_sources_to_use - .into_iter() - .map(|f| { - ( - f.factor_source_id(), - QuantifiedDerivationPresets { - quantity: addresses_of_entities.len(), - derivation_preset, - }, - ) - }) - .collect(), - &NextDerivationEntityIndexAssigner::new(network_id, Some(profile)), interactors, ) .await?; @@ -292,540 +276,160 @@ impl FactorInstancesProvider { } } -struct TakeFromCacheResult { - pf_found_in_cache: IndexMap, - pf_quantity_remaining_not_satisfied_by_cache: - IndexMap, - need_to_derive_more_instances: bool, -} - -struct SplitFactorInstances { - pf_to_cache: IndexMap, +struct Split { pf_to_use_directly: IndexMap, - /// "unsplit" - pf_newly_derived: IndexMap, + pf_to_cache: IndexMap, } impl FactorInstancesProvider { async fn with( network_id: NetworkID, - cache: &mut FactorInstancesCache, factor_sources: IndexSet, - index_agnostic_path_and_quantity_per_factor_source: IndexMap< - FactorSourceIDFromHash, - QuantifiedDerivationPresets, - >, - next_index_assigner: &NextDerivationEntityIndexAssigner, - interactors: Arc, - ) -> Result { - // clone cache so that we do not mutate the cache itself, later, if - // derivation is successful, we will write back the changes made to - // this cloned cache, on top of which we will save the newly derived - // instances. - let mut cloned_cache = cache.clone(); - - // take (consume) the cache and derive more instances if necessary - let outcome = Self::with_copy_of_cache( - network_id, - &mut cloned_cache, - factor_sources, - index_agnostic_path_and_quantity_per_factor_source, - next_index_assigner, - interactors, - ) - .await?; - - // derivation was successful, safe to write back the changes to the cache - *cache = cloned_cache; - - // and now lets save all `to_cache` (newly derived minus enough instances - // to satisfy the initial request) into the cache. - cache.insert_all( - outcome - .per_factor - .clone() - .into_iter() - .map(|(k, v)| { - // We are only saving the instances `to_cache` into the cache, - // the other instances should be used directly (if any). - let to_cache = v.to_cache; - (k, to_cache) - }) - .collect::>(), - )?; - - Ok(outcome) - } - - /// You should pass this a clone of the cache and not the cache itself. - /// since this mutates the cache. - #[allow(clippy::nonminimal_bool)] - async fn with_copy_of_cache( - network_id: NetworkID, + originally_requested_quantified_derivation_preset: QuantifiedDerivationPresets, + profile: impl Into>, cache: &mut FactorInstancesCache, - factor_sources: IndexSet, - index_agnostic_path_and_quantity_per_factor_source: IndexMap< - FactorSourceIDFromHash, - QuantifiedDerivationPresets, - >, - next_index_assigner: &NextDerivationEntityIndexAssigner, interactors: Arc, ) -> Result { - let pf_cache_result = Self::take_from_cache( + let profile = profile.into(); + let factor_source_ids = factor_sources + .iter() + .map(|f| f.factor_source_id()) + .collect::>(); + + let cached = cache.get_poly_factor_with_quantities( + &factor_source_ids, + &originally_requested_quantified_derivation_preset, network_id, - cache, - &index_agnostic_path_and_quantity_per_factor_source, )?; - if !pf_cache_result.need_to_derive_more_instances { - // `need_to_derive_more_instances` was never set to true, so we - // can satisfy the request with what we found in the cache. + if let Some(satisfied_by_cache) = cached.satisfied() { + cache.delete(&satisfied_by_cache); // Consume! return Ok(InternalFactorInstancesProviderOutcome::satisfied_by_cache( - pf_cache_result.pf_found_in_cache, + satisfied_by_cache.clone(), )); } - // We need to derive more instances, since we could not satisfy the request - // We will derive more and save them into the cache, and return - // the concatenation of what was found in cache (if any) with the remaining - // quantity from newly derived to satisfy the request. - Self::derive_more_instances( + let pf_newly_derived = Self::derive_more( + factor_sources, + cached.quantities_to_derive()?, network_id, + profile, cache, - next_index_assigner, - interactors, - factor_sources, - index_agnostic_path_and_quantity_per_factor_source, - pf_cache_result.pf_quantity_remaining_not_satisfied_by_cache, - pf_cache_result.pf_found_in_cache, + interactors.clone(), ) - .await - } - - fn take_from_cache( - network_id: NetworkID, - cache: &mut FactorInstancesCache, - index_agnostic_path_and_quantity_per_factor_source: &IndexMap< - FactorSourceIDFromHash, - QuantifiedDerivationPresets, - >, - ) -> Result { - // `pf` is short for `Per FactorSource` - let mut pf_found_in_cache = IndexMap::::new(); - - // For every factor source found in this map, we derive the remaining - // quantity as to satisfy the request PLUS we are deriving to fill the - // cache since we are deriving anyway, i.e. derive for all `IndexAgnosticPath` - // "presets" (Account Veci, Identity Veci, Account MFA, Identity MFA). - let mut pf_quantity_remaining_not_satisfied_by_cache = - IndexMap::::new(); - - // if false we will not derive any more instances, we could satisfy the request - // with what we found in the cache. - let mut need_to_derive_more_instances: bool = false; - - for (factor_source_id, quantified_agnostic_path) in - index_agnostic_path_and_quantity_per_factor_source.iter() - { - let from_cache: FactorInstances; - let unsatisfied_quantity: usize; - let cache_key = - IndexAgnosticPath::from((network_id, quantified_agnostic_path.derivation_preset)); - - // the quantity of factor instances needed to satisfy the request - // this will be `0` in case of PRE_DERIVE_KEYS_FOR_NEW_FACTOR_SOURCE (hacky). - // this will be `accounts.len()` in case of `for_account_mfa` (and analog for identities) and will - // be `1` for account_veci / identity_veci. - let quantity = quantified_agnostic_path.quantity; - - // we are mutating the cache, reading out `quantity` OR LESS instances. - // we must check how many we got - match cache.remove(factor_source_id, &cache_key, quantity) { - // Found nothing in the cache - QuantityOutcome::Empty => { - // need to derive more since cache was empty - need_to_derive_more_instances = true; - // nothing found in the cache, use empty... - from_cache = FactorInstances::default(); - // ALL `quantity` many instances are "unsatisfied". - unsatisfied_quantity = quantity; - } - // Found some instances in the cache, but `remaining` many are still needed - QuantityOutcome::Partial { - instances, - remaining, - } => { - // we need to derive more since cache could only partially satisfy the request - need_to_derive_more_instances = true; - // use all found (and we will need to derive more) - from_cache = instances; - // `remaining` many instances are "unsatisfied", for this factor source - unsatisfied_quantity = remaining; - } - // Found all instances needed in the cache - QuantityOutcome::Full { instances } => { - // we do not set `need_to_derive_more_instances` to `false` - // since an earlier iteration might have set it to true (for another factor source). - // so we do not change it. - - // use all found (and we will not need to derive more for this factor source) - from_cache = instances; - // none unsatisfied for this factor source. - unsatisfied_quantity = 0; - } - } - - if unsatisfied_quantity > 0 { - // need to save which IndexAgnosticPath we need to derive more for - // and how many instances we need to derive to satisfy the request, - // we might in fact derive even more than `unsatisfied_quantity` for - // this unsatisfied `IndexAgnosticPath` so that the cache is filled - // even after the request has been satisfied. - pf_quantity_remaining_not_satisfied_by_cache.insert( - *factor_source_id, - QuantifiedDerivationPresets { - quantity: unsatisfied_quantity, - derivation_preset: quantified_agnostic_path.derivation_preset, - }, - ); - } - - // We don't wanna insert empty instances into `pf_found_in_cache` - if !from_cache.is_empty() { - pf_found_in_cache.insert(*factor_source_id, from_cache.clone()); - } - } + .await?; - Ok(TakeFromCacheResult { - need_to_derive_more_instances, - pf_found_in_cache, - pf_quantity_remaining_not_satisfied_by_cache, - }) - } + let pf_found_in_cache_leq_requested = cached.partially_satisfied()?; - /// Derives more instances for the factor sources in `factor_sources` - /// uses `pf_quantity_remaining_not_satisfied_by_cache` to split the newly - /// derived ones in half, for each `IndexAgnosticPath`, for each factor sources - /// and uses some of the newly derived to satisfy the request and the rest - /// is saved them into the cache. - /// - /// We are also not deriving only for the `IndexAgnosticPath` that was unsatisfied, - /// but rather we are deriving for all `IndexAgnosticPath` "presets" (account veci, - /// identity veci, account mfa, identity mfa) for each factor source, i.e. - /// we are filling the cache. - #[allow(clippy::too_many_arguments)] - async fn derive_more_instances( - network_id: NetworkID, - cache: &mut FactorInstancesCache, - next_index_assigner: &NextDerivationEntityIndexAssigner, - interactors: Arc, - factor_sources: IndexSet, - index_agnostic_path_and_quantity_per_factor_source: IndexMap< - FactorSourceIDFromHash, - QuantifiedDerivationPresets, - >, - pf_quantity_remaining_not_satisfied_by_cache: IndexMap< - FactorSourceIDFromHash, - QuantifiedDerivationPresets, - >, - pf_found_in_cache: IndexMap, - ) -> Result { - // Based on contents of cache and the original request, calculate derivation paths - // for instances to be derived. - let paths = Self::calculate_derivation_paths( - network_id, - cache, - next_index_assigner, - &pf_found_in_cache, - &index_agnostic_path_and_quantity_per_factor_source, - &pf_quantity_remaining_not_satisfied_by_cache, - )?; - - // Actually derive more factor instances. - let keys_collector = KeysCollector::new(factor_sources, paths, interactors)?; - let newly_derived_instances = keys_collector.collect_keys().await; + let Split { + pf_to_use_directly, + pf_to_cache, + } = Self::split( + &originally_requested_quantified_derivation_preset, + &pf_found_in_cache_leq_requested, + &pf_newly_derived, + ); - // Split instances into `(to_use_directly, to_cache)`, per factor source. - // We include the "unsplit" `pf_newly_derived` just for debug/test purposes. - let split_factor_instances = Self::split( - network_id, - newly_derived_instances, - index_agnostic_path_and_quantity_per_factor_source, - &pf_quantity_remaining_not_satisfied_by_cache, - &pf_found_in_cache, - )?; + cache.delete(&pf_found_in_cache_leq_requested); + cache.insert(&pf_to_cache); - // Build of the "collection" of FactorInstances: - // * to_cache - // * to_use_directly - // * (for tests/debug) found in cache - // * (for tests/debug) newly derived - // - // And "transpose" them, into one collection per FactorSource. let outcome = InternalFactorInstancesProviderOutcome::transpose( - split_factor_instances.pf_to_cache, - split_factor_instances - .pf_to_use_directly - .into_iter() - .map(|(k, v)| (k, v.clone())) - .collect(), - pf_found_in_cache, - split_factor_instances.pf_newly_derived, + pf_to_cache, + pf_to_use_directly, + pf_found_in_cache_leq_requested, + pf_newly_derived, ); + let outcome = outcome; Ok(outcome) } fn split( - network_id: NetworkID, - newly_derived: KeyDerivationOutcome, - index_agnostic_path_and_quantity_per_factor_source: IndexMap< - FactorSourceIDFromHash, - QuantifiedDerivationPresets, - >, - pf_quantity_remaining_not_satisfied_by_cache: &IndexMap< - FactorSourceIDFromHash, - QuantifiedDerivationPresets, - >, - pf_found_in_cache: &IndexMap, - ) -> Result { - // used to filter out factor instances to use directly from the newly derived, based on - // `index_agnostic_path_and_quantity_per_factor_source` we map - // from: `IndexMap::` - // to `IndexSet::` - // - // If any `IndexAgnosticPath` is in `index_agnostic_paths_originally_requested`, - // it means we should not cache instances of that IndexAgnosticPath unconditionally, - // rather we need to perform a split, to see which `remaining` quantity should - // be used directly, and which instances should be cached. - let index_agnostic_paths_originally_requested = - index_agnostic_path_and_quantity_per_factor_source - .values() - .cloned() - .map(|q| IndexAgnosticPath::from((network_id, q.derivation_preset))) - .collect::>(); - + originally_requested_quantified_derivation_preset: &QuantifiedDerivationPresets, + pf_found_in_cache_leq_requested: &IndexMap, + newly_derived: &IndexMap, + ) -> Split { + let mut pf_to_use_directly = pf_found_in_cache_leq_requested.clone(); let mut pf_to_cache = IndexMap::::new(); - let mut pf_newly_derived = IndexMap::::new(); - - // we will use directly what was found in clone, but later when - // we derive more, we will add those to `pf_to_use_directly`, but - // not to `pf_found_in_cache`, but we will include `pf_found_in_cache` for - // unit tests. - let mut pf_to_use_directly = pf_found_in_cache.clone(); - - // Now split the newly derived FactorInstances, per factor source, into - // `to_cache` and into `to_use_directly`. - for (f, instances) in newly_derived.factors_by_source.into_iter() { - pf_newly_derived.insert(f, instances.clone().into()); - let instances: Vec = - instances.into_iter().collect_vec(); - let mut to_use_directly = IndexSet::::new(); - - // to use directly - let remaining = pf_quantity_remaining_not_satisfied_by_cache - .get(&f) - .map(|q| q.quantity) - .unwrap_or(0); - - let mut to_cache = IndexSet::::new(); - for instance in instances { - // `instance_matches_original_request` should be `false` if we used - // the `FactorInstancesProvider` for purpose "PRE_DERIVE_KEYS_FOR_NEW_FACTOR_SOURCE", - - let instance_matches_original_request = index_agnostic_paths_originally_requested - .contains(&instance.derivation_path().agnostic()); - - if instance_matches_original_request { - // Here we ensure to only use `remaining` many - // instances for `to_use_directly`, the rest - // should be cached! - if to_use_directly.len() < remaining { - to_use_directly.insert(instance); + let target_qty = originally_requested_quantified_derivation_preset.quantity; + for (f, instances) in newly_derived { + for instance in instances.factor_instances() { + let derivation_preset = + DerivationPreset::try_from(instance.agnostic_path()).unwrap(); + if derivation_preset + == originally_requested_quantified_derivation_preset.derivation_preset + { + // relevant for pf_to_use_directly + let instances_to_use_dir = pf_to_use_directly.get_mut(f).unwrap(); + if instances_to_use_dir.len() < target_qty { + instances_to_use_dir.append(FactorInstances::just(instance)); } else { - to_cache.insert(instance); + pf_to_cache.append_or_insert_element_to(f, instance); } } else { - // Does not match original request, cache all! - to_cache.insert(instance); + pf_to_cache.append_or_insert_element_to(f, instance); } } - - pf_to_cache.insert(f, to_cache.into()); - - if let Some(existing_to_use_directly) = pf_to_use_directly.get_mut(&f) { - // We already have inserted some FactorInstances to use directly for this - // FactorSource, this is possible we have made a "composite" request - // loading some AccountMFA FactorInstances **and** a ROLA key for example. - existing_to_use_directly.extend(to_use_directly.into_iter()); - } else { - pf_to_use_directly.insert(f, FactorInstances::from(to_use_directly)); - } } - Ok(SplitFactorInstances { - pf_to_cache, + Split { pf_to_use_directly, - pf_newly_derived, - }) + pf_to_cache, + } } - fn calculate_derivation_paths( - network_id: NetworkID, - cache: &FactorInstancesCache, // not mutated - next_index_assigner: &NextDerivationEntityIndexAssigner, - pf_found_in_cache: &IndexMap, - index_agnostic_path_and_quantity_per_factor_source: &IndexMap< - FactorSourceIDFromHash, - QuantifiedDerivationPresets, - >, - pf_quantity_remaining_not_satisfied_by_cache: &IndexMap< + async fn derive_more( + factor_sources: IndexSet, + pf_pdp_quantity_to_derive: IndexMap< FactorSourceIDFromHash, - QuantifiedDerivationPresets, + IndexMap, >, - ) -> Result>> { - // Per FactorSource a set of NetworkIndexAgnostic Paths ("presets") to derive for - // and the quantity to derive, will be built up using `DerivationPreset::all()` - // and the originally requested in `index_agnostic_path_and_quantity_per_factor_source`. - let mut pf_quantified_network_agnostic_paths_for_derivation = IndexMap::< - FactorSourceIDFromHash, - IndexSet, - >::new(); - - // Lets build up `pf_quantified_network_agnostic_paths_for_derivation`, which - // contains - // `index_agnostic_path_and_quantity_per_factor_source` contains the QuantifiedDerivationPresets for - // each FactorSource originally requested, we are gonna fill it with - // `DerivationPreset::all()` and for each gonna look up - // how many instances we need to derive to fill the cache, but first - // we are gonna check if any of the `DerivationPreset::all()` was - // matches the IndexAgnosticPath of the original request. - for factor_source_id in index_agnostic_path_and_quantity_per_factor_source.keys() { - let partial = pf_quantity_remaining_not_satisfied_by_cache - .get(factor_source_id) - .cloned(); - for derivation_preset in DerivationPreset::all() { - let to_derive = partial - .and_then(|p| { - if p.derivation_preset == derivation_preset { - Some(QuantifiedToCacheToUseDerivationPresets { - quantity: QuantityToCacheToUseDirectly::ToCacheToUseDirectly { - remaining: p.quantity, - extra_to_fill_cache: CACHE_FILLING_QUANTITY, - }, - derivation_preset: p.derivation_preset, + network_id: NetworkID, + profile: Option, + cache: &FactorInstancesCache, + interactors: Arc, + ) -> Result> { + let next_index_assigner = + NextDerivationEntityIndexAssigner::new(network_id, profile, cache.clone()); + + let pf_paths = pf_pdp_quantity_to_derive + .into_iter() + .map(|(factor_source_id, pdp_quantity_to_derive)| { + let paths = pdp_quantity_to_derive + .into_iter() + .map(|(derivation_preset, qty)| { + // `qty` many paths + let paths = (0..qty) + .map(|_| { + let index_agnostic_path = + derivation_preset.index_agnostic_path_on_network(network_id); + let index = next_index_assigner + .next(factor_source_id, index_agnostic_path)?; + Ok(DerivationPath::from((index_agnostic_path, index))) }) - } else { - None - } + .collect::>>()?; + + Ok(paths) }) - .unwrap_or_else(|| { - let cache_key = - derivation_preset.index_agnostic_path_on_network(network_id); - - let instances_in_cache = cache - .peek_all_instances_of_factor_source(*factor_source_id) - .and_then(|c| c.get(&cache_key).cloned()) - .unwrap_or_default(); - - let number_of_instances_in_cache = instances_in_cache.len(); - let instance_with_max_index = - instances_in_cache.into_iter().max_by(|a, b| { - a.derivation_entity_index() - .cmp(&b.derivation_entity_index()) - }); - let number_of_instances_to_derive_to_fill_cache = - CACHE_FILLING_QUANTITY - number_of_instances_in_cache; - - QuantifiedToCacheToUseDerivationPresets { - quantity: QuantityToCacheToUseDirectly::OnlyCacheFilling { - fill_cache: number_of_instances_to_derive_to_fill_cache, - instance_with_max_index, - }, - derivation_preset, - } - }); - - if let Some(existing) = - pf_quantified_network_agnostic_paths_for_derivation.get_mut(factor_source_id) - { - existing.insert(to_derive); - } else { - pf_quantified_network_agnostic_paths_for_derivation - .insert(*factor_source_id, IndexSet::just(to_derive)); - } - } - } + .collect::>>>()?; + + // flatten (I was unable to use `flat_map` above combined with `Result`...) + let paths = paths.into_iter().flatten().collect::>(); + + Ok((factor_source_id, paths)) + }) + .collect::>>>()?; + + let keys_collector = KeysCollector::new(factor_sources, pf_paths, interactors)?; + + let pf_instances = keys_collector + .collect_keys() + .await + .factors_by_source + .into_iter() + .map(|(k, v)| (k, v.into_iter().collect::())) + .collect::>(); - // Map `NetworkAgnostic -> IndexAgnosticPath`, by using `network_id`. - let pf_quantified_index_agnostic_paths_for_derivation = - pf_quantified_network_agnostic_paths_for_derivation - .into_iter() - .map(|(factor_source_id, quantified_network_agnostic_paths)| { - let index_agnostic_paths = quantified_network_agnostic_paths - .into_iter() - .map(|q| QuantifiedToCacheToUseIndexAgnosticPath { - agnostic_path: q - .derivation_preset - .index_agnostic_path_on_network(network_id), - quantity: q.quantity, - }) - .collect::>(); - (factor_source_id, index_agnostic_paths) - }) - .collect::>(); - - // Now map from IndexAgnostic paths to index aware paths, a.k.a. DerivationPath - // but ALSO we need to retain the information of how many factor instances of - // the newly derived to append to the factor instances to use directly, and how many to cache. - let paths: Result>> = - pf_quantified_index_agnostic_paths_for_derivation - .clone() - .into_iter() - .map(|(f, agnostic_paths)| { - let paths: Vec> = agnostic_paths - .clone() - .into_iter() - .map(|quantified_agnostic_path| { - // IMPORTANT! We are not mapping one `IndexAgnosticPath` to one `DerivationPath`, but - // rather we are mapping one `IndexAgnosticPath` to **MANY** `DerivationPath`s! Equal to - // the same number as the specified quantity! - (0..quantified_agnostic_path.quantity.total_quantity_to_derive()) - .map(|_| { - let index_agnostic_path = - quantified_agnostic_path.agnostic_path; - - let index = next_index_assigner.next( - f, - index_agnostic_path, - // Must also use cache based offsets, checking max between cache and - // profile. Since profile might not contain the highest entity - // derivation index, the cache might! - quantified_agnostic_path - .quantity - .max_index() - .map(|max_index| OffsetFromCache::KnownMax { - instance: max_index, - }) - .unwrap_or(OffsetFromCache::FindMaxInRemoved { - pf_found_in_cache: pf_found_in_cache.clone(), - }), - )?; - Ok(DerivationPath::from(( - quantified_agnostic_path.agnostic_path, - index, - ))) - }) - .collect::>>() - }) - .collect::>>>()?; - - let paths = paths.into_iter().flatten().collect::>(); - - Ok((f, paths)) - }) - .collect::>>>(); - - paths + Ok(pf_instances) } } diff --git a/src/factor_instances_provider/provider/factor_instances_provider_unit_tests.rs b/src/factor_instances_provider/provider/factor_instances_provider_unit_tests.rs index bfc0a147..955c0da6 100644 --- a/src/factor_instances_provider/provider/factor_instances_provider_unit_tests.rs +++ b/src/factor_instances_provider/provider/factor_instances_provider_unit_tests.rs @@ -1073,14 +1073,12 @@ async fn securify_accounts_when_cache_is_half_full_single_factor_source() { .primary_role_instances() .into_iter() .map(|f| f.derivation_entity_index()) - .map(|x| format!("{:?}", x)) .next() .unwrap()) // single factor per role text .collect_vec(), - [ - "0^", "1^", "2^", "3^", "4^", "5^", "6^", "7^", "8^", "9^", "10^", "11^", "12^", "13^", - "14^" - ] + (0..CACHE_FILLING_QUANTITY / 2) + .map(|i| HDPathComponent::securifying_base_index(i as u32)) + .collect_vec() ); let (second_half_securified_accounts, stats) = os @@ -1108,15 +1106,12 @@ async fn securify_accounts_when_cache_is_half_full_single_factor_source() { .primary_role_instances() .into_iter() .map(|f| f.derivation_entity_index()) - .map(|x| format!("{:?}", x)) .next() .unwrap()) // single factor per role text .collect_vec(), - [ - "15^", "16^", "17^", "18^", "19^", "20^", "21^", "22^", "23^", "24^", "25^", "26^", - "27^", "28^", "29^", "30^", "31^", "32^", "33^", "34^", "35^", "36^", "37^", "38^", - "39^", "40^", "41^", "42^", "43^", "44^" - ] + (CACHE_FILLING_QUANTITY / 2..(CACHE_FILLING_QUANTITY / 2 + CACHE_FILLING_QUANTITY)) + .map(|i| HDPathComponent::securifying_base_index(i as u32)) + .collect_vec() ); } @@ -1186,7 +1181,6 @@ async fn securify_accounts_when_cache_is_half_full_multiple_factor_sources() { ) .await .unwrap(); - assert!( !stats.derived_any_new_instance_for_any_factor_source(), "should have used cache" @@ -1404,6 +1398,34 @@ async fn securify_personas_when_cache_is_half_full_single_factor_source() { ); } +#[actix_rt::test] +async fn create_single_account() { + let (mut os, bdfs) = SargonOS::with_bdfs().await; + let (alice, stats) = os.new_mainnet_account_with_bdfs("alice").await.unwrap(); + assert!(stats.debug_was_derived.is_empty(), "should have used cache"); + let (sec_accounts, stats) = os + .securify_accounts( + IndexSet::just(alice.entity_address()), + MatrixOfFactorSources::new([], 0, [bdfs]), + ) + .await + .unwrap(); + assert!( + !stats.derived_any_new_instance_for_any_factor_source(), + "should have used cache" + ); + let alice_sec = sec_accounts.into_iter().next().unwrap(); + assert_eq!( + alice_sec + .securified_entity_control() + .primary_role_instances() + .first() + .unwrap() + .derivation_entity_index(), + HDPathComponent::securifying_base_index(0) + ); +} + #[actix_rt::test] async fn securified_personas() { let (mut os, bdfs) = SargonOS::with_bdfs().await; diff --git a/src/factor_instances_provider/provider/test_sargon_os.rs b/src/factor_instances_provider/provider/test_sargon_os.rs index bc3f6e19..da6083fa 100644 --- a/src/factor_instances_provider/provider/test_sargon_os.rs +++ b/src/factor_instances_provider/provider/test_sargon_os.rs @@ -187,6 +187,18 @@ impl SargonOS { .map(|(k, outcome_per_factor)| (k, outcome_per_factor.to_use_directly)) .collect::>(); + assert_eq!( + instance_per_factor + .keys() + .cloned() + .collect::>(), + shield + .all_factors() + .into_iter() + .map(|f| f.factor_source_id()) + .collect::>() + ); + // Now we need to map the flat set of instances into many MatrixOfFactorInstances, and assign // one to each account let updated_entities = addresses_of_entities @@ -200,6 +212,7 @@ impl SargonOS { shield.clone(), ) .unwrap(); + let access_controller = match entity.security_state() { EntitySecurityState::Unsecured(_) => { AccessController::from_unsecurified_address(a) diff --git a/src/types/new_types/appendable_collection.rs b/src/types/new_types/appendable_collection.rs new file mode 100644 index 00000000..530b2c5e --- /dev/null +++ b/src/types/new_types/appendable_collection.rs @@ -0,0 +1,84 @@ +use crate::prelude::*; +use std::borrow::Borrow; + +pub trait AppendableCollection: FromIterator { + type Element: Eq + std::hash::Hash; + fn append>(&mut self, iter: T); +} +impl AppendableCollection for IndexSet { + type Element = V; + + fn append>(&mut self, iter: T) { + self.extend(iter) + } +} + +impl AppendableCollection for FactorInstances { + type Element = HierarchicalDeterministicFactorInstance; + + fn append>(&mut self, iter: T) { + self.extend(iter) + } +} + +pub trait AppendableMap { + type Key: Eq + std::hash::Hash + Clone; + type AC: AppendableCollection; + fn append_or_insert_to::Element>>( + &mut self, + key: impl Borrow, + items: I, + ); + + fn append_or_insert_element_to( + &mut self, + key: impl Borrow, + element: ::Element, + ) { + self.append_or_insert_to(key.borrow(), [element]); + } +} + +impl AppendableMap for IndexMap +where + K: Eq + std::hash::Hash + Clone, + V: AppendableCollection, +{ + type Key = K; + type AC = V; + fn append_or_insert_to::Element>>( + &mut self, + key: impl Borrow, + items: I, + ) { + let key = key.borrow(); + if let Some(existing) = self.get_mut(key) { + existing.append(items); + } else { + self.insert(key.clone(), V::from_iter(items)); + } + } +} + +#[cfg(test)] +mod test_appendable_collection { + use super::*; + + #[test] + fn test_append_element() { + type Sut = IndexMap>; + let mut map = Sut::new(); + map.append_or_insert_element_to(-3, 5); + map.append_or_insert_element_to(-3, 6); + map.append_or_insert_element_to(-3, 6); + map.append_or_insert_to(-3, [42, 237]); + map.append_or_insert_to(-9, [64, 128]); + assert_eq!( + map, + Sut::from_iter([ + (-3, IndexSet::::from_iter([5, 6, 42, 237, 237])), + (-9, IndexSet::::from_iter([64, 128])), + ]) + ); + } +} diff --git a/src/types/new_types/factor_instances.rs b/src/types/new_types/factor_instances.rs index c4527ee2..0d947600 100644 --- a/src/types/new_types/factor_instances.rs +++ b/src/types/new_types/factor_instances.rs @@ -1,12 +1,14 @@ use crate::prelude::*; /// A collection of factor instances. -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive(Default, Clone, PartialEq, Eq, derive_more::Debug)] +#[debug("FIS[{:?}]", self.factor_instances)] pub struct FactorInstances { #[allow(dead_code)] hidden: HiddenConstructor, factor_instances: IndexSet, } + impl FactorInstances { pub fn extend( &mut self, diff --git a/src/types/new_types/key_space.rs b/src/types/new_types/key_space.rs index 73f947e3..aec97c4c 100644 --- a/src/types/new_types/key_space.rs +++ b/src/types/new_types/key_space.rs @@ -34,4 +34,11 @@ impl KeySpace { pub fn both() -> [Self; 2] { [Self::Unsecurified, Self::Securified] } + + pub fn indicator(&self) -> String { + match self { + Self::Unsecurified => "'".to_owned(), + Self::Securified => "^".to_owned(), + } + } } diff --git a/src/types/new_types/mod.rs b/src/types/new_types/mod.rs index 2df728eb..fbdbf986 100644 --- a/src/types/new_types/mod.rs +++ b/src/types/new_types/mod.rs @@ -1,4 +1,5 @@ mod accounts; +mod appendable_collection; mod factor_instances; mod is_securified_entity; mod key_space; @@ -11,6 +12,7 @@ mod unsecurified_entity; mod veci; pub use accounts::*; +pub use appendable_collection::*; pub use factor_instances::*; pub use is_securified_entity::*; pub use key_space::*; diff --git a/src/types/sargon_types.rs b/src/types/sargon_types.rs index 126e8b8c..1e04be53 100644 --- a/src/types/sargon_types.rs +++ b/src/types/sargon_types.rs @@ -347,8 +347,8 @@ impl UnhardenedIndex { #[derive( Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display, derive_more::Debug, )] -#[display("{}'", self.base_index())] -#[debug("{}'", self.base_index())] +#[display("{}{}", self.base_index(), KeySpace::Unsecurified.indicator())] +#[debug("{}{}", self.base_index(), KeySpace::Unsecurified.indicator())] pub struct UnsecurifiedIndex(HDPathValue); impl UnsecurifiedIndex { /// # Panics @@ -382,8 +382,8 @@ impl UnsecurifiedIndex { #[derive( Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display, derive_more::Debug, )] -#[display("{}^", self.base_index())] -#[debug("{}^", self.base_index())] +#[display("{}{}", self.base_index(), KeySpace::Securified.indicator())] +#[debug("{}{}", self.base_index(), KeySpace::Securified.indicator())] pub struct SecurifiedIndex(u32); impl SecurifiedIndex { /// # Panics @@ -904,7 +904,7 @@ impl HierarchicalDeterministicPublicKey { } #[derive(Clone, PartialEq, Eq, std::hash::Hash, derive_more::Debug)] -#[debug("{}", self.debug_str())] +#[debug("{}", self.derivation_entity_index())] pub struct HierarchicalDeterministicFactorInstance { pub factor_source_id: FactorSourceIDFromHash, pub public_key: HierarchicalDeterministicPublicKey,