From 896b6278e023bcf562e018a7c28a8d84090964ea Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 10:30:22 +0200 Subject: [PATCH 01/17] Bump LDK for async store migration Bump LDK to 3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c Co-Authored-By: HAL 9000 --- Cargo.toml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2a60c8ced..bed984f07 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,18 +41,18 @@ postgres = ["dep:tokio-postgres", "dep:native-tls", "dep:postgres-native-tls"] #lightning-macros = { version = "0.2.0" } #lightning-dns-resolver = { version = "0.3.0" } -lightning = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182", features = ["std"] } -lightning-types = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182" } -lightning-invoice = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182", features = ["std"] } -lightning-net-tokio = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182" } -lightning-persister = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182", features = ["tokio"] } -lightning-background-processor = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182" } -lightning-rapid-gossip-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182" } -lightning-block-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182", features = ["rest-client", "rpc-client", "tokio"] } -lightning-transaction-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182", features = ["esplora-async-https", "time", "electrum-rustls-ring"] } -lightning-liquidity = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182", features = ["std"] } -lightning-macros = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182" } -lightning-dns-resolver = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182" } +lightning = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c", features = ["std"] } +lightning-types = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c" } +lightning-invoice = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c", features = ["std"] } +lightning-net-tokio = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c" } +lightning-persister = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c", features = ["tokio"] } +lightning-background-processor = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c" } +lightning-rapid-gossip-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c" } +lightning-block-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c", features = ["rest-client", "rpc-client", "tokio"] } +lightning-transaction-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c", features = ["esplora-async-https", "time", "electrum-rustls-ring"] } +lightning-liquidity = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c", features = ["std"] } +lightning-macros = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c" } +lightning-dns-resolver = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c" } bdk_chain = { version = "0.23.0", default-features = false, features = ["std"] } bdk_esplora = { version = "0.22.0", default-features = false, features = ["async-https-rustls", "tokio"]} @@ -85,13 +85,13 @@ postgres-native-tls = { version = "0.5", default-features = false, features = [" vss-client = { package = "vss-client-ng", version = "0.5" } prost = { version = "0.11.6", default-features = false} #bitcoin-payment-instructions = { version = "0.6" } -bitcoin-payment-instructions = { git = "https://github.com/tnull/bitcoin-payment-instructions", rev = "ed8657dee284f987b6791cd291d0f0f18811ee76" } +bitcoin-payment-instructions = { git = "https://github.com/tnull/bitcoin-payment-instructions", rev = "ff09ce9401afa448549a8f101172700bcd14d7bb" } [target.'cfg(windows)'.dependencies] winapi = { version = "0.3", features = ["winbase"] } [dev-dependencies] -lightning = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "090e09f3992694040d3a55c1c798b3a92fb77182", features = ["std", "_test_utils"] } +lightning = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "3dfcc4cca1866c5e5d4d4eaf3b82e09584e2ce5c", features = ["std", "_test_utils"] } rand = { version = "0.9.2", default-features = false, features = ["std", "thread_rng", "os_rng"] } proptest = "1.0.0" regex = "1.5.6" From f6d3570e656c917a63d7a0b4f714466bcc5485a7 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 10:35:46 +0200 Subject: [PATCH 02/17] Move BDK wallet helpers onto async KV storage Read and write BDK wallet state through async KVStore helpers while keeping the current WalletPersister entry points bridged through the node runtime. This reduces the wallet persistence surface that still depends on KVStoreSync. Co-Authored-By: HAL 9000 --- src/builder.rs | 7 +++++-- src/io/utils.rs | 27 ++++++++++++++++----------- src/wallet/persist.rs | 39 ++++++++++++++++++++++++++------------- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 03ded494f..c217850e3 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -1537,8 +1537,11 @@ fn build_with_store_internal( let descriptor = Bip84(xprv, KeychainKind::External); let change_descriptor = Bip84(xprv, KeychainKind::Internal); - let mut wallet_persister = - KVStoreWalletPersister::new(Arc::clone(&kv_store), Arc::clone(&logger)); + let mut wallet_persister = KVStoreWalletPersister::new( + Arc::clone(&kv_store), + Arc::clone(&runtime), + Arc::clone(&logger), + ); let wallet_opt = BdkWallet::load() .descriptor(KeychainKind::External, Some(descriptor.clone())) .descriptor(KeychainKind::Internal, Some(change_descriptor.clone())) diff --git a/src/io/utils.rs b/src/io/utils.rs index 89d4afc5c..11d1255b8 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -469,14 +469,15 @@ macro_rules! impl_read_write_change_set_type { $secondary_namespace:expr, $key:expr ) => { - pub(crate) fn $read_name( + pub(crate) async fn $read_name( kv_store: &DynStore, logger: L, ) -> Result, std::io::Error> where L::Target: LdkLogger, { let reader = - match KVStoreSync::read(&*kv_store, $primary_namespace, $secondary_namespace, $key) + match KVStore::read(&*kv_store, $primary_namespace, $secondary_namespace, $key) + .await { Ok(bytes) => bytes, Err(e) => { @@ -510,14 +511,15 @@ macro_rules! impl_read_write_change_set_type { } } - pub(crate) fn $write_name( + pub(crate) async fn $write_name( value: &$change_set_type, kv_store: &DynStore, logger: L, ) -> Result<(), std::io::Error> where L::Target: LdkLogger, { let data = ChangeSetSerWrapper(value).encode(); - KVStoreSync::write(&*kv_store, $primary_namespace, $secondary_namespace, $key, data) + KVStore::write(&*kv_store, $primary_namespace, $secondary_namespace, $key, data) + .await .map_err(|e| { log_error!( logger, @@ -588,36 +590,39 @@ impl_read_write_change_set_type!( ); // Reads the full BdkWalletChangeSet or returns default fields -pub(crate) fn read_bdk_wallet_change_set( +pub(crate) async fn read_bdk_wallet_change_set( kv_store: &DynStore, logger: &Logger, ) -> Result, std::io::Error> { let mut change_set = BdkWalletChangeSet::default(); // We require a descriptor and return `None` to signal creation of a new wallet otherwise. - if let Some(descriptor) = read_bdk_wallet_descriptor(kv_store, logger)? { + if let Some(descriptor) = read_bdk_wallet_descriptor(kv_store, logger).await? { change_set.descriptor = Some(descriptor); } else { return Ok(None); } // We require a change_descriptor and return `None` to signal creation of a new wallet otherwise. - if let Some(change_descriptor) = read_bdk_wallet_change_descriptor(kv_store, logger)? { + if let Some(change_descriptor) = read_bdk_wallet_change_descriptor(kv_store, logger).await? { change_set.change_descriptor = Some(change_descriptor); } else { return Ok(None); } // We require a network and return `None` to signal creation of a new wallet otherwise. - if let Some(network) = read_bdk_wallet_network(kv_store, logger)? { + if let Some(network) = read_bdk_wallet_network(kv_store, logger).await? { change_set.network = Some(network); } else { return Ok(None); } - read_bdk_wallet_local_chain(&*kv_store, logger)? + read_bdk_wallet_local_chain(&*kv_store, logger) + .await? .map(|local_chain| change_set.local_chain = local_chain); - read_bdk_wallet_tx_graph(&*kv_store, logger)?.map(|tx_graph| change_set.tx_graph = tx_graph); - read_bdk_wallet_indexer(&*kv_store, logger)?.map(|indexer| change_set.indexer = indexer); + read_bdk_wallet_tx_graph(&*kv_store, logger) + .await? + .map(|tx_graph| change_set.tx_graph = tx_graph); + read_bdk_wallet_indexer(&*kv_store, logger).await?.map(|indexer| change_set.indexer = indexer); Ok(Some(change_set)) } diff --git a/src/wallet/persist.rs b/src/wallet/persist.rs index 10be1fac0..cc5005780 100644 --- a/src/wallet/persist.rs +++ b/src/wallet/persist.rs @@ -16,16 +16,19 @@ use crate::io::utils::{ write_bdk_wallet_tx_graph, }; use crate::logger::{log_error, LdkLogger, Logger}; +use crate::runtime::Runtime; use crate::types::DynStore; + pub(crate) struct KVStoreWalletPersister { latest_change_set: Option, kv_store: Arc, + runtime: Arc, logger: Arc, } impl KVStoreWalletPersister { - pub(crate) fn new(kv_store: Arc, logger: Arc) -> Self { - Self { latest_change_set: None, kv_store, logger } + pub(crate) fn new(kv_store: Arc, runtime: Arc, logger: Arc) -> Self { + Self { latest_change_set: None, kv_store, runtime, logger } } } @@ -38,7 +41,9 @@ impl WalletPersister for KVStoreWalletPersister { return Ok(latest_change_set.clone()); } - let change_set_opt = read_bdk_wallet_change_set(&*persister.kv_store, &*persister.logger)?; + let change_set_opt = persister + .runtime + .block_on(read_bdk_wallet_change_set(&*persister.kv_store, &*persister.logger))?; let change_set = match change_set_opt { Some(persisted_change_set) => persisted_change_set, @@ -84,7 +89,11 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.descriptor = Some(descriptor.clone()); - write_bdk_wallet_descriptor(&descriptor, &*persister.kv_store, &*persister.logger)?; + persister.runtime.block_on(write_bdk_wallet_descriptor( + &descriptor, + &*persister.kv_store, + &*persister.logger, + ))?; } } @@ -103,11 +112,11 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.change_descriptor = Some(change_descriptor.clone()); - write_bdk_wallet_change_descriptor( + persister.runtime.block_on(write_bdk_wallet_change_descriptor( &change_descriptor, &*persister.kv_store, &*persister.logger, - )?; + ))?; } } @@ -124,7 +133,11 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.network = Some(network); - write_bdk_wallet_network(&network, &*persister.kv_store, &*persister.logger)?; + persister.runtime.block_on(write_bdk_wallet_network( + &network, + &*persister.kv_store, + &*persister.logger, + ))?; } } @@ -144,29 +157,29 @@ impl WalletPersister for KVStoreWalletPersister { // particular order. if !change_set.indexer.is_empty() { latest_change_set.indexer.merge(change_set.indexer.clone()); - write_bdk_wallet_indexer( + persister.runtime.block_on(write_bdk_wallet_indexer( &latest_change_set.indexer, &*persister.kv_store, Arc::clone(&persister.logger), - )?; + ))?; } if !change_set.tx_graph.is_empty() { latest_change_set.tx_graph.merge(change_set.tx_graph.clone()); - write_bdk_wallet_tx_graph( + persister.runtime.block_on(write_bdk_wallet_tx_graph( &latest_change_set.tx_graph, &*persister.kv_store, Arc::clone(&persister.logger), - )?; + ))?; } if !change_set.local_chain.is_empty() { latest_change_set.local_chain.merge(change_set.local_chain.clone()); - write_bdk_wallet_local_chain( + persister.runtime.block_on(write_bdk_wallet_local_chain( &latest_change_set.local_chain, &*persister.kv_store, Arc::clone(&persister.logger), - )?; + ))?; } Ok(()) From e67b7a1ee75f56f3e07e3e75e5adac113a2a91ae Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 10:39:05 +0200 Subject: [PATCH 03/17] Use async KV storage for static invoices Static invoice persistence already runs from async handlers, so use KVStore directly instead of routing those reads and writes through the blocking KVStoreSync trait. Co-Authored-By: HAL 9000 --- src/payment/asynchronous/static_invoice_store.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/payment/asynchronous/static_invoice_store.rs b/src/payment/asynchronous/static_invoice_store.rs index 6fb406334..f1e2378c2 100644 --- a/src/payment/asynchronous/static_invoice_store.rs +++ b/src/payment/asynchronous/static_invoice_store.rs @@ -15,7 +15,7 @@ use bitcoin::hashes::Hash; use lightning::blinded_path::message::BlindedMessagePath; use lightning::impl_writeable_tlv_based; use lightning::offers::static_invoice::StaticInvoice; -use lightning::util::persist::KVStoreSync; +use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, Writeable}; use crate::hex_utils; @@ -78,12 +78,13 @@ impl StaticInvoiceStore { let (secondary_namespace, key) = Self::get_storage_location(invoice_slot, recipient_id); - KVStoreSync::read( + KVStore::read( &*self.kv_store, STATIC_INVOICE_STORE_PRIMARY_NAMESPACE, &secondary_namespace, &key, ) + .await .and_then(|data| { PersistedStaticInvoice::read(&mut &*data) .map(|persisted_invoice| { @@ -124,13 +125,14 @@ impl StaticInvoiceStore { // Static invoices will be persisted at "static_invoices//". // // Example: static_invoices/039058c6f2c0cb492c533b0a4d14ef77cc0f78abccced5287d84a1a2011cfb81/00001 - KVStoreSync::write( + KVStore::write( &*self.kv_store, STATIC_INVOICE_STORE_PRIMARY_NAMESPACE, &secondary_namespace, &key, buf, ) + .await } fn get_storage_location(invoice_slot: u16, recipient_id: &[u8]) -> (String, String) { From d49b681144776c4313dc1ff5f93156a433a9deea Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 10:42:45 +0200 Subject: [PATCH 04/17] Move peer persistence onto async KV storage Persist peer store updates through async KVStore operations. The synchronous node APIs keep bridging at their runtime boundary while async event handling awaits peer persistence directly. Co-Authored-By: HAL 9000 --- src/event.rs | 62 +++++++++++++++++++------------------- src/lib.rs | 8 ++--- src/payment/bolt11.rs | 2 +- src/peer_store.rs | 70 +++++++++++++++++++++++++++---------------- 4 files changed, 82 insertions(+), 60 deletions(-) diff --git a/src/event.rs b/src/event.rs index 7d23be99a..48a69db06 100644 --- a/src/event.rs +++ b/src/event.rs @@ -1524,36 +1524,38 @@ where }, }; - let network_graph = self.network_graph.read_only(); - let channels = - self.channel_manager.list_channels_with_counterparty(&counterparty_node_id); - if let Some(pending_channel) = - channels.into_iter().find(|c| c.channel_id == channel_id) - { - if !pending_channel.is_outbound - && self.peer_store.get_peer(&counterparty_node_id).is_none() - { - if let Some(address) = network_graph - .nodes() - .get(&NodeId::from_pubkey(&counterparty_node_id)) - .and_then(|node_info| node_info.announcement_info.as_ref()) - .and_then(|ann_info| ann_info.addresses().first()) - { - let peer = PeerInfo { - node_id: counterparty_node_id, - address: address.clone(), - }; - - self.peer_store.add_peer(peer).unwrap_or_else(|e| { - log_error!( - self.logger, - "Failed to add peer {} to peer store: {}", - counterparty_node_id, - e - ); - }); - } - } + let peer_to_store = { + let network_graph = self.network_graph.read_only(); + let channels = + self.channel_manager.list_channels_with_counterparty(&counterparty_node_id); + channels + .into_iter() + .find(|c| c.channel_id == channel_id) + .filter(|pending_channel| { + !pending_channel.is_outbound + && self.peer_store.get_peer(&counterparty_node_id).is_none() + }) + .and_then(|_| { + network_graph + .nodes() + .get(&NodeId::from_pubkey(&counterparty_node_id)) + .and_then(|node_info| node_info.announcement_info.as_ref()) + .and_then(|ann_info| ann_info.addresses().first()) + .map(|address| PeerInfo { + node_id: counterparty_node_id, + address: address.clone(), + }) + }) + }; + if let Some(peer) = peer_to_store { + self.peer_store.add_peer(peer).await.unwrap_or_else(|e| { + log_error!( + self.logger, + "Failed to add peer {} to peer store: {}", + counterparty_node_id, + e + ); + }); } }, LdkEvent::ChannelReady { diff --git a/src/lib.rs b/src/lib.rs index 614be098b..3f536d3ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1121,7 +1121,7 @@ impl Node { log_info!(self.logger, "Connected to peer {}@{}. ", peer_info.node_id, peer_info.address); if persist { - self.peer_store.add_peer(peer_info)?; + self.runtime.block_on(self.peer_store.add_peer(peer_info))?; } Ok(()) @@ -1138,7 +1138,7 @@ impl Node { log_info!(self.logger, "Disconnecting peer {}..", counterparty_node_id); - match self.peer_store.remove_peer(&counterparty_node_id) { + match self.runtime.block_on(self.peer_store.remove_peer(&counterparty_node_id)) { Ok(()) => {}, Err(e) => { log_error!(self.logger, "Failed to remove peer {}: {}", counterparty_node_id, e) @@ -1255,7 +1255,7 @@ impl Node { zero_reserve_string, peer_info.node_id ); - self.peer_store.add_peer(peer_info)?; + self.runtime.block_on(self.peer_store.add_peer(peer_info))?; Ok(UserChannelId(user_channel_id)) }, Err(e) => { @@ -1861,7 +1861,7 @@ impl Node { // Check if this was the last open channel, if so, forget the peer. if open_channels.len() == 1 { - self.peer_store.remove_peer(&counterparty_node_id)?; + self.runtime.block_on(self.peer_store.remove_peer(&counterparty_node_id))?; } } diff --git a/src/payment/bolt11.rs b/src/payment/bolt11.rs index e81aa51f7..816698d4a 100644 --- a/src/payment/bolt11.rs +++ b/src/payment/bolt11.rs @@ -242,7 +242,7 @@ impl Bolt11Payment { self.payment_store.insert(payment)?; // Persist LSP peer to make sure we reconnect on restart. - self.peer_store.add_peer(peer_info)?; + self.runtime.block_on(self.peer_store.add_peer(peer_info))?; Ok(invoice) } diff --git a/src/peer_store.rs b/src/peer_store.rs index 307fb6929..8037f9347 100644 --- a/src/peer_store.rs +++ b/src/peer_store.rs @@ -11,7 +11,7 @@ use std::sync::{Arc, RwLock}; use bitcoin::secp256k1::PublicKey; use lightning::impl_writeable_tlv_based; -use lightning::util::persist::KVStoreSync; +use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, ReadableArgs, Writeable, Writer}; use crate::io::{ @@ -27,6 +27,7 @@ where L::Target: LdkLogger, { peers: RwLock>, + mutation_lock: tokio::sync::Mutex<()>, kv_store: Arc, logger: L, } @@ -37,44 +38,60 @@ where { pub(crate) fn new(kv_store: Arc, logger: L) -> Self { let peers = RwLock::new(HashMap::new()); - Self { peers, kv_store, logger } + let mutation_lock = tokio::sync::Mutex::new(()); + Self { peers, mutation_lock, kv_store, logger } } - pub(crate) fn add_peer(&self, peer_info: PeerInfo) -> Result<(), Error> { - let mut locked_peers = self.peers.write().expect("lock"); - - if locked_peers.contains_key(&peer_info.node_id) { - return Ok(()); - } - - locked_peers.insert(peer_info.node_id, peer_info); - self.persist_peers(&*locked_peers) + pub(crate) async fn add_peer(&self, peer_info: PeerInfo) -> Result<(), Error> { + let _guard = self.mutation_lock.lock().await; + let data = { + let mut locked_peers = self.peers.write().expect("lock"); + if locked_peers.contains_key(&peer_info.node_id) { + return Ok(()); + } + locked_peers.insert(peer_info.node_id, peer_info); + PeerStoreSerWrapper(&locked_peers).encode() + }; + self.persist_peers(data).await } - pub(crate) fn remove_peer(&self, node_id: &PublicKey) -> Result<(), Error> { - let mut locked_peers = self.peers.write().expect("lock"); - - locked_peers.remove(node_id); - self.persist_peers(&*locked_peers) + pub(crate) async fn remove_peer(&self, node_id: &PublicKey) -> Result<(), Error> { + let _guard = self.mutation_lock.lock().await; + let data = { + let mut locked_peers = self.peers.write().expect("lock"); + locked_peers.remove(node_id); + PeerStoreSerWrapper(&locked_peers).encode() + }; + self.persist_peers(data).await } + /// Returns the current in-memory peer set. + /// + /// The async mutation lock serializes `add_peer` and `remove_peer`, but this synchronous + /// reader cannot wait on it. Until peer-store reads are async, callers may observe peer + /// changes that are still being persisted. pub(crate) fn list_peers(&self) -> Vec { self.peers.read().expect("lock").values().cloned().collect() } + /// Returns the current in-memory peer info for `node_id`. + /// + /// The async mutation lock serializes `add_peer` and `remove_peer`, but this synchronous + /// reader cannot wait on it. Until peer-store reads are async, callers may observe peer + /// changes that are still being persisted. pub(crate) fn get_peer(&self, node_id: &PublicKey) -> Option { self.peers.read().expect("lock").get(node_id).cloned() } - fn persist_peers(&self, locked_peers: &HashMap) -> Result<(), Error> { - let data = PeerStoreSerWrapper(&*locked_peers).encode(); - KVStoreSync::write( + async fn persist_peers(&self, data: Vec) -> Result<(), Error> { + KVStore::write( &*self.kv_store, PEER_INFO_PERSISTENCE_PRIMARY_NAMESPACE, PEER_INFO_PERSISTENCE_SECONDARY_NAMESPACE, PEER_INFO_PERSISTENCE_KEY, data, ) + .await .map_err(|e| { log_error!( self.logger, @@ -101,7 +118,8 @@ where let (kv_store, logger) = args; let read_peers: PeerStoreDeserWrapper = Readable::read(reader)?; let peers: RwLock> = RwLock::new(read_peers.0); - Ok(Self { peers, kv_store, logger }) + let mutation_lock = tokio::sync::Mutex::new(()); + Ok(Self { peers, mutation_lock, kv_store, logger }) } } @@ -158,8 +176,8 @@ mod tests { use crate::io::test_utils::InMemoryStore; use crate::types::DynStoreWrapper; - #[test] - fn peer_info_persistence() { + #[tokio::test] + async fn peer_info_persistence() { let store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let logger = Arc::new(TestLogger::new()); let peer_store = PeerStore::new(Arc::clone(&store), Arc::clone(&logger)); @@ -170,22 +188,24 @@ mod tests { .unwrap(); let address = SocketAddress::from_str("127.0.0.1:9738").unwrap(); let expected_peer_info = PeerInfo { node_id, address }; - assert!(KVStoreSync::read( + assert!(KVStore::read( &*store, PEER_INFO_PERSISTENCE_PRIMARY_NAMESPACE, PEER_INFO_PERSISTENCE_SECONDARY_NAMESPACE, PEER_INFO_PERSISTENCE_KEY, ) + .await .is_err()); - peer_store.add_peer(expected_peer_info.clone()).unwrap(); + peer_store.add_peer(expected_peer_info.clone()).await.unwrap(); // Check we can read back what we persisted. - let persisted_bytes = KVStoreSync::read( + let persisted_bytes = KVStore::read( &*store, PEER_INFO_PERSISTENCE_PRIMARY_NAMESPACE, PEER_INFO_PERSISTENCE_SECONDARY_NAMESPACE, PEER_INFO_PERSISTENCE_KEY, ) + .await .unwrap(); let deser_peer_store = PeerStore::read(&mut &persisted_bytes[..], (Arc::clone(&store), logger)).unwrap(); From b3f947c3b9d73b1336fffa95cef6066f6e834bce Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 10:53:05 +0200 Subject: [PATCH 05/17] Move DataStore persistence onto async KV storage Persist DataStore mutations through async KVStore operations while keeping the existing synchronous APIs bridged through the node runtime. Async event handling now awaits payment store writes directly. Co-Authored-By: HAL 9000 --- src/builder.rs | 1 + src/data_store.rs | 149 ++++++++++++++++++++++++------------- src/event.rs | 28 +++---- src/lib.rs | 6 +- src/payment/bolt11.rs | 14 ++-- src/payment/bolt12.rs | 22 +++--- src/payment/spontaneous.rs | 14 ++-- src/wallet/mod.rs | 39 ++++++---- 8 files changed, 171 insertions(+), 102 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index c217850e3..779b2bd39 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -1622,6 +1622,7 @@ fn build_with_store_internal( Arc::clone(&fee_estimator), Arc::clone(&chain_source), Arc::clone(&payment_store), + Arc::clone(&runtime), Arc::clone(&config), Arc::clone(&logger), Arc::clone(&pending_payment_store), diff --git a/src/data_store.rs b/src/data_store.rs index f80ec0891..70abfcc3f 100644 --- a/src/data_store.rs +++ b/src/data_store.rs @@ -9,7 +9,7 @@ use std::collections::{hash_map, HashMap}; use std::ops::Deref; use std::sync::{Arc, Mutex}; -use lightning::util::persist::KVStoreSync; +use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, Writeable}; use crate::logger::{log_error, LdkLogger}; @@ -45,6 +45,7 @@ where L::Target: LdkLogger, { objects: Mutex>, + mutation_lock: tokio::sync::Mutex<()>, primary_namespace: String, secondary_namespace: String, kv_store: Arc, @@ -61,50 +62,64 @@ where ) -> Self { let objects = Mutex::new(HashMap::from_iter(objects.into_iter().map(|obj| (obj.id(), obj)))); - Self { objects, primary_namespace, secondary_namespace, kv_store, logger } + Self { + objects, + mutation_lock: tokio::sync::Mutex::new(()), + primary_namespace, + secondary_namespace, + kv_store, + logger, + } } - pub(crate) fn insert(&self, object: SO) -> Result { - let mut locked_objects = self.objects.lock().expect("lock"); + pub(crate) async fn insert(&self, object: SO) -> Result { + let _guard = self.mutation_lock.lock().await; - self.persist(&object)?; + self.persist(&object).await?; + let mut locked_objects = self.objects.lock().expect("lock"); let updated = locked_objects.insert(object.id(), object).is_some(); Ok(updated) } - pub(crate) fn insert_or_update(&self, object: SO) -> Result { - let mut locked_objects = self.objects.lock().expect("lock"); + pub(crate) async fn insert_or_update(&self, object: SO) -> Result { + let _guard = self.mutation_lock.lock().await; + let (updated, data_to_persist) = { + let mut locked_objects = self.objects.lock().expect("lock"); + match locked_objects.entry(object.id()) { + hash_map::Entry::Occupied(mut e) => { + let update = object.to_update(); + let updated = e.get_mut().update(update); + let data_to_persist = + if updated { Some(Self::encode_object(e.get())) } else { None }; + (updated, data_to_persist) + }, + hash_map::Entry::Vacant(e) => { + let data_to_persist = Self::encode_object(&object); + e.insert(object); + (true, Some(data_to_persist)) + }, + } + }; - let updated; - match locked_objects.entry(object.id()) { - hash_map::Entry::Occupied(mut e) => { - let update = object.to_update(); - updated = e.get_mut().update(update); - if updated { - self.persist(&e.get())?; - } - }, - hash_map::Entry::Vacant(e) => { - e.insert(object.clone()); - self.persist(&object)?; - updated = true; - }, + if let Some((store_key, data)) = data_to_persist { + self.persist_encoded(store_key, data).await?; } - Ok(updated) } - pub(crate) fn remove(&self, id: &SO::Id) -> Result<(), Error> { - let removed = self.objects.lock().expect("lock").remove(id).is_some(); + pub(crate) async fn remove(&self, id: &SO::Id) -> Result<(), Error> { + let _guard = self.mutation_lock.lock().await; + let removed = { self.objects.lock().expect("lock").remove(id).is_some() }; if removed { let store_key = id.encode_to_hex_str(); - KVStoreSync::remove( + KVStore::remove( &*self.kv_store, &self.primary_namespace, &self.secondary_namespace, &store_key, false, ) + .await .map_err(|e| { log_error!( self.logger, @@ -120,40 +135,63 @@ where Ok(()) } + /// Returns the current in-memory object for `id`. + /// + /// The async mutation lock serializes writers, but this synchronous reader cannot wait on it. + /// Until store reads are async, callers may temporarily see in-memory state that is either + /// still being persisted or has not yet caught up to a write in progress. pub(crate) fn get(&self, id: &SO::Id) -> Option { self.objects.lock().expect("lock").get(id).cloned() } - pub(crate) fn update(&self, update: SO::Update) -> Result { - let mut locked_objects = self.objects.lock().expect("lock"); - - if let Some(object) = locked_objects.get_mut(&update.id()) { - let updated = object.update(update); - if updated { - self.persist(&object)?; - Ok(DataStoreUpdateResult::Updated) + pub(crate) async fn update(&self, update: SO::Update) -> Result { + let _guard = self.mutation_lock.lock().await; + let (res, data_to_persist) = { + let mut locked_objects = self.objects.lock().expect("lock"); + if let Some(object) = locked_objects.get_mut(&update.id()) { + let updated = object.update(update); + if updated { + (DataStoreUpdateResult::Updated, Some(Self::encode_object(object))) + } else { + (DataStoreUpdateResult::Unchanged, None) + } } else { - Ok(DataStoreUpdateResult::Unchanged) + (DataStoreUpdateResult::NotFound, None) } - } else { - Ok(DataStoreUpdateResult::NotFound) + }; + if let Some((store_key, data)) = data_to_persist { + self.persist_encoded(store_key, data).await?; } + Ok(res) } + /// Returns in-memory objects matching `f`. + /// + /// The async mutation lock serializes writers, but this synchronous reader cannot wait on it. + /// Until store reads are async, callers may temporarily see in-memory state that is either + /// still being persisted or has not yet caught up to a write in progress. pub(crate) fn list_filter bool>(&self, f: F) -> Vec { self.objects.lock().expect("lock").values().filter(f).cloned().collect::>() } - fn persist(&self, object: &SO) -> Result<(), Error> { - let store_key = object.id().encode_to_hex_str(); - let data = object.encode(); - KVStoreSync::write( + async fn persist(&self, object: &SO) -> Result<(), Error> { + let (store_key, data) = Self::encode_object(object); + self.persist_encoded(store_key, data).await + } + + fn encode_object(object: &SO) -> (String, Vec) { + (object.id().encode_to_hex_str(), object.encode()) + } + + async fn persist_encoded(&self, store_key: String, data: Vec) -> Result<(), Error> { + KVStore::write( &*self.kv_store, &self.primary_namespace, &self.secondary_namespace, &store_key, data, ) + .await .map_err(|e| { log_error!( self.logger, @@ -168,6 +206,11 @@ where Ok(()) } + /// Returns whether the in-memory store contains `id`. + /// + /// The async mutation lock serializes writers, but this synchronous reader cannot wait on it. + /// Until store reads are async, callers may temporarily see in-memory state that is either + /// still being persisted or has not yet caught up to a write in progress. pub(crate) fn contains_key(&self, id: &SO::Id) -> bool { self.objects.lock().expect("lock").contains_key(id) } @@ -238,8 +281,8 @@ mod tests { (2, data, required), }); - #[test] - fn data_is_persisted() { + #[tokio::test] + async fn data_is_persisted() { let store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let logger = Arc::new(TestLogger::new()); let primary_namespace = "datastore_test_primary".to_string(); @@ -258,47 +301,49 @@ mod tests { let store_key = id.encode_to_hex_str(); // Check we start empty. - assert!(KVStoreSync::read(&*store, &primary_namespace, &secondary_namespace, &store_key) + assert!(KVStore::read(&*store, &primary_namespace, &secondary_namespace, &store_key) + .await .is_err()); // Check we successfully store an object and return `false` let object = TestObject { id, data: [23u8; 3] }; - assert_eq!(Ok(false), data_store.insert(object.clone())); + assert_eq!(Ok(false), data_store.insert(object.clone()).await); assert_eq!(Some(object), data_store.get(&id)); - assert!(KVStoreSync::read(&*store, &primary_namespace, &secondary_namespace, &store_key) + assert!(KVStore::read(&*store, &primary_namespace, &secondary_namespace, &store_key) + .await .is_ok()); // Test re-insertion returns `true` let mut override_object = object.clone(); override_object.data = [24u8; 3]; - assert_eq!(Ok(true), data_store.insert(override_object)); + assert_eq!(Ok(true), data_store.insert(override_object).await); assert_eq!(Some(override_object), data_store.get(&id)); // Check update returns `Updated` let update = TestObjectUpdate { id, data: [25u8; 3] }; - assert_eq!(Ok(DataStoreUpdateResult::Updated), data_store.update(update)); + assert_eq!(Ok(DataStoreUpdateResult::Updated), data_store.update(update).await); assert_eq!(data_store.get(&id).unwrap().data, [25u8; 3]); // Check no-op update yields `Unchanged` let update = TestObjectUpdate { id, data: [25u8; 3] }; - assert_eq!(Ok(DataStoreUpdateResult::Unchanged), data_store.update(update)); + assert_eq!(Ok(DataStoreUpdateResult::Unchanged), data_store.update(update).await); // Check bogus update yields `NotFound` let bogus_id = TestObjectId { id: [84u8; 4] }; let update = TestObjectUpdate { id: bogus_id, data: [12u8; 3] }; - assert_eq!(Ok(DataStoreUpdateResult::NotFound), data_store.update(update)); + assert_eq!(Ok(DataStoreUpdateResult::NotFound), data_store.update(update).await); // Check `insert_or_update` inserts unknown objects let iou_id = TestObjectId { id: [55u8; 4] }; let iou_object = TestObject { id: iou_id, data: [34u8; 3] }; - assert_eq!(Ok(true), data_store.insert_or_update(iou_object.clone())); + assert_eq!(Ok(true), data_store.insert_or_update(iou_object.clone()).await); // Check `insert_or_update` doesn't update the same object - assert_eq!(Ok(false), data_store.insert_or_update(iou_object.clone())); + assert_eq!(Ok(false), data_store.insert_or_update(iou_object.clone()).await); // Check `insert_or_update` updates if object changed let mut new_iou_object = iou_object; new_iou_object.data[0] += 1; - assert_eq!(Ok(true), data_store.insert_or_update(new_iou_object)); + assert_eq!(Ok(true), data_store.insert_or_update(new_iou_object).await); } } diff --git a/src/event.rs b/src/event.rs index 48a69db06..86ee7bb05 100644 --- a/src/event.rs +++ b/src/event.rs @@ -581,7 +581,7 @@ where } } - fn fail_claimable_payment( + async fn fail_claimable_payment( &self, payment_id: PaymentId, payment_hash: &PaymentHash, ) -> Result<(), ReplayEvent> { self.channel_manager.fail_htlc_backwards(payment_hash); @@ -591,7 +591,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => Ok(()), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -738,7 +738,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => return Ok(()), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -781,7 +781,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => return Ok(()), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -809,7 +809,7 @@ where hex_utils::to_string(&payment_hash.0), counterparty_skimmed_fee_msat, ); - self.fail_claimable_payment(payment_id, &payment_hash)?; + self.fail_claimable_payment(payment_id, &payment_hash).await?; return Ok(()); }; @@ -821,7 +821,7 @@ where counterparty_skimmed_fee_msat, max_total_opening_fee_msat, ); - self.fail_claimable_payment(payment_id, &payment_hash)?; + self.fail_claimable_payment(payment_id, &payment_hash).await?; return Ok(()); } @@ -832,14 +832,14 @@ where counterparty_skimmed_fee_msat: Some(Some(counterparty_skimmed_fee_msat)), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => (), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); return Err(ReplayEvent()); }, }; - } + }, _ => debug_assert!(false, "We only expect the counterparty to get away with withholding fees for BOLT11 payments."), } } @@ -923,7 +923,7 @@ where PaymentStatus::Pending, ); - match self.payment_store.insert(payment) { + match self.payment_store.insert(payment).await { Ok(false) => (), Ok(true) => { log_error!( @@ -964,7 +964,7 @@ where PaymentStatus::Pending, ); - match self.payment_store.insert(payment) { + match self.payment_store.insert(payment).await { Ok(false) => (), Ok(true) => { log_error!( @@ -1004,7 +1004,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => return Ok(()), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -1072,7 +1072,7 @@ where }, }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(DataStoreUpdateResult::Updated) | Ok(DataStoreUpdateResult::Unchanged) => ( // No need to do anything if the idempotent update was applied, which might // be the result of a replayed event. @@ -1134,7 +1134,7 @@ where ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => {}, Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -1189,7 +1189,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => {}, Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); diff --git a/src/lib.rs b/src/lib.rs index 3f536d3ea..792f7c9f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -925,6 +925,7 @@ impl Node { #[cfg(not(feature = "uniffi"))] pub fn bolt12_payment(&self) -> Bolt12Payment { Bolt12Payment::new( + Arc::clone(&self.runtime), Arc::clone(&self.channel_manager), Arc::clone(&self.keys_manager), Arc::clone(&self.payment_store), @@ -941,6 +942,7 @@ impl Node { #[cfg(feature = "uniffi")] pub fn bolt12_payment(&self) -> Arc { Arc::new(Bolt12Payment::new( + Arc::clone(&self.runtime), Arc::clone(&self.channel_manager), Arc::clone(&self.keys_manager), Arc::clone(&self.payment_store), @@ -955,6 +957,7 @@ impl Node { #[cfg(not(feature = "uniffi"))] pub fn spontaneous_payment(&self) -> SpontaneousPayment { SpontaneousPayment::new( + Arc::clone(&self.runtime), Arc::clone(&self.channel_manager), Arc::clone(&self.keys_manager), Arc::clone(&self.payment_store), @@ -968,6 +971,7 @@ impl Node { #[cfg(feature = "uniffi")] pub fn spontaneous_payment(&self) -> Arc { Arc::new(SpontaneousPayment::new( + Arc::clone(&self.runtime), Arc::clone(&self.channel_manager), Arc::clone(&self.keys_manager), Arc::clone(&self.payment_store), @@ -1899,7 +1903,7 @@ impl Node { /// Remove the payment with the given id from the store. pub fn remove_payment(&self, payment_id: &PaymentId) -> Result<(), Error> { - self.payment_store.remove(&payment_id) + self.runtime.block_on(self.payment_store.remove(&payment_id)) } /// Retrieves an overview of all known balances. diff --git a/src/payment/bolt11.rs b/src/payment/bolt11.rs index 816698d4a..068269997 100644 --- a/src/payment/bolt11.rs +++ b/src/payment/bolt11.rs @@ -158,7 +158,7 @@ impl Bolt11Payment { PaymentDirection::Inbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(invoice) } @@ -239,7 +239,7 @@ impl Bolt11Payment { PaymentDirection::Inbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; // Persist LSP peer to make sure we reconnect on restart. self.runtime.block_on(self.peer_store.add_peer(peer_info))?; @@ -341,7 +341,7 @@ impl Bolt11Payment { PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -371,7 +371,7 @@ impl Bolt11Payment { PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::PaymentSendingFailed) }, } @@ -457,7 +457,7 @@ impl Bolt11Payment { PaymentDirection::Outbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -488,7 +488,7 @@ impl Bolt11Payment { PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::PaymentSendingFailed) }, } @@ -582,7 +582,7 @@ impl Bolt11Payment { ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.runtime.block_on(self.payment_store.update(update)) { Ok(DataStoreUpdateResult::Updated) | Ok(DataStoreUpdateResult::Unchanged) => (), Ok(DataStoreUpdateResult::NotFound) => { log_error!( diff --git a/src/payment/bolt12.rs b/src/payment/bolt12.rs index 2e5a5fb45..d79aca6c2 100644 --- a/src/payment/bolt12.rs +++ b/src/payment/bolt12.rs @@ -29,6 +29,7 @@ use crate::error::Error; use crate::ffi::{maybe_deref, maybe_wrap}; use crate::logger::{log_error, log_info, LdkLogger, Logger}; use crate::payment::store::{PaymentDetails, PaymentDirection, PaymentKind, PaymentStatus}; +use crate::runtime::Runtime; use crate::types::{ChannelManager, KeysManager, PaymentStore}; #[cfg(not(feature = "uniffi"))] @@ -59,6 +60,7 @@ type HumanReadableName = Arc; /// [`Node::bolt12_payment`]: crate::Node::bolt12_payment #[cfg_attr(feature = "uniffi", derive(uniffi::Object))] pub struct Bolt12Payment { + runtime: Arc, channel_manager: Arc, keys_manager: Arc, payment_store: Arc, @@ -70,11 +72,13 @@ pub struct Bolt12Payment { impl Bolt12Payment { pub(crate) fn new( - channel_manager: Arc, keys_manager: Arc, - payment_store: Arc, config: Arc, is_running: Arc>, - logger: Arc, async_payments_role: Option, + runtime: Arc, channel_manager: Arc, + keys_manager: Arc, payment_store: Arc, config: Arc, + is_running: Arc>, logger: Arc, + async_payments_role: Option, ) -> Self { Self { + runtime, channel_manager, keys_manager, payment_store, @@ -163,7 +167,7 @@ impl Bolt12Payment { PaymentDirection::Outbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -188,7 +192,7 @@ impl Bolt12Payment { PaymentDirection::Outbound, PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::PaymentSendingFailed) }, } @@ -325,7 +329,7 @@ impl Bolt12Payment { PaymentDirection::Outbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -350,7 +354,7 @@ impl Bolt12Payment { PaymentDirection::Outbound, PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::InvoiceRequestCreationFailed) }, } @@ -457,7 +461,7 @@ impl Bolt12Payment { PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(maybe_wrap(invoice)) } @@ -526,7 +530,7 @@ impl Bolt12Payment { PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(maybe_wrap(refund)) } diff --git a/src/payment/spontaneous.rs b/src/payment/spontaneous.rs index 1c819582e..45dab644d 100644 --- a/src/payment/spontaneous.rs +++ b/src/payment/spontaneous.rs @@ -22,6 +22,7 @@ use crate::config::{Config, LDK_PAYMENT_RETRY_TIMEOUT}; use crate::error::Error; use crate::logger::{log_error, log_info, LdkLogger, Logger}; use crate::payment::store::{PaymentDetails, PaymentDirection, PaymentKind, PaymentStatus}; +use crate::runtime::Runtime; use crate::types::{ChannelManager, CustomTlvRecord, KeysManager, PaymentStore}; // The default `final_cltv_expiry_delta` we apply when not set. @@ -34,6 +35,7 @@ const LDK_DEFAULT_FINAL_CLTV_EXPIRY_DELTA: u32 = 144; /// [`Node::spontaneous_payment`]: crate::Node::spontaneous_payment #[cfg_attr(feature = "uniffi", derive(uniffi::Object))] pub struct SpontaneousPayment { + runtime: Arc, channel_manager: Arc, keys_manager: Arc, payment_store: Arc, @@ -44,11 +46,11 @@ pub struct SpontaneousPayment { impl SpontaneousPayment { pub(crate) fn new( - channel_manager: Arc, keys_manager: Arc, - payment_store: Arc, config: Arc, is_running: Arc>, - logger: Arc, + runtime: Arc, channel_manager: Arc, + keys_manager: Arc, payment_store: Arc, config: Arc, + is_running: Arc>, logger: Arc, ) -> Self { - Self { channel_manager, keys_manager, payment_store, config, is_running, logger } + Self { runtime, channel_manager, keys_manager, payment_store, config, is_running, logger } } fn send_inner( @@ -130,7 +132,7 @@ impl SpontaneousPayment { PaymentDirection::Outbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -153,7 +155,7 @@ impl SpontaneousPayment { PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::PaymentSendingFailed) }, } diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index 13b1f384f..cdae34a0a 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -58,6 +58,7 @@ use crate::payment::store::ConfirmationStatus; use crate::payment::{ PaymentDetails, PaymentDirection, PaymentKind, PaymentStatus, PendingPaymentDetails, }; +use crate::runtime::Runtime; use crate::types::{Broadcaster, PaymentStore, PendingPaymentStore}; use crate::{ChainSource, Error}; @@ -85,6 +86,7 @@ pub(crate) struct Wallet { fee_estimator: Arc, chain_source: Arc, payment_store: Arc, + runtime: Arc, config: Arc, logger: Arc, pending_payment_store: Arc, @@ -95,8 +97,8 @@ impl Wallet { wallet: bdk_wallet::PersistedWallet, wallet_persister: KVStoreWalletPersister, broadcaster: Arc, fee_estimator: Arc, chain_source: Arc, - payment_store: Arc, config: Arc, logger: Arc, - pending_payment_store: Arc, + payment_store: Arc, runtime: Arc, config: Arc, + logger: Arc, pending_payment_store: Arc, ) -> Self { let inner = Mutex::new(wallet); let persister = Mutex::new(wallet_persister); @@ -107,6 +109,7 @@ impl Wallet { fee_estimator, chain_source, payment_store, + runtime, config, logger, pending_payment_store, @@ -278,13 +281,15 @@ impl Wallet { confirmation_status, ); - self.payment_store.insert_or_update(payment.clone())?; + self.runtime.block_on(self.payment_store.insert_or_update(payment.clone()))?; if payment_status == PaymentStatus::Pending { let pending_payment = self.create_pending_payment_from_tx(payment, Vec::new()); - self.pending_payment_store.insert_or_update(pending_payment)?; + self.runtime.block_on( + self.pending_payment_store.insert_or_update(pending_payment), + )?; } }, WalletEvent::ChainTipChanged { new_tip, .. } => { @@ -310,8 +315,11 @@ impl Wallet { let payment_id = payment.details.id; if new_tip.height >= height + ANTI_REORG_DELAY - 1 { payment.details.status = PaymentStatus::Succeeded; - self.payment_store.insert_or_update(payment.details)?; - self.pending_payment_store.remove(&payment_id)?; + self.runtime.block_on( + self.payment_store.insert_or_update(payment.details), + )?; + self.runtime + .block_on(self.pending_payment_store.remove(&payment_id))?; } }, PaymentKind::Onchain { @@ -367,8 +375,9 @@ impl Wallet { ); let pending_payment = self.create_pending_payment_from_tx(payment.clone(), Vec::new()); - self.payment_store.insert_or_update(payment)?; - self.pending_payment_store.insert_or_update(pending_payment)?; + self.runtime.block_on(self.payment_store.insert_or_update(payment))?; + self.runtime + .block_on(self.pending_payment_store.insert_or_update(pending_payment))?; }, WalletEvent::TxReplaced { txid, conflicts, .. } => { let Some(payment_id) = self.find_payment_by_txid(txid) else { @@ -398,7 +407,9 @@ impl Wallet { let pending_payment_details = self.create_pending_payment_from_tx(payment, conflict_txids.clone()); - self.pending_payment_store.insert_or_update(pending_payment_details)?; + self.runtime.block_on( + self.pending_payment_store.insert_or_update(pending_payment_details), + )?; }, WalletEvent::TxDropped { txid, tx } => { let payment_id = self @@ -414,8 +425,9 @@ impl Wallet { ); let pending_payment = self.create_pending_payment_from_tx(payment.clone(), Vec::new()); - self.payment_store.insert_or_update(payment)?; - self.pending_payment_store.insert_or_update(pending_payment)?; + self.runtime.block_on(self.payment_store.insert_or_update(payment))?; + self.runtime + .block_on(self.pending_payment_store.insert_or_update(pending_payment))?; }, _ => { continue; @@ -1416,8 +1428,9 @@ impl Wallet { let pending_payment_store = self.create_pending_payment_from_tx(new_payment.clone(), Vec::new()); - self.pending_payment_store.insert_or_update(pending_payment_store)?; - self.payment_store.insert_or_update(new_payment)?; + self.runtime + .block_on(self.pending_payment_store.insert_or_update(pending_payment_store))?; + self.runtime.block_on(self.payment_store.insert_or_update(new_payment))?; log_info!(self.logger, "RBF successful: replaced {} with {}", txid, new_txid); From 0d48be8cf1f7cf63ba5929fe6f416ca456b88ad2 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 11:02:55 +0200 Subject: [PATCH 06/17] Move node metrics persistence onto async KV storage Persist node metric updates through async KVStore writes and await them from the chain, gossip, and scoring tasks. This removes the remaining blocking metrics writer while keeping the helper name stable. Co-Authored-By: HAL 9000 --- src/builder.rs | 6 +-- src/chain/bitcoind.rs | 15 +++++--- src/chain/electrum.rs | 87 ++++++++++++++++++++++++++----------------- src/chain/esplora.rs | 17 +++++---- src/chain/mod.rs | 12 +++--- src/io/utils.rs | 35 ++++++++--------- src/lib.rs | 39 ++++++++++++++++++- src/scoring.rs | 11 +++--- 8 files changed, 143 insertions(+), 79 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 779b2bd39..15f656600 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -85,7 +85,7 @@ use crate::types::{ }; use crate::wallet::persist::KVStoreWalletPersister; use crate::wallet::Wallet; -use crate::{Node, NodeMetrics}; +use crate::{Node, NodeMetrics, PersistedNodeMetrics}; const LSPS_HARDENED_CHILD_INDEX: u32 = 577; const PERSISTER_MAX_PENDING_UPDATES: u64 = 100; @@ -1415,10 +1415,10 @@ fn build_with_store_internal( // Initialize the status fields. let node_metrics = match node_metris_res { - Ok(metrics) => Arc::new(RwLock::new(metrics)), + Ok(metrics) => Arc::new(PersistedNodeMetrics::new(metrics)), Err(e) => { if e.kind() == std::io::ErrorKind::NotFound { - Arc::new(RwLock::new(NodeMetrics::default())) + Arc::new(PersistedNodeMetrics::new(NodeMetrics::default())) } else { log_error!(logger, "Failed to read node metrics from store: {}", e); return Err(BuildError::ReadFailed); diff --git a/src/chain/bitcoind.rs b/src/chain/bitcoind.rs index 2582f32f6..6bfa8ffd2 100644 --- a/src/chain/bitcoind.rs +++ b/src/chain/bitcoind.rs @@ -42,7 +42,7 @@ use crate::fee_estimator::{ use crate::io::utils::update_and_persist_node_metrics; use crate::logger::{log_bytes, log_debug, log_error, log_info, log_trace, LdkLogger, Logger}; use crate::types::{ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; -use crate::{Error, NodeMetrics}; +use crate::{Error, PersistedNodeMetrics}; const CHAIN_POLLING_INTERVAL_SECS: u64 = 2; const CHAIN_POLLING_TIMEOUT_SECS: u64 = 10; @@ -55,14 +55,14 @@ pub(super) struct BitcoindChainSource { kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, } impl BitcoindChainSource { pub(crate) fn new_rpc( rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String, fee_estimator: Arc, kv_store: Arc, config: Arc, - logger: Arc, node_metrics: Arc>, + logger: Arc, node_metrics: Arc, ) -> Self { let api_client = Arc::new(BitcoindClient::new_rpc( rpc_host.clone(), @@ -89,7 +89,7 @@ impl BitcoindChainSource { rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String, fee_estimator: Arc, kv_store: Arc, config: Arc, rest_client_config: BitcoindRestClientConfig, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, ) -> Self { let api_client = Arc::new(BitcoindClient::new_rest( rest_client_config.rest_host, @@ -204,6 +204,7 @@ impl BitcoindChainSource { m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt; }, ) + .await .unwrap_or_else(|e| { log_error!(self.logger, "Failed to persist node metrics: {}", e); }); @@ -451,7 +452,8 @@ impl BitcoindChainSource { update_and_persist_node_metrics(&self.node_metrics, &*self.kv_store, &*self.logger, |m| { m.latest_lightning_wallet_sync_timestamp = unix_time_secs_opt; m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt; - })?; + }) + .await?; Ok(()) } @@ -563,7 +565,8 @@ impl BitcoindChainSource { SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); update_and_persist_node_metrics(&self.node_metrics, &*self.kv_store, &*self.logger, |m| { m.latest_fee_rate_cache_update_timestamp = unix_time_secs_opt - })?; + }) + .await?; Ok(()) } diff --git a/src/chain/electrum.rs b/src/chain/electrum.rs index 54e7fff0c..ad0ef1b7b 100644 --- a/src/chain/electrum.rs +++ b/src/chain/electrum.rs @@ -34,7 +34,7 @@ use crate::io::utils::update_and_persist_node_metrics; use crate::logger::{log_bytes, log_debug, log_error, log_trace, LdkLogger, Logger}; use crate::runtime::Runtime; use crate::types::{ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; -use crate::NodeMetrics; +use crate::PersistedNodeMetrics; const BDK_ELECTRUM_CLIENT_BATCH_SIZE: usize = 5; const ELECTRUM_CLIENT_NUM_RETRIES: u8 = 3; @@ -49,14 +49,14 @@ pub(super) struct ElectrumChainSource { kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, } impl ElectrumChainSource { pub(super) fn new( server_url: String, sync_config: ElectrumSyncConfig, fee_estimator: Arc, kv_store: Arc, config: Arc, - logger: Arc, node_metrics: Arc>, + logger: Arc, node_metrics: Arc, ) -> Self { let electrum_runtime_status = RwLock::new(ElectrumRuntimeStatus::new()); let onchain_wallet_sync_status = Mutex::new(WalletSyncStatus::Completed); @@ -129,31 +129,6 @@ impl ElectrumChainSource { let incremental_sync = self.node_metrics.read().expect("lock").latest_onchain_wallet_sync_timestamp.is_some(); - let apply_wallet_update = - |update_res: Result, now: Instant| match update_res { - Ok(update) => match onchain_wallet.apply_update(update) { - Ok(()) => { - log_debug!( - self.logger, - "{} of on-chain wallet finished in {}ms.", - if incremental_sync { "Incremental sync" } else { "Sync" }, - now.elapsed().as_millis() - ); - let unix_time_secs_opt = - SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); - update_and_persist_node_metrics( - &self.node_metrics, - &*self.kv_store, - &*self.logger, - |m| m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt, - )?; - Ok(()) - }, - Err(e) => Err(e), - }, - Err(e) => Err(e), - }; - let cached_txs = onchain_wallet.get_cached_txs(); let res = if incremental_sync { @@ -162,20 +137,62 @@ impl ElectrumChainSource { .get_incremental_sync_wallet_update(incremental_sync_request, cached_txs); let now = Instant::now(); - let update_res = incremental_sync_fut.await.map(|u| u.into()); - apply_wallet_update(update_res, now) + let update_res: Result = incremental_sync_fut.await.map(|u| u.into()); + self.apply_onchain_wallet_update( + onchain_wallet.as_ref(), + incremental_sync, + update_res, + now, + ) + .await } else { let full_scan_request = onchain_wallet.get_full_scan_request(); let full_scan_fut = electrum_client.get_full_scan_wallet_update(full_scan_request, cached_txs); let now = Instant::now(); - let update_res = full_scan_fut.await.map(|u| u.into()); - apply_wallet_update(update_res, now) + let update_res: Result = full_scan_fut.await.map(|u| u.into()); + self.apply_onchain_wallet_update( + onchain_wallet.as_ref(), + incremental_sync, + update_res, + now, + ) + .await }; res } + async fn apply_onchain_wallet_update( + &self, onchain_wallet: &Wallet, incremental_sync: bool, + update_res: Result, now: Instant, + ) -> Result<(), Error> { + match update_res { + Ok(update) => match onchain_wallet.apply_update(update) { + Ok(()) => { + log_debug!( + self.logger, + "{} of on-chain wallet finished in {}ms.", + if incremental_sync { "Incremental sync" } else { "Sync" }, + now.elapsed().as_millis() + ); + let unix_time_secs_opt = + SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); + update_and_persist_node_metrics( + &self.node_metrics, + &*self.kv_store, + &*self.logger, + |m| m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt, + ) + .await?; + Ok(()) + }, + Err(e) => Err(e), + }, + Err(e) => Err(e), + } + } + pub(crate) async fn sync_lightning_wallet( &self, channel_manager: Arc, chain_monitor: Arc, output_sweeper: Arc, @@ -239,7 +256,8 @@ impl ElectrumChainSource { &*self.kv_store, &*self.logger, |m| m.latest_lightning_wallet_sync_timestamp = unix_time_secs_opt, - )?; + ) + .await?; } res @@ -270,7 +288,8 @@ impl ElectrumChainSource { SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); update_and_persist_node_metrics(&self.node_metrics, &*self.kv_store, &*self.logger, |m| { m.latest_fee_rate_cache_update_timestamp = unix_time_secs_opt - })?; + }) + .await?; Ok(()) } diff --git a/src/chain/esplora.rs b/src/chain/esplora.rs index 5825a0984..eb23a395d 100644 --- a/src/chain/esplora.rs +++ b/src/chain/esplora.rs @@ -6,7 +6,7 @@ // accordance with one or both of these licenses. use std::collections::HashMap; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use bdk_esplora::EsploraAsyncExt; @@ -25,7 +25,7 @@ use crate::fee_estimator::{ use crate::io::utils::update_and_persist_node_metrics; use crate::logger::{log_bytes, log_debug, log_error, log_trace, LdkLogger, Logger}; use crate::types::{ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; -use crate::{Error, NodeMetrics}; +use crate::{Error, PersistedNodeMetrics}; pub(super) struct EsploraChainSource { pub(super) sync_config: EsploraSyncConfig, @@ -37,14 +37,14 @@ pub(super) struct EsploraChainSource { kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, } impl EsploraChainSource { pub(crate) fn new( server_url: String, headers: HashMap, sync_config: EsploraSyncConfig, fee_estimator: Arc, kv_store: Arc, config: Arc, - logger: Arc, node_metrics: Arc>, + logger: Arc, node_metrics: Arc, ) -> Result { let mut client_builder = esplora_client::Builder::new(&server_url); client_builder = @@ -127,7 +127,8 @@ impl EsploraChainSource { &*self.kv_store, &*self.logger, |m| m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt, - )?; + ) + .await?; Ok(()) }, Err(e) => Err(e), @@ -265,7 +266,8 @@ impl EsploraChainSource { &*self.kv_store, &*self.logger, |m| m.latest_lightning_wallet_sync_timestamp = unix_time_secs_opt, - )?; + ) + .await?; Ok(()) }, Err(e) => { @@ -347,7 +349,8 @@ impl EsploraChainSource { SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); update_and_persist_node_metrics(&self.node_metrics, &*self.kv_store, &*self.logger, |m| { m.latest_fee_rate_cache_update_timestamp = unix_time_secs_opt - })?; + }) + .await?; Ok(()) } diff --git a/src/chain/mod.rs b/src/chain/mod.rs index cb8541be6..92c4bdb64 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -10,7 +10,7 @@ mod electrum; mod esplora; use std::collections::HashMap; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use bitcoin::{Script, Txid}; @@ -27,7 +27,7 @@ use crate::fee_estimator::OnchainFeeEstimator; use crate::logger::{log_debug, log_info, log_trace, LdkLogger, Logger}; use crate::runtime::Runtime; use crate::types::{Broadcaster, ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; -use crate::{Error, NodeMetrics}; +use crate::{Error, PersistedNodeMetrics}; pub(crate) enum WalletSyncStatus { Completed, @@ -100,7 +100,7 @@ impl ChainSource { server_url: String, headers: HashMap, sync_config: EsploraSyncConfig, fee_estimator: Arc, tx_broadcaster: Arc, kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, ) -> Result<(Self, Option), ()> { let esplora_chain_source = EsploraChainSource::new( server_url, @@ -121,7 +121,7 @@ impl ChainSource { server_url: String, sync_config: ElectrumSyncConfig, fee_estimator: Arc, tx_broadcaster: Arc, kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, ) -> (Self, Option) { let electrum_chain_source = ElectrumChainSource::new( server_url, @@ -141,7 +141,7 @@ impl ChainSource { rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String, fee_estimator: Arc, tx_broadcaster: Arc, kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, ) -> (Self, Option) { let bitcoind_chain_source = BitcoindChainSource::new_rpc( rpc_host, @@ -164,7 +164,7 @@ impl ChainSource { rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String, fee_estimator: Arc, tx_broadcaster: Arc, kv_store: Arc, config: Arc, rest_client_config: BitcoindRestClientConfig, - logger: Arc, node_metrics: Arc>, + logger: Arc, node_metrics: Arc, ) -> (Self, Option) { let bitcoind_chain_source = BitcoindChainSource::new_rest( rpc_host, diff --git a/src/io/utils.rs b/src/io/utils.rs index 11d1255b8..b13e0cf5d 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -11,7 +11,7 @@ use std::ops::Deref; #[cfg(unix)] use std::os::unix::fs::OpenOptionsExt; use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use bdk_chain::indexer::keychain_txout::ChangeSet as BdkIndexerChangeSet; use bdk_chain::local_chain::ChangeSet as BdkLocalChainChangeSet; @@ -26,12 +26,12 @@ use lightning::routing::scoring::{ ChannelLiquidities, ProbabilisticScorer, ProbabilisticScoringDecayParameters, }; use lightning::util::persist::{ - migrate_kv_store_data, KVStore, KVStoreSync, KVSTORE_NAMESPACE_KEY_ALPHABET, - KVSTORE_NAMESPACE_KEY_MAX_LEN, NETWORK_GRAPH_PERSISTENCE_KEY, - NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, - OUTPUT_SWEEPER_PERSISTENCE_KEY, OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE, - OUTPUT_SWEEPER_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_KEY, - SCORER_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + migrate_kv_store_data, KVStore, KVSTORE_NAMESPACE_KEY_ALPHABET, KVSTORE_NAMESPACE_KEY_MAX_LEN, + NETWORK_GRAPH_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, OUTPUT_SWEEPER_PERSISTENCE_KEY, + OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE, OUTPUT_SWEEPER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, }; use lightning::util::ser::{Readable, ReadableArgs, Writeable}; use lightning_persister::fs_store::v1::FilesystemStore; @@ -49,7 +49,7 @@ use crate::logger::{log_error, LdkLogger, Logger}; use crate::peer_store::PeerStore; use crate::types::{Broadcaster, DynStore, KeysManager, Sweeper}; use crate::wallet::ser::{ChangeSetDeserWrapper, ChangeSetSerWrapper}; -use crate::{BuildError, Error, EventQueue, NodeMetrics}; +use crate::{BuildError, Error, EventQueue, NodeMetrics, PersistedNodeMetrics}; pub const EXTERNAL_PATHFINDING_SCORES_CACHE_KEY: &str = "external_pathfinding_scores_cache"; @@ -336,26 +336,27 @@ where } /// Take a write lock on `node_metrics`, apply `update`, and persist the result to `kv_store`. -/// -/// The write lock is held across the KV-store write, preserving the invariant that readers only -/// observe the mutation once it has been durably persisted (or the persist has failed). -pub(crate) fn update_and_persist_node_metrics( - node_metrics: &RwLock, kv_store: &DynStore, logger: L, +pub(crate) async fn update_and_persist_node_metrics( + node_metrics: &PersistedNodeMetrics, kv_store: &DynStore, logger: L, update: impl FnOnce(&mut NodeMetrics), ) -> Result<(), Error> where L::Target: LdkLogger, { - let mut locked_node_metrics = node_metrics.write().expect("lock"); - update(&mut *locked_node_metrics); - let data = locked_node_metrics.encode(); - KVStoreSync::write( + let _guard = node_metrics.lock_mutation().await; + let data = { + let mut locked_node_metrics = node_metrics.write().expect("lock"); + update(&mut *locked_node_metrics); + locked_node_metrics.encode() + }; + KVStore::write( &*kv_store, NODE_METRICS_PRIMARY_NAMESPACE, NODE_METRICS_SECONDARY_NAMESPACE, NODE_METRICS_KEY, data, ) + .await .map_err(|e| { log_error!( logger, diff --git a/src/lib.rs b/src/lib.rs index 792f7c9f8..2ee74ecc5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -243,7 +243,7 @@ pub struct Node { payment_store: Arc, lnurl_auth: Arc, is_running: Arc>, - node_metrics: Arc>, + node_metrics: Arc, om_mailbox: Option>, async_payments_role: Option, hrn_resolver: HRNResolver, @@ -556,6 +556,7 @@ impl Node { Arc::clone(&bcast_logger), |m| m.latest_node_announcement_broadcast_timestamp = unix_time_secs_opt, ) + .await .unwrap_or_else(|e| { log_error!(bcast_logger, "Persistence failed: {}", e); }); @@ -2185,6 +2186,42 @@ impl Default for NodeMetrics { } } +pub(crate) struct PersistedNodeMetrics { + metrics: RwLock, + mutation_lock: tokio::sync::Mutex<()>, +} + +impl PersistedNodeMetrics { + pub(crate) fn new(metrics: NodeMetrics) -> Self { + Self { metrics: RwLock::new(metrics), mutation_lock: tokio::sync::Mutex::new(()) } + } + + /// Returns the current in-memory metrics. + /// + /// The async mutation lock serializes persistence updates, but this synchronous reader cannot + /// wait on it. Until metrics reads are async, callers may observe metrics changes that are + /// still being persisted. + pub(crate) fn read( + &self, + ) -> std::sync::LockResult> { + self.metrics.read() + } + + /// Returns the in-memory metrics write lock. + /// + /// Persistence updates should go through `update_and_persist_node_metrics` so writers are + /// serialized by the async mutation lock. + pub(crate) fn write( + &self, + ) -> std::sync::LockResult> { + self.metrics.write() + } + + pub(crate) async fn lock_mutation(&self) -> tokio::sync::MutexGuard<'_, ()> { + self.mutation_lock.lock().await + } +} + impl_writeable_tlv_based!(NodeMetrics, { (0, latest_lightning_wallet_sync_timestamp, option), (1, latest_pathfinding_scores_sync_timestamp, option), diff --git a/src/scoring.rs b/src/scoring.rs index 8abc4eab6..401d9c3f1 100644 --- a/src/scoring.rs +++ b/src/scoring.rs @@ -1,4 +1,4 @@ -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex}; use std::time::SystemTime; use lightning::routing::scoring::ChannelLiquidities; @@ -13,12 +13,12 @@ use crate::io::utils::write_external_pathfinding_scores_to_cache; use crate::logger::LdkLogger; use crate::runtime::Runtime; use crate::types::DynStore; -use crate::{update_and_persist_node_metrics, Logger, NodeMetrics, Scorer}; +use crate::{update_and_persist_node_metrics, Logger, PersistedNodeMetrics, Scorer}; /// Start a background task that periodically downloads scores via an external url and merges them into the local /// pathfinding scores. pub fn setup_background_pathfinding_scores_sync( - url: String, scorer: Arc>, node_metrics: Arc>, + url: String, scorer: Arc>, node_metrics: Arc, kv_store: Arc, logger: Arc, runtime: Arc, mut stop_receiver: tokio::sync::watch::Receiver<()>, ) { @@ -51,7 +51,7 @@ pub fn setup_background_pathfinding_scores_sync( } async fn sync_external_scores( - logger: &Logger, scorer: &Mutex, node_metrics: &RwLock, + logger: &Logger, scorer: &Mutex, node_metrics: &PersistedNodeMetrics, kv_store: Arc, url: &String, ) -> () { let request = bitreq::get(url) @@ -86,9 +86,10 @@ async fn sync_external_scores( .duration_since(SystemTime::UNIX_EPOCH) .expect("system time must be after Unix epoch"); scorer.lock().expect("lock").merge(liquidities, duration_since_epoch); - update_and_persist_node_metrics(&node_metrics, &*kv_store, logger, |m| { + update_and_persist_node_metrics(node_metrics, &*kv_store, logger, |m| { m.latest_pathfinding_scores_sync_timestamp = Some(duration_since_epoch.as_secs()); }) + .await .unwrap_or_else(|e| { log_error!(logger, "Persisting node metrics failed: {}", e); }); From 5ac2ed1787b47d256eb78ad5616939ef377c8264 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 11:07:30 +0200 Subject: [PATCH 07/17] Use BDK's async wallet persister Persist the on-chain wallet through BDK's AsyncWalletPersister so wallet state writes use the async KVStore path. Existing synchronous wallet APIs keep bridging through the node runtime until their callers are made async. Co-Authored-By: HAL 9000 --- src/builder.rs | 33 ++++++++------ src/wallet/mod.rs | 46 +++++++++++-------- src/wallet/persist.rs | 103 ++++++++++++++++++++++-------------------- 3 files changed, 100 insertions(+), 82 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 15f656600..4107ca686 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -1537,17 +1537,18 @@ fn build_with_store_internal( let descriptor = Bip84(xprv, KeychainKind::External); let change_descriptor = Bip84(xprv, KeychainKind::Internal); - let mut wallet_persister = KVStoreWalletPersister::new( - Arc::clone(&kv_store), - Arc::clone(&runtime), - Arc::clone(&logger), - ); - let wallet_opt = BdkWallet::load() - .descriptor(KeychainKind::External, Some(descriptor.clone())) - .descriptor(KeychainKind::Internal, Some(change_descriptor.clone())) - .extract_keys() - .check_network(config.network) - .load_wallet(&mut wallet_persister) + let mut wallet_persister = + KVStoreWalletPersister::new(Arc::clone(&kv_store), Arc::clone(&logger)); + let wallet_opt = runtime + .block_on(async { + BdkWallet::load() + .descriptor(KeychainKind::External, Some(descriptor.clone())) + .descriptor(KeychainKind::Internal, Some(change_descriptor.clone())) + .extract_keys() + .check_network(config.network) + .load_wallet_async(&mut wallet_persister) + .await + }) .map_err(|e| match e { bdk_wallet::LoadWithPersistError::InvalidChangeSet( bdk_wallet::LoadError::Mismatch(bdk_wallet::LoadMismatch::Network { @@ -1571,9 +1572,13 @@ fn build_with_store_internal( let bdk_wallet = match wallet_opt { Some(wallet) => wallet, None => { - let mut wallet = BdkWallet::create(descriptor, change_descriptor) - .network(config.network) - .create_wallet(&mut wallet_persister) + let mut wallet = runtime + .block_on(async { + BdkWallet::create(descriptor, change_descriptor) + .network(config.network) + .create_wallet_async(&mut wallet_persister) + .await + }) .map_err(|e| { log_error!(logger, "Failed to set up wallet: {}", e); BuildError::WalletSetupFailed diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index cdae34a0a..76f2aa9ce 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -161,10 +161,12 @@ impl Wallet { })?; let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { - log_error!(self.logger, "Failed to persist wallet: {}", e); - Error::PersistenceFailed - })?; + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err( + |e| { + log_error!(self.logger, "Failed to persist wallet: {}", e); + Error::PersistenceFailed + }, + )?; Ok(()) }, @@ -214,7 +216,7 @@ impl Wallet { })?; let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -474,7 +476,7 @@ impl Wallet { } let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -492,7 +494,7 @@ impl Wallet { let mut locked_persister = self.persister.lock().expect("lock"); let address_info = locked_wallet.reveal_next_address(KeychainKind::External); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -504,7 +506,7 @@ impl Wallet { let mut locked_persister = self.persister.lock().expect("lock"); let address_info = locked_wallet.next_unused_address(KeychainKind::Internal); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -516,7 +518,7 @@ impl Wallet { let mut locked_persister = self.persister.lock().expect("lock"); locked_wallet.cancel_tx(tx); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -854,10 +856,12 @@ impl Wallet { } let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { - log_error!(self.logger, "Failed to persist wallet: {}", e); - Error::PersistenceFailed - })?; + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err( + |e| { + log_error!(self.logger, "Failed to persist wallet: {}", e); + Error::PersistenceFailed + }, + )?; psbt.extract_tx().map_err(|e| { log_error!(self.logger, "Failed to extract transaction: {}", e); @@ -972,10 +976,12 @@ impl Wallet { .find(|txout| must_pay_to.iter().all(|output| output != txout)); if change_output.is_some() { - locked_wallet.persist(&mut locked_persister).map_err(|e| { - log_error!(self.logger, "Failed to persist wallet: {}", e); - () - })?; + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err( + |e| { + log_error!(self.logger, "Failed to persist wallet: {}", e); + () + }, + )?; } Ok(CoinSelection { confirmed_utxos, change_output }) @@ -1080,7 +1086,7 @@ impl Wallet { let mut locked_persister = self.persister.lock().expect("lock"); let address_info = locked_wallet.next_unused_address(KeychainKind::Internal); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); () })?; @@ -1399,7 +1405,7 @@ impl Wallet { } let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet after fee bump of {}: {}", txid, e); Error::PersistenceFailed })?; @@ -1501,7 +1507,7 @@ impl Listen for Wallet { }; let mut locked_persister = self.persister.lock().expect("lock"); - match locked_wallet.persist(&mut locked_persister) { + match self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)) { Ok(_) => (), Err(e) => { log_error!(self.logger, "Failed to persist on-chain wallet: {}", e); diff --git a/src/wallet/persist.rs b/src/wallet/persist.rs index cc5005780..364dc4b47 100644 --- a/src/wallet/persist.rs +++ b/src/wallet/persist.rs @@ -5,10 +5,12 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use bdk_chain::Merge; -use bdk_wallet::{ChangeSet, WalletPersister}; +use bdk_wallet::{AsyncWalletPersister, ChangeSet}; use crate::io::utils::{ read_bdk_wallet_change_set, write_bdk_wallet_change_descriptor, write_bdk_wallet_descriptor, @@ -16,34 +18,26 @@ use crate::io::utils::{ write_bdk_wallet_tx_graph, }; use crate::logger::{log_error, LdkLogger, Logger}; -use crate::runtime::Runtime; use crate::types::DynStore; pub(crate) struct KVStoreWalletPersister { latest_change_set: Option, kv_store: Arc, - runtime: Arc, logger: Arc, } impl KVStoreWalletPersister { - pub(crate) fn new(kv_store: Arc, runtime: Arc, logger: Arc) -> Self { - Self { latest_change_set: None, kv_store, runtime, logger } + pub(crate) fn new(kv_store: Arc, logger: Arc) -> Self { + Self { latest_change_set: None, kv_store, logger } } -} - -impl WalletPersister for KVStoreWalletPersister { - type Error = std::io::Error; - fn initialize(persister: &mut Self) -> Result { + async fn initialize_inner(&mut self) -> Result { // Return immediately if we have already been initialized. - if let Some(latest_change_set) = persister.latest_change_set.as_ref() { + if let Some(latest_change_set) = self.latest_change_set.as_ref() { return Ok(latest_change_set.clone()); } - let change_set_opt = persister - .runtime - .block_on(read_bdk_wallet_change_set(&*persister.kv_store, &*persister.logger))?; + let change_set_opt = read_bdk_wallet_change_set(&*self.kv_store, &*self.logger).await?; let change_set = match change_set_opt { Some(persisted_change_set) => persisted_change_set, @@ -54,18 +48,21 @@ impl WalletPersister for KVStoreWalletPersister { ChangeSet::default() }, }; - persister.latest_change_set = Some(change_set.clone()); + self.latest_change_set = Some(change_set.clone()); Ok(change_set) } - fn persist(persister: &mut Self, change_set: &ChangeSet) -> Result<(), Self::Error> { + async fn persist_inner(&mut self, change_set: &ChangeSet) -> Result<(), std::io::Error> { if change_set.is_empty() { return Ok(()); } + let kv_store = Arc::clone(&self.kv_store); + let logger = Arc::clone(&self.logger); + // We're allowed to fail here if we're not initialized, BDK docs state: "This method can fail if the // persister is not initialized." - let latest_change_set = persister.latest_change_set.as_mut().ok_or_else(|| { + let latest_change_set = self.latest_change_set.as_mut().ok_or_else(|| { std::io::Error::new( std::io::ErrorKind::Other, "Wallet must be initialized before calling persist", @@ -80,7 +77,7 @@ impl WalletPersister for KVStoreWalletPersister { { debug_assert!(false, "Wallet descriptor must never change"); log_error!( - persister.logger, + logger, "Wallet change set doesn't match persisted descriptor. This should never happen." ); return Err(std::io::Error::new( @@ -89,11 +86,7 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.descriptor = Some(descriptor.clone()); - persister.runtime.block_on(write_bdk_wallet_descriptor( - &descriptor, - &*persister.kv_store, - &*persister.logger, - ))?; + write_bdk_wallet_descriptor(&descriptor, &*kv_store, Arc::clone(&logger)).await?; } } @@ -103,7 +96,7 @@ impl WalletPersister for KVStoreWalletPersister { { debug_assert!(false, "Wallet change_descriptor must never change"); log_error!( - persister.logger, + logger, "Wallet change set doesn't match persisted change_descriptor. This should never happen." ); return Err(std::io::Error::new( @@ -112,11 +105,12 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.change_descriptor = Some(change_descriptor.clone()); - persister.runtime.block_on(write_bdk_wallet_change_descriptor( + write_bdk_wallet_change_descriptor( &change_descriptor, - &*persister.kv_store, - &*persister.logger, - ))?; + &*kv_store, + Arc::clone(&logger), + ) + .await?; } } @@ -124,7 +118,7 @@ impl WalletPersister for KVStoreWalletPersister { if latest_change_set.network.is_some() && latest_change_set.network != Some(network) { debug_assert!(false, "Wallet network must never change"); log_error!( - persister.logger, + logger, "Wallet change set doesn't match persisted network. This should never happen." ); return Err(std::io::Error::new( @@ -133,11 +127,7 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.network = Some(network); - persister.runtime.block_on(write_bdk_wallet_network( - &network, - &*persister.kv_store, - &*persister.logger, - ))?; + write_bdk_wallet_network(&network, &*kv_store, Arc::clone(&logger)).await?; } } @@ -157,31 +147,48 @@ impl WalletPersister for KVStoreWalletPersister { // particular order. if !change_set.indexer.is_empty() { latest_change_set.indexer.merge(change_set.indexer.clone()); - persister.runtime.block_on(write_bdk_wallet_indexer( - &latest_change_set.indexer, - &*persister.kv_store, - Arc::clone(&persister.logger), - ))?; + write_bdk_wallet_indexer(&latest_change_set.indexer, &*kv_store, Arc::clone(&logger)) + .await?; } if !change_set.tx_graph.is_empty() { latest_change_set.tx_graph.merge(change_set.tx_graph.clone()); - persister.runtime.block_on(write_bdk_wallet_tx_graph( - &latest_change_set.tx_graph, - &*persister.kv_store, - Arc::clone(&persister.logger), - ))?; + write_bdk_wallet_tx_graph(&latest_change_set.tx_graph, &*kv_store, Arc::clone(&logger)) + .await?; } if !change_set.local_chain.is_empty() { latest_change_set.local_chain.merge(change_set.local_chain.clone()); - persister.runtime.block_on(write_bdk_wallet_local_chain( + write_bdk_wallet_local_chain( &latest_change_set.local_chain, - &*persister.kv_store, - Arc::clone(&persister.logger), - ))?; + &*kv_store, + Arc::clone(&logger), + ) + .await?; } Ok(()) } } + +impl AsyncWalletPersister for KVStoreWalletPersister { + type Error = std::io::Error; + + fn initialize<'a>( + persister: &'a mut Self, + ) -> Pin> + Send + 'a>> + where + Self: 'a, + { + Box::pin(persister.initialize_inner()) + } + + fn persist<'a>( + persister: &'a mut Self, change_set: &'a ChangeSet, + ) -> Pin> + Send + 'a>> + where + Self: 'a, + { + Box::pin(persister.persist_inner(change_set)) + } +} From a72905c815c906cd750a7e319aeb2a30fd1e0930 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 11:09:53 +0200 Subject: [PATCH 08/17] Use async KVStore migration for filesystem stores Open filesystem stores through the async LDK migration helper so v1-to-v2 store migration no longer depends on the blocking KVStoreSync migration path. Co-Authored-By: HAL 9000 --- src/builder.rs | 5 +++-- src/io/utils.rs | 51 +++++++++++++++++++++++++------------------------ 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 4107ca686..9fd7f832c 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -697,11 +697,12 @@ impl NodeBuilder { /// [`FilesystemStoreV2`]: lightning_persister::fs_store::v2::FilesystemStoreV2 pub fn build_with_fs_store(&self, node_entropy: NodeEntropy) -> Result { let logger = setup_logger(&self.log_writer_config, &self.config)?; + let runtime = self.setup_runtime(&logger)?; let mut storage_dir_path: PathBuf = self.config.storage_dir_path.clone().into(); storage_dir_path.push("fs_store"); - let kv_store = open_or_migrate_fs_store(storage_dir_path)?; - self.build_with_store_and_logger(node_entropy, kv_store, logger) + let kv_store = runtime.block_on(open_or_migrate_fs_store(storage_dir_path))?; + self.build_with_store_runtime_and_logger(node_entropy, kv_store, runtime, logger) } /// Builds a [`Node`] instance with a [VSS] backend and according to the options diff --git a/src/io/utils.rs b/src/io/utils.rs index b13e0cf5d..cf3905332 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -26,12 +26,12 @@ use lightning::routing::scoring::{ ChannelLiquidities, ProbabilisticScorer, ProbabilisticScoringDecayParameters, }; use lightning::util::persist::{ - migrate_kv_store_data, KVStore, KVSTORE_NAMESPACE_KEY_ALPHABET, KVSTORE_NAMESPACE_KEY_MAX_LEN, - NETWORK_GRAPH_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, - NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, OUTPUT_SWEEPER_PERSISTENCE_KEY, - OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE, OUTPUT_SWEEPER_PERSISTENCE_SECONDARY_NAMESPACE, - SCORER_PERSISTENCE_KEY, SCORER_PERSISTENCE_PRIMARY_NAMESPACE, - SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + migrate_kv_store_data_async, KVStore, KVSTORE_NAMESPACE_KEY_ALPHABET, + KVSTORE_NAMESPACE_KEY_MAX_LEN, NETWORK_GRAPH_PERSISTENCE_KEY, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + OUTPUT_SWEEPER_PERSISTENCE_KEY, OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE, + OUTPUT_SWEEPER_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_KEY, + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE, }; use lightning::util::ser::{Readable, ReadableArgs, Writeable}; use lightning_persister::fs_store::v1::FilesystemStore; @@ -632,7 +632,7 @@ pub(crate) async fn read_bdk_wallet_change_set( /// If the directory contains v1 data (files at the top level), the data is migrated to v2 format /// in a temporary directory, the original is renamed to `fs_store_v1_backup`, and the migrated /// directory is moved into place. -pub(crate) fn open_or_migrate_fs_store( +pub(crate) async fn open_or_migrate_fs_store( storage_dir_path: PathBuf, ) -> Result { let parent_dir = storage_dir_path.parent().ok_or(BuildError::StoragePathAccessFailed)?; @@ -647,14 +647,15 @@ pub(crate) fn open_or_migrate_fs_store( Ok(store) => Ok(store), Err(FilesystemStoreV2Error::V1DataDetected(_)) => { // The directory contains v1 data, migrate to v2. - let mut v1_store = FilesystemStore::new(storage_dir_path.clone()); + let v1_store = FilesystemStore::new(storage_dir_path.clone()); let v2_dir = fs_store_sibling_path(&storage_dir_path, "fs_store_v2_migrating"); fs::create_dir_all(v2_dir.clone()).map_err(|_| BuildError::StoragePathAccessFailed)?; - let mut v2_store = FilesystemStoreV2::new(v2_dir.clone()) + let v2_store = FilesystemStoreV2::new(v2_dir.clone()) .map_err(|_| BuildError::KVStoreSetupFailed)?; - migrate_kv_store_data(&mut v1_store, &mut v2_store) + migrate_kv_store_data_async(&v1_store, &v2_store) + .await .map_err(|_| BuildError::KVStoreSetupFailed)?; // Swap directories: rename v1 out of the way, move v2 into place. @@ -739,15 +740,15 @@ mod tests { assert_eq!(expected_seed_bytes, read_seed_bytes); } - #[test] - fn fs_store_migration_recovers_before_v1_backup_rename() { + #[tokio::test] + async fn fs_store_migration_recovers_before_v1_backup_rename() { let fs_store_path = fs_store_path(); let mut v1_store = write_v1_test_data(&fs_store_path); let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); let mut v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); migrate_kv_store_data(&mut v1_store, &mut v2_store).unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( KVStoreSync::read( &migrated_store, @@ -762,8 +763,8 @@ mod tests { assert!(!v2_migrating_path.exists()); } - #[test] - fn fs_store_migration_recovers_after_v1_backup_rename() { + #[tokio::test] + async fn fs_store_migration_recovers_after_v1_backup_rename() { let fs_store_path = fs_store_path(); let mut v1_store = write_v1_test_data(&fs_store_path); let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); @@ -773,7 +774,7 @@ mod tests { let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, backup_path).unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( KVStoreSync::read( &migrated_store, @@ -788,8 +789,8 @@ mod tests { assert!(!v2_migrating_path.exists()); } - #[test] - fn fs_store_migration_recovers_after_v2_rename() { + #[tokio::test] + async fn fs_store_migration_recovers_after_v2_rename() { let fs_store_path = fs_store_path(); let mut v1_store = write_v1_test_data(&fs_store_path); let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); @@ -800,7 +801,7 @@ mod tests { fs::rename(&fs_store_path, &backup_path).unwrap(); fs::rename(&v2_migrating_path, &fs_store_path).unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( KVStoreSync::read( &migrated_store, @@ -816,15 +817,15 @@ mod tests { assert!(!v2_migrating_path.exists()); } - #[test] - fn fs_store_migration_recovers_backup_without_migrating_dir() { + #[tokio::test] + async fn fs_store_migration_recovers_backup_without_migrating_dir() { let fs_store_path = fs_store_path(); write_v1_test_data(&fs_store_path); let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, backup_path).unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( KVStoreSync::read( &migrated_store, @@ -839,8 +840,8 @@ mod tests { assert!(!sibling_path(&fs_store_path, "fs_store_v1_backup").exists()); } - #[test] - fn fs_store_migration_recovers_unexpected_migrating_dir_without_backup() { + #[tokio::test] + async fn fs_store_migration_recovers_unexpected_migrating_dir_without_backup() { let fs_store_path = fs_store_path(); let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); @@ -853,7 +854,7 @@ mod tests { ) .unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( KVStoreSync::read( &migrated_store, From 98170fe151dae3504fcb4104869532900c910551 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 11:57:31 +0200 Subject: [PATCH 09/17] Move test InMemoryStore into shared module Move the existing in-memory test store into a shared module without changing its behavior. This lets integration tests reuse it while keeping the later async TestSyncStore change separate. Co-Authored-By: HAL 9000 --- src/io/in_memory_store.rs | 225 ++++++++++++++++++++++++++++++++++++++ src/io/test_utils.rs | 220 +------------------------------------ 2 files changed, 230 insertions(+), 215 deletions(-) create mode 100644 src/io/in_memory_store.rs diff --git a/src/io/in_memory_store.rs b/src/io/in_memory_store.rs new file mode 100644 index 000000000..8d0d24a4e --- /dev/null +++ b/src/io/in_memory_store.rs @@ -0,0 +1,225 @@ +// This file is Copyright its original authors, visible in version control history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license , at your option. You may not use this file except in +// accordance with one or both of these licenses. + +use std::collections::{hash_map, HashMap}; +use std::future::Future; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; + +use lightning::io; +use lightning::util::persist::{ + KVStore, KVStoreSync, PageToken, PaginatedKVStore, PaginatedKVStoreSync, PaginatedListResponse, +}; + +const IN_MEMORY_PAGE_SIZE: usize = 50; + +pub struct InMemoryStore { + persisted_bytes: Mutex>>>, + creation_counter: AtomicU64, + creation_times: Mutex>>, +} + +impl InMemoryStore { + pub fn new() -> Self { + let persisted_bytes = Mutex::new(HashMap::new()); + let creation_counter = AtomicU64::new(1); + let creation_times = Mutex::new(HashMap::new()); + Self { persisted_bytes, creation_counter, creation_times } + } + + fn read_internal( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result> { + let persisted_lock = self.persisted_bytes.lock().unwrap(); + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + + if let Some(outer_ref) = persisted_lock.get(&prefixed) { + if let Some(inner_ref) = outer_ref.get(key) { + let bytes = inner_ref.clone(); + Ok(bytes) + } else { + Err(io::Error::new(io::ErrorKind::NotFound, "Key not found")) + } + } else { + Err(io::Error::new(io::ErrorKind::NotFound, "Namespace not found")) + } + } + + fn write_internal( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> io::Result<()> { + let mut persisted_lock = self.persisted_bytes.lock().unwrap(); + + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + let outer_e = persisted_lock.entry(prefixed.clone()).or_insert(HashMap::new()); + outer_e.insert(key.to_string(), buf); + + // Only assign creation time on first write (not on update) + let mut ct_lock = self.creation_times.lock().unwrap(); + let ct_ns = ct_lock.entry(prefixed).or_insert(HashMap::new()); + ct_ns + .entry(key.to_string()) + .or_insert_with(|| self.creation_counter.fetch_add(1, Ordering::Relaxed)); + + Ok(()) + } + + fn remove_internal( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, _lazy: bool, + ) -> io::Result<()> { + let mut persisted_lock = self.persisted_bytes.lock().unwrap(); + + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + if let Some(outer_ref) = persisted_lock.get_mut(&prefixed) { + outer_ref.remove(&key.to_string()); + } + + // Remove creation time entry + let mut ct_lock = self.creation_times.lock().unwrap(); + if let Some(ct_ns) = ct_lock.get_mut(&prefixed) { + ct_ns.remove(key); + } + + Ok(()) + } + + fn list_internal( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result> { + let mut persisted_lock = self.persisted_bytes.lock().unwrap(); + + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + match persisted_lock.entry(prefixed) { + hash_map::Entry::Occupied(e) => Ok(e.get().keys().cloned().collect()), + hash_map::Entry::Vacant(_) => Ok(Vec::new()), + } + } +} + +impl KVStore for InMemoryStore { + fn read( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> impl Future, io::Error>> + 'static + Send { + let res = self.read_internal(&primary_namespace, &secondary_namespace, &key); + async move { res } + } + fn write( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> impl Future> + 'static + Send { + let res = self.write_internal(&primary_namespace, &secondary_namespace, &key, buf); + async move { res } + } + fn remove( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> impl Future> + 'static + Send { + let res = self.remove_internal(&primary_namespace, &secondary_namespace, &key, lazy); + async move { res } + } + fn list( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> impl Future, io::Error>> + 'static + Send { + let res = self.list_internal(primary_namespace, secondary_namespace); + async move { res } + } +} + +impl KVStoreSync for InMemoryStore { + fn read( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result> { + self.read_internal(primary_namespace, secondary_namespace, key) + } + + fn write( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> io::Result<()> { + self.write_internal(primary_namespace, secondary_namespace, key, buf) + } + + fn remove( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> io::Result<()> { + self.remove_internal(primary_namespace, secondary_namespace, key, lazy) + } + + fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { + self.list_internal(primary_namespace, secondary_namespace) + } +} + +impl InMemoryStore { + fn list_paginated_internal( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> io::Result { + let ct_lock = self.creation_times.lock().unwrap(); + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + + let ct_ns = match ct_lock.get(&prefixed) { + Some(m) => m, + None => { + return Ok(PaginatedListResponse { keys: Vec::new(), next_page_token: None }); + }, + }; + + // Build list of (key, sort_order) sorted by sort_order DESC (newest first). + let mut entries: Vec<(&String, &u64)> = ct_ns.iter().collect(); + entries.sort_by(|a, b| b.1.cmp(a.1)); + + // Apply page token filter + let start_idx = if let Some(ref token) = page_token { + let token_sort_order: u64 = token + .as_str() + .parse() + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Invalid page token"))?; + + entries + .iter() + .position(|(_, sort_order)| **sort_order < token_sort_order) + .unwrap_or(entries.len()) + } else { + 0 + }; + + // Fetch one extra entry beyond page size to determine whether a next page exists. + let mut page: Vec<(&String, &u64)> = + entries[start_idx..].iter().take(IN_MEMORY_PAGE_SIZE + 1).cloned().collect(); + + let has_more = page.len() > IN_MEMORY_PAGE_SIZE; + page.truncate(IN_MEMORY_PAGE_SIZE); + + let next_page_token = if has_more { + let (_, last_sort_order) = page.last().unwrap(); + Some(PageToken::new(last_sort_order.to_string())) + } else { + None + }; + + let page: Vec = page.into_iter().map(|(k, _)| k.clone()).collect(); + + Ok(PaginatedListResponse { keys: page, next_page_token }) + } +} + +impl PaginatedKVStoreSync for InMemoryStore { + fn list_paginated( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> io::Result { + self.list_paginated_internal(primary_namespace, secondary_namespace, page_token) + } +} + +impl PaginatedKVStore for InMemoryStore { + fn list_paginated( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> impl Future> + 'static + Send { + let res = self.list_paginated_internal(primary_namespace, secondary_namespace, page_token); + async move { res } + } +} + +unsafe impl Sync for InMemoryStore {} +unsafe impl Send for InMemoryStore {} diff --git a/src/io/test_utils.rs b/src/io/test_utils.rs index eed8c3e2d..0b16e525e 100644 --- a/src/io/test_utils.rs +++ b/src/io/test_utils.rs @@ -5,15 +5,10 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. -use std::collections::{hash_map, HashMap}; -use std::future::Future; use std::panic::RefUnwindSafe; use std::path::PathBuf; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Mutex; use lightning::events::ClosureReason; -use lightning::io; use lightning::ln::functional_test_utils::{ check_added_monitors, check_closed_broadcast, check_closed_event, connect_block, create_announced_chan_between_nodes, create_chanmon_cfgs, create_dummy_block, create_network, @@ -21,13 +16,15 @@ use lightning::ln::functional_test_utils::{ TestChanMonCfg, }; use lightning::util::persist::{ - KVStore, KVStoreSync, MonitorUpdatingPersister, PageToken, PaginatedKVStore, - PaginatedKVStoreSync, PaginatedListResponse, KVSTORE_NAMESPACE_KEY_MAX_LEN, + KVStoreSync, MonitorUpdatingPersister, KVSTORE_NAMESPACE_KEY_MAX_LEN, }; use lightning::util::test_utils; use rand::distr::Alphanumeric; use rand::{rng, Rng}; +#[path = "in_memory_store.rs"] +mod in_memory_store; + type TestMonitorUpdatePersister<'a, K> = MonitorUpdatingPersister< &'a K, &'a test_utils::TestLogger, @@ -39,214 +36,7 @@ type TestMonitorUpdatePersister<'a, K> = MonitorUpdatingPersister< const EXPECTED_UPDATES_PER_PAYMENT: u64 = 5; -const IN_MEMORY_PAGE_SIZE: usize = 50; - -pub struct InMemoryStore { - persisted_bytes: Mutex>>>, - creation_counter: AtomicU64, - creation_times: Mutex>>, -} - -impl InMemoryStore { - pub fn new() -> Self { - let persisted_bytes = Mutex::new(HashMap::new()); - let creation_counter = AtomicU64::new(1); - let creation_times = Mutex::new(HashMap::new()); - Self { persisted_bytes, creation_counter, creation_times } - } - - fn read_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - let persisted_lock = self.persisted_bytes.lock().unwrap(); - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - - if let Some(outer_ref) = persisted_lock.get(&prefixed) { - if let Some(inner_ref) = outer_ref.get(key) { - let bytes = inner_ref.clone(); - Ok(bytes) - } else { - Err(io::Error::new(io::ErrorKind::NotFound, "Key not found")) - } - } else { - Err(io::Error::new(io::ErrorKind::NotFound, "Namespace not found")) - } - } - - fn write_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - let mut persisted_lock = self.persisted_bytes.lock().unwrap(); - - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - let outer_e = persisted_lock.entry(prefixed.clone()).or_insert(HashMap::new()); - outer_e.insert(key.to_string(), buf); - - // Only assign creation time on first write (not on update) - let mut ct_lock = self.creation_times.lock().unwrap(); - let ct_ns = ct_lock.entry(prefixed).or_insert(HashMap::new()); - ct_ns - .entry(key.to_string()) - .or_insert_with(|| self.creation_counter.fetch_add(1, Ordering::Relaxed)); - - Ok(()) - } - - fn remove_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, _lazy: bool, - ) -> io::Result<()> { - let mut persisted_lock = self.persisted_bytes.lock().unwrap(); - - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - if let Some(outer_ref) = persisted_lock.get_mut(&prefixed) { - outer_ref.remove(&key.to_string()); - } - - // Remove creation time entry - let mut ct_lock = self.creation_times.lock().unwrap(); - if let Some(ct_ns) = ct_lock.get_mut(&prefixed) { - ct_ns.remove(key); - } - - Ok(()) - } - - fn list_internal( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> io::Result> { - let mut persisted_lock = self.persisted_bytes.lock().unwrap(); - - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - match persisted_lock.entry(prefixed) { - hash_map::Entry::Occupied(e) => Ok(e.get().keys().cloned().collect()), - hash_map::Entry::Vacant(_) => Ok(Vec::new()), - } - } -} - -impl KVStore for InMemoryStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> impl Future, io::Error>> + 'static + Send { - let res = self.read_internal(&primary_namespace, &secondary_namespace, &key); - async move { res } - } - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> impl Future> + 'static + Send { - let res = self.write_internal(&primary_namespace, &secondary_namespace, &key, buf); - async move { res } - } - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> impl Future> + 'static + Send { - let res = self.remove_internal(&primary_namespace, &secondary_namespace, &key, lazy); - async move { res } - } - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> impl Future, io::Error>> + 'static + Send { - let res = self.list_internal(primary_namespace, secondary_namespace); - async move { res } - } -} - -impl KVStoreSync for InMemoryStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - self.read_internal(primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - self.write_internal(primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> io::Result<()> { - self.remove_internal(primary_namespace, secondary_namespace, key, lazy) - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - self.list_internal(primary_namespace, secondary_namespace) - } -} - -impl InMemoryStore { - fn list_paginated_internal( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - let ct_lock = self.creation_times.lock().unwrap(); - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - - let ct_ns = match ct_lock.get(&prefixed) { - Some(m) => m, - None => { - return Ok(PaginatedListResponse { keys: Vec::new(), next_page_token: None }); - }, - }; - - // Build list of (key, sort_order) sorted by sort_order DESC (newest first). - let mut entries: Vec<(&String, &u64)> = ct_ns.iter().collect(); - entries.sort_by(|a, b| b.1.cmp(a.1)); - - // Apply page token filter - let start_idx = if let Some(ref token) = page_token { - let token_sort_order: u64 = token - .as_str() - .parse() - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Invalid page token"))?; - - entries - .iter() - .position(|(_, sort_order)| **sort_order < token_sort_order) - .unwrap_or(entries.len()) - } else { - 0 - }; - - // Fetch one extra entry beyond page size to determine whether a next page exists. - let mut page: Vec<(&String, &u64)> = - entries[start_idx..].iter().take(IN_MEMORY_PAGE_SIZE + 1).cloned().collect(); - - let has_more = page.len() > IN_MEMORY_PAGE_SIZE; - page.truncate(IN_MEMORY_PAGE_SIZE); - - let next_page_token = if has_more { - let (_, last_sort_order) = page.last().unwrap(); - Some(PageToken::new(last_sort_order.to_string())) - } else { - None - }; - - let page: Vec = page.into_iter().map(|(k, _)| k.clone()).collect(); - - Ok(PaginatedListResponse { keys: page, next_page_token }) - } -} - -impl PaginatedKVStoreSync for InMemoryStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - self.list_paginated_internal(primary_namespace, secondary_namespace, page_token) - } -} - -impl PaginatedKVStore for InMemoryStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> impl Future> + 'static + Send { - let res = self.list_paginated_internal(primary_namespace, secondary_namespace, page_token); - async move { res } - } -} - -unsafe impl Sync for InMemoryStore {} -unsafe impl Send for InMemoryStore {} +pub(crate) use in_memory_store::InMemoryStore; pub(crate) fn random_storage_path() -> PathBuf { let mut temp_path = std::env::temp_dir(); From d6fd1dc96150665eda84d45e82e4629da90aa8d6 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 11:58:16 +0200 Subject: [PATCH 10/17] Move test store checks onto async KV storage Exercise async KVStore operations in TestSyncStore and filesystem migration tests while keeping the temporary sync comparison path until the final KVStoreSync removal. Co-Authored-By: HAL 9000 --- src/io/utils.rs | 45 ++++++---- tests/common/mod.rs | 211 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 195 insertions(+), 61 deletions(-) diff --git a/src/io/utils.rs b/src/io/utils.rs index cf3905332..4657688f5 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -719,7 +719,7 @@ mod tests { use std::fs; use std::path::{Path, PathBuf}; - use lightning::util::persist::{migrate_kv_store_data, KVStoreSync}; + use lightning::util::persist::{migrate_kv_store_data_async, KVStore}; use lightning_persister::fs_store::v1::FilesystemStore; use lightning_persister::fs_store::v2::FilesystemStoreV2; @@ -743,19 +743,20 @@ mod tests { #[tokio::test] async fn fs_store_migration_recovers_before_v1_backup_rename() { let fs_store_path = fs_store_path(); - let mut v1_store = write_v1_test_data(&fs_store_path); + let v1_store = write_v1_test_data(&fs_store_path).await; let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); - let mut v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); - migrate_kv_store_data(&mut v1_store, &mut v2_store).unwrap(); + let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); + migrate_kv_store_data_async(&v1_store, &v2_store).await.unwrap(); let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -766,22 +767,23 @@ mod tests { #[tokio::test] async fn fs_store_migration_recovers_after_v1_backup_rename() { let fs_store_path = fs_store_path(); - let mut v1_store = write_v1_test_data(&fs_store_path); + let v1_store = write_v1_test_data(&fs_store_path).await; let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); - let mut v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); - migrate_kv_store_data(&mut v1_store, &mut v2_store).unwrap(); + let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); + migrate_kv_store_data_async(&v1_store, &v2_store).await.unwrap(); let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, backup_path).unwrap(); let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -792,10 +794,10 @@ mod tests { #[tokio::test] async fn fs_store_migration_recovers_after_v2_rename() { let fs_store_path = fs_store_path(); - let mut v1_store = write_v1_test_data(&fs_store_path); + let v1_store = write_v1_test_data(&fs_store_path).await; let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); - let mut v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); - migrate_kv_store_data(&mut v1_store, &mut v2_store).unwrap(); + let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); + migrate_kv_store_data_async(&v1_store, &v2_store).await.unwrap(); let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, &backup_path).unwrap(); @@ -803,12 +805,13 @@ mod tests { let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -820,19 +823,20 @@ mod tests { #[tokio::test] async fn fs_store_migration_recovers_backup_without_migrating_dir() { let fs_store_path = fs_store_path(); - write_v1_test_data(&fs_store_path); + write_v1_test_data(&fs_store_path).await; let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, backup_path).unwrap(); let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -845,23 +849,25 @@ mod tests { let fs_store_path = fs_store_path(); let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); - KVStoreSync::write( + KVStore::write( &v2_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY, TEST_VALUE.to_vec(), ) + .await .unwrap(); let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -881,15 +887,16 @@ mod tests { sibling_path } - fn write_v1_test_data(fs_store_path: &Path) -> FilesystemStore { + async fn write_v1_test_data(fs_store_path: &Path) -> FilesystemStore { let v1_store = FilesystemStore::new(fs_store_path.to_path_buf()); - KVStoreSync::write( + KVStore::write( &v1_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY, TEST_VALUE.to_vec(), ) + .await .unwrap(); v1_store } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 30d9a4387..f2e7191ae 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -24,7 +24,7 @@ use std::future::Future; use std::path::PathBuf; use std::str::FromStr; use std::sync::atomic::{AtomicU16, Ordering}; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use std::time::Duration; use bitcoin::hashes::hex::FromHex; @@ -51,7 +51,6 @@ use lightning::io; use lightning::ln::msgs::SocketAddress; use lightning::routing::gossip::NodeAlias; use lightning::util::persist::{KVStore, KVStoreSync}; -use lightning::util::test_utils::TestStore; use lightning_invoice::{Bolt11InvoiceDescription, Description}; use lightning_persister::fs_store::v1::FilesystemStore; use lightning_types::payment::{PaymentHash, PaymentPreimage}; @@ -60,6 +59,10 @@ use rand::distr::Alphanumeric; use rand::{rng, Rng}; use serde_json::{json, Value}; +#[path = "../../src/io/in_memory_store.rs"] +mod in_memory_store; +use in_memory_store::InMemoryStore; + /// Shared timeout (in seconds) for waiting on LDK events and external node operations. pub(crate) const INTEROP_TIMEOUT_SECS: u64 = 60; @@ -1660,16 +1663,9 @@ impl KVStore for TestSyncStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); - let fut = tokio::task::spawn_blocking(move || { - inner.read_internal(&primary_namespace, &secondary_namespace, &key) - }); - async move { - fut.await.unwrap_or_else(|e| { - let msg = format!("Failed to IO operation due join error: {}", e); - Err(io::Error::new(io::ErrorKind::Other, msg)) - }) - } + async move { inner.read_internal_async(&primary_namespace, &secondary_namespace, &key).await } } + fn write( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, ) -> impl Future> + 'static + Send { @@ -1677,16 +1673,11 @@ impl KVStore for TestSyncStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); - let fut = tokio::task::spawn_blocking(move || { - inner.write_internal(&primary_namespace, &secondary_namespace, &key, buf) - }); async move { - fut.await.unwrap_or_else(|e| { - let msg = format!("Failed to IO operation due join error: {}", e); - Err(io::Error::new(io::ErrorKind::Other, msg)) - }) + inner.write_internal_async(&primary_namespace, &secondary_namespace, &key, buf).await } } + fn remove( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, ) -> impl Future> + 'static + Send { @@ -1694,31 +1685,18 @@ impl KVStore for TestSyncStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); - let fut = tokio::task::spawn_blocking(move || { - inner.remove_internal(&primary_namespace, &secondary_namespace, &key, lazy) - }); async move { - fut.await.unwrap_or_else(|e| { - let msg = format!("Failed to IO operation due join error: {}", e); - Err(io::Error::new(io::ErrorKind::Other, msg)) - }) + inner.remove_internal_async(&primary_namespace, &secondary_namespace, &key, lazy).await } } + fn list( &self, primary_namespace: &str, secondary_namespace: &str, ) -> impl Future, io::Error>> + 'static + Send { let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let inner = Arc::clone(&self.inner); - let fut = tokio::task::spawn_blocking(move || { - inner.list_internal(&primary_namespace, &secondary_namespace) - }); - async move { - fut.await.unwrap_or_else(|e| { - let msg = format!("Failed to IO operation due join error: {}", e); - Err(io::Error::new(io::ErrorKind::Other, msg)) - }) - } + async move { inner.list_internal_async(&primary_namespace, &secondary_namespace).await } } } @@ -1749,15 +1727,15 @@ impl KVStoreSync for TestSyncStore { } struct TestSyncStoreInner { - serializer: RwLock<()>, - test_store: TestStore, + serializer: tokio::sync::RwLock<()>, + test_store: InMemoryStore, fs_store: FilesystemStore, sqlite_store: SqliteStore, } impl TestSyncStoreInner { fn new(dest_dir: PathBuf) -> Self { - let serializer = RwLock::new(()); + let serializer = tokio::sync::RwLock::new(()); let mut fs_dir = dest_dir.clone(); fs_dir.push("fs_store"); let fs_store = FilesystemStore::new(fs_dir); @@ -1769,7 +1747,7 @@ impl TestSyncStoreInner { Some("test_sync_table".to_string()), ) .unwrap(); - let test_store = TestStore::new(false); + let test_store = InMemoryStore::new(); Self { serializer, fs_store, sqlite_store, test_store } } @@ -1803,10 +1781,159 @@ impl TestSyncStoreInner { } } + async fn do_list_async( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> lightning::io::Result> { + let fs_res = KVStore::list(&self.fs_store, primary_namespace, secondary_namespace).await; + let sqlite_res = + KVStore::list(&self.sqlite_store, primary_namespace, secondary_namespace).await; + let test_res = + KVStore::list(&self.test_store, primary_namespace, secondary_namespace).await; + + match fs_res { + Ok(mut list) => { + list.sort(); + + let mut sqlite_list = sqlite_res.unwrap(); + sqlite_list.sort(); + assert_eq!(list, sqlite_list); + + let mut test_list = test_res.unwrap(); + test_list.sort(); + assert_eq!(list, test_list); + + Ok(list) + }, + Err(e) => { + assert!(sqlite_res.is_err()); + assert!(test_res.is_err()); + Err(e) + }, + } + } + + async fn list_internal_async( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> lightning::io::Result> { + let _guard = self.serializer.read().await; + self.do_list_async(primary_namespace, secondary_namespace).await + } + + async fn read_internal_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> lightning::io::Result> { + let _guard = self.serializer.read().await; + + let fs_res = + KVStore::read(&self.fs_store, primary_namespace, secondary_namespace, key).await; + let sqlite_res = + KVStore::read(&self.sqlite_store, primary_namespace, secondary_namespace, key).await; + let test_res = + KVStore::read(&self.test_store, primary_namespace, secondary_namespace, key).await; + + match fs_res { + Ok(read) => { + assert_eq!(read, sqlite_res.unwrap()); + assert_eq!(read, test_res.unwrap()); + Ok(read) + }, + Err(e) => { + assert!(sqlite_res.is_err()); + assert_eq!(e.kind(), unsafe { sqlite_res.unwrap_err_unchecked().kind() }); + assert!(test_res.is_err()); + assert_eq!(e.kind(), unsafe { test_res.unwrap_err_unchecked().kind() }); + Err(e) + }, + } + } + + async fn write_internal_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> lightning::io::Result<()> { + let _guard = self.serializer.write().await; + let fs_res = KVStore::write( + &self.fs_store, + primary_namespace, + secondary_namespace, + key, + buf.clone(), + ) + .await; + let sqlite_res = KVStore::write( + &self.sqlite_store, + primary_namespace, + secondary_namespace, + key, + buf.clone(), + ) + .await; + let test_res = KVStore::write( + &self.test_store, + primary_namespace, + secondary_namespace, + key, + buf.clone(), + ) + .await; + + assert!(self + .do_list_async(primary_namespace, secondary_namespace) + .await + .unwrap() + .contains(&key.to_string())); + + match fs_res { + Ok(()) => { + assert!(sqlite_res.is_ok()); + assert!(test_res.is_ok()); + Ok(()) + }, + Err(e) => { + assert!(sqlite_res.is_err()); + assert!(test_res.is_err()); + Err(e) + }, + } + } + + async fn remove_internal_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> lightning::io::Result<()> { + let _guard = self.serializer.write().await; + let fs_res = + KVStore::remove(&self.fs_store, primary_namespace, secondary_namespace, key, lazy) + .await; + let sqlite_res = + KVStore::remove(&self.sqlite_store, primary_namespace, secondary_namespace, key, lazy) + .await; + let test_res = + KVStore::remove(&self.test_store, primary_namespace, secondary_namespace, key, lazy) + .await; + + assert!(!self + .do_list_async(primary_namespace, secondary_namespace) + .await + .unwrap() + .contains(&key.to_string())); + + match fs_res { + Ok(()) => { + assert!(sqlite_res.is_ok()); + assert!(test_res.is_ok()); + Ok(()) + }, + Err(e) => { + assert!(sqlite_res.is_err()); + assert!(test_res.is_err()); + Err(e) + }, + } + } + fn read_internal( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> lightning::io::Result> { - let _guard = self.serializer.read().unwrap(); + let _guard = self.serializer.blocking_read(); let fs_res = KVStoreSync::read(&self.fs_store, primary_namespace, secondary_namespace, key); let sqlite_res = @@ -1833,7 +1960,7 @@ impl TestSyncStoreInner { fn write_internal( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, ) -> lightning::io::Result<()> { - let _guard = self.serializer.write().unwrap(); + let _guard = self.serializer.blocking_write(); let fs_res = KVStoreSync::write( &self.fs_store, primary_namespace, @@ -1878,7 +2005,7 @@ impl TestSyncStoreInner { fn remove_internal( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, ) -> lightning::io::Result<()> { - let _guard = self.serializer.write().unwrap(); + let _guard = self.serializer.blocking_write(); let fs_res = KVStoreSync::remove(&self.fs_store, primary_namespace, secondary_namespace, key, lazy); let sqlite_res = KVStoreSync::remove( @@ -1918,7 +2045,7 @@ impl TestSyncStoreInner { fn list_internal( &self, primary_namespace: &str, secondary_namespace: &str, ) -> lightning::io::Result> { - let _guard = self.serializer.read().unwrap(); + let _guard = self.serializer.blocking_read(); self.do_list(primary_namespace, secondary_namespace) } } From 99fe33b283b7472adcaef524bcd185ed78df2a61 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 8 Jun 2026 16:31:25 +0200 Subject: [PATCH 11/17] Add async test monitor persister Use a local async KVStore-backed monitor persister in store tests so the later blocking KVStore removal can drop the old synchronous MonitorUpdatingPersister path without mixing the test adapter into that change. Co-Authored-By: HAL 9000 --- src/io/test_utils.rs | 163 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 142 insertions(+), 21 deletions(-) diff --git a/src/io/test_utils.rs b/src/io/test_utils.rs index 0b16e525e..685377853 100644 --- a/src/io/test_utils.rs +++ b/src/io/test_utils.rs @@ -7,8 +7,12 @@ use std::panic::RefUnwindSafe; use std::path::PathBuf; +use std::sync::Arc; +use lightning::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate}; +use lightning::chain::{chainmonitor, BlockLocator, ChannelMonitorUpdateStatus}; use lightning::events::ClosureReason; +use lightning::io; use lightning::ln::functional_test_utils::{ check_added_monitors, check_closed_broadcast, check_closed_event, connect_block, create_announced_chan_between_nodes, create_chanmon_cfgs, create_dummy_block, create_network, @@ -16,8 +20,13 @@ use lightning::ln::functional_test_utils::{ TestChanMonCfg, }; use lightning::util::persist::{ - KVStoreSync, MonitorUpdatingPersister, KVSTORE_NAMESPACE_KEY_MAX_LEN, + KVStore, KVStoreSync, MonitorName, ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + KVSTORE_NAMESPACE_KEY_MAX_LEN, }; +use lightning::util::ser::{ReadableArgs, Writeable}; +use lightning::util::test_channel_signer::TestChannelSigner; use lightning::util::test_utils; use rand::distr::Alphanumeric; use rand::{rng, Rng}; @@ -25,14 +34,127 @@ use rand::{rng, Rng}; #[path = "in_memory_store.rs"] mod in_memory_store; -type TestMonitorUpdatePersister<'a, K> = MonitorUpdatingPersister< - &'a K, - &'a test_utils::TestLogger, - &'a test_utils::TestKeysInterface, - &'a test_utils::TestKeysInterface, - &'a test_utils::TestBroadcaster, - &'a test_utils::TestFeeEstimator, ->; +use crate::logger::Logger; +use crate::runtime::Runtime; + +pub(crate) struct TestMonitorUpdatePersister<'a, K> { + store: &'a K, + runtime: Runtime, + entropy_source: &'a test_utils::TestKeysInterface, + signer_provider: &'a test_utils::TestKeysInterface, +} + +impl TestMonitorUpdatePersister<'_, K> { + pub(crate) fn read_all_channel_monitors_with_updates( + &self, + ) -> Result)>, io::Error> { + self.runtime.block_on(async { + let stored_keys = KVStore::list( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await?; + + let mut res = Vec::with_capacity(stored_keys.len()); + for stored_key in stored_keys { + let data = KVStore::read( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &stored_key, + ) + .await?; + match )>>::read( + &mut io::Cursor::new(data), + (self.entropy_source, self.signer_provider), + ) { + Ok(Some((best_block, channel_monitor))) => { + res.push((best_block, channel_monitor)); + }, + Ok(None) => {}, + Err(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Failed to read ChannelMonitor", + )); + }, + } + } + Ok(res) + }) + } + + fn write_monitor( + &self, monitor_name: MonitorName, monitor: &ChannelMonitor, + ) -> ChannelMonitorUpdateStatus { + let write_res = self.runtime.block_on(KVStore::write( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &monitor_name.to_string(), + monitor.encode(), + )); + match write_res { + Ok(()) => ChannelMonitorUpdateStatus::Completed, + Err(_) => ChannelMonitorUpdateStatus::UnrecoverableError, + } + } +} + +impl chainmonitor::Persist + for TestMonitorUpdatePersister<'_, K> +{ + fn persist_new_channel( + &self, monitor_name: MonitorName, monitor: &ChannelMonitor, + ) -> ChannelMonitorUpdateStatus { + self.write_monitor(monitor_name, monitor) + } + + fn update_persisted_channel( + &self, monitor_name: MonitorName, _monitor_update: Option<&ChannelMonitorUpdate>, + monitor: &ChannelMonitor, + ) -> ChannelMonitorUpdateStatus { + self.write_monitor(monitor_name, monitor) + } + + fn archive_persisted_channel(&self, monitor_name: MonitorName) { + let key = monitor_name.to_string(); + self.runtime.block_on(async { + let monitor = match KVStore::read( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &key, + ) + .await + { + Ok(monitor) => monitor, + Err(_) => return, + }; + + if KVStore::write( + self.store, + ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &key, + monitor, + ) + .await + .is_ok() + { + let _ = KVStore::remove( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &key, + true, + ) + .await; + } + }); + } +} const EXPECTED_UPDATES_PER_PAYMENT: u64 = 5; @@ -96,21 +218,20 @@ pub(crate) fn do_read_write_remove_list_persist( assert_eq!(listed_keys.len(), 0); } -pub(crate) fn create_persister<'a, K: KVStoreSync + Sync>( - store: &'a K, chanmon_cfg: &'a TestChanMonCfg, max_pending_updates: u64, +pub(crate) fn create_persister<'a, K: KVStore + Sync>( + store: &'a K, chanmon_cfg: &'a TestChanMonCfg, _max_pending_updates: u64, ) -> TestMonitorUpdatePersister<'a, K> { - MonitorUpdatingPersister::new( + let runtime = + Runtime::new(Arc::new(Logger::new_log_facade())).expect("Failed to setup runtime"); + TestMonitorUpdatePersister { store, - &chanmon_cfg.logger, - max_pending_updates, - &chanmon_cfg.keys_manager, - &chanmon_cfg.keys_manager, - &chanmon_cfg.tx_broadcaster, - &chanmon_cfg.fee_estimator, - ) + runtime, + entropy_source: &chanmon_cfg.keys_manager, + signer_provider: &chanmon_cfg.keys_manager, + } } -pub(crate) fn create_chain_monitor<'a, K: KVStoreSync + Sync>( +pub(crate) fn create_chain_monitor<'a, K: KVStore + Sync>( chanmon_cfg: &'a TestChanMonCfg, persister: &'a TestMonitorUpdatePersister<'a, K>, ) -> test_utils::TestChainMonitor<'a> { test_utils::TestChainMonitor::new( @@ -125,7 +246,7 @@ pub(crate) fn create_chain_monitor<'a, K: KVStoreSync + Sync>( // Integration-test the given KVStore implementation. Test relaying a few payments and check that // the persisted data is updated the appropriate number of times. -pub(crate) fn do_test_store(store_0: &K, store_1: &K) { +pub(crate) fn do_test_store(store_0: &K, store_1: &K) { // This value is used later to limit how many iterations we perform. let persister_0_max_pending_updates = 7; // Intentionally set this to a smaller value to test a different alignment. From 458d660bb2a3d9d7affd76dcf431e4d8420d7d25 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 11:29:24 +0200 Subject: [PATCH 12/17] Remove blocking KV store support Drop the remaining synchronous KV store trait bounds and implementations. After the preceding migrations, custom stores only need to provide async KVStore persistence, and pathfinding score export also reads from the async store. Co-Authored-By: HAL 9000 --- src/builder.rs | 16 +-- src/io/in_memory_store.rs | 36 +---- src/io/postgres_store/mod.rs | 178 +++++++++---------------- src/io/sqlite_store/migrations.rs | 24 ++-- src/io/sqlite_store/mod.rs | 212 +++++++++++------------------- src/io/test_utils.rs | 67 +++++++--- src/io/vss_store.rs | 126 ++---------------- src/lib.rs | 33 ++--- src/types.rs | 80 +---------- tests/common/mod.rs | 177 +------------------------ 10 files changed, 235 insertions(+), 714 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 9fd7f832c..c88c867cc 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -81,7 +81,7 @@ use crate::tx_broadcaster::TransactionBroadcaster; use crate::types::{ AsyncPersister, ChainMonitor, ChannelManager, DynStore, DynStoreRef, DynStoreWrapper, GossipSync, Graph, HRNResolver, KeysManager, MessageRouter, OnionMessenger, PaymentStore, - PeerManager, PendingPaymentStore, SyncAndAsyncKVStore, + PeerManager, PendingPaymentStore, }; use crate::wallet::persist::KVStoreWalletPersister; use crate::wallet::Wallet; @@ -176,17 +176,17 @@ pub enum BuildError { RuntimeSetupFailed, /// We failed to read data from the [`KVStore`]. /// - /// [`KVStore`]: lightning::util::persist::KVStoreSync + /// [`KVStore`]: lightning::util::persist::KVStore ReadFailed, /// We failed to write data to the [`KVStore`]. /// - /// [`KVStore`]: lightning::util::persist::KVStoreSync + /// [`KVStore`]: lightning::util::persist::KVStore WriteFailed, /// We failed to access the given `storage_dir_path`. StoragePathAccessFailed, /// We failed to setup our [`KVStore`]. /// - /// [`KVStore`]: lightning::util::persist::KVStoreSync + /// [`KVStore`]: lightning::util::persist::KVStore KVStoreSetupFailed, /// We failed to setup the onchain wallet. WalletSetupFailed, @@ -826,7 +826,7 @@ impl NodeBuilder { } /// Builds a [`Node`] instance according to the options previously configured. - pub fn build_with_store( + pub fn build_with_store( &self, node_entropy: NodeEntropy, kv_store: S, ) -> Result { let logger = setup_logger(&self.log_writer_config, &self.config)?; @@ -845,14 +845,14 @@ impl NodeBuilder { } } - fn build_with_store_and_logger( + fn build_with_store_and_logger( &self, node_entropy: NodeEntropy, kv_store: S, logger: Arc, ) -> Result { let runtime = self.setup_runtime(&logger)?; self.build_with_store_runtime_and_logger(node_entropy, kv_store, runtime, logger) } - fn build_with_store_runtime_and_logger( + fn build_with_store_runtime_and_logger( &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc, logger: Arc, ) -> Result { let seed_bytes = node_entropy.to_seed_bytes(); @@ -1346,7 +1346,7 @@ impl ArcedNodeBuilder { /// Builds a [`Node`] instance according to the options previously configured. // Note that the generics here don't actually work for Uniffi, but we don't currently expose // this so its not needed. - pub fn build_with_store( + pub fn build_with_store( &self, node_entropy: Arc, kv_store: S, ) -> Result, BuildError> { self.inner.read().expect("lock").build_with_store(*node_entropy, kv_store).map(Arc::new) diff --git a/src/io/in_memory_store.rs b/src/io/in_memory_store.rs index 8d0d24a4e..8b7d41c84 100644 --- a/src/io/in_memory_store.rs +++ b/src/io/in_memory_store.rs @@ -11,9 +11,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Mutex; use lightning::io; -use lightning::util::persist::{ - KVStore, KVStoreSync, PageToken, PaginatedKVStore, PaginatedKVStoreSync, PaginatedListResponse, -}; +use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; const IN_MEMORY_PAGE_SIZE: usize = 50; @@ -127,30 +125,6 @@ impl KVStore for InMemoryStore { } } -impl KVStoreSync for InMemoryStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - self.read_internal(primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - self.write_internal(primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> io::Result<()> { - self.remove_internal(primary_namespace, secondary_namespace, key, lazy) - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - self.list_internal(primary_namespace, secondary_namespace) - } -} - impl InMemoryStore { fn list_paginated_internal( &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, @@ -204,14 +178,6 @@ impl InMemoryStore { } } -impl PaginatedKVStoreSync for InMemoryStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - self.list_paginated_internal(primary_namespace, secondary_namespace, page_token) - } -} - impl PaginatedKVStore for InMemoryStore { fn list_paginated( &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, diff --git a/src/io/postgres_store/mod.rs b/src/io/postgres_store/mod.rs index 7319d0898..c54cbdef1 100644 --- a/src/io/postgres_store/mod.rs +++ b/src/io/postgres_store/mod.rs @@ -12,9 +12,7 @@ use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use lightning::io; -use lightning::util::persist::{ - KVStore, KVStoreSync, PageToken, PaginatedKVStore, PaginatedKVStoreSync, PaginatedListResponse, -}; +use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; use lightning_types::string::PrintableString; use native_tls::TlsConnector; use postgres_native_tls::MakeTlsConnector; @@ -91,10 +89,9 @@ macro_rules! query_with_retry { }}; } -/// A [`KVStoreSync`] implementation that writes to and reads from a [PostgreSQL] database. +/// A [`KVStore`] implementation that writes to and reads from a [PostgreSQL] database. /// -/// Maintains an internal runtime for the underlying tokio-postgres connection drivers and for -/// synchronous [`KVStoreSync`] and [`PaginatedKVStoreSync`] calls. +/// Maintains an internal runtime for the underlying tokio-postgres connection drivers. /// /// [PostgreSQL]: https://www.postgresql.org pub struct PostgresStore { @@ -104,14 +101,14 @@ pub struct PostgresStore { // operations aren't sensitive to the order of execution. next_write_version: AtomicU64, - // A store-internal runtime used for setup, connection driver tasks, and sync store access. + // A store-internal runtime used for setup and connection driver tasks. internal_runtime: Option, } // tokio::sync::Mutex (used for the DB client) contains UnsafeCell which opts out of // RefUnwindSafe. std::sync::Mutex (used by SqliteStore) doesn't have this issue because // it poisons on panic. This impl is needed for do_read_write_remove_list_persist which -// requires K: KVStoreSync + RefUnwindSafe. +// requires K: KVStore + RefUnwindSafe. #[cfg(test)] impl std::panic::RefUnwindSafe for PostgresStore {} @@ -302,48 +299,6 @@ impl PostgresStore { io::Error::new(io::ErrorKind::Other, "Failed to access internal PostgreSQL runtime") }) } - - fn block_on(&self, fut: F) -> io::Result { - let internal_runtime = self.internal_runtime()?; - Ok(tokio::task::block_in_place(move || internal_runtime.block_on(fut))) - } -} - -impl KVStoreSync for PostgresStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - self.block_on(KVStore::read(self, primary_namespace, secondary_namespace, key))? - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - self.block_on(KVStore::write(self, primary_namespace, secondary_namespace, key, buf))? - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> io::Result<()> { - self.block_on(KVStore::remove(self, primary_namespace, secondary_namespace, key, lazy))? - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - self.block_on(KVStore::list(self, primary_namespace, secondary_namespace))? - } -} - -impl PaginatedKVStoreSync for PostgresStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - self.block_on(PaginatedKVStore::list_paginated( - self, - primary_namespace, - secondary_namespace, - page_token, - ))? - } } impl PaginatedKVStore for PostgresStore { @@ -901,7 +856,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn read_write_remove_list_persist() { let store = create_test_store("test_rwrl").await; - do_read_write_remove_list_persist(&store); + do_read_write_remove_list_persist(&store).await; cleanup_store(&store).await; } @@ -931,17 +886,17 @@ mod tests { let sub = "test_sub"; // Write a value before disconnecting. - KVStoreSync::write(&store, ns, sub, "key_a", vec![1u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, "key_a", vec![1u8; 8]).await.unwrap(); // Read should auto-reconnect and return the previously written value. kill_connection(&store).await; - let data = KVStoreSync::read(&store, ns, sub, "key_a").unwrap(); + let data = KVStore::read(&store, ns, sub, "key_a").await.unwrap(); assert_eq!(data, vec![1u8; 8]); // Write should auto-reconnect without a preceding read. kill_connection(&store).await; - KVStoreSync::write(&store, ns, sub, "key_b", vec![2u8; 8]).unwrap(); - let data = KVStoreSync::read(&store, ns, sub, "key_b").unwrap(); + KVStore::write(&store, ns, sub, "key_b", vec![2u8; 8]).await.unwrap(); + let data = KVStore::read(&store, ns, sub, "key_b").await.unwrap(); assert_eq!(data, vec![2u8; 8]); cleanup_store(&store).await; @@ -958,7 +913,9 @@ mod tests { for i in 0..num_entries { let key = format!("key_{:04}", i); let data = vec![i as u8; 32]; - KVStoreSync::write(&store, primary_namespace, secondary_namespace, &key, data).unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, &key, data) + .await + .unwrap(); } // Paginate through all entries and collect them @@ -967,12 +924,13 @@ mod tests { let mut page_count = 0; loop { - let response = PaginatedKVStoreSync::list_paginated( + let response = PaginatedKVStore::list_paginated( &store, primary_namespace, secondary_namespace, page_token, ) + .await .unwrap(); all_keys.extend(response.keys.clone()); @@ -1010,32 +968,33 @@ mod tests { let primary_namespace = "test_ns"; let secondary_namespace = "test_sub"; - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "first", vec![1u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "first", vec![1u8; 8]) + .await .unwrap(); - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "second", vec![2u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "second", vec![2u8; 8]) + .await .unwrap(); - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "third", vec![3u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "third", vec![3u8; 8]) + .await .unwrap(); // Update the first entry - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "first", vec![99u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "first", vec![99u8; 8]) + .await .unwrap(); // Paginated listing should still show "first" with its original creation order - let response = PaginatedKVStoreSync::list_paginated( - &store, - primary_namespace, - secondary_namespace, - None, - ) - .unwrap(); + let response = + PaginatedKVStore::list_paginated(&store, primary_namespace, secondary_namespace, None) + .await + .unwrap(); // Newest first: third, second, first assert_eq!(response.keys, vec!["third", "second", "first"]); // Verify the updated value was persisted let data = - KVStoreSync::read(&store, primary_namespace, secondary_namespace, "first").unwrap(); + KVStore::read(&store, primary_namespace, secondary_namespace, "first").await.unwrap(); assert_eq!(data, vec![99u8; 8]); cleanup_store(&store).await; @@ -1047,7 +1006,7 @@ mod tests { // Paginating an empty or unknown namespace returns an empty result with no token. let response = - PaginatedKVStoreSync::list_paginated(&store, "nonexistent", "ns", None).unwrap(); + PaginatedKVStore::list_paginated(&store, "nonexistent", "ns", None).await.unwrap(); assert!(response.keys.is_empty()); assert!(response.next_page_token.is_none()); @@ -1058,22 +1017,23 @@ mod tests { async fn test_postgres_store_paginated_namespace_isolation() { let store = create_test_store("test_pg_paginated_isolation").await; - KVStoreSync::write(&store, "ns_a", "sub", "key_1", vec![1u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_a", "sub", "key_2", vec![2u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_b", "sub", "key_3", vec![3u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_a", "other", "key_4", vec![4u8; 8]).unwrap(); + KVStore::write(&store, "ns_a", "sub", "key_1", vec![1u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_a", "sub", "key_2", vec![2u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_b", "sub", "key_3", vec![3u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_a", "other", "key_4", vec![4u8; 8]).await.unwrap(); // ns_a/sub should only contain key_1 and key_2 (newest first). - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_a", "sub", None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, "ns_a", "sub", None).await.unwrap(); assert_eq!(response.keys, vec!["key_2", "key_1"]); assert!(response.next_page_token.is_none()); // ns_b/sub should only contain key_3. - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_b", "sub", None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, "ns_b", "sub", None).await.unwrap(); assert_eq!(response.keys, vec!["key_3"]); // ns_a/other should only contain key_4. - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_a", "other", None).unwrap(); + let response = + PaginatedKVStore::list_paginated(&store, "ns_a", "other", None).await.unwrap(); assert_eq!(response.keys, vec!["key_4"]); cleanup_store(&store).await; @@ -1086,13 +1046,13 @@ mod tests { let ns = "test_ns"; let sub = "test_sub"; - KVStoreSync::write(&store, ns, sub, "a", vec![1u8; 8]).unwrap(); - KVStoreSync::write(&store, ns, sub, "b", vec![2u8; 8]).unwrap(); - KVStoreSync::write(&store, ns, sub, "c", vec![3u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, "a", vec![1u8; 8]).await.unwrap(); + KVStore::write(&store, ns, sub, "b", vec![2u8; 8]).await.unwrap(); + KVStore::write(&store, ns, sub, "c", vec![3u8; 8]).await.unwrap(); - KVStoreSync::remove(&store, ns, sub, "b", false).unwrap(); + KVStore::remove(&store, ns, sub, "b", false).await.unwrap(); - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys, vec!["c", "a"]); assert!(response.next_page_token.is_none()); @@ -1109,24 +1069,24 @@ mod tests { // Write exactly PAGE_SIZE entries (50). for i in 0..PAGE_SIZE { let key = format!("key_{:04}", i); - KVStoreSync::write(&store, ns, sub, &key, vec![i as u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, &key, vec![i as u8; 8]).await.unwrap(); } // Exactly PAGE_SIZE entries: all returned in one page with no next-page token. - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), PAGE_SIZE); assert!(response.next_page_token.is_none()); // Add one more entry (PAGE_SIZE + 1 total). First page should now have a token. - KVStoreSync::write(&store, ns, sub, "key_extra", vec![0u8; 8]).unwrap(); - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + KVStore::write(&store, ns, sub, "key_extra", vec![0u8; 8]).await.unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), PAGE_SIZE); assert!(response.next_page_token.is_some()); // Second page should have exactly 1 entry and no token. - let response = - PaginatedKVStoreSync::list_paginated(&store, ns, sub, response.next_page_token) - .unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, response.next_page_token) + .await + .unwrap(); assert_eq!(response.keys.len(), 1); assert!(response.next_page_token.is_none()); @@ -1143,10 +1103,10 @@ mod tests { // Write fewer entries than PAGE_SIZE. for i in 0..5 { let key = format!("key_{i}"); - KVStoreSync::write(&store, ns, sub, &key, vec![i as u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, &key, vec![i as u8; 8]).await.unwrap(); } - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), 5); // Fewer than PAGE_SIZE means no next page. assert!(response.next_page_token.is_none()); @@ -1165,22 +1125,12 @@ mod tests { { let store = create_test_store(table_name).await; - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_a", - vec![1u8; 8], - ) - .unwrap(); - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_b", - vec![2u8; 8], - ) - .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_a", vec![1u8; 8]) + .await + .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_b", vec![2u8; 8]) + .await + .unwrap(); // Don't clean up since we want to reopen } @@ -1189,22 +1139,18 @@ mod tests { { let store = create_test_store(table_name).await; - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_c", - vec![3u8; 8], - ) - .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_c", vec![3u8; 8]) + .await + .unwrap(); // Paginated listing should show newest first: key_c, key_b, key_a - let response = PaginatedKVStoreSync::list_paginated( + let response = PaginatedKVStore::list_paginated( &store, primary_namespace, secondary_namespace, None, ) + .await .unwrap(); assert_eq!(response.keys, vec!["key_c", "key_b", "key_a"]); diff --git a/src/io/sqlite_store/migrations.rs b/src/io/sqlite_store/migrations.rs index f596b1a42..1b4de9aa0 100644 --- a/src/io/sqlite_store/migrations.rs +++ b/src/io/sqlite_store/migrations.rs @@ -169,14 +169,14 @@ fn migrate_v2_to_v3(connection: &mut Connection, kv_table_name: &str) -> io::Res mod tests { use std::fs; - use lightning::util::persist::{KVStoreSync, PaginatedKVStoreSync}; + use lightning::util::persist::{KVStore, PaginatedKVStore}; use rusqlite::{named_params, Connection}; use crate::io::sqlite_store::SqliteStore; use crate::io::test_utils::{do_read_write_remove_list_persist, random_storage_path}; - #[test] - fn rwrl_post_schema_1_migration() { + #[tokio::test] + async fn rwrl_post_schema_1_migration() { let old_schema_version = 1; let mut temp_path = random_storage_path(); @@ -253,15 +253,15 @@ mod tests { // Check we migrate the db just fine without losing our written data. let store = SqliteStore::new(temp_path, Some(db_file_name), Some(kv_table_name)).unwrap(); - let res = store.read(&test_namespace, "", &test_key).unwrap(); + let res = KVStore::read(&store, &test_namespace, "", &test_key).await.unwrap(); assert_eq!(res, test_data); // Check we can continue to use the store just fine. - do_read_write_remove_list_persist(&store); + do_read_write_remove_list_persist(&store).await; } - #[test] - fn rwrl_post_schema_2_migration() { + #[tokio::test] + async fn rwrl_post_schema_2_migration() { let old_schema_version = 2u16; let mut temp_path = random_storage_path(); @@ -325,24 +325,24 @@ mod tests { // Verify data survived for i in 0..3 { let key = format!("key_{}", i); - let data = store.read(test_ns, test_sub, &key).unwrap(); + let data = KVStore::read(&store, test_ns, test_sub, &key).await.unwrap(); assert_eq!(data, vec![i as u8; 8]); } // Verify paginated listing works and returns entries in ROWID-backfilled order (newest first) let response = - PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap(); + PaginatedKVStore::list_paginated(&store, test_ns, test_sub, None).await.unwrap(); assert_eq!(response.keys.len(), 3); // ROWIDs were 1, 2, 3 so sort_order was backfilled as 1, 2, 3; newest first assert_eq!(response.keys, vec!["key_2", "key_1", "key_0"]); // Verify we can write new entries and they get proper ordering - KVStoreSync::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).unwrap(); + KVStore::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).await.unwrap(); let response = - PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap(); + PaginatedKVStore::list_paginated(&store, test_ns, test_sub, None).await.unwrap(); assert_eq!(response.keys[0], "key_new"); // Check we can continue to use the store just fine. - do_read_write_remove_list_persist(&store); + do_read_write_remove_list_persist(&store).await; } } diff --git a/src/io/sqlite_store/mod.rs b/src/io/sqlite_store/mod.rs index 84af03adc..076aeef9b 100644 --- a/src/io/sqlite_store/mod.rs +++ b/src/io/sqlite_store/mod.rs @@ -14,9 +14,7 @@ use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use lightning::io; -use lightning::util::persist::{ - KVStore, KVStoreSync, PageToken, PaginatedKVStore, PaginatedKVStoreSync, PaginatedListResponse, -}; +use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; use lightning_types::string::PrintableString; use rusqlite::{named_params, Connection}; @@ -41,7 +39,7 @@ const SCHEMA_USER_VERSION: u16 = 3; // The number of entries returned per page in paginated list operations. const PAGE_SIZE: usize = 50; -/// A [`KVStoreSync`] implementation that writes to and reads from an [SQLite] database. +/// A [`KVStore`] implementation that writes to and reads from an [SQLite] database. /// /// [SQLite]: https://sqlite.org pub struct SqliteStore { @@ -185,57 +183,6 @@ impl KVStore for SqliteStore { } } -impl KVStoreSync for SqliteStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - self.inner.read_internal(primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - let locking_key = self.build_locking_key(primary_namespace, secondary_namespace, key); - let (inner_lock_ref, version) = self.get_new_version_and_lock_ref(locking_key.clone()); - self.inner.write_internal( - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - buf, - ) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, _lazy: bool, - ) -> io::Result<()> { - let locking_key = self.build_locking_key(primary_namespace, secondary_namespace, key); - let (inner_lock_ref, version) = self.get_new_version_and_lock_ref(locking_key.clone()); - self.inner.remove_internal( - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - ) - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - self.inner.list_internal(primary_namespace, secondary_namespace) - } -} - -impl PaginatedKVStoreSync for SqliteStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - self.inner.list_paginated_internal(primary_namespace, secondary_namespace, page_token) - } -} - impl PaginatedKVStore for SqliteStore { fn list_paginated( &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, @@ -700,8 +647,8 @@ mod tests { } } - #[test] - fn read_write_remove_list_persist() { + #[tokio::test] + async fn read_write_remove_list_persist() { let mut temp_path = random_storage_path(); temp_path.push("read_write_remove_list_persist"); let store = SqliteStore::new( @@ -710,11 +657,11 @@ mod tests { Some("test_table".to_string()), ) .unwrap(); - do_read_write_remove_list_persist(&store); + do_read_write_remove_list_persist(&store).await; } - #[test] - fn test_sqlite_store() { + #[tokio::test(flavor = "multi_thread")] + async fn test_sqlite_store() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store"); let store_0 = SqliteStore::new( @@ -732,8 +679,8 @@ mod tests { do_test_store(&store_0, &store_1) } - #[test] - fn test_sqlite_store_paginated_listing() { + #[tokio::test] + async fn test_sqlite_store_paginated_listing() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_listing"); let store = SqliteStore::new( @@ -750,7 +697,9 @@ mod tests { for i in 0..num_entries { let key = format!("key_{:04}", i); let data = vec![i as u8; 32]; - KVStoreSync::write(&store, primary_namespace, secondary_namespace, &key, data).unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, &key, data) + .await + .unwrap(); } // Paginate through all entries and collect them @@ -759,12 +708,13 @@ mod tests { let mut page_count = 0; loop { - let response = PaginatedKVStoreSync::list_paginated( + let response = PaginatedKVStore::list_paginated( &store, primary_namespace, secondary_namespace, page_token, ) + .await .unwrap(); all_keys.extend(response.keys.clone()); @@ -795,8 +745,8 @@ mod tests { assert_eq!(all_keys[num_entries - 1], "key_0000"); } - #[test] - fn test_sqlite_store_paginated_update_preserves_order() { + #[tokio::test] + async fn test_sqlite_store_paginated_update_preserves_order() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_update"); let store = SqliteStore::new( @@ -809,37 +759,38 @@ mod tests { let primary_namespace = "test_ns"; let secondary_namespace = "test_sub"; - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "first", vec![1u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "first", vec![1u8; 8]) + .await .unwrap(); - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "second", vec![2u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "second", vec![2u8; 8]) + .await .unwrap(); - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "third", vec![3u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "third", vec![3u8; 8]) + .await .unwrap(); // Update the first entry - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "first", vec![99u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "first", vec![99u8; 8]) + .await .unwrap(); // Paginated listing should still show "first" with its original creation order - let response = PaginatedKVStoreSync::list_paginated( - &store, - primary_namespace, - secondary_namespace, - None, - ) - .unwrap(); + let response = + PaginatedKVStore::list_paginated(&store, primary_namespace, secondary_namespace, None) + .await + .unwrap(); // Newest first: third, second, first assert_eq!(response.keys, vec!["third", "second", "first"]); // Verify the updated value was persisted let data = - KVStoreSync::read(&store, primary_namespace, secondary_namespace, "first").unwrap(); + KVStore::read(&store, primary_namespace, secondary_namespace, "first").await.unwrap(); assert_eq!(data, vec![99u8; 8]); } - #[test] - fn test_sqlite_store_paginated_empty_namespace() { + #[tokio::test] + async fn test_sqlite_store_paginated_empty_namespace() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_empty"); let store = SqliteStore::new( @@ -851,13 +802,13 @@ mod tests { // Paginating an empty or unknown namespace returns an empty result with no token. let response = - PaginatedKVStoreSync::list_paginated(&store, "nonexistent", "ns", None).unwrap(); + PaginatedKVStore::list_paginated(&store, "nonexistent", "ns", None).await.unwrap(); assert!(response.keys.is_empty()); assert!(response.next_page_token.is_none()); } - #[test] - fn test_sqlite_store_paginated_namespace_isolation() { + #[tokio::test] + async fn test_sqlite_store_paginated_namespace_isolation() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_isolation"); let store = SqliteStore::new( @@ -867,27 +818,28 @@ mod tests { ) .unwrap(); - KVStoreSync::write(&store, "ns_a", "sub", "key_1", vec![1u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_a", "sub", "key_2", vec![2u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_b", "sub", "key_3", vec![3u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_a", "other", "key_4", vec![4u8; 8]).unwrap(); + KVStore::write(&store, "ns_a", "sub", "key_1", vec![1u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_a", "sub", "key_2", vec![2u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_b", "sub", "key_3", vec![3u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_a", "other", "key_4", vec![4u8; 8]).await.unwrap(); // ns_a/sub should only contain key_1 and key_2 (newest first). - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_a", "sub", None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, "ns_a", "sub", None).await.unwrap(); assert_eq!(response.keys, vec!["key_2", "key_1"]); assert!(response.next_page_token.is_none()); // ns_b/sub should only contain key_3. - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_b", "sub", None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, "ns_b", "sub", None).await.unwrap(); assert_eq!(response.keys, vec!["key_3"]); // ns_a/other should only contain key_4. - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_a", "other", None).unwrap(); + let response = + PaginatedKVStore::list_paginated(&store, "ns_a", "other", None).await.unwrap(); assert_eq!(response.keys, vec!["key_4"]); } - #[test] - fn test_sqlite_store_paginated_removal() { + #[tokio::test] + async fn test_sqlite_store_paginated_removal() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_removal"); let store = SqliteStore::new( @@ -900,19 +852,19 @@ mod tests { let ns = "test_ns"; let sub = "test_sub"; - KVStoreSync::write(&store, ns, sub, "a", vec![1u8; 8]).unwrap(); - KVStoreSync::write(&store, ns, sub, "b", vec![2u8; 8]).unwrap(); - KVStoreSync::write(&store, ns, sub, "c", vec![3u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, "a", vec![1u8; 8]).await.unwrap(); + KVStore::write(&store, ns, sub, "b", vec![2u8; 8]).await.unwrap(); + KVStore::write(&store, ns, sub, "c", vec![3u8; 8]).await.unwrap(); - KVStoreSync::remove(&store, ns, sub, "b", false).unwrap(); + KVStore::remove(&store, ns, sub, "b", false).await.unwrap(); - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys, vec!["c", "a"]); assert!(response.next_page_token.is_none()); } - #[test] - fn test_sqlite_store_paginated_exact_page_boundary() { + #[tokio::test] + async fn test_sqlite_store_paginated_exact_page_boundary() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_boundary"); let store = SqliteStore::new( @@ -928,30 +880,30 @@ mod tests { // Write exactly PAGE_SIZE entries (50). for i in 0..PAGE_SIZE { let key = format!("key_{:04}", i); - KVStoreSync::write(&store, ns, sub, &key, vec![i as u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, &key, vec![i as u8; 8]).await.unwrap(); } // Exactly PAGE_SIZE entries: all returned in one page with no next-page token. - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), PAGE_SIZE); assert!(response.next_page_token.is_none()); // Add one more entry (PAGE_SIZE + 1 total). First page should now have a token. - KVStoreSync::write(&store, ns, sub, "key_extra", vec![0u8; 8]).unwrap(); - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + KVStore::write(&store, ns, sub, "key_extra", vec![0u8; 8]).await.unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), PAGE_SIZE); assert!(response.next_page_token.is_some()); // Second page should have exactly 1 entry and no token. - let response = - PaginatedKVStoreSync::list_paginated(&store, ns, sub, response.next_page_token) - .unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, response.next_page_token) + .await + .unwrap(); assert_eq!(response.keys.len(), 1); assert!(response.next_page_token.is_none()); } - #[test] - fn test_sqlite_store_paginated_fewer_than_page_size() { + #[tokio::test] + async fn test_sqlite_store_paginated_fewer_than_page_size() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_few"); let store = SqliteStore::new( @@ -967,10 +919,10 @@ mod tests { // Write fewer entries than PAGE_SIZE. for i in 0..5 { let key = format!("key_{}", i); - KVStoreSync::write(&store, ns, sub, &key, vec![i as u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, &key, vec![i as u8; 8]).await.unwrap(); } - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), 5); // Fewer than PAGE_SIZE means no next page. assert!(response.next_page_token.is_none()); @@ -978,8 +930,8 @@ mod tests { assert_eq!(response.keys, vec!["key_4", "key_3", "key_2", "key_1", "key_0"]); } - #[test] - fn test_sqlite_store_write_version_persists_across_restart() { + #[tokio::test] + async fn test_sqlite_store_write_version_persists_across_restart() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_write_version_restart"); @@ -994,22 +946,12 @@ mod tests { ) .unwrap(); - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_a", - vec![1u8; 8], - ) - .unwrap(); - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_b", - vec![2u8; 8], - ) - .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_a", vec![1u8; 8]) + .await + .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_b", vec![2u8; 8]) + .await + .unwrap(); // Don't drop/cleanup since we want to reopen std::mem::forget(store); @@ -1024,22 +966,18 @@ mod tests { ) .unwrap(); - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_c", - vec![3u8; 8], - ) - .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_c", vec![3u8; 8]) + .await + .unwrap(); // Paginated listing should show newest first: key_c, key_b, key_a - let response = PaginatedKVStoreSync::list_paginated( + let response = PaginatedKVStore::list_paginated( &store, primary_namespace, secondary_namespace, None, ) + .await .unwrap(); assert_eq!(response.keys, vec!["key_c", "key_b", "key_a"]); diff --git a/src/io/test_utils.rs b/src/io/test_utils.rs index 685377853..aadb4b79a 100644 --- a/src/io/test_utils.rs +++ b/src/io/test_utils.rs @@ -5,6 +5,7 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. +use std::future::Future; use std::panic::RefUnwindSafe; use std::path::PathBuf; use std::sync::Arc; @@ -20,7 +21,7 @@ use lightning::ln::functional_test_utils::{ TestChanMonCfg, }; use lightning::util::persist::{ - KVStore, KVStoreSync, MonitorName, ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + KVStore, MonitorName, ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, KVSTORE_NAMESPACE_KEY_MAX_LEN, @@ -168,7 +169,32 @@ pub(crate) fn random_storage_path() -> PathBuf { temp_path } -pub(crate) fn do_read_write_remove_list_persist(kv_store: &K) { +async fn catch_future_unwind(future: F) -> std::thread::Result { + let mut future = std::pin::pin!(future); + std::future::poll_fn(|cx| { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| future.as_mut().poll(cx))) { + Ok(std::task::Poll::Ready(output)) => std::task::Poll::Ready(Ok(output)), + Ok(std::task::Poll::Pending) => std::task::Poll::Pending, + Err(panic) => std::task::Poll::Ready(Err(panic)), + } + }) + .await +} + +async fn assert_invalid_write_fails( + kv_store: &K, primary_namespace: &str, secondary_namespace: &str, key: &str, data: Vec, +) { + let res = std::panic::catch_unwind(|| { + KVStore::write(kv_store, primary_namespace, secondary_namespace, key, data) + }); + if let Ok(fut) = res { + if let Ok(write_res) = catch_future_unwind(fut).await { + assert!(write_res.is_err()); + } + } +} + +pub(crate) async fn do_read_write_remove_list_persist(kv_store: &K) { let data = vec![42u8; 32]; let primary_namespace = "testspace"; @@ -176,45 +202,46 @@ pub(crate) fn do_read_write_remove_list_persist( let key = "testkey"; // Test the basic KVStore operations. - kv_store.write(primary_namespace, secondary_namespace, key, data.clone()).unwrap(); + KVStore::write(kv_store, primary_namespace, secondary_namespace, key, data.clone()) + .await + .unwrap(); // Test empty primary/secondary namespaces are allowed, but not empty primary namespace and non-empty // secondary primary_namespace, and not empty key. - kv_store.write("", "", key, data.clone()).unwrap(); - let res = - std::panic::catch_unwind(|| kv_store.write("", secondary_namespace, key, data.clone())); - assert!(res.is_err()); - let res = std::panic::catch_unwind(|| { - kv_store.write(primary_namespace, secondary_namespace, "", data.clone()) - }); - assert!(res.is_err()); + KVStore::write(kv_store, "", "", key, data.clone()).await.unwrap(); + assert_invalid_write_fails(kv_store, "", secondary_namespace, key, data.clone()).await; + assert_invalid_write_fails(kv_store, primary_namespace, secondary_namespace, "", data.clone()) + .await; - let listed_keys = kv_store.list(primary_namespace, secondary_namespace).unwrap(); + let listed_keys = + KVStore::list(kv_store, primary_namespace, secondary_namespace).await.unwrap(); assert_eq!(listed_keys.len(), 1); assert_eq!(listed_keys[0], key); - let read_data = kv_store.read(primary_namespace, secondary_namespace, key).unwrap(); + let read_data = + KVStore::read(kv_store, primary_namespace, secondary_namespace, key).await.unwrap(); assert_eq!(data, &*read_data); - kv_store.remove(primary_namespace, secondary_namespace, key, false).unwrap(); + KVStore::remove(kv_store, primary_namespace, secondary_namespace, key, false).await.unwrap(); - let listed_keys = kv_store.list(primary_namespace, secondary_namespace).unwrap(); + let listed_keys = + KVStore::list(kv_store, primary_namespace, secondary_namespace).await.unwrap(); assert_eq!(listed_keys.len(), 0); // Ensure we have no issue operating with primary_namespace/secondary_namespace/key being KVSTORE_NAMESPACE_KEY_MAX_LEN let max_chars: String = std::iter::repeat('A').take(KVSTORE_NAMESPACE_KEY_MAX_LEN).collect(); - kv_store.write(&max_chars, &max_chars, &max_chars, data.clone()).unwrap(); + KVStore::write(kv_store, &max_chars, &max_chars, &max_chars, data.clone()).await.unwrap(); - let listed_keys = kv_store.list(&max_chars, &max_chars).unwrap(); + let listed_keys = KVStore::list(kv_store, &max_chars, &max_chars).await.unwrap(); assert_eq!(listed_keys.len(), 1); assert_eq!(listed_keys[0], max_chars); - let read_data = kv_store.read(&max_chars, &max_chars, &max_chars).unwrap(); + let read_data = KVStore::read(kv_store, &max_chars, &max_chars, &max_chars).await.unwrap(); assert_eq!(data, &*read_data); - kv_store.remove(&max_chars, &max_chars, &max_chars, false).unwrap(); + KVStore::remove(kv_store, &max_chars, &max_chars, &max_chars, false).await.unwrap(); - let listed_keys = kv_store.list(&max_chars, &max_chars).unwrap(); + let listed_keys = KVStore::list(kv_store, &max_chars, &max_chars).await.unwrap(); assert_eq!(listed_keys.len(), 0); } diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs index 97883b5d5..e146bfc39 100644 --- a/src/io/vss_store.rs +++ b/src/io/vss_store.rs @@ -24,7 +24,7 @@ use bitcoin::Network; use lightning::impl_writeable_tlv_based_enum; use lightning::io::{self, Error, ErrorKind}; use lightning::sign::{EntropySource as LdkEntropySource, RandomBytes}; -use lightning::util::persist::{KVStore, KVStoreSync}; +use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, Writeable}; use prost::Message; use vss_client::client::VssClient; @@ -77,7 +77,7 @@ const VSS_SCHEMA_VERSION_KEY: &str = "vss_schema_version"; // would hit a blocking case const INTERNAL_RUNTIME_WORKERS: usize = 2; -/// A [`KVStore`]/[`KVStoreSync`] implementation that writes to and reads from a [VSS] backend. +/// A [`KVStore`] implementation that writes to and reads from a [VSS] backend. /// /// [VSS]: https://github.com/lightningdevkit/vss-server/blob/main/README.md pub struct VssStore { @@ -149,7 +149,6 @@ impl VssStore { let inner = Arc::new(VssStoreInner::new( schema_version, - blocking_client, async_client, store_id, data_encryption_key, @@ -193,111 +192,6 @@ impl VssStore { } } -impl KVStoreSync for VssStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - let internal_runtime = self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal runtime"); - let msg = format!("Failed to access internal runtime"); - Error::new(ErrorKind::Other, msg) - })?; - let primary_namespace = primary_namespace.to_string(); - let secondary_namespace = secondary_namespace.to_string(); - let key = key.to_string(); - let inner = Arc::clone(&self.inner); - let fut = async move { - inner - .read_internal(&inner.blocking_client, primary_namespace, secondary_namespace, key) - .await - }; - tokio::task::block_in_place(move || internal_runtime.block_on(fut)) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - let internal_runtime = self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal runtime"); - let msg = format!("Failed to access internal runtime"); - Error::new(ErrorKind::Other, msg) - })?; - let primary_namespace = primary_namespace.to_string(); - let secondary_namespace = secondary_namespace.to_string(); - let key = key.to_string(); - let inner = Arc::clone(&self.inner); - let locking_key = self.build_locking_key(&primary_namespace, &secondary_namespace, &key); - let (inner_lock_ref, version) = self.get_new_version_and_lock_ref(locking_key.clone()); - let fut = async move { - inner - .write_internal( - &inner.blocking_client, - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - buf, - ) - .await - }; - tokio::task::block_in_place(move || internal_runtime.block_on(fut)) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> io::Result<()> { - let internal_runtime = self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal runtime"); - let msg = format!("Failed to access internal runtime"); - Error::new(ErrorKind::Other, msg) - })?; - let primary_namespace = primary_namespace.to_string(); - let secondary_namespace = secondary_namespace.to_string(); - let key = key.to_string(); - let inner = Arc::clone(&self.inner); - let locking_key = self.build_locking_key(&primary_namespace, &secondary_namespace, &key); - let (inner_lock_ref, version) = self.get_new_version_and_lock_ref(locking_key.clone()); - let fut = async move { - inner - .remove_internal( - &inner.blocking_client, - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - ) - .await - }; - if lazy { - internal_runtime.spawn(async { fut.await }); - Ok(()) - } else { - tokio::task::block_in_place(move || internal_runtime.block_on(fut)) - } - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - let internal_runtime = self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal runtime"); - let msg = format!("Failed to access internal runtime"); - Error::new(ErrorKind::Other, msg) - })?; - let primary_namespace = primary_namespace.to_string(); - let secondary_namespace = secondary_namespace.to_string(); - let inner = Arc::clone(&self.inner); - let fut = async move { - inner - .list_internal(&inner.blocking_client, primary_namespace, secondary_namespace) - .await - }; - tokio::task::block_in_place(move || internal_runtime.block_on(fut)) - } -} - impl KVStore for VssStore { fn read( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, @@ -388,7 +282,6 @@ impl Drop for VssStore { struct VssStoreInner { schema_version: VssSchemaVersion, - blocking_client: VssClient, // A secondary client that will only be used for async persistence via `KVStore`, to ensure TCP // connections aren't shared between our outer and the internal runtime. async_client: VssClient, @@ -403,14 +296,13 @@ struct VssStoreInner { impl VssStoreInner { pub(crate) fn new( - schema_version: VssSchemaVersion, blocking_client: VssClient, - async_client: VssClient, store_id: String, - data_encryption_key: [u8; 32], key_obfuscator: KeyObfuscator, entropy_source: RandomBytes, + schema_version: VssSchemaVersion, async_client: VssClient, + store_id: String, data_encryption_key: [u8; 32], key_obfuscator: KeyObfuscator, + entropy_source: RandomBytes, ) -> Self { let locks = Mutex::new(HashMap::new()); Self { schema_version, - blocking_client, async_client, store_id, data_encryption_key, @@ -1026,8 +918,8 @@ mod tests { use super::*; use crate::io::test_utils::do_read_write_remove_list_persist; - #[test] - fn vss_read_write_remove_list_persist() { + #[tokio::test] + async fn vss_read_write_remove_list_persist() { let vss_base_url = std::env::var("TEST_VSS_BASE_URL").unwrap(); let mut rng = rng(); let rand_store_id: String = (0..7).map(|_| rng.sample(Alphanumeric) as char).collect(); @@ -1038,7 +930,7 @@ mod tests { VssStoreBuilder::new(entropy, vss_base_url, rand_store_id, Network::Testnet) .build_with_sigs_auth(HashMap::new()) .unwrap(); - do_read_write_remove_list_persist(&vss_store); + do_read_write_remove_list_persist(&vss_store).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -1054,7 +946,7 @@ mod tests { .build_with_sigs_auth(HashMap::new()) .unwrap(); - do_read_write_remove_list_persist(&vss_store); + do_read_write_remove_list_persist(&vss_store).await; drop(vss_store) } } diff --git a/src/lib.rs b/src/lib.rs index 2ee74ecc5..7465dfabf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -156,7 +156,7 @@ use lightning::ln::msgs::{BaseMessageHandler, SocketAddress}; use lightning::ln::peer_handler::CustomMessageHandler; use lightning::routing::gossip::NodeAlias; use lightning::sign::EntropySource; -use lightning::util::persist::KVStoreSync; +use lightning::util::persist::KVStore; use lightning::util::wallet_utils::{Input, Wallet as LdkWallet}; use lightning_background_processor::process_events_async; pub use lightning_invoice; @@ -180,7 +180,7 @@ use types::{ HRNResolver, KeysManager, OnionMessenger, PaymentStore, PeerManager, Router, Scorer, Sweeper, Wallet, }; -pub use types::{ChannelDetails, CustomTlvRecord, PeerDetails, SyncAndAsyncKVStore, UserChannelId}; +pub use types::{ChannelDetails, CustomTlvRecord, PeerDetails, UserChannelId}; pub use vss_client; use crate::scoring::setup_background_pathfinding_scores_sync; @@ -2062,20 +2062,21 @@ impl Node { /// Exports the current state of the scorer. The result can be shared with and merged by light nodes that only have /// a limited view of the network. pub fn export_pathfinding_scores(&self) -> Result, Error> { - KVStoreSync::read( - &*self.kv_store, - lightning::util::persist::SCORER_PERSISTENCE_PRIMARY_NAMESPACE, - lightning::util::persist::SCORER_PERSISTENCE_SECONDARY_NAMESPACE, - lightning::util::persist::SCORER_PERSISTENCE_KEY, - ) - .map_err(|e| { - log_error!( - self.logger, - "Failed to access store while exporting pathfinding scores: {}", - e - ); - Error::PersistenceFailed - }) + self.runtime + .block_on(KVStore::read( + &*self.kv_store, + lightning::util::persist::SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + lightning::util::persist::SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + lightning::util::persist::SCORER_PERSISTENCE_KEY, + )) + .map_err(|e| { + log_error!( + self.logger, + "Failed to access store while exporting pathfinding scores: {}", + e + ); + Error::PersistenceFailed + }) } /// Return the features used in node announcement. diff --git a/src/types.rs b/src/types.rs index 06e65fbd0..64209430b 100644 --- a/src/types.rs +++ b/src/types.rs @@ -29,7 +29,7 @@ use lightning::routing::gossip; use lightning::routing::router::DefaultRouter; use lightning::routing::scoring::{CombinedScorer, ProbabilisticScoringFeeParameters}; use lightning::sign::InMemorySigner; -use lightning::util::persist::{KVStore, KVStoreSync, MonitorUpdatingPersisterAsync}; +use lightning::util::persist::{KVStore, MonitorUpdatingPersisterAsync}; use lightning::util::ser::{Readable, Writeable, Writer}; use lightning::util::sweep::OutputSweeper; use lightning_block_sync::gossip::GossipVerifier; @@ -46,17 +46,6 @@ use crate::message_handler::NodeCustomMessageHandler; use crate::payment::{PaymentDetails, PendingPaymentDetails}; use crate::runtime::RuntimeSpawner; -/// A supertrait that requires that a type implements both [`KVStore`] and [`KVStoreSync`] at the -/// same time. -pub trait SyncAndAsyncKVStore: KVStore + KVStoreSync {} - -impl SyncAndAsyncKVStore for T -where - T: KVStore, - T: KVStoreSync, -{ -} - pub(crate) trait DynStoreTrait: Send + Sync { fn read_async( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, @@ -70,19 +59,6 @@ pub(crate) trait DynStoreTrait: Send + Sync { fn list_async( &self, primary_namespace: &str, secondary_namespace: &str, ) -> Pin, bitcoin::io::Error>> + Send + 'static>>; - - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> Result, bitcoin::io::Error>; - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> Result<(), bitcoin::io::Error>; - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> Result<(), bitcoin::io::Error>; - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> Result, bitcoin::io::Error>; } impl<'a> KVStore for dyn DynStoreTrait + 'a { @@ -111,32 +87,6 @@ impl<'a> KVStore for dyn DynStoreTrait + 'a { } } -impl<'a> KVStoreSync for dyn DynStoreTrait + 'a { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> Result, bitcoin::io::Error> { - DynStoreTrait::read(self, primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> Result<(), bitcoin::io::Error> { - DynStoreTrait::write(self, primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> Result<(), bitcoin::io::Error> { - DynStoreTrait::remove(self, primary_namespace, secondary_namespace, key, lazy) - } - - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> Result, bitcoin::io::Error> { - DynStoreTrait::list(self, primary_namespace, secondary_namespace) - } -} - pub(crate) type DynStore = dyn DynStoreTrait; // Newtype wrapper that implements `KVStore` for `Arc`. This is needed because `KVStore` @@ -172,9 +122,9 @@ impl KVStore for DynStoreRef { } } -pub(crate) struct DynStoreWrapper(pub(crate) T); +pub(crate) struct DynStoreWrapper(pub(crate) T); -impl DynStoreTrait for DynStoreWrapper { +impl DynStoreTrait for DynStoreWrapper { fn read_async( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> Pin, bitcoin::io::Error>> + Send + 'static>> { @@ -198,30 +148,6 @@ impl DynStoreTrait for DynStoreWrapper ) -> Pin, bitcoin::io::Error>> + Send + 'static>> { Box::pin(KVStore::list(&self.0, primary_namespace, secondary_namespace)) } - - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> Result, bitcoin::io::Error> { - KVStoreSync::read(&self.0, primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> Result<(), bitcoin::io::Error> { - KVStoreSync::write(&self.0, primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> Result<(), bitcoin::io::Error> { - KVStoreSync::remove(&self.0, primary_namespace, secondary_namespace, key, lazy) - } - - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> Result, bitcoin::io::Error> { - KVStoreSync::list(&self.0, primary_namespace, secondary_namespace) - } } pub(crate) type AsyncPersister = MonitorUpdatingPersisterAsync< diff --git a/tests/common/mod.rs b/tests/common/mod.rs index f2e7191ae..d7775e67b 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -50,7 +50,7 @@ use ldk_node::{ use lightning::io; use lightning::ln::msgs::SocketAddress; use lightning::routing::gossip::NodeAlias; -use lightning::util::persist::{KVStore, KVStoreSync}; +use lightning::util::persist::KVStore; use lightning_invoice::{Bolt11InvoiceDescription, Description}; use lightning_persister::fs_store::v1::FilesystemStore; use lightning_types::payment::{PaymentHash, PaymentPreimage}; @@ -1700,32 +1700,6 @@ impl KVStore for TestSyncStore { } } -impl KVStoreSync for TestSyncStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> lightning::io::Result> { - self.inner.read_internal(primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> lightning::io::Result<()> { - self.inner.write_internal(primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> lightning::io::Result<()> { - self.inner.remove_internal(primary_namespace, secondary_namespace, key, lazy) - } - - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> lightning::io::Result> { - self.inner.list_internal(primary_namespace, secondary_namespace) - } -} - struct TestSyncStoreInner { serializer: tokio::sync::RwLock<()>, test_store: InMemoryStore, @@ -1751,36 +1725,6 @@ impl TestSyncStoreInner { Self { serializer, fs_store, sqlite_store, test_store } } - fn do_list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> lightning::io::Result> { - let fs_res = KVStoreSync::list(&self.fs_store, primary_namespace, secondary_namespace); - let sqlite_res = - KVStoreSync::list(&self.sqlite_store, primary_namespace, secondary_namespace); - let test_res = KVStoreSync::list(&self.test_store, primary_namespace, secondary_namespace); - - match fs_res { - Ok(mut list) => { - list.sort(); - - let mut sqlite_list = sqlite_res.unwrap(); - sqlite_list.sort(); - assert_eq!(list, sqlite_list); - - let mut test_list = test_res.unwrap(); - test_list.sort(); - assert_eq!(list, test_list); - - Ok(list) - }, - Err(e) => { - assert!(sqlite_res.is_err()); - assert!(test_res.is_err()); - Err(e) - }, - } - } - async fn do_list_async( &self, primary_namespace: &str, secondary_namespace: &str, ) -> lightning::io::Result> { @@ -1929,123 +1873,4 @@ impl TestSyncStoreInner { }, } } - - fn read_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> lightning::io::Result> { - let _guard = self.serializer.blocking_read(); - - let fs_res = KVStoreSync::read(&self.fs_store, primary_namespace, secondary_namespace, key); - let sqlite_res = - KVStoreSync::read(&self.sqlite_store, primary_namespace, secondary_namespace, key); - let test_res = - KVStoreSync::read(&self.test_store, primary_namespace, secondary_namespace, key); - - match fs_res { - Ok(read) => { - assert_eq!(read, sqlite_res.unwrap()); - assert_eq!(read, test_res.unwrap()); - Ok(read) - }, - Err(e) => { - assert!(sqlite_res.is_err()); - assert_eq!(e.kind(), unsafe { sqlite_res.unwrap_err_unchecked().kind() }); - assert!(test_res.is_err()); - assert_eq!(e.kind(), unsafe { test_res.unwrap_err_unchecked().kind() }); - Err(e) - }, - } - } - - fn write_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> lightning::io::Result<()> { - let _guard = self.serializer.blocking_write(); - let fs_res = KVStoreSync::write( - &self.fs_store, - primary_namespace, - secondary_namespace, - key, - buf.clone(), - ); - let sqlite_res = KVStoreSync::write( - &self.sqlite_store, - primary_namespace, - secondary_namespace, - key, - buf.clone(), - ); - let test_res = KVStoreSync::write( - &self.test_store, - primary_namespace, - secondary_namespace, - key, - buf.clone(), - ); - - assert!(self - .do_list(primary_namespace, secondary_namespace) - .unwrap() - .contains(&key.to_string())); - - match fs_res { - Ok(()) => { - assert!(sqlite_res.is_ok()); - assert!(test_res.is_ok()); - Ok(()) - }, - Err(e) => { - assert!(sqlite_res.is_err()); - assert!(test_res.is_err()); - Err(e) - }, - } - } - - fn remove_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> lightning::io::Result<()> { - let _guard = self.serializer.blocking_write(); - let fs_res = - KVStoreSync::remove(&self.fs_store, primary_namespace, secondary_namespace, key, lazy); - let sqlite_res = KVStoreSync::remove( - &self.sqlite_store, - primary_namespace, - secondary_namespace, - key, - lazy, - ); - let test_res = KVStoreSync::remove( - &self.test_store, - primary_namespace, - secondary_namespace, - key, - lazy, - ); - - assert!(!self - .do_list(primary_namespace, secondary_namespace) - .unwrap() - .contains(&key.to_string())); - - match fs_res { - Ok(()) => { - assert!(sqlite_res.is_ok()); - assert!(test_res.is_ok()); - Ok(()) - }, - Err(e) => { - assert!(sqlite_res.is_err()); - assert!(test_res.is_err()); - Err(e) - }, - } - } - - fn list_internal( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> lightning::io::Result> { - let _guard = self.serializer.blocking_read(); - self.do_list(primary_namespace, secondary_namespace) - } } From 431be00c4d566f6152724fdb9223a3070ff286ae Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 5 Jun 2026 09:40:59 +0200 Subject: [PATCH 13/17] Add shared store runtime wrapper Add a crate-local runtime wrapper for store backends that need to keep their I/O isolated while shutting down safely from async contexts. Co-Authored-By: HAL 9000 --- src/runtime.rs | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/src/runtime.rs b/src/runtime.rs index 1d8eb32b0..030a01bb8 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -6,6 +6,8 @@ // accordance with one or both of these licenses. use std::future::Future; +use std::io; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -223,6 +225,90 @@ enum RuntimeMode { Handle(tokio::runtime::Handle), } +/// Runtime used by async store backends while ldk-node still exposes synchronous APIs. +/// +/// This is a temporary bridge for store implementations that need Tokio-driven I/O, such as VSS +/// and PostgreSQL. Many public ldk-node methods are still synchronous, so they call +/// [`Runtime::block_on`] when they need to wait for async persistence. If that persistence work is +/// driven by the same Tokio runtime as the synchronous caller, a blocking call can deadlock in a +/// narrow but realistic scheduler state. +/// +/// The failure mode is that `block_on` parks the current worker with `block_in_place` while it +/// waits for an async store operation. Suppose that store operation is waiting for an I/O future, +/// and the connection driver or I/O driver task that can make the future progress is assigned to +/// the same worker thread that just entered `block_in_place`. The blocked sync caller is waiting +/// for the persistence future to complete, while the persistence future is waiting for an I/O task +/// that cannot be polled because its worker is occupied by the blocking caller. With no worker +/// driving that I/O resource, neither side can make progress. +/// +/// A simple example is a synchronous node API calling `block_on(store.write(...))` for a +/// tokio-postgres-backed store. The write future may wait for the postgres connection task or +/// socket readiness. If the runtime worker that should poll that connection task is also the +/// worker currently blocked in the synchronous API, the write cannot complete, and the synchronous +/// API cannot unblock. +/// +/// `StoreRuntime` gives each such store backend its own small runtime, workers, and I/O driver. +/// Synchronous node APIs may still block the node runtime while waiting for persistence, but the +/// persistence tasks they wait on are driven independently and can continue polling sockets and +/// connection drivers. +/// +/// Once ldk-node switches the remaining store-backed APIs to be fully async, callers will await +/// persistence directly and these `block_on` bridges will be disallowed. At that point the store +/// runtimes should be removed again and store I/O can run on the node runtime directly. +pub(crate) struct StoreRuntime { + runtime: Option, +} + +impl StoreRuntime { + pub(crate) fn new( + thread_name_prefix: &'static str, worker_threads: usize, runtime_name: &'static str, + ) -> io::Result { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .thread_name_fn(move || { + static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); + let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); + format!("{}-{}", thread_name_prefix, id) + }) + .worker_threads(worker_threads) + .max_blocking_threads(worker_threads) + .build() + .map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("Failed to build {runtime_name} runtime: {e}"), + ) + })?; + Ok(Self { runtime: Some(runtime) }) + } + + pub(crate) fn handle(&self) -> &tokio::runtime::Handle { + self.runtime.as_ref().expect("store runtime must be available").handle() + } + + pub(crate) fn spawn(&self, future: F) -> JoinHandle + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + self.handle().spawn(future) + } + + pub(crate) fn shutdown_background(mut self) { + if let Some(runtime) = self.runtime.take() { + runtime.shutdown_background(); + } + } +} + +impl Drop for StoreRuntime { + fn drop(&mut self) { + if let Some(runtime) = self.runtime.take() { + runtime.shutdown_background(); + } + } +} + pub(crate) struct RuntimeSpawner { runtime: Arc, } From 0359c225635016e94a2b2ed00376b0bc71ce5841 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 4 Jun 2026 17:55:51 +0200 Subject: [PATCH 14/17] Isolate VSS persistence from the node runtime Co-Authored-By: HAL 9000 --- src/io/vss_store.rs | 135 ++++++++++++++++++++++++++------------------ 1 file changed, 79 insertions(+), 56 deletions(-) diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs index e146bfc39..6c3535627 100644 --- a/src/io/vss_store.rs +++ b/src/io/vss_store.rs @@ -13,7 +13,7 @@ use std::fmt; use std::future::Future; #[cfg(test)] use std::panic::RefUnwindSafe; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -45,6 +45,7 @@ use vss_client::util::storable_builder::{EntropySource, StorableBuilder}; use crate::entropy::NodeEntropy; use crate::io::utils::check_namespace_key_validity; use crate::lnurl_auth::LNURL_AUTH_HARDENED_CHILD_INDEX; +use crate::runtime::StoreRuntime; type CustomRetryPolicy = FilteredRetryPolicy< JitteredRetryPolicy< @@ -85,13 +86,8 @@ pub struct VssStore { // Version counter to ensure that writes are applied in the correct order. It is assumed that read and list // operations aren't sensitive to the order of execution. next_version: AtomicU64, - // A VSS-internal runtime we use to avoid any deadlocks we could hit when waiting on a spawned - // blocking task to finish while the blocked thread had acquired the reactor. In particular, - // this works around a previously-hit case where a concurrent call to - // `PeerManager::process_pending_events` -> `ChannelManager::get_and_clear_pending_msg_events` - // would deadlock when trying to acquire sync `Mutex` locks that are held by the thread - // currently being blocked waiting on the VSS operation to finish. - internal_runtime: Option, + // A VSS-internal runtime that drives VSS I/O independently from the node runtime. + internal_runtime: Option>, } impl VssStore { @@ -100,52 +96,46 @@ impl VssStore { header_provider: Arc, ) -> io::Result { let next_version = AtomicU64::new(1); - let internal_runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .thread_name_fn(|| { - static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); - let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); - format!("ldk-node-vss-runtime-{}", id) - }) - .worker_threads(INTERNAL_RUNTIME_WORKERS) - .max_blocking_threads(INTERNAL_RUNTIME_WORKERS) - .build() - .map_err(|e| { - io::Error::new(io::ErrorKind::Other, format!("Failed to build VSS runtime: {}", e)) - })?; + let internal_runtime = + Arc::new(StoreRuntime::new("ldk-node-vss-runtime", INTERNAL_RUNTIME_WORKERS, "VSS")?); let (data_encryption_key, obfuscation_master_key) = derive_data_encryption_and_obfuscation_keys(&vss_seed); let key_obfuscator = KeyObfuscator::new(obfuscation_master_key); + let setup_key_obfuscator = KeyObfuscator::new(obfuscation_master_key); let mut entropy_seed = [0u8; 32]; getrandom::fill(&mut entropy_seed).expect("Failed to generate random bytes"); let entropy_source = RandomBytes::new(entropy_seed); + let setup_entropy_source = RandomBytes::new(entropy_seed); - let sync_retry_policy = retry_policy(); - let blocking_client = VssClient::new_with_headers( + let setup_retry_policy = retry_policy(); + let setup_client = VssClient::new_with_headers( base_url.clone(), - sync_retry_policy, - header_provider.clone(), + setup_retry_policy, + Arc::clone(&header_provider), ); - let runtime_handle = internal_runtime.handle(); - let schema_version = tokio::task::block_in_place(|| { + let async_retry_policy = retry_policy(); + let async_client = + VssClient::new_with_headers(base_url, async_retry_policy, header_provider); + + let setup_store_id = store_id.clone(); + let runtime_handle = internal_runtime.handle().clone(); + let schema_version = std::thread::spawn(move || { runtime_handle.block_on(async { determine_and_write_schema_version( - &blocking_client, - &store_id, + &setup_client, + &setup_store_id, data_encryption_key, - &key_obfuscator, - &entropy_source, + &setup_key_obfuscator, + &setup_entropy_source, ) .await }) - })?; - - let async_retry_policy = retry_policy(); - let async_client = - VssClient::new_with_headers(base_url, async_retry_policy, header_provider); + }) + .join() + .map_err(|_| io::Error::new(io::ErrorKind::Other, "VSS schema setup task panicked"))??; let inner = Arc::new(VssStoreInner::new( schema_version, @@ -158,6 +148,10 @@ impl VssStore { Ok(Self { inner, next_version, internal_runtime: Some(internal_runtime) }) } + + fn internal_runtime(&self) -> Arc { + Arc::clone(self.internal_runtime.as_ref().expect("VSS runtime must be available")) + } /// Returns a [`VssStoreBuilder`] allowing to build a [`VssStore`]. pub fn builder( node_entropy: NodeEntropy, vss_url: String, store_id: String, network: Network, @@ -200,10 +194,16 @@ impl KVStore for VssStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .read_internal(&inner.async_client, primary_namespace, secondary_namespace, key) - .await + let task = runtime.spawn(async move { + inner + .read_internal(&inner.async_client, primary_namespace, secondary_namespace, key) + .await + }); + task.await.map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("VSS runtime task failed: {}", e)) + })? } } fn write( @@ -215,19 +215,25 @@ impl KVStore for VssStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .write_internal( - &inner.async_client, - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - buf, - ) - .await + let task = runtime.spawn(async move { + inner + .write_internal( + &inner.async_client, + inner_lock_ref, + locking_key, + version, + primary_namespace, + secondary_namespace, + key, + buf, + ) + .await + }); + task.await.map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("VSS runtime task failed: {}", e)) + })? } } fn remove( @@ -239,6 +245,7 @@ impl KVStore for VssStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); let fut = async move { inner .remove_internal( @@ -254,10 +261,15 @@ impl KVStore for VssStore { }; async move { if lazy { - tokio::task::spawn(async move { fut.await }); + runtime.spawn(async move { + let _ = fut.await; + }); Ok(()) } else { - fut.await + let task = runtime.spawn(fut); + task.await.map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("VSS runtime task failed: {}", e)) + })? } } } @@ -267,16 +279,27 @@ impl KVStore for VssStore { let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner.list_internal(&inner.async_client, primary_namespace, secondary_namespace).await + let task = runtime.spawn(async move { + inner + .list_internal(&inner.async_client, primary_namespace, secondary_namespace) + .await + }); + task.await.map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("VSS runtime task failed: {}", e)) + })? } } } impl Drop for VssStore { fn drop(&mut self) { - let internal_runtime = self.internal_runtime.take(); - tokio::task::block_in_place(move || drop(internal_runtime)); + if let Some(runtime) = self.internal_runtime.take() { + if let Ok(runtime) = Arc::try_unwrap(runtime) { + runtime.shutdown_background(); + } + } } } From 81ba551a10a4c57588602d8171a3d44695133ce3 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 4 Jun 2026 17:58:56 +0200 Subject: [PATCH 15/17] Isolate PostgreSQL persistence from the node runtime Co-Authored-By: HAL 9000 --- src/io/postgres_store/mod.rs | 155 +++++++++++++++++++++-------------- 1 file changed, 95 insertions(+), 60 deletions(-) diff --git a/src/io/postgres_store/mod.rs b/src/io/postgres_store/mod.rs index c54cbdef1..c0770de5f 100644 --- a/src/io/postgres_store/mod.rs +++ b/src/io/postgres_store/mod.rs @@ -8,7 +8,7 @@ //! Objects related to [`PostgresStore`] live here. use std::collections::HashMap; use std::future::Future; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use lightning::io; @@ -22,6 +22,7 @@ use tokio_postgres::{Config, Error as PgError}; use self::pool::{make_config_connection, ClientConnection, PgTlsConnector, SmallPool}; use crate::io::utils::check_namespace_key_validity; use crate::logger::{log_debug, log_info, LdkLogger, Logger}; +use crate::runtime::StoreRuntime; mod migrations; mod pool; @@ -101,8 +102,8 @@ pub struct PostgresStore { // operations aren't sensitive to the order of execution. next_write_version: AtomicU64, - // A store-internal runtime used for setup and connection driver tasks. - internal_runtime: Option, + // A store-internal runtime that drives PostgreSQL I/O independently from the node runtime. + internal_runtime: Option>, } // tokio::sync::Mutex (used for the DB client) contains UnsafeCell which opts out of @@ -145,30 +146,18 @@ impl PostgresStore { connection_string: String, db_name: Option, kv_table_name: Option, certificate_pem: Option, logger: Option>, ) -> io::Result { - let internal_runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .thread_name_fn(|| { - static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); - let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); - format!("ldk-node-postgres-runtime-{}", id) - }) - .worker_threads(INTERNAL_RUNTIME_WORKERS) - .max_blocking_threads(INTERNAL_RUNTIME_WORKERS) - .build() - .map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("Failed to build PostgreSQL runtime: {e}"), - ) - })?; + let internal_runtime = Arc::new(StoreRuntime::new( + "ldk-node-postgres-runtime", + INTERNAL_RUNTIME_WORKERS, + "PostgreSQL", + )?); let tls = Self::build_tls_connector(certificate_pem)?; - let runtime_handle = internal_runtime.handle(); - let inner = tokio::task::block_in_place(|| { - runtime_handle.block_on(async { - PostgresStoreInner::new(connection_string, db_name, kv_table_name, tls, logger) - .await - }) - })?; + let task = internal_runtime.spawn(async move { + PostgresStoreInner::new(connection_string, db_name, kv_table_name, tls, logger).await + }); + let inner = task.await.map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("PostgreSQL runtime task failed: {}", e)) + })??; let inner = Arc::new(inner); let next_write_version = AtomicU64::new(1); Ok(Self { inner, next_write_version, internal_runtime: Some(internal_runtime) }) @@ -214,12 +203,18 @@ impl PostgresStore { (inner_lock_ref, version) } + + fn internal_runtime(&self) -> Arc { + Arc::clone(self.internal_runtime.as_ref().expect("PostgreSQL runtime must be available")) + } } impl Drop for PostgresStore { fn drop(&mut self) { if let Some(internal_runtime) = self.internal_runtime.take() { - internal_runtime.shutdown_background(); + if let Ok(internal_runtime) = Arc::try_unwrap(internal_runtime) { + internal_runtime.shutdown_background(); + } } } } @@ -232,7 +227,18 @@ impl KVStore for PostgresStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); - async move { inner.read_internal(&primary_namespace, &secondary_namespace, &key).await } + let runtime = self.internal_runtime(); + async move { + let task = runtime.spawn(async move { + inner.read_internal(&primary_namespace, &secondary_namespace, &key).await + }); + task.await.map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("PostgreSQL runtime task failed: {}", e), + ) + })? + } } fn write( @@ -244,18 +250,27 @@ impl KVStore for PostgresStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .write_internal( - inner_lock_ref, - locking_key, - version, - &primary_namespace, - &secondary_namespace, - &key, - buf, + let task = runtime.spawn(async move { + inner + .write_internal( + inner_lock_ref, + locking_key, + version, + &primary_namespace, + &secondary_namespace, + &key, + buf, + ) + .await + }); + task.await.map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("PostgreSQL runtime task failed: {}", e), ) - .await + })? } } @@ -268,17 +283,26 @@ impl KVStore for PostgresStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .remove_internal( - inner_lock_ref, - locking_key, - version, - &primary_namespace, - &secondary_namespace, - &key, + let task = runtime.spawn(async move { + inner + .remove_internal( + inner_lock_ref, + locking_key, + version, + &primary_namespace, + &secondary_namespace, + &key, + ) + .await + }); + task.await.map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("PostgreSQL runtime task failed: {}", e), ) - .await + })? } } @@ -288,16 +312,18 @@ impl KVStore for PostgresStore { let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let inner = Arc::clone(&self.inner); - async move { inner.list_internal(&primary_namespace, &secondary_namespace).await } - } -} - -impl PostgresStore { - fn internal_runtime(&self) -> io::Result<&tokio::runtime::Runtime> { - self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal PostgreSQL runtime"); - io::Error::new(io::ErrorKind::Other, "Failed to access internal PostgreSQL runtime") - }) + let runtime = self.internal_runtime(); + async move { + let task = runtime.spawn(async move { + inner.list_internal(&primary_namespace, &secondary_namespace).await + }); + task.await.map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("PostgreSQL runtime task failed: {}", e), + ) + })? + } } } @@ -308,10 +334,19 @@ impl PaginatedKVStore for PostgresStore { let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .list_paginated_internal(&primary_namespace, &secondary_namespace, page_token) - .await + let task = runtime.spawn(async move { + inner + .list_paginated_internal(&primary_namespace, &secondary_namespace, page_token) + .await + }); + task.await.map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("PostgreSQL runtime task failed: {}", e), + ) + })? } } } From 2380b6fc12a533376fdfe82e4abb7bde5923d20e Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 5 Jun 2026 10:22:27 +0200 Subject: [PATCH 16/17] Enable Tokio eager handoff for node runtimes Enable Tokio's eager driver handoff when building with tokio_unstable so node-owned runtimes can use the dedicated driver handoff path where available. Build binding artifacts and selected CI coverage with tokio_unstable so the cfg-gated runtime path remains exercised. Co-Authored-By: HAL 9000 --- .github/workflows/benchmarks.yml | 2 +- .github/workflows/rust.yml | 4 ++-- scripts/uniffi_bindgen_generate_kotlin.sh | 5 +++++ scripts/uniffi_bindgen_generate_kotlin_android.sh | 11 ++++++++--- scripts/uniffi_bindgen_generate_python.sh | 5 +++++ scripts/uniffi_bindgen_generate_swift.sh | 5 +++++ src/runtime.rs | 15 ++++++++++++++- 7 files changed, 40 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 6d0056e9a..cd3980b9a 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -43,4 +43,4 @@ jobs: echo "ELECTRS_EXE=$( pwd )/bin/electrs-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV" - name: Run benchmarks run: | - cargo bench + RUSTFLAGS="--cfg tokio_unstable" cargo bench diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b2575aca1..16064fa45 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -84,7 +84,7 @@ jobs: - name: Test on Rust ${{ matrix.toolchain }} if: "matrix.platform != 'windows-latest'" run: | - RUSTFLAGS="--cfg no_download --cfg cycle_tests" cargo test + RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test - name: Test with UniFFI support on Rust ${{ matrix.toolchain }} if: "matrix.platform != 'windows-latest' && matrix.build-uniffi" run: | @@ -114,4 +114,4 @@ jobs: - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - uses: dtolnay/install@cargo-docs-rs - - run: cargo docs-rs \ No newline at end of file + - run: cargo docs-rs diff --git a/scripts/uniffi_bindgen_generate_kotlin.sh b/scripts/uniffi_bindgen_generate_kotlin.sh index dc0237ba6..f82d5c0d0 100755 --- a/scripts/uniffi_bindgen_generate_kotlin.sh +++ b/scripts/uniffi_bindgen_generate_kotlin.sh @@ -5,6 +5,11 @@ PROJECT_DIR="ldk-node-jvm" PACKAGE_DIR="org/lightningdevkit/ldknode" UNIFFI_BINDGEN_BIN="cargo run --manifest-path bindings/uniffi-bindgen/Cargo.toml" +case " ${RUSTFLAGS:-} " in + *" --cfg tokio_unstable "*|*" --cfg=tokio_unstable "*) ;; + *) export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }--cfg tokio_unstable" ;; +esac + if [[ "$OSTYPE" == "linux-gnu"* ]]; then rustup target add x86_64-unknown-linux-gnu || exit 1 cargo build --release --target x86_64-unknown-linux-gnu --features uniffi || exit 1 diff --git a/scripts/uniffi_bindgen_generate_kotlin_android.sh b/scripts/uniffi_bindgen_generate_kotlin_android.sh index 161292857..d0eb8654d 100755 --- a/scripts/uniffi_bindgen_generate_kotlin_android.sh +++ b/scripts/uniffi_bindgen_generate_kotlin_android.sh @@ -5,6 +5,11 @@ TARGET_DIR="target" PROJECT_DIR="ldk-node-android" UNIFFI_BINDGEN_BIN="cargo run --manifest-path bindings/uniffi-bindgen/Cargo.toml" +case " ${RUSTFLAGS:-} " in + *" --cfg tokio_unstable "*|*" --cfg=tokio_unstable "*) RUSTFLAGS_WITH_TOKIO_UNSTABLE="${RUSTFLAGS:-}" ;; + *) RUSTFLAGS_WITH_TOKIO_UNSTABLE="${RUSTFLAGS:+$RUSTFLAGS }--cfg tokio_unstable" ;; +esac + export_variable_if_not_present() { local name="$1" local value="$2" @@ -35,9 +40,9 @@ case "$OSTYPE" in PATH="$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/$LLVM_ARCH_PATH/bin:$PATH" rustup target add x86_64-linux-android aarch64-linux-android armv7-linux-androideabi -RUSTFLAGS="-C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER="x86_64-linux-android21-clang" CC="x86_64-linux-android21-clang" cargo build --profile release-smaller --features uniffi --target x86_64-linux-android || exit 1 -RUSTFLAGS="-C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER="armv7a-linux-androideabi21-clang" CC="armv7a-linux-androideabi21-clang" cargo build --profile release-smaller --features uniffi --target armv7-linux-androideabi || exit 1 -RUSTFLAGS="-C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="aarch64-linux-android21-clang" CC="aarch64-linux-android21-clang" cargo build --profile release-smaller --features uniffi --target aarch64-linux-android || exit 1 +RUSTFLAGS="$RUSTFLAGS_WITH_TOKIO_UNSTABLE -C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER="x86_64-linux-android21-clang" CC="x86_64-linux-android21-clang" cargo build --profile release-smaller --features uniffi --target x86_64-linux-android || exit 1 +RUSTFLAGS="$RUSTFLAGS_WITH_TOKIO_UNSTABLE -C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER="armv7a-linux-androideabi21-clang" CC="armv7a-linux-androideabi21-clang" cargo build --profile release-smaller --features uniffi --target armv7-linux-androideabi || exit 1 +RUSTFLAGS="$RUSTFLAGS_WITH_TOKIO_UNSTABLE -C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="aarch64-linux-android21-clang" CC="aarch64-linux-android21-clang" cargo build --profile release-smaller --features uniffi --target aarch64-linux-android || exit 1 $UNIFFI_BINDGEN_BIN generate bindings/ldk_node.udl --lib-file "$TARGET_DIR"/x86_64-linux-android/release-smaller/libldk_node.so --language kotlin --config uniffi-android.toml -o "$BINDINGS_DIR"/"$PROJECT_DIR"/lib/src/main/kotlin || exit 1 JNI_LIB_DIR="$BINDINGS_DIR"/"$PROJECT_DIR"/lib/src/main/jniLibs/ diff --git a/scripts/uniffi_bindgen_generate_python.sh b/scripts/uniffi_bindgen_generate_python.sh index 50ba450b7..abc4088ff 100755 --- a/scripts/uniffi_bindgen_generate_python.sh +++ b/scripts/uniffi_bindgen_generate_python.sh @@ -2,6 +2,11 @@ BINDINGS_DIR="./bindings/python/src/ldk_node" UNIFFI_BINDGEN_BIN="cargo run --manifest-path bindings/uniffi-bindgen/Cargo.toml" +case " ${RUSTFLAGS:-} " in + *" --cfg tokio_unstable "*|*" --cfg=tokio_unstable "*) ;; + *) export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }--cfg tokio_unstable" ;; +esac + if [[ "$OSTYPE" == "linux-gnu"* ]]; then DYNAMIC_LIB_PATH="./target/release-smaller/libldk_node.so" else diff --git a/scripts/uniffi_bindgen_generate_swift.sh b/scripts/uniffi_bindgen_generate_swift.sh index d4c900e40..d69ac1fbe 100755 --- a/scripts/uniffi_bindgen_generate_swift.sh +++ b/scripts/uniffi_bindgen_generate_swift.sh @@ -4,6 +4,11 @@ set -eox pipefail BINDINGS_DIR="./bindings/swift" UNIFFI_BINDGEN_BIN="cargo run --manifest-path bindings/uniffi-bindgen/Cargo.toml" +case " ${RUSTFLAGS:-} " in + *" --cfg tokio_unstable "*|*" --cfg=tokio_unstable "*) ;; + *) export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }--cfg tokio_unstable" ;; +esac + mkdir -p $BINDINGS_DIR # Install rust target toolchains diff --git a/src/runtime.rs b/src/runtime.rs index 030a01bb8..b86b2e7cd 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -32,7 +32,20 @@ impl Runtime { let mode = match tokio::runtime::Handle::try_current() { Ok(handle) => RuntimeMode::Handle(handle), Err(_) => { - let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; + let mut runtime_builder = tokio::runtime::Builder::new_multi_thread(); + runtime_builder.enable_all(); + // Eager driver handoff lets Tokio move the I/O driver to another worker sooner + // when this runtime's current worker enters `block_in_place` via `block_on`. + // That marginally reduces the chance that a synchronous caller blocks the same + // worker that would otherwise drive the I/O resource it is waiting on. It does + // not solve the issue completely: it only applies to node runtimes we build + // ourselves under `tokio_unstable`, does not affect externally supplied runtime + // handles, and cannot guarantee that every persistence driver task needed by the + // blocked future is already polling elsewhere. See the `StoreRuntime` docs below + // for the full deadlock scenario and the temporary store-runtime isolation. + #[cfg(tokio_unstable)] + runtime_builder.enable_eager_driver_handoff(); + let rt = runtime_builder.build()?; RuntimeMode::Owned(rt) }, }; From a3e7653bb41a43e4bb2e5b1c5f55bae0ee8d1880 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 8 Jun 2026 16:37:25 +0200 Subject: [PATCH 17/17] Name owned node runtime threads Give ldk-node-created runtime workers a stable ldk-node-prefixed thread name so runtime activity is easier to identify in debuggers and logs, matching the store runtime naming convention. Co-Authored-By: HAL 9000 --- src/runtime.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/runtime.rs b/src/runtime.rs index b86b2e7cd..9673d0eb7 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -34,6 +34,11 @@ impl Runtime { Err(_) => { let mut runtime_builder = tokio::runtime::Builder::new_multi_thread(); runtime_builder.enable_all(); + runtime_builder.thread_name_fn(|| { + static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); + let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); + format!("ldk-node-runtime-{}", id) + }); // Eager driver handoff lets Tokio move the I/O driver to another worker sooner // when this runtime's current worker enters `block_in_place` via `block_on`. // That marginally reduces the chance that a synchronous caller blocks the same