feat: integration of frigate ephemeral scanning

cli/v2/Cargo.toml

@@ -21,5 +21,6 @@ rand = "0.9.0"
 indexer = { version = "0.1.0", path = "../../indexer", features = ["serde"]}
 bdk_sp_wallet = { version = "0.1.0", path = "../../wallet", features = ["serde"]}
 bdk_sp_oracles = { version = "0.1.0", path = "../../oracles" }
+electrum_streaming_client = { git = "https://github.com/sdmg15/electrum_streaming_client", branch = "master"}
 tracing = "0.1.41"
 tracing-subscriber = "0.3.19"

cli/v2/src/main.rs

@@ -10,7 +10,7 @@ use bdk_sp::{
         self,
         address::NetworkUnchecked,
         bip32,
-        consensus::Decodable,
+        consensus::{self, deserialize, Decodable},
         hex::{DisplayHex, FromHex},
         key::Secp256k1,
         script::PushBytesBuf,
@@ -34,6 +34,7 @@ use bdk_sp_oracles::{
         TrustedPeer, UnboundedReceiver, Warning,
     },
     filters::kyoto::{FilterEvent, FilterSubscriber},
+    frigate::{FrigateClient, StreamExt, SubscribeRequest, UnsubscribeRequest, DUMMY_COINBASE},
     tweaks::blindbit::{BlindbitSubscriber, TweakEvent},
 };
 use bdk_sp_wallet::{
@@ -45,6 +46,7 @@ use bdk_sp_wallet::{
     ChangeSet, SpWallet,
 };
 use clap::{self, ArgGroup, Args, Parser, Subcommand};
+use electrum_streaming_client::{notification::Notification, Event};
 use indexer::bdk_chain::BlockId;
 use rand::RngCore;
 use serde_json::json;
@@ -161,6 +163,14 @@ pub enum Commands {
         #[clap(long)]
         hash: Option<BlockHash>,
     },
+    ScanFrigate {
+        #[clap(flatten)]
+        rpc_args: RpcArgs,
+        #[clap(long)]
+        height: Option<u32>,
+        #[clap(long)]
+        hash: Option<BlockHash>,
+    },
     Create {
         /// Network
         #[clap(long, short, default_value = "signet")]
@@ -561,6 +571,132 @@ async fn main() -> anyhow::Result<()> {
                 );
             }
         }
+        Commands::ScanFrigate {
+            rpc_args,
+            height,
+            hash,
+        } => {
+            // The implementation done here differs from what is mentioned in the section
+            // https://github.com/sparrowwallet/frigate/tree/master?tab=readme-ov-file#blockchainsilentpaymentssubscribe
+            // This implementation is doing a one time scanning only. So instead of calling
+            // `blockchain.scripthash.subscribe` on each script from the wallet, we just subscribe
+            // and read the scanning result from the stream. On each result received we update the
+            // wallet state and once scanning progress reaches 1.0 (100%) we stop.
+            let sync_point = if let (Some(height), Some(hash)) = (height, hash) {
+                HeaderCheckpoint::new(height, hash)
+            } else if wallet.birthday.height <= wallet.chain().tip().height() {
+                let height = wallet.chain().tip().height();
+                let hash = wallet.chain().tip().hash();
+                HeaderCheckpoint::new(height, hash)
+            } else {
+                let checkpoint = wallet
+                    .chain()
+                    .get(wallet.birthday.height)
+                    .expect("should be something");
+                let height = checkpoint.height();
+                let hash = checkpoint.hash();
+                HeaderCheckpoint::new(height, hash)
+            };
+
+            let mut client = FrigateClient::connect(&rpc_args.url)
+                .await
+                .unwrap()
+                .with_timeout(tokio::time::Duration::from_secs(60));
+
+            let labels = wallet
+                .indexer()
+                .index()
+                .num_to_label
+                .clone()
+                .into_keys()
+                .collect::<Vec<u32>>();
+            let labels = if !labels.is_empty() {
+                Some(labels)
+            } else {
+                None
+            };
+
+            let subscribe_params = SubscribeRequest {
+                scan_priv_key: *wallet.indexer().scan_sk(),
+                spend_pub_key: *wallet.indexer().spend_pk(),
+                start_height: Some(sync_point.height),
+                labels,
+            };
+
+            client.version().await?;
+            client.subscribe(&subscribe_params).await?;
+
+            while let Some(event) = client.events.next().await {
+                if let Event::Notification(notification) = event {
+                    match notification {
+                        Notification::SpSubscribe(sp_subscribe_notification) => {
+                            let histories = sp_subscribe_notification.history().clone();
+                            let progress = sp_subscribe_notification.progress();
+
+                            let mut secrets_by_height: HashMap<u32, HashMap<Txid, PublicKey>> =
+                                HashMap::new();
+
+                            tracing::debug!("Received history {:#?}", histories);
+
+                            histories.iter().for_each(|h| {
+                                secrets_by_height
+                                    .entry(h.height)
+                                    .and_modify(|v| {
+                                        v.insert(h.tx_hash, h.tweak_key);
+                                    })
+                                    .or_insert(HashMap::from([(h.tx_hash, h.tweak_key)]));
+                            });
+
+                            // Filter when the height is 0, because that would mean mempool transaction
+                            for secret in secrets_by_height.into_iter().filter(|v| v.0 > 0) {
+                                // Since frigate doesn't provide a blockchain.getblock we will mimick that here
+                                // By constructing a block from the block header and the list of transactions
+                                // received from the scan request
+                                let header = client.get_block_header(secret.0).await?.header;
+                                let mut raw_blk = consensus::serialize(&header);
+                                raw_blk.push(0);
+
+                                // Push dummy coinbase
+                                let coinbase: Transaction =
+                                    deserialize(&Vec::<u8>::from_hex(DUMMY_COINBASE).unwrap())
+                                        .unwrap();
+                                let mut block: Block = deserialize(&raw_blk).unwrap();
+
+                                let blockhash = header.block_hash();
+                                let mut txs: Vec<Transaction> = vec![coinbase];
+
+                                for key in secret.1.keys() {
+                                    let tx_result = client.get_transaction(*key).await?;
+                                    txs.push(tx_result.tx);
+                                }
+
+                                block.txdata = txs;
+                                tracing::debug!("Final block {:?}", block);
+                                wallet.apply_block_relevant(&block, secret.1, secret.0);
+
+                                let checkpoint = wallet.chain().tip().insert(BlockId {
+                                    height: secret.0,
+                                    hash: blockhash,
+                                });
+                                wallet.update_chain(checkpoint);
+                            }
+
+                            if progress >= 1.0 {
+                                tracing::info!("Scanning completed");
+                                break;
+                            }
+                        }
+                        _ => tracing::error!("Notification event not supported"),
+                    }
+                }
+            }
+            // Unsubscribe once scanning is done
+            let unsub_req = UnsubscribeRequest {
+                scan_priv_key: *wallet.indexer().scan_sk(),
+                spend_pub_key: *wallet.indexer().spend_pk(),
+            };
+            client.unsubscribe(&unsub_req).await?;
+        }
         Commands::Balance => {
             fn print_balances<'a>(
                 title_str: &'a str,

doc/tabconf7/frigate_playbook.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+########################### STAGE 1: setup ####################################
+
+# 1. Install dependencies locally and setup regtest environment
+just non_nix_init
+# 2. Check bitcoind is running on regtest
+just cli getblockchaininfo
+# 3. Check bdk-cli wallet was created correctly
+just regtest-bdk balance
+# 4. Check sp-cli wallet was created correctly
+just regtest-sp balance
+# 5. Synchronize bdk-cli wallet
+just regtest-bdk sync
+
+###################### STAGE 2: fund bdk-cli wallet ###########################
+
+# 6. Get a new address from bdk-cli wallet
+REGTEST_ADDRESS=$(just regtest-bdk unused_address | jq -r '.address' | tr -d '\n')
+# 7. Mine a few more blocks to fund the wallet
+just mine 1 $REGTEST_ADDRESS
+# 8. Mine some of them to the internal wallet to confirm the bdk-cli balance
+just mine 101
+# 9. Synchronize bdk-cli wallet
+just regtest-bdk sync
+# 10. Check balance
+just regtest-bdk balance
+
+################ STAGE 3: create a silent payment output ######################
+
+# 11. Get a silent payment code from sp-cli2 wallet
+SP_CODE=$(just regtest-sp code | jq -r '.silent_payment_code' | tr -d '\n')
+# 12. Create a transaction spending bdk-cli wallet UTXOs to a the previous silent payment code
+RAW_TX=$(just regtest-bdk create_sp_tx --to-sp $SP_CODE:10000 --fee_rate 5 | jq -r '.raw_tx' | tr -d '\n')
+TXID=$(just regtest-bdk broadcast --tx $RAW_TX | jq -r '.txid' | tr -d '\n')
+# 14. Mine a new block
+just mine 1
+# 15. Once the new transaction has been mined, synchronize bdk-cli wallet again
+just regtest-bdk sync
+
+# ################## STAGE 4: find a silent payment output ######################
+
+# 16. Now synchronize sp-cli2 wallet using frigate ephemeral scanning
+FRIGATE_HOST="127.0.0.1:57001"
+just regtest-sp scan-frigate --url $FRIGATE_HOST
+# 17. Check balance on sp-cli2 wallet
+just regtest-sp balance
+# 18. Check balance on bdk-cli wallet
+just regtest-bdk balance
+
+# At this point we will able to see SP outputs paid to out wallet!

doc/tabconf7/justfile

@@ -121,7 +121,7 @@ build TAG="1.0.0" VERSION="29.0" RELEASE="29.0": machine
     RUN mkdir -p /build/frigate
     RUN mkdir -p /frigate
     WORKDIR /frigate
-    RUN git clone --recursive --branch 1.1.0 --depth 1 https://github.com/sparrowwallet/frigate.git .
+    RUN git clone --recursive --branch 1.3.2 --depth 1 https://github.com/sparrowwallet/frigate.git .
     RUN ./gradlew jpackage
     RUN cp -r ./build/jpackage/frigate /build/frigate
     RUN rm -rf /frigate

oracles/Cargo.toml

@@ -13,9 +13,11 @@ redb = "2.4.0"
 rayon = "1.11.0"
 reqwest = { version = "0.12.23", features = ["json", "rustls-tls", "http2", "charset"], default-features = false }
 serde = { version = "1.0.219", features = ["serde_derive"] }
-serde_json = "1.0.142"
+serde_json = { version = "1.0.142", features = ["raw_value"]}
+electrum_streaming_client = { path = "/home/sdmg15/workspace/electrum_streaming_client"}
 url = "2.5.4"
 tracing = "0.1.41"
+jsonrpc = "=0.18.0"
 
 [lints]
 workspace = true