From: Joey Guinta <jguinta@squareup.com>
Date: Thu, 13 Feb 2025 17:29:39 +0000 (-0500)
Subject: Address out of memory issue caused by many txs.
X-Git-Tag: v0.30.2~1^2~1
X-Git-Url: http://internal-gitweb-vhost/?a=commitdiff_plain;h=7530b22b6fa59832776a224fa087db1ddde9e589;p=bdk

Address out of memory issue caused by many txs.

For some wallets there exists a pathological case where we may try to fetch many thousands
        of transactions at once, which creates enormous memory pressure. By chunking the batch
        into more reasonably sized sub-queries, we allow time for memory to be freed.
---

diff --git a/src/blockchain/electrum.rs b/src/blockchain/electrum.rs
index 5bfd51ef..acc367dd 100644
--- a/src/blockchain/electrum.rs
+++ b/src/blockchain/electrum.rs
@@ -278,15 +278,20 @@ impl<'a, 'b, D: Database> TxCache<'a, 'b, D> {
             }
         }
 
-        if !need_fetch.is_empty() {
+        // For some wallets there exists a pathological case where we may try to fetch many thousands
+        // of transactions at once, which creates enormous memory pressure. By chunking the batch
+        // into more reasonably sized sub-queries, we allow time for memory to be freed.
+        for chunk in need_fetch.chunks(1000) {
             let txs = self
                 .client
-                .batch_transaction_get(need_fetch.clone())
+                .batch_transaction_get(chunk)
                 .map_err(Error::Electrum)?;
+
             let mut txs: HashMap<_, _> = txs.into_iter().map(|tx| (tx.txid(), tx)).collect();
-            for txid in need_fetch {
-                if let Some(tx) = txs.remove(txid) {
-                    self.cache.insert(*txid, tx);
+
+            for txid in chunk {
+                if let Some(tx) = txs.remove(*txid) {
+                    self.cache.insert(**txid, tx);
                 }
             }
         }