From 8aa4925750b8f540cbec2260f90c7c948eb48b05 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 9 Nov 2023 13:47:56 +0200 Subject: [PATCH 1/6] feat: ios pre-populated scorer download --- lib/ios/Ldk.m | 6 +++- lib/ios/Ldk.swift | 49 +++++++++++++++++++++++++++++-- lib/src/ldk.ts | 28 ++++++++++++++++++ lib/src/lightning-manager.ts | 57 ++++++++++++++++++++++++++++++++---- lib/src/utils/types.ts | 7 +++++ 5 files changed, 138 insertions(+), 9 deletions(-) diff --git a/lib/ios/Ldk.m b/lib/ios/Ldk.m index 7c31a22b..16e22852 100644 --- a/lib/ios/Ldk.m +++ b/lib/ios/Ldk.m @@ -13,7 +13,6 @@ @interface RCT_EXTERN_MODULE(Ldk, NSObject) RCT_EXTERN_METHOD(writeToLogFile:(NSString *)line resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject) - RCT_EXTERN_METHOD(initChainMonitor:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject) RCT_EXTERN_METHOD(initKeysManager:(NSString *)seed @@ -22,8 +21,13 @@ @interface RCT_EXTERN_MODULE(Ldk, NSObject) RCT_EXTERN_METHOD(initUserConfig:(NSDictionary *)userConfig resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject) +RCT_EXTERN_METHOD(downloadScorer:(NSString *)scorerSyncUrl + skipHoursThreshold:(NSInteger *)skipHoursThreshold + resolve:(RCTPromiseResolveBlock)resolve + reject:(RCTPromiseRejectBlock)reject) RCT_EXTERN_METHOD(initNetworkGraph:(NSString *)network rapidGossipSyncUrl:(NSString *)rapidGossipSyncUrl + skipHoursThreshold:(NSInteger *)skipHoursThreshold resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject) RCT_EXTERN_METHOD(initChannelManager:(NSString *)network diff --git a/lib/ios/Ldk.swift b/lib/ios/Ldk.swift index 63efa6e7..29ab3287 100644 --- a/lib/ios/Ldk.swift +++ b/lib/ios/Ldk.swift @@ -73,6 +73,7 @@ enum LdkErrors: String { case backup_restore_failed = "backup_restore_failed" case backup_restore_failed_existing_files = "backup_restore_failed_existing_files" case backup_list_files_failed = "backup_list_files_failed" + case scorer_download_fail = "scorer_download_fail" } enum LdkCallbackResponses: String { @@ -101,6 +102,8 @@ enum LdkCallbackResponses: String { case backup_client_setup_success = "backup_client_setup_success" case backup_restore_success = "backup_restore_success" case backup_client_check_success = "backup_client_check_success" + case scorer_download_success = "scorer_download_success" + case scorer_download_skip = "scorer_download_skip" } enum LdkFileNames: String { @@ -228,7 +231,43 @@ class Ldk: NSObject { } @objc - func initNetworkGraph(_ network: NSString, rapidGossipSyncUrl: NSString, resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock) { + func downloadScorer(_ scorerSyncUrl: NSString, skipHoursThreshold: NSInteger, resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock) { + guard let accountStoragePath = Ldk.accountStoragePath else { + return handleReject(reject, .init_storage_path) + } + + let destinationFile = accountStoragePath.appendingPathComponent(LdkFileNames.scorer.rawValue) + + //If old one is still recent, skip download. Else delete it. + if FileManager().fileExists(atPath: destinationFile.path) { + let fileAttributes = try? FileManager().attributesOfItem(atPath: destinationFile.path) + if let creationDate = fileAttributes?[.creationDate] as? Date { + let currentTime = Date() + let timeInterval = currentTime.timeIntervalSince(creationDate) + let hoursPassed = timeInterval / 3600 + + if hoursPassed <= Double(skipHoursThreshold) { + return handleResolve(resolve, .scorer_download_skip) + } + } + + try? FileManager().removeItem(atPath: destinationFile.path) + } + + let url = URL(string: String(scorerSyncUrl))! + let task = url.downloadTask(destination: destinationFile) { error in + if let error = error { + return handleReject(reject, .scorer_download_fail, error) + } + + handleResolve(resolve, .scorer_download_success) + } + + task?.resume() + } + + @objc + func initNetworkGraph(_ network: NSString, rapidGossipSyncUrl: NSString, skipHoursThreshold: NSInteger, resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock) { guard networkGraph == nil else { return handleReject(reject, .already_init) } @@ -260,6 +299,7 @@ class Ldk: NSObject { } print("rapidGossipSyncUrl: \(rapidGossipSyncUrl)") + print("accountStoragePath: \(accountStoragePath)") //Download url passed, enable rapid gossip sync do { @@ -274,7 +314,7 @@ class Ldk: NSObject { let timestamp = networkGraph?.getLastRapidGossipSyncTimestamp() ?? 0 let minutesDiffSinceLastRGS = (Calendar.current.dateComponents([.minute], from: Date.init(timeIntervalSince1970: TimeInterval(timestamp)), to: Date()).minute)! - guard minutesDiffSinceLastRGS > 60 else { + guard minutesDiffSinceLastRGS > 60 * skipHoursThreshold else { LdkEventEmitter.shared.send(withEvent: .native_log, body: "Skipping rapid gossip sync. Last updated \(minutesDiffSinceLastRGS/60) hours ago.") return handleResolve(resolve, .network_graph_init_success) } @@ -442,6 +482,11 @@ class Ldk: NSObject { currentBlockchainTipHash = blockHash currentBlockchainHeight = blockHeight addForegroundObserver() + + print("Scorer:") +// Task { +// probabilisticScorer.debugLogLiquidityStats() +// } return handleResolve(resolve, .channel_manager_init_success) } diff --git a/lib/src/ldk.ts b/lib/src/ldk.ts index 0397e01b..cc989e81 100644 --- a/lib/src/ldk.ts +++ b/lib/src/ldk.ts @@ -35,6 +35,7 @@ import { TBackupServerDetails, TNodeSignReq, TBackedUpFileList, + TDownloadScorer, } from './utils/types'; import { extractPaymentRequest } from './utils/helpers'; @@ -97,23 +98,50 @@ class LDK { } } + /** + * Downloads a pre-populated scorer from remote server to be used on startup. + * Should be called before channel manager is initialized. + * @param scorerDownloadUrl + * @param skipHoursThreshold + */ + async downloadScorer({ + scorerDownloadUrl, + skipHoursThreshold, + }: TDownloadScorer): Promise> { + try { + const res = await NativeLDK.downloadScorer( + scorerDownloadUrl, + skipHoursThreshold ?? 3, + ); + this.writeDebugToLog('downloadScorer'); + return ok(res); + } catch (e) { + this.writeErrorToLog('downloadScorer', e); + return err(e); + } + } + /** * Inits the network graph from previous cache or syncs from scratch. * By passing in rapidGossipSyncUrl p2p gossip sync will be disabled in favor out rapid gossip sync. * For local regtest p2p works fine but for mainnet it is better to enable rapid gossip sync. + * Use skipHoursThreshold to avoid unnecessary http calls that check if there is an update. * https://docs.rs/lightning/latest/lightning/routing/network_graph/struct.NetworkGraph.html * @param network * @param rapidGossipSyncUrl + * @param skipHoursThreshold * @returns {Promise | Ok | Err>>} */ async initNetworkGraph({ network, rapidGossipSyncUrl, + skipHoursThreshold, }: TInitNetworkGraphReq): Promise> { try { const res = await NativeLDK.initNetworkGraph( network, rapidGossipSyncUrl ?? '', + skipHoursThreshold ?? 3, // default to 3 hours as that is the current RGS schedule. ); this.writeDebugToLog( 'initNetworkGraph', diff --git a/lib/src/lightning-manager.ts b/lib/src/lightning-manager.ts index 29a6bc30..51601a0b 100644 --- a/lib/src/lightning-manager.ts +++ b/lib/src/lightning-manager.ts @@ -290,6 +290,7 @@ class LightningManager { broadcastTransaction, network, rapidGossipSyncUrl = 'https://rapidsync.lightningdevkit.org/snapshot/', + scorerDownloadUrl = 'https://rapidsync.lightningdevkit.org/scoring/scorer.bin', forceCloseOnStartup, userConfig = defaultUserConfig, trustedZeroConfPeers = [], @@ -450,15 +451,59 @@ class LightningManager { // Handled in initChannelManager below if (network !== ENetworks.mainnet) { - //RGS only currently working for mainnet + //RGS and pre-populated scorer only available for mainnet rapidGossipSyncUrl = ''; + scorerDownloadUrl = ''; } - // Step 11: Optional: Initialize the NetGraphMsgHandler - const networkGraphRes = await ldk.initNetworkGraph({ - network, - rapidGossipSyncUrl, - }); + let promises: Promise>[] = []; + + if (scorerDownloadUrl) { + promises.push( + ldk.downloadScorer({ + scorerDownloadUrl, + skipHoursThreshold: 3, + }), + ); + } else { + promises.push(Promise.resolve(ok(''))); + } + + promises.push( + ldk.initNetworkGraph({ + network, + rapidGossipSyncUrl, + skipHoursThreshold: 3, + }), + ); + + // + // + // + // if (scorerDownloadUrl) { + // const scorerRes = await ldk.downloadScorer({ + // scorerDownloadUrl, + // skipHoursThreshold: 3, + // }); + // if (scorerRes.isErr()) { + // return this.handleStartError(scorerRes); + // } + // } + // + // // Step 11: Optional: Initialize the NetGraphMsgHandler + // const networkGraphRes = await ldk.initNetworkGraph({ + // network, + // rapidGossipSyncUrl, + // skipHoursThreshold: 3, + // }); + // if (networkGraphRes.isErr()) { + // return this.handleStartError(networkGraphRes); + // } + + const [scorerRes, networkGraphRes] = await Promise.all(promises); + if (scorerRes.isErr()) { + return this.handleStartError(scorerRes); + } if (networkGraphRes.isErr()) { return this.handleStartError(networkGraphRes); } diff --git a/lib/src/utils/types.ts b/lib/src/utils/types.ts index 650c485e..68d1371a 100644 --- a/lib/src/utils/types.ts +++ b/lib/src/utils/types.ts @@ -310,9 +310,15 @@ export type TInitChannelManagerReq = { }; }; +export type TDownloadScorer = { + scorerDownloadUrl: string; + skipHoursThreshold?: number; +}; + export type TInitNetworkGraphReq = { network: ENetworks; rapidGossipSyncUrl?: string; + skipHoursThreshold?: number; }; export type TChannelHandshakeConfig = { @@ -553,6 +559,7 @@ export type TLdkStart = { broadcastTransaction: TBroadcastTransaction; network: ENetworks; rapidGossipSyncUrl?: string; + scorerDownloadUrl?: string; forceCloseOnStartup?: TForceCloseOnStartup; userConfig?: TUserConfig; trustedZeroConfPeers?: string[]; From 3ec65c6c0382b664a38e4fa08f6ce521414b4aa7 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 9 Nov 2023 14:09:40 +0200 Subject: [PATCH 2/6] feat: grouped startup functions that can run in parallel and option to skip startup param check --- lib/src/lightning-manager.ts | 180 +++++++++++------------------------ lib/src/utils/types.ts | 1 + 2 files changed, 57 insertions(+), 124 deletions(-) diff --git a/lib/src/lightning-manager.ts b/lib/src/lightning-manager.ts index 51601a0b..a7569a1c 100644 --- a/lib/src/lightning-manager.ts +++ b/lib/src/lightning-manager.ts @@ -295,6 +295,7 @@ class LightningManager { userConfig = defaultUserConfig, trustedZeroConfPeers = [], backupServerDetails, + skipParamCheck = false, }: TLdkStart): Promise> { if (!account) { return err( @@ -329,20 +330,24 @@ class LightningManager { } this.isStarting = true; - // Ensure the start params function as expected. - const paramCheckResponse = await startParamCheck({ - account, - getBestBlock, - getTransactionData, - getTransactionPosition, - broadcastTransaction, - getAddress, - getScriptPubKeyHistory, - getFees, - network, - }); - if (paramCheckResponse.isErr()) { - return this.handleStartError(paramCheckResponse); + if (!skipParamCheck) { + // Ensure the start params function as expected. + const paramCheckResponse = await startParamCheck({ + account, + getBestBlock, + getTransactionData, + getTransactionPosition, + broadcastTransaction, + getAddress, + getScriptPubKeyHistory, + getFees, + network, + }); + if (paramCheckResponse.isErr()) { + return this.handleStartError(paramCheckResponse); + } + } else { + console.warn('Skipping start param check. Switch back on for debugging.'); } this.getBestBlock = getBestBlock; @@ -402,61 +407,28 @@ class LightningManager { } } - //Setup remote backup server - if (backupServerDetails) { - const backupSetupRes = await ldk.backupSetup({ - seed: account.seed, - network: this.network, - details: backupServerDetails, - }); - if (backupSetupRes.isErr()) { - return this.handleStartError(err(backupSetupRes.error)); - } - } - - // Step 1: Initialize the FeeEstimator - // Lazy loaded in native code - // https://docs.rs/lightning/latest/lightning/chain/chaininterface/trait.FeeEstimator.html - - // Step 2: Initialize the Logger - // Lazy loaded in native code - // https://docs.rs/lightning/latest/lightning/util/logger/index.html - - //Switch on log levels we're interested in. All levels are false by default. - await ldk.setLogLevel(ELdkLogLevels.info, true); - await ldk.setLogLevel(ELdkLogLevels.warn, true); - await ldk.setLogLevel(ELdkLogLevels.error, true); - await ldk.setLogLevel(ELdkLogLevels.debug, true); - - //TODO might not always need this one as they make the logs a little noisy - // await ldk.setLogLevel(ELdkLogLevels.trace, true); - - // Step 3: Initialize the BroadcasterInterface - // Lazy loaded in native code - // https://docs.rs/lightning/latest/lightning/chain/chaininterface/trait.BroadcasterInterface.html - - // Step 4: Initialize Persist - // Lazy loaded in native code - // https://docs.rs/lightning/latest/lightning/chain/chainmonitor/trait.Persist.html - - // Step 5: Initialize the ChainMonitor (happens when we init the ChannelManager) - - // Step 6: Initialize the KeysManager - const keysManager = await ldk.initKeysManager(this.account.seed); - if (keysManager.isErr()) { - return this.handleStartError(keysManager); - } - - // Step 7: Read ChannelMonitors state from disk - // Handled in initChannelManager below - if (network !== ENetworks.mainnet) { //RGS and pre-populated scorer only available for mainnet rapidGossipSyncUrl = ''; scorerDownloadUrl = ''; } - let promises: Promise>[] = []; + //All these calls don't need to be done in any particular sequence + let promises: Promise>[] = [ + ldk.setLogLevel(ELdkLogLevels.info, true), + ldk.setLogLevel(ELdkLogLevels.warn, true), + ldk.setLogLevel(ELdkLogLevels.error, true), + ldk.setLogLevel(ELdkLogLevels.debug, true), + // ldk.setLogLevel(ELdkLogLevels.trace, true), + ldk.initKeysManager(this.account.seed), + ldk.initNetworkGraph({ + network, + rapidGossipSyncUrl, + skipHoursThreshold: 3, + }), + this.setFees(), + ldk.initUserConfig(userConfig), + ]; if (scorerDownloadUrl) { promises.push( @@ -465,54 +437,24 @@ class LightningManager { skipHoursThreshold: 3, }), ); - } else { - promises.push(Promise.resolve(ok(''))); } - promises.push( - ldk.initNetworkGraph({ - network, - rapidGossipSyncUrl, - skipHoursThreshold: 3, - }), - ); + if (backupServerDetails) { + promises.push( + ldk.backupSetup({ + seed: account.seed, + network: this.network, + details: backupServerDetails, + }), + ); + } - // - // - // - // if (scorerDownloadUrl) { - // const scorerRes = await ldk.downloadScorer({ - // scorerDownloadUrl, - // skipHoursThreshold: 3, - // }); - // if (scorerRes.isErr()) { - // return this.handleStartError(scorerRes); - // } - // } - // - // // Step 11: Optional: Initialize the NetGraphMsgHandler - // const networkGraphRes = await ldk.initNetworkGraph({ - // network, - // rapidGossipSyncUrl, - // skipHoursThreshold: 3, - // }); - // if (networkGraphRes.isErr()) { - // return this.handleStartError(networkGraphRes); - // } - - const [scorerRes, networkGraphRes] = await Promise.all(promises); - if (scorerRes.isErr()) { - return this.handleStartError(scorerRes); - } - if (networkGraphRes.isErr()) { - return this.handleStartError(networkGraphRes); - } - - // Step 8: Initialize the UserConfig ChannelManager - const confRes = await ldk.initUserConfig(userConfig); - - if (confRes.isErr()) { - return this.handleStartError(confRes); + // Check for any errors before starting channel manager. + const results = await Promise.all(promises); + for (const result of results) { + if (result.isErr()) { + return this.handleStartError(result); + } } const channelManagerRes = await ldk.initChannelManager({ @@ -534,28 +476,18 @@ class LightningManager { ); } - // Set fee estimates - await this.setFees(); - //Force close all channels on startup. Likely to recover funds after restoring from a stale backup. if (forceCloseOnStartup && forceCloseOnStartup.forceClose) { await ldk.forceCloseAllChannels(forceCloseOnStartup.broadcastLatestTx); } if (!forceCloseOnStartup || forceCloseOnStartup.broadcastLatestTx) { - // If we're force closing without broadcasting latest state don't add peers as we're likely doing this to recovery from a stale backup - await this.addPeers(); + // If we're force closing without broadcasting the latest state don't add peers as we're likely doing this to recovery from a stale backup + this.addPeers().catch(console.error); } - // Step 9: Sync ChannelMonitors and ChannelManager to chain tip - await this.syncLdk(); - - // Step 10: Give ChannelMonitors to ChainMonitor - - // Step 12: Initialize the PeerManager - // Done with initChannelManager - // Step 13: Initialize networking - // Done with initChannelManager + //Can continue in the background + this.syncLdk().catch(console.error); //Writes node state to log files ldk.nodeStateDump().catch(console.error); @@ -593,9 +525,9 @@ class LightningManager { ); } - async setFees(): Promise { + async setFees(): Promise> { const fees = await this.getFees(); - await ldk.updateFees(fees); + return await ldk.updateFees(fees); } /** diff --git a/lib/src/utils/types.ts b/lib/src/utils/types.ts index 68d1371a..bcb83e9a 100644 --- a/lib/src/utils/types.ts +++ b/lib/src/utils/types.ts @@ -564,6 +564,7 @@ export type TLdkStart = { userConfig?: TUserConfig; trustedZeroConfPeers?: string[]; backupServerDetails?: TBackupServerDetails; + skipParamCheck?: boolean; }; export type TGetAddress = () => Promise; From d995d4203aa58ed3f44d474085ecc0d30d2918f6 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 9 Nov 2023 14:49:22 +0200 Subject: [PATCH 3/6] chore: remove old backup logic from example app and add deprecated warnings to old backup and restore functions fix: lint --- example/Dev.tsx | 74 +------------------------------ example/ldk/index.ts | 43 ------------------ example/tests/lnd.ts | 20 ++++++++- example/tests/unit.ts | 11 +++-- example/utils/helpers.ts | 85 +----------------------------------- lib/src/lightning-manager.ts | 17 +++++++- 6 files changed, 45 insertions(+), 205 deletions(-) diff --git a/example/Dev.tsx b/example/Dev.tsx index f01f31c5..f7084f25 100644 --- a/example/Dev.tsx +++ b/example/Dev.tsx @@ -12,14 +12,7 @@ import { View, } from 'react-native'; import Clipboard from '@react-native-clipboard/clipboard'; -import { - backupAccount, - getAddressBalance, - importAccount, - setupLdk, - syncLdk, - updateHeader, -} from './ldk'; +import { getAddressBalance, setupLdk, syncLdk, updateHeader } from './ldk'; import { connectToElectrum, subscribeToHeader } from './electrum'; import ldk from '@synonymdev/react-native-ldk/dist/ldk'; import lm, { @@ -31,12 +24,7 @@ import lm, { TChannelUpdate, } from '@synonymdev/react-native-ldk'; import { backupServerDetails, peers } from './utils/constants'; -import { - createNewAccount, - getAccount, - getAddress, - simulateStaleRestore, -} from './utils/helpers'; +import { createNewAccount, getAccount, getAddress } from './utils/helpers'; import RNFS from 'react-native-fs'; let logSubscription: EmitterSubscription | undefined; @@ -44,7 +32,6 @@ let paymentSubscription: EmitterSubscription | undefined; let onChannelSubscription: EmitterSubscription | undefined; let paymentFailedSubscription: EmitterSubscription | undefined; let paymentPathSuccess: EmitterSubscription | undefined; -let backupSubscriptionId: string | undefined; const Dev = (): ReactElement => { const [message, setMessage] = useState('...'); @@ -144,25 +131,12 @@ const Dev = (): ReactElement => { ); } - if (!backupSubscriptionId) { - backupSubscriptionId = lm.subscribeToBackups((backupRes) => { - if (backupRes.isErr()) { - return alert('Backup required but failed to export account'); - } - - console.log( - `Backup updated for account ${backupRes.value.account.name}`, - ); - }); - } - return (): void => { logSubscription && logSubscription.remove(); paymentSubscription && paymentSubscription.remove(); paymentFailedSubscription && paymentFailedSubscription.remove(); paymentPathSuccess && paymentPathSuccess.remove(); onChannelSubscription && onChannelSubscription.remove(); - backupSubscriptionId && lm.unsubscribeFromBackups(backupSubscriptionId); }; }, []); @@ -589,50 +563,6 @@ const Dev = (): ReactElement => { }} /> -