feat: allow for backing up any file to remote backup server

backup-server/src/server.js

@@ -16,7 +16,7 @@ const challenges = new Map(); // pubkey -> {challenge, expires}
 
 const signedMessagePrefix = 'react-native-ldk backup server auth:';
 
-let labels = [
+const ldkLabels = [
     'ping',
     'channel_manager',
     'channel_monitor',
@@ -29,6 +29,15 @@ let labels = [
     'payments_sent',
     'bolt11_invoices',
 ];
+const bitkitLabels = [
+    'bitkit_settings',
+    'bitkit_widgets',
+    'bitkit_metadata',
+    'bitkit_blocktank_orders',
+    'bitkit_slashtags_contacts',
+];
+const labels = [...ldkLabels, ...bitkitLabels];
+
 let networks = ['bitcoin', 'testnet', 'regtest', 'signet'];
 
 const version = 'v1';
@@ -264,6 +273,7 @@ fastify.route({
             type: 'object',
             properties: {
                 network: { type: 'string', enum: networks },
+                fileGroup: { type: 'string', enum: ['all', 'bitkit', 'ldk'] },
             },
             required: ['network'],
         },
@@ -272,12 +282,22 @@ fastify.route({
     handler: async (request, reply) => {
         const {query, headers} = request;
 
-        const {network} = query;
+        const {network, fileGroup} = query;
         const bearerToken = headers.authorization;
         const {pubkey} = users.get(bearerToken);
 
-        const list = await storage.list({pubkey, network});
-        const channelMonitorList = await storage.list({pubkey, network, subdir: 'channel_monitors'});
+        let list = await storage.list({pubkey, network});
+        let channelMonitorList = await storage.list({pubkey, network, subdir: 'channel_monitors'});
+
+        //If not set then assume older version of the app
+        if (!fileGroup || fileGroup === 'ldk') {
+            list = list.filter(file => ldkLabels.includes(file.replace('.bin', '')));
+        } else if (fileGroup === 'bitkit') {
+            list = list.filter(file => bitkitLabels.includes(file.replace('.bin', '')));
+            channelMonitorList = [];
+        } else if (fileGroup === 'all') {
+            //Do nothing
+        }
 
         const allFiles = {
             list,

example/Dev.tsx

@@ -94,7 +94,7 @@
 			// @ts-ignore
 			paymentSubscription = ldk.onEvent(
 				EEventTypes.channel_manager_payment_claimed,
 				(res: TChannelManagerClaim) => alert(`Received ${res.amount_sat} sats`),
 			);
 		}
 
@@ -125,7 +125,7 @@
 			onChannelSubscription = ldk.onEvent(
 				EEventTypes.new_channel,
 				(res: TChannelUpdate) =>
 					alert(
 						`Channel received from ${res.counterparty_node_id} Channel ${res.channel_id}`,
 					),
 			);
@@ -645,6 +645,46 @@
 							setMessage(res.value);
 						}}
 					/>
+
+					<Button
+						title={'List backed up files'}
+						onPress={async (): Promise<void> => {
+							setMessage('Backing up...');
+							const res = await ldk.backupListFiles();
+							if (res.isErr()) {
+								setMessage(res.error.message);
+								return;
+							}
+
+							setMessage(JSON.stringify(res.value));
+						}}
+					/>
+
+					<Button
+						title={'Test file backup'}
+						onPress={async (): Promise<void> => {
+							setMessage('Backing up...');
+							const content = `look at me I'm a file ${Date().toString()}`;
+							const res = await ldk.backupFile('ping', content);
+							if (res.isErr()) {
+								setMessage(res.error.message);
+								return;
+							}
+
+							const res2 = await ldk.fetchBackupFile('ping');
+							if (res2.isErr()) {
+								setMessage(res2.error.message);
+								return;
+							}
+
+							if (res2.value !== content) {
+								setMessage('File backup failed');
+								return;
+							}
+
+							setMessage(`Retrieved remote content: "${res2.value}"`);
+						}}
+					/>
 				</View>
 			</ScrollView>
 

example/ios/Podfile.lock

@@ -375,7 +375,7 @@ PODS:
   - React-jsinspector (0.72.4)
   - React-logger (0.72.4):
     - glog
-  - react-native-ldk (0.0.125):
+  - react-native-ldk (0.0.127):
     - React
   - react-native-randombytes (3.6.1):
     - React-Core
@@ -723,7 +723,7 @@ SPEC CHECKSUMS:
   React-jsiexecutor: c7f826e40fa9cab5d37cab6130b1af237332b594
   React-jsinspector: aaed4cf551c4a1c98092436518c2d267b13a673f
   React-logger: da1ebe05ae06eb6db4b162202faeafac4b435e77
-  react-native-ldk: c78b18c8c8fe218481721f7083e3e0f825aa957e
+  react-native-ldk: 4ab3d26d5e1356313c572814289cc516dc18dd88
   react-native-randombytes: 421f1c7d48c0af8dbcd471b0324393ebf8fe7846
   react-native-tcp-socket: c1b7297619616b4c9caae6889bcb0aba78086989
   React-NativeModulesApple: edb5ace14f73f4969df6e7b1f3e41bef0012740f

example/ldk/index.ts

@@ -109,6 +109,15 @@ export const setupLdk = async (
 			return err(storageRes.error);
 		}
 
+		const res = await ldk.backupSetup({
+			network: ldkNetwork(selectedNetwork),
+			seed: account.seed,
+			details: backupServerDetails!,
+		});
+		if (res.isErr()) {
+			return err(res.error);
+		}
+
 		const lmStart = await lm.start({
 			getBestBlock,
 			account,
@@ -140,7 +149,7 @@ export const setupLdk = async (
 				manually_accept_inbound_channels: true,
 			},
 			trustedZeroConfPeers: [peers.lnd.pubKey],
-			backupServerDetails,
+			skipRemoteBackups: !backupServerDetails,
 		});
 
 		if (lmStart.isErr()) {

lib/android/src/main/java/com/reactnativeldk/LdkModule.kt

@@ -33,7 +33,6 @@ enum class EventTypes {
     register_tx,
     register_output,
     broadcast_transaction,
-    backup,
     channel_manager_funding_generation_ready,
     channel_manager_payment_claimable,
     channel_manager_payment_sent,
@@ -94,6 +93,7 @@ enum class LdkErrors {
     backup_setup_failed,
     backup_restore_failed,
     backup_restore_failed_existing_files,
+    backup_file_failed,
     scorer_download_fail
 }
 
@@ -122,6 +122,7 @@ enum class LdkCallbackResponses {
     backup_client_setup_success,
     backup_restore_success,
     backup_client_check_success,
+    backup_file_success,
     scorer_download_success,
     scorer_download_skip
 }
@@ -1291,6 +1292,35 @@ class LdkModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaMod
             handleReject(promise, LdkErrors.backup_check_failed, Error(e))
         }
     }
+
+    @ReactMethod
+    fun backupFile(fileName: String, content: String, promise: Promise) {
+        if (BackupClient.requiresSetup) {
+            return handleReject(promise, LdkErrors.backup_setup_required)
+        }
+
+        BackupClient.addToPersistQueue(BackupClient.Label.MISC(fileName), content.toByteArray(Charsets.UTF_8)) { error ->
+            if (error != null) {
+                handleReject(promise, LdkErrors.backup_file_failed, Error(error))
+            } else {
+                handleResolve(promise, LdkCallbackResponses.backup_file_success)
+            }
+        }
+    }
+
+    @ReactMethod
+    fun fetchBackupFile(fileName: String, promise: Promise) {
+        if (BackupClient.requiresSetup) {
+            return handleReject(promise, LdkErrors.backup_setup_required)
+        }
+
+        try {
+            val bytes = BackupClient.retrieve(BackupClient.Label.MISC(fileName))
+            promise.resolve(bytes.toString(Charsets.UTF_8))
+        } catch (e: Exception) {
+            handleReject(promise, LdkErrors.backup_file_failed, Error(e))
+        }
+    }
 }
 
 object LdkEventEmitter {

lib/android/src/main/java/com/reactnativeldk/classes/BackupClient.kt

@@ -40,7 +40,7 @@ class CompleteBackup(
     val channelFiles: Map<String, ByteArray>
 )
 
-typealias BackupCompleteCallback = () -> Unit
+typealias BackupCompleteCallback = (Exception?) -> Unit
 
 class BackupQueueEntry(
     val uuid: UUID,
@@ -141,6 +141,10 @@ class BackupClient {
                 urlString += "&channelId=${label.channelId}"
             }
 
+            if (method == Method.LIST) {
+                urlString += "&fileGroup=ldk"
+            }
+
             return URL(urlString)
         }
 
@@ -508,9 +512,9 @@ class BackupClient {
         }
 
         //Backup queue management
-        fun addToPersistQueue(label: BackupClient.Label, bytes: ByteArray, callback: (() -> Unit)? = null) {
+        fun addToPersistQueue(label: Label, bytes: ByteArray, callback: BackupCompleteCallback? = null) {
             if (BackupClient.skipRemoteBackup) {
-                callback?.invoke()
+                callback?.invoke(null)
                 LdkEventEmitter.send(
                     EventTypes.native_log,
                     "Skipping remote backup queue append for ${label.string}"
@@ -548,12 +552,13 @@ class BackupClient {
             Thread {
                 try {
                     persist(backupEntry!!.label, backupEntry.bytes, 10)
-                    backupEntry.callback?.invoke()
+                    backupEntry.callback?.invoke(null)
                 } catch (e: Exception) {
                     LdkEventEmitter.send(
                         EventTypes.native_log,
                         "Remote persist failed for ${label.string} with error ${e.message}"
                     )
+                    backupEntry?.callback?.invoke(e)
                 } finally {
                     backupQueueLock.lock()
                     try {

lib/android/src/main/java/com/reactnativeldk/classes/LdkChannelManagerPersister.kt

@@ -185,14 +185,16 @@ class LdkChannelManagerPersister: ChannelManagerConstructor.EventHandler {
 
     override fun persist_manager(channel_manager_bytes: ByteArray?) {
         if (channel_manager_bytes != null && LdkModule.accountStoragePath != "") {
-            BackupClient.addToPersistQueue(BackupClient.Label.CHANNEL_MANAGER(), channel_manager_bytes) {
-                LdkEventEmitter.send(EventTypes.native_log, "Remote persisted channel manager to disk")
+            BackupClient.addToPersistQueue(BackupClient.Label.CHANNEL_MANAGER(), channel_manager_bytes) { error ->
+                if (error != null) {
+                    LdkEventEmitter.send(EventTypes.native_log, "Failed to remotely persist channel manager to disk")
+                } else {
+                    LdkEventEmitter.send(EventTypes.native_log, "Remote persisted channel manager to disk")
+                }
             }
 
             File(LdkModule.accountStoragePath + "/" + LdkFileNames.channel_manager.fileName).writeBytes(channel_manager_bytes)
-
             LdkEventEmitter.send(EventTypes.native_log, "Locally persisted channel manager to disk")
-            LdkEventEmitter.send(EventTypes.backup, "")
         }
     }
 
@@ -324,8 +326,6 @@ class LdkChannelManagerPersister: ChannelManagerConstructor.EventHandler {
         } catch (e: Exception) {
             LdkEventEmitter.send(EventTypes.native_log, "Error could not read existing ChannelOpenedWithNewCustomKeysManager")
         }
-
-        println("**** existingIds: $existingIds")
     }
 
     private fun channelWasOpenedWithNewCustomKeysManager(channelId: ByteArray): Boolean {

lib/android/src/main/java/com/reactnativeldk/classes/LdkPersister.kt

@@ -24,7 +24,20 @@ class LdkPersister {
 
             val isNew = !file.exists()
 
-            BackupClient.addToPersistQueue(BackupClient.Label.CHANNEL_MONITOR(channelId=channelId), data.write()) {
+            if (BackupClient.skipRemoteBackup) {
+                file.writeBytes(data.write())
+                if (isNew) {
+                    LdkEventEmitter.send(EventTypes.new_channel, body)
+                }
+                return ChannelMonitorUpdateStatus.LDKChannelMonitorUpdateStatus_Completed
+            }
+
+            BackupClient.addToPersistQueue(BackupClient.Label.CHANNEL_MONITOR(channelId=channelId), data.write()) { error ->
+                if (error != null) {
+                    LdkEventEmitter.send(EventTypes.native_log, "Failed to persist channel (${id.to_channel_id().hexEncodedString()}) to remote backup: $error")
+                    return@addToPersistQueue
+                }
+
                 try {
                     file.writeBytes(data.write())
                 } catch (e: Exception) {
@@ -33,20 +46,18 @@ class LdkPersister {
                     return@addToPersistQueue
                 }
 
-                //Update chainmonitor with successful persist
+                //Update chain monitor with successful persist
                 val res = LdkModule.chainMonitor?.channel_monitor_updated(id, update_id)
                 if (res == null || !res.is_ok) {
                     LdkEventEmitter.send(EventTypes.native_log, "Failed to update chain monitor with persisted channel (${id.to_channel_id().hexEncodedString()})")
                 } else {
                     LdkEventEmitter.send(EventTypes.native_log, "Persisted channel (${id.to_channel_id().hexEncodedString()}) to disk")
-                    LdkEventEmitter.send(EventTypes.backup, "")
+                    if (isNew) {
+                        LdkEventEmitter.send(EventTypes.new_channel, body)
+                    }
                 }
             }
 
-            if (isNew) {
-                LdkEventEmitter.send(EventTypes.new_channel, body)
-            }
-
             return ChannelMonitorUpdateStatus.LDKChannelMonitorUpdateStatus_InProgress
         } catch (e: Exception) {
             return ChannelMonitorUpdateStatus.LDKChannelMonitorUpdateStatus_UnrecoverableError

lib/ios/Classes/BackupClient.swift

@@ -145,6 +145,12 @@ class BackupClient {
             urlString = "\(urlString)&channelId=\(id)"
         }
 
+        //Only include files related to this library
+        if method == .list {
+            //TODO add this to android
+            urlString = "\(urlString)&fileGroup=ldk"
+        }
+
         return URL(string: urlString)!
     }
 
@@ -393,7 +399,7 @@ class BackupClient {
 
     static func retrieveCompleteBackup() throws -> CompleteBackup {
         let backedUpFilenames = try listFiles()
-
+                
         var allFiles: [String: Data] = [:]
         var channelFiles: [String: Data] = [:]
 
@@ -586,9 +592,9 @@ class BackupClient {
 
 //Backup queue management
 extension BackupClient {
-    static func addToPersistQueue(_ label: Label, _ bytes: [UInt8], callback: (() -> Void)? = nil) {
+    static func addToPersistQueue(_ label: Label, _ bytes: [UInt8], callback: ((Error?) -> Void)? = nil) {
         guard !skipRemoteBackup else {
-            callback?()
+            callback?(nil)
             LdkEventEmitter.shared.send(withEvent: .native_log, body: "Skipping remote backup queue append for \(label.string)")
             return
         }
@@ -618,9 +624,10 @@ extension BackupClient {
         backupQueue.async {
             do {
                 try persist(label, bytes, retry: 10)
-                callback?()
+                callback?(nil)
             } catch {
-                LdkEventEmitter.shared.send(withEvent: .native_log, body: "Failed to persist channel manager backup. \(error.localizedDescription)")
+                LdkEventEmitter.shared.send(withEvent: .native_log, body: "Failed to persist remote backup \(label.string). \(error.localizedDescription)")
+                callback?(error)
             }
         }
     }