Backup your account without a second device

Cargo.toml

@@ -72,6 +72,10 @@ web-time = "1.1"
 # - https://github.com/seanmonstar/reqwest/issues/2159
 # - https://github.com/hyperium/tonic/pull/1974
 # - https://github.com/rustls/rustls-platform-verifier/issues/58
+async-compression = { default-features = false, version = "0.4", features = [
+  "futures-io",
+  "zstd",
+] }
 bincode = "1.3"
 console_error_panic_hook = "0.1"
 const_format = "0.2"

bindings_ffi/src/lib.rs

@@ -56,6 +56,10 @@ pub enum GenericError {
     DeviceSync(#[from] xmtp_mls::groups::device_sync::DeviceSyncError),
     #[error(transparent)]
     Identity(#[from] xmtp_mls::identity::IdentityError),
+    #[error(transparent)]
+    JoinError(#[from] tokio::task::JoinError),
+    #[error(transparent)]
+    IoError(#[from] tokio::io::Error),
 }
 
 #[derive(uniffi::Error, thiserror::Error, Debug)]

bindings_ffi/src/mls.rs

@@ -18,7 +18,9 @@ use xmtp_id::{
     },
     InboxId,
 };
+use xmtp_mls::groups::device_sync::backup::{BackupImporter, BackupOptions};
 use xmtp_mls::groups::device_sync::preference_sync::UserPreferenceUpdate;
+use xmtp_mls::groups::device_sync::ENC_KEY_SIZE;
 use xmtp_mls::groups::scoped_client::LocalScopedGroupClient;
 use xmtp_mls::groups::HmacKey;
 use xmtp_mls::storage::group::ConversationType;
@@ -49,6 +51,7 @@ use xmtp_mls::{
     },
     AbortHandle, GenericStreamHandle, StreamHandle,
 };
+use xmtp_proto::xmtp::device_sync::{BackupElementSelection, BackupMetadata};
 use xmtp_proto::xmtp::mls::message_contents::content_types::ReactionV2;
 use xmtp_proto::xmtp::mls::message_contents::{DeviceSyncKind, EncodedContent};
 pub type RustXmtpClient = MlsClient<TonicApiClient>;
@@ -377,11 +380,13 @@ impl FfiXmtpClient {
         Ok(result.into())
     }
 
+    /// A utility function to sign a piece of text with this installation's private key.
     pub fn sign_with_installation_key(&self, text: &str) -> Result<Vec<u8>, GenericError> {
         let inner = self.inner_client.as_ref();
         Ok(inner.context().sign_with_public_context(text)?)
     }
 
+    /// A utility function to easily verify that a piece of text was signed by this installation.
     pub fn verify_signed_with_installation_key(
         &self,
         signature_text: &str,
@@ -393,6 +398,8 @@ impl FfiXmtpClient {
         self.verify_signed_with_public_key(signature_text, signature_bytes, public_key)
     }
 
+    /// A utility function to easily verify that a string has been signed by another libXmtp installation.
+    /// Only works for verifying libXmtp public context signatures.
     pub fn verify_signed_with_public_key(
         &self,
         signature_text: &str,
@@ -454,6 +461,7 @@ impl FfiXmtpClient {
         Ok(())
     }
 
+    /// Manually trigger a device sync request to sync records from another active device on this account.
     pub async fn send_sync_request(&self, kind: FfiDeviceSyncKind) -> Result<(), GenericError> {
         let provider = self.inner_client.mls_provider()?;
         self.inner_client
@@ -493,7 +501,7 @@ impl FfiXmtpClient {
         Ok(())
     }
 
-    /// Revokes or removes an identity - really a wallet address - from the existing client
+    /// Revokes or removes an identity from the existing client
     pub async fn revoke_wallet(
         &self,
         wallet_address: &str,
@@ -556,6 +564,111 @@ impl FfiXmtpClient {
             scw_verifier: self.inner_client.scw_verifier().clone().clone(),
         }))
     }
+
+    /// Backup your application to file for later restoration.
+    pub async fn backup_to_file(
+        &self,
+        path: String,
+        opts: FfiBackupOptions,
+        key: Vec<u8>,
+    ) -> Result<(), GenericError> {
+        let provider = self.inner_client.mls_provider()?;
+        let opts: BackupOptions = opts.into();
+        opts.export_to_file(provider, path, &check_key(key)?)
+            .await?;
+
+        Ok(())
+    }
+
+    /// Import a previous backup
+    pub async fn import_from_file(&self, path: String, key: Vec<u8>) -> Result<(), GenericError> {
+        let provider = self.inner_client.mls_provider()?;
+        let mut importer = BackupImporter::from_file(path, &check_key(key)?).await?;
+        importer.insert(&provider).await?;
+        Ok(())
+    }
+
+    /// Load the metadata for a backup to see what it contains.
+    /// Reads only the metadata without loading the entire file, so this function is quick.
+    pub async fn backup_metadata(
+        &self,
+        path: String,
+        key: Vec<u8>,
+    ) -> Result<FfiBackupMetadata, GenericError> {
+        let importer = BackupImporter::from_file(path, &check_key(key)?).await?;
+        Ok(importer.metadata.into())
+    }
+}
+
+fn check_key(mut key: Vec<u8>) -> Result<Vec<u8>, GenericError> {
+    if key.len() < 32 {
+        return Err(GenericError::Generic {
+            err: format!(
+                "The encryption key must be at least {} bytes long.",
+                ENC_KEY_SIZE
+            ),
+        });
+    }
+    key.truncate(ENC_KEY_SIZE);
+    Ok(key)
+}
+
+#[derive(uniffi::Record)]
+pub struct FfiBackupMetadata {
+    backup_version: u32,
+    elements: Vec<FfiBackupElementSelection>,
+    exported_at_ns: i64,
+    start_ns: Option<i64>,
+    end_ns: Option<i64>,
+}
+impl From<BackupMetadata> for FfiBackupMetadata {
+    fn from(value: BackupMetadata) -> Self {
+        Self {
+            backup_version: value.backup_version,
+            elements: value.elements().map(Into::into).collect(),
+            start_ns: value.start_ns,
+            end_ns: value.end_ns,
+            exported_at_ns: value.exported_at_ns,
+        }
+    }
+}
+
+#[derive(uniffi::Record)]
+pub struct FfiBackupOptions {
+    start_ns: Option<i64>,
+    end_ns: Option<i64>,
+    elements: Vec<FfiBackupElementSelection>,
+}
+impl From<FfiBackupOptions> for BackupOptions {
+    fn from(value: FfiBackupOptions) -> Self {
+        Self {
+            start_ns: value.start_ns,
+            end_ns: value.start_ns,
+            elements: value.elements.into_iter().map(Into::into).collect(),
+        }
+    }
+}
+
+#[derive(uniffi::Enum)]
+pub enum FfiBackupElementSelection {
+    Messages,
+    Consent,
+}
+impl From<FfiBackupElementSelection> for BackupElementSelection {
+    fn from(value: FfiBackupElementSelection) -> Self {
+        match value {
+            FfiBackupElementSelection::Consent => Self::Consent,
+            FfiBackupElementSelection::Messages => Self::Messages,
+        }
+    }
+}
+impl From<BackupElementSelection> for FfiBackupElementSelection {
+    fn from(value: BackupElementSelection) -> Self {
+        match value {
+            BackupElementSelection::Consent => Self::Consent,
+            BackupElementSelection::Messages => Self::Messages,
+        }
+    }
 }
 
 impl From<HmacKey> for FfiHmacKey {

bindings_wasm/Cargo.toml

@@ -11,14 +11,14 @@ console_error_panic_hook.workspace = true
 hex.workspace = true
 js-sys.workspace = true
 prost.workspace = true
-serde-wasm-bindgen = "0.6.5"
 serde.workspace = true
-tokio.workspace = true
+serde-wasm-bindgen = "0.6.5"
+tokio = { workspace = true, features = ["io-util"] }
+tracing.workspace = true
 tracing-subscriber = { workspace = true, features = ["env-filter", "json"] }
 tracing-web = "0.1"
-tracing.workspace = true
-wasm-bindgen-futures.workspace = true
 wasm-bindgen.workspace = true
+wasm-bindgen-futures.workspace = true
 xmtp_api_http = { path = "../xmtp_api_http" }
 xmtp_common.workspace = true
 xmtp_cryptography = { path = "../xmtp_cryptography" }

dev/docker/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   node:
     image: xmtp/node-go:latest
@@ -59,7 +57,7 @@ services:
     environment:
       POSTGRES_PASSWORD: xmtp
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s
       timeout: 5s
       retries: 5
@@ -93,6 +91,6 @@ services:
       chain:
         condition: service_started
       replicationdb:
-          condition: service_healthy
+        condition: service_healthy
     ports:
-      - 5050:5050
+      - 5050:5050

dev/gen_protos.sh

@@ -6,7 +6,7 @@ if ! cargo install --list | grep "protoc-gen-prost-crate" > /dev/null; then
     fi
 fi
 
-if ! buf generate https://github.com/xmtp/proto.git#branch=main,subdir=proto; then
+if ! buf generate https://github.com/xmtp/proto.git#branch=coda/backups,subdir=proto; then
     echo "Failed to generate protobuf definitions"
     exit 1
 fi

xmtp_id/src/associations/serialization.rs

@@ -1,7 +1,3 @@
-use std::collections::{HashMap, HashSet};
-
-use crate::scw_verifier::ValidationResponse;
-
 use super::{
     member::Member,
     signature::{AccountId, ValidatedLegacySignedPublicKey},
@@ -19,8 +15,10 @@ use super::{
     verified_signature::VerifiedSignature,
     MemberIdentifier, SignatureError,
 };
+use crate::scw_verifier::ValidationResponse;
 use prost::{DecodeError, Message};
 use regex::Regex;
+use std::collections::{HashMap, HashSet};
 use thiserror::Error;
 use xmtp_cryptography::signature::sanitize_evm_addresses;
 use xmtp_proto::xmtp::{

xmtp_mls/Cargo.toml

@@ -24,7 +24,7 @@ bench = [
     "dep:fdlimit",
     "dep:ethers",
     "dep:const_format",
-    "xmtp_common/bench"
+    "xmtp_common/bench",
 ]
 default = ["grpc-api"]
 grpc-api = ["dep:xmtp_api_grpc"]
@@ -45,6 +45,7 @@ update-schema = ["toml"]
 
 [dependencies]
 aes-gcm = { version = "0.10.3", features = ["std"] }
+async-compression.workspace = true
 async-stream.workspace = true
 async-trait.workspace = true
 bincode.workspace = true
@@ -107,19 +108,23 @@ diesel = { workspace = true, features = [
     "r2d2",
     "returning_clauses_for_sqlite_3_35",
     "sqlite",
-    "32-column-tables"
+    "32-column-tables",
 ] }
 dyn-clone.workspace = true
 libsqlite3-sys = { workspace = true }
 openmls.workspace = true
-openssl-sys.workspace = true
 openssl.workspace = true
+openssl-sys.workspace = true
 tokio = { workspace = true, features = [
+    "io-util",
     "macros",
     "tracing",
     "rt",
     "rt-multi-thread",
 ] }
+tokio-util = { version = "0.7", default-features = false, features = [
+    "compat",
+] }
 xmtp_api_grpc = { path = "../xmtp_api_grpc", optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]