From 4424a6da4096429668a21bcd792326868cb30fdc Mon Sep 17 00:00:00 2001 From: ok300 <106775972+ok300@users.noreply.github.com> Date: Fri, 15 Nov 2024 11:00:00 +0100 Subject: [PATCH 1/3] Persist SendPay.groupid as TEXT to cover full u64 range --- libs/sdk-core/src/greenlight/node_api.rs | 6 +++--- libs/sdk-core/src/persist/migrations.rs | 27 ++++++++++++++++++++++++ libs/sdk-core/src/persist/send_pays.rs | 10 +++++++-- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/libs/sdk-core/src/greenlight/node_api.rs b/libs/sdk-core/src/greenlight/node_api.rs index 658fb6d1d..f2f68d053 100644 --- a/libs/sdk-core/src/greenlight/node_api.rs +++ b/libs/sdk-core/src/greenlight/node_api.rs @@ -808,7 +808,7 @@ impl Greenlight { let mut key = hex::encode(&p.payment_hash); key.push('|'); key.push_str(&p.groupid.to_string()); - (key, (p.payment_hash.clone(), p.groupid)) + (key, (p.payment_hash.clone(), p.groupid.to_string())) }) .collect(); let hash_group_values: Vec<_> = hash_groups.values().cloned().collect(); @@ -840,7 +840,7 @@ impl Greenlight { for send_pay in send_pays { let mut key = hex::encode(&send_pay.payment_hash); key.push('|'); - key.push_str(&send_pay.groupid.to_string()); + key.push_str(&send_pay.groupid); let payment = outbound_payments.entry(key).or_insert(SendPayAgg { state: 0, created_at: send_pay.created_at, @@ -2147,7 +2147,7 @@ impl TryFrom for SendPay { .created_index .ok_or(NodeError::generic("missing created index"))?, updated_index: value.updated_index, - groupid: value.groupid, + groupid: value.groupid.to_string(), partid: value.partid, payment_hash: value.payment_hash, status: value.status.try_into()?, diff --git a/libs/sdk-core/src/persist/migrations.rs b/libs/sdk-core/src/persist/migrations.rs index 75f4d4fc4..ca9120e9d 100644 --- a/libs/sdk-core/src/persist/migrations.rs +++ b/libs/sdk-core/src/persist/migrations.rs @@ -457,6 +457,33 @@ pub(crate) fn current_migrations() -> Vec<&'static str> { ", "DELETE FROM payments", "DELETE FROM cached_items WHERE key = 'sync_state'", + // Change groupid type to TEXT + " + ALTER TABLE send_pays RENAME TO send_pays_old; + + CREATE TABLE send_pays ( + created_index INTEGER PRIMARY KEY NOT NULL, + updated_index INTEGER, + groupid TEXT NOT NULL, + partid INTEGER, + payment_hash BLOB NOT NULL, + status INTEGER NOT NULL, + amount_msat INTEGER, + destination BLOB, + created_at INTEGER NOT NULL, + amount_sent_msat INTEGER, + label TEXT, + bolt11 TEXT, + description TEXT, + bolt12 TEXT, + payment_preimage BLOB, + erroronion BLOB + ) STRICT; + + INSERT INTO send_pays SELECT * FROM send_pays_old; + + DROP TABLE send_pays_old; + ", ] } diff --git a/libs/sdk-core/src/persist/send_pays.rs b/libs/sdk-core/src/persist/send_pays.rs index e4055c8e3..fadb9be16 100644 --- a/libs/sdk-core/src/persist/send_pays.rs +++ b/libs/sdk-core/src/persist/send_pays.rs @@ -14,7 +14,13 @@ pub(crate) enum SendPayStatus { pub(crate) struct SendPay { pub created_index: u64, pub updated_index: Option, - pub groupid: u64, + + // The cln_grpc groupid (u64) can contain 64 bit values (e.g. > 2^63). This leads to errors when + // trying to persist them in an SQLite INTEGER column, which holds signed 64 bit numbers (63 bit + // value + 1 bit for the sign). + // To map the full range of u64 values in SQLite, we treat this field as String, stored as TEXT. + pub groupid: String, + pub partid: Option, pub payment_hash: Vec, pub status: SendPayStatus, @@ -74,7 +80,7 @@ impl SqliteStorage { pub(crate) fn list_send_pays( &self, - hash_groups: &[(Vec, u64)], + hash_groups: &[(Vec, String)], ) -> PersistResult> { let conn = self.get_connection()?; let mut stmt = conn.prepare( From 7c16a500c01f39518353e3d2067d9ce99e7b03c9 Mon Sep 17 00:00:00 2001 From: ok300 <106775972+ok300@users.noreply.github.com> Date: Fri, 15 Nov 2024 13:38:10 +0100 Subject: [PATCH 2/3] Don't keep old send_pays data, force re-sync --- libs/sdk-core/src/persist/migrations.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libs/sdk-core/src/persist/migrations.rs b/libs/sdk-core/src/persist/migrations.rs index ca9120e9d..ee1d322e8 100644 --- a/libs/sdk-core/src/persist/migrations.rs +++ b/libs/sdk-core/src/persist/migrations.rs @@ -457,9 +457,10 @@ pub(crate) fn current_migrations() -> Vec<&'static str> { ", "DELETE FROM payments", "DELETE FROM cached_items WHERE key = 'sync_state'", - // Change groupid type to TEXT + // Delete send_pays, re-create it with groupid column as TEXT " ALTER TABLE send_pays RENAME TO send_pays_old; + DROP TABLE send_pays_old; CREATE TABLE send_pays ( created_index INTEGER PRIMARY KEY NOT NULL, @@ -480,9 +481,7 @@ pub(crate) fn current_migrations() -> Vec<&'static str> { erroronion BLOB ) STRICT; - INSERT INTO send_pays SELECT * FROM send_pays_old; - - DROP TABLE send_pays_old; + DELETE FROM cached_items WHERE key = 'sync_state'; ", ] } From d1a52f5f0f04e90e0f5a758729dd7c75b6680e60 Mon Sep 17 00:00:00 2001 From: ok300 <106775972+ok300@users.noreply.github.com> Date: Fri, 15 Nov 2024 14:52:15 +0000 Subject: [PATCH 3/3] Update libs/sdk-core/src/persist/migrations.rs Co-authored-by: Jesse de Wit --- libs/sdk-core/src/persist/migrations.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/sdk-core/src/persist/migrations.rs b/libs/sdk-core/src/persist/migrations.rs index ee1d322e8..f24235448 100644 --- a/libs/sdk-core/src/persist/migrations.rs +++ b/libs/sdk-core/src/persist/migrations.rs @@ -459,8 +459,7 @@ pub(crate) fn current_migrations() -> Vec<&'static str> { "DELETE FROM cached_items WHERE key = 'sync_state'", // Delete send_pays, re-create it with groupid column as TEXT " - ALTER TABLE send_pays RENAME TO send_pays_old; - DROP TABLE send_pays_old; + DROP TABLE send_pays; CREATE TABLE send_pays ( created_index INTEGER PRIMARY KEY NOT NULL,