diff --git a/.changes/syncing.md b/.changes/syncing.md new file mode 100644 index 000000000..b80284209 --- /dev/null +++ b/.changes/syncing.md @@ -0,0 +1,6 @@ +--- +"nodejs-binding": patch +--- + +Improve syncing speed and return possible syncing errors that got previously ignored. +Fix parsing in generateAddresses(). \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 0091cc790..85bfaa16c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -284,9 +284,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.57.0" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd4865004a46a0aafb2a0a5eb19d3c9fc46ee5f063a6cfc605c69ac9ecf5263d" +checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" dependencies = [ "bitflags", "cexpr", @@ -370,6 +370,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cache-padded" version = "1.1.1" @@ -396,11 +407,11 @@ dependencies = [ [[package]] name = "cexpr" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.0", ] [[package]] @@ -518,7 +529,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3" dependencies = [ "lazy_static", - "nom", + "nom 5.1.2", "rust-ini", "serde 1.0.130", "serde-hjson", @@ -1337,8 +1348,8 @@ dependencies = [ [[package]] name = "iota-client" -version = "1.1.0" -source = "git+https://github.com/iotaledger/iota.rs?rev=981136e9118a092cd4de6735e616858140b33019#981136e9118a092cd4de6735e616858140b33019" +version = "1.1.1" +source = "git+https://github.com/iotaledger/iota.rs?rev=47e5f6684e2c59237c5fac88be85950a6e5eec4f#47e5f6684e2c59237c5fac88be85950a6e5eec4f" dependencies = [ "async-trait", "bee-common", @@ -1358,6 +1369,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", + "ureq", "url", "zeroize", ] @@ -1511,7 +1523,7 @@ dependencies = [ "futures", "getset", "hex", - "iota-client 1.1.0", + "iota-client 1.1.1", "iota-core", "iota-crypto 0.5.1", "iota-ledger", @@ -1698,13 +1710,15 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "6.17.3" -source = "git+https://github.com/iotaledger/rust-rocksdb?rev=d2c579660e6c72ec1f14f0f2781cef5e3171d380#d2c579660e6c72ec1f14f0f2781cef5e3171d380" +version = "6.20.3" +source = "git+https://github.com/rust-rocksdb/rust-rocksdb?rev=86d983987e7cafce90ad8a147b0b325e6007eba6#86d983987e7cafce90ad8a147b0b325e6007eba6" dependencies = [ "bindgen", + "bzip2-sys", "cc", "glob", "libc", + "libz-sys", ] [[package]] @@ -1719,6 +1733,17 @@ dependencies = [ "walkdir", ] +[[package]] +name = "libz-sys" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2bf61678a0a521c3f7daf815d2e6717d85a272a7dcd02c9768272b32bd1e2a" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.3.0" @@ -1796,6 +1821,12 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -1861,6 +1892,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "nom" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +dependencies = [ + "memchr", + "minimal-lexical", + "version_check", +] + [[package]] name = "ntapi" version = "0.3.6" @@ -2400,8 +2442,8 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.16.0" -source = "git+https://github.com/iotaledger/rust-rocksdb?rev=d2c579660e6c72ec1f14f0f2781cef5e3171d380#d2c579660e6c72ec1f14f0f2781cef5e3171d380" +version = "0.17.0" +source = "git+https://github.com/rust-rocksdb/rust-rocksdb?rev=86d983987e7cafce90ad8a147b0b325e6007eba6#86d983987e7cafce90ad8a147b0b325e6007eba6" dependencies = [ "libc", "librocksdb-sys", @@ -2669,9 +2711,9 @@ dependencies = [ [[package]] name = "shlex" -version = "0.1.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signal-hook-registry" @@ -3180,6 +3222,12 @@ dependencies = [ "getrandom 0.2.3", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.3" @@ -3421,18 +3469,18 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.4.3" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68d9dcec5f9b43a30d38c49f91dfedfaac384cb8f085faca366c26207dd1619" +checksum = "50344758e2f40e3a1fcfc8f6f91aa57b5f8ebd8d27919fe6451f15aaaf9ee608" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.2.2" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65f1a51723ec88c66d5d1fe80c841f17f63587d6691901d66be9bec6c3b51f73" +checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17" dependencies = [ "proc-macro2 1.0.32", "quote 1.0.10", diff --git a/Cargo.toml b/Cargo.toml index 9af950a7a..26ac34a83 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ chrono = { version = "0.4.19", default-features = false, features = ["serde"] } futures = { version = "0.3.17", default-features = false } getset = { version = "0.1.1", default-features = false } hex = { version = "0.4.3", default-features = false } -iota-client = { git = "https://github.com/iotaledger/iota.rs", rev = "981136e9118a092cd4de6735e616858140b33019", default-features = false, features = ["async", "mqtt"] } +iota-client = { git = "https://github.com/iotaledger/iota.rs", rev = "47e5f6684e2c59237c5fac88be85950a6e5eec4f", default-features = false, features = ["async", "mqtt", "tls"] } log = { version = "0.4.14", default-features = false } once_cell = { version = "1.8.0", default-features = false } rand = { version = "0.8.4", default-features = false } diff --git a/bindings/java/Cargo.lock b/bindings/java/Cargo.lock index c4ef213af..a5b745d69 100644 --- a/bindings/java/Cargo.lock +++ b/bindings/java/Cargo.lock @@ -211,7 +211,7 @@ dependencies = [ "digest", "hex", "iota-crypto 0.9.1", - "serde 1.0.130", + "serde 1.0.131", "thiserror", ] @@ -259,7 +259,7 @@ checksum = "8e8479555c8e9bf3cad36daf16e6a2f77771774e977276a08df4d8ac5f1dda66" dependencies = [ "autocfg", "num-traits 0.2.14", - "serde 1.0.130", + "serde 1.0.131", ] [[package]] @@ -282,25 +282,6 @@ dependencies = [ "serde 1.0.131", ] -[[package]] -name = "bindgen" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd4865004a46a0aafb2a0a5eb19d3c9fc46ee5f063a6cfc605c69ac9ecf5263d" -dependencies = [ - "bitflags", - "cexpr 0.4.0", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "proc-macro2 1.0.33", - "quote 1.0.10", - "regex", - "rustc-hash", - "shlex 0.1.1", -] - [[package]] name = "bindgen" version = "0.59.2" @@ -308,7 +289,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" dependencies = [ "bitflags", - "cexpr 0.6.0", + "cexpr", "clang-sys", "lazy_static", "lazycell", @@ -317,7 +298,7 @@ dependencies = [ "quote 1.0.10", "regex", "rustc-hash", - "shlex 1.1.0", + "shlex", ] [[package]] @@ -389,6 +370,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cache-padded" version = "1.1.1" @@ -419,15 +411,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" -[[package]] -name = "cexpr" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" -dependencies = [ - "nom 5.1.2", -] - [[package]] name = "cexpr" version = "0.6.0" @@ -483,7 +466,7 @@ dependencies = [ "libc", "num-integer", "num-traits 0.2.14", - "serde 1.0.130", + "serde 1.0.131", "time 0.1.43", "winapi", ] @@ -1088,7 +1071,7 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89f17b21645bc4ed773c69af9c9a0effd4a3f1a3876eadd453469f8854e7fdd" dependencies = [ - "proc-macro2 1.0.32", + "proc-macro2 1.0.33", "quote 1.0.10", "syn 1.0.82", ] @@ -1414,8 +1397,8 @@ dependencies = [ [[package]] name = "iota-client" -version = "1.1.0" -source = "git+https://github.com/iotaledger/iota.rs?rev=c7d1cc4bae4ef00f16314239463ce115f8c6b35a#c7d1cc4bae4ef00f16314239463ce115f8c6b35a" +version = "1.1.1" +source = "git+https://github.com/iotaledger/iota.rs?rev=47e5f6684e2c59237c5fac88be85950a6e5eec4f#47e5f6684e2c59237c5fac88be85950a6e5eec4f" dependencies = [ "async-trait", "bee-common", @@ -1435,6 +1418,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", + "ureq", "url", "zeroize", ] @@ -1587,7 +1571,7 @@ dependencies = [ "futures", "getset", "hex", - "iota-client 1.1.0", + "iota-client 1.1.1", "iota-core", "iota-crypto 0.5.1", "iota-ledger", @@ -1614,7 +1598,8 @@ dependencies = [ "anyhow", "bee-common", "bee-message", - "bindgen 0.59.2", + "bindgen", + "cc", "chrono", "env_logger", "flapigen", @@ -1815,13 +1800,15 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "6.17.3" -source = "git+https://github.com/iotaledger/rust-rocksdb?rev=d2c579660e6c72ec1f14f0f2781cef5e3171d380#d2c579660e6c72ec1f14f0f2781cef5e3171d380" +version = "6.20.3" +source = "git+https://github.com/rust-rocksdb/rust-rocksdb?rev=86d983987e7cafce90ad8a147b0b325e6007eba6#86d983987e7cafce90ad8a147b0b325e6007eba6" dependencies = [ - "bindgen 0.57.0", + "bindgen", + "bzip2-sys", "cc", "glob", "libc", + "libz-sys", ] [[package]] @@ -1836,6 +1823,17 @@ dependencies = [ "walkdir", ] +[[package]] +name = "libz-sys" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.3.0" @@ -2476,8 +2474,8 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.16.0" -source = "git+https://github.com/iotaledger/rust-rocksdb?rev=d2c579660e6c72ec1f14f0f2781cef5e3171d380#d2c579660e6c72ec1f14f0f2781cef5e3171d380" +version = "0.17.0" +source = "git+https://github.com/rust-rocksdb/rust-rocksdb?rev=86d983987e7cafce90ad8a147b0b325e6007eba6#86d983987e7cafce90ad8a147b0b325e6007eba6" dependencies = [ "libc", "librocksdb-sys", @@ -2741,12 +2739,6 @@ dependencies = [ "opaque-debug", ] -[[package]] -name = "shlex" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" - [[package]] name = "shlex" version = "1.1.0" @@ -3246,7 +3238,7 @@ dependencies = [ "log", "once_cell", "rustls 0.20.2", - "serde 1.0.130", + "serde 1.0.131", "serde_json", "url", "webpki 0.22.0", @@ -3281,6 +3273,12 @@ dependencies = [ "getrandom 0.2.3", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.3" @@ -3449,17 +3447,6 @@ dependencies = [ "libc", ] -[[package]] -name = "which" -version = "4.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea187a8ef279bc014ec368c27a920da2024d2a711109bfbe3440585d5cf27ad9" -dependencies = [ - "either", - "lazy_static", - "libc", -] - [[package]] name = "winapi" version = "0.3.9" @@ -3542,18 +3529,18 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.4.3" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68d9dcec5f9b43a30d38c49f91dfedfaac384cb8f085faca366c26207dd1619" +checksum = "50344758e2f40e3a1fcfc8f6f91aa57b5f8ebd8d27919fe6451f15aaaf9ee608" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.2.2" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65f1a51723ec88c66d5d1fe80c841f17f63587d6691901d66be9bec6c3b51f73" +checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17" dependencies = [ "proc-macro2 1.0.33", "quote 1.0.10", diff --git a/bindings/nodejs/Cargo.lock b/bindings/nodejs/Cargo.lock index 277cffbf9..59894b52d 100644 --- a/bindings/nodejs/Cargo.lock +++ b/bindings/nodejs/Cargo.lock @@ -292,9 +292,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.57.0" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd4865004a46a0aafb2a0a5eb19d3c9fc46ee5f063a6cfc605c69ac9ecf5263d" +checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" dependencies = [ "bitflags", "cexpr", @@ -378,6 +378,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cache-padded" version = "1.1.1" @@ -404,11 +415,11 @@ dependencies = [ [[package]] name = "cexpr" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.0", ] [[package]] @@ -526,7 +537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3" dependencies = [ "lazy_static", - "nom", + "nom 5.1.2", "rust-ini", "serde 1.0.130", "serde-hjson", @@ -1311,8 +1322,8 @@ dependencies = [ [[package]] name = "iota-client" -version = "1.1.0" -source = "git+https://github.com/iotaledger/iota.rs?rev=981136e9118a092cd4de6735e616858140b33019#981136e9118a092cd4de6735e616858140b33019" +version = "1.1.1" +source = "git+https://github.com/iotaledger/iota.rs?rev=47e5f6684e2c59237c5fac88be85950a6e5eec4f#47e5f6684e2c59237c5fac88be85950a6e5eec4f" dependencies = [ "async-trait", "bee-common 0.5.0", @@ -1332,6 +1343,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", + "ureq", "url", "zeroize", ] @@ -1465,7 +1477,7 @@ dependencies = [ "futures", "getset", "hex", - "iota-client 1.1.0", + "iota-client 1.1.1", "iota-core", "iota-crypto 0.5.1", "iota_stronghold", @@ -1604,13 +1616,15 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "6.17.3" -source = "git+https://github.com/iotaledger/rust-rocksdb?rev=d2c579660e6c72ec1f14f0f2781cef5e3171d380#d2c579660e6c72ec1f14f0f2781cef5e3171d380" +version = "6.20.3" +source = "git+https://github.com/rust-rocksdb/rust-rocksdb?rev=86d983987e7cafce90ad8a147b0b325e6007eba6#86d983987e7cafce90ad8a147b0b325e6007eba6" dependencies = [ "bindgen", + "bzip2-sys", "cc", "glob", "libc", + "libz-sys", ] [[package]] @@ -1625,6 +1639,17 @@ dependencies = [ "walkdir", ] +[[package]] +name = "libz-sys" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2bf61678a0a521c3f7daf815d2e6717d85a272a7dcd02c9768272b32bd1e2a" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.3.0" @@ -1693,6 +1718,12 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -1786,6 +1817,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "nom" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +dependencies = [ + "memchr", + "minimal-lexical", + "version_check", +] + [[package]] name = "ntapi" version = "0.3.6" @@ -2265,8 +2307,8 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.16.0" -source = "git+https://github.com/iotaledger/rust-rocksdb?rev=d2c579660e6c72ec1f14f0f2781cef5e3171d380#d2c579660e6c72ec1f14f0f2781cef5e3171d380" +version = "0.17.0" +source = "git+https://github.com/rust-rocksdb/rust-rocksdb?rev=86d983987e7cafce90ad8a147b0b325e6007eba6#86d983987e7cafce90ad8a147b0b325e6007eba6" dependencies = [ "libc", "librocksdb-sys", @@ -2538,9 +2580,9 @@ dependencies = [ [[package]] name = "shlex" -version = "0.1.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signature" @@ -3012,6 +3054,12 @@ dependencies = [ "getrandom 0.2.3", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.3" @@ -3268,18 +3316,18 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.4.3" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68d9dcec5f9b43a30d38c49f91dfedfaac384cb8f085faca366c26207dd1619" +checksum = "50344758e2f40e3a1fcfc8f6f91aa57b5f8ebd8d27919fe6451f15aaaf9ee608" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.2.2" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65f1a51723ec88c66d5d1fe80c841f17f63587d6691901d66be9bec6c3b51f73" +checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17" dependencies = [ "proc-macro2 1.0.32", "quote 1.0.10", diff --git a/bindings/nodejs/lib/binding/account.js b/bindings/nodejs/lib/binding/account.js index ae5484e37..ad79a6dbb 100644 --- a/bindings/nodejs/lib/binding/account.js +++ b/bindings/nodejs/lib/binding/account.js @@ -60,7 +60,7 @@ class Account { } generateAddresses(amount) { - return JSON.parse(generateAddresses.apply(this.account, [amount])); + return generateAddresses.apply(this.account, [amount]).map((address) => JSON.parse(address)); } latestAddress() { diff --git a/bindings/nodejs/src/classes/account/mod.rs b/bindings/nodejs/src/classes/account/mod.rs index 4e258945f..5dcc67439 100644 --- a/bindings/nodejs/src/classes/account/mod.rs +++ b/bindings/nodejs/src/classes/account/mod.rs @@ -526,7 +526,7 @@ pub fn sync(mut cx: FunctionContext) -> JsResult { if let Some(gap_limit) = options.gap_limit { synchronizer = synchronizer.gap_limit(gap_limit); } - let _synced_account = synchronizer.execute().await; + let _synced_account = synchronizer.execute().await.unwrap(); account_wrapper.channel.send(move |mut cx| { let cb = callback.into_inner(&mut cx); diff --git a/bindings/python/native/Cargo.lock b/bindings/python/native/Cargo.lock index c6562da31..312d11d00 100644 --- a/bindings/python/native/Cargo.lock +++ b/bindings/python/native/Cargo.lock @@ -284,9 +284,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.57.0" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd4865004a46a0aafb2a0a5eb19d3c9fc46ee5f063a6cfc605c69ac9ecf5263d" +checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" dependencies = [ "bitflags", "cexpr", @@ -370,6 +370,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cache-padded" version = "1.1.1" @@ -396,11 +407,11 @@ dependencies = [ [[package]] name = "cexpr" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.0", ] [[package]] @@ -518,7 +529,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3" dependencies = [ "lazy_static", - "nom", + "nom 5.1.2", "rust-ini", "serde 1.0.130", "serde-hjson", @@ -1414,8 +1425,8 @@ dependencies = [ [[package]] name = "iota-client" -version = "1.1.0" -source = "git+https://github.com/iotaledger/iota.rs?rev=981136e9118a092cd4de6735e616858140b33019#981136e9118a092cd4de6735e616858140b33019" +version = "1.1.1" +source = "git+https://github.com/iotaledger/iota.rs?rev=47e5f6684e2c59237c5fac88be85950a6e5eec4f#47e5f6684e2c59237c5fac88be85950a6e5eec4f" dependencies = [ "async-trait", "bee-common", @@ -1435,6 +1446,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", + "ureq", "url", "zeroize", ] @@ -1587,7 +1599,7 @@ dependencies = [ "futures", "getset", "hex", - "iota-client 1.1.0", + "iota-client 1.1.1", "iota-core", "iota-crypto 0.5.1", "iota-ledger", @@ -1615,7 +1627,7 @@ dependencies = [ "dict_derive", "futures", "hex", - "iota-client 1.1.0", + "iota-client 1.1.1", "iota-wallet", "once_cell", "pyo3", @@ -1788,13 +1800,15 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "6.17.3" -source = "git+https://github.com/iotaledger/rust-rocksdb?rev=d2c579660e6c72ec1f14f0f2781cef5e3171d380#d2c579660e6c72ec1f14f0f2781cef5e3171d380" +version = "6.20.3" +source = "git+https://github.com/rust-rocksdb/rust-rocksdb?rev=86d983987e7cafce90ad8a147b0b325e6007eba6#86d983987e7cafce90ad8a147b0b325e6007eba6" dependencies = [ "bindgen", + "bzip2-sys", "cc", "glob", "libc", + "libz-sys", ] [[package]] @@ -1809,6 +1823,17 @@ dependencies = [ "walkdir", ] +[[package]] +name = "libz-sys" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.3.0" @@ -1886,6 +1911,12 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -1951,6 +1982,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "nom" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +dependencies = [ + "memchr", + "minimal-lexical", + "version_check", +] + [[package]] name = "ntapi" version = "0.3.6" @@ -2513,8 +2555,8 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.16.0" -source = "git+https://github.com/iotaledger/rust-rocksdb?rev=d2c579660e6c72ec1f14f0f2781cef5e3171d380#d2c579660e6c72ec1f14f0f2781cef5e3171d380" +version = "0.17.0" +source = "git+https://github.com/rust-rocksdb/rust-rocksdb?rev=86d983987e7cafce90ad8a147b0b325e6007eba6#86d983987e7cafce90ad8a147b0b325e6007eba6" dependencies = [ "libc", "librocksdb-sys", @@ -2771,9 +2813,9 @@ dependencies = [ [[package]] name = "shlex" -version = "0.1.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signature" @@ -3262,6 +3304,12 @@ dependencies = [ "getrandom 0.2.3", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.3" @@ -3503,18 +3551,18 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.4.3" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68d9dcec5f9b43a30d38c49f91dfedfaac384cb8f085faca366c26207dd1619" +checksum = "50344758e2f40e3a1fcfc8f6f91aa57b5f8ebd8d27919fe6451f15aaaf9ee608" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.2.2" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65f1a51723ec88c66d5d1fe80c841f17f63587d6691901d66be9bec6c3b51f73" +checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17" dependencies = [ "proc-macro2 1.0.32", "quote 1.0.10", diff --git a/bindings/python/native/Cargo.toml b/bindings/python/native/Cargo.toml index 01001db87..b56f41c03 100644 --- a/bindings/python/native/Cargo.toml +++ b/bindings/python/native/Cargo.toml @@ -20,7 +20,7 @@ chrono = { version = "0.4.19", default-features = false } dict_derive = "0.3.0" futures = { version = "0.3.17", default-features = false } hex = { version = "0.4.3", default-features = false } -iota-client = { git = "https://github.com/iotaledger/iota.rs", rev = "981136e9118a092cd4de6735e616858140b33019", default-features = false, features = ["async", "mqtt"] } +iota-client = { git = "https://github.com/iotaledger/iota.rs", rev = "47e5f6684e2c59237c5fac88be85950a6e5eec4f", default-features = false, features = ["async", "mqtt", "tls"] } iota-wallet = { path = "../../../", version = "0.2.0", features = ["stronghold", "ledger-nano", "ledger-nano-simulator"] } once_cell = { version = "1.8.0", default-features = false } serde = { version = "1.0.130", default-features = false, features = ["derive"] } diff --git a/bindings/python/native/src/types/message.rs b/bindings/python/native/src/types/message.rs index 28282dea4..f7d554db8 100644 --- a/bindings/python/native/src/types/message.rs +++ b/bindings/python/native/src/types/message.rs @@ -471,8 +471,8 @@ pub async fn to_rust_payload( let unlock_blocks: Result> = transaction_payload[0] .unlock_blocks - .to_vec() - .into_iter() + .iter() + .cloned() .map(|u| u.try_into()) .collect(); transaction = transaction.with_unlock_blocks(RustUnlockBlocks::new(unlock_blocks?)?); @@ -489,13 +489,12 @@ pub async fn to_rust_payload( ))) } else { let indexation = RustIndexationPayload::new( - &(&payload + &payload .indexation .as_ref() .unwrap_or_else(|| panic!("Invalid Payload: {:?}", payload))[0] .index - .clone()) - .to_owned(), + .clone(), &payload .indexation .as_ref() diff --git a/documentation/docs/specification.md b/documentation/docs/specification.md index 369c14ff5..6f0a73e1e 100644 --- a/documentation/docs/specification.md +++ b/documentation/docs/specification.md @@ -337,21 +337,20 @@ The following should be considered when initializing an account: | Errors | List of error messages [TBD] | | Required client library methods | None | -#### sync_addresses() +#### check_for_new_used_addresses() Sync addresses with the Tangle. The method ensures that the wallet's local state contains all used addresses and an unused address. The following should be considered when implementing this method: - The updated address history should not be written down in the database/persistent storage. Instead, the method should only return the updated address history (with transaction hashes). This ensures that there are no partial writes to the database. -- To sync addresses for an account from scratch, _index = 0_ and _gap_limit = 10_ should be sent as arguments. -* To sync addresses from the latest address, _index = latest address index_ and _gap_limit = 1_ should be sent as arguments. +- To sync addresses for an account from scratch, _gap_limit = 10_ should be sent as arguments. +* To sync addresses from the latest address, _gap_limit = 1_ should be sent as arguments. ##### Parameters | *Name* | *Required* | *Type* | *Description* | | --------- | ---------- | ------ | ----------------------------------------------------------------------------------------------------- | -| index | ✔ | number | Address index. By default the length of addresses stored for this account should be used as an index. | | gap_limit | ✔ | number | Number of address indexes that are generated. | @@ -372,7 +371,7 @@ The following should be considered when implementing this method: | Errors | List of error messages [TBD] | | Required client library methods | [get_address_balances()](https://github.com/iotaledger/iota.rs/blob/dev/specs/iota-rs-ENGINEERING-SPEC-0000.md#get_address_balances)\| [find_messages()](https://github.com/iotaledger/iota.rs/blob/dev/specs/iota-rs-ENGINEERING-SPEC-0000.md#find_messages) \| [find_outputs()](https://github.com/iotaledger/iota.rs/blob/dev/specs/iota-rs-ENGINEERING-SPEC-0000.md#find_outputs) | -#### sync_messages() +#### sync_addresses_and_messages() Sync messages with the Tangle. The method should ensure that the wallet's local state has all messages associated with the address history. @@ -530,8 +529,8 @@ Please note that it is a proposed design decision to enforce account syncing bef If you want to sync an account, you can use the following process: -1. Sync addresses using [sync_addresses()](#sync_addresses). -2. Sync messages using [sync_messages()](#sync_messages). +1. Sync addresses using [check_for_new_used_addresses()](#check_for_new_used_addresses). +2. Sync messages using [sync_addresses_and_messages()](#sync_messages). 3. Store updated addresses and messages information in persistent storage (if not explicitly set otherwise by the user). ##### Parameters diff --git a/specs/wallet-ENGINEERING-SPEC-0000.md b/specs/wallet-ENGINEERING-SPEC-0000.md index 683c63566..76fc7e456 100644 --- a/specs/wallet-ENGINEERING-SPEC-0000.md +++ b/specs/wallet-ENGINEERING-SPEC-0000.md @@ -33,8 +33,8 @@ - [Account](#account) - [API](#api) - [Initialisation](#initialisation) - - [sync_addresses()](#sync_addresses) - - [sync_messages()](#sync_messages) + - [check_for_new_used_addresses()](#check_for_new_used_addresses) + - [sync_addresses_and_messages()](#sync_messages) - [select_inputs()](#select_inputs) - [send()](#send) - [retry()](#retry) @@ -1125,15 +1125,15 @@ The following should be considered when initialising an account: -#### sync_addresses() +#### check_for_new_used_addresses() Syncs addresses with the Tangle. The method should ensure that the wallet's local state contains all used addresses and an unused address. The following should be considered when implementing this method: * The updated address history should not be written down in the database/persistent storage. Instead the method should only return the updated address history (with transaction hashes). This will ensure that there are no partial writes to the database; -* To sync addresses for an account from scratch, index = 0 and gap_limit = 10 should be provided; -* To sync addresses from the latest address, index = latest address index and gap_limit = 1 should be provided. +* To sync addresses for an account from scratch, gap_limit = 10 should be provided; +* To sync addresses from the latest address, gap_limit = 1 should be provided. @@ -1145,12 +1145,6 @@ The following should be considered when implementing this method: - - - - - - @@ -1202,7 +1196,7 @@ The following should be considered when implementing this method:
Type Description
index✔numberAddress index. By default the length of addresses stored for this account should be used as an index.
gap_limit ✔
-#### sync_messages() +#### sync_addresses_and_messages() Syncs messages with the Tangle. The method should ensure that the wallet's local state has all messages associated with the address history. @@ -1483,8 +1477,8 @@ Note that it is a proposed design decision to enforce account syncing before eve The process for account syncing:_ -* Sync addresses using [sync_addresses()](#syncaddresses); -* Sync messages using [sync_messages()](#syncmessages); +* Sync addresses using [check_for_new_used_addresses()](#syncaddresses); +* Sync messages using [sync_addresses_and_messages()](#syncmessages); * Store updated addresses and messages information in persistent storage (if not explicitly set otherwise by the user). diff --git a/src/account/mod.rs b/src/account/mod.rs index d0bb2f9f3..377e1a0d9 100644 --- a/src/account/mod.rs +++ b/src/account/mod.rs @@ -1221,7 +1221,9 @@ impl Account { /// Gets the account balance information. pub async fn balance(&self) -> crate::Result { let sent_messages = self.list_messages(0, 0, Some(MessageType::Sent)).await?; - Ok(self.balance_internal(&sent_messages).await) + let balance = self.balance_internal(&sent_messages).await; + log::debug!("Account balance: {:?}", balance); + Ok(balance) } /// Updates the account alias. diff --git a/src/account/sync/mod.rs b/src/account/sync/mod.rs index 3d68c97da..d8964e2b2 100644 --- a/src/account/sync/mod.rs +++ b/src/account/sync/mod.rs @@ -61,43 +61,35 @@ async fn get_address_outputs( address: String, client: &Client, fetch_spent_outputs: bool, -) -> crate::Result> { - let address_outputs = client - .get_address() - .outputs( - &address, - AddressOutputsOptions { - include_spent: fetch_spent_outputs, - ..Default::default() - }, - ) - .await? - .to_vec(); - - // fetch only the unspent outputs so we can diff it - if fetch_spent_outputs { - let unspent_address_outputs = client - .get_address() - .outputs( - &address, - AddressOutputsOptions { - include_spent: false, - ..Default::default() - }, - ) - .await?; - let spent_address_outputs: Vec<(UtxoInput, bool)> = address_outputs - .into_iter() - .filter(|o| !unspent_address_outputs.contains(o)) - .map(|o| (o, true)) - .collect(); - - let mut outputs: Vec<(UtxoInput, bool)> = unspent_address_outputs.iter().map(|o| (o.clone(), false)).collect(); - outputs.extend(spent_address_outputs); - Ok(outputs) - } else { - Ok(address_outputs.into_iter().map(|output| (output, false)).collect()) - } +) -> crate::Result> { + let outputs = { + if fetch_spent_outputs { + let spent_and_unspent_outputs = client + .get_address() + .outputs( + &address, + AddressOutputsOptions { + include_spent: false, + ..Default::default() + }, + ) + .await?; + spent_and_unspent_outputs + } else { + let address_outputs = client + .get_address() + .outputs( + &address, + AddressOutputsOptions { + include_spent: fetch_spent_outputs, + ..Default::default() + }, + ) + .await?; + address_outputs + } + }; + Ok(outputs.to_vec()) } async fn get_message(client: &Client, message_id: &MessageId) -> crate::Result> { @@ -134,38 +126,29 @@ pub(crate) async fn sync_address( // if we previously had an output that wasn't returned by the node, mark it as spent if !address_outputs .iter() - .any(|(utxo_input, _)| utxo_input.output_id() == output_id) + .any(|utxo_input| utxo_input.output_id() == output_id) { output.set_is_spent(true); } } let mut tasks = Vec::new(); - for (utxo_input, is_spent) in address_outputs.iter() { + for utxo_input in address_outputs.iter() { let utxo_input = utxo_input.clone(); - // if we already have the output we don't need to get the info from the node - if let Some(existing_output) = outputs.get_mut(utxo_input.output_id()) { - existing_output.set_is_spent(*is_spent); - continue; + if let Some(existing_output) = outputs.get(utxo_input.output_id()) { + // If we have the output already and it got spent, then we don't need to get it again from the node + if existing_output.is_spent { + continue; + } } let client_guard = client_guard.clone(); let bech32_hrp = bech32_hrp.clone(); - let spent = *is_spent; let account_messages = account_messages.clone(); tasks.push(async move { tokio::spawn(async move { let client = client_guard.read().await; - let output = match client.get_output(&utxo_input).await { - Ok(output) => output, - Err(err) => { - if spent { - return Err(crate::Error::SpentOutputNotFound); - } else { - return Err(err.into()); - } - } - }; + let output = client.get_output(&utxo_input).await?; let found_output = AddressOutput::from_output_response(output, bech32_hrp.to_string())?; let message_id = *found_output.message_id(); @@ -196,7 +179,6 @@ pub(crate) async fn sync_address( } for res in futures::future::try_join_all(tasks).await? { - // Don't return an error if we didn't got a spent output match res { Ok((found_output, found_message)) => { outputs.insert(found_output.id()?, found_output); @@ -204,12 +186,14 @@ pub(crate) async fn sync_address( found_messages.push(m); } } - Err(e) => match e { - crate::Error::SpentOutputNotFound => { - log::debug!("[SYNC] output not found in syncing address") + Err(e) => { + // Don't return errors if we sync spent outputs, because they could be pruned already + if !options.sync_spent_outputs { + log::debug!("[SYNC] error during syncing with spent address: {}", e) + } else { + return Err(e); } - _ => return Err(e), - }, + } } } @@ -316,44 +300,62 @@ async fn sync_address_list( Ok((found_addresses, found_messages)) } -/// Syncs addresses with the tangle. +/// Generates new addresses and syncs them with the tangle. /// The method ensures that the wallet local state has all used addresses plus an unused address. /// -/// To sync addresses for an account from scratch, `address_index` = 0 and `gap_limit` = 10 should be provided. -/// To sync addresses from the latest address, `address_index` = latest address index and `gap_limit` = 1 should be -/// provided. +/// To sync addresses for an account from scratch, `gap_limit` = 10 should be provided. +/// To sync addresses later, `gap_limit` = 1 should be provided. /// /// # Arguments /// -/// * `address_index` The address index. /// * `gap_limit` Number of addresses indexes that are generated. /// /// # Return value /// /// Returns a (addresses, messages) tuples representing the address history up to latest unused address, /// and the messages associated with the addresses. -async fn sync_addresses( +async fn check_for_new_used_addresses( account_handle: &AccountHandle, internal: bool, - address_index: usize, gap_limit: usize, options: AccountOptions, return_all_addresses: bool, ) -> crate::Result<(Vec
, Vec)> { - log::debug!("[SYNC] sync_addresses internal: {}", internal); - let mut address_index = address_index; + log::debug!("[SYNC] check_for_new_used_addresses internal: {}", internal); + let account = account_handle.read().await.clone(); + // get the latest address index +1 for public or internal addresses + let mut address_index_to_start_from = if internal { + let internal_addresses = account.addresses.iter().filter(|a| *a.internal()); + let latest_internal_address_index = internal_addresses + .clone() + .max_by_key(|a| a.key_index()) + .map(|a| a.key_index()) + .cloned() + .unwrap_or(0); + latest_internal_address_index + 1 + } else { + let public_addresses = account.addresses.iter().filter(|a| !a.internal()); + let latest_public_address_index = public_addresses + .clone() + .max_by_key(|a| a.key_index()) + .map(|a| a.key_index()) + .cloned() + .unwrap_or(0); + latest_public_address_index + 1 + }; let mut generated_addresses = vec![]; let mut found_messages = vec![]; - let account = account_handle.read().await.clone(); let bech32_hrp = account.bech32_hrp().clone(); drop(account); + // Generate addresses and check if they have outputs, if amount of gap_limit addresses don't have outputs in a row, + // it breaks loop { let mut address_generation_locked = false; let mut generated_iota_addresses = vec![]; // collection of (address_index, address) pairs - for i in address_index..(address_index + gap_limit) { + for i in address_index_to_start_from..(address_index_to_start_from + gap_limit) { // generate addresses let account = account_handle.read().await.clone(); if let Some(address) = get_address_for_sync(&account, bech32_hrp.to_string(), i, internal).await? { @@ -365,6 +367,11 @@ async fn sync_addresses( drop(account); } + if address_generation_locked { + log::debug!("[SYNC] finishing check_for_new_used_addresses because stronghold is locked"); + break; + } + let mut curr_generated_addresses = vec![]; let mut curr_found_messages = vec![]; @@ -381,7 +388,7 @@ async fn sync_addresses( drop(account); let mut addresses_to_sync = Vec::new(); - for (iota_address_index, iota_address) in generated_iota_addresses.to_vec() { + for (iota_address_index, iota_address) in generated_iota_addresses { let outputs = account_addresses .iter() .find(|(a, _)| a == &iota_address) @@ -407,7 +414,7 @@ async fn sync_addresses( curr_generated_addresses.extend(found_addresses_); curr_found_messages.extend(found_messages_); - address_index += gap_limit; + address_index_to_start_from += gap_limit; let is_empty = curr_found_messages.is_empty() && curr_generated_addresses @@ -419,15 +426,10 @@ async fn sync_addresses( if is_empty { log::debug!( - "[SYNC] finishing address syncing because the current messages list and address list are empty" + "[SYNC] finishing check_for_new_used_addresses because the current messages list and address outputs list are empty" ); break; } - - if address_generation_locked { - log::debug!("[SYNC] finishing address syncing because stronghold is locked"); - break; - } } Ok((generated_addresses, found_messages)) @@ -435,7 +437,7 @@ async fn sync_addresses( /// Syncs messages with the tangle. /// The method should ensures that the wallet local state has messages associated with the address history. -async fn sync_messages( +async fn sync_addresses_and_messages( account_handle: &AccountHandle, skip_addresses: &[Address], options: AccountOptions, @@ -444,7 +446,7 @@ async fn sync_messages( // only sync messages for addresses >= this index address_start_index: usize, ) -> crate::Result<(Vec
, Vec)> { - log::debug!("[SYNC] sync_messages"); + log::debug!("[SYNC] sync_addresses_and_messages"); let mut messages = vec![]; let account = account_handle.read().await.clone(); @@ -471,7 +473,10 @@ async fn sync_messages( .filter(|address| address.key_index() >= &address_start_index) .cloned() .collect(); - log::debug!("[SYNC] sync_messages for {} addresses", account_addresses.len()); + log::debug!( + "[SYNC] sync_addresses_and_messages for {} addresses", + account_addresses.len() + ); drop(account); for addresses_chunk in account_addresses .to_vec() @@ -493,75 +498,67 @@ async fn sync_messages( let mut outputs = address.outputs.clone(); tasks.push(async move { - tokio::spawn(async move { + tokio::spawn(async move { let client = client.read().await; - // get address balance first, because it's faster than the outputs endpoint - let address_balance = client.get_address().balance(&address.address().to_bech32()).await?; - let unspent_output_amount = address - .outputs() - .iter() - .map(|(_, o)| o.is_spent) - .filter(|is_spent| !is_spent) - .count(); - - log::debug!( - "[SYNC] syncing messages and outputs for address {} index: {}, got balance: {}, known unspent outputs: {}", - address.address().to_bech32(), - address.key_index(), - address_balance.balance, - unspent_output_amount - ); - // return early if the address has no balance and we don't have any unspent outputs - if address_balance.balance == 0 && unspent_output_amount == 0 { - return crate::Result::Ok((address, vec![])); - } let address_outputs = get_address_outputs(address.address().to_bech32(), &client, options.sync_spent_outputs).await?; + let address_output_ids: Vec = + address_outputs.into_iter().map(|o| *o.output_id()).collect(); - for (output_id, output) in outputs.iter_mut() { - // if we previously had an output that wasn't returned by the node, mark it as spent - if !address_outputs - .iter() - .any(|(utxo_input, _)| utxo_input.output_id() == output_id) - { - output.set_is_spent(true); + // if the node doesn't have this output anymore, then it got pruned and spent + for (output_id, pruned_output) in address + .outputs + .iter() + .filter(|(output_id, _)| !address_output_ids.contains(output_id)) + { + // Only add outputs that aren't set as spent + if !pruned_output.is_spent { + let mut spent_output = pruned_output.clone(); + spent_output.set_is_spent(true); + outputs.insert(*output_id, spent_output); + log::debug!("[SYNC] output {} got pruned, setting it as spent", output_id); } } log::debug!( "[SYNC] syncing messages and outputs for address {}, got {} outputs", address.address().to_bech32(), - address_outputs.len(), + address_output_ids.len(), ); let mut messages = vec![]; - for (utxo_input, is_spent) in address_outputs.iter() { - let output = match address.outputs().get(utxo_input.output_id()) { - // if we already have the output we don't need to get the info from the node - Some(output) => { - let mut output = output.clone(); - output.set_is_spent(*is_spent); - output + for output_id in address_output_ids.iter() { + let output = match client.get_output(&((*output_id).into())).await { + Ok(output) => { + let address_output = AddressOutput::from_output_response( + output, + address.address().bech32_hrp().to_string(), + )?; + address_output } - None => { - // if the output got spent and we didn't get it from the node we ignore it and don't return an error - if *is_spent { - match client.get_output(utxo_input).await{ - Ok(output) => { - AddressOutput::from_output_response(output, address.address().bech32_hrp().to_string())? - } - Err(_) =>{ - log::debug!( - "[SYNC] couldn't get spent output: {}", - utxo_input.output_id().transaction_id().to_string(), - ); - continue; + Err(err) => { + // Don't return errors if we sync spent outputs, because they could be pruned + // already + log::error!("[SYNC] couldn't get output: {}", output_id.transaction_id().to_string(),); + match err { + iota_client::Error::ResponseError(status_code, _) => { + // if the output got pruned and the node doesn't have it anymore, set it as + // spent + if status_code == 404 { + if let Some(output) = address.outputs().get(&(*output_id)) { + let mut output = output.clone(); + output.set_is_spent(true); + output + } else { + // output is unknown, so we can just skip it + continue; + } + } else { + return Err(err.into()); } } - } else { - let output = client.get_output(utxo_input).await?; - AddressOutput::from_output_response(output, address.address().bech32_hrp().to_string())? + err => return Err(err.into()), } } }; @@ -657,24 +654,12 @@ async fn perform_sync( ) .await? } else { - let (found_public_addresses, mut messages) = sync_addresses( - &account_handle, - false, - address_index, - gap_limit, - options, - return_all_addresses, - ) - .await?; - let (found_change_addresses, synced_messages) = sync_addresses( - &account_handle, - true, - address_index, - gap_limit, - options, - return_all_addresses, - ) - .await?; + let (found_public_addresses, mut messages) = + check_for_new_used_addresses(&account_handle, false, gap_limit, options, return_all_addresses) + .await?; + let (found_change_addresses, synced_messages) = + check_for_new_used_addresses(&account_handle, true, gap_limit, options, return_all_addresses) + .await?; let mut found_addresses = found_public_addresses; found_addresses.extend(found_change_addresses); messages.extend(synced_messages); @@ -699,7 +684,7 @@ async fn perform_sync( } if steps.contains(&AccountSynchronizeStep::SyncMessages) { - let (synced_addresses, synced_messages) = sync_messages( + let (synced_addresses, synced_messages) = sync_addresses_and_messages( &account_handle, &found_addresses, options, @@ -946,6 +931,7 @@ async fn perform_sync( addresses_to_save.sort_unstable_by_key(|a| *a.internal()); addresses_to_save.dedup(); + log::debug!("[SYNC] addresses to save: {:#?}", addresses_to_save); log::debug!("[SYNC] perform_sync finished"); Ok(SyncedAccountData { messages: new_messages, @@ -1030,7 +1016,7 @@ impl SyncedAccountData { account: &Account, ) -> crate::Result> { let mut tasks = Vec::new(); - for new_message in self.messages.to_vec() { + for new_message in self.messages.iter().cloned() { let client_options = account.client_options().clone(); let account_id = account.id().to_string(); let account_addresses = account.addresses().to_vec(); @@ -1179,7 +1165,7 @@ impl AccountSynchronizer { } /// Sets the steps to run on the sync process. - /// By default it runs all steps (sync_addresses and sync_messages), + /// By default it runs all steps (check_for_new_used_addresses and sync_messages), /// but the library can pick what to run here. pub(crate) fn steps(mut self, steps: Vec) -> Self { self.steps = steps; diff --git a/src/account_manager.rs b/src/account_manager.rs index 2d25815f2..8bd7ee9a5 100644 --- a/src/account_manager.rs +++ b/src/account_manager.rs @@ -443,7 +443,7 @@ impl AccountManager { balance: metadata.balance, last_checked_address_index: metadata.last_checked_address_index, spent_addresses: metadata.spent_addresses, - inputs: metadata.inputs.into_iter().map(|(_, v)| v).flatten().collect(), + inputs: metadata.inputs.into_iter().flat_map(|(_, v)| v).collect(), }) } @@ -1574,7 +1574,7 @@ impl AccountsSynchronizer { } /// Sets the steps to run on the sync process. - /// By default it runs all steps (sync_addresses and sync_messages), + /// By default it runs all steps (check_for_new_used_addresses and sync_messages), /// but the library can pick what to run here. pub(crate) fn steps(mut self, steps: Vec) -> Self { self.steps.replace(steps); @@ -2042,7 +2042,7 @@ async fn discover_accounts( } } } - log::error!("[SYNC] finished discover_accounts"); + log::debug!("[SYNC] finished discover_accounts"); Ok(synced_accounts) } diff --git a/src/monitor.rs b/src/monitor.rs index 623c9132e..b155fd794 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -195,11 +195,10 @@ async fn process_output(payload: String, account_handle: AccountHandle) -> crate let input_addresses: Vec = essence .inputs() .iter() - .map(|input| match input { + .filter_map(|input| match input { TransactionInput::Utxo(i) => i.metadata.as_ref().map(|m| m.address.clone()), _ => unimplemented!(), }) - .flatten() .collect(); let mut addresses = output_addresses; addresses.extend(input_addresses);