From a50a0d3d28e16bfcee4aa9d30d7d654eb36bb439 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 8 Mar 2023 17:53:40 +0800 Subject: [PATCH 01/43] feat: database garbage collection --- Cargo.lock | 206 ++++++++++++++++++---------------- node/db/Cargo.toml | 3 +- node/db/src/lib.rs | 33 +----- node/db/src/rolling/impls.rs | 154 +++++++++++++++++++++++++ node/db/src/rolling/index.rs | 2 + node/db/src/rolling/mod.rs | 64 +++++++++++ node/db/tests/mem_test.rs | 12 -- node/db/tests/parity_test.rs | 12 -- node/db/tests/rocks_test.rs | 12 -- node/db/tests/subtests/mod.rs | 40 ------- 10 files changed, 337 insertions(+), 201 deletions(-) create mode 100644 node/db/src/rolling/impls.rs create mode 100644 node/db/src/rolling/index.rs create mode 100644 node/db/src/rolling/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 6f782b717caa..45a2988c8402 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,9 +264,9 @@ dependencies = [ [[package]] name = "asn1-rs" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf6690c370453db30743b373a60ba498fc0d6d83b11f4abfd87a84a075db5dd4" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" dependencies = [ "asn1-rs-derive 0.4.0", "asn1-rs-impl", @@ -395,12 +395,11 @@ dependencies = [ [[package]] name = "async-lock" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685" +checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" dependencies = [ "event-listener", - "futures-lite", ] [[package]] @@ -470,9 +469,9 @@ checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" [[package]] name = "async-trait" -version = "0.1.64" +version = "0.1.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +checksum = "b84f9ebcc6c1f5b8cb160f6990096a5c127f423fcb6e1ccc46c370cbdfb75dfc" dependencies = [ "proc-macro2", "quote", @@ -559,9 +558,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.9" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6137c6234afb339e75e764c866e3594900f0211e1315d33779f269bbe2ec6967" +checksum = "2fb79c228270dcf2426e74864cabc94babb5dbab01a4314e702d2f16540e1591" dependencies = [ "async-trait", "axum-core", @@ -588,16 +587,16 @@ dependencies = [ "tokio", "tokio-tungstenite", "tower", - "tower-http", + "tower-http 0.3.5", "tower-layer", "tower-service", ] [[package]] name = "axum-core" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34" +checksum = "b2f958c80c248b34b9a877a643811be8dbca03ca5ba827f2b63baf3a81e5fc4e" dependencies = [ "async-trait", "bytes 1.4.0", @@ -1923,9 +1922,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" +checksum = "9a140f260e6f3f79013b8bfc65e7ce630c9ab4388c6a89c71e07226f49487b72" dependencies = [ "cc", "cxxbridge-flags", @@ -1935,9 +1934,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" +checksum = "da6383f459341ea689374bf0a42979739dc421874f112ff26f829b8040b8e613" dependencies = [ "cc", "codespan-reporting", @@ -1950,15 +1949,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" +checksum = "90201c1a650e95ccff1c8c0bb5a343213bdd317c6e600a93075bca2eff54ec97" [[package]] name = "cxxbridge-macro" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" +checksum = "0b75aed41bb2e6367cae39e6326ef817a851db13c13e4f3263714ca3cfb8de56" dependencies = [ "proc-macro2", "quote", @@ -2064,11 +2063,11 @@ dependencies = [ [[package]] name = "der-parser" -version = "8.1.0" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d4bc9b0db0a0df9ae64634ac5bdefb7afcb534e182275ca0beadbe486701c1" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" dependencies = [ - "asn1-rs 0.5.1", + "asn1-rs 0.5.2", "displaydoc", "nom", "num-bigint", @@ -2311,9 +2310,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "dtoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00704156a7de8df8da0911424e30c2049957b0a714542a44e05fe693dd85313" +checksum = "65d09067bfacaa79114679b279d7f5885b53295b1e2cfb4e79c8e4bd3d633169" [[package]] name = "dyn-clone" @@ -2461,9 +2460,9 @@ dependencies = [ [[package]] name = "erased-serde" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ca605381c017ec7a5fef5e548f1cfaa419ed0f6df6367339300db74c92aa7d" +checksum = "4f2b0c2380453a92ea8b6c8e5f64ecaafccddde8ceab55ff7a8ac1029f894569" dependencies = [ "serde", ] @@ -2585,7 +2584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ef1a30ae415c3a691a4f41afddc2dbcd6d70baf338368d85ebc1e8ed92cedb9" dependencies = [ "cfg-if 1.0.0", - "rustix 0.36.8", + "rustix 0.36.9", "windows-sys 0.45.0", ] @@ -2611,9 +2610,9 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a214f5bb88731d436478f3ae1f8a277b62124089ba9fb67f4f93fb100ef73c90" +checksum = "54b2f3c51e4dd999930845da5d10a48775b8fe4ca9f4f9ec1f9161f334da5dfe" [[package]] name = "fil_actor_account_v10" @@ -3557,7 +3556,7 @@ dependencies = [ "time 0.3.20", "tokio", "toml 0.7.2", - "tower-http", + "tower-http 0.4.0", "tracing-appender", "tracing-loki", "tracing-subscriber", @@ -3583,6 +3582,7 @@ version = "0.6.0" dependencies = [ "ahash 0.8.3", "anyhow", + "chrono", "cid", "forest_libp2p_bitswap", "fvm_ipld_blockstore", @@ -4506,7 +4506,7 @@ dependencies = [ [[package]] name = "fvm" version = "3.0.0" -source = "git+https://github.com/ChainSafe/ref-fvm?branch=lemmih-wibbles#2bd2bd557bf0a318ea1b4fa076e6cdaba11fb2e9" +source = "git+https://github.com/ChainSafe/ref-fvm?branch=lemmih-wibbles#3da7a1047837328e1395acf372a41a3b8b630ad1" dependencies = [ "anyhow", "blake2b_simd", @@ -4790,7 +4790,7 @@ dependencies = [ [[package]] name = "fvm_shared" version = "3.0.0" -source = "git+https://github.com/ChainSafe/ref-fvm?branch=lemmih-wibbles#2bd2bd557bf0a318ea1b4fa076e6cdaba11fb2e9" +source = "git+https://github.com/ChainSafe/ref-fvm?branch=lemmih-wibbles#3da7a1047837328e1395acf372a41a3b8b630ad1" dependencies = [ "anyhow", "arbitrary", @@ -5404,9 +5404,9 @@ checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074" [[package]] name = "io-lifetimes" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" +checksum = "cfa919a82ea574332e2de6e74b4c36e74d41982b335080fa59d4ef31be20fdf3" dependencies = [ "libc", "windows-sys 0.45.0", @@ -5437,8 +5437,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" dependencies = [ "hermit-abi 0.3.1", - "io-lifetimes 1.0.5", - "rustix 0.36.8", + "io-lifetimes 1.0.6", + "rustix 0.36.9", "windows-sys 0.45.0", ] @@ -5462,9 +5462,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "jobserver" @@ -6381,7 +6381,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b20a59d985586e4a5aef64564ac77299f8586d8be6cf9106a5a40207e8908efb" dependencies = [ - "rustix 0.36.8", + "rustix 0.36.9", ] [[package]] @@ -6732,9 +6732,9 @@ dependencies = [ [[package]] name = "netlink-sys" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "260e21fbb6f3d253a14df90eb0000a6066780a15dd901a7519ce02d77a94985b" +checksum = "6471bf08e7ac0135876a9581bf3217ef0333c191c128d34878079f42ee150411" dependencies = [ "bytes 1.4.0", "futures", @@ -6973,7 +6973,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" dependencies = [ - "asn1-rs 0.5.1", + "asn1-rs 0.5.2", ] [[package]] @@ -7205,9 +7205,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" [[package]] name = "pathdiff" @@ -7486,9 +7486,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78" +checksum = "4ebcd279d20a4a0a2404a33056388e950504d891c855c7975b9a8fef75f3bf04" dependencies = [ "proc-macro2", "syn", @@ -7562,7 +7562,7 @@ dependencies = [ "byteorder", "hex", "lazy_static", - "rustix 0.36.8", + "rustix 0.36.9", ] [[package]] @@ -7984,9 +7984,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -7994,9 +7994,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -8436,13 +8436,13 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.8" +version = "0.36.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +checksum = "fd5c6ff11fecd55b40746d1995a02f2eb375bf8c00d192d521ee09f42bef37bc" dependencies = [ "bitflags", "errno", - "io-lifetimes 1.0.5", + "io-lifetimes 1.0.6", "libc", "linux-raw-sys 0.1.4", "windows-sys 0.45.0", @@ -8496,9 +8496,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" [[package]] name = "rustyline" @@ -8555,9 +8555,9 @@ checksum = "53bc79743f9a66c2fb1f951cd83735f275d46bfe466259fbc5897bb60a0d00ee" [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "ryu-js" @@ -8609,9 +8609,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "sct" @@ -8702,9 +8702,9 @@ checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" [[package]] name = "serde" -version = "1.0.152" +version = "1.0.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "3a382c72b4ba118526e187430bb4963cd6d55051ebf13d9b25574d379cc98d20" dependencies = [ "serde_derive", ] @@ -8741,9 +8741,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "1ef476a5790f0f6decbc66726b6e5d63680ed518283e64c7df415989d880954f" dependencies = [ "proc-macro2", "quote", @@ -8764,9 +8764,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ "itoa", "ryu", @@ -8775,18 +8775,18 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341" +checksum = "db0969fff533976baadd92e08b1d102c5a3d8a8049eadfd69d4d1e3c5b2ed189" dependencies = [ "serde", ] [[package]] name = "serde_repr" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a5ec9fa74a20ebbe5d9ac23dac1fc96ba0ecfe9f50f2843b52e537b10fbcb4e" +checksum = "395627de918015623b32e7669714206363a7fc00382bf477e72c1f7533e8eafc" dependencies = [ "proc-macro2", "quote", @@ -9135,9 +9135,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -9467,7 +9467,7 @@ dependencies = [ "cfg-if 1.0.0", "fastrand", "redox_syscall", - "rustix 0.36.8", + "rustix 0.36.9", "windows-sys 0.42.0", ] @@ -9488,18 +9488,18 @@ checksum = "95059e91184749cb66be6dc994f67f182b6d897cb3df74a5bf66b5e709295fd8" [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "a5ab016db510546d856297882807df8da66a16fb8c4101cb8b30054b0d5b2d9c" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e" dependencies = [ "proc-macro2", "quote", @@ -9616,9 +9616,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes 1.4.0", @@ -9632,7 +9632,7 @@ dependencies = [ "socket2", "tokio-macros", "tracing", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -9800,6 +9800,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +dependencies = [ + "bitflags", + "bytes 1.4.0", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite 0.2.9", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-http" version = "0.4.0" @@ -9820,7 +9839,6 @@ dependencies = [ "pin-project-lite 0.2.9", "tokio", "tokio-util", - "tower", "tower-layer", "tower-service", "tracing", @@ -10085,9 +10103,9 @@ checksum = "2281c8c1d221438e373249e065ca4989c4c36952c211ff21a0ee91c44a3869e7" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -10407,9 +10425,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.101.1" +version = "0.102.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf2f22ef84ac5666544afa52f326f13e16f3d019d2e61e704fd8091c9358b130" +checksum = "48134de3d7598219ab9eaf6b91b15d8e50d31da76b8519fe4ecfcec2cf35104b" dependencies = [ "indexmap", "url", @@ -10417,12 +10435,12 @@ dependencies = [ [[package]] name = "wasmprinter" -version = "0.2.52" +version = "0.2.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "003f2e37b9b7caac949d388e185ecd9139f51441249a23880b0cf38e10cdf647" +checksum = "5aa44d546e4e4479f2e91035fa497c0a05cffbf22413ad05bf0b06a789b9118f" dependencies = [ "anyhow", - "wasmparser 0.101.1", + "wasmparser 0.102.0", ] [[package]] @@ -10680,7 +10698,7 @@ dependencies = [ "byteorder", "ccm", "curve25519-dalek 3.2.0", - "der-parser 8.1.0", + "der-parser 8.2.0", "elliptic-curve", "hkdf", "hmac 0.12.1", @@ -11088,9 +11106,9 @@ checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "winnow" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf09497b8f8b5ac5d3bb4d05c0a99be20f26fd3d5f2db7b0716e946d5103658" +checksum = "ee7b2c67f962bf5042bfd8b6a916178df33a26eec343ae064cb8e069f638fa6f" dependencies = [ "memchr", ] @@ -11160,10 +11178,10 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0ecbeb7b67ce215e40e3cc7f2ff902f94a223acf44995934763467e7b1febc8" dependencies = [ - "asn1-rs 0.5.1", + "asn1-rs 0.5.2", "base64 0.13.1", "data-encoding", - "der-parser 8.1.0", + "der-parser 8.2.0", "lazy_static", "nom", "oid-registry 0.6.1", diff --git a/node/db/Cargo.toml b/node/db/Cargo.toml index b2de74147988..ce026fac3935 100644 --- a/node/db/Cargo.toml +++ b/node/db/Cargo.toml @@ -25,6 +25,7 @@ paritydb = ["dep:parity-db"] [dependencies] ahash.workspace = true anyhow.workspace = true +chrono.workspace = true cid.workspace = true forest_libp2p_bitswap.workspace = true fvm_ipld_blockstore.workspace = true @@ -38,7 +39,7 @@ serde = { workspace = true, features = ["derive"] } thiserror.workspace = true # optional -parity-db = { version = "0.4", default-features = false, optional = true } +parity-db = { version = "=0.4.3", default-features = false, optional = true } rocksdb = { version = "0.20", default-features = false, optional = true } [dev-dependencies] diff --git a/node/db/src/lib.rs b/node/db/src/lib.rs index a12d9b3b7726..9b714b91b534 100644 --- a/node/db/src/lib.rs +++ b/node/db/src/lib.rs @@ -17,6 +17,9 @@ pub mod rocks_config; pub use errors::Error; pub use memory::MemoryDB; +mod rolling; +pub use rolling::*; + /// Store interface used as a KV store implementation pub trait Store { /// Read single value from data store and return `None` if key doesn't @@ -41,14 +44,6 @@ pub trait Store { where K: AsRef<[u8]>; - /// Read slice of keys and return a vector of optional values. - fn bulk_read(&self, keys: &[K]) -> Result>>, Error> - where - K: AsRef<[u8]>, - { - keys.iter().map(|key| self.read(key)).collect() - } - /// Write slice of KV pairs. fn bulk_write( &self, @@ -59,14 +54,6 @@ pub trait Store { .try_for_each(|(key, value)| self.write(key.into(), value.into())) } - /// Bulk delete keys from the data store. - fn bulk_delete(&self, keys: &[K]) -> Result<(), Error> - where - K: AsRef<[u8]>, - { - keys.iter().try_for_each(|key| self.delete(key)) - } - /// Flush writing buffer if there is any. Default implementation is blank fn flush(&self) -> Result<(), Error> { Ok(()) @@ -103,26 +90,12 @@ impl Store for &BS { (*self).exists(key) } - fn bulk_read(&self, keys: &[K]) -> Result>>, Error> - where - K: AsRef<[u8]>, - { - (*self).bulk_read(keys) - } - fn bulk_write( &self, values: impl IntoIterator>, impl Into>)>, ) -> Result<(), Error> { (*self).bulk_write(values) } - - fn bulk_delete(&self, keys: &[K]) -> Result<(), Error> - where - K: AsRef<[u8]>, - { - (*self).bulk_delete(keys) - } } /// Traits for collecting DB stats diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs new file mode 100644 index 000000000000..acc5f5f2a75d --- /dev/null +++ b/node/db/src/rolling/impls.rs @@ -0,0 +1,154 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use cid::Cid; +use forest_libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite}; + +use super::*; +use crate::*; + +impl Blockstore for RollingDB { + fn has(&self, k: &Cid) -> anyhow::Result { + for db in self.dbs.iter() { + if let Ok(true) = Blockstore::has(db, k) { + return Ok(true); + } + } + + Ok(false) + } + + fn get(&self, k: &Cid) -> anyhow::Result>> { + for db in self.dbs.iter() { + if let Ok(Some(v)) = Blockstore::get(db, k) { + return Ok(Some(v)); + } + } + + Ok(None) + } + + fn put( + &self, + mh_code: cid::multihash::Code, + block: &fvm_ipld_blockstore::Block, + ) -> anyhow::Result + where + Self: Sized, + D: AsRef<[u8]>, + { + Blockstore::put(self.current(), mh_code, block) + } + + fn put_many(&self, blocks: I) -> anyhow::Result<()> + where + Self: Sized, + D: AsRef<[u8]>, + I: IntoIterator)>, + { + Blockstore::put_many(self.current(), blocks) + } + + fn put_many_keyed(&self, blocks: I) -> anyhow::Result<()> + where + Self: Sized, + D: AsRef<[u8]>, + I: IntoIterator, + { + Blockstore::put_many_keyed(self.current(), blocks) + } + + fn put_keyed(&self, k: &Cid, block: &[u8]) -> anyhow::Result<()> { + Blockstore::put_keyed(self.current(), k, block) + } +} + +impl Store for RollingDB { + fn read(&self, key: K) -> Result>, crate::Error> + where + K: AsRef<[u8]>, + { + for db in self.dbs.iter() { + if let Ok(Some(v)) = Store::read(db, key.as_ref()) { + return Ok(Some(v)); + } + } + + Ok(None) + } + + fn exists(&self, key: K) -> Result + where + K: AsRef<[u8]>, + { + for db in self.dbs.iter() { + if let Ok(true) = Store::exists(db, key.as_ref()) { + return Ok(true); + } + } + + Ok(false) + } + + fn write(&self, key: K, value: V) -> Result<(), crate::Error> + where + K: AsRef<[u8]>, + V: AsRef<[u8]>, + { + Store::write(self.current(), key, value) + } + + fn delete(&self, key: K) -> Result<(), crate::Error> + where + K: AsRef<[u8]>, + { + Store::delete(self.current(), key) + } + + fn bulk_write( + &self, + values: impl IntoIterator>, impl Into>)>, + ) -> Result<(), crate::Error> { + Store::bulk_write(self.current(), values) + } + + fn flush(&self) -> Result<(), crate::Error> { + Store::flush(self.current()) + } +} + +impl BitswapStoreRead for RollingDB { + fn contains(&self, cid: &Cid) -> anyhow::Result { + for db in self.dbs.iter() { + if let Ok(true) = BitswapStoreRead::contains(db, cid) { + return Ok(true); + } + } + + Ok(false) + } + + fn get(&self, cid: &Cid) -> anyhow::Result>> { + for db in self.dbs.iter() { + if let Ok(Some(v)) = BitswapStoreRead::get(db, cid) { + return Ok(Some(v)); + } + } + + Ok(None) + } +} + +impl BitswapStoreReadWrite for RollingDB { + type Params = ::Params; + + fn insert(&self, block: &libipld::Block) -> anyhow::Result<()> { + BitswapStoreReadWrite::insert(self.current(), block) + } +} + +impl DBStatistics for RollingDB { + fn get_statistics(&self) -> Option { + DBStatistics::get_statistics(self.current()) + } +} diff --git a/node/db/src/rolling/index.rs b/node/db/src/rolling/index.rs new file mode 100644 index 000000000000..3ec2696da862 --- /dev/null +++ b/node/db/src/rolling/index.rs @@ -0,0 +1,2 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT diff --git a/node/db/src/rolling/mod.rs b/node/db/src/rolling/mod.rs new file mode 100644 index 000000000000..83e0f3657195 --- /dev/null +++ b/node/db/src/rolling/mod.rs @@ -0,0 +1,64 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +mod impls; +mod index; + +use std::{ + collections::VecDeque, + path::{Path, PathBuf}, + time::SystemTime, +}; + +use chrono::Utc; +use fvm_ipld_blockstore::Blockstore; + +use crate::db_engine::{open_db, Db, DbConfig}; + +pub struct RollingDB { + db_root: PathBuf, + db_config: DbConfig, + dbs: VecDeque, +} + +impl RollingDB { + pub fn load_or_create(db_root: PathBuf, db_config: DbConfig) -> anyhow::Result { + let dbs = load_dbs(db_root.as_path()); + let mut rolling = Self { + db_root, + db_config, + dbs, + }; + + if rolling.dbs.is_empty() { + let name = Utc::now().timestamp(); + let db = open_db(&rolling.db_root.join(name.to_string()), &rolling.db_config)?; + rolling.add_as_current(db)?; + } + + Ok(rolling) + } + + pub fn add_as_current(&mut self, db: Db) -> anyhow::Result<()> { + self.dbs.push_front(db); + self.flush_index_to_file() + } + + fn current(&self) -> &Db { + self.dbs + .get(0) + .expect("RollingDB should contain at least one DB reference") + } + + fn flush_index_to_file(&self) -> anyhow::Result<()> { + Ok(()) + } +} + +fn load_dbs(db_root: &Path) -> VecDeque { + todo!() +} + +fn get_db_index_name(db_root: &Path) -> PathBuf { + db_root.join("db_index.yaml") +} diff --git a/node/db/tests/mem_test.rs b/node/db/tests/mem_test.rs index 2449c1346743..c6f90ad21da0 100644 --- a/node/db/tests/mem_test.rs +++ b/node/db/tests/mem_test.rs @@ -40,15 +40,3 @@ fn mem_db_bulk_write() { let db = MemoryDB::default(); subtests::bulk_write(&db); } - -#[test] -fn mem_db_bulk_read() { - let db = MemoryDB::default(); - subtests::bulk_read(&db); -} - -#[test] -fn mem_db_bulk_delete() { - let db = MemoryDB::default(); - subtests::bulk_delete(&db); -} diff --git a/node/db/tests/parity_test.rs b/node/db/tests/parity_test.rs index 38dd0d6c8cfe..8b0625014520 100644 --- a/node/db/tests/parity_test.rs +++ b/node/db/tests/parity_test.rs @@ -46,16 +46,4 @@ mod paritydb_tests { let db = TempParityDB::new(); subtests::bulk_write(&*db); } - - #[test] - fn db_bulk_read() { - let db = TempParityDB::new(); - subtests::bulk_read(&*db); - } - - #[test] - fn db_bulk_delete() { - let db = TempParityDB::new(); - subtests::bulk_delete(&*db); - } } diff --git a/node/db/tests/rocks_test.rs b/node/db/tests/rocks_test.rs index 18934db82633..31863b5ba112 100644 --- a/node/db/tests/rocks_test.rs +++ b/node/db/tests/rocks_test.rs @@ -47,16 +47,4 @@ mod rocksdb_tests { let db = TempRocksDB::new(); subtests::bulk_write(&*db); } - - #[test] - fn db_bulk_read() { - let db = TempRocksDB::new(); - subtests::bulk_read(&*db); - } - - #[test] - fn db_bulk_delete() { - let db = TempRocksDB::new(); - subtests::bulk_delete(&*db); - } } diff --git a/node/db/tests/subtests/mod.rs b/node/db/tests/subtests/mod.rs index 926b8b1d8079..decd989e91fe 100644 --- a/node/db/tests/subtests/mod.rs +++ b/node/db/tests/subtests/mod.rs @@ -68,43 +68,3 @@ where assert!(res); } } - -pub fn bulk_read(db: &DB) -where - DB: Store, -{ - let keys = [[0], [1], [2]]; - let values = [[0], [1], [2]]; - let kvs: Vec<_> = keys - .iter() - .zip(values.iter()) - .map(|(k, v)| (k.to_vec(), v.to_vec())) - .collect(); - db.bulk_write(kvs).unwrap(); - let results = db.bulk_read(&keys).unwrap(); - for (result, value) in results.iter().zip(values.iter()) { - match result { - Some(v) => assert_eq!(v, value), - None => panic!("No values found!"), - } - } -} - -pub fn bulk_delete(db: &DB) -where - DB: Store, -{ - let keys = [[0], [1], [2]]; - let values = [[0], [1], [2]]; - let kvs: Vec<_> = keys - .iter() - .zip(values.iter()) - .map(|(k, v)| (k.to_vec(), v.to_vec())) - .collect(); - db.bulk_write(kvs).unwrap(); - db.bulk_delete(&keys).unwrap(); - for k in keys.iter() { - let res = db.exists(*k).unwrap(); - assert!(!res); - } -} From d27067378f0462dad1971d17ce292aa31c8245bc Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 8 Mar 2023 20:47:29 +0800 Subject: [PATCH 02/43] rolling db impl and tests --- Cargo.lock | 24 +++ Cargo.toml | 3 +- node/db/Cargo.toml | 5 + node/db/src/rolling/impls.rs | 186 ++++++++++++++++++- node/db/src/rolling/mod.rs | 53 +----- node/db/tests/rolling_test.rs | 75 ++++++++ utils/forest_utils/src/db/file_backed_obj.rs | 77 ++++++++ utils/forest_utils/src/db/mod.rs | 2 + 8 files changed, 374 insertions(+), 51 deletions(-) create mode 100644 node/db/tests/rolling_test.rs create mode 100644 utils/forest_utils/src/db/file_backed_obj.rs diff --git a/Cargo.lock b/Cargo.lock index 45a2988c8402..c23092293acf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3585,7 +3585,10 @@ dependencies = [ "chrono", "cid", "forest_libp2p_bitswap", + "forest_utils", + "fs_extra", "fvm_ipld_blockstore", + "human-repr", "lazy_static", "libipld", "log", @@ -3593,8 +3596,10 @@ dependencies = [ "parity-db", "parking_lot 0.12.1", "prometheus", + "rand 0.8.5", "rocksdb", "serde", + "serde_yaml", "tempfile", "thiserror", ] @@ -8863,6 +8868,19 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_yaml" +version = "0.9.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82e6c8c047aa50a7328632d067bcae6ef38772a79e28daf32f735e0e4f3dd10" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "serialization_tests" version = "0.6.0" @@ -10154,6 +10172,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad2024452afd3874bf539695e04af6732ba06517424dbf958fdb16a01f3bef6c" + [[package]] name = "unsigned-varint" version = "0.7.1" diff --git a/Cargo.toml b/Cargo.toml index a66b1ce517ea..c1849fd18dba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,7 +63,7 @@ bls-signatures = { version = "0.12", default-features = false, features = ["blst byteorder = "1.4.3" bytes = "1.2" cfg-if = "1" -chrono = { version = "0.4", default-features = false, features = [] } +chrono = { version = "0.4", default-features = false, features = ["clock"] } cid = { version = "0.8", default-features = false, features = ["std"] } clap = { version = "4.0", features = ["derive"] } console-subscriber = { version = "0.1", features = ["parking_lot"] } @@ -128,6 +128,7 @@ serde_ipld_dagcbor = "0.2" serde_json = "1.0" serde_repr = "0.1.8" serde_with = { version = "2.0.1", features = ["chrono_0_4"] } +serde_yaml = "0.9" sha2 = { version = "0.10.5", default-features = false } tempfile = "3.4" thiserror = "1.0" diff --git a/node/db/Cargo.toml b/node/db/Cargo.toml index ce026fac3935..cd2fa20e8240 100644 --- a/node/db/Cargo.toml +++ b/node/db/Cargo.toml @@ -28,7 +28,10 @@ anyhow.workspace = true chrono.workspace = true cid.workspace = true forest_libp2p_bitswap.workspace = true +forest_utils.workspace = true +fs_extra.workspace = true fvm_ipld_blockstore.workspace = true +human-repr.workspace = true lazy_static.workspace = true libipld.workspace = true log.workspace = true @@ -36,6 +39,7 @@ num_cpus.workspace = true parking_lot.workspace = true prometheus = { workspace = true } serde = { workspace = true, features = ["derive"] } +serde_yaml.workspace = true thiserror.workspace = true # optional @@ -43,4 +47,5 @@ parity-db = { version = "=0.4.3", default-features = false, optional = true } rocksdb = { version = "0.20", default-features = false, optional = true } [dev-dependencies] +rand.workspace = true tempfile.workspace = true diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs index acc5f5f2a75d..0c99fd49c1be 100644 --- a/node/db/src/rolling/impls.rs +++ b/node/db/src/rolling/impls.rs @@ -1,15 +1,19 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +use chrono::Utc; use cid::Cid; use forest_libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite}; +use forest_utils::db::file_backed_obj::FileBackedObject; +use fvm_ipld_blockstore::Blockstore; +use human_repr::HumanCount; use super::*; use crate::*; impl Blockstore for RollingDB { fn has(&self, k: &Cid) -> anyhow::Result { - for db in self.dbs.iter() { + for db in self.db_queue.iter() { if let Ok(true) = Blockstore::has(db, k) { return Ok(true); } @@ -19,7 +23,7 @@ impl Blockstore for RollingDB { } fn get(&self, k: &Cid) -> anyhow::Result>> { - for db in self.dbs.iter() { + for db in self.db_queue.iter() { if let Ok(Some(v)) = Blockstore::get(db, k) { return Ok(Some(v)); } @@ -68,7 +72,7 @@ impl Store for RollingDB { where K: AsRef<[u8]>, { - for db in self.dbs.iter() { + for db in self.db_queue.iter() { if let Ok(Some(v)) = Store::read(db, key.as_ref()) { return Ok(Some(v)); } @@ -81,7 +85,7 @@ impl Store for RollingDB { where K: AsRef<[u8]>, { - for db in self.dbs.iter() { + for db in self.db_queue.iter() { if let Ok(true) = Store::exists(db, key.as_ref()) { return Ok(true); } @@ -119,7 +123,7 @@ impl Store for RollingDB { impl BitswapStoreRead for RollingDB { fn contains(&self, cid: &Cid) -> anyhow::Result { - for db in self.dbs.iter() { + for db in self.db_queue.iter() { if let Ok(true) = BitswapStoreRead::contains(db, cid) { return Ok(true); } @@ -129,7 +133,7 @@ impl BitswapStoreRead for RollingDB { } fn get(&self, cid: &Cid) -> anyhow::Result>> { - for db in self.dbs.iter() { + for db in self.db_queue.iter() { if let Ok(Some(v)) = BitswapStoreRead::get(db, cid) { return Ok(Some(v)); } @@ -152,3 +156,173 @@ impl DBStatistics for RollingDB { DBStatistics::get_statistics(self.current()) } } + +impl FileBackedObject for DbIndex { + fn serialize(&self) -> anyhow::Result> { + Ok(serde_yaml::to_string(self)?.as_bytes().to_vec()) + } + + fn deserialize(bytes: &[u8]) -> anyhow::Result { + Ok(serde_yaml::from_slice(bytes)?) + } +} + +impl Drop for RollingDB { + fn drop(&mut self) { + if let Err(err) = self.flush() { + warn!( + "Error flushing rolling db under {}: {err}", + self.db_root.display() + ); + } + } +} + +impl RollingDB { + pub fn load_or_create(db_root: PathBuf, db_config: DbConfig) -> anyhow::Result { + let (db_index, db_queue) = load_db_queue(db_root.as_path(), &db_config)?; + let mut rolling = Self { + db_root, + db_config, + db_index, + db_queue, + }; + + if rolling.db_queue.is_empty() { + let (name, db) = rolling.create_untracked()?; + rolling.track_as_current(name, db)?; + } + rolling.ensure_db_index_integrity()?; + Ok(rolling) + } + + pub fn track_as_current(&mut self, name: String, db: Db) -> anyhow::Result<()> { + self.db_queue.push_front(db); + self.db_index + .inner_mut_or_default() + .db_names + .push_front(name); + self.db_index.flush_to_file() + } + + pub fn create_untracked(&self) -> anyhow::Result<(String, Db)> { + let name = Utc::now().timestamp_millis().to_string(); + let db = open_db(&self.db_root.join(&name), &self.db_config)?; + Ok((name, db)) + } + + pub fn clean_tracked(&mut self, n_db_to_reserve: usize, delete: bool) -> anyhow::Result<()> { + anyhow::ensure!(n_db_to_reserve > 0); + + while self.db_queue.len() > n_db_to_reserve { + if let Some(db) = self.db_queue.pop_back() { + db.flush()?; + } + if let Some(db_index) = self.db_index.inner_mut() { + if let Some(name) = db_index.db_names.pop_back() { + info!("Closing DB {name}"); + if delete { + let db_path = self.db_root.join(name); + delete_db(&db_path); + } + } + } + self.ensure_db_index_integrity()?; + } + + self.db_index.flush_to_file() + } + + pub fn clean_untracked(&self) -> anyhow::Result<()> { + if let Ok(dir) = std::fs::read_dir(&self.db_root) { + dir.flatten() + .filter(|entry| { + entry.path().is_dir() + && self + .db_index + .inner() + .as_ref() + .map(|db_index| { + db_index + .db_names + .iter() + .all(|name| entry.path() != self.db_root.join(name).as_path()) + }) + .unwrap_or_default() + }) + .for_each(|entry| delete_db(&entry.path())); + } + Ok(()) + } + + pub fn size_in_bytes(&self) -> anyhow::Result { + Ok(fs_extra::dir::get_size(&self.db_root)?) + } + + pub fn len(&self) -> usize { + self.db_queue.len() + } + + fn current(&self) -> &Db { + self.db_queue + .get(0) + .expect("RollingDB should contain at least one DB reference") + } + + fn ensure_db_index_integrity(&self) -> anyhow::Result<()> { + anyhow::ensure!( + self.db_queue.len() + == self + .db_index + .inner() + .as_ref() + .map(|index| index.db_names.len()) + .unwrap_or_default() + ); + Ok(()) + } +} + +fn load_db_queue( + db_root: &Path, + db_config: &DbConfig, +) -> anyhow::Result<(FileBacked, VecDeque)> { + let mut db_index = FileBacked::load_from_file_or_new(db_root.join("db_index.yaml"))?; + let mut db_queue = VecDeque::new(); + let index_inner_mut: &mut DbIndex = db_index.inner_mut_or_default(); + for i in (0..index_inner_mut.db_names.len()).rev() { + let name = index_inner_mut.db_names[i].as_str(); + let db_path = db_root.join(name); + if !db_path.is_dir() { + index_inner_mut.db_names.remove(i); + continue; + } + match open_db(&db_path, db_config) { + Ok(db) => db_queue.push_front(db), + Err(err) => { + index_inner_mut.db_names.remove(i); + warn!("Failed to open database under {}: {err}", db_path.display()); + } + } + } + + db_index.flush_to_file()?; + Ok((db_index, db_queue)) +} + +fn delete_db(db_path: &Path) { + let size = fs_extra::dir::get_size(db_path).unwrap_or_default(); + if let Err(err) = std::fs::remove_dir_all(db_path) { + warn!( + "Error deleting database under {}, size: {}. {err}", + db_path.display(), + size.human_count_bytes() + ); + } else { + info!( + "Deleted database under {}, size: {}", + db_path.display(), + size.human_count_bytes() + ); + } +} diff --git a/node/db/src/rolling/mod.rs b/node/db/src/rolling/mod.rs index 83e0f3657195..652522ac10aa 100644 --- a/node/db/src/rolling/mod.rs +++ b/node/db/src/rolling/mod.rs @@ -7,58 +7,23 @@ mod index; use std::{ collections::VecDeque, path::{Path, PathBuf}, - time::SystemTime, }; -use chrono::Utc; -use fvm_ipld_blockstore::Blockstore; +use forest_utils::db::file_backed_obj::FileBacked; +use log::{info, warn}; +use serde::{Deserialize, Serialize}; use crate::db_engine::{open_db, Db, DbConfig}; pub struct RollingDB { db_root: PathBuf, db_config: DbConfig, - dbs: VecDeque, + db_index: FileBacked, + /// A queue of active databases, from youngest to oldest + db_queue: VecDeque, } -impl RollingDB { - pub fn load_or_create(db_root: PathBuf, db_config: DbConfig) -> anyhow::Result { - let dbs = load_dbs(db_root.as_path()); - let mut rolling = Self { - db_root, - db_config, - dbs, - }; - - if rolling.dbs.is_empty() { - let name = Utc::now().timestamp(); - let db = open_db(&rolling.db_root.join(name.to_string()), &rolling.db_config)?; - rolling.add_as_current(db)?; - } - - Ok(rolling) - } - - pub fn add_as_current(&mut self, db: Db) -> anyhow::Result<()> { - self.dbs.push_front(db); - self.flush_index_to_file() - } - - fn current(&self) -> &Db { - self.dbs - .get(0) - .expect("RollingDB should contain at least one DB reference") - } - - fn flush_index_to_file(&self) -> anyhow::Result<()> { - Ok(()) - } -} - -fn load_dbs(db_root: &Path) -> VecDeque { - todo!() -} - -fn get_db_index_name(db_root: &Path) -> PathBuf { - db_root.join("db_index.yaml") +#[derive(Debug, Default, Serialize, Deserialize)] +struct DbIndex { + db_names: VecDeque, } diff --git a/node/db/tests/rolling_test.rs b/node/db/tests/rolling_test.rs new file mode 100644 index 000000000000..f519b553022c --- /dev/null +++ b/node/db/tests/rolling_test.rs @@ -0,0 +1,75 @@ +#[cfg(test)] +mod tests { + use std::{thread::sleep, time::Duration}; + + use anyhow::*; + use cid::{multihash::MultihashDigest, Cid}; + use forest_db::RollingDB; + use forest_libp2p_bitswap::BitswapStoreRead; + use fvm_ipld_blockstore::Blockstore; + use rand::Rng; + use tempfile::TempDir; + + #[test] + fn rolling_db_behaviour_tests() -> Result<()> { + let db_root = TempDir::new()?; + println!("Creating rolling db under {}", db_root.path().display()); + let mut rolling_db = RollingDB::load_or_create(db_root.path().into(), Default::default())?; + println!("Generating random blocks"); + let pairs: Vec<_> = (0..1000) + .map(|_| { + let mut bytes = [0; 1024]; + rand::rngs::OsRng.fill(&mut bytes); + let cid = + Cid::new_v0(cid::multihash::Code::Sha2_256.digest(bytes.as_slice())).unwrap(); + (cid, bytes.to_vec()) + }) + .collect(); + + let split_index = 500; + + for (i, (k, block)) in pairs.iter().enumerate() { + if i == split_index { + sleep(Duration::from_millis(1)); + println!("Creating another inner db"); + let (name, db) = rolling_db.create_untracked()?; + rolling_db.track_as_current(name, db)?; + } + rolling_db.put_keyed(k, block)?; + } + + for (i, (k, block)) in pairs.iter().enumerate() { + ensure!(rolling_db.contains(k)?, "{i}"); + ensure!( + Blockstore::get(&rolling_db, k)?.unwrap().as_slice() == block, + "{i}" + ); + } + + rolling_db.clean_tracked(1, false)?; + ensure!(rolling_db.len() == 1); + + for (i, (k, _)) in pairs.iter().enumerate() { + if i < split_index { + ensure!(!rolling_db.contains(k)?, "{i}"); + } else { + ensure!(rolling_db.contains(k)?, "{i}"); + } + } + + rolling_db.clean_untracked()?; + drop(rolling_db); + + let rolling_db = RollingDB::load_or_create(db_root.path().into(), Default::default())?; + ensure!(rolling_db.len() == 1); + for (i, (k, _)) in pairs.iter().enumerate() { + if i < split_index { + ensure!(!rolling_db.contains(k)?); + } else { + ensure!(rolling_db.contains(k)?); + } + } + + Ok(()) + } +} diff --git a/utils/forest_utils/src/db/file_backed_obj.rs b/utils/forest_utils/src/db/file_backed_obj.rs new file mode 100644 index 000000000000..25ae33dbe508 --- /dev/null +++ b/utils/forest_utils/src/db/file_backed_obj.rs @@ -0,0 +1,77 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use std::path::PathBuf; + +use log::warn; + +pub struct FileBacked { + inner: Option, + path: PathBuf, +} + +impl FileBacked { + /// Gets a borrow of the inner object + pub fn inner(&self) -> &Option { + &self.inner + } + + /// Gets a mutable borrow of the inner object + pub fn inner_mut(&mut self) -> &mut Option { + &mut self.inner + } + + /// Sets the inner object and flushes to file + pub fn set_inner(&mut self, inner: T) -> anyhow::Result<()> { + self.inner = Some(inner); + self.flush_to_file() + } + + /// Creates a new file backed object + pub fn new(inner: Option, path: PathBuf) -> Self { + Self { inner, path } + } + + /// Loads an object from a file and creates a new instance + pub fn load_from_file_or_new(path: PathBuf) -> anyhow::Result { + if path.is_file() { + let bytes = std::fs::read(path.as_path())?; + Ok(Self { + inner: T::deserialize(&bytes) + .map_err(|e| { + warn!("Error loading object from {}", path.display()); + e + }) + .ok(), + path, + }) + } else { + Ok(Self { inner: None, path }) + } + } + + /// Flushes the object to the file + pub fn flush_to_file(&self) -> anyhow::Result<()> { + if let Some(inner) = &self.inner { + let bytes = inner.serialize()?; + Ok(std::fs::write(&self.path, bytes)?) + } else { + anyhow::bail!("Inner object is not set") + } + } +} + +impl FileBacked { + pub fn inner_mut_or_default(&mut self) -> &mut T { + self.inner_mut().get_or_insert_with(Default::default) + } +} + +/// An object that is backed by a single file on disk +pub trait FileBackedObject: Sized { + /// Serializes into a byte array + fn serialize(&self) -> anyhow::Result>; + + /// Deserializes from a byte array + fn deserialize(bytes: &[u8]) -> anyhow::Result; +} diff --git a/utils/forest_utils/src/db/mod.rs b/utils/forest_utils/src/db/mod.rs index 60227ab6f4ce..9580e19e63d1 100644 --- a/utils/forest_utils/src/db/mod.rs +++ b/utils/forest_utils/src/db/mod.rs @@ -1,6 +1,8 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +pub mod file_backed_obj; + use cid::{ multihash::{Code, MultihashDigest}, Cid, From da3ce6ae52a753d1ef5f045cb76ef221d9f52702 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 8 Mar 2023 20:52:02 +0800 Subject: [PATCH 03/43] fix lints --- node/db/src/rolling/impls.rs | 2 +- node/db/tests/rolling_test.rs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs index 0c99fd49c1be..3e3a9d942e4b 100644 --- a/node/db/src/rolling/impls.rs +++ b/node/db/src/rolling/impls.rs @@ -259,7 +259,7 @@ impl RollingDB { Ok(fs_extra::dir::get_size(&self.db_root)?) } - pub fn len(&self) -> usize { + pub fn size(&self) -> usize { self.db_queue.len() } diff --git a/node/db/tests/rolling_test.rs b/node/db/tests/rolling_test.rs index f519b553022c..681f440613ef 100644 --- a/node/db/tests/rolling_test.rs +++ b/node/db/tests/rolling_test.rs @@ -1,3 +1,6 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + #[cfg(test)] mod tests { use std::{thread::sleep, time::Duration}; @@ -47,7 +50,7 @@ mod tests { } rolling_db.clean_tracked(1, false)?; - ensure!(rolling_db.len() == 1); + ensure!(rolling_db.size() == 1); for (i, (k, _)) in pairs.iter().enumerate() { if i < split_index { @@ -61,7 +64,7 @@ mod tests { drop(rolling_db); let rolling_db = RollingDB::load_or_create(db_root.path().into(), Default::default())?; - ensure!(rolling_db.len() == 1); + ensure!(rolling_db.size() == 1); for (i, (k, _)) in pairs.iter().enumerate() { if i < split_index { ensure!(!rolling_db.contains(k)?); From 621c11fa557666dd0ba3ea2ce53d4a231470436d Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 9 Mar 2023 00:44:11 +0800 Subject: [PATCH 04/43] ri FileBacked change --- node/db/src/lib.rs | 4 +- node/db/src/rolling/impls.rs | 43 ++++--------- node/db/tests/rolling_test.rs | 3 +- utils/forest_utils/src/db/file_backed_obj.rs | 64 ++++++++++++-------- 4 files changed, 57 insertions(+), 57 deletions(-) diff --git a/node/db/src/lib.rs b/node/db/src/lib.rs index 9b714b91b534..d167fa119637 100644 --- a/node/db/src/lib.rs +++ b/node/db/src/lib.rs @@ -17,8 +17,8 @@ pub mod rocks_config; pub use errors::Error; pub use memory::MemoryDB; -mod rolling; -pub use rolling::*; +#[cfg(any(feature = "paritydb", feature = "rocksdb"))] +pub mod rolling; /// Store interface used as a KV store implementation pub trait Store { diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs index 3e3a9d942e4b..9f8e462347ce 100644 --- a/node/db/src/rolling/impls.rs +++ b/node/db/src/rolling/impls.rs @@ -198,10 +198,7 @@ impl RollingDB { pub fn track_as_current(&mut self, name: String, db: Db) -> anyhow::Result<()> { self.db_queue.push_front(db); - self.db_index - .inner_mut_or_default() - .db_names - .push_front(name); + self.db_index.inner_mut().db_names.push_front(name); self.db_index.flush_to_file() } @@ -218,13 +215,11 @@ impl RollingDB { if let Some(db) = self.db_queue.pop_back() { db.flush()?; } - if let Some(db_index) = self.db_index.inner_mut() { - if let Some(name) = db_index.db_names.pop_back() { - info!("Closing DB {name}"); - if delete { - let db_path = self.db_root.join(name); - delete_db(&db_path); - } + if let Some(name) = self.db_index.inner_mut().db_names.pop_back() { + info!("Closing DB {name}"); + if delete { + let db_path = self.db_root.join(name); + delete_db(&db_path); } } self.ensure_db_index_integrity()?; @@ -241,14 +236,9 @@ impl RollingDB { && self .db_index .inner() - .as_ref() - .map(|db_index| { - db_index - .db_names - .iter() - .all(|name| entry.path() != self.db_root.join(name).as_path()) - }) - .unwrap_or_default() + .db_names + .iter() + .all(|name| entry.path() != self.db_root.join(name).as_path()) }) .for_each(|entry| delete_db(&entry.path())); } @@ -270,15 +260,7 @@ impl RollingDB { } fn ensure_db_index_integrity(&self) -> anyhow::Result<()> { - anyhow::ensure!( - self.db_queue.len() - == self - .db_index - .inner() - .as_ref() - .map(|index| index.db_names.len()) - .unwrap_or_default() - ); + anyhow::ensure!(self.db_queue.len() == self.db_index.inner().db_names.len()); Ok(()) } } @@ -287,9 +269,10 @@ fn load_db_queue( db_root: &Path, db_config: &DbConfig, ) -> anyhow::Result<(FileBacked, VecDeque)> { - let mut db_index = FileBacked::load_from_file_or_new(db_root.join("db_index.yaml"))?; + let mut db_index = + FileBacked::load_from_file_or_create(db_root.join("db_index.yaml"), Default::default)?; let mut db_queue = VecDeque::new(); - let index_inner_mut: &mut DbIndex = db_index.inner_mut_or_default(); + let index_inner_mut: &mut DbIndex = db_index.inner_mut(); for i in (0..index_inner_mut.db_names.len()).rev() { let name = index_inner_mut.db_names[i].as_str(); let db_path = db_root.join(name); diff --git a/node/db/tests/rolling_test.rs b/node/db/tests/rolling_test.rs index 681f440613ef..4821dfe08964 100644 --- a/node/db/tests/rolling_test.rs +++ b/node/db/tests/rolling_test.rs @@ -2,12 +2,13 @@ // SPDX-License-Identifier: Apache-2.0, MIT #[cfg(test)] +#[cfg(any(feature = "paritydb", feature = "rocksdb"))] mod tests { use std::{thread::sleep, time::Duration}; use anyhow::*; use cid::{multihash::MultihashDigest, Cid}; - use forest_db::RollingDB; + use forest_db::rolling::RollingDB; use forest_libp2p_bitswap::BitswapStoreRead; use fvm_ipld_blockstore::Blockstore; use rand::Rng; diff --git a/utils/forest_utils/src/db/file_backed_obj.rs b/utils/forest_utils/src/db/file_backed_obj.rs index 25ae33dbe508..4d75b685a265 100644 --- a/utils/forest_utils/src/db/file_backed_obj.rs +++ b/utils/forest_utils/src/db/file_backed_obj.rs @@ -1,69 +1,75 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use std::path::PathBuf; +use std::{path::PathBuf, str::FromStr}; +use cid::Cid; use log::warn; pub struct FileBacked { - inner: Option, + inner: T, path: PathBuf, } impl FileBacked { /// Gets a borrow of the inner object - pub fn inner(&self) -> &Option { + pub fn inner(&self) -> &T { &self.inner } /// Gets a mutable borrow of the inner object - pub fn inner_mut(&mut self) -> &mut Option { + pub fn inner_mut(&mut self) -> &mut T { &mut self.inner } /// Sets the inner object and flushes to file pub fn set_inner(&mut self, inner: T) -> anyhow::Result<()> { - self.inner = Some(inner); + self.inner = inner; self.flush_to_file() } /// Creates a new file backed object - pub fn new(inner: Option, path: PathBuf) -> Self { + pub fn new(inner: T, path: PathBuf) -> Self { Self { inner, path } } /// Loads an object from a file and creates a new instance - pub fn load_from_file_or_new(path: PathBuf) -> anyhow::Result { - if path.is_file() { + pub fn load_from_file_or_create T>( + path: PathBuf, + create: F, + ) -> anyhow::Result { + let mut need_flush = false; + let obj = if path.is_file() { let bytes = std::fs::read(path.as_path())?; - Ok(Self { + Self { inner: T::deserialize(&bytes) .map_err(|e| { warn!("Error loading object from {}", path.display()); + need_flush = true; e }) - .ok(), + .unwrap_or_else(|_| create()), path, - }) + } } else { - Ok(Self { inner: None, path }) + need_flush = true; + Self { + inner: create(), + path, + } + }; + + if need_flush { + obj.flush_to_file()?; } + + Ok(obj) } /// Flushes the object to the file pub fn flush_to_file(&self) -> anyhow::Result<()> { - if let Some(inner) = &self.inner { - let bytes = inner.serialize()?; - Ok(std::fs::write(&self.path, bytes)?) - } else { - anyhow::bail!("Inner object is not set") - } - } -} - -impl FileBacked { - pub fn inner_mut_or_default(&mut self) -> &mut T { - self.inner_mut().get_or_insert_with(Default::default) + let bytes = self.inner().serialize()?; + Ok(std::fs::write(&self.path, bytes)?) } } @@ -75,3 +81,13 @@ pub trait FileBackedObject: Sized { /// Deserializes from a byte array fn deserialize(bytes: &[u8]) -> anyhow::Result; } + +impl FileBackedObject for Cid { + fn serialize(&self) -> anyhow::Result> { + Ok(self.to_string().into_bytes()) + } + + fn deserialize(bytes: &[u8]) -> anyhow::Result { + Ok(Cid::from_str(String::from_utf8_lossy(bytes).trim())?) + } +} From e87c67ceba9dfc89fdb237e49f5ddb6da67af5ff Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 9 Mar 2023 01:41:50 +0800 Subject: [PATCH 05/43] remove &mut self from pub APIs --- node/db/src/rolling/impls.rs | 80 +++++++++++++++++------------------ node/db/src/rolling/mod.rs | 5 ++- node/db/tests/rolling_test.rs | 2 +- 3 files changed, 44 insertions(+), 43 deletions(-) diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs index 9f8e462347ce..b4ef96938568 100644 --- a/node/db/src/rolling/impls.rs +++ b/node/db/src/rolling/impls.rs @@ -7,13 +7,14 @@ use forest_libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite}; use forest_utils::db::file_backed_obj::FileBackedObject; use fvm_ipld_blockstore::Blockstore; use human_repr::HumanCount; +use parking_lot::RwLock; use super::*; use crate::*; impl Blockstore for RollingDB { fn has(&self, k: &Cid) -> anyhow::Result { - for db in self.db_queue.iter() { + for db in self.db_queue.read().iter() { if let Ok(true) = Blockstore::has(db, k) { return Ok(true); } @@ -23,7 +24,7 @@ impl Blockstore for RollingDB { } fn get(&self, k: &Cid) -> anyhow::Result>> { - for db in self.db_queue.iter() { + for db in self.db_queue.read().iter() { if let Ok(Some(v)) = Blockstore::get(db, k) { return Ok(Some(v)); } @@ -41,7 +42,7 @@ impl Blockstore for RollingDB { Self: Sized, D: AsRef<[u8]>, { - Blockstore::put(self.current(), mh_code, block) + Blockstore::put(&self.current(), mh_code, block) } fn put_many(&self, blocks: I) -> anyhow::Result<()> @@ -50,7 +51,7 @@ impl Blockstore for RollingDB { D: AsRef<[u8]>, I: IntoIterator)>, { - Blockstore::put_many(self.current(), blocks) + Blockstore::put_many(&self.current(), blocks) } fn put_many_keyed(&self, blocks: I) -> anyhow::Result<()> @@ -59,11 +60,11 @@ impl Blockstore for RollingDB { D: AsRef<[u8]>, I: IntoIterator, { - Blockstore::put_many_keyed(self.current(), blocks) + Blockstore::put_many_keyed(&self.current(), blocks) } fn put_keyed(&self, k: &Cid, block: &[u8]) -> anyhow::Result<()> { - Blockstore::put_keyed(self.current(), k, block) + Blockstore::put_keyed(&self.current(), k, block) } } @@ -72,7 +73,7 @@ impl Store for RollingDB { where K: AsRef<[u8]>, { - for db in self.db_queue.iter() { + for db in self.db_queue.read().iter() { if let Ok(Some(v)) = Store::read(db, key.as_ref()) { return Ok(Some(v)); } @@ -85,7 +86,7 @@ impl Store for RollingDB { where K: AsRef<[u8]>, { - for db in self.db_queue.iter() { + for db in self.db_queue.read().iter() { if let Ok(true) = Store::exists(db, key.as_ref()) { return Ok(true); } @@ -99,31 +100,31 @@ impl Store for RollingDB { K: AsRef<[u8]>, V: AsRef<[u8]>, { - Store::write(self.current(), key, value) + Store::write(&self.current(), key, value) } fn delete(&self, key: K) -> Result<(), crate::Error> where K: AsRef<[u8]>, { - Store::delete(self.current(), key) + Store::delete(&self.current(), key) } fn bulk_write( &self, values: impl IntoIterator>, impl Into>)>, ) -> Result<(), crate::Error> { - Store::bulk_write(self.current(), values) + Store::bulk_write(&self.current(), values) } fn flush(&self) -> Result<(), crate::Error> { - Store::flush(self.current()) + Store::flush(&self.current()) } } impl BitswapStoreRead for RollingDB { fn contains(&self, cid: &Cid) -> anyhow::Result { - for db in self.db_queue.iter() { + for db in self.db_queue.read().iter() { if let Ok(true) = BitswapStoreRead::contains(db, cid) { return Ok(true); } @@ -133,7 +134,7 @@ impl BitswapStoreRead for RollingDB { } fn get(&self, cid: &Cid) -> anyhow::Result>> { - for db in self.db_queue.iter() { + for db in self.db_queue.read().iter() { if let Ok(Some(v)) = BitswapStoreRead::get(db, cid) { return Ok(Some(v)); } @@ -147,13 +148,13 @@ impl BitswapStoreReadWrite for RollingDB { type Params = ::Params; fn insert(&self, block: &libipld::Block) -> anyhow::Result<()> { - BitswapStoreReadWrite::insert(self.current(), block) + BitswapStoreReadWrite::insert(&self.current(), block) } } impl DBStatistics for RollingDB { fn get_statistics(&self) -> Option { - DBStatistics::get_statistics(self.current()) + DBStatistics::get_statistics(&self.current()) } } @@ -181,25 +182,26 @@ impl Drop for RollingDB { impl RollingDB { pub fn load_or_create(db_root: PathBuf, db_config: DbConfig) -> anyhow::Result { let (db_index, db_queue) = load_db_queue(db_root.as_path(), &db_config)?; - let mut rolling = Self { + let rolling = Self { db_root, db_config, - db_index, - db_queue, + db_index: RwLock::new(db_index), + db_queue: db_queue.into(), }; - if rolling.db_queue.is_empty() { + if rolling.db_queue.read().is_empty() { let (name, db) = rolling.create_untracked()?; rolling.track_as_current(name, db)?; } - rolling.ensure_db_index_integrity()?; + Ok(rolling) } - pub fn track_as_current(&mut self, name: String, db: Db) -> anyhow::Result<()> { - self.db_queue.push_front(db); - self.db_index.inner_mut().db_names.push_front(name); - self.db_index.flush_to_file() + pub fn track_as_current(&self, name: String, db: Db) -> anyhow::Result<()> { + self.db_queue.write().push_front(db); + let mut db_index = self.db_index.write(); + db_index.inner_mut().db_names.push_front(name); + db_index.flush_to_file() } pub fn create_untracked(&self) -> anyhow::Result<(String, Db)> { @@ -208,33 +210,34 @@ impl RollingDB { Ok((name, db)) } - pub fn clean_tracked(&mut self, n_db_to_reserve: usize, delete: bool) -> anyhow::Result<()> { + pub fn clean_tracked(&self, n_db_to_reserve: usize, delete: bool) -> anyhow::Result<()> { anyhow::ensure!(n_db_to_reserve > 0); - while self.db_queue.len() > n_db_to_reserve { - if let Some(db) = self.db_queue.pop_back() { + let mut db_index = self.db_index.write(); + let mut db_queue = self.db_queue.write(); + while db_queue.len() > n_db_to_reserve { + if let Some(db) = db_queue.pop_back() { db.flush()?; } - if let Some(name) = self.db_index.inner_mut().db_names.pop_back() { + if let Some(name) = db_index.inner_mut().db_names.pop_back() { info!("Closing DB {name}"); if delete { let db_path = self.db_root.join(name); delete_db(&db_path); } } - self.ensure_db_index_integrity()?; } - self.db_index.flush_to_file() + db_index.flush_to_file() } pub fn clean_untracked(&self) -> anyhow::Result<()> { if let Ok(dir) = std::fs::read_dir(&self.db_root) { + let db_index = self.db_index.read(); dir.flatten() .filter(|entry| { entry.path().is_dir() - && self - .db_index + && db_index .inner() .db_names .iter() @@ -250,19 +253,16 @@ impl RollingDB { } pub fn size(&self) -> usize { - self.db_queue.len() + self.db_queue.read().len() } - fn current(&self) -> &Db { + fn current(&self) -> Db { self.db_queue + .read() .get(0) + .cloned() .expect("RollingDB should contain at least one DB reference") } - - fn ensure_db_index_integrity(&self) -> anyhow::Result<()> { - anyhow::ensure!(self.db_queue.len() == self.db_index.inner().db_names.len()); - Ok(()) - } } fn load_db_queue( diff --git a/node/db/src/rolling/mod.rs b/node/db/src/rolling/mod.rs index 652522ac10aa..57cee2d6bff3 100644 --- a/node/db/src/rolling/mod.rs +++ b/node/db/src/rolling/mod.rs @@ -11,6 +11,7 @@ use std::{ use forest_utils::db::file_backed_obj::FileBacked; use log::{info, warn}; +use parking_lot::RwLock; use serde::{Deserialize, Serialize}; use crate::db_engine::{open_db, Db, DbConfig}; @@ -18,9 +19,9 @@ use crate::db_engine::{open_db, Db, DbConfig}; pub struct RollingDB { db_root: PathBuf, db_config: DbConfig, - db_index: FileBacked, + db_index: RwLock>, /// A queue of active databases, from youngest to oldest - db_queue: VecDeque, + db_queue: RwLock>, } #[derive(Debug, Default, Serialize, Deserialize)] diff --git a/node/db/tests/rolling_test.rs b/node/db/tests/rolling_test.rs index 4821dfe08964..c79c67b32dda 100644 --- a/node/db/tests/rolling_test.rs +++ b/node/db/tests/rolling_test.rs @@ -18,7 +18,7 @@ mod tests { fn rolling_db_behaviour_tests() -> Result<()> { let db_root = TempDir::new()?; println!("Creating rolling db under {}", db_root.path().display()); - let mut rolling_db = RollingDB::load_or_create(db_root.path().into(), Default::default())?; + let rolling_db = RollingDB::load_or_create(db_root.path().into(), Default::default())?; println!("Generating random blocks"); let pairs: Vec<_> = (0..1000) .map(|_| { From baf8d07bded59d54765224ae7f6a9b60954083a2 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 9 Mar 2023 02:22:13 +0800 Subject: [PATCH 06/43] hook up proxy_db --- forest/cli/src/cli/db_cmd.rs | 6 ++--- forest/cli/src/cli/snapshot_cmd.rs | 9 ++++--- forest/daemon/src/daemon.rs | 9 ++++--- forest/daemon/src/main.rs | 17 +++---------- node/db/src/lib.rs | 40 +++++++++++++++++------------- node/db/src/rolling/impls.rs | 15 ++++++----- node/db/src/rolling/mod.rs | 10 +++++--- utils/statediff/src/main.rs | 4 +-- 8 files changed, 58 insertions(+), 52 deletions(-) diff --git a/forest/cli/src/cli/db_cmd.rs b/forest/cli/src/cli/db_cmd.rs index b6d75a68cbf3..38e28caa66a6 100644 --- a/forest/cli/src/cli/db_cmd.rs +++ b/forest/cli/src/cli/db_cmd.rs @@ -3,7 +3,7 @@ use clap::Subcommand; use forest_cli_shared::{chain_path, cli::Config}; -use forest_db::db_engine::db_path; +use forest_db::db_engine::db_root; use log::error; use crate::cli::prompt_confirm; @@ -26,14 +26,14 @@ impl DBCommands { Self::Stats => { use human_repr::HumanCount; - let dir = db_path(&chain_path(config)); + let dir = db_root(&chain_path(config)); println!("Database path: {}", dir.display()); let size = fs_extra::dir::get_size(dir).unwrap_or_default(); println!("Database size: {}", size.human_count_bytes()); Ok(()) } Self::Clean { force } => { - let dir = db_path(&chain_path(config)); + let dir = db_root(&chain_path(config)); if !dir.is_dir() { println!( "Aborted. Database path {} is not a valid directory", diff --git a/forest/cli/src/cli/snapshot_cmd.rs b/forest/cli/src/cli/snapshot_cmd.rs index a725362ec9d1..864897efa0bd 100644 --- a/forest/cli/src/cli/snapshot_cmd.rs +++ b/forest/cli/src/cli/snapshot_cmd.rs @@ -11,7 +11,10 @@ use forest_chain::ChainStore; use forest_cli_shared::cli::{ default_snapshot_dir, is_car_or_tmp, snapshot_fetch, SnapshotServer, SnapshotStore, }; -use forest_db::{db_engine::open_db, Store}; +use forest_db::{ + db_engine::{db_root, open_proxy_db}, + Store, +}; use forest_genesis::{forest_load_car, read_genesis_header}; use forest_ipld::{recurse_links_hash, CidHashSet}; use forest_rpc_api::chain_api::ChainExportParams; @@ -393,8 +396,8 @@ async fn validate( if confirm { let tmp_db_path = TempDir::new()?; - let db_path = tmp_db_path.path().join(&config.chain.name); - let db = open_db(&db_path, config.db_config())?; + let db_path = db_root(tmp_db_path.path().join(&config.chain.name).as_path()); + let db = open_proxy_db(db_path, config.db_config().clone())?; let genesis = read_genesis_header( config.client.genesis_file.as_ref(), diff --git a/forest/daemon/src/daemon.rs b/forest/daemon/src/daemon.rs index a7e612f5dbe9..64b573648f4d 100644 --- a/forest/daemon/src/daemon.rs +++ b/forest/daemon/src/daemon.rs @@ -17,7 +17,8 @@ use forest_cli_shared::{ }, }; use forest_db::{ - db_engine::{db_path, open_db, Db}, + db_engine::{db_root, open_proxy_db}, + rolling::RollingDB, Store, }; use forest_genesis::{get_network_name_from_genesis, import_chain, read_genesis_header}; @@ -69,7 +70,7 @@ fn unblock_parent_process() -> anyhow::Result<()> { } /// Starts daemon process -pub(super) async fn start(opts: CliOpts, config: Config) -> anyhow::Result { +pub(super) async fn start(opts: CliOpts, config: Config) -> anyhow::Result { if config.chain.name == "calibnet" { forest_shim::address::set_current_network(forest_shim::address::Network::Testnet); } @@ -112,7 +113,7 @@ pub(super) async fn start(opts: CliOpts, config: Config) -> anyhow::Result { let keystore = Arc::new(RwLock::new(keystore)); - let db = open_db(&db_path(&chain_path(&config)), config.db_config())?; + let db = open_proxy_db(db_root(&chain_path(&config)), config.db_config().clone())?; let mut services = JoinSet::new(); @@ -125,7 +126,7 @@ pub(super) async fn start(opts: CliOpts, config: Config) -> anyhow::Result { "Prometheus server started at {}", config.client.metrics_address ); - let db_directory = forest_db::db_engine::db_path(&chain_path(&config)); + let db_directory = forest_db::db_engine::db_root(&chain_path(&config)); let db = db.clone(); services.spawn(async { forest_metrics::init_prometheus(prometheus_listener, db_directory, db) diff --git a/forest/daemon/src/main.rs b/forest/daemon/src/main.rs index ce35a58da01f..1f2ff3931069 100644 --- a/forest/daemon/src/main.rs +++ b/forest/daemon/src/main.rs @@ -16,7 +16,7 @@ static GLOBAL: MiMalloc = MiMalloc; mod cli; mod daemon; -use std::{cmp::max, fs::File, process, sync::Arc, time::Duration}; +use std::{cmp::max, fs::File, process, time::Duration}; use anyhow::Context; use clap::Parser; @@ -26,10 +26,10 @@ use forest_cli_shared::{ cli::{check_for_unknown_keys, cli_error_and_die, ConfigPath, DaemonConfig}, logger, }; -use forest_db::{db_engine::Db, Store}; +use forest_db::Store; use forest_utils::io::ProgressBar; use lazy_static::lazy_static; -use log::{error, info, warn}; +use log::{info, warn}; use raw_sync::{ events::{Event, EventInit}, Timeout, @@ -170,21 +170,12 @@ fn main() -> anyhow::Result<()> { if let Some(loki_task) = loki_task { rt.spawn(loki_task); } - let db: Db = rt.block_on(daemon::start(opts, cfg))?; - + let db = rt.block_on(daemon::start(opts, cfg))?; info!("Shutting down tokio..."); rt.shutdown_timeout(Duration::from_secs(10)); - db.flush()?; - let db_weak_ref = Arc::downgrade(&db.db); drop(db); - if db_weak_ref.strong_count() != 0 { - error!( - "Dangling reference to DB detected: {}. Tracking issue: https://github.com/ChainSafe/forest/issues/1891", - db_weak_ref.strong_count() - ); - } info!("Forest finish shutdown"); } } diff --git a/node/db/src/lib.rs b/node/db/src/lib.rs index d167fa119637..405125d071e6 100644 --- a/node/db/src/lib.rs +++ b/node/db/src/lib.rs @@ -105,35 +105,41 @@ pub trait DBStatistics { } } -#[cfg(feature = "rocksdb")] +#[cfg(any(feature = "paritydb", feature = "rocksdb"))] pub mod db_engine { use std::path::{Path, PathBuf}; + use crate::rolling::*; + + #[cfg(feature = "rocksdb")] pub type Db = crate::rocks::RocksDb; + #[cfg(feature = "paritydb")] + pub type Db = crate::parity_db::ParityDb; + #[cfg(feature = "rocksdb")] pub type DbConfig = crate::rocks_config::RocksDbConfig; + #[cfg(feature = "paritydb")] + pub type DbConfig = crate::parity_db_config::ParityDbConfig; + + #[cfg(feature = "rocksdb")] + const DIR_NAME: &str = "rocksdb"; + #[cfg(feature = "paritydb")] + const DIR_NAME: &str = "paritydb"; - pub fn db_path(path: &Path) -> PathBuf { - path.join("rocksdb") + pub fn db_root(chain_data_root: &Path) -> PathBuf { + chain_data_root.join(DIR_NAME) } - pub fn open_db(path: &std::path::Path, config: &DbConfig) -> anyhow::Result { + #[cfg(feature = "rocksdb")] + pub(crate) fn open_db(path: &Path, config: &DbConfig) -> anyhow::Result { crate::rocks::RocksDb::open(path, config).map_err(Into::into) } -} - -#[cfg(feature = "paritydb")] -pub mod db_engine { - use std::path::{Path, PathBuf}; - - pub type Db = crate::parity_db::ParityDb; - pub type DbConfig = crate::parity_db_config::ParityDbConfig; - pub fn db_path(path: &Path) -> PathBuf { - path.join("paritydb") + #[cfg(feature = "paritydb")] + pub(crate) fn open_db(path: &Path, config: &DbConfig) -> anyhow::Result { + crate::parity_db::ParityDb::open(path.into(), config).map_err(Into::into) } - pub fn open_db(path: &std::path::Path, config: &DbConfig) -> anyhow::Result { - use crate::parity_db::ParityDb; - ParityDb::open(path.to_owned(), config) + pub fn open_proxy_db(db_root: PathBuf, db_config: DbConfig) -> anyhow::Result { + RollingDB::load_or_create(db_root, db_config) } } diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs index b4ef96938568..23d44b37dcbd 100644 --- a/node/db/src/rolling/impls.rs +++ b/node/db/src/rolling/impls.rs @@ -181,12 +181,15 @@ impl Drop for RollingDB { impl RollingDB { pub fn load_or_create(db_root: PathBuf, db_config: DbConfig) -> anyhow::Result { + if !db_root.exists() { + std::fs::create_dir_all(db_root.as_path())?; + } let (db_index, db_queue) = load_db_queue(db_root.as_path(), &db_config)?; let rolling = Self { - db_root, - db_config, - db_index: RwLock::new(db_index), - db_queue: db_queue.into(), + db_root: db_root.into(), + db_config: db_config.into(), + db_index: RwLock::new(db_index).into(), + db_queue: RwLock::new(db_queue).into(), }; if rolling.db_queue.read().is_empty() { @@ -232,7 +235,7 @@ impl RollingDB { } pub fn clean_untracked(&self) -> anyhow::Result<()> { - if let Ok(dir) = std::fs::read_dir(&self.db_root) { + if let Ok(dir) = std::fs::read_dir(self.db_root.as_path()) { let db_index = self.db_index.read(); dir.flatten() .filter(|entry| { @@ -249,7 +252,7 @@ impl RollingDB { } pub fn size_in_bytes(&self) -> anyhow::Result { - Ok(fs_extra::dir::get_size(&self.db_root)?) + Ok(fs_extra::dir::get_size(self.db_root.as_path())?) } pub fn size(&self) -> usize { diff --git a/node/db/src/rolling/mod.rs b/node/db/src/rolling/mod.rs index 57cee2d6bff3..5c49452794ba 100644 --- a/node/db/src/rolling/mod.rs +++ b/node/db/src/rolling/mod.rs @@ -7,6 +7,7 @@ mod index; use std::{ collections::VecDeque, path::{Path, PathBuf}, + sync::Arc, }; use forest_utils::db::file_backed_obj::FileBacked; @@ -16,12 +17,13 @@ use serde::{Deserialize, Serialize}; use crate::db_engine::{open_db, Db, DbConfig}; +#[derive(Clone)] pub struct RollingDB { - db_root: PathBuf, - db_config: DbConfig, - db_index: RwLock>, + db_root: Arc, + db_config: Arc, + db_index: Arc>>, /// A queue of active databases, from youngest to oldest - db_queue: RwLock>, + db_queue: Arc>>, } #[derive(Debug, Default, Serialize, Deserialize)] diff --git a/utils/statediff/src/main.rs b/utils/statediff/src/main.rs index 4ec7860d6c43..fbabb9edb2e9 100644 --- a/utils/statediff/src/main.rs +++ b/utils/statediff/src/main.rs @@ -4,7 +4,7 @@ use cid::Cid; use clap::Parser; use directories::ProjectDirs; use forest_cli_shared::cli::HELP_MESSAGE; -use forest_db::db_engine::{db_path, open_db, DbConfig}; +use forest_db::db_engine::{db_root, open_proxy_db}; use forest_statediff::print_state_diff; impl crate::Subcommand { @@ -19,7 +19,7 @@ impl crate::Subcommand { let dir = ProjectDirs::from("com", "ChainSafe", "Forest") .ok_or(anyhow::Error::msg("no such path"))?; let chain_path = dir.data_dir().join(chain); - let blockstore = open_db(&db_path(&chain_path), &DbConfig::default())?; + let blockstore = open_proxy_db(db_root(&chain_path), Default::default())?; if let Err(err) = print_state_diff(&blockstore, pre, post, *depth) { eprintln!("Failed to print state diff: {err}"); From 0a6f64279f1af5ec66fda338258545e4a3d8f29c Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 9 Mar 2023 05:00:39 +0800 Subject: [PATCH 07/43] background gc task --- Cargo.lock | 6 +- blockchain/chain/Cargo.toml | 2 - blockchain/chain/src/store/chain_store.rs | 76 ++-------------------- forest/daemon/src/daemon.rs | 12 +++- ipld/Cargo.toml | 1 + ipld/src/util.rs | 73 ++++++++++++++++++++- node/db/Cargo.toml | 3 + node/db/src/lib.rs | 6 ++ node/db/src/rolling/gc.rs | 78 +++++++++++++++++++++++ node/db/src/rolling/impls.rs | 25 +++++++- node/db/src/rolling/mod.rs | 2 + node/db/tests/rolling_test.rs | 4 +- utils/genesis/src/lib.rs | 3 + 13 files changed, 209 insertions(+), 82 deletions(-) create mode 100644 node/db/src/rolling/gc.rs diff --git a/Cargo.lock b/Cargo.lock index 6c0e6e8f4609..44771924aefc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3435,7 +3435,6 @@ dependencies = [ "cid", "digest 0.10.6", "flume", - "forest_actor_interface", "forest_beacon", "forest_blocks", "forest_db", @@ -3448,7 +3447,6 @@ dependencies = [ "forest_networks", "forest_shim", "forest_utils", - "futures", "fvm_ipld_amt 0.5.1", "fvm_ipld_blockstore", "fvm_ipld_car", @@ -3584,6 +3582,8 @@ dependencies = [ "anyhow", "chrono", "cid", + "forest_blocks", + "forest_ipld", "forest_libp2p_bitswap", "forest_utils", "fs_extra", @@ -3602,6 +3602,7 @@ dependencies = [ "serde_yaml", "tempfile", "thiserror", + "tokio", ] [[package]] @@ -3751,6 +3752,7 @@ dependencies = [ "async-recursion", "async-trait", "cid", + "forest_blocks", "forest_db", "forest_json", "forest_utils", diff --git a/blockchain/chain/Cargo.toml b/blockchain/chain/Cargo.toml index 72190c94bcff..e45c050c291c 100644 --- a/blockchain/chain/Cargo.toml +++ b/blockchain/chain/Cargo.toml @@ -15,7 +15,6 @@ bls-signatures.workspace = true cid.workspace = true digest.workspace = true flume.workspace = true -forest_actor_interface.workspace = true forest_beacon.workspace = true forest_blocks.workspace = true forest_db.workspace = true @@ -28,7 +27,6 @@ forest_metrics.workspace = true forest_networks.workspace = true forest_shim.workspace = true forest_utils.workspace = true -futures.workspace = true fvm_ipld_amt.workspace = true fvm_ipld_blockstore.workspace = true fvm_ipld_car.workspace = true diff --git a/blockchain/chain/src/store/chain_store.rs b/blockchain/chain/src/store/chain_store.rs index e277e26eda4b..c4b12221ee79 100644 --- a/blockchain/chain/src/store/chain_store.rs +++ b/blockchain/chain/src/store/chain_store.rs @@ -1,24 +1,20 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use std::{collections::VecDeque, num::NonZeroUsize, sync::Arc, time::SystemTime}; +use std::{num::NonZeroUsize, sync::Arc, time::SystemTime}; use ahash::{HashMap, HashMapExt}; use anyhow::Result; use async_stream::stream; use bls_signatures::Serialize as SerializeBls; -use cid::{ - multihash::{Code, Code::Blake2b256}, - Cid, -}; +use cid::{multihash::Code::Blake2b256, Cid}; use digest::Digest; -use forest_actor_interface::EPOCHS_IN_DAY; use forest_beacon::{BeaconEntry, IGNORE_DRAND_VAR}; use forest_blocks::{Block, BlockHeader, FullTipset, Tipset, TipsetKeys, TxMeta}; use forest_db::Store; use forest_encoding::de::DeserializeOwned; use forest_interpreter::BlockMessages; -use forest_ipld::{recurse_links_hash, CidHashSet}; +use forest_ipld::{should_save_block_to_snapshot, walk_snapshot}; use forest_libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite}; use forest_message::{ChainMessage, Message as MessageTrait, SignedMessage}; use forest_metrics::metrics; @@ -32,7 +28,6 @@ use forest_shim::{ state_tree::StateTree, }; use forest_utils::{db::BlockstoreExt, io::Checksum}; -use futures::Future; use fvm_ipld_amt::Amtv0 as Amt; use fvm_ipld_blockstore::Blockstore; use fvm_ipld_car::CarHeader; @@ -561,7 +556,7 @@ where // Walks over tipset and historical data, sending all blocks visited into the // car writer. - Self::walk_snapshot(tipset, recent_roots, |cid| { + walk_snapshot(tipset, recent_roots, |cid| { let tx_clone = tx.clone(); async move { let block = self @@ -569,15 +564,10 @@ where .get(&cid)? .ok_or_else(|| Error::Other(format!("Cid {cid} not found in blockstore")))?; - // Don't include identity CIDs. - // We only include raw and dagcbor, for now. - // Raw for "code" CIDs. - if u64::from(Code::Identity) != cid.hash().code() - && (cid.codec() == fvm_shared::IPLD_RAW - || cid.codec() == fvm_ipld_encoding::DAG_CBOR) - { + if should_save_block_to_snapshot(&cid) { tx_clone.send_async((cid, block.clone())).await?; } + Ok(block) } }) @@ -603,60 +593,6 @@ where let digest = writer.lock().await.get_mut().finalize(); Ok(digest) } - - /// Walks over tipset and state data and loads all blocks not yet seen. - /// This is tracked based on the callback function loading blocks. - pub async fn walk_snapshot( - tipset: &Tipset, - recent_roots: ChainEpoch, - mut load_block: F, - ) -> Result<(), Error> - where - F: FnMut(Cid) -> T + Send, - T: Future, anyhow::Error>> + Send, - { - let mut seen = CidHashSet::default(); - let mut blocks_to_walk: VecDeque = tipset.cids().to_vec().into(); - let mut current_min_height = tipset.epoch(); - let incl_roots_epoch = tipset.epoch() - recent_roots; - - while let Some(next) = blocks_to_walk.pop_front() { - if !seen.insert(&next) { - continue; - } - - let data = load_block(next).await?; - - let h = BlockHeader::unmarshal_cbor(&data)?; - - if current_min_height > h.epoch() { - current_min_height = h.epoch(); - if current_min_height % EPOCHS_IN_DAY == 0 { - info!(target: "chain_api", "export at: {}", current_min_height); - } - } - - if h.epoch() > incl_roots_epoch { - recurse_links_hash(&mut seen, *h.messages(), &mut load_block).await?; - } - - if h.epoch() > 0 { - for p in h.parents().cids() { - blocks_to_walk.push_back(*p); - } - } else { - for p in h.parents().cids() { - load_block(*p).await?; - } - } - - if h.epoch() == 0 || h.epoch() > incl_roots_epoch { - recurse_links_hash(&mut seen, *h.state_root(), &mut load_block).await?; - } - } - - Ok(()) - } } pub(crate) type TipsetCache = Mutex>>; diff --git a/forest/daemon/src/daemon.rs b/forest/daemon/src/daemon.rs index 64b573648f4d..13a4ddfd67a7 100644 --- a/forest/daemon/src/daemon.rs +++ b/forest/daemon/src/daemon.rs @@ -18,7 +18,7 @@ use forest_cli_shared::{ }; use forest_db::{ db_engine::{db_root, open_proxy_db}, - rolling::RollingDB, + rolling::{DbGarbageCollector, RollingDB}, Store, }; use forest_genesis::{get_network_name_from_genesis, import_chain, read_genesis_header}; @@ -154,6 +154,16 @@ pub(super) async fn start(opts: CliOpts, config: Config) -> anyhow::Result( + tipset: &Tipset, + recent_roots: ChainEpoch, + mut load_block: F, +) -> anyhow::Result<()> +where + F: FnMut(Cid) -> T + Send, + T: Future>> + Send, +{ + let mut seen = CidHashSet::default(); + let mut blocks_to_walk: VecDeque = tipset.cids().to_vec().into(); + let mut current_min_height = tipset.epoch(); + let incl_roots_epoch = tipset.epoch() - recent_roots; + + while let Some(next) = blocks_to_walk.pop_front() { + if !seen.insert(&next) { + continue; + } + + let data = load_block(next).await?; + + let h = BlockHeader::unmarshal_cbor(&data)?; + + if current_min_height > h.epoch() { + current_min_height = h.epoch(); + } + + if h.epoch() > incl_roots_epoch { + recurse_links_hash(&mut seen, *h.messages(), &mut load_block).await?; + } + + if h.epoch() > 0 { + for p in h.parents().cids() { + blocks_to_walk.push_back(*p); + } + } else { + for p in h.parents().cids() { + load_block(*p).await?; + } + } + + if h.epoch() == 0 || h.epoch() > incl_roots_epoch { + recurse_links_hash(&mut seen, *h.state_root(), &mut load_block).await?; + } + } + + Ok(()) +} + +pub fn should_save_block_to_snapshot(cid: &Cid) -> bool { + // Don't include identity CIDs. + // We only include raw and dagcbor, for now. + // Raw for "code" CIDs. + if cid.hash().code() == u64::from(cid::multihash::Code::Identity) { + false + } else { + matches!( + cid.codec(), + fvm_shared::IPLD_RAW | fvm_ipld_encoding::DAG_CBOR + ) + } +} diff --git a/node/db/Cargo.toml b/node/db/Cargo.toml index 53935b954d3c..c78dd4345dc9 100644 --- a/node/db/Cargo.toml +++ b/node/db/Cargo.toml @@ -27,6 +27,8 @@ ahash.workspace = true anyhow.workspace = true chrono.workspace = true cid.workspace = true +forest_blocks.workspace = true +forest_ipld.workspace = true forest_libp2p_bitswap.workspace = true forest_utils.workspace = true fs_extra.workspace = true @@ -41,6 +43,7 @@ prometheus = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_yaml.workspace = true thiserror.workspace = true +tokio = { workspace = true, features = ["sync"] } # optional parity-db = { version = "0.4", default-features = false, optional = true } diff --git a/node/db/src/lib.rs b/node/db/src/lib.rs index 405125d071e6..280e8a5d5729 100644 --- a/node/db/src/lib.rs +++ b/node/db/src/lib.rs @@ -58,6 +58,12 @@ pub trait Store { fn flush(&self) -> Result<(), Error> { Ok(()) } + + /// Create a new physical partition for writing when supported, defaults to + /// doing nothing + fn next_partition(&self) -> anyhow::Result<()> { + Ok(()) + } } impl Store for &BS { diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs new file mode 100644 index 000000000000..8938ab72b14c --- /dev/null +++ b/node/db/src/rolling/gc.rs @@ -0,0 +1,78 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use std::time::Duration; + +use chrono::Utc; +use forest_blocks::Tipset; +use forest_ipld::util::*; +use fvm_ipld_blockstore::Blockstore; +use tokio::sync::Mutex; + +use super::*; +use crate::Store; + +pub struct DbGarbageCollector Tipset> { + db: RollingDB, + get_tipset: F, + lock: Mutex<()>, +} + +impl Tipset> DbGarbageCollector { + pub fn new(db: RollingDB, get_tipset: F) -> Self { + Self { + db, + get_tipset, + lock: Default::default(), + } + } + + pub async fn collect_loop(&self) -> anyhow::Result<()> { + loop { + if let Ok(total_size) = self.db.total_size_in_bytes() { + if let Ok(current_size) = self.db.current_size_in_bytes() { + if total_size > 0 && self.db.db_count() > 1 && current_size * 3 > total_size { + if let Err(err) = self.collect_once().await { + warn!("Garbage collection failed: {err}"); + } + } + } + } + tokio::time::sleep(Duration::from_secs(60)).await; + } + } + + pub async fn collect_once(&self) -> anyhow::Result<()> { + if self.lock.try_lock().is_ok() { + let start = Utc::now(); + let tipset = (self.get_tipset)(); + info!("Garbage collection started at epoch {}", tipset.epoch()); + let db = &self.db; + walk_snapshot(&tipset, DEFAULT_RECENT_ROOTS, |cid| { + let db = db.clone(); + async move { + let block = db + .get(&cid)? + .ok_or_else(|| anyhow::anyhow!("Cid {cid} not found in blockstore"))?; + if should_save_block_to_snapshot(&cid) && !db.current().has(&cid)? { + db.current().put_keyed(&cid, &block)?; + } + + Ok(block) + } + }) + .await?; + info!( + "Garbage collection finished at epoch {}, took {}s", + tipset.epoch(), + (Utc::now() - start).num_seconds() + ); + db.clean_tracked(1, true)?; + db.next_partition()?; + db.clean_untracked()?; + Ok(()) + } else { + anyhow::bail!("Another garbage collection task is in progress."); + } + } +} diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs index 23d44b37dcbd..0663d2551e6b 100644 --- a/node/db/src/rolling/impls.rs +++ b/node/db/src/rolling/impls.rs @@ -120,6 +120,11 @@ impl Store for RollingDB { fn flush(&self) -> Result<(), crate::Error> { Store::flush(&self.current()) } + + fn next_partition(&self) -> anyhow::Result<()> { + let (name, db) = self.create_untracked()?; + self.track_as_current(name, db) + } } impl BitswapStoreRead for RollingDB { @@ -201,6 +206,7 @@ impl RollingDB { } pub fn track_as_current(&self, name: String, db: Db) -> anyhow::Result<()> { + info!("Setting db {name} as current"); self.db_queue.write().push_front(db); let mut db_index = self.db_index.write(); db_index.inner_mut().db_names.push_front(name); @@ -251,15 +257,28 @@ impl RollingDB { Ok(()) } - pub fn size_in_bytes(&self) -> anyhow::Result { + pub fn total_size_in_bytes(&self) -> anyhow::Result { Ok(fs_extra::dir::get_size(self.db_root.as_path())?) } - pub fn size(&self) -> usize { + pub fn current_size_in_bytes(&self) -> anyhow::Result { + Ok(fs_extra::dir::get_size( + self.db_root.as_path().join( + self.db_index + .read() + .inner() + .db_names + .get(0) + .expect("RollingDB should contain at least one DB in index"), + ), + )?) + } + + pub fn db_count(&self) -> usize { self.db_queue.read().len() } - fn current(&self) -> Db { + pub fn current(&self) -> Db { self.db_queue .read() .get(0) diff --git a/node/db/src/rolling/mod.rs b/node/db/src/rolling/mod.rs index 5c49452794ba..8e479fa3a98f 100644 --- a/node/db/src/rolling/mod.rs +++ b/node/db/src/rolling/mod.rs @@ -1,6 +1,8 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +mod gc; +pub use gc::*; mod impls; mod index; diff --git a/node/db/tests/rolling_test.rs b/node/db/tests/rolling_test.rs index c79c67b32dda..5ae7fbe0e599 100644 --- a/node/db/tests/rolling_test.rs +++ b/node/db/tests/rolling_test.rs @@ -51,7 +51,7 @@ mod tests { } rolling_db.clean_tracked(1, false)?; - ensure!(rolling_db.size() == 1); + ensure!(rolling_db.db_count() == 1); for (i, (k, _)) in pairs.iter().enumerate() { if i < split_index { @@ -65,7 +65,7 @@ mod tests { drop(rolling_db); let rolling_db = RollingDB::load_or_create(db_root.path().into(), Default::default())?; - ensure!(rolling_db.size() == 1); + ensure!(rolling_db.db_count() == 1); for (i, (k, _)) in pairs.iter().enumerate() { if i < split_index { ensure!(!rolling_db.contains(k)?); diff --git a/utils/genesis/src/lib.rs b/utils/genesis/src/lib.rs index ef148ce4b802..72fa9139ef50 100644 --- a/utils/genesis/src/lib.rs +++ b/utils/genesis/src/lib.rs @@ -159,6 +159,9 @@ where info!("Accepting {:?} as new head.", ts.cids()); + // Creates a new partition after snapshot import if supported + sm.blockstore().next_partition()?; + Ok(()) } From 1a40602db924bf541efde538b47221c95913b833 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 9 Mar 2023 14:19:25 +0800 Subject: [PATCH 08/43] cli command and CI --- .github/workflows/rust.yml | 5 +++ Cargo.lock | 2 ++ forest/cli/Cargo.toml | 1 + forest/cli/src/cli/db_cmd.rs | 22 ++++++++++-- forest/cli/src/subcommand.rs | 2 +- forest/daemon/src/daemon.rs | 24 ++++++++----- node/db/Cargo.toml | 1 + node/db/src/rolling/gc.rs | 61 +++++++++++++++++++++++----------- node/rpc-api/src/data_types.rs | 1 + node/rpc-api/src/lib.rs | 11 ++++++ node/rpc-client/src/db_ops.rs | 11 ++++++ node/rpc-client/src/lib.rs | 1 + node/rpc/src/db_api.rs | 18 ++++++++++ node/rpc/src/lib.rs | 7 ++-- node/rpc/src/sync_api.rs | 2 ++ scripts/gen_coverage_report.sh | 2 ++ 16 files changed, 139 insertions(+), 32 deletions(-) create mode 100644 node/rpc-client/src/db_ops.rs create mode 100644 node/rpc/src/db_api.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index add6a2c26690..6c817156c6f9 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -174,6 +174,11 @@ jobs: run: | forest-cli net listen forest-cli net peers + - name: db gc + run: | + du -hS ~/.local/share/forest/calibnet + forest-cli --chain calibnet db gc + du -hS ~/.local/share/forest/calibnet - name: validate snapshot run: | forest-cli --chain mainnet snapshot validate $SNAPSHOT_DIRECTORY/*.car --force && diff --git a/Cargo.lock b/Cargo.lock index 44771924aefc..5425757987ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3223,6 +3223,7 @@ dependencies = [ "atty", "base64 0.21.0", "boa_engine", + "chrono", "cid", "clap", "convert_case 0.6.0", @@ -3582,6 +3583,7 @@ dependencies = [ "anyhow", "chrono", "cid", + "flume", "forest_blocks", "forest_ipld", "forest_libp2p_bitswap", diff --git a/forest/cli/Cargo.toml b/forest/cli/Cargo.toml index 05a41db53a9b..d0b4e4a2b5f8 100644 --- a/forest/cli/Cargo.toml +++ b/forest/cli/Cargo.toml @@ -13,6 +13,7 @@ anyhow.workspace = true atty = "0.2" base64.workspace = true boa_engine = { version = "0.16.0", features = ["console"] } +chrono.workspace = true cid.workspace = true clap.workspace = true convert_case = "0.6.0" diff --git a/forest/cli/src/cli/db_cmd.rs b/forest/cli/src/cli/db_cmd.rs index 38e28caa66a6..d1b9b8bbc0c0 100644 --- a/forest/cli/src/cli/db_cmd.rs +++ b/forest/cli/src/cli/db_cmd.rs @@ -1,17 +1,21 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +use chrono::Utc; use clap::Subcommand; use forest_cli_shared::{chain_path, cli::Config}; use forest_db::db_engine::db_root; +use forest_rpc_client::db_ops::db_gc; use log::error; -use crate::cli::prompt_confirm; +use crate::cli::{handle_rpc_err, prompt_confirm}; #[derive(Debug, Subcommand)] pub enum DBCommands { /// Show DB stats Stats, + /// Run DB garbage collection + GC, /// DB Clean up Clean { /// Answer yes to all forest-cli yes/no questions without prompting @@ -21,7 +25,7 @@ pub enum DBCommands { } impl DBCommands { - pub fn run(&self, config: &Config) -> anyhow::Result<()> { + pub async fn run(&self, config: &Config) -> anyhow::Result<()> { match self { Self::Stats => { use human_repr::HumanCount; @@ -32,6 +36,20 @@ impl DBCommands { println!("Database size: {}", size.human_count_bytes()); Ok(()) } + Self::GC => { + let start = Utc::now(); + + db_gc((), &config.client.rpc_token) + .await + .map_err(handle_rpc_err)?; + + println!( + "DB GC completed. took {}s", + (Utc::now() - start).num_seconds() + ); + + Ok(()) + } Self::Clean { force } => { let dir = db_root(&chain_path(config)); if !dir.is_dir() { diff --git a/forest/cli/src/subcommand.rs b/forest/cli/src/subcommand.rs index 78b7efcd4a02..292e28a33974 100644 --- a/forest/cli/src/subcommand.rs +++ b/forest/cli/src/subcommand.rs @@ -20,7 +20,7 @@ pub(super) async fn process(command: Subcommand, config: Config) -> anyhow::Resu Subcommand::State(cmd) => cmd.run(config), Subcommand::Config(cmd) => cmd.run(&config, &mut std::io::stdout()), Subcommand::Send(cmd) => cmd.run(config).await, - Subcommand::DB(cmd) => cmd.run(&config), + Subcommand::DB(cmd) => cmd.run(&config).await, Subcommand::Snapshot(cmd) => cmd.run(config).await, Subcommand::Attach(cmd) => cmd.run(config), Subcommand::Shutdown(cmd) => cmd.run(config).await, diff --git a/forest/daemon/src/daemon.rs b/forest/daemon/src/daemon.rs index 13a4ddfd67a7..5c68f1bf960f 100644 --- a/forest/daemon/src/daemon.rs +++ b/forest/daemon/src/daemon.rs @@ -153,16 +153,21 @@ pub(super) async fn start(opts: CliOpts, config: Config) -> anyhow::Result anyhow::Result anyhow::Result Tipset> { db: RollingDB, get_tipset: F, lock: Mutex<()>, + gc_tx: flume::Sender>>, + gc_rx: flume::Receiver>>, } impl Tipset> DbGarbageCollector { pub fn new(db: RollingDB, get_tipset: F) -> Self { + let (gc_tx, gc_rx) = flume::unbounded(); + Self { db, get_tipset, lock: Default::default(), + gc_tx, + gc_rx, } } - pub async fn collect_loop(&self) -> anyhow::Result<()> { + pub fn get_tx(&self) -> flume::Sender>> { + self.gc_tx.clone() + } + + pub async fn collect_loop_passive(&self) -> anyhow::Result<()> { loop { if let Ok(total_size) = self.db.total_size_in_bytes() { if let Ok(current_size) = self.db.current_size_in_bytes() { @@ -42,19 +52,34 @@ impl Tipset> DbGarbageCollector { } } - pub async fn collect_once(&self) -> anyhow::Result<()> { - if self.lock.try_lock().is_ok() { - let start = Utc::now(); - let tipset = (self.get_tipset)(); - info!("Garbage collection started at epoch {}", tipset.epoch()); - let db = &self.db; + pub async fn collect_loop_event(&self) -> anyhow::Result<()> { + while let Ok(responder) = self.gc_rx.recv_async().await { + if let Err(e) = responder.send(self.collect_once().await) { + warn!("{e}"); + } + } + + Ok(()) + } + + async fn collect_once(&self) -> anyhow::Result<()> { + let guard = self.lock.try_lock(); + if guard.is_err() { + anyhow::bail!("Another garbage collection task is in progress."); + } + + let start = Utc::now(); + let tipset = (self.get_tipset)(); + info!("Garbage collection started at epoch {}", tipset.epoch()); + let db = &self.db; + if db.db_count() > 1 { walk_snapshot(&tipset, DEFAULT_RECENT_ROOTS, |cid| { let db = db.clone(); async move { let block = db .get(&cid)? .ok_or_else(|| anyhow::anyhow!("Cid {cid} not found in blockstore"))?; - if should_save_block_to_snapshot(&cid) && !db.current().has(&cid)? { + if !db.current().has(&cid)? { db.current().put_keyed(&cid, &block)?; } @@ -62,17 +87,15 @@ impl Tipset> DbGarbageCollector { } }) .await?; - info!( - "Garbage collection finished at epoch {}, took {}s", - tipset.epoch(), - (Utc::now() - start).num_seconds() - ); - db.clean_tracked(1, true)?; - db.next_partition()?; - db.clean_untracked()?; - Ok(()) - } else { - anyhow::bail!("Another garbage collection task is in progress."); } + info!( + "Garbage collection finished at epoch {}, took {}s", + tipset.epoch(), + (Utc::now() - start).num_seconds() + ); + db.clean_tracked(1, true)?; + db.next_partition()?; + db.clean_untracked()?; + Ok(()) } } diff --git a/node/rpc-api/src/data_types.rs b/node/rpc-api/src/data_types.rs index 33e46ca1a2ef..959630606c38 100644 --- a/node/rpc-api/src/data_types.rs +++ b/node/rpc-api/src/data_types.rs @@ -42,6 +42,7 @@ where pub network_name: String, pub new_mined_block_tx: flume::Sender>, pub beacon: Arc>, + pub gc_event_tx: flume::Sender>>, } #[derive(Debug, Serialize, Deserialize)] diff --git a/node/rpc-api/src/lib.rs b/node/rpc-api/src/lib.rs index 342d59a1cd7c..1b6f3d29a776 100644 --- a/node/rpc-api/src/lib.rs +++ b/node/rpc-api/src/lib.rs @@ -89,6 +89,10 @@ pub static ACCESS_MAP: Lazy> = Lazy::new(|| { access.insert(net_api::NET_CONNECT, Access::Write); access.insert(net_api::NET_DISCONNECT, Access::Write); + // DB API + // FIXME: Use Read for testing, change this to Write later + access.insert(db_api::DB_GC, Access::Read); + access }); @@ -411,3 +415,10 @@ pub mod net_api { pub type NetDisconnectParams = (String,); pub type NetDisconnectResult = (); } + +/// DB API +pub mod db_api { + pub const DB_GC: &str = "Filecoin.DatabaseGarbageCollection"; + pub type DBGCParams = (); + pub type DBGCResult = (); +} diff --git a/node/rpc-client/src/db_ops.rs b/node/rpc-client/src/db_ops.rs new file mode 100644 index 000000000000..d142b690b3e1 --- /dev/null +++ b/node/rpc-client/src/db_ops.rs @@ -0,0 +1,11 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use forest_rpc_api::db_api::*; +use jsonrpc_v2::Error; + +use crate::call; + +pub async fn db_gc(params: DBGCParams, auth_token: &Option) -> Result { + call(DB_GC, params, auth_token).await +} diff --git a/node/rpc-client/src/lib.rs b/node/rpc-client/src/lib.rs index 5addee1f3a3e..37d17438033c 100644 --- a/node/rpc-client/src/lib.rs +++ b/node/rpc-client/src/lib.rs @@ -5,6 +5,7 @@ pub mod auth_ops; pub mod chain_ops; pub mod common_ops; +pub mod db_ops; pub mod mpool_ops; pub mod net_ops; pub mod state_ops; diff --git a/node/rpc/src/db_api.rs b/node/rpc/src/db_api.rs new file mode 100644 index 000000000000..a31bf1f8d85f --- /dev/null +++ b/node/rpc/src/db_api.rs @@ -0,0 +1,18 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use forest_beacon::Beacon; +use forest_db::Store; +use forest_rpc_api::{data_types::RPCState, db_api::*}; +use fvm_ipld_blockstore::Blockstore; +use jsonrpc_v2::{Data, Error as JsonRpcError, Params}; + +pub(crate) async fn db_gc( + data: Data>, + Params(_): Params, +) -> Result { + let (tx, rx) = flume::bounded(1); + data.gc_event_tx.send_async(tx).await?; + rx.recv_async().await??; + Ok(()) +} diff --git a/node/rpc/src/lib.rs b/node/rpc/src/lib.rs index c092c50aa191..67d5034628e3 100644 --- a/node/rpc/src/lib.rs +++ b/node/rpc/src/lib.rs @@ -5,6 +5,7 @@ mod auth_api; mod beacon_api; mod chain_api; mod common_api; +mod db_api; mod gas_api; mod mpool_api; mod net_api; @@ -22,8 +23,8 @@ use forest_beacon::Beacon; use forest_chain::Scale; use forest_db::Store; use forest_rpc_api::{ - auth_api::*, beacon_api::*, chain_api::*, common_api::*, data_types::RPCState, gas_api::*, - mpool_api::*, net_api::*, state_api::*, sync_api::*, wallet_api::*, + auth_api::*, beacon_api::*, chain_api::*, common_api::*, data_types::RPCState, db_api::*, + gas_api::*, mpool_api::*, net_api::*, state_api::*, sync_api::*, wallet_api::*, }; use fvm_ipld_blockstore::Blockstore; use jsonrpc_v2::{Data, Error as JSONRPCError, Server}; @@ -127,6 +128,8 @@ where .with_method(NET_PEERS, net_api::net_peers::) .with_method(NET_CONNECT, net_api::net_connect::) .with_method(NET_DISCONNECT, net_api::net_disconnect::) + // DB API + .with_method(DB_GC, db_api::db_gc::) .finish_unwrapped(), ); diff --git a/node/rpc/src/sync_api.rs b/node/rpc/src/sync_api.rs index 40005633d4fb..886c821eb8cb 100644 --- a/node/rpc/src/sync_api.rs +++ b/node/rpc/src/sync_api.rs @@ -141,6 +141,7 @@ mod tests { .unwrap() }; let (new_mined_block_tx, _) = flume::bounded(5); + let (gc_event_tx, _) = flume::unbounded(); let state = Arc::new(RPCState { state_manager, keystore: Arc::new(RwLock::new(KeyStore::new(KeyStoreConfig::Memory).unwrap())), @@ -152,6 +153,7 @@ mod tests { chain_store: cs_for_chain, beacon, new_mined_block_tx, + gc_event_tx, }); (state, network_rx) } diff --git a/scripts/gen_coverage_report.sh b/scripts/gen_coverage_report.sh index d7cec82f33a1..e7569b0468da 100755 --- a/scripts/gen_coverage_report.sh +++ b/scripts/gen_coverage_report.sh @@ -35,6 +35,8 @@ cov forest --chain calibnet --encrypt-keystore false --import-snapshot "$SNAPSHO cov forest-cli sync wait cov forest-cli sync status cov forest-cli chain validate-tipset-checkpoints +cov forest-cli --chain calibnet db gc +cov forest-cli --chain calibnet db stats cov forest-cli snapshot export cov forest-cli attach --exec 'showPeers()' cov forest-cli net listen From ac96d4e29fa1caef30024986ed192baa1f9413a4 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 9 Mar 2023 15:31:04 +0800 Subject: [PATCH 09/43] fix gc event --- node/db/src/rolling/gc.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 23f9e3d55632..d8d30d867741 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -12,7 +12,10 @@ use tokio::sync::Mutex; use super::*; use crate::Store; -pub struct DbGarbageCollector Tipset> { +pub struct DbGarbageCollector +where + F: Fn() -> Tipset + Send + Sync + 'static, +{ db: RollingDB, get_tipset: F, lock: Mutex<()>, @@ -20,7 +23,10 @@ pub struct DbGarbageCollector Tipset> { gc_rx: flume::Receiver>>, } -impl Tipset> DbGarbageCollector { +impl DbGarbageCollector +where + F: Fn() -> Tipset + Send + Sync + 'static, +{ pub fn new(db: RollingDB, get_tipset: F) -> Self { let (gc_tx, gc_rx) = flume::unbounded(); @@ -52,11 +58,15 @@ impl Tipset> DbGarbageCollector { } } - pub async fn collect_loop_event(&self) -> anyhow::Result<()> { + pub async fn collect_loop_event(self: &Arc) -> anyhow::Result<()> { while let Ok(responder) = self.gc_rx.recv_async().await { - if let Err(e) = responder.send(self.collect_once().await) { - warn!("{e}"); - } + let this = self.clone(); + tokio::spawn(async move { + let result = this.collect_once().await; + if let Err(e) = responder.send(result) { + warn!("{e}"); + } + }); } Ok(()) From 9d8eb59f7c9ce8434cbefb50b99af2d3d4b36dd5 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 9 Mar 2023 22:19:08 +0800 Subject: [PATCH 10/43] buffered copy during GC --- Cargo.lock | 7 ++++--- forest/cli/src/cli/snapshot_cmd.rs | 2 +- node/db/Cargo.toml | 3 ++- node/db/src/ext.rs | 29 +++++++++++++++++++++++++++++ node/db/src/lib.rs | 2 ++ node/db/src/rolling/gc.rs | 14 ++++++++++++-- utils/genesis/Cargo.toml | 1 + utils/genesis/src/lib.rs | 27 ++++++++++++--------------- 8 files changed, 63 insertions(+), 22 deletions(-) create mode 100644 node/db/src/ext.rs diff --git a/Cargo.lock b/Cargo.lock index c020247e48bf..7618e8f97564 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3581,6 +3581,7 @@ version = "0.6.0" dependencies = [ "ahash 0.8.3", "anyhow", + "async-trait", "chrono", "cid", "flume", @@ -3696,6 +3697,7 @@ version = "0.6.0" dependencies = [ "anyhow", "cid", + "flume", "forest_blocks", "forest_db", "forest_state_manager", @@ -7106,9 +7108,9 @@ dependencies = [ [[package]] name = "parity-db" -version = "0.4.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df89dd8311063c54ae4e03d9aeb597b04212a57e82c339344130a9cad9b3e2d9" +checksum = "dd684a725651d9588ef21f140a328b6b4f64e646b2e931f3e6f14f75eedf9980" dependencies = [ "blake2", "crc32fast", @@ -7120,7 +7122,6 @@ dependencies = [ "memmap2", "parking_lot 0.12.1", "rand 0.8.5", - "siphasher", "snap", ] diff --git a/forest/cli/src/cli/snapshot_cmd.rs b/forest/cli/src/cli/snapshot_cmd.rs index 864897efa0bd..c5babeac9d4d 100644 --- a/forest/cli/src/cli/snapshot_cmd.rs +++ b/forest/cli/src/cli/snapshot_cmd.rs @@ -411,7 +411,7 @@ async fn validate( let cids = { let file = tokio::fs::File::open(&snapshot).await?; let reader = FetchProgress::fetch_from_file(file).await?; - forest_load_car(chain_store.blockstore(), reader.compat()).await? + forest_load_car(chain_store.blockstore().clone(), reader.compat()).await? }; let ts = chain_store.tipset_from_keys(&TipsetKeys::new(cids))?; diff --git a/node/db/Cargo.toml b/node/db/Cargo.toml index f4b66439cf47..8918b0ec6d7f 100644 --- a/node/db/Cargo.toml +++ b/node/db/Cargo.toml @@ -25,6 +25,7 @@ paritydb = ["dep:parity-db"] [dependencies] ahash.workspace = true anyhow.workspace = true +async-trait.workspace = true chrono.workspace = true cid.workspace = true flume.workspace = true @@ -47,7 +48,7 @@ thiserror.workspace = true tokio = { workspace = true, features = ["sync"] } # optional -parity-db = { version = "0.4", default-features = false, optional = true } +parity-db = { version = "=0.4.3", default-features = false, optional = true } rocksdb = { version = "0.20", default-features = false, optional = true } [dev-dependencies] diff --git a/node/db/src/ext.rs b/node/db/src/ext.rs new file mode 100644 index 000000000000..b810afa3b45e --- /dev/null +++ b/node/db/src/ext.rs @@ -0,0 +1,29 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use async_trait::async_trait; + +use crate::*; + +#[async_trait] +pub trait StoreExt: Store { + async fn buffered_write( + &self, + rx: flume::Receiver<(Vec, Vec)>, + buffer_capacity_bytes: usize, + ) -> anyhow::Result<()> { + let mut estimated_size = 0; + let mut buffer = vec![]; + while let Ok((key, value)) = rx.recv_async().await { + estimated_size += key.len() + value.len(); + buffer.push((key, value)); + if estimated_size >= buffer_capacity_bytes { + self.bulk_write(std::mem::take(&mut buffer))?; + estimated_size = 0; + } + } + Ok(self.bulk_write(buffer)?) + } +} + +impl StoreExt for T {} diff --git a/node/db/src/lib.rs b/node/db/src/lib.rs index 280e8a5d5729..35c3c09237ad 100644 --- a/node/db/src/lib.rs +++ b/node/db/src/lib.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0, MIT mod errors; +mod ext; mod memory; mod metrics; @@ -15,6 +16,7 @@ pub mod parity_db_config; pub mod rocks_config; pub use errors::Error; +pub use ext::StoreExt; pub use memory::MemoryDB; #[cfg(any(feature = "paritydb", feature = "rocksdb"))] diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index d8d30d867741..e8ae16610954 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -10,7 +10,7 @@ use fvm_ipld_blockstore::Blockstore; use tokio::sync::Mutex; use super::*; -use crate::Store; +use crate::{Store, StoreExt}; pub struct DbGarbageCollector where @@ -83,20 +83,30 @@ where info!("Garbage collection started at epoch {}", tipset.epoch()); let db = &self.db; if db.db_count() > 1 { + // 512MB + const BUFFER_CAPCITY_BYTES: usize = 512 * 1024 * 1024; + let (tx, rx) = flume::unbounded(); + let write_task = tokio::spawn({ + let db = db.current(); + async move { db.buffered_write(rx, BUFFER_CAPCITY_BYTES).await } + }); walk_snapshot(&tipset, DEFAULT_RECENT_ROOTS, |cid| { let db = db.clone(); + let tx = tx.clone(); async move { let block = db .get(&cid)? .ok_or_else(|| anyhow::anyhow!("Cid {cid} not found in blockstore"))?; if !db.current().has(&cid)? { - db.current().put_keyed(&cid, &block)?; + tx.send_async((cid.to_bytes(), block.clone())).await?; } Ok(block) } }) .await?; + drop(tx); + write_task.await??; } info!( "Garbage collection finished at epoch {}, took {}s", diff --git a/utils/genesis/Cargo.toml b/utils/genesis/Cargo.toml index f4141fdb59fb..174699edcaae 100644 --- a/utils/genesis/Cargo.toml +++ b/utils/genesis/Cargo.toml @@ -12,6 +12,7 @@ testing = [] [dependencies] anyhow.workspace = true cid.workspace = true +flume.workspace = true forest_blocks.workspace = true forest_db.workspace = true forest_state_manager.workspace = true diff --git a/utils/genesis/src/lib.rs b/utils/genesis/src/lib.rs index 72fa9139ef50..38da88cd8dcf 100644 --- a/utils/genesis/src/lib.rs +++ b/utils/genesis/src/lib.rs @@ -6,7 +6,7 @@ use std::{sync::Arc, time}; use anyhow::bail; use cid::Cid; use forest_blocks::{BlockHeader, Tipset, TipsetKeys}; -use forest_db::Store; +use forest_db::{Store, StoreExt}; use forest_state_manager::StateManager; use forest_utils::{db::BlockstoreExt, net::FetchProgress}; use fvm_ipld_blockstore::Blockstore; @@ -118,12 +118,12 @@ where info!("Downloading file..."); let url = Url::parse(path)?; let reader = FetchProgress::fetch_from_url(url).await?; - load_and_retrieve_header(sm.blockstore(), reader, skip_load).await? + load_and_retrieve_header(sm.blockstore().clone(), reader, skip_load).await? } else { info!("Reading file..."); let file = File::open(&path).await?; let reader = FetchProgress::fetch_from_file(file).await?; - load_and_retrieve_header(sm.blockstore(), reader, skip_load).await? + load_and_retrieve_header(sm.blockstore().clone(), reader, skip_load).await? }; info!("Loaded .car file in {}s", stopwatch.elapsed().as_secs()); @@ -168,12 +168,12 @@ where /// Loads car file into database, and returns the block header CIDs from the CAR /// header. async fn load_and_retrieve_header( - store: &DB, + store: DB, reader: FetchProgress, skip_load: bool, ) -> anyhow::Result> where - DB: Store, + DB: Store + Sync + Send + 'static, R: AsyncRead + Send + Unpin, { let mut compat = reader.compat(); @@ -194,22 +194,19 @@ where pub async fn forest_load_car(store: DB, reader: R) -> anyhow::Result> where R: futures::AsyncRead + Send + Unpin, - DB: Store, + DB: Store + Sync + Send + 'static, { // 1GB const BUFFER_CAPCITY_BYTES: usize = 1024 * 1024 * 1024; + let (tx, rx) = flume::unbounded(); + let write_task = + tokio::spawn(async move { store.buffered_write(rx, BUFFER_CAPCITY_BYTES).await }); let mut car_reader = CarReader::new(reader).await?; - let mut estimated_size = 0; - let mut buffer = vec![]; while let Some(block) = car_reader.next_block().await? { - estimated_size += 64 + block.data.len(); - buffer.push((block.cid.to_bytes(), block.data)); - if estimated_size >= BUFFER_CAPCITY_BYTES { - store.bulk_write(std::mem::take(&mut buffer))?; - estimated_size = 0; - } + tx.send((block.cid.to_bytes(), block.data))?; } - store.bulk_write(buffer)?; + drop(tx); + write_task.await??; Ok(car_reader.header.roots) } From ee240afc485a4373f4cb20ca7f91357817a16f33 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 9 Mar 2023 23:20:29 +0800 Subject: [PATCH 11/43] resolve review comments --- forest/daemon/src/daemon.rs | 6 ++---- ipld/src/util.rs | 5 ++--- node/db/src/lib.rs | 12 ------------ node/db/src/memory.rs | 8 -------- node/db/src/parity_db.rs | 8 -------- node/db/src/rocks.rs | 7 ------- node/db/src/rolling/impls.rs | 29 +++++++++++++---------------- node/db/src/rolling/index.rs | 2 -- node/db/src/rolling/mod.rs | 1 - node/db/tests/mem_test.rs | 6 ------ node/db/tests/parity_test.rs | 6 ------ node/db/tests/rocks_test.rs | 6 ------ node/db/tests/subtests/mod.rs | 14 -------------- 13 files changed, 17 insertions(+), 93 deletions(-) delete mode 100644 node/db/src/rolling/index.rs diff --git a/forest/daemon/src/daemon.rs b/forest/daemon/src/daemon.rs index 5c68f1bf960f..58b2719f069a 100644 --- a/forest/daemon/src/daemon.rs +++ b/forest/daemon/src/daemon.rs @@ -311,10 +311,8 @@ pub(super) async fn start(opts: CliOpts, config: Config) -> anyhow::Result( tipset: &Tipset, - recent_roots: ChainEpoch, + recent_roots: i64, mut load_block: F, ) -> anyhow::Result<()> where diff --git a/node/db/src/lib.rs b/node/db/src/lib.rs index 35c3c09237ad..767aea29d784 100644 --- a/node/db/src/lib.rs +++ b/node/db/src/lib.rs @@ -36,11 +36,6 @@ pub trait Store { K: AsRef<[u8]>, V: AsRef<[u8]>; - /// Delete value at key. - fn delete(&self, key: K) -> Result<(), Error> - where - K: AsRef<[u8]>; - /// Returns `Ok(true)` if key exists in store fn exists(&self, key: K) -> Result where @@ -84,13 +79,6 @@ impl Store for &BS { (*self).write(key, value) } - fn delete(&self, key: K) -> Result<(), Error> - where - K: AsRef<[u8]>, - { - (*self).delete(key) - } - fn exists(&self, key: K) -> Result where K: AsRef<[u8]>, diff --git a/node/db/src/memory.rs b/node/db/src/memory.rs index 0bfb598108c3..020453ac49a2 100644 --- a/node/db/src/memory.rs +++ b/node/db/src/memory.rs @@ -30,14 +30,6 @@ impl Store for MemoryDB { Ok(()) } - fn delete(&self, key: K) -> Result<(), Error> - where - K: AsRef<[u8]>, - { - self.db.write().remove(key.as_ref()); - Ok(()) - } - fn read(&self, key: K) -> Result>, Error> where K: AsRef<[u8]>, diff --git a/node/db/src/parity_db.rs b/node/db/src/parity_db.rs index ebf17f4a9543..2700aa2bd421 100644 --- a/node/db/src/parity_db.rs +++ b/node/db/src/parity_db.rs @@ -106,14 +106,6 @@ impl Store for ParityDb { // ``` } - fn delete(&self, key: K) -> Result<(), Error> - where - K: AsRef<[u8]>, - { - let tx = [(0, key.as_ref(), None)]; - self.db.commit(tx).map_err(Error::from) - } - fn exists(&self, key: K) -> Result where K: AsRef<[u8]>, diff --git a/node/db/src/rocks.rs b/node/db/src/rocks.rs index 4867d2772330..c1f02331c005 100644 --- a/node/db/src/rocks.rs +++ b/node/db/src/rocks.rs @@ -234,13 +234,6 @@ impl Store for RocksDb { Ok(self.db.put_opt(key, value, &WRITE_OPT_NO_WAL)?) } - fn delete(&self, key: K) -> Result<(), Error> - where - K: AsRef<[u8]>, - { - Ok(self.db.delete(key)?) - } - fn exists(&self, key: K) -> Result where K: AsRef<[u8]>, diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs index 0663d2551e6b..97d005dcca8e 100644 --- a/node/db/src/rolling/impls.rs +++ b/node/db/src/rolling/impls.rs @@ -7,14 +7,14 @@ use forest_libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite}; use forest_utils::db::file_backed_obj::FileBackedObject; use fvm_ipld_blockstore::Blockstore; use human_repr::HumanCount; -use parking_lot::RwLock; +use parking_lot::{RwLock, RwLockReadGuard}; use super::*; use crate::*; impl Blockstore for RollingDB { fn has(&self, k: &Cid) -> anyhow::Result { - for db in self.db_queue.read().iter() { + for db in self.db_queue().iter() { if let Ok(true) = Blockstore::has(db, k) { return Ok(true); } @@ -24,7 +24,7 @@ impl Blockstore for RollingDB { } fn get(&self, k: &Cid) -> anyhow::Result>> { - for db in self.db_queue.read().iter() { + for db in self.db_queue().iter() { if let Ok(Some(v)) = Blockstore::get(db, k) { return Ok(Some(v)); } @@ -73,7 +73,7 @@ impl Store for RollingDB { where K: AsRef<[u8]>, { - for db in self.db_queue.read().iter() { + for db in self.db_queue().iter() { if let Ok(Some(v)) = Store::read(db, key.as_ref()) { return Ok(Some(v)); } @@ -86,7 +86,7 @@ impl Store for RollingDB { where K: AsRef<[u8]>, { - for db in self.db_queue.read().iter() { + for db in self.db_queue().iter() { if let Ok(true) = Store::exists(db, key.as_ref()) { return Ok(true); } @@ -103,13 +103,6 @@ impl Store for RollingDB { Store::write(&self.current(), key, value) } - fn delete(&self, key: K) -> Result<(), crate::Error> - where - K: AsRef<[u8]>, - { - Store::delete(&self.current(), key) - } - fn bulk_write( &self, values: impl IntoIterator>, impl Into>)>, @@ -129,7 +122,7 @@ impl Store for RollingDB { impl BitswapStoreRead for RollingDB { fn contains(&self, cid: &Cid) -> anyhow::Result { - for db in self.db_queue.read().iter() { + for db in self.db_queue().iter() { if let Ok(true) = BitswapStoreRead::contains(db, cid) { return Ok(true); } @@ -139,7 +132,7 @@ impl BitswapStoreRead for RollingDB { } fn get(&self, cid: &Cid) -> anyhow::Result>> { - for db in self.db_queue.read().iter() { + for db in self.db_queue().iter() { if let Ok(Some(v)) = BitswapStoreRead::get(db, cid) { return Ok(Some(v)); } @@ -197,7 +190,7 @@ impl RollingDB { db_queue: RwLock::new(db_queue).into(), }; - if rolling.db_queue.read().is_empty() { + if rolling.db_queue().is_empty() { let (name, db) = rolling.create_untracked()?; rolling.track_as_current(name, db)?; } @@ -275,7 +268,7 @@ impl RollingDB { } pub fn db_count(&self) -> usize { - self.db_queue.read().len() + self.db_queue().len() } pub fn current(&self) -> Db { @@ -285,6 +278,10 @@ impl RollingDB { .cloned() .expect("RollingDB should contain at least one DB reference") } + + fn db_queue(&self) -> RwLockReadGuard> { + self.db_queue.read() + } } fn load_db_queue( diff --git a/node/db/src/rolling/index.rs b/node/db/src/rolling/index.rs deleted file mode 100644 index 3ec2696da862..000000000000 --- a/node/db/src/rolling/index.rs +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright 2019-2023 ChainSafe Systems -// SPDX-License-Identifier: Apache-2.0, MIT diff --git a/node/db/src/rolling/mod.rs b/node/db/src/rolling/mod.rs index 8e479fa3a98f..cdf993969801 100644 --- a/node/db/src/rolling/mod.rs +++ b/node/db/src/rolling/mod.rs @@ -4,7 +4,6 @@ mod gc; pub use gc::*; mod impls; -mod index; use std::{ collections::VecDeque, diff --git a/node/db/tests/mem_test.rs b/node/db/tests/mem_test.rs index c6f90ad21da0..729295d609eb 100644 --- a/node/db/tests/mem_test.rs +++ b/node/db/tests/mem_test.rs @@ -29,12 +29,6 @@ fn mem_db_does_not_exist() { subtests::does_not_exist(&db); } -#[test] -fn mem_db_delete() { - let db = MemoryDB::default(); - subtests::delete(&db); -} - #[test] fn mem_db_bulk_write() { let db = MemoryDB::default(); diff --git a/node/db/tests/parity_test.rs b/node/db/tests/parity_test.rs index 8b0625014520..0b422e9e9a57 100644 --- a/node/db/tests/parity_test.rs +++ b/node/db/tests/parity_test.rs @@ -35,12 +35,6 @@ mod paritydb_tests { subtests::does_not_exist(&*db); } - #[test] - fn db_delete() { - let db = TempParityDB::new(); - subtests::delete(&*db); - } - #[test] fn db_bulk_write() { let db = TempParityDB::new(); diff --git a/node/db/tests/rocks_test.rs b/node/db/tests/rocks_test.rs index 31863b5ba112..4db645b8c532 100644 --- a/node/db/tests/rocks_test.rs +++ b/node/db/tests/rocks_test.rs @@ -36,12 +36,6 @@ mod rocksdb_tests { subtests::does_not_exist(&*db); } - #[test] - fn db_delete() { - let db = TempRocksDB::new(); - subtests::delete(&*db); - } - #[test] fn db_bulk_write() { let db = TempRocksDB::new(); diff --git a/node/db/tests/subtests/mod.rs b/node/db/tests/subtests/mod.rs index decd989e91fe..9909382342bf 100644 --- a/node/db/tests/subtests/mod.rs +++ b/node/db/tests/subtests/mod.rs @@ -43,20 +43,6 @@ where assert!(!res); } -pub fn delete(db: &DB) -where - DB: Store, -{ - let key = [0]; - let value = [1]; - db.write(key, value).unwrap(); - let res = db.exists(key).unwrap(); - assert!(res); - db.delete(key).unwrap(); - let res = db.exists(key).unwrap(); - assert!(!res); -} - pub fn bulk_write(db: &DB) where DB: Store, From f89a7a77b2ccf029df29ad00fb3465a9647c24ee Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 9 Mar 2023 23:57:08 +0800 Subject: [PATCH 12/43] aggregated error --- CHANGELOG.md | 2 + node/db/src/errors.rs | 9 ++++ node/db/src/rolling/impls.rs | 81 +++++++++++++++++++++------- utils/forest_utils/src/common/mod.rs | 37 +++++++++++++ utils/forest_utils/src/lib.rs | 2 + 5 files changed, 111 insertions(+), 20 deletions(-) create mode 100644 utils/forest_utils/src/common/mod.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e8b88c72686f..2e6be593d209 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,13 @@ Notable updates: ### Added * [database] added ParityDb statistics to the stats endpoint. [#2444](https://github.com/ChainSafe/forest/pull/2444) * [api|cli] Add RPC `Filecoin.Shutdown` endpoint and `forest-cli shutdown` subcommand. [#2538](https://github.com/ChainSafe/forest/pull/2538) +* [api|cli] Add RPC `Filecoin.DatabaseGarbageCollection` endpoint and `forest-cli db gc` subcommand. [#2638](https://github.com/ChainSafe/forest/pull/2638) * [cli] A JavaScript console to interact with Filecoin API. [#2492](https://github.com/ChainSafe/forest/pull/2492) * [docker] Multi-platform Docker image support. [#2552](https://github.com/ChainSafe/forest/pull/2552) * [forest-cli] added `--dry-run` flag to `snapshot export` command. [#2549](https://github.com/ChainSafe/forest/pull/2549) * [forest daemon] Added `--exit-after-init` and `--save-token` flags. [#2577](https://github.com/ChainSafe/forest/pull/2577) * [forest daemon] Support for NV18. [#2558](https://github.com/ChainSafe/forest/pull/2558) [#2579](https://github.com/ChainSafe/forest/pull/2579) +* [forest daemon] Automatic database garbage collection. [#2638](https://github.com/ChainSafe/forest/pull/2638) ### Changed * [cli] Remove Forest ctrl-c hard shutdown behavior on subsequent ctrl-c signals. [#2538](https://github.com/ChainSafe/forest/pull/2538) diff --git a/node/db/src/errors.rs b/node/db/src/errors.rs index f90014096465..59f3f310bb09 100644 --- a/node/db/src/errors.rs +++ b/node/db/src/errors.rs @@ -1,6 +1,9 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +use std::fmt::Display; + +use forest_utils::common::AggregatedError; use thiserror::Error; /// Database error @@ -42,3 +45,9 @@ impl From for String { e.to_string() } } + +impl From> for Error { + fn from(value: AggregatedError) -> Self { + Self::Other(value.to_string()) + } +} diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs index 97d005dcca8e..6bc7b138898c 100644 --- a/node/db/src/rolling/impls.rs +++ b/node/db/src/rolling/impls.rs @@ -4,7 +4,7 @@ use chrono::Utc; use cid::Cid; use forest_libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite}; -use forest_utils::db::file_backed_obj::FileBackedObject; +use forest_utils::{common::AggregatedError, db::file_backed_obj::FileBackedObject}; use fvm_ipld_blockstore::Blockstore; use human_repr::HumanCount; use parking_lot::{RwLock, RwLockReadGuard}; @@ -14,23 +14,36 @@ use crate::*; impl Blockstore for RollingDB { fn has(&self, k: &Cid) -> anyhow::Result { + let mut errors = AggregatedError::new(); for db in self.db_queue().iter() { - if let Ok(true) = Blockstore::has(db, k) { - return Ok(true); + match Blockstore::has(db, k) { + Ok(true) => return Ok(true), + Ok(false) => {} + Err(e) => errors.push(e), } } - Ok(false) + if errors.is_empty() { + Ok(false) + } else { + Err(errors.into()) + } } fn get(&self, k: &Cid) -> anyhow::Result>> { + let mut errors = AggregatedError::new(); for db in self.db_queue().iter() { - if let Ok(Some(v)) = Blockstore::get(db, k) { - return Ok(Some(v)); + match Blockstore::get(db, k) { + Ok(Some(v)) => return Ok(Some(v)), + Ok(None) => {} + Err(e) => errors.push(e), } } - - Ok(None) + if errors.is_empty() { + Ok(None) + } else { + Err(errors.into()) + } } fn put( @@ -73,26 +86,40 @@ impl Store for RollingDB { where K: AsRef<[u8]>, { + let mut errors = AggregatedError::new(); for db in self.db_queue().iter() { - if let Ok(Some(v)) = Store::read(db, key.as_ref()) { - return Ok(Some(v)); + match Store::read(db, key.as_ref()) { + Ok(Some(v)) => return Ok(Some(v)), + Ok(None) => {} + Err(e) => errors.push(e), } } - Ok(None) + if errors.is_empty() { + Ok(None) + } else { + Err(errors.into()) + } } fn exists(&self, key: K) -> Result where K: AsRef<[u8]>, { + let mut errors = AggregatedError::new(); for db in self.db_queue().iter() { - if let Ok(true) = Store::exists(db, key.as_ref()) { - return Ok(true); + match Store::exists(db, key.as_ref()) { + Ok(true) => return Ok(true), + Ok(false) => {} + Err(e) => errors.push(e), } } - Ok(false) + if errors.is_empty() { + Ok(false) + } else { + Err(errors.into()) + } } fn write(&self, key: K, value: V) -> Result<(), crate::Error> @@ -122,23 +149,37 @@ impl Store for RollingDB { impl BitswapStoreRead for RollingDB { fn contains(&self, cid: &Cid) -> anyhow::Result { + let mut errors = AggregatedError::new(); for db in self.db_queue().iter() { - if let Ok(true) = BitswapStoreRead::contains(db, cid) { - return Ok(true); + match BitswapStoreRead::contains(db, cid) { + Ok(true) => return Ok(true), + Ok(false) => {} + Err(e) => errors.push(e), } } - Ok(false) + if errors.is_empty() { + Ok(false) + } else { + Err(errors.into()) + } } fn get(&self, cid: &Cid) -> anyhow::Result>> { + let mut errors = AggregatedError::new(); for db in self.db_queue().iter() { - if let Ok(Some(v)) = BitswapStoreRead::get(db, cid) { - return Ok(Some(v)); + match BitswapStoreRead::get(db, cid) { + Ok(Some(v)) => return Ok(Some(v)), + Ok(None) => {} + Err(e) => errors.push(e), } } - Ok(None) + if errors.is_empty() { + Ok(None) + } else { + Err(errors.into()) + } } } diff --git a/utils/forest_utils/src/common/mod.rs b/utils/forest_utils/src/common/mod.rs new file mode 100644 index 000000000000..892f1979972d --- /dev/null +++ b/utils/forest_utils/src/common/mod.rs @@ -0,0 +1,37 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use std::fmt::Display; + +#[derive(Debug, thiserror::Error)] +pub struct AggregatedError(Vec); + +impl AggregatedError { + pub fn new() -> Self { + Default::default() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn push(&mut self, item: T) { + self.0.push(item); + } +} + +impl Default for AggregatedError { + fn default() -> Self { + Self(vec![]) + } +} + +impl Display for AggregatedError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "Aggregated errors:")?; + for e in self.0.iter() { + writeln!(f, " {e}")?; + } + Ok(()) + } +} diff --git a/utils/forest_utils/src/lib.rs b/utils/forest_utils/src/lib.rs index aa0976d36cca..bd65dee2fcec 100644 --- a/utils/forest_utils/src/lib.rs +++ b/utils/forest_utils/src/lib.rs @@ -1,7 +1,9 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT + pub extern crate const_format; +pub mod common; pub mod db; pub mod io; pub mod json; From 307588ddfd9e052cf094aa525c05e459b8cee617 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 10 Mar 2023 00:22:30 +0800 Subject: [PATCH 13/43] fix(ci): move gc to before snapshot export --- .github/workflows/rust.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e1a0adc105ef..d5867307ac2c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -164,6 +164,11 @@ jobs: run: | forest-cli --chain calibnet sync wait forest-cli --chain calibnet db stats + - name: db gc + run: | + du -hS ~/.local/share/forest/calibnet + forest-cli --chain calibnet db gc + du -hS ~/.local/share/forest/calibnet - name: export snapshot run: | forest-cli --chain calibnet snapshot export @@ -174,11 +179,6 @@ jobs: run: | forest-cli net listen forest-cli net peers - - name: db gc - run: | - du -hS ~/.local/share/forest/calibnet - forest-cli --chain calibnet db gc - du -hS ~/.local/share/forest/calibnet - name: validate snapshot run: | forest-cli --chain mainnet snapshot validate $SNAPSHOT_DIRECTORY/*.car --force && From fa06eec4efb46f72371de7407ea3cd3a48210cee Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 10 Mar 2023 11:10:46 +0800 Subject: [PATCH 14/43] reduce loop depth --- node/db/src/rolling/gc.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index e8ae16610954..cc83a1566f51 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -45,15 +45,18 @@ where pub async fn collect_loop_passive(&self) -> anyhow::Result<()> { loop { - if let Ok(total_size) = self.db.total_size_in_bytes() { - if let Ok(current_size) = self.db.current_size_in_bytes() { - if total_size > 0 && self.db.db_count() > 1 && current_size * 3 > total_size { - if let Err(err) = self.collect_once().await { - warn!("Garbage collection failed: {err}"); - } + if let (Ok(total_size), Ok(current_size)) = ( + self.db.total_size_in_bytes(), + self.db.current_size_in_bytes(), + ) { + // Collect when size of young partition > 0.5 * size of old partition + if total_size > 0 && self.db.db_count() > 1 && current_size * 3 > total_size { + if let Err(err) = self.collect_once().await { + warn!("Garbage collection failed: {err}"); } } } + tokio::time::sleep(Duration::from_secs(60)).await; } } From 9edd0f46d3584ec4647c0c3edebade6de80ef7c9 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 10 Mar 2023 21:18:52 +0800 Subject: [PATCH 15/43] resolve review comments --- node/db/src/rolling/gc.rs | 4 +- node/db/src/rolling/impls.rs | 148 ++++++++++++++--------------------- node/db/src/rolling/mod.rs | 2 + 3 files changed, 61 insertions(+), 93 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index cc83a1566f51..16a44a8f5177 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -86,8 +86,8 @@ where info!("Garbage collection started at epoch {}", tipset.epoch()); let db = &self.db; if db.db_count() > 1 { - // 512MB - const BUFFER_CAPCITY_BYTES: usize = 512 * 1024 * 1024; + // 128MB + const BUFFER_CAPCITY_BYTES: usize = 128 * 1024 * 1024; let (tx, rx) = flume::unbounded(); let write_task = tokio::spawn({ let db = db.current(); diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs index 6bc7b138898c..abf220e63822 100644 --- a/node/db/src/rolling/impls.rs +++ b/node/db/src/rolling/impls.rs @@ -4,7 +4,7 @@ use chrono::Utc; use cid::Cid; use forest_libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite}; -use forest_utils::{common::AggregatedError, db::file_backed_obj::FileBackedObject}; +use forest_utils::db::file_backed_obj::FileBackedObject; use fvm_ipld_blockstore::Blockstore; use human_repr::HumanCount; use parking_lot::{RwLock, RwLockReadGuard}; @@ -14,36 +14,23 @@ use crate::*; impl Blockstore for RollingDB { fn has(&self, k: &Cid) -> anyhow::Result { - let mut errors = AggregatedError::new(); for db in self.db_queue().iter() { - match Blockstore::has(db, k) { - Ok(true) => return Ok(true), - Ok(false) => {} - Err(e) => errors.push(e), + if Blockstore::has(db, k)? { + return Ok(true); } } - if errors.is_empty() { - Ok(false) - } else { - Err(errors.into()) - } + Ok(false) } fn get(&self, k: &Cid) -> anyhow::Result>> { - let mut errors = AggregatedError::new(); for db in self.db_queue().iter() { - match Blockstore::get(db, k) { - Ok(Some(v)) => return Ok(Some(v)), - Ok(None) => {} - Err(e) => errors.push(e), + if let Some(v) = Blockstore::get(db, k)? { + return Ok(Some(v)); } } - if errors.is_empty() { - Ok(None) - } else { - Err(errors.into()) - } + + Ok(None) } fn put( @@ -86,40 +73,26 @@ impl Store for RollingDB { where K: AsRef<[u8]>, { - let mut errors = AggregatedError::new(); for db in self.db_queue().iter() { - match Store::read(db, key.as_ref()) { - Ok(Some(v)) => return Ok(Some(v)), - Ok(None) => {} - Err(e) => errors.push(e), + if let Some(v) = Store::read(db, key.as_ref())? { + return Ok(Some(v)); } } - if errors.is_empty() { - Ok(None) - } else { - Err(errors.into()) - } + Ok(None) } fn exists(&self, key: K) -> Result where K: AsRef<[u8]>, { - let mut errors = AggregatedError::new(); for db in self.db_queue().iter() { - match Store::exists(db, key.as_ref()) { - Ok(true) => return Ok(true), - Ok(false) => {} - Err(e) => errors.push(e), + if Store::exists(db, key.as_ref())? { + return Ok(true); } } - if errors.is_empty() { - Ok(false) - } else { - Err(errors.into()) - } + Ok(false) } fn write(&self, key: K, value: V) -> Result<(), crate::Error> @@ -149,37 +122,23 @@ impl Store for RollingDB { impl BitswapStoreRead for RollingDB { fn contains(&self, cid: &Cid) -> anyhow::Result { - let mut errors = AggregatedError::new(); for db in self.db_queue().iter() { - match BitswapStoreRead::contains(db, cid) { - Ok(true) => return Ok(true), - Ok(false) => {} - Err(e) => errors.push(e), + if BitswapStoreRead::contains(db, cid)? { + return Ok(true); } } - if errors.is_empty() { - Ok(false) - } else { - Err(errors.into()) - } + Ok(false) } fn get(&self, cid: &Cid) -> anyhow::Result>> { - let mut errors = AggregatedError::new(); for db in self.db_queue().iter() { - match BitswapStoreRead::get(db, cid) { - Ok(Some(v)) => return Ok(Some(v)), - Ok(None) => {} - Err(e) => errors.push(e), + if let Some(v) = BitswapStoreRead::get(db, cid)? { + return Ok(Some(v)); } } - if errors.is_empty() { - Ok(None) - } else { - Err(errors.into()) - } + Ok(None) } } @@ -223,34 +182,34 @@ impl RollingDB { if !db_root.exists() { std::fs::create_dir_all(db_root.as_path())?; } - let (db_index, db_queue) = load_db_queue(db_root.as_path(), &db_config)?; - let rolling = Self { + let (mut db_index, mut db_queue) = load_db_queue(&db_root, &db_config)?; + let current = if db_queue.is_empty() { + let (name, db) = create_untracked(&db_root, &db_config)?; + track_as_youngest(&mut db_index, &mut db_queue, name.clone(), db.clone())?; + (name, db) + } else { + (db_index.inner().db_names[0].clone(), db_queue[0].clone()) + }; + + Ok(Self { db_root: db_root.into(), db_config: db_config.into(), db_index: RwLock::new(db_index).into(), db_queue: RwLock::new(db_queue).into(), - }; - - if rolling.db_queue().is_empty() { - let (name, db) = rolling.create_untracked()?; - rolling.track_as_current(name, db)?; - } - - Ok(rolling) + current: RwLock::new(current).into(), + }) } pub fn track_as_current(&self, name: String, db: Db) -> anyhow::Result<()> { - info!("Setting db {name} as current"); - self.db_queue.write().push_front(db); let mut db_index = self.db_index.write(); - db_index.inner_mut().db_names.push_front(name); - db_index.flush_to_file() + let mut db_queue = self.db_queue.write(); + track_as_youngest(&mut db_index, &mut db_queue, name.clone(), db.clone())?; + *self.current.write() = (name, db); + Ok(()) } pub fn create_untracked(&self) -> anyhow::Result<(String, Db)> { - let name = Utc::now().timestamp_millis().to_string(); - let db = open_db(&self.db_root.join(&name), &self.db_config)?; - Ok((name, db)) + create_untracked(&self.db_root, &self.db_config) } pub fn clean_tracked(&self, n_db_to_reserve: usize, delete: bool) -> anyhow::Result<()> { @@ -297,14 +256,7 @@ impl RollingDB { pub fn current_size_in_bytes(&self) -> anyhow::Result { Ok(fs_extra::dir::get_size( - self.db_root.as_path().join( - self.db_index - .read() - .inner() - .db_names - .get(0) - .expect("RollingDB should contain at least one DB in index"), - ), + self.db_root.as_path().join(self.current.read().0.as_str()), )?) } @@ -313,11 +265,7 @@ impl RollingDB { } pub fn current(&self) -> Db { - self.db_queue - .read() - .get(0) - .cloned() - .expect("RollingDB should contain at least one DB reference") + self.current.read().1.clone() } fn db_queue(&self) -> RwLockReadGuard> { @@ -369,3 +317,21 @@ fn delete_db(db_path: &Path) { ); } } + +fn track_as_youngest( + db_index: &mut FileBacked, + db_queue: &mut VecDeque, + name: String, + db: Db, +) -> anyhow::Result<()> { + info!("Setting db {name} as current"); + db_queue.push_front(db); + db_index.inner_mut().db_names.push_front(name); + db_index.flush_to_file() +} + +fn create_untracked(db_root: &Path, db_config: &DbConfig) -> anyhow::Result<(String, Db)> { + let name = Utc::now().timestamp_millis().to_string(); + let db = open_db(&db_root.join(&name), db_config)?; + Ok((name, db)) +} diff --git a/node/db/src/rolling/mod.rs b/node/db/src/rolling/mod.rs index cdf993969801..d8b0fdad246a 100644 --- a/node/db/src/rolling/mod.rs +++ b/node/db/src/rolling/mod.rs @@ -25,6 +25,8 @@ pub struct RollingDB { db_index: Arc>>, /// A queue of active databases, from youngest to oldest db_queue: Arc>>, + /// The current writable DB + current: Arc>, } #[derive(Debug, Default, Serialize, Deserialize)] From 18003a75fb5953c36a001fb598c94803adcceea5 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 12:59:01 +0800 Subject: [PATCH 16/43] changes as requested --- Cargo.lock | 1 + blockchain/chain/src/store/chain_store.rs | 1 - node/db/Cargo.toml | 1 + node/db/src/rolling/gc.rs | 66 ++++++----- node/db/src/rolling/impls.rs | 138 +++++----------------- node/db/src/rolling/mod.rs | 10 +- node/db/tests/rolling_test.rs | 13 +- 7 files changed, 80 insertions(+), 150 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a1842bec375..f6fbdad8bd52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3608,6 +3608,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "uuid", ] [[package]] diff --git a/blockchain/chain/src/store/chain_store.rs b/blockchain/chain/src/store/chain_store.rs index 211d2cfa0926..b13d139b7a99 100644 --- a/blockchain/chain/src/store/chain_store.rs +++ b/blockchain/chain/src/store/chain_store.rs @@ -21,7 +21,6 @@ use forest_metrics::metrics; use forest_networks::ChainConfig; use forest_shim::{ address::Address, - clock::EPOCHS_IN_DAY, crypto::{Signature, SignatureType}, econ::TokenAmount, executor::Receipt, diff --git a/node/db/Cargo.toml b/node/db/Cargo.toml index 8918b0ec6d7f..6652f7e61c0b 100644 --- a/node/db/Cargo.toml +++ b/node/db/Cargo.toml @@ -46,6 +46,7 @@ serde = { workspace = true, features = ["derive"] } serde_yaml.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["sync"] } +uuid = { version = "1.3", features = ["v4"] } # optional parity-db = { version = "=0.4.3", default-features = false, optional = true } diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 16a44a8f5177..d01185e1870a 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -45,19 +45,28 @@ where pub async fn collect_loop_passive(&self) -> anyhow::Result<()> { loop { + // Check every 10 mins + tokio::time::sleep(Duration::from_secs(10 * 60)).await; + + // Bypass size checking when lock is held + { + let lock = self.lock.try_lock(); + if lock.is_err() { + continue; + } + } + if let (Ok(total_size), Ok(current_size)) = ( self.db.total_size_in_bytes(), self.db.current_size_in_bytes(), ) { // Collect when size of young partition > 0.5 * size of old partition - if total_size > 0 && self.db.db_count() > 1 && current_size * 3 > total_size { + if total_size > 0 && current_size * 3 > total_size { if let Err(err) = self.collect_once().await { warn!("Garbage collection failed: {err}"); } } } - - tokio::time::sleep(Duration::from_secs(60)).await; } } @@ -85,40 +94,37 @@ where let tipset = (self.get_tipset)(); info!("Garbage collection started at epoch {}", tipset.epoch()); let db = &self.db; - if db.db_count() > 1 { - // 128MB - const BUFFER_CAPCITY_BYTES: usize = 128 * 1024 * 1024; - let (tx, rx) = flume::unbounded(); - let write_task = tokio::spawn({ - let db = db.current(); - async move { db.buffered_write(rx, BUFFER_CAPCITY_BYTES).await } - }); - walk_snapshot(&tipset, DEFAULT_RECENT_ROOTS, |cid| { - let db = db.clone(); - let tx = tx.clone(); - async move { - let block = db - .get(&cid)? - .ok_or_else(|| anyhow::anyhow!("Cid {cid} not found in blockstore"))?; - if !db.current().has(&cid)? { - tx.send_async((cid.to_bytes(), block.clone())).await?; - } - - Ok(block) + // 128MB + const BUFFER_CAPCITY_BYTES: usize = 128 * 1024 * 1024; + let (tx, rx) = flume::unbounded(); + let write_task = tokio::spawn({ + let db = db.current(); + async move { db.buffered_write(rx, BUFFER_CAPCITY_BYTES).await } + }); + walk_snapshot(&tipset, DEFAULT_RECENT_ROOTS, |cid| { + let db = db.clone(); + let tx = tx.clone(); + async move { + let block = db + .get(&cid)? + .ok_or_else(|| anyhow::anyhow!("Cid {cid} not found in blockstore"))?; + if !db.current().has(&cid)? { + tx.send_async((cid.to_bytes(), block.clone())).await?; } - }) - .await?; - drop(tx); - write_task.await??; - } + + Ok(block) + } + }) + .await?; + drop(tx); + write_task.await??; + info!( "Garbage collection finished at epoch {}, took {}s", tipset.epoch(), (Utc::now() - start).num_seconds() ); - db.clean_tracked(1, true)?; db.next_partition()?; - db.clean_untracked()?; Ok(()) } } diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs index abf220e63822..46f7d2a48300 100644 --- a/node/db/src/rolling/impls.rs +++ b/node/db/src/rolling/impls.rs @@ -1,13 +1,13 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use chrono::Utc; use cid::Cid; use forest_libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite}; use forest_utils::db::file_backed_obj::FileBackedObject; use fvm_ipld_blockstore::Blockstore; use human_repr::HumanCount; -use parking_lot::{RwLock, RwLockReadGuard}; +use parking_lot::RwLock; +use uuid::Uuid; use super::*; use crate::*; @@ -115,8 +115,7 @@ impl Store for RollingDB { } fn next_partition(&self) -> anyhow::Result<()> { - let (name, db) = self.create_untracked()?; - self.track_as_current(name, db) + self.next_current() } } @@ -182,71 +181,30 @@ impl RollingDB { if !db_root.exists() { std::fs::create_dir_all(db_root.as_path())?; } - let (mut db_index, mut db_queue) = load_db_queue(&db_root, &db_config)?; - let current = if db_queue.is_empty() { - let (name, db) = create_untracked(&db_root, &db_config)?; - track_as_youngest(&mut db_index, &mut db_queue, name.clone(), db.clone())?; - (name, db) - } else { - (db_index.inner().db_names[0].clone(), db_queue[0].clone()) - }; + let (db_index, current, old) = load_dbs(&db_root, &db_config)?; Ok(Self { db_root: db_root.into(), db_config: db_config.into(), db_index: RwLock::new(db_index).into(), - db_queue: RwLock::new(db_queue).into(), current: RwLock::new(current).into(), + old: RwLock::new(old).into(), }) } - pub fn track_as_current(&self, name: String, db: Db) -> anyhow::Result<()> { + fn next_current(&self) -> anyhow::Result<()> { + let new_db_name = Uuid::new_v4().simple().to_string(); + let db = open_db(&self.db_root.join(&new_db_name), &self.db_config)?; + *self.old.write() = self.current.read().clone(); + *self.current.write() = db; let mut db_index = self.db_index.write(); - let mut db_queue = self.db_queue.write(); - track_as_youngest(&mut db_index, &mut db_queue, name.clone(), db.clone())?; - *self.current.write() = (name, db); - Ok(()) - } - - pub fn create_untracked(&self) -> anyhow::Result<(String, Db)> { - create_untracked(&self.db_root, &self.db_config) - } - - pub fn clean_tracked(&self, n_db_to_reserve: usize, delete: bool) -> anyhow::Result<()> { - anyhow::ensure!(n_db_to_reserve > 0); - - let mut db_index = self.db_index.write(); - let mut db_queue = self.db_queue.write(); - while db_queue.len() > n_db_to_reserve { - if let Some(db) = db_queue.pop_back() { - db.flush()?; - } - if let Some(name) = db_index.inner_mut().db_names.pop_back() { - info!("Closing DB {name}"); - if delete { - let db_path = self.db_root.join(name); - delete_db(&db_path); - } - } - } - - db_index.flush_to_file() - } + let db_index_inner_mut = db_index.inner_mut(); + let old_db_path = self.db_root.join(&db_index_inner_mut.old); + db_index_inner_mut.old = db_index_inner_mut.current.clone(); + db_index_inner_mut.current = new_db_name; + db_index.flush_to_file()?; + delete_db(&old_db_path); - pub fn clean_untracked(&self) -> anyhow::Result<()> { - if let Ok(dir) = std::fs::read_dir(self.db_root.as_path()) { - let db_index = self.db_index.read(); - dir.flatten() - .filter(|entry| { - entry.path().is_dir() - && db_index - .inner() - .db_names - .iter() - .all(|name| entry.path() != self.db_root.join(name).as_path()) - }) - .for_each(|entry| delete_db(&entry.path())); - } Ok(()) } @@ -256,49 +214,35 @@ impl RollingDB { pub fn current_size_in_bytes(&self) -> anyhow::Result { Ok(fs_extra::dir::get_size( - self.db_root.as_path().join(self.current.read().0.as_str()), + self.db_root + .as_path() + .join(self.db_index.read().inner().current.as_str()), )?) } - pub fn db_count(&self) -> usize { - self.db_queue().len() - } - pub fn current(&self) -> Db { - self.current.read().1.clone() + self.current.read().clone() } - fn db_queue(&self) -> RwLockReadGuard> { - self.db_queue.read() + fn db_queue(&self) -> [Db; 2] { + [self.current.read().clone(), self.old.read().clone()] } } -fn load_db_queue( - db_root: &Path, - db_config: &DbConfig, -) -> anyhow::Result<(FileBacked, VecDeque)> { +fn load_dbs(db_root: &Path, db_config: &DbConfig) -> anyhow::Result<(FileBacked, Db, Db)> { let mut db_index = FileBacked::load_from_file_or_create(db_root.join("db_index.yaml"), Default::default)?; - let mut db_queue = VecDeque::new(); - let index_inner_mut: &mut DbIndex = db_index.inner_mut(); - for i in (0..index_inner_mut.db_names.len()).rev() { - let name = index_inner_mut.db_names[i].as_str(); - let db_path = db_root.join(name); - if !db_path.is_dir() { - index_inner_mut.db_names.remove(i); - continue; - } - match open_db(&db_path, db_config) { - Ok(db) => db_queue.push_front(db), - Err(err) => { - index_inner_mut.db_names.remove(i); - warn!("Failed to open database under {}: {err}", db_path.display()); - } - } + let db_index_mut: &mut DbIndex = db_index.inner_mut(); + if db_index_mut.current.is_empty() { + db_index_mut.current = Uuid::new_v4().simple().to_string(); } - + if db_index_mut.old.is_empty() { + db_index_mut.old = Uuid::new_v4().simple().to_string(); + } + let current = open_db(&db_root.join(&db_index_mut.current), db_config)?; + let old = open_db(&db_root.join(&db_index_mut.old), db_config)?; db_index.flush_to_file()?; - Ok((db_index, db_queue)) + Ok((db_index, current, old)) } fn delete_db(db_path: &Path) { @@ -317,21 +261,3 @@ fn delete_db(db_path: &Path) { ); } } - -fn track_as_youngest( - db_index: &mut FileBacked, - db_queue: &mut VecDeque, - name: String, - db: Db, -) -> anyhow::Result<()> { - info!("Setting db {name} as current"); - db_queue.push_front(db); - db_index.inner_mut().db_names.push_front(name); - db_index.flush_to_file() -} - -fn create_untracked(db_root: &Path, db_config: &DbConfig) -> anyhow::Result<(String, Db)> { - let name = Utc::now().timestamp_millis().to_string(); - let db = open_db(&db_root.join(&name), db_config)?; - Ok((name, db)) -} diff --git a/node/db/src/rolling/mod.rs b/node/db/src/rolling/mod.rs index d8b0fdad246a..1a79342f6aa9 100644 --- a/node/db/src/rolling/mod.rs +++ b/node/db/src/rolling/mod.rs @@ -6,7 +6,6 @@ pub use gc::*; mod impls; use std::{ - collections::VecDeque, path::{Path, PathBuf}, sync::Arc, }; @@ -23,13 +22,14 @@ pub struct RollingDB { db_root: Arc, db_config: Arc, db_index: Arc>>, - /// A queue of active databases, from youngest to oldest - db_queue: Arc>>, /// The current writable DB - current: Arc>, + current: Arc>, + /// The old writable DB + old: Arc>, } #[derive(Debug, Default, Serialize, Deserialize)] struct DbIndex { - db_names: VecDeque, + current: String, + old: String, } diff --git a/node/db/tests/rolling_test.rs b/node/db/tests/rolling_test.rs index 5ae7fbe0e599..e9e02990884a 100644 --- a/node/db/tests/rolling_test.rs +++ b/node/db/tests/rolling_test.rs @@ -8,7 +8,7 @@ mod tests { use anyhow::*; use cid::{multihash::MultihashDigest, Cid}; - use forest_db::rolling::RollingDB; + use forest_db::{rolling::RollingDB, Store}; use forest_libp2p_bitswap::BitswapStoreRead; use fvm_ipld_blockstore::Blockstore; use rand::Rng; @@ -35,9 +35,9 @@ mod tests { for (i, (k, block)) in pairs.iter().enumerate() { if i == split_index { sleep(Duration::from_millis(1)); - println!("Creating another inner db"); - let (name, db) = rolling_db.create_untracked()?; - rolling_db.track_as_current(name, db)?; + println!("Creating a new current db"); + rolling_db.next_partition()?; + println!("Created a new current db"); } rolling_db.put_keyed(k, block)?; } @@ -50,8 +50,7 @@ mod tests { ); } - rolling_db.clean_tracked(1, false)?; - ensure!(rolling_db.db_count() == 1); + rolling_db.next_partition()?; for (i, (k, _)) in pairs.iter().enumerate() { if i < split_index { @@ -61,11 +60,9 @@ mod tests { } } - rolling_db.clean_untracked()?; drop(rolling_db); let rolling_db = RollingDB::load_or_create(db_root.path().into(), Default::default())?; - ensure!(rolling_db.db_count() == 1); for (i, (k, _)) in pairs.iter().enumerate() { if i < split_index { ensure!(!rolling_db.contains(k)?); From 6e3339cfbbc6ee272ef34d19504d37aeeb3d368c Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 14:05:14 +0800 Subject: [PATCH 17/43] bypass GC at epoch 0 --- node/db/src/rolling/gc.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index d01185e1870a..18b2622aa4b0 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -92,6 +92,10 @@ where let start = Utc::now(); let tipset = (self.get_tipset)(); + if tipset.epoch() == 0 { + return Ok(()); + } + info!("Garbage collection started at epoch {}", tipset.epoch()); let db = &self.db; // 128MB From 9f60ea2baa5278ed266c91ab6366a4ee47653528 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 14:17:23 +0800 Subject: [PATCH 18/43] Bypass size checking during import --- node/db/src/rolling/gc.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 18b2622aa4b0..394a69d08719 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -48,6 +48,12 @@ where // Check every 10 mins tokio::time::sleep(Duration::from_secs(10 * 60)).await; + // Bypass size checking during import + let tipset = (self.get_tipset)(); + if tipset.epoch() == 0 { + continue; + } + // Bypass size checking when lock is held { let lock = self.lock.try_lock(); @@ -62,7 +68,7 @@ where ) { // Collect when size of young partition > 0.5 * size of old partition if total_size > 0 && current_size * 3 > total_size { - if let Err(err) = self.collect_once().await { + if let Err(err) = self.collect_once(tipset).await { warn!("Garbage collection failed: {err}"); } } @@ -73,8 +79,9 @@ where pub async fn collect_loop_event(self: &Arc) -> anyhow::Result<()> { while let Ok(responder) = self.gc_rx.recv_async().await { let this = self.clone(); + let tipset = (self.get_tipset)(); tokio::spawn(async move { - let result = this.collect_once().await; + let result = this.collect_once(tipset).await; if let Err(e) = responder.send(result) { warn!("{e}"); } @@ -84,17 +91,13 @@ where Ok(()) } - async fn collect_once(&self) -> anyhow::Result<()> { + async fn collect_once(&self, tipset: Tipset) -> anyhow::Result<()> { let guard = self.lock.try_lock(); if guard.is_err() { anyhow::bail!("Another garbage collection task is in progress."); } let start = Utc::now(); - let tipset = (self.get_tipset)(); - if tipset.epoch() == 0 { - return Ok(()); - } info!("Garbage collection started at epoch {}", tipset.epoch()); let db = &self.db; From 6c930bb02672d2be28e1a5647788472c6b30bcf4 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 17:11:08 +0800 Subject: [PATCH 19/43] use bounded channel to reduce mem usage during db re-index --- node/db/src/rolling/gc.rs | 2 +- utils/genesis/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 394a69d08719..dd5ed75be7b4 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -103,7 +103,7 @@ where let db = &self.db; // 128MB const BUFFER_CAPCITY_BYTES: usize = 128 * 1024 * 1024; - let (tx, rx) = flume::unbounded(); + let (tx, rx) = flume::bounded(100); let write_task = tokio::spawn({ let db = db.current(); async move { db.buffered_write(rx, BUFFER_CAPCITY_BYTES).await } diff --git a/utils/genesis/src/lib.rs b/utils/genesis/src/lib.rs index 38da88cd8dcf..55273ad6329e 100644 --- a/utils/genesis/src/lib.rs +++ b/utils/genesis/src/lib.rs @@ -199,7 +199,7 @@ where // 1GB const BUFFER_CAPCITY_BYTES: usize = 1024 * 1024 * 1024; - let (tx, rx) = flume::unbounded(); + let (tx, rx) = flume::bounded(100); let write_task = tokio::spawn(async move { store.buffered_write(rx, BUFFER_CAPCITY_BYTES).await }); let mut car_reader = CarReader::new(reader).await?; From 01810039b3e78d9357c41e1f4c6a06eda7af9522 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 18:30:30 +0800 Subject: [PATCH 20/43] docs and fixes --- node/db/src/errors.rs | 9 ------- node/db/src/rolling/gc.rs | 33 +++++++++++++++++++++++++ node/db/src/rolling/mod.rs | 7 ++++++ utils/forest_utils/src/common/mod.rs | 37 ---------------------------- utils/forest_utils/src/lib.rs | 1 - 5 files changed, 40 insertions(+), 47 deletions(-) delete mode 100644 utils/forest_utils/src/common/mod.rs diff --git a/node/db/src/errors.rs b/node/db/src/errors.rs index 59f3f310bb09..f90014096465 100644 --- a/node/db/src/errors.rs +++ b/node/db/src/errors.rs @@ -1,9 +1,6 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use std::fmt::Display; - -use forest_utils::common::AggregatedError; use thiserror::Error; /// Database error @@ -45,9 +42,3 @@ impl From for String { e.to_string() } } - -impl From> for Error { - fn from(value: AggregatedError) -> Self { - Self::Other(value.to_string()) - } -} diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index dd5ed75be7b4..93b5109203d8 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -1,6 +1,39 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +//! +//! The current implementation of the garbage collector is a concurrent, +//! semi-space one. +//! +//! ## GC workflow +//! 1. Walk back from the current heaviest tipset to the genesis block, collect +//! all the blocks that are reachable from the snapshot +//! 2. writes blocks that are absent from the `current` database to it +//! 3. delete `old` database(s) +//! 4. sets `current` database to a newly created one +//! +//! ## Correctness +//! This alroghrim considers all blocks that are visited during the snapshot +//! export task reachable, and ensures they are all transfered and kept in the +//! current DB space. A snapshot can be used to bootstrap the node from +//! scratch thus the algorithm is considered approapriate when the post-GC +//! database contains blocks that are sufficient for exporting a snapshot +//! +//! ## Disk usage +//! During `walk_snapshot`, data from the `old` DB is duplicated in the +//! `current` DB, which uses extra disk space of up-to 100% of the snapshot file +//! size +//! +//! ## Memory usage +//! During the data carry-over process, a memory buffer with a fixed capacity is +//! used to speed up the database write operation +//! +//! ## Scheduling +//! 1. GC is triggered automatically when current DB size is greater than 50% of +//! the old DB size +//! 2. GC can be triggered manually by `forest-cli db gc` command +//! 3. There's global GC lock to ensure at most one GC job running + use std::time::Duration; use chrono::Utc; diff --git a/node/db/src/rolling/mod.rs b/node/db/src/rolling/mod.rs index 1a79342f6aa9..bd0bcfb05ccf 100644 --- a/node/db/src/rolling/mod.rs +++ b/node/db/src/rolling/mod.rs @@ -1,6 +1,13 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +//! +//! This DB wrapper is specially designed for supporting the concurrent, +//! semi-space GC algorithm that is implemented in the [gc] module, containing a +//! reference to the `old` DB space and a reference to the `current` DB space. +//! Both underlying key-vale DB are supposed to contain only block data as value and +//! its content-addressed CID as key + mod gc; pub use gc::*; mod impls; diff --git a/utils/forest_utils/src/common/mod.rs b/utils/forest_utils/src/common/mod.rs deleted file mode 100644 index 892f1979972d..000000000000 --- a/utils/forest_utils/src/common/mod.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019-2023 ChainSafe Systems -// SPDX-License-Identifier: Apache-2.0, MIT - -use std::fmt::Display; - -#[derive(Debug, thiserror::Error)] -pub struct AggregatedError(Vec); - -impl AggregatedError { - pub fn new() -> Self { - Default::default() - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn push(&mut self, item: T) { - self.0.push(item); - } -} - -impl Default for AggregatedError { - fn default() -> Self { - Self(vec![]) - } -} - -impl Display for AggregatedError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - writeln!(f, "Aggregated errors:")?; - for e in self.0.iter() { - writeln!(f, " {e}")?; - } - Ok(()) - } -} diff --git a/utils/forest_utils/src/lib.rs b/utils/forest_utils/src/lib.rs index bd65dee2fcec..9aed941b2ede 100644 --- a/utils/forest_utils/src/lib.rs +++ b/utils/forest_utils/src/lib.rs @@ -3,7 +3,6 @@ pub extern crate const_format; -pub mod common; pub mod db; pub mod io; pub mod json; From 8b4fd5c10a4b49a605fd7eb2ae31be230cc059d5 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 18:37:51 +0800 Subject: [PATCH 21/43] log total entries and total size --- node/db/src/ext.rs | 25 ++++++++++++++++++++----- node/db/src/rolling/mod.rs | 4 ++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/node/db/src/ext.rs b/node/db/src/ext.rs index b810afa3b45e..6ca2be7c3122 100644 --- a/node/db/src/ext.rs +++ b/node/db/src/ext.rs @@ -2,6 +2,9 @@ // SPDX-License-Identifier: Apache-2.0, MIT use async_trait::async_trait; +use chrono::Utc; +use human_repr::HumanCount; +use log::info; use crate::*; @@ -12,17 +15,29 @@ pub trait StoreExt: Store { rx: flume::Receiver<(Vec, Vec)>, buffer_capacity_bytes: usize, ) -> anyhow::Result<()> { - let mut estimated_size = 0; + let start = Utc::now(); + let mut total_bytes = 0; + let mut total_entries = 0; + let mut estimated_buffer_bytes = 0; let mut buffer = vec![]; while let Ok((key, value)) = rx.recv_async().await { - estimated_size += key.len() + value.len(); + estimated_buffer_bytes += key.len() + value.len(); + total_bytes += key.len() + value.len(); + total_entries += 1; buffer.push((key, value)); - if estimated_size >= buffer_capacity_bytes { + if estimated_buffer_bytes >= buffer_capacity_bytes { self.bulk_write(std::mem::take(&mut buffer))?; - estimated_size = 0; + estimated_buffer_bytes = 0; } } - Ok(self.bulk_write(buffer)?) + self.bulk_write(buffer)?; + info!( + "Buffered write completed: total entries: {total_entries}, total size: {}, took: {}s", + total_bytes.human_count_bytes(), + (Utc::now() - start).num_seconds() + ); + + Ok(()) } } diff --git a/node/db/src/rolling/mod.rs b/node/db/src/rolling/mod.rs index bd0bcfb05ccf..00f78f6c4d70 100644 --- a/node/db/src/rolling/mod.rs +++ b/node/db/src/rolling/mod.rs @@ -5,8 +5,8 @@ //! This DB wrapper is specially designed for supporting the concurrent, //! semi-space GC algorithm that is implemented in the [gc] module, containing a //! reference to the `old` DB space and a reference to the `current` DB space. -//! Both underlying key-vale DB are supposed to contain only block data as value and -//! its content-addressed CID as key +//! Both underlying key-vale DB are supposed to contain only block data as value +//! and its content-addressed CID as key mod gc; pub use gc::*; From 7a4fda3c29c6072bf105fac5fd92f06484190b46 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 18:49:53 +0800 Subject: [PATCH 22/43] fix docs ci --- node/db/src/rolling/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/node/db/src/rolling/mod.rs b/node/db/src/rolling/mod.rs index 00f78f6c4d70..e7892c6d79a0 100644 --- a/node/db/src/rolling/mod.rs +++ b/node/db/src/rolling/mod.rs @@ -3,10 +3,10 @@ //! //! This DB wrapper is specially designed for supporting the concurrent, -//! semi-space GC algorithm that is implemented in the [gc] module, containing a -//! reference to the `old` DB space and a reference to the `current` DB space. -//! Both underlying key-vale DB are supposed to contain only block data as value -//! and its content-addressed CID as key +//! semi-space GC algorithm that is implemented in [DbGarbageCollector], +//! containing a reference to the `old` DB space and a reference to the +//! `current` DB space. Both underlying key-vale DB are supposed to contain only +//! block data as value and its content-addressed CID as key mod gc; pub use gc::*; From db49a6a4658b6f661b953c7acedd85cb856506a0 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 19:02:28 +0800 Subject: [PATCH 23/43] remove invalid options from `chain export` --- forest/cli/src/cli/snapshot_cmd.rs | 14 +++----------- ipld/src/util.rs | 2 +- node/db/src/rolling/gc.rs | 2 +- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/forest/cli/src/cli/snapshot_cmd.rs b/forest/cli/src/cli/snapshot_cmd.rs index 40e263a4aaa2..d59240d7bc9b 100644 --- a/forest/cli/src/cli/snapshot_cmd.rs +++ b/forest/cli/src/cli/snapshot_cmd.rs @@ -16,7 +16,7 @@ use forest_db::{ Store, }; use forest_genesis::{forest_load_car, read_genesis_header}; -use forest_ipld::{recurse_links_hash, CidHashSet}; +use forest_ipld::{recurse_links_hash, CidHashSet, DEFAULT_RECENT_STATE_ROOTS}; use forest_rpc_api::chain_api::ChainExportParams; use forest_rpc_client::chain_ops::*; use forest_utils::{io::parser::parse_duration, net::FetchProgress, retry}; @@ -38,12 +38,6 @@ pub(crate) const OUTPUT_PATH_DEFAULT_FORMAT: &str = pub enum SnapshotCommands { /// Export a snapshot of the chain to `` Export { - /// Tipset to start the export from, default is the chain head - #[arg(short, long)] - tipset: Option, - /// Specify the number of recent state roots to include in the export. - #[arg(short, long, default_value = "2000")] - recent_stateroots: i64, /// Snapshot output path. Default to /// `forest_snapshot_{chain}_{year}-{month}-{day}_height_{height}.car` /// Date is in ISO 8601 date format. @@ -153,8 +147,6 @@ impl SnapshotCommands { pub async fn run(&self, config: Config) -> anyhow::Result<()> { match self { Self::Export { - tipset, - recent_stateroots, output_path, skip_checksum, dry_run, @@ -164,7 +156,7 @@ impl SnapshotCommands { Err(_) => cli_error_and_die("Could not get network head", 1), }; - let epoch = tipset.unwrap_or(chain_head.epoch()); + let epoch = chain_head.epoch(); let now = OffsetDateTime::now_utc(); @@ -199,7 +191,7 @@ impl SnapshotCommands { let params = ChainExportParams { epoch, - recent_roots: *recent_stateroots, + recent_roots: DEFAULT_RECENT_STATE_ROOTS, output_path, tipset_keys: TipsetKeysJson(chain_head.key().clone()), skip_checksum: *skip_checksum, diff --git a/ipld/src/util.rs b/ipld/src/util.rs index c2b93a1f11a7..29eec737798b 100644 --- a/ipld/src/util.rs +++ b/ipld/src/util.rs @@ -80,7 +80,7 @@ where Ok(()) } -pub const DEFAULT_RECENT_ROOTS: i64 = 2000; +pub const DEFAULT_RECENT_STATE_ROOTS: i64 = 2000; /// Walks over tipset and state data and loads all blocks not yet seen. /// This is tracked based on the callback function loading blocks. diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 93b5109203d8..22f553c9444f 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -141,7 +141,7 @@ where let db = db.current(); async move { db.buffered_write(rx, BUFFER_CAPCITY_BYTES).await } }); - walk_snapshot(&tipset, DEFAULT_RECENT_ROOTS, |cid| { + walk_snapshot(&tipset, DEFAULT_RECENT_STATE_ROOTS, |cid| { let db = db.clone(); let tx = tx.clone(); async move { From df4abb46367bfadf89aba42b1747596be9163b13 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 19:24:32 +0800 Subject: [PATCH 24/43] fix typos, add design goals and reasoning --- node/db/src/rolling/gc.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 22f553c9444f..1a457f4c39e4 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -5,6 +5,15 @@ //! The current implementation of the garbage collector is a concurrent, //! semi-space one. //! +//! ## Design goals +//! Implement a correct GC algorithm that is simple and efficient for forest scenarios. +//! +//! ## GC algorithm +//! We choose `semi-space` GC algorithm for simplicity and sufficiency +//! Besides `semi-space`, `mark-and-sweep` was also considered and evaluated. +//! However, it's not feasible because of the limitations of the underlying DB +//! we use, more specifically, iterating the DB and retrieve the original key. See +//! //! ## GC workflow //! 1. Walk back from the current heaviest tipset to the genesis block, collect //! all the blocks that are reachable from the snapshot @@ -13,10 +22,10 @@ //! 4. sets `current` database to a newly created one //! //! ## Correctness -//! This alroghrim considers all blocks that are visited during the snapshot -//! export task reachable, and ensures they are all transfered and kept in the -//! current DB space. A snapshot can be used to bootstrap the node from -//! scratch thus the algorithm is considered approapriate when the post-GC +//! This algorithm considers all blocks that are visited during the snapshot +//! export task reachable, and ensures they are all transferred and kept in the +//! current DB space. A snapshot can be used to bootstrap a node from +//! scratch thus the algorithm is considered appropriate when the post-GC //! database contains blocks that are sufficient for exporting a snapshot //! //! ## Disk usage @@ -32,7 +41,7 @@ //! 1. GC is triggered automatically when current DB size is greater than 50% of //! the old DB size //! 2. GC can be triggered manually by `forest-cli db gc` command -//! 3. There's global GC lock to ensure at most one GC job running +//! 3. There's global GC lock to ensure at most one GC job is running use std::time::Duration; From 18c824a5d20695ca9144ab07a4d50a69b2d326ea Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 19:27:03 +0800 Subject: [PATCH 25/43] wordsmith --- node/db/src/rolling/gc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 1a457f4c39e4..4bb55d329963 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -12,7 +12,7 @@ //! We choose `semi-space` GC algorithm for simplicity and sufficiency //! Besides `semi-space`, `mark-and-sweep` was also considered and evaluated. //! However, it's not feasible because of the limitations of the underlying DB -//! we use, more specifically, iterating the DB and retrieve the original key. See +//! we use, more specifically, limitations in iterating the DB and retrieving the original key. See //! //! ## GC workflow //! 1. Walk back from the current heaviest tipset to the genesis block, collect From 314f742b72f1ded35276617f285fc09d8f28eca3 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 19:36:16 +0800 Subject: [PATCH 26/43] fix lint --- node/db/src/rolling/gc.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 4bb55d329963..da4aac3af3fc 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -6,8 +6,9 @@ //! semi-space one. //! //! ## Design goals -//! Implement a correct GC algorithm that is simple and efficient for forest scenarios. -//! +//! Implement a correct GC algorithm that is simple and efficient for forest +//! scenarios. +//! //! ## GC algorithm //! We choose `semi-space` GC algorithm for simplicity and sufficiency //! Besides `semi-space`, `mark-and-sweep` was also considered and evaluated. From 1a37851418da1b27dcb0a069d787ed6a8fa11916 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 20:02:28 +0800 Subject: [PATCH 27/43] update doc --- node/db/src/rolling/gc.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index da4aac3af3fc..e8d66d1ae230 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -2,8 +2,15 @@ // SPDX-License-Identifier: Apache-2.0, MIT //! -//! The current implementation of the garbage collector is a concurrent, -//! semi-space one. +//! The state of the Filecoin Blockchain is a persistent, directed acyclic +//! graph. Data in this graph is never mutated nor explicitly deleted but may +//! become unreachable over time. +//! +//! This module contains a concurrent, semi-space garbage collector. The garbage +//! collector is guaranteed to be non-blocking and can be expected to run with a +//! fixed memory overhead and require disk space proportional to the size of the +//! reachable graph. For example, if the size of the reachable graph is 100GiB, +//! expect this garbage collector to use 3x100GiB = 300GiB of storage. //! //! ## Design goals //! Implement a correct GC algorithm that is simple and efficient for forest From 59901c01c38adbbfd3afa214a3f116c3d2774587 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 21:19:57 +0800 Subject: [PATCH 28/43] fix doc and auto-gc trigger condition --- node/db/src/rolling/gc.rs | 51 ++++++++++++++++++++++++-------------- node/db/src/rolling/mod.rs | 18 ++++++++++---- 2 files changed, 46 insertions(+), 23 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index e8d66d1ae230..dfc8e83d5557 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -2,15 +2,8 @@ // SPDX-License-Identifier: Apache-2.0, MIT //! -//! The state of the Filecoin Blockchain is a persistent, directed acyclic -//! graph. Data in this graph is never mutated nor explicitly deleted but may -//! become unreachable over time. -//! -//! This module contains a concurrent, semi-space garbage collector. The garbage -//! collector is guaranteed to be non-blocking and can be expected to run with a -//! fixed memory overhead and require disk space proportional to the size of the -//! reachable graph. For example, if the size of the reachable graph is 100GiB, -//! expect this garbage collector to use 3x100GiB = 300GiB of storage. +//! The current implementation of the garbage collector is a concurrent, +//! semi-space one. //! //! ## Design goals //! Implement a correct GC algorithm that is simple and efficient for forest @@ -46,17 +39,21 @@ //! used to speed up the database write operation //! //! ## Scheduling -//! 1. GC is triggered automatically when current DB size is greater than 50% of -//! the old DB size +//! 1. GC is triggered automatically when total DB size is greater than 2x of +//! the last reachable data size //! 2. GC can be triggered manually by `forest-cli db gc` command //! 3. There's global GC lock to ensure at most one GC job is running -use std::time::Duration; +use std::{ + sync::atomic::{self, AtomicU64, AtomicUsize}, + time::Duration, +}; use chrono::Utc; use forest_blocks::Tipset; use forest_ipld::util::*; use fvm_ipld_blockstore::Blockstore; +use human_repr::HumanCount; use tokio::sync::Mutex; use super::*; @@ -71,6 +68,7 @@ where lock: Mutex<()>, gc_tx: flume::Sender>>, gc_rx: flume::Receiver>>, + last_reachable_bytes: AtomicU64, } impl DbGarbageCollector @@ -86,6 +84,7 @@ where lock: Default::default(), gc_tx, gc_rx, + last_reachable_bytes: AtomicU64::new(0), } } @@ -112,12 +111,18 @@ where } } - if let (Ok(total_size), Ok(current_size)) = ( + if let (Ok(total_size), Ok(current_size), last_reachable_bytes) = ( self.db.total_size_in_bytes(), self.db.current_size_in_bytes(), + self.last_reachable_bytes.load(atomic::Ordering::Relaxed), ) { - // Collect when size of young partition > 0.5 * size of old partition - if total_size > 0 && current_size * 3 > total_size { + let should_collect = if last_reachable_bytes > 0 { + total_size > 2 * last_reachable_bytes + } else { + total_size > 0 && current_size * 3 > total_size + }; + + if should_collect { if let Err(err) = self.collect_once(tipset).await { warn!("Garbage collection failed: {err}"); } @@ -148,6 +153,7 @@ where } let start = Utc::now(); + let reachable_bytes = Arc::new(AtomicUsize::new(0)); info!("Garbage collection started at epoch {}", tipset.epoch()); let db = &self.db; @@ -161,12 +167,16 @@ where walk_snapshot(&tipset, DEFAULT_RECENT_STATE_ROOTS, |cid| { let db = db.clone(); let tx = tx.clone(); + let reachable_bytes = reachable_bytes.clone(); async move { let block = db .get(&cid)? .ok_or_else(|| anyhow::anyhow!("Cid {cid} not found in blockstore"))?; if !db.current().has(&cid)? { - tx.send_async((cid.to_bytes(), block.clone())).await?; + let pair = (cid.to_bytes(), block.clone()); + reachable_bytes + .fetch_add(pair.0.len() + pair.1.len(), atomic::Ordering::Relaxed); + tx.send_async(pair).await?; } Ok(block) @@ -176,11 +186,16 @@ where drop(tx); write_task.await??; + let reachable_bytes = reachable_bytes.load(atomic::Ordering::Relaxed); + self.last_reachable_bytes + .store(reachable_bytes as _, atomic::Ordering::Relaxed); info!( - "Garbage collection finished at epoch {}, took {}s", + "Garbage collection finished at epoch {}, took {}s, reachable data size: {}", tipset.epoch(), - (Utc::now() - start).num_seconds() + (Utc::now() - start).num_seconds(), + reachable_bytes.human_count_bytes(), ); + db.next_partition()?; Ok(()) } diff --git a/node/db/src/rolling/mod.rs b/node/db/src/rolling/mod.rs index e7892c6d79a0..e59657bd240f 100644 --- a/node/db/src/rolling/mod.rs +++ b/node/db/src/rolling/mod.rs @@ -1,12 +1,15 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +//! The state of the Filecoin Blockchain is a persistent, directed acyclic +//! graph. Data in this graph is never mutated nor explicitly deleted but may +//! become unreachable over time. //! -//! This DB wrapper is specially designed for supporting the concurrent, -//! semi-space GC algorithm that is implemented in [DbGarbageCollector], -//! containing a reference to the `old` DB space and a reference to the -//! `current` DB space. Both underlying key-vale DB are supposed to contain only -//! block data as value and its content-addressed CID as key +//! This module contains a concurrent, semi-space garbage collector. The garbage +//! collector is guaranteed to be non-blocking and can be expected to run with a +//! fixed memory overhead and require disk space proportional to the size of the +//! reachable graph. For example, if the size of the reachable graph is 100GiB, +//! expect this garbage collector to use 3x100GiB = 300GiB of storage. mod gc; pub use gc::*; @@ -24,6 +27,11 @@ use serde::{Deserialize, Serialize}; use crate::db_engine::{open_db, Db, DbConfig}; +/// This DB wrapper is specially designed for supporting the concurrent, +/// semi-space GC algorithm that is implemented in [DbGarbageCollector], +/// containing a reference to the `old` DB space and a reference to the +/// `current` DB space. Both underlying key-vale DB are supposed to contain only +/// block data as value and its content-addressed CID as key #[derive(Clone)] pub struct RollingDB { db_root: Arc, From 6829e7ed420f6bf8963695d8f535cd1e5854ec85 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 13 Mar 2023 21:56:25 +0800 Subject: [PATCH 29/43] fix ut --- forest/daemon/src/daemon.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/forest/daemon/src/daemon.rs b/forest/daemon/src/daemon.rs index f78e87b28b24..6385edb76e93 100644 --- a/forest/daemon/src/daemon.rs +++ b/forest/daemon/src/daemon.rs @@ -586,7 +586,7 @@ mod test { use super::*; - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn import_snapshot_from_file_valid() -> anyhow::Result<()> { anyhow::ensure!(import_snapshot_from_file("test_files/chain4.car") .await @@ -594,20 +594,20 @@ mod test { Ok(()) } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn import_snapshot_from_file_invalid() -> anyhow::Result<()> { anyhow::ensure!(import_snapshot_from_file("Cargo.toml").await.is_err()); Ok(()) } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn import_snapshot_from_file_not_found() -> anyhow::Result<()> { anyhow::ensure!(import_snapshot_from_file("dummy.car").await.is_err()); Ok(()) } #[cfg(feature = "slow_tests")] - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn import_snapshot_from_url_not_found() -> anyhow::Result<()> { anyhow::ensure!(import_snapshot_from_file("https://dummy.com/dummy.car") .await @@ -640,7 +640,7 @@ mod test { Ok(()) } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn import_chain_from_file() -> anyhow::Result<()> { let db = MemoryDB::default(); let chain_config = Arc::new(ChainConfig::default()); From 6d35873f12d9881c022070ef2c71abaf853fc5e0 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 16 Mar 2023 09:02:59 +0800 Subject: [PATCH 30/43] docs --- node/db/src/rolling/gc.rs | 44 +++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index dfc8e83d5557..171916c967f7 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -43,6 +43,22 @@ //! the last reachable data size //! 2. GC can be triggered manually by `forest-cli db gc` command //! 3. There's global GC lock to ensure at most one GC job is running +//! +//! ## Performance +//! GC performance is typically 1x-1.5x of `snapshot export`, depending on +//! number of write operations to the `current` DB space. +//! +//! ### Look up performance +//! DB lookup performance is almost on-par between from single DB and two DBs. +//! Time cost of `forest-cli snapshot export --dry-run` on DO droplet with 16GiB +//! ram is between `9000s` to `11000s` for both scenarios, no significant +//! performance regression has been observed +//! +//! ### Write performance +//! DB write performance is typically on-par with `snapshot import`, note that +//! when the `current` DB space is very large, it tends to trigger DB re-index +//! more frequently, each DB re-index could pause the GC process for a few +//! minutes. The same behaviour is observed during snapshot import as well. use std::{ sync::atomic::{self, AtomicU64, AtomicUsize}, @@ -59,6 +75,9 @@ use tokio::sync::Mutex; use super::*; use crate::{Store, StoreExt}; +/// 100GiB +const ESTIMATED_LAST_REACHABLE_BYTES_FOR_COLD_START: u64 = 100 * 1024_u64.pow(3); + pub struct DbGarbageCollector where F: Fn() -> Tipset + Send + Sync + 'static, @@ -84,7 +103,7 @@ where lock: Default::default(), gc_tx, gc_rx, - last_reachable_bytes: AtomicU64::new(0), + last_reachable_bytes: AtomicU64::new(ESTIMATED_LAST_REACHABLE_BYTES_FOR_COLD_START), } } @@ -92,6 +111,8 @@ where self.gc_tx.clone() } + /// This loop automatically triggers `collect_once` when the total DB size + /// is greater than 2x of the last reachable data size pub async fn collect_loop_passive(&self) -> anyhow::Result<()> { loop { // Check every 10 mins @@ -111,18 +132,15 @@ where } } - if let (Ok(total_size), Ok(current_size), last_reachable_bytes) = ( + if let (Ok(total_size), mut last_reachable_bytes) = ( self.db.total_size_in_bytes(), - self.db.current_size_in_bytes(), self.last_reachable_bytes.load(atomic::Ordering::Relaxed), ) { - let should_collect = if last_reachable_bytes > 0 { - total_size > 2 * last_reachable_bytes - } else { - total_size > 0 && current_size * 3 > total_size - }; + if last_reachable_bytes == 0 { + last_reachable_bytes = ESTIMATED_LAST_REACHABLE_BYTES_FOR_COLD_START; + } - if should_collect { + if total_size > 2 * last_reachable_bytes { if let Err(err) = self.collect_once(tipset).await { warn!("Garbage collection failed: {err}"); } @@ -131,6 +149,8 @@ where } } + /// This loop listens on events emitted by `forest-cli db gc` and triggers + /// `collect_once` pub async fn collect_loop_event(self: &Arc) -> anyhow::Result<()> { while let Ok(responder) = self.gc_rx.recv_async().await { let this = self.clone(); @@ -146,6 +166,12 @@ where Ok(()) } + /// ## GC workflow + /// 1. Walk back from the current heaviest tipset to the genesis block, + /// collect all the blocks that are reachable from the snapshot + /// 2. writes blocks that are absent from the `current` database to it + /// 3. delete `old` database(s) + /// 4. sets `current` database to a newly created one async fn collect_once(&self, tipset: Tipset) -> anyhow::Result<()> { let guard = self.lock.try_lock(); if guard.is_err() { From ed76a294aa4fe0c39b85cd765b8533facb9f1b75 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 16 Mar 2023 17:17:27 +0800 Subject: [PATCH 31/43] suppress clippy warnings --- forest/daemon/src/daemon.rs | 2 ++ node/db/src/rolling/gc.rs | 1 + utils/genesis/src/lib.rs | 1 + 3 files changed, 4 insertions(+) diff --git a/forest/daemon/src/daemon.rs b/forest/daemon/src/daemon.rs index 50cde91f428e..696a79ce4b7b 100644 --- a/forest/daemon/src/daemon.rs +++ b/forest/daemon/src/daemon.rs @@ -168,10 +168,12 @@ pub(super) async fn start(opts: CliOpts, config: Config) -> anyhow::Result Date: Thu, 16 Mar 2023 20:24:31 +0800 Subject: [PATCH 32/43] remove fn next_partition --- node/db/src/lib.rs | 6 ------ node/db/src/rolling/gc.rs | 2 +- node/db/src/rolling/impls.rs | 8 +++----- node/db/tests/rolling_test.rs | 4 ++-- utils/genesis/src/lib.rs | 3 --- 5 files changed, 6 insertions(+), 17 deletions(-) diff --git a/node/db/src/lib.rs b/node/db/src/lib.rs index 767aea29d784..6a91b1ce1d6b 100644 --- a/node/db/src/lib.rs +++ b/node/db/src/lib.rs @@ -55,12 +55,6 @@ pub trait Store { fn flush(&self) -> Result<(), Error> { Ok(()) } - - /// Create a new physical partition for writing when supported, defaults to - /// doing nothing - fn next_partition(&self) -> anyhow::Result<()> { - Ok(()) - } } impl Store for &BS { diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 0f5b71a288a1..43ae010577fc 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -223,7 +223,7 @@ where reachable_bytes.human_count_bytes(), ); - db.next_partition()?; + db.next_current()?; Ok(()) } } diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs index 46f7d2a48300..de539a8434e7 100644 --- a/node/db/src/rolling/impls.rs +++ b/node/db/src/rolling/impls.rs @@ -113,10 +113,6 @@ impl Store for RollingDB { fn flush(&self) -> Result<(), crate::Error> { Store::flush(&self.current()) } - - fn next_partition(&self) -> anyhow::Result<()> { - self.next_current() - } } impl BitswapStoreRead for RollingDB { @@ -192,7 +188,9 @@ impl RollingDB { }) } - fn next_current(&self) -> anyhow::Result<()> { + /// Sets `current` as `old`, and sets a new DB as `current`, finally delete + /// the dangling `old` DB. + pub fn next_current(&self) -> anyhow::Result<()> { let new_db_name = Uuid::new_v4().simple().to_string(); let db = open_db(&self.db_root.join(&new_db_name), &self.db_config)?; *self.old.write() = self.current.read().clone(); diff --git a/node/db/tests/rolling_test.rs b/node/db/tests/rolling_test.rs index e9e02990884a..987a5f7511d5 100644 --- a/node/db/tests/rolling_test.rs +++ b/node/db/tests/rolling_test.rs @@ -36,7 +36,7 @@ mod tests { if i == split_index { sleep(Duration::from_millis(1)); println!("Creating a new current db"); - rolling_db.next_partition()?; + rolling_db.next_current()?; println!("Created a new current db"); } rolling_db.put_keyed(k, block)?; @@ -50,7 +50,7 @@ mod tests { ); } - rolling_db.next_partition()?; + rolling_db.next_current()?; for (i, (k, _)) in pairs.iter().enumerate() { if i < split_index { diff --git a/utils/genesis/src/lib.rs b/utils/genesis/src/lib.rs index baa3378ab1f3..6ffaf493c451 100644 --- a/utils/genesis/src/lib.rs +++ b/utils/genesis/src/lib.rs @@ -159,9 +159,6 @@ where info!("Accepting {:?} as new head.", ts.cids()); - // Creates a new partition after snapshot import if supported - sm.blockstore().next_partition()?; - Ok(()) } From f90703c23b1f1cb071a635c60ec328277334ae7c Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 16 Mar 2023 20:33:33 +0800 Subject: [PATCH 33/43] fix clippy --- node/db/src/rolling/gc.rs | 2 +- node/db/tests/rolling_test.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 43ae010577fc..07848b968288 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -73,7 +73,7 @@ use human_repr::HumanCount; use tokio::sync::Mutex; use super::*; -use crate::{Store, StoreExt}; +use crate::StoreExt; /// 100GiB const ESTIMATED_LAST_REACHABLE_BYTES_FOR_COLD_START: u64 = 100 * 1024_u64.pow(3); diff --git a/node/db/tests/rolling_test.rs b/node/db/tests/rolling_test.rs index 987a5f7511d5..94aff5074328 100644 --- a/node/db/tests/rolling_test.rs +++ b/node/db/tests/rolling_test.rs @@ -8,7 +8,7 @@ mod tests { use anyhow::*; use cid::{multihash::MultihashDigest, Cid}; - use forest_db::{rolling::RollingDB, Store}; + use forest_db::rolling::RollingDB; use forest_libp2p_bitswap::BitswapStoreRead; use fvm_ipld_blockstore::Blockstore; use rand::Rng; From 578ed1ece673b922cc2d42ae8da4a0880dec25ae Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 16 Mar 2023 21:13:51 +0800 Subject: [PATCH 34/43] set visibility of next_current to pub(crate) --- node/db/src/rolling/impls.rs | 76 ++++++++++++++++++++++++++++++++++- node/db/tests/rolling_test.rs | 76 ----------------------------------- 2 files changed, 75 insertions(+), 77 deletions(-) delete mode 100644 node/db/tests/rolling_test.rs diff --git a/node/db/src/rolling/impls.rs b/node/db/src/rolling/impls.rs index de539a8434e7..d98ee945c392 100644 --- a/node/db/src/rolling/impls.rs +++ b/node/db/src/rolling/impls.rs @@ -190,7 +190,7 @@ impl RollingDB { /// Sets `current` as `old`, and sets a new DB as `current`, finally delete /// the dangling `old` DB. - pub fn next_current(&self) -> anyhow::Result<()> { + pub(crate) fn next_current(&self) -> anyhow::Result<()> { let new_db_name = Uuid::new_v4().simple().to_string(); let db = open_db(&self.db_root.join(&new_db_name), &self.db_config)?; *self.old.write() = self.current.read().clone(); @@ -259,3 +259,77 @@ fn delete_db(db_path: &Path) { ); } } + +#[cfg(test)] +mod tests { + use std::{thread::sleep, time::Duration}; + + use anyhow::*; + use cid::{multihash::MultihashDigest, Cid}; + use forest_libp2p_bitswap::BitswapStoreRead; + use fvm_ipld_blockstore::Blockstore; + use rand::Rng; + use tempfile::TempDir; + + use super::*; + + #[test] + fn rolling_db_behaviour_tests() -> Result<()> { + let db_root = TempDir::new()?; + println!("Creating rolling db under {}", db_root.path().display()); + let rolling_db = RollingDB::load_or_create(db_root.path().into(), Default::default())?; + println!("Generating random blocks"); + let pairs: Vec<_> = (0..1000) + .map(|_| { + let mut bytes = [0; 1024]; + rand::rngs::OsRng.fill(&mut bytes); + let cid = + Cid::new_v0(cid::multihash::Code::Sha2_256.digest(bytes.as_slice())).unwrap(); + (cid, bytes.to_vec()) + }) + .collect(); + + let split_index = 500; + + for (i, (k, block)) in pairs.iter().enumerate() { + if i == split_index { + sleep(Duration::from_millis(1)); + println!("Creating a new current db"); + rolling_db.next_current()?; + println!("Created a new current db"); + } + rolling_db.put_keyed(k, block)?; + } + + for (i, (k, block)) in pairs.iter().enumerate() { + ensure!(rolling_db.contains(k)?, "{i}"); + ensure!( + Blockstore::get(&rolling_db, k)?.unwrap().as_slice() == block, + "{i}" + ); + } + + rolling_db.next_current()?; + + for (i, (k, _)) in pairs.iter().enumerate() { + if i < split_index { + ensure!(!rolling_db.contains(k)?, "{i}"); + } else { + ensure!(rolling_db.contains(k)?, "{i}"); + } + } + + drop(rolling_db); + + let rolling_db = RollingDB::load_or_create(db_root.path().into(), Default::default())?; + for (i, (k, _)) in pairs.iter().enumerate() { + if i < split_index { + ensure!(!rolling_db.contains(k)?); + } else { + ensure!(rolling_db.contains(k)?); + } + } + + Ok(()) + } +} diff --git a/node/db/tests/rolling_test.rs b/node/db/tests/rolling_test.rs deleted file mode 100644 index 94aff5074328..000000000000 --- a/node/db/tests/rolling_test.rs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2019-2023 ChainSafe Systems -// SPDX-License-Identifier: Apache-2.0, MIT - -#[cfg(test)] -#[cfg(any(feature = "paritydb", feature = "rocksdb"))] -mod tests { - use std::{thread::sleep, time::Duration}; - - use anyhow::*; - use cid::{multihash::MultihashDigest, Cid}; - use forest_db::rolling::RollingDB; - use forest_libp2p_bitswap::BitswapStoreRead; - use fvm_ipld_blockstore::Blockstore; - use rand::Rng; - use tempfile::TempDir; - - #[test] - fn rolling_db_behaviour_tests() -> Result<()> { - let db_root = TempDir::new()?; - println!("Creating rolling db under {}", db_root.path().display()); - let rolling_db = RollingDB::load_or_create(db_root.path().into(), Default::default())?; - println!("Generating random blocks"); - let pairs: Vec<_> = (0..1000) - .map(|_| { - let mut bytes = [0; 1024]; - rand::rngs::OsRng.fill(&mut bytes); - let cid = - Cid::new_v0(cid::multihash::Code::Sha2_256.digest(bytes.as_slice())).unwrap(); - (cid, bytes.to_vec()) - }) - .collect(); - - let split_index = 500; - - for (i, (k, block)) in pairs.iter().enumerate() { - if i == split_index { - sleep(Duration::from_millis(1)); - println!("Creating a new current db"); - rolling_db.next_current()?; - println!("Created a new current db"); - } - rolling_db.put_keyed(k, block)?; - } - - for (i, (k, block)) in pairs.iter().enumerate() { - ensure!(rolling_db.contains(k)?, "{i}"); - ensure!( - Blockstore::get(&rolling_db, k)?.unwrap().as_slice() == block, - "{i}" - ); - } - - rolling_db.next_current()?; - - for (i, (k, _)) in pairs.iter().enumerate() { - if i < split_index { - ensure!(!rolling_db.contains(k)?, "{i}"); - } else { - ensure!(rolling_db.contains(k)?, "{i}"); - } - } - - drop(rolling_db); - - let rolling_db = RollingDB::load_or_create(db_root.path().into(), Default::default())?; - for (i, (k, _)) in pairs.iter().enumerate() { - if i < split_index { - ensure!(!rolling_db.contains(k)?); - } else { - ensure!(rolling_db.contains(k)?); - } - } - - Ok(()) - } -} From f757e84c034d5b968a1ab2439d28264793d75a29 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 16 Mar 2023 21:33:30 +0800 Subject: [PATCH 35/43] fix cold start issue for last_reachable_bytes --- node/db/src/rolling/gc.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 07848b968288..9728f50395af 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -75,9 +75,6 @@ use tokio::sync::Mutex; use super::*; use crate::StoreExt; -/// 100GiB -const ESTIMATED_LAST_REACHABLE_BYTES_FOR_COLD_START: u64 = 100 * 1024_u64.pow(3); - pub struct DbGarbageCollector where F: Fn() -> Tipset + Send + Sync + 'static, @@ -103,7 +100,7 @@ where lock: Default::default(), gc_tx, gc_rx, - last_reachable_bytes: AtomicU64::new(ESTIMATED_LAST_REACHABLE_BYTES_FOR_COLD_START), + last_reachable_bytes: AtomicU64::new(0), } } @@ -132,15 +129,18 @@ where } } - if let (Ok(total_size), mut last_reachable_bytes) = ( + if let (Ok(total_size), Ok(current_size), last_reachable_bytes) = ( self.db.total_size_in_bytes(), + self.db.current_size_in_bytes(), self.last_reachable_bytes.load(atomic::Ordering::Relaxed), ) { - if last_reachable_bytes == 0 { - last_reachable_bytes = ESTIMATED_LAST_REACHABLE_BYTES_FOR_COLD_START; - } + let should_collect = if last_reachable_bytes > 0 { + total_size > 2 * last_reachable_bytes + } else { + total_size > 0 && current_size * 3 > total_size + }; - if total_size > 2 * last_reachable_bytes { + if should_collect { if let Err(err) = self.collect_once(tipset).await { warn!("Garbage collection failed: {err}"); } From c13ebb9065b61a99dd31f317bf39b262784a3b45 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 16 Mar 2023 23:32:11 +0800 Subject: [PATCH 36/43] fix reachable_bytes calculation --- node/db/src/rolling/gc.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 9728f50395af..86d68e1d83ef 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -199,10 +199,10 @@ where let block = db .get(&cid)? .ok_or_else(|| anyhow::anyhow!("Cid {cid} not found in blockstore"))?; + + let pair = (cid.to_bytes(), block.clone()); + reachable_bytes.fetch_add(pair.0.len() + pair.1.len(), atomic::Ordering::Relaxed); if !db.current().has(&cid)? { - let pair = (cid.to_bytes(), block.clone()); - reachable_bytes - .fetch_add(pair.0.len() + pair.1.len(), atomic::Ordering::Relaxed); tx.send_async(pair).await?; } From b7d4c8d23564c2d763839922593290c5139e719f Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 20 Mar 2023 09:03:46 +0800 Subject: [PATCH 37/43] simplify open_db --- node/db/src/lib.rs | 8 +------- node/db/src/parity_db.rs | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/node/db/src/lib.rs b/node/db/src/lib.rs index 6dc875979dcc..de2665fa8a1b 100644 --- a/node/db/src/lib.rs +++ b/node/db/src/lib.rs @@ -117,14 +117,8 @@ pub mod db_engine { chain_data_root.join(DIR_NAME) } - #[cfg(feature = "rocksdb")] - pub(crate) fn open_db(path: &Path, config: &DbConfig) -> anyhow::Result { - crate::rocks::RocksDb::open(path, config).map_err(Into::into) - } - - #[cfg(feature = "paritydb")] pub(crate) fn open_db(path: &Path, config: &DbConfig) -> anyhow::Result { - crate::parity_db::ParityDb::open(path.into(), config).map_err(Into::into) + Db::open(path, config).map_err(Into::into) } pub fn open_proxy_db(db_root: PathBuf, db_config: DbConfig) -> anyhow::Result { diff --git a/node/db/src/parity_db.rs b/node/db/src/parity_db.rs index d8a005c5e86d..b61ea8bd939a 100644 --- a/node/db/src/parity_db.rs +++ b/node/db/src/parity_db.rs @@ -50,8 +50,8 @@ impl ParityDb { }) } - pub fn open(path: PathBuf, config: &ParityDbConfig) -> anyhow::Result { - let opts = Self::to_options(path, config)?; + pub fn open(path: impl Into, config: &ParityDbConfig) -> anyhow::Result { + let opts = Self::to_options(path.into(), config)?; Ok(Self { db: Arc::new(Db::open_or_create(&opts)?), statistics_enabled: opts.stats, From ee3087282d1e9455d30dd53cf2e346ee452405e7 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 20 Mar 2023 16:41:49 +0800 Subject: [PATCH 38/43] change db_gc rpc access to write --- node/rpc-api/src/lib.rs | 3 +-- scripts/calibnet_health_check.sh | 9 +++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/node/rpc-api/src/lib.rs b/node/rpc-api/src/lib.rs index 1b6f3d29a776..4dfa29e6cb04 100644 --- a/node/rpc-api/src/lib.rs +++ b/node/rpc-api/src/lib.rs @@ -90,8 +90,7 @@ pub static ACCESS_MAP: Lazy> = Lazy::new(|| { access.insert(net_api::NET_DISCONNECT, Access::Write); // DB API - // FIXME: Use Read for testing, change this to Write later - access.insert(db_api::DB_GC, Access::Read); + access.insert(db_api::DB_GC, Access::Write); access }); diff --git a/scripts/calibnet_health_check.sh b/scripts/calibnet_health_check.sh index 031bb39b5e0a..b28a169b125f 100755 --- a/scripts/calibnet_health_check.sh +++ b/scripts/calibnet_health_check.sh @@ -46,6 +46,11 @@ $FOREST_CLI_PATH chain validate-tipset-checkpoints echo "Waiting for sync and check health" timeout 30m $FOREST_CLI_PATH --chain calibnet sync wait && $FOREST_CLI_PATH --chain calibnet db stats +# Admin token used when interacting with wallet +ADMIN_TOKEN=$(cat admin_token) +# Set environment variable +export FULLNODE_API_INFO="$ADMIN_TOKEN:/ip4/127.0.0.1/tcp/1234/http" + echo "Running database garbage collection" du -hS ~/.local/share/forest/calibnet $FOREST_CLI_PATH --chain calibnet db gc @@ -82,10 +87,6 @@ echo "Wallet tests" # Amount to send to FIL_AMT=500 -# Admin token used when interacting with wallet -ADMIN_TOKEN=$(cat admin_token) -# Set environment variable -export FULLNODE_API_INFO="$ADMIN_TOKEN:/ip4/127.0.0.1/tcp/1234/http" echo "Importing preloaded wallet key" $FOREST_CLI_PATH --chain calibnet wallet import preloaded_wallet.key From 6896b5322ce25afe7260822216fdc7f35d47c4be Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 21 Mar 2023 18:18:08 +0800 Subject: [PATCH 39/43] Apply suggestions from code review Co-authored-by: Hubert --- node/db/src/rolling/gc.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index ad16e3f6f465..89d95b56fae5 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -10,7 +10,7 @@ //! scenarios. //! //! ## GC algorithm -//! We choose `semi-space` GC algorithm for simplicity and sufficiency +//! We chose the `semi-space` GC algorithm for simplicity and sufficiency //! Besides `semi-space`, `mark-and-sweep` was also considered and evaluated. //! However, it's not feasible because of the limitations of the underlying DB //! we use, more specifically, limitations in iterating the DB and retrieving the original key. See @@ -31,7 +31,7 @@ //! //! ## Disk usage //! During `walk_snapshot`, data from the `old` DB is duplicated in the -//! `current` DB, which uses extra disk space of up-to 100% of the snapshot file +//! `current` DB, which uses extra disk space of up to 100% of the snapshot file //! size //! //! ## Memory usage @@ -42,7 +42,7 @@ //! 1. GC is triggered automatically when total DB size is greater than 2x of //! the last reachable data size //! 2. GC can be triggered manually by `forest-cli db gc` command -//! 3. There's global GC lock to ensure at most one GC job is running +//! 3. There's a global GC lock to ensure at most one GC job is running //! //! ## Performance //! GC performance is typically 1x-1.5x of `snapshot export`, depending on @@ -55,7 +55,7 @@ //! performance regression has been observed //! //! ### Write performance -//! DB write performance is typically on-par with `snapshot import`, note that +//! DB write performance is typically on par with `snapshot import`. Note that //! when the `current` DB space is very large, it tends to trigger DB re-index //! more frequently, each DB re-index could pause the GC process for a few //! minutes. The same behaviour is observed during snapshot import as well. From a66158d0cd695079ac4b117e7d82d9e32dcb43e6 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 21 Mar 2023 18:34:20 +0800 Subject: [PATCH 40/43] const DB_KEY_SIZE --- node/db/src/rolling/gc.rs | 5 ++--- utils/forest_utils/src/db/mod.rs | 9 +++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 89d95b56fae5..32c209ef6c65 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -68,7 +68,7 @@ use std::{ use chrono::Utc; use forest_blocks::Tipset; use forest_ipld::util::*; -use forest_utils::db::BlockstoreBufferedWriteExt; +use forest_utils::db::{BlockstoreBufferedWriteExt, DB_KEY_BYTES}; use fvm_ipld_blockstore::Blockstore; use human_repr::HumanCount; use tokio::sync::Mutex; @@ -201,8 +201,7 @@ where .ok_or_else(|| anyhow::anyhow!("Cid {cid} not found in blockstore"))?; let pair = (cid, block.clone()); - // Key size is 32 bytes in paritydb - reachable_bytes.fetch_add(32 + pair.1.len(), atomic::Ordering::Relaxed); + reachable_bytes.fetch_add(DB_KEY_BYTES + pair.1.len(), atomic::Ordering::Relaxed); if !db.current().has(&cid)? { tx.send_async(pair).await?; } diff --git a/utils/forest_utils/src/db/mod.rs b/utils/forest_utils/src/db/mod.rs index 6c1f8a169136..f65621f10e6a 100644 --- a/utils/forest_utils/src/db/mod.rs +++ b/utils/forest_utils/src/db/mod.rs @@ -15,6 +15,11 @@ use fvm_ipld_encoding::{from_slice, to_vec, DAG_CBOR}; use human_repr::HumanCount; use log::info; +/// DB key size in bytes for estimating reachable data size. Use parity-db value +/// for simplicity. The actual value for other underlying DB might be slightly +/// different but that is negligible for calculating the total reachable data +/// size +pub const DB_KEY_BYTES: usize = 32; /// Extension methods for inserting and retrieving IPLD data with CIDs pub trait BlockstoreExt: Blockstore { /// Get typed object from block store by CID @@ -88,8 +93,8 @@ pub trait BlockstoreBufferedWriteExt: Blockstore + Sized { let mut buffer = vec![]; while let Ok((key, value)) = rx.recv_async().await { // Key is stored in 32 bytes in paritydb - estimated_buffer_bytes += 32 + value.len(); - total_bytes += 32 + value.len(); + estimated_buffer_bytes += DB_KEY_BYTES + value.len(); + total_bytes += DB_KEY_BYTES + value.len(); total_entries += 1; buffer.push((key, value)); if estimated_buffer_bytes >= buffer_capacity_bytes { From 350fd3f666a219d316dfb551b19bc25a162de28a Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 21 Mar 2023 18:52:42 +0800 Subject: [PATCH 41/43] revert snapshot import tests to using single thread --- forest/daemon/src/daemon.rs | 10 +++++----- utils/genesis/src/lib.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/forest/daemon/src/daemon.rs b/forest/daemon/src/daemon.rs index cecbd410690b..23fdb515b84f 100644 --- a/forest/daemon/src/daemon.rs +++ b/forest/daemon/src/daemon.rs @@ -600,7 +600,7 @@ mod test { use super::*; - #[tokio::test(flavor = "multi_thread")] + #[tokio::test] async fn import_snapshot_from_file_valid() -> anyhow::Result<()> { anyhow::ensure!(import_snapshot_from_file("test_files/chain4.car") .await @@ -608,20 +608,20 @@ mod test { Ok(()) } - #[tokio::test(flavor = "multi_thread")] + #[tokio::test] async fn import_snapshot_from_file_invalid() -> anyhow::Result<()> { anyhow::ensure!(import_snapshot_from_file("Cargo.toml").await.is_err()); Ok(()) } - #[tokio::test(flavor = "multi_thread")] + #[tokio::test] async fn import_snapshot_from_file_not_found() -> anyhow::Result<()> { anyhow::ensure!(import_snapshot_from_file("dummy.car").await.is_err()); Ok(()) } #[cfg(feature = "slow_tests")] - #[tokio::test(flavor = "multi_thread")] + #[tokio::test] async fn import_snapshot_from_url_not_found() -> anyhow::Result<()> { anyhow::ensure!(import_snapshot_from_file("https://dummy.com/dummy.car") .await @@ -654,7 +654,7 @@ mod test { Ok(()) } - #[tokio::test(flavor = "multi_thread")] + #[tokio::test] async fn import_chain_from_file() -> anyhow::Result<()> { let db = MemoryDB::default(); let chain_config = Arc::new(ChainConfig::default()); diff --git a/utils/genesis/src/lib.rs b/utils/genesis/src/lib.rs index bb2f6cd5a994..c5664c12a610 100644 --- a/utils/genesis/src/lib.rs +++ b/utils/genesis/src/lib.rs @@ -198,7 +198,7 @@ where tokio::spawn(async move { store.buffered_write(rx, BUFFER_CAPCITY_BYTES).await }); let mut car_reader = CarReader::new(reader).await?; while let Some(block) = car_reader.next_block().await? { - tx.send((block.cid, block.data))?; + tx.send_async((block.cid, block.data)).await?; } drop(tx); write_task.await??; From ae67da5fda877898f4334b7952fb3b8aebb34101 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 21 Mar 2023 22:10:49 +0800 Subject: [PATCH 42/43] sample numbers --- node/db/src/rolling/gc.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index 32c209ef6c65..d6cbc79b3536 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -59,6 +59,13 @@ //! when the `current` DB space is very large, it tends to trigger DB re-index //! more frequently, each DB re-index could pause the GC process for a few //! minutes. The same behaviour is observed during snapshot import as well. +//! +//! ### Sample mainnet log +//! ``` +//! 2023-03-16T19:50:40.323860Z INFO forest_db::rolling::gc: Garbage collection started at epoch 2689660 +//! 2023-03-16T22:27:36.484245Z INFO forest_db::rolling::gc: Garbage collection finished at epoch 2689660, took 9416s, reachable data size: 135.71GB +//! 2023-03-16T22:27:38.793717Z INFO forest_db::rolling::impls: Deleted database under /root/.local/share/forest/mainnet/paritydb/14d0f80992374fb8b20e3b1bd70d5d7b, size: 139.01GB +//! ``` use std::{ sync::atomic::{self, AtomicU64, AtomicUsize}, From 2d077eb1585e3f4a634e13e4c13195ba487dac7c Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 22 Mar 2023 19:18:02 +0800 Subject: [PATCH 43/43] FOREST_GC_TRIGGER_FACTOR env var --- node/db/src/rolling/gc.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/node/db/src/rolling/gc.rs b/node/db/src/rolling/gc.rs index d6cbc79b3536..f7e6666e226b 100644 --- a/node/db/src/rolling/gc.rs +++ b/node/db/src/rolling/gc.rs @@ -142,7 +142,7 @@ where self.last_reachable_bytes.load(atomic::Ordering::Relaxed), ) { let should_collect = if last_reachable_bytes > 0 { - total_size > 2 * last_reachable_bytes + total_size > (gc_trigger_factor() * last_reachable_bytes as f64) as _ } else { total_size > 0 && current_size * 3 > total_size }; @@ -234,3 +234,13 @@ where Ok(()) } } + +fn gc_trigger_factor() -> f64 { + const DEFAULT_GC_TRIGGER_FACTOR: f64 = 2.0; + + if let Ok(factor) = std::env::var("FOREST_GC_TRIGGER_FACTOR") { + factor.parse().unwrap_or(DEFAULT_GC_TRIGGER_FACTOR) + } else { + DEFAULT_GC_TRIGGER_FACTOR + } +}