From 801a0ed225110c99fc8b05f79330a8e8dc9bb6bf Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Sun, 14 May 2023 22:35:09 +0200 Subject: [PATCH 01/22] Add tantivy index (cherry picked from commit 22227a2a1df59bd072f48654c9c62d9a2838c39c) --- Cargo.lock | 563 ++++++++++++++++++++ alexandrie.toml | 3 + crates/alexandrie/Cargo.toml | 4 + crates/alexandrie/src/api/crates/suggest.rs | 41 +- crates/alexandrie/src/config/mod.rs | 25 +- crates/alexandrie/src/error.rs | 18 + crates/alexandrie/src/frontend/indexer.rs | 118 ++++ crates/alexandrie/src/frontend/mod.rs | 2 + crates/alexandrie/src/frontend/search.rs | 97 ++-- crates/alexandrie/src/fts/document.rs | 160 ++++++ crates/alexandrie/src/fts/index.rs | 278 ++++++++++ crates/alexandrie/src/fts/mod.rs | 24 + crates/alexandrie/src/main.rs | 6 +- 13 files changed, 1263 insertions(+), 76 deletions(-) create mode 100644 crates/alexandrie/src/frontend/indexer.rs create mode 100644 crates/alexandrie/src/fts/document.rs create mode 100644 crates/alexandrie/src/fts/index.rs create mode 100644 crates/alexandrie/src/fts/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 73e31b35..738f2f70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,6 +62,17 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom 0.2.9", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.20" @@ -106,6 +117,8 @@ dependencies = [ "slog-scope", "slog-stdlog", "slog-term", + "tantivy", + "tantivy-analysis-contrib", "tar", "thiserror", "tide", @@ -547,6 +560,15 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitpacking" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c7d2ac73c167c06af4a5f37e6e59d84148d57ccbe4480b76f0273eefea82d7" +dependencies = [ + "crunchy", +] + [[package]] name = "blake3" version = "0.3.8" @@ -645,6 +667,12 @@ dependencies = [ "jobserver", ] +[[package]] +name = "census" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fafee10a5dd1cffcb5cc560e0d0df8803d7355a2b12272e3557dee57314cb6e" + [[package]] name = "cfg-if" version = "0.1.10" @@ -726,6 +754,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.2.0" @@ -814,6 +851,30 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.15" @@ -823,6 +884,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -926,6 +993,72 @@ dependencies = [ "syn 2.0.15", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", +] + [[package]] name = "diesel" version = "2.0.3" @@ -1018,6 +1151,18 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "downcast-rs" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + [[package]] name = "encoding_rs" version = "0.8.32" @@ -1054,6 +1199,37 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "fail" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c" +dependencies = [ + "log", + "once_cell", + "rand 0.8.5", +] + +[[package]] +name = "fastdivide" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25c7df09945d65ea8d70b3321547ed414bbc540aad5bac6883d021b970f35b04" + +[[package]] +name = "fastfield_codecs" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374a3a53c1bd5fb31b10084229290eafb0a05f260ec90f1f726afffda4877a8a" +dependencies = [ + "fastdivide", + "itertools", + "log", + "ownedbytes", + "tantivy-bitpacker", + "tantivy-common", +] + [[package]] name = "fastrand" version = "1.9.0" @@ -1115,6 +1291,22 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" + [[package]] name = "futf" version = "0.1.5" @@ -1229,6 +1421,19 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3e123d9ae7c02966b4d892e550bdc32164f05853cd40ab570650ad600596a8a" +dependencies = [ + "cc", + "libc", + "log", + "rustversion", + "windows", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1348,6 +1553,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] [[package]] name = "hermit-abi" @@ -1433,6 +1641,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + [[package]] name = "http" version = "0.2.9" @@ -1575,6 +1789,12 @@ dependencies = [ "cxx-build", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.3.0" @@ -1608,6 +1828,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -1678,6 +1901,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + [[package]] name = "libc" version = "0.2.141" @@ -1784,6 +2013,35 @@ dependencies = [ "value-bag", ] +[[package]] +name = "loom" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5" +dependencies = [ + "cfg-if 1.0.0", + "generator", + "pin-utils", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "lz4_flex" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8cbbb2831780bc3b9c15a41f5b49222ef756b6730a95f3decfdd15903eb5a3" + [[package]] name = "mac" version = "0.1.1" @@ -1810,6 +2068,15 @@ dependencies = [ "tendril", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + [[package]] name = "md-5" version = "0.9.1" @@ -1821,12 +2088,40 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "measure_time" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56220900f1a0923789ecd6bf25fbae8af3b2f1ff3e9e297fc9b6b8674dd4d852" +dependencies = [ + "instant", + "log", +] + [[package]] name = "memchr" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + [[package]] name = "migrations_internals" version = "2.0.0" @@ -1875,6 +2170,15 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "murmurhash32" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d736ff882f0e85fe9689fb23db229616c4c00aee2b3ac282f666d8f20eb25d4a" +dependencies = [ + "byteorder", +] + [[package]] name = "mysqlclient-sys" version = "0.2.5" @@ -1909,6 +2213,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.3" @@ -1994,6 +2308,15 @@ version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +[[package]] +name = "oneshot" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc22d22931513428ea6cc089e942d38600e3d00976eef8c86de6b8a3aadec6eb" +dependencies = [ + "loom", +] + [[package]] name = "onig" version = "6.4.0" @@ -2066,6 +2389,21 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "ownedbytes" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e957eaa64a299f39755416e5b3128c505e9d63a91d0453771ad2ccd3907f8db" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "parking" version = "2.1.0" @@ -2440,6 +2778,28 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2480,6 +2840,15 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.6.29" @@ -2631,6 +3000,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.2.3" @@ -2729,6 +3114,12 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.1.0" @@ -3033,6 +3424,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "standback" version = "0.2.17" @@ -3193,6 +3590,108 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" +[[package]] +name = "tantivy" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb26a6b22c84d8be41d99a14016d6f04d30d8d31a2ea411a8ab553af5cc490d" +dependencies = [ + "aho-corasick", + "arc-swap", + "async-trait", + "base64 0.13.1", + "bitpacking", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "downcast-rs", + "fail", + "fastdivide", + "fastfield_codecs", + "fs2", + "htmlescape", + "itertools", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "murmurhash32", + "num_cpus", + "once_cell", + "oneshot", + "ownedbytes", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "smallvec", + "stable_deref_trait", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tempfile", + "thiserror", + "time 0.3.20", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-analysis-contrib" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4240ab300b15d81dca023ad3e27b7b6b45522340594634f0c7e1dfbaec82c53b" +dependencies = [ + "derive_builder", + "either", + "fst", + "tantivy", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e71a0c95b82d4292b097a09b989a6380d28c3a86800c841a2d03bae1fc8b9fa6" + +[[package]] +name = "tantivy-common" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14fef4182bb60df9a4b92cd8ecab39ba2e50a05542934af17eef1f49660705cb" +dependencies = [ + "byteorder", + "ownedbytes", +] + +[[package]] +name = "tantivy-fst" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc3c506b1a8443a3a65352df6382a1fb6a7afe1a02e871cee0d25e2c3d5f3944" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343e3ada4c1c480953f6960f8a21ce9c76611480ffdd4f4e230fdddce0fc5331" +dependencies = [ + "combine", + "once_cell", + "regex", +] + [[package]] name = "tar" version = "0.4.38" @@ -3514,9 +4013,21 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if 1.0.0", "pin-project-lite 0.2.9", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + [[package]] name = "tracing-core" version = "0.1.30" @@ -3524,6 +4035,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -3614,12 +4155,34 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + [[package]] name = "utf8parse" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" +dependencies = [ + "getrandom 0.2.9", + "serde", +] + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "value-bag" version = "1.0.0-alpha.9" diff --git a/alexandrie.toml b/alexandrie.toml index b28f4caf..b323db29 100644 --- a/alexandrie.toml +++ b/alexandrie.toml @@ -76,3 +76,6 @@ path = "syntect/dumps/syntaxes.dump" type = "dump" path = "syntect/dumps/themes.dump" theme_name = "frontier-contrast" + +[search] +directory = "/tmp/tantivy" \ No newline at end of file diff --git a/crates/alexandrie/Cargo.toml b/crates/alexandrie/Cargo.toml index 5fa1249c..9c500f5d 100644 --- a/crates/alexandrie/Cargo.toml +++ b/crates/alexandrie/Cargo.toml @@ -50,6 +50,10 @@ percent-encoding = "2.2.0" diesel = { version = "2.0.3", features = ["r2d2", "chrono"] } diesel_migrations = "2.0.0" +# Text indexation and search +tantivy = "0.19" +tantivy-analysis-contrib = { version = "0.8", default-features = false, features = ["commons"] } + # async primitives async-std = { version = "1.12.0", features = ["attributes", "tokio1"] } diff --git a/crates/alexandrie/src/api/crates/suggest.rs b/crates/alexandrie/src/api/crates/suggest.rs index 9f6e4712..49ea7635 100644 --- a/crates/alexandrie/src/api/crates/suggest.rs +++ b/crates/alexandrie/src/api/crates/suggest.rs @@ -1,17 +1,15 @@ use std::num::NonZeroU32; -use diesel::prelude::*; +use log::info; use semver::Version; use serde::{Deserialize, Serialize}; use tide::Request; use alexandrie_index::Indexer; -use crate::db::models::Crate; -use crate::db::schema::*; use crate::error::{AlexError, Error}; -use crate::utils; use crate::State; +use crate::utils; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct APIResponse { @@ -37,37 +35,26 @@ pub(crate) async fn get(req: Request) -> tide::Result { .map_err(|_| AlexError::MissingQueryParams { missing_params: &["q"], })?; - let state = req.state().clone(); - let db = &state.db; - - //? Fetch the latest index changes. - // state.index.refresh()?; let name = utils::canonical_name(params.q); + let limit = params.limit.map_or(10, |limit| limit.get() as usize); - //? Build the search pattern. - let name_pattern = format!("%{0}%", name.replace('\\', "\\\\").replace('%', "\\%")); - - //? Limit the result count depending on parameters. - let limit = params.limit.map_or(10, |limit| i64::from(limit.get())); - - //? Fetch results. - let results = db - .run(move |conn| { - crates::table - .filter(crates::canon_name.like(name_pattern.as_str())) - .limit(limit) - .load::(conn) - }) - .await?; + info!("Suggester : {} & {}", name, limit); + let state = req.state().clone(); + let index = &state.index; - //? Fetch version information about these crates. + let results = { + let tantivy = (&state.search) + .read() + .map_err(|error| Error::PoisonedError(error.to_string()))?; + tantivy.suggest(name, limit)? + }; let suggestions = results .into_iter() .map(|krate| { - let latest = state.index.latest_record(krate.name.as_str())?; + let latest = index.latest_record(krate.to_lowercase().as_str())?; Ok(Suggestion { - name: krate.name, + name: krate, vers: latest.vers, }) }) diff --git a/crates/alexandrie/src/config/mod.rs b/crates/alexandrie/src/config/mod.rs index 642ef246..6dda1f93 100644 --- a/crates/alexandrie/src/config/mod.rs +++ b/crates/alexandrie/src/config/mod.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use std::sync::RwLock; /// Database configuration (`[database]` section). pub mod database; @@ -19,6 +20,8 @@ use crate::db::Database; #[cfg(feature = "frontend")] pub use crate::config::frontend::*; +use crate::error::Error; +use crate::fts::Tantivy; use self::database::DatabaseConfig; @@ -32,6 +35,13 @@ pub struct GeneralConfig { max_crate_size: Option, } +/// Configuration for search index. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SearchConfig { + /// Path to the directory where Tantivy will store its index. + pub directory: String, +} + /// The application configuration struct. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Config { @@ -45,6 +55,8 @@ pub struct Config { pub database: DatabaseConfig, /// The syntax-highlighting configuration. pub syntect: SyntectConfig, + /// Search config + pub search: SearchConfig, /// The frontend configuration. #[cfg(feature = "frontend")] pub frontend: FrontendConfig, @@ -68,6 +80,8 @@ pub struct State { pub db: Database, /// The syntect configuration. pub syntect: SyntectState, + /// Search config + pub search: RwLock, /// The frontend configured state. #[cfg(feature = "frontend")] pub frontend: FrontendState, @@ -81,17 +95,20 @@ impl From for GeneralState { } } -impl From for State { - fn from(config: Config) -> State { - State { +impl TryFrom for State { + type Error = Error; + + fn try_from(config: Config) -> Result { + Ok(State { general: config.general.into(), index: config.index.into(), storage: config.storage.into(), db: Database::new(&config.database), syntect: config.syntect.into(), + search: RwLock::new(config.search.try_into()?), #[cfg(feature = "frontend")] frontend: config.frontend.into(), - } + }) } } diff --git a/crates/alexandrie/src/error.rs b/crates/alexandrie/src/error.rs index f4aa52f4..44cfc3df 100644 --- a/crates/alexandrie/src/error.rs +++ b/crates/alexandrie/src/error.rs @@ -6,6 +6,8 @@ use hex::FromHexError as HexError; use io::Error as IOError; use json::Error as JSONError; use semver::{Error as SemverError, Version}; +use tantivy::directory::error::OpenDirectoryError; +use tantivy::TantivyError; use thiserror::Error; use toml::de::Error as TOMLError; @@ -49,6 +51,22 @@ pub enum Error { /// Storage-specific errors. #[error("{0}")] StorageError(#[from] StorageError), + /// Tantivy's errors. + #[error("{0}")] + TantivyError(#[from] TantivyError), + /// Open directory error + #[error("{0}")] + OpenDirectoryError(#[from] OpenDirectoryError), + /// Empty stop words. + #[error("Empty stop word filter")] + EmptyStopWord, + /// Tantivy's index is poisoned + #[error("Tantivy's index is poisoned: {0}")] + PoisonedError(String), + /// Missing id field or on of nae's field in index schema + /// Should never happen... + #[error("Missing {0} in Tantivy's schema")] + MissingField(&'static str), } /// The Error type for Alexandrie's own errors. diff --git a/crates/alexandrie/src/frontend/indexer.rs b/crates/alexandrie/src/frontend/indexer.rs new file mode 100644 index 00000000..cc0f86fe --- /dev/null +++ b/crates/alexandrie/src/frontend/indexer.rs @@ -0,0 +1,118 @@ +use diesel::prelude::*; +use log::{info, warn}; +use tide::{Request, Response}; + +use crate::db::models::Crate; +use crate::db::schema::*; +use crate::error::Error; +use crate::fts::TantivyDocument; +use crate::State; + +const NUMBER_RESULT_PER_PAGE: i64 = 1000; + + +pub(crate) async fn get(req: Request) -> tide::Result { + let state = req.state().clone(); + let repo = &state.db; + + let transaction: Result<(), Error> = repo + .transaction(move |conn| { + let state = req.state(); + let mut tantivy = state.search.write().unwrap(); + tantivy.delete_all_documents()?; + tantivy.commit()?; + let mut start: i64 = 0; + let mut count_crate = 0; + + loop { + let krates = crates::table + .order_by(crates::id.asc()) + .limit(NUMBER_RESULT_PER_PAGE) + .offset(start) + .load::(conn)?; + if krates.is_empty() { + info!("End indexing"); + break; + } + + let ids = krates + .clone() + .into_iter() + .map(|c| c.id) + .collect::>(); + start = start + krates.len() as i64; + + info!("Crates {:?}", ids); + + let keywords = keywords::table + .inner_join(crate_keywords::table) + .select((crate_keywords::crate_id, keywords::name)) + .filter(crate_keywords::crate_id.eq_any(&ids)) + .order_by(crate_keywords::crate_id.asc()) + .load::<(i64, String)>(conn)?; + + let categories = categories::table + .inner_join(crate_categories::table) + .select((crate_categories::crate_id, categories::name)) + .filter(crate_categories::crate_id.eq_any(&ids)) + .order_by(crate_categories::crate_id.asc()) + .load::<(i64, String)>(conn)?; + + let mut keywords_iterator = keywords.into_iter(); + let mut categories_iterator = categories.into_iter(); + + let mut current_keyword: Option<(i64, String)> = keywords_iterator.next(); + let mut current_category: Option<(i64, String)> = categories_iterator.next(); + + for krate in krates.into_iter() { + info!("crate {:?}", krate); + // Create a document with database ID and crate name + let mut doc: TantivyDocument = + TantivyDocument::new(krate.id, krate.name.clone(), &tantivy.schema); + + // If there is some description, then set it + if let Some(description) = krate.description.as_ref() { + doc.set_description(description.clone()); + } + + // Add all keywords + while current_keyword.is_some() + && current_keyword.as_ref().unwrap().0 == krate.id + { + doc.add_keyword(current_keyword.unwrap().1); + current_keyword = keywords_iterator.next(); + } + + // Add all cateogries + while current_category.is_some() + && current_category.as_ref().unwrap().0 == krate.id + { + doc.add_category(current_category.unwrap().1); + current_category = categories_iterator.next(); + } + + // TODO get README + + match doc.try_into() { + Ok(document) => { + tantivy.create_or_update(krate.id, document)?; + } + Err(error) => { + warn!("Can't convert crate '{}' ({}) into Tantivy document : {error}", krate.id, krate.name.clone()); + } + } + count_crate += 1; + } + + (*tantivy).commit()?; + info!("{} crates indexed", count_crate); + } + + Ok(()) + }) + .await; + + transaction?; + let res = Response::new(200); + Ok(res) +} \ No newline at end of file diff --git a/crates/alexandrie/src/frontend/mod.rs b/crates/alexandrie/src/frontend/mod.rs index 78f4f0c2..eb3a6d9b 100644 --- a/crates/alexandrie/src/frontend/mod.rs +++ b/crates/alexandrie/src/frontend/mod.rs @@ -14,3 +14,5 @@ pub mod me; pub mod most_downloaded; /// Search pages (eg. "/search?q=\"). pub mod search; +/// Index all crates into search index. +pub mod indexer; diff --git a/crates/alexandrie/src/frontend/search.rs b/crates/alexandrie/src/frontend/search.rs index f4f8004c..aa7a82dc 100644 --- a/crates/alexandrie/src/frontend/search.rs +++ b/crates/alexandrie/src/frontend/search.rs @@ -1,6 +1,5 @@ use std::num::NonZeroU32; -use diesel::dsl as sql; use diesel::prelude::*; use json::json; use serde::{Deserialize, Serialize}; @@ -8,14 +7,16 @@ use tide::Request; use alexandrie_index::Indexer; +use crate::db::DATETIME_FORMAT; use crate::db::models::Crate; use crate::db::schema::*; -use crate::db::DATETIME_FORMAT; use crate::error::Error; use crate::frontend::helpers; +use crate::State; use crate::utils; use crate::utils::auth::AuthExt; -use crate::State; + +const RESULT_PER_PAGE: i64 = 15; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct SearchParams { @@ -23,71 +24,79 @@ struct SearchParams { pub page: Option, } +/// Route to search through crates (used by `cargo search`) using tantivy index pub(crate) async fn get(req: Request) -> tide::Result { - let params = req.query::()?; - let searched_text = utils::canonical_name(params.q.as_str()); - let q = format!( - "%{0}%", - searched_text.replace('\\', "\\\\").replace('%', "\\%") - ); - + let params = req.query::().unwrap(); + let searched_text = params.q.clone(); let page_number = params.page.map_or_else(|| 1, |page| page.get()); + let offset = (page_number - 1) * RESULT_PER_PAGE as u32; + let user = req.get_author(); if req.state().is_login_required() && user.is_none() { return Ok(utils::response::redirect("/account/login")); } let state = req.state().clone(); - let db = &state.db; - - let transaction = db.transaction(move |conn| { + let repo = &state.db; + + let (count, results) = { + let tantivy = (&state.search) + .read() + .map_err(|error| Error::PoisonedError(error.to_string()))?; + tantivy.search( + searched_text.clone(), + offset as usize, + RESULT_PER_PAGE as usize, + )? + }; + + let page_count = (count / RESULT_PER_PAGE as usize + + if count > 0 && count % RESULT_PER_PAGE as usize == 0 { + 0 + } else { + 1 + }) as u32; + + let transaction = repo.transaction(move |conn| { let state = req.state(); - //? Get the total count of search results. - let total_results = crates::table - .select(sql::count(crates::id)) - .filter(crates::canon_name.like(q.as_str())) - .first::(conn)?; - - //? Get the search results for the given page number. - let results: Vec = crates::table - .filter(crates::canon_name.like(q.as_str())) - .limit(15) - .offset(15 * i64::from(page_number - 1)) - .load(conn)?; - - let results = results + let results: Vec<(Crate, Vec)> = results .into_iter() - .map(|result| { + .map(|v| { + let krate = crates::table + .filter(crates::id.eq(v)) + .first::(conn) + .unwrap(); let keywords = crate_keywords::table .inner_join(keywords::table) .select(keywords::name) - .filter(crate_keywords::crate_id.eq(result.id)) - .load::(conn)?; - Ok((result, keywords)) + .filter(crate_keywords::crate_id.eq(krate.id)) + .load::(conn) + .unwrap(); + (krate, keywords) }) - .collect::)>, Error>>()?; - - //? Make page number starts counting from 1 (instead of 0). - let page_count = (total_results / 15 - + if total_results > 0 && total_results % 15 == 0 { - 0 - } else { - 1 - }) as u32; + .collect(); let encoded_q = percent_encoding::percent_encode( params.q.as_bytes(), percent_encoding::NON_ALPHANUMERIC, ); let next_page = if page_number < page_count { - Some(format!("/search?q={0}&page={1}", encoded_q, page_number + 1)) + Some(format!( + "/search?q={0}&page={1}", + encoded_q, + page_number + 1 + )) } else { None }; let prev_page = if page_number > 1 { - Some(format!("/search?q={0}&page={1}", encoded_q, page_number - 1)) + Some(format!( + "/search?q={0}&page={1}", + encoded_q, + page_number - 1 + )) } else { None }; @@ -96,8 +105,8 @@ pub(crate) async fn get(req: Request) -> tide::Result { let context = json!({ "user": user, "instance": &state.frontend.config, - "searched_text": params.q, - "total_results": total_results, + "searched_text": searched_text, + "total_results": count, "pagination": { "current": page_number, "total_count": page_count, diff --git a/crates/alexandrie/src/fts/document.rs b/crates/alexandrie/src/fts/document.rs new file mode 100644 index 00000000..aff03192 --- /dev/null +++ b/crates/alexandrie/src/fts/document.rs @@ -0,0 +1,160 @@ +use std::fmt::Formatter; +use tantivy::Document; +use tantivy::schema::Schema; +use crate::error::Error; + +/// Represent a crate. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TantivyDocument<'a> { + schema: &'a Schema, + id: i64, + name: String, + description: Option, + readme: Option, + keywords: Vec, + categories: Vec, +} + +impl std::fmt::Display for TantivyDocument<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "id: {}, name: {}", self.id, self.name)?; + if let Some(description) = &self.description { + write!(f, ", '{}'", description)?; + } + // Don't write the README, it might be big, and + // log will be unreadable. Just say that it has + // a README. + if let Some(_) = &self.readme { + write!(f, ", crate has README", )?; + } + + if self.keywords.is_empty() { + write!(f, ", no keyword")?; + } else { + let keywords = self.keywords.join(", "); + write!(f, ", keywords : {keywords}")?; + } + + if self.categories.is_empty() { + write!(f, ", no categories")?; + } else { + let categories = self.categories.join(", "); + write!(f, ", categories : {categories}")?; + } + + Ok(()) + } +} + +impl TryFrom> for Document { + type Error = Error; + + fn try_from(value: TantivyDocument) -> Result { + // Can't implement From because I need to use a tuple to hold schema and both + // tuple and Document are in another crate :-( + let mut document = Document::new(); + + let id_field = value.schema.get_field(super::ID_FIELD_NAME); + let name_field = value.schema.get_field(super::NAME_FIELD_NAME); + let name_full_field = value.schema.get_field(super::NAME_FIELD_NAME_FULL); + let name_prefix_field = value.schema.get_field(super::NAME_FIELD_PREFIX_NAME); + let description_field = value.schema.get_field(super::DESCRIPTION_FIELD_NAME); + let readme_field = value.schema.get_field(super::README_FIELD_NAME); + let category_field = value.schema.get_field(super::CATEGORY_FIELD_NAME); + let keyword_field = value.schema.get_field(super::KEYWORD_FIELD_NAME); + + // None of the fields should be `None`. + // But we check that anyway. + if id_field.is_none() { + return Err(Error::MissingField(super::ID_FIELD_NAME)); + } + if name_field.is_none() { + return Err(Error::MissingField(super::NAME_FIELD_NAME)); + } + if name_full_field.is_none() { + return Err(Error::MissingField(super::NAME_FIELD_NAME_FULL)); + } + if name_prefix_field.is_none() { + return Err(Error::MissingField(super::NAME_FIELD_PREFIX_NAME)); + } + + document.add_i64(id_field.unwrap(), value.id); + document.add_text(name_field.unwrap(), &value.name); + document.add_text(name_full_field.unwrap(), &value.name); + document.add_text(name_prefix_field.unwrap(), &value.name); + + // For the following fields we will not fail if they are not in schema + // but TODO add warn + if let Some(description) = &value.description { + match description_field { + Some(field) => document.add_text(field, description), + None => (), + } + } + + if let Some(readme) = &value.readme { + match readme_field { + Some(field) => document.add_text(field, readme), + None => (), + } + } + + if !value.keywords.is_empty() { + match keyword_field { + Some(field) => value + .keywords + .clone() + .into_iter() + .for_each(|v| document.add_text(field, v)), + None => (), + } + } + + if !value.categories.is_empty() { + match category_field { + Some(field) => value + .categories + .clone() + .into_iter() + .for_each(|v| document.add_text(field, v)), + None => (), + } + } + + Ok(document) + } +} + +impl<'a> TantivyDocument<'a> { + pub(crate) fn new(id: i64, name: String, schema: &'a Schema) -> Self { + Self { + schema, + id, + name, + description: None, + readme: None, + keywords: Vec::with_capacity(5), + categories: Vec::with_capacity(5), + } + } + + /// Set crate's description + pub(crate) fn set_description(&mut self, description: String) { + self.description = Some(description); + } + + /// Set crate's README + pub(crate) fn set_readme(&mut self, readme: String) { + self.readme = Some(readme); + } + + /// Add new crate's keyword + pub(crate) fn add_keyword(&mut self, keyword: String) { + self.keywords.push(keyword); + } + + /// Add new crate's category + pub(crate) fn add_category(&mut self, category: String) { + self.categories.push(category); + } +} \ No newline at end of file diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs new file mode 100644 index 00000000..3828364b --- /dev/null +++ b/crates/alexandrie/src/fts/index.rs @@ -0,0 +1,278 @@ +use std::convert::TryFrom; +use std::num::NonZeroUsize; + +use log::{error, info, warn}; +use tantivy::{Document, Index as TantivyIndex, IndexWriter, Opstamp, TantivyError, Term}; +use tantivy::collector::{Count, TopDocs}; +use tantivy::directory::MmapDirectory; +use tantivy::query::QueryParser; +use tantivy::schema::{NumericOptions, Schema, TextFieldIndexing, TextOptions}; +use tantivy::tokenizer::{Language, LowerCaser, RawTokenizer, SimpleTokenizer, StopWordFilter, TextAnalyzer, TokenizerManager}; +use tantivy_analysis_contrib::commons::EdgeNgramTokenFilter; + +use crate::config::SearchConfig; +use crate::error::Error; + +/// Helper for using Tantivy +pub struct Tantivy { + index: TantivyIndex, + /// There can only be one index writer at a time (see https://tantivy-search.github.io/examples/basic_search.html) + /// so we keep only one here. It has its own pool. + index_writer: IndexWriter, + pub schema: Schema, + /// Search tokenizer manager + search_tokenizer_manager: TokenizerManager, +} + +impl TryFrom for Tantivy { + type Error = crate::error::Error; + + fn try_from(search: SearchConfig) -> Result { + let analyzer_name = "alexandrie"; + // Not tokenized + let analyzer_name_full = "alexandrie_full"; + // Prefix + let analyzer_prefix_name = "alexandrie_prefix"; + + // Create index directory + let path = search.directory.as_str(); + std::fs::create_dir_all(path)?; + let directory = MmapDirectory::open(path)?; + + // Index options for all fields (perhaps keywords and category could have another analysis & options) + let options = TextOptions::default() + .set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer(analyzer_name) + .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions), + ) + .set_stored(); + + let options_full = TextOptions::default() + .set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer(analyzer_name_full) + .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions), + ) + .set_stored(); + + let options_prefixes = TextOptions::default() + .set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer(analyzer_prefix_name) + .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions), + ) + .set_stored(); + + let id_options = NumericOptions::default().set_stored().set_indexed(); + + // Schema of a document, we index and store (though storing isn't really necessary): + // * name : crate's name + // * description: crate's description + // * categories: crate's categories + // * keywords: crate's keywords + let mut schema_builder = Schema::builder(); + // Easier with i64 than u64 because IDs are i64 i db module. + schema_builder.add_i64_field(super::ID_FIELD_NAME, id_options); + + // For these fields, we could avoid storing original data if we get things in the database right + // after searching indices + schema_builder.add_text_field(super::NAME_FIELD_NAME, options.clone()); + schema_builder.add_text_field(super::NAME_FIELD_NAME_FULL, options_full.clone()); + schema_builder.add_text_field(super::NAME_FIELD_PREFIX_NAME, options_prefixes.clone()); + schema_builder.add_text_field(super::DESCRIPTION_FIELD_NAME, options.clone()); + schema_builder.add_text_field(super::README_FIELD_NAME, options.clone()); + schema_builder.add_text_field(super::CATEGORY_FIELD_NAME, options_full); + schema_builder.add_text_field(super::KEYWORD_FIELD_NAME, options); + let schema = schema_builder.build(); + + // Analysis a tokenizer that tokenizes on non-alphanumeric characters + // A filter that removes common english words (the, a, ...etc) + // A filter that lowercase words + let stop_words = StopWordFilter::new(Language::English).ok_or(Self::Error::EmptyStopWord)?; + let analyzer = TextAnalyzer::from(SimpleTokenizer) + .filter(stop_words) + .filter(LowerCaser); + + let analyzer_full = TextAnalyzer::from(RawTokenizer).filter(LowerCaser); + + let analyzer_prefix = TextAnalyzer::from(SimpleTokenizer) + .filter(LowerCaser) + .filter(EdgeNgramTokenFilter::new(NonZeroUsize::new(1).unwrap(), None, false).unwrap()); + + let index = TantivyIndex::open_or_create(directory, schema.clone())?; + // Register analyzer + index.tokenizers().register(analyzer_name, analyzer.clone()); + index + .tokenizers() + .register(analyzer_name_full, analyzer_full.clone()); + index + .tokenizers() + .register(analyzer_prefix_name, analyzer_prefix); + + // Create an analyzer manager for search: on name prefix we do not want to apply + // the edge ngram filter: more efficient & less noise + // We need to have the other tokenizer registered so that search work on any field properly + let search_tokenizer_manager = TokenizerManager::new(); + search_tokenizer_manager.register(analyzer_name, analyzer); + search_tokenizer_manager.register(analyzer_name_full, analyzer_full); + search_tokenizer_manager.register( + analyzer_prefix_name, + TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser), + ); + + // Get an index writer with 50MB of heap + let index_writer = index.writer(50_000_000)?; + + Ok(Self { + index, + index_writer, + schema, + search_tokenizer_manager, + }) + } +} + +impl Tantivy { + /// Method that create or update a document in Tantivy index. As there is no update, we need + /// to first delete the document then create a new document. + pub fn create_or_update(&self, id: i64, document: Document) -> Result<(), Error> { + if let Some(field) = self.schema.get_field(super::ID_FIELD_NAME) { + let term = Term::from_field_i64(field, id); + self.index_writer.delete_term(term); + self.index_writer.add_document(document)?; + } else { + error!("There is no field {} in schema", super::ID_FIELD_NAME); + } + + Ok(()) + } + + pub fn delete_all_documents(&mut self) -> Result { + self.index_writer.delete_all_documents() + } + + /// Commit all pending changes inside the index. + pub fn commit(&mut self) -> Result { + self.index_writer.commit() + } + + /// Search document by default through all crate's name index. This allows having search + /// as you type (using prefixes) while increasing relevance when there's a matching word + /// or if the whole text matches a crate's name (using the other crate's name indices). + pub fn suggest(&self, query: String, limit: usize) -> Result, TantivyError> { + let searcher = self.index.reader()?.searcher(); + + let name = self.schema.get_field(super::NAME_FIELD_NAME).unwrap(); + let name_full = self.schema.get_field(super::NAME_FIELD_NAME_FULL).unwrap(); + let name_prefix = self.schema.get_field(super::NAME_FIELD_PREFIX_NAME).unwrap(); + + let mut query_parser = QueryParser::new( + self.schema.clone(), + vec![name_full, name_prefix], + self.search_tokenizer_manager.clone(), + ); + + query_parser.set_field_boost(name_full, 10.0); + query_parser.set_field_boost(name, 5.0); + query_parser.set_field_boost(name_prefix, 1.0); + + let query = query_parser.parse_query(&query)?; + + info!("Query : {:?}", query); + + let results = searcher.search(&query, &TopDocs::with_limit(limit))?; + + info!("Result : {:?}", results); + + let results = results + .into_iter() + .map(|(score, doc_address)| { + let retrieve_doc = searcher.doc(doc_address).unwrap(); + + let x = retrieve_doc.get_all(name).next(); + if let Some(n) = x { + info!("Score : {} / Crate : {:?}", score, n); + } + x.cloned() + }) + .map(|v| v.map(|i| i.as_text().map(|t| t.to_owned())).flatten()) + .filter(|v| v.is_some()) + .map(|v| v.unwrap()) + .collect(); + + Ok(results) + } + + /// Search documents. Return document count & database IDs. + pub fn search( + &self, + query: String, + offset: usize, + limit: usize, + ) -> Result<(usize, Vec), TantivyError> { + let searcher = self.index.reader()?.searcher(); + + let id = self.schema.get_field(super::ID_FIELD_NAME).unwrap(); + let name = self.schema.get_field(super::NAME_FIELD_NAME).unwrap(); + let name_full = self.schema.get_field(super::NAME_FIELD_NAME_FULL).unwrap(); + let description = self.schema.get_field(super::DESCRIPTION_FIELD_NAME).unwrap(); + let readme = self.schema.get_field(super::README_FIELD_NAME).unwrap(); + let categories = self.schema.get_field(super::CATEGORY_FIELD_NAME).unwrap(); + let keywords = self.schema.get_field(super::KEYWORD_FIELD_NAME).unwrap(); + + let mut query_parser = QueryParser::for_index( + &self.index, + vec![name, name_full, description, readme, categories, keywords], + ); + + // Exact matches (on name_full) have a big boost + query_parser.set_field_boost(name_full, 10.0); + query_parser.set_field_boost(name, 5.0); + // Categories shouldn't be free (there is a list) so a nice boost + query_parser.set_field_boost(categories, 1.0); + // Keywords are free + query_parser.set_field_boost(keywords, 0.5); + // description & readme are full text they got a lower boost (if there is a match, that might not be relevant) + query_parser.set_field_boost(description, 0.2); + query_parser.set_field_boost(readme, 0.2); + + let query = query_parser.parse_query(&query)?; + + info!("Query offset={} query limit={}", offset, limit); + + let (count, results) = searcher.search( + &query, + &(Count, TopDocs::with_limit(limit).and_offset(offset)), + )?; + + let results = results + .into_iter() + .map(|(score, doc_address)| { + let retrieve_doc = match searcher.doc(doc_address) { + Ok(retrieve_doc) => retrieve_doc, + Err(error) => { + warn!("Could not find document {doc_address:?} : {error}"); + return None; + } + }; + + if let Some(name) = retrieve_doc.get_all(name).next() { + info!("Score : {} / Crate : {:?}", score, name); + } + + let mut field = retrieve_doc.get_all(id); + if let Some(x) = field.next() { + x.as_i64().clone() + } else { + warn!("Could not find field id"); + None + } + }) + .filter(|v| v.is_some()) + .map(|v| v.unwrap()) + .collect(); + + Ok((count, results)) + } +} \ No newline at end of file diff --git a/crates/alexandrie/src/fts/mod.rs b/crates/alexandrie/src/fts/mod.rs new file mode 100644 index 00000000..7cb56a22 --- /dev/null +++ b/crates/alexandrie/src/fts/mod.rs @@ -0,0 +1,24 @@ +//! Full-text search module +mod index; +mod document; + +pub(crate) use index::Tantivy; +pub(crate) use document::TantivyDocument; + +/// Database ID. +const ID_FIELD_NAME: &str = "id"; +/// Tokenized version of crate's name. +const NAME_FIELD_NAME: &str = "name"; +/// Another index of crate's name, this one +/// isn't tokenized. So it's an exact match +/// but case-insensitive. It's here to improve +/// results relevance. +const NAME_FIELD_NAME_FULL: &str = "name.full"; +/// A third index for crate's name. This one is for +/// suggestion in the search bar. It's tokenized and +/// contains word's prefixes to do "search as you type". +const NAME_FIELD_PREFIX_NAME: &str = "name.prefix"; +const DESCRIPTION_FIELD_NAME: &str = "description"; +const README_FIELD_NAME: &str = "readme"; +const CATEGORY_FIELD_NAME: &str = "category"; +const KEYWORD_FIELD_NAME: &str = "keyword"; diff --git a/crates/alexandrie/src/main.rs b/crates/alexandrie/src/main.rs index 71340839..6d97dd7c 100644 --- a/crates/alexandrie/src/main.rs +++ b/crates/alexandrie/src/main.rs @@ -57,6 +57,7 @@ pub mod utils; /// Frontend endpoints definitions. #[cfg(feature = "frontend")] pub mod frontend; +mod fts; use crate::config::Config; use crate::error::Error; @@ -149,6 +150,9 @@ fn frontend_routes(state: State, frontend_config: FrontendConfig) -> io::Result< log::info!("mounting '/account/manage/tokens/:token-id/revoke'"); app.at("/account/manage/tokens/:token-id/revoke") .get(frontend::account::manage::tokens::revoke::get); + log::info!("mounting '/indexer'"); + app.at("/indexer") + .get(frontend::indexer::get); log::info!("mounting '/assets/*path'"); app.at("/assets").serve_dir(frontend_config.assets.path)?; @@ -241,7 +245,7 @@ async fn run() -> Result<(), Error> { #[cfg(feature = "frontend")] let frontend_config = config.frontend.clone(); - let state: Arc = Arc::new(config.into()); + let state: Arc = Arc::new(config.try_into()?); log::info!("starting Alexandrie (version: {})", build::short()); From 021dbc79ce770f2bd34b8f9478e005e94e9746f1 Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Sun, 14 May 2023 23:39:16 +0200 Subject: [PATCH 02/22] Run ft (cherry picked from commit e5bb5b0c6ff39c36d287a6f105889852d6ca3950) --- crates/alexandrie/src/api/crates/suggest.rs | 2 +- crates/alexandrie/src/frontend/indexer.rs | 9 ++++++--- crates/alexandrie/src/frontend/mod.rs | 4 ++-- crates/alexandrie/src/frontend/search.rs | 12 +++++------ crates/alexandrie/src/fts/document.rs | 8 ++++---- crates/alexandrie/src/fts/index.rs | 22 +++++++++++++++------ crates/alexandrie/src/fts/mod.rs | 4 ++-- crates/alexandrie/src/main.rs | 3 +-- 8 files changed, 38 insertions(+), 26 deletions(-) diff --git a/crates/alexandrie/src/api/crates/suggest.rs b/crates/alexandrie/src/api/crates/suggest.rs index 49ea7635..fd5ebc13 100644 --- a/crates/alexandrie/src/api/crates/suggest.rs +++ b/crates/alexandrie/src/api/crates/suggest.rs @@ -8,8 +8,8 @@ use tide::Request; use alexandrie_index::Indexer; use crate::error::{AlexError, Error}; -use crate::State; use crate::utils; +use crate::State; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct APIResponse { diff --git a/crates/alexandrie/src/frontend/indexer.rs b/crates/alexandrie/src/frontend/indexer.rs index cc0f86fe..7f8e039c 100644 --- a/crates/alexandrie/src/frontend/indexer.rs +++ b/crates/alexandrie/src/frontend/indexer.rs @@ -10,7 +10,6 @@ use crate::State; const NUMBER_RESULT_PER_PAGE: i64 = 1000; - pub(crate) async fn get(req: Request) -> tide::Result { let state = req.state().clone(); let repo = &state.db; @@ -98,7 +97,11 @@ pub(crate) async fn get(req: Request) -> tide::Result { tantivy.create_or_update(krate.id, document)?; } Err(error) => { - warn!("Can't convert crate '{}' ({}) into Tantivy document : {error}", krate.id, krate.name.clone()); + warn!( + "Can't convert crate '{}' ({}) into Tantivy document : {error}", + krate.id, + krate.name.clone() + ); } } count_crate += 1; @@ -115,4 +118,4 @@ pub(crate) async fn get(req: Request) -> tide::Result { transaction?; let res = Response::new(200); Ok(res) -} \ No newline at end of file +} diff --git a/crates/alexandrie/src/frontend/mod.rs b/crates/alexandrie/src/frontend/mod.rs index eb3a6d9b..bc45c442 100644 --- a/crates/alexandrie/src/frontend/mod.rs +++ b/crates/alexandrie/src/frontend/mod.rs @@ -4,6 +4,8 @@ pub mod account; pub mod helpers; /// The index page (eg. "/"). pub mod index; +/// Index all crates into search index. +pub mod indexer; /// Crate-dedicated pages (eg. "/crates/\"). pub mod krate; /// Last updated crates (eg. "/last-updated"). @@ -14,5 +16,3 @@ pub mod me; pub mod most_downloaded; /// Search pages (eg. "/search?q=\"). pub mod search; -/// Index all crates into search index. -pub mod indexer; diff --git a/crates/alexandrie/src/frontend/search.rs b/crates/alexandrie/src/frontend/search.rs index aa7a82dc..92031a9a 100644 --- a/crates/alexandrie/src/frontend/search.rs +++ b/crates/alexandrie/src/frontend/search.rs @@ -7,14 +7,14 @@ use tide::Request; use alexandrie_index::Indexer; -use crate::db::DATETIME_FORMAT; use crate::db::models::Crate; use crate::db::schema::*; +use crate::db::DATETIME_FORMAT; use crate::error::Error; use crate::frontend::helpers; -use crate::State; use crate::utils; use crate::utils::auth::AuthExt; +use crate::State; const RESULT_PER_PAGE: i64 = 15; @@ -53,10 +53,10 @@ pub(crate) async fn get(req: Request) -> tide::Result { let page_count = (count / RESULT_PER_PAGE as usize + if count > 0 && count % RESULT_PER_PAGE as usize == 0 { - 0 - } else { - 1 - }) as u32; + 0 + } else { + 1 + }) as u32; let transaction = repo.transaction(move |conn| { let state = req.state(); diff --git a/crates/alexandrie/src/fts/document.rs b/crates/alexandrie/src/fts/document.rs index aff03192..7e2832e7 100644 --- a/crates/alexandrie/src/fts/document.rs +++ b/crates/alexandrie/src/fts/document.rs @@ -1,7 +1,7 @@ +use crate::error::Error; use std::fmt::Formatter; -use tantivy::Document; use tantivy::schema::Schema; -use crate::error::Error; +use tantivy::Document; /// Represent a crate. #[derive(Clone, Debug, Eq, PartialEq)] @@ -25,7 +25,7 @@ impl std::fmt::Display for TantivyDocument<'_> { // log will be unreadable. Just say that it has // a README. if let Some(_) = &self.readme { - write!(f, ", crate has README", )?; + write!(f, ", crate has README",)?; } if self.keywords.is_empty() { @@ -157,4 +157,4 @@ impl<'a> TantivyDocument<'a> { pub(crate) fn add_category(&mut self, category: String) { self.categories.push(category); } -} \ No newline at end of file +} diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index 3828364b..0ac823c2 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -2,12 +2,15 @@ use std::convert::TryFrom; use std::num::NonZeroUsize; use log::{error, info, warn}; -use tantivy::{Document, Index as TantivyIndex, IndexWriter, Opstamp, TantivyError, Term}; use tantivy::collector::{Count, TopDocs}; use tantivy::directory::MmapDirectory; use tantivy::query::QueryParser; use tantivy::schema::{NumericOptions, Schema, TextFieldIndexing, TextOptions}; -use tantivy::tokenizer::{Language, LowerCaser, RawTokenizer, SimpleTokenizer, StopWordFilter, TextAnalyzer, TokenizerManager}; +use tantivy::tokenizer::{ + Language, LowerCaser, RawTokenizer, SimpleTokenizer, StopWordFilter, TextAnalyzer, + TokenizerManager, +}; +use tantivy::{Document, Index as TantivyIndex, IndexWriter, Opstamp, TantivyError, Term}; use tantivy_analysis_contrib::commons::EdgeNgramTokenFilter; use crate::config::SearchConfig; @@ -89,7 +92,8 @@ impl TryFrom for Tantivy { // Analysis a tokenizer that tokenizes on non-alphanumeric characters // A filter that removes common english words (the, a, ...etc) // A filter that lowercase words - let stop_words = StopWordFilter::new(Language::English).ok_or(Self::Error::EmptyStopWord)?; + let stop_words = + StopWordFilter::new(Language::English).ok_or(Self::Error::EmptyStopWord)?; let analyzer = TextAnalyzer::from(SimpleTokenizer) .filter(stop_words) .filter(LowerCaser); @@ -165,7 +169,10 @@ impl Tantivy { let name = self.schema.get_field(super::NAME_FIELD_NAME).unwrap(); let name_full = self.schema.get_field(super::NAME_FIELD_NAME_FULL).unwrap(); - let name_prefix = self.schema.get_field(super::NAME_FIELD_PREFIX_NAME).unwrap(); + let name_prefix = self + .schema + .get_field(super::NAME_FIELD_PREFIX_NAME) + .unwrap(); let mut query_parser = QueryParser::new( self.schema.clone(), @@ -216,7 +223,10 @@ impl Tantivy { let id = self.schema.get_field(super::ID_FIELD_NAME).unwrap(); let name = self.schema.get_field(super::NAME_FIELD_NAME).unwrap(); let name_full = self.schema.get_field(super::NAME_FIELD_NAME_FULL).unwrap(); - let description = self.schema.get_field(super::DESCRIPTION_FIELD_NAME).unwrap(); + let description = self + .schema + .get_field(super::DESCRIPTION_FIELD_NAME) + .unwrap(); let readme = self.schema.get_field(super::README_FIELD_NAME).unwrap(); let categories = self.schema.get_field(super::CATEGORY_FIELD_NAME).unwrap(); let keywords = self.schema.get_field(super::KEYWORD_FIELD_NAME).unwrap(); @@ -275,4 +285,4 @@ impl Tantivy { Ok((count, results)) } -} \ No newline at end of file +} diff --git a/crates/alexandrie/src/fts/mod.rs b/crates/alexandrie/src/fts/mod.rs index 7cb56a22..19bf37bf 100644 --- a/crates/alexandrie/src/fts/mod.rs +++ b/crates/alexandrie/src/fts/mod.rs @@ -1,9 +1,9 @@ //! Full-text search module -mod index; mod document; +mod index; -pub(crate) use index::Tantivy; pub(crate) use document::TantivyDocument; +pub(crate) use index::Tantivy; /// Database ID. const ID_FIELD_NAME: &str = "id"; diff --git a/crates/alexandrie/src/main.rs b/crates/alexandrie/src/main.rs index 6d97dd7c..f350d822 100644 --- a/crates/alexandrie/src/main.rs +++ b/crates/alexandrie/src/main.rs @@ -151,8 +151,7 @@ fn frontend_routes(state: State, frontend_config: FrontendConfig) -> io::Result< app.at("/account/manage/tokens/:token-id/revoke") .get(frontend::account::manage::tokens::revoke::get); log::info!("mounting '/indexer'"); - app.at("/indexer") - .get(frontend::indexer::get); + app.at("/indexer").get(frontend::indexer::get); log::info!("mounting '/assets/*path'"); app.at("/assets").serve_dir(frontend_config.assets.path)?; From 56136dc5a8741791811726cda8c96587b4fdb912 Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Sun, 14 May 2023 23:49:39 +0200 Subject: [PATCH 03/22] Fix clippy warning (cherry picked from commit 217fc6892c376c82e33c1d45c62ed529905c6ec3) --- crates/alexandrie/src/frontend/indexer.rs | 2 +- crates/alexandrie/src/frontend/search.rs | 3 +- crates/alexandrie/src/fts/document.rs | 46 ++++++++++------------- crates/alexandrie/src/fts/index.rs | 12 +++--- 4 files changed, 28 insertions(+), 35 deletions(-) diff --git a/crates/alexandrie/src/frontend/indexer.rs b/crates/alexandrie/src/frontend/indexer.rs index 7f8e039c..264f2ce4 100644 --- a/crates/alexandrie/src/frontend/indexer.rs +++ b/crates/alexandrie/src/frontend/indexer.rs @@ -39,7 +39,7 @@ pub(crate) async fn get(req: Request) -> tide::Result { .into_iter() .map(|c| c.id) .collect::>(); - start = start + krates.len() as i64; + start += krates.len() as i64; info!("Crates {:?}", ids); diff --git a/crates/alexandrie/src/frontend/search.rs b/crates/alexandrie/src/frontend/search.rs index 92031a9a..e6dc5369 100644 --- a/crates/alexandrie/src/frontend/search.rs +++ b/crates/alexandrie/src/frontend/search.rs @@ -41,7 +41,8 @@ pub(crate) async fn get(req: Request) -> tide::Result { let repo = &state.db; let (count, results) = { - let tantivy = (&state.search) + let tantivy = state + .search .read() .map_err(|error| Error::PoisonedError(error.to_string()))?; tantivy.search( diff --git a/crates/alexandrie/src/fts/document.rs b/crates/alexandrie/src/fts/document.rs index 7e2832e7..66ec8a2f 100644 --- a/crates/alexandrie/src/fts/document.rs +++ b/crates/alexandrie/src/fts/document.rs @@ -1,8 +1,10 @@ -use crate::error::Error; use std::fmt::Formatter; + use tantivy::schema::Schema; use tantivy::Document; +use crate::error::Error; + /// Represent a crate. #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct TantivyDocument<'a> { @@ -24,7 +26,7 @@ impl std::fmt::Display for TantivyDocument<'_> { // Don't write the README, it might be big, and // log will be unreadable. Just say that it has // a README. - if let Some(_) = &self.readme { + if self.readme.is_some() { write!(f, ", crate has README",)?; } @@ -86,39 +88,31 @@ impl TryFrom> for Document { // For the following fields we will not fail if they are not in schema // but TODO add warn if let Some(description) = &value.description { - match description_field { - Some(field) => document.add_text(field, description), - None => (), + if let Some(field) = description_field { + document.add_text(field, description) } } if let Some(readme) = &value.readme { - match readme_field { - Some(field) => document.add_text(field, readme), - None => (), + if let Some(field) = readme_field { + document.add_text(field, readme) } } - if !value.keywords.is_empty() { - match keyword_field { - Some(field) => value - .keywords - .clone() - .into_iter() - .for_each(|v| document.add_text(field, v)), - None => (), - } + if let Some(field) = keyword_field { + value + .keywords + .clone() + .into_iter() + .for_each(|v| document.add_text(field, v)) } - if !value.categories.is_empty() { - match category_field { - Some(field) => value - .categories - .clone() - .into_iter() - .for_each(|v| document.add_text(field, v)), - None => (), - } + if let Some(field) = category_field { + value + .categories + .clone() + .into_iter() + .for_each(|v| document.add_text(field, v)) } Ok(document) diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index 0ac823c2..49f6ec3a 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -82,7 +82,7 @@ impl TryFrom for Tantivy { // after searching indices schema_builder.add_text_field(super::NAME_FIELD_NAME, options.clone()); schema_builder.add_text_field(super::NAME_FIELD_NAME_FULL, options_full.clone()); - schema_builder.add_text_field(super::NAME_FIELD_PREFIX_NAME, options_prefixes.clone()); + schema_builder.add_text_field(super::NAME_FIELD_PREFIX_NAME, options_prefixes); schema_builder.add_text_field(super::DESCRIPTION_FIELD_NAME, options.clone()); schema_builder.add_text_field(super::README_FIELD_NAME, options.clone()); schema_builder.add_text_field(super::CATEGORY_FIELD_NAME, options_full); @@ -203,9 +203,8 @@ impl Tantivy { } x.cloned() }) - .map(|v| v.map(|i| i.as_text().map(|t| t.to_owned())).flatten()) - .filter(|v| v.is_some()) - .map(|v| v.unwrap()) + .map(|v| v.and_then(|i| i.as_text().map(|t| t.to_owned()))) + .flatten() .collect(); Ok(results) @@ -273,14 +272,13 @@ impl Tantivy { let mut field = retrieve_doc.get_all(id); if let Some(x) = field.next() { - x.as_i64().clone() + x.as_i64() } else { warn!("Could not find field id"); None } }) - .filter(|v| v.is_some()) - .map(|v| v.unwrap()) + .flatten() .collect(); Ok((count, results)) From 96fc11f376244e7360ce8bf966289cc6ad089a7a Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Thu, 18 May 2023 15:27:27 +0200 Subject: [PATCH 04/22] Reduce logs (cherry picked from commit 67067972e4e0aa7df0106e51bcb28d8e3584bcbf) --- crates/alexandrie/src/frontend/indexer.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/alexandrie/src/frontend/indexer.rs b/crates/alexandrie/src/frontend/indexer.rs index 264f2ce4..db1c2f83 100644 --- a/crates/alexandrie/src/frontend/indexer.rs +++ b/crates/alexandrie/src/frontend/indexer.rs @@ -1,5 +1,5 @@ use diesel::prelude::*; -use log::{info, warn}; +use log::{info, warn, debug}; use tide::{Request, Response}; use crate::db::models::Crate; @@ -41,7 +41,7 @@ pub(crate) async fn get(req: Request) -> tide::Result { .collect::>(); start += krates.len() as i64; - info!("Crates {:?}", ids); + debug!("Crates {:?}", ids); let keywords = keywords::table .inner_join(crate_keywords::table) @@ -64,7 +64,7 @@ pub(crate) async fn get(req: Request) -> tide::Result { let mut current_category: Option<(i64, String)> = categories_iterator.next(); for krate in krates.into_iter() { - info!("crate {:?}", krate); + debug!("crate {:?}", krate); // Create a document with database ID and crate name let mut doc: TantivyDocument = TantivyDocument::new(krate.id, krate.name.clone(), &tantivy.schema); @@ -105,6 +105,10 @@ pub(crate) async fn get(req: Request) -> tide::Result { } } count_crate += 1; + + if count_crate % 1000 == 0 { + info!("{} crates indexed", count_crate); + } } (*tantivy).commit()?; From b6d6959267b9ce51373a1d198f52e8c0d62a4bc9 Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Thu, 18 May 2023 16:13:27 +0200 Subject: [PATCH 05/22] Remove RwLock on Tantivy Instead, the RwLock is in the index writer to allow interior mutability for commit operation. Though IndexWriter should be thread safe, I don't know how to achieve mutability without the RwLock. (cherry picked from commit b2987f546e0d85a3f5cbc4542a203ca734440011) --- crates/alexandrie/src/api/crates/suggest.rs | 9 ++----- crates/alexandrie/src/config/mod.rs | 5 ++-- crates/alexandrie/src/frontend/indexer.rs | 4 +-- crates/alexandrie/src/frontend/search.rs | 28 ++++++++------------- crates/alexandrie/src/fts/index.rs | 28 ++++++++++++++------- 5 files changed, 36 insertions(+), 38 deletions(-) diff --git a/crates/alexandrie/src/api/crates/suggest.rs b/crates/alexandrie/src/api/crates/suggest.rs index fd5ebc13..f0ecf5d1 100644 --- a/crates/alexandrie/src/api/crates/suggest.rs +++ b/crates/alexandrie/src/api/crates/suggest.rs @@ -8,8 +8,8 @@ use tide::Request; use alexandrie_index::Indexer; use crate::error::{AlexError, Error}; -use crate::utils; use crate::State; +use crate::utils; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct APIResponse { @@ -43,12 +43,7 @@ pub(crate) async fn get(req: Request) -> tide::Result { let state = req.state().clone(); let index = &state.index; - let results = { - let tantivy = (&state.search) - .read() - .map_err(|error| Error::PoisonedError(error.to_string()))?; - tantivy.suggest(name, limit)? - }; + let results = state.search.suggest(name, limit)?; let suggestions = results .into_iter() .map(|krate| { diff --git a/crates/alexandrie/src/config/mod.rs b/crates/alexandrie/src/config/mod.rs index 6dda1f93..f55dd019 100644 --- a/crates/alexandrie/src/config/mod.rs +++ b/crates/alexandrie/src/config/mod.rs @@ -1,5 +1,4 @@ use serde::{Deserialize, Serialize}; -use std::sync::RwLock; /// Database configuration (`[database]` section). pub mod database; @@ -81,7 +80,7 @@ pub struct State { /// The syntect configuration. pub syntect: SyntectState, /// Search config - pub search: RwLock, + pub search: Tantivy, /// The frontend configured state. #[cfg(feature = "frontend")] pub frontend: FrontendState, @@ -105,7 +104,7 @@ impl TryFrom for State { storage: config.storage.into(), db: Database::new(&config.database), syntect: config.syntect.into(), - search: RwLock::new(config.search.try_into()?), + search: config.search.try_into()?, #[cfg(feature = "frontend")] frontend: config.frontend.into(), }) diff --git a/crates/alexandrie/src/frontend/indexer.rs b/crates/alexandrie/src/frontend/indexer.rs index db1c2f83..6cb8eb1a 100644 --- a/crates/alexandrie/src/frontend/indexer.rs +++ b/crates/alexandrie/src/frontend/indexer.rs @@ -1,5 +1,5 @@ use diesel::prelude::*; -use log::{info, warn, debug}; +use log::{debug, info, warn}; use tide::{Request, Response}; use crate::db::models::Crate; @@ -17,7 +17,7 @@ pub(crate) async fn get(req: Request) -> tide::Result { let transaction: Result<(), Error> = repo .transaction(move |conn| { let state = req.state(); - let mut tantivy = state.search.write().unwrap(); + let tantivy = &state.search; tantivy.delete_all_documents()?; tantivy.commit()?; let mut start: i64 = 0; diff --git a/crates/alexandrie/src/frontend/search.rs b/crates/alexandrie/src/frontend/search.rs index e6dc5369..b259b49c 100644 --- a/crates/alexandrie/src/frontend/search.rs +++ b/crates/alexandrie/src/frontend/search.rs @@ -7,14 +7,14 @@ use tide::Request; use alexandrie_index::Indexer; +use crate::db::DATETIME_FORMAT; use crate::db::models::Crate; use crate::db::schema::*; -use crate::db::DATETIME_FORMAT; use crate::error::Error; use crate::frontend::helpers; +use crate::State; use crate::utils; use crate::utils::auth::AuthExt; -use crate::State; const RESULT_PER_PAGE: i64 = 15; @@ -40,24 +40,18 @@ pub(crate) async fn get(req: Request) -> tide::Result { let state = req.state().clone(); let repo = &state.db; - let (count, results) = { - let tantivy = state - .search - .read() - .map_err(|error| Error::PoisonedError(error.to_string()))?; - tantivy.search( - searched_text.clone(), - offset as usize, - RESULT_PER_PAGE as usize, - )? - }; + let (count, results) = state.search.search( + searched_text.clone(), + offset as usize, + RESULT_PER_PAGE as usize, + )?; let page_count = (count / RESULT_PER_PAGE as usize + if count > 0 && count % RESULT_PER_PAGE as usize == 0 { - 0 - } else { - 1 - }) as u32; + 0 + } else { + 1 + }) as u32; let transaction = repo.transaction(move |conn| { let state = req.state(); diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index 49f6ec3a..fb50a3a6 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -1,7 +1,9 @@ use std::convert::TryFrom; use std::num::NonZeroUsize; +use std::sync::RwLock; use log::{error, info, warn}; +use tantivy::{Document, Index as TantivyIndex, IndexWriter, Opstamp, TantivyError, Term}; use tantivy::collector::{Count, TopDocs}; use tantivy::directory::MmapDirectory; use tantivy::query::QueryParser; @@ -10,7 +12,6 @@ use tantivy::tokenizer::{ Language, LowerCaser, RawTokenizer, SimpleTokenizer, StopWordFilter, TextAnalyzer, TokenizerManager, }; -use tantivy::{Document, Index as TantivyIndex, IndexWriter, Opstamp, TantivyError, Term}; use tantivy_analysis_contrib::commons::EdgeNgramTokenFilter; use crate::config::SearchConfig; @@ -21,7 +22,7 @@ pub struct Tantivy { index: TantivyIndex, /// There can only be one index writer at a time (see https://tantivy-search.github.io/examples/basic_search.html) /// so we keep only one here. It has its own pool. - index_writer: IndexWriter, + index_writer: RwLock, pub schema: Schema, /// Search tokenizer manager search_tokenizer_manager: TokenizerManager, @@ -126,7 +127,7 @@ impl TryFrom for Tantivy { ); // Get an index writer with 50MB of heap - let index_writer = index.writer(50_000_000)?; + let index_writer = RwLock::new(index.writer(50_000_000)?); Ok(Self { index, @@ -143,8 +144,11 @@ impl Tantivy { pub fn create_or_update(&self, id: i64, document: Document) -> Result<(), Error> { if let Some(field) = self.schema.get_field(super::ID_FIELD_NAME) { let term = Term::from_field_i64(field, id); - self.index_writer.delete_term(term); - self.index_writer.add_document(document)?; + let index_writer = self.index_writer + .read() + .map_err(|error| Error::PoisonedError(error.to_string()))?; + index_writer.delete_term(term); + index_writer.add_document(document)?; } else { error!("There is no field {} in schema", super::ID_FIELD_NAME); } @@ -152,13 +156,19 @@ impl Tantivy { Ok(()) } - pub fn delete_all_documents(&mut self) -> Result { - self.index_writer.delete_all_documents() + pub fn delete_all_documents(&self) -> Result { + let index_writer = self.index_writer + .read() + .map_err(|error| Error::PoisonedError(error.to_string()))?; + Ok(index_writer.delete_all_documents()?) } /// Commit all pending changes inside the index. - pub fn commit(&mut self) -> Result { - self.index_writer.commit() + pub fn commit(&self) -> Result { + let mut index_writer = self.index_writer + .write() + .map_err(|error| Error::PoisonedError(error.to_string()))?; + Ok(index_writer.commit()?) } /// Search document by default through all crate's name index. This allows having search From e761e4a3f0ed772ee94187c225a24fec82d4e63a Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Thu, 18 May 2023 18:45:29 +0200 Subject: [PATCH 06/22] Remove Index and add IndexReader In Tantivy's example, it is stated that for a search server you typically want one reader for the lifetime of the application (cherry picked from commit 6313c4e734b2909d1505a6515b2ef7fc6d8fadfa) --- crates/alexandrie/src/frontend/indexer.rs | 11 +--- crates/alexandrie/src/fts/document.rs | 74 +++++++++++------------ crates/alexandrie/src/fts/index.rs | 25 +++++--- 3 files changed, 54 insertions(+), 56 deletions(-) diff --git a/crates/alexandrie/src/frontend/indexer.rs b/crates/alexandrie/src/frontend/indexer.rs index 6cb8eb1a..879e9d59 100644 --- a/crates/alexandrie/src/frontend/indexer.rs +++ b/crates/alexandrie/src/frontend/indexer.rs @@ -67,7 +67,7 @@ pub(crate) async fn get(req: Request) -> tide::Result { debug!("crate {:?}", krate); // Create a document with database ID and crate name let mut doc: TantivyDocument = - TantivyDocument::new(krate.id, krate.name.clone(), &tantivy.schema); + TantivyDocument::new(krate.id, krate.name.clone()); // If there is some description, then set it if let Some(description) = krate.description.as_ref() { @@ -92,17 +92,12 @@ pub(crate) async fn get(req: Request) -> tide::Result { // TODO get README - match doc.try_into() { - Ok(document) => { - tantivy.create_or_update(krate.id, document)?; - } - Err(error) => { - warn!( + if let Err(error) = tantivy.create_or_update(krate.id, doc) { + warn!( "Can't convert crate '{}' ({}) into Tantivy document : {error}", krate.id, krate.name.clone() ); - } } count_crate += 1; diff --git a/crates/alexandrie/src/fts/document.rs b/crates/alexandrie/src/fts/document.rs index 66ec8a2f..8899c7a5 100644 --- a/crates/alexandrie/src/fts/document.rs +++ b/crates/alexandrie/src/fts/document.rs @@ -7,8 +7,7 @@ use crate::error::Error; /// Represent a crate. #[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct TantivyDocument<'a> { - schema: &'a Schema, +pub struct TantivyDocument { id: i64, name: String, description: Option, @@ -17,7 +16,7 @@ pub(crate) struct TantivyDocument<'a> { categories: Vec, } -impl std::fmt::Display for TantivyDocument<'_> { +impl std::fmt::Display for TantivyDocument { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "id: {}, name: {}", self.id, self.name)?; if let Some(description) = &self.description { @@ -48,22 +47,31 @@ impl std::fmt::Display for TantivyDocument<'_> { } } -impl TryFrom> for Document { - type Error = Error; +impl TantivyDocument { + pub fn new(id: i64, name: String) -> Self { + Self { + id, + name, + description: None, + readme: None, + keywords: Vec::with_capacity(5), + categories: Vec::with_capacity(5), + } + } - fn try_from(value: TantivyDocument) -> Result { + pub fn try_into(self, schema: &Schema) -> Result { // Can't implement From because I need to use a tuple to hold schema and both // tuple and Document are in another crate :-( let mut document = Document::new(); - let id_field = value.schema.get_field(super::ID_FIELD_NAME); - let name_field = value.schema.get_field(super::NAME_FIELD_NAME); - let name_full_field = value.schema.get_field(super::NAME_FIELD_NAME_FULL); - let name_prefix_field = value.schema.get_field(super::NAME_FIELD_PREFIX_NAME); - let description_field = value.schema.get_field(super::DESCRIPTION_FIELD_NAME); - let readme_field = value.schema.get_field(super::README_FIELD_NAME); - let category_field = value.schema.get_field(super::CATEGORY_FIELD_NAME); - let keyword_field = value.schema.get_field(super::KEYWORD_FIELD_NAME); + let id_field = schema.get_field(super::ID_FIELD_NAME); + let name_field = schema.get_field(super::NAME_FIELD_NAME); + let name_full_field = schema.get_field(super::NAME_FIELD_NAME_FULL); + let name_prefix_field = schema.get_field(super::NAME_FIELD_PREFIX_NAME); + let description_field = schema.get_field(super::DESCRIPTION_FIELD_NAME); + let readme_field = schema.get_field(super::README_FIELD_NAME); + let category_field = schema.get_field(super::CATEGORY_FIELD_NAME); + let keyword_field = schema.get_field(super::KEYWORD_FIELD_NAME); // None of the fields should be `None`. // But we check that anyway. @@ -80,27 +88,27 @@ impl TryFrom> for Document { return Err(Error::MissingField(super::NAME_FIELD_PREFIX_NAME)); } - document.add_i64(id_field.unwrap(), value.id); - document.add_text(name_field.unwrap(), &value.name); - document.add_text(name_full_field.unwrap(), &value.name); - document.add_text(name_prefix_field.unwrap(), &value.name); + document.add_i64(id_field.unwrap(), self.id); + document.add_text(name_field.unwrap(), &self.name); + document.add_text(name_full_field.unwrap(), &self.name); + document.add_text(name_prefix_field.unwrap(), self.name); // For the following fields we will not fail if they are not in schema // but TODO add warn - if let Some(description) = &value.description { + if let Some(description) = &self.description { if let Some(field) = description_field { document.add_text(field, description) } } - if let Some(readme) = &value.readme { + if let Some(readme) = &self.readme { if let Some(field) = readme_field { document.add_text(field, readme) } } if let Some(field) = keyword_field { - value + self .keywords .clone() .into_iter() @@ -108,7 +116,7 @@ impl TryFrom> for Document { } if let Some(field) = category_field { - value + self .categories .clone() .into_iter() @@ -117,38 +125,24 @@ impl TryFrom> for Document { Ok(document) } -} - -impl<'a> TantivyDocument<'a> { - pub(crate) fn new(id: i64, name: String, schema: &'a Schema) -> Self { - Self { - schema, - id, - name, - description: None, - readme: None, - keywords: Vec::with_capacity(5), - categories: Vec::with_capacity(5), - } - } /// Set crate's description - pub(crate) fn set_description(&mut self, description: String) { + pub fn set_description(&mut self, description: String) { self.description = Some(description); } /// Set crate's README - pub(crate) fn set_readme(&mut self, readme: String) { + pub fn set_readme(&mut self, readme: String) { self.readme = Some(readme); } /// Add new crate's keyword - pub(crate) fn add_keyword(&mut self, keyword: String) { + pub fn add_keyword(&mut self, keyword: String) { self.keywords.push(keyword); } /// Add new crate's category - pub(crate) fn add_category(&mut self, category: String) { + pub fn add_category(&mut self, category: String) { self.categories.push(category); } } diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index fb50a3a6..c3c20ef8 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -3,7 +3,7 @@ use std::num::NonZeroUsize; use std::sync::RwLock; use log::{error, info, warn}; -use tantivy::{Document, Index as TantivyIndex, IndexWriter, Opstamp, TantivyError, Term}; +use tantivy::{Index as TantivyIndex, IndexReader, IndexWriter, Opstamp, ReloadPolicy, TantivyError, Term}; use tantivy::collector::{Count, TopDocs}; use tantivy::directory::MmapDirectory; use tantivy::query::QueryParser; @@ -16,14 +16,16 @@ use tantivy_analysis_contrib::commons::EdgeNgramTokenFilter; use crate::config::SearchConfig; use crate::error::Error; +use crate::fts::TantivyDocument; /// Helper for using Tantivy pub struct Tantivy { - index: TantivyIndex, + index_reader: IndexReader, /// There can only be one index writer at a time (see https://tantivy-search.github.io/examples/basic_search.html) /// so we keep only one here. It has its own pool. index_writer: RwLock, - pub schema: Schema, + /// Index schema + schema: Schema, /// Search tokenizer manager search_tokenizer_manager: TokenizerManager, } @@ -129,8 +131,10 @@ impl TryFrom for Tantivy { // Get an index writer with 50MB of heap let index_writer = RwLock::new(index.writer(50_000_000)?); + let index_reader = index.reader_builder().reload_policy(ReloadPolicy::OnCommit).try_into()?; + Ok(Self { - index, + index_reader, index_writer, schema, search_tokenizer_manager, @@ -139,9 +143,14 @@ impl TryFrom for Tantivy { } impl Tantivy { + pub fn schema(&self) -> &Schema { + &self.schema + } + /// Method that create or update a document in Tantivy index. As there is no update, we need /// to first delete the document then create a new document. - pub fn create_or_update(&self, id: i64, document: Document) -> Result<(), Error> { + pub fn create_or_update(&self, id: i64, document: TantivyDocument) -> Result<(), Error> { + let document = document.try_into(&self.schema)?; if let Some(field) = self.schema.get_field(super::ID_FIELD_NAME) { let term = Term::from_field_i64(field, id); let index_writer = self.index_writer @@ -175,7 +184,7 @@ impl Tantivy { /// as you type (using prefixes) while increasing relevance when there's a matching word /// or if the whole text matches a crate's name (using the other crate's name indices). pub fn suggest(&self, query: String, limit: usize) -> Result, TantivyError> { - let searcher = self.index.reader()?.searcher(); + let searcher = self.index_reader.searcher(); let name = self.schema.get_field(super::NAME_FIELD_NAME).unwrap(); let name_full = self.schema.get_field(super::NAME_FIELD_NAME_FULL).unwrap(); @@ -227,7 +236,7 @@ impl Tantivy { offset: usize, limit: usize, ) -> Result<(usize, Vec), TantivyError> { - let searcher = self.index.reader()?.searcher(); + let searcher = self.index_reader.searcher(); let id = self.schema.get_field(super::ID_FIELD_NAME).unwrap(); let name = self.schema.get_field(super::NAME_FIELD_NAME).unwrap(); @@ -241,7 +250,7 @@ impl Tantivy { let keywords = self.schema.get_field(super::KEYWORD_FIELD_NAME).unwrap(); let mut query_parser = QueryParser::for_index( - &self.index, + searcher.index(), vec![name, name_full, description, readme, categories, keywords], ); From abb8bf3d6bbf83a39d86099e1a00ee8fb412a705 Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Thu, 18 May 2023 22:05:31 +0200 Subject: [PATCH 07/22] Index at startup and remove endpoint Indexing at startup is faster than indexing in an endpoint. 114k crates from crates.io get indexed within less than 10s. (cherry picked from commit efbcb7c212f5ea10e947e37a0394bedb9b1697e9) --- crates/alexandrie/src/frontend/indexer.rs | 120 --------------------- crates/alexandrie/src/frontend/mod.rs | 2 - crates/alexandrie/src/fts/index.rs | 122 +++++++++++++++++++++- crates/alexandrie/src/main.rs | 9 +- 4 files changed, 127 insertions(+), 126 deletions(-) delete mode 100644 crates/alexandrie/src/frontend/indexer.rs diff --git a/crates/alexandrie/src/frontend/indexer.rs b/crates/alexandrie/src/frontend/indexer.rs deleted file mode 100644 index 879e9d59..00000000 --- a/crates/alexandrie/src/frontend/indexer.rs +++ /dev/null @@ -1,120 +0,0 @@ -use diesel::prelude::*; -use log::{debug, info, warn}; -use tide::{Request, Response}; - -use crate::db::models::Crate; -use crate::db::schema::*; -use crate::error::Error; -use crate::fts::TantivyDocument; -use crate::State; - -const NUMBER_RESULT_PER_PAGE: i64 = 1000; - -pub(crate) async fn get(req: Request) -> tide::Result { - let state = req.state().clone(); - let repo = &state.db; - - let transaction: Result<(), Error> = repo - .transaction(move |conn| { - let state = req.state(); - let tantivy = &state.search; - tantivy.delete_all_documents()?; - tantivy.commit()?; - let mut start: i64 = 0; - let mut count_crate = 0; - - loop { - let krates = crates::table - .order_by(crates::id.asc()) - .limit(NUMBER_RESULT_PER_PAGE) - .offset(start) - .load::(conn)?; - if krates.is_empty() { - info!("End indexing"); - break; - } - - let ids = krates - .clone() - .into_iter() - .map(|c| c.id) - .collect::>(); - start += krates.len() as i64; - - debug!("Crates {:?}", ids); - - let keywords = keywords::table - .inner_join(crate_keywords::table) - .select((crate_keywords::crate_id, keywords::name)) - .filter(crate_keywords::crate_id.eq_any(&ids)) - .order_by(crate_keywords::crate_id.asc()) - .load::<(i64, String)>(conn)?; - - let categories = categories::table - .inner_join(crate_categories::table) - .select((crate_categories::crate_id, categories::name)) - .filter(crate_categories::crate_id.eq_any(&ids)) - .order_by(crate_categories::crate_id.asc()) - .load::<(i64, String)>(conn)?; - - let mut keywords_iterator = keywords.into_iter(); - let mut categories_iterator = categories.into_iter(); - - let mut current_keyword: Option<(i64, String)> = keywords_iterator.next(); - let mut current_category: Option<(i64, String)> = categories_iterator.next(); - - for krate in krates.into_iter() { - debug!("crate {:?}", krate); - // Create a document with database ID and crate name - let mut doc: TantivyDocument = - TantivyDocument::new(krate.id, krate.name.clone()); - - // If there is some description, then set it - if let Some(description) = krate.description.as_ref() { - doc.set_description(description.clone()); - } - - // Add all keywords - while current_keyword.is_some() - && current_keyword.as_ref().unwrap().0 == krate.id - { - doc.add_keyword(current_keyword.unwrap().1); - current_keyword = keywords_iterator.next(); - } - - // Add all cateogries - while current_category.is_some() - && current_category.as_ref().unwrap().0 == krate.id - { - doc.add_category(current_category.unwrap().1); - current_category = categories_iterator.next(); - } - - // TODO get README - - if let Err(error) = tantivy.create_or_update(krate.id, doc) { - warn!( - "Can't convert crate '{}' ({}) into Tantivy document : {error}", - krate.id, - krate.name.clone() - ); - } - count_crate += 1; - - if count_crate % 1000 == 0 { - info!("{} crates indexed", count_crate); - } - } - - (*tantivy).commit()?; - info!("{} crates indexed", count_crate); - } - - Ok(()) - }) - .await; - - transaction?; - let res = Response::new(200); - Ok(res) -} diff --git a/crates/alexandrie/src/frontend/mod.rs b/crates/alexandrie/src/frontend/mod.rs index bc45c442..78f4f0c2 100644 --- a/crates/alexandrie/src/frontend/mod.rs +++ b/crates/alexandrie/src/frontend/mod.rs @@ -4,8 +4,6 @@ pub mod account; pub mod helpers; /// The index page (eg. "/"). pub mod index; -/// Index all crates into search index. -pub mod indexer; /// Crate-dedicated pages (eg. "/crates/\"). pub mod krate; /// Last updated crates (eg. "/last-updated"). diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index c3c20ef8..8af21dca 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -2,7 +2,7 @@ use std::convert::TryFrom; use std::num::NonZeroUsize; use std::sync::RwLock; -use log::{error, info, warn}; +use log::{debug, error, info, warn}; use tantivy::{Index as TantivyIndex, IndexReader, IndexWriter, Opstamp, ReloadPolicy, TantivyError, Term}; use tantivy::collector::{Count, TopDocs}; use tantivy::directory::MmapDirectory; @@ -15,9 +15,19 @@ use tantivy::tokenizer::{ use tantivy_analysis_contrib::commons::EdgeNgramTokenFilter; use crate::config::SearchConfig; +use crate::db::Database; +use diesel::prelude::*; + +use crate::db::models::Crate; +use crate::db::schema::*; use crate::error::Error; use crate::fts::TantivyDocument; + +const NUMBER_RESULT_PER_PAGE: i64 = 1000; + +type CrateKeywordCategory = (Vec, Vec<(i64, String)>, Vec<(i64, String)>); + /// Helper for using Tantivy pub struct Tantivy { index_reader: IndexReader, @@ -302,4 +312,114 @@ impl Tantivy { Ok((count, results)) } + + pub async fn index_all(&self, repo:&Database) -> Result<(), Error> { + info!("Index all crates"); + self.delete_all_documents()?; + self.commit()?; + let mut start: i64 = 0; + let mut count_crate = 0; + + 'indexing: loop { + let result: Result, Error> = repo + .run(move |conn| { + debug!("Querying crates from {start} to {}", start + NUMBER_RESULT_PER_PAGE); + let krates = crates::table + .order_by(crates::id.asc()) + .limit(NUMBER_RESULT_PER_PAGE) + .offset(start) + .load::(conn)?; + + debug!("{} crates fetched", krates.len()); + + if krates.is_empty() { + return Ok(None); + } + + let ids = krates + .clone() + .into_iter() + .map(|c| c.id) + .collect::>(); + + debug!("Crates {:?}", ids); + + let keywords = keywords::table + .inner_join(crate_keywords::table) + .select((crate_keywords::crate_id, keywords::name)) + .filter(crate_keywords::crate_id.eq_any(&ids)) + .order_by(crate_keywords::crate_id.asc()) + .load::<(i64, String)>(conn)?; + + let categories = categories::table + .inner_join(crate_categories::table) + .select((crate_categories::crate_id, categories::name)) + .filter(crate_categories::crate_id.eq_any(&ids)) + .order_by(crate_categories::crate_id.asc()) + .load::<(i64, String)>(conn)?; + + Ok(Some((krates, keywords, categories))) + }).await; + + let result = result?; + + if let Some((krates, keywords, categories)) = result { + start = start + krates.len() as i64; + let mut keywords_iterator = keywords.into_iter(); + let mut categories_iterator = categories.into_iter(); + + let mut current_keyword: Option<(i64, String)> = keywords_iterator.next(); + let mut current_category: Option<(i64, String)> = categories_iterator.next(); + + for krate in krates.into_iter() { + debug!("crate {:?}", krate); + // Create a document with database ID and crate name + let mut doc: TantivyDocument = + TantivyDocument::new(krate.id, krate.name.clone()); + + // If there is some description, then set it + if let Some(description) = krate.description.as_ref() { + doc.set_description(description.clone()); + } + + // Add all keywords + while current_keyword.is_some() + && current_keyword.as_ref().unwrap().0 == krate.id + { + doc.add_keyword(current_keyword.unwrap().1); + current_keyword = keywords_iterator.next(); + } + + // Add all cateogries + while current_category.is_some() + && current_category.as_ref().unwrap().0 == krate.id + { + doc.add_category(current_category.unwrap().1); + current_category = categories_iterator.next(); + } + + // TODO get README + + if let Err(error) = self.create_or_update(krate.id, doc) { + warn!( + "Can't convert crate '{}' ({}) into Tantivy document : {error}", + krate.id, + krate.name.clone() + ); + } + count_crate += 1; + + if count_crate % 1000 == 0 { + info!("{} crates indexed", count_crate); + } + } + } else { + info!("End indexing {start} crates"); + self.commit()?; + break 'indexing; + } + } + + Ok(()) + } } diff --git a/crates/alexandrie/src/main.rs b/crates/alexandrie/src/main.rs index f350d822..af94e6af 100644 --- a/crates/alexandrie/src/main.rs +++ b/crates/alexandrie/src/main.rs @@ -150,8 +150,6 @@ fn frontend_routes(state: State, frontend_config: FrontendConfig) -> io::Result< log::info!("mounting '/account/manage/tokens/:token-id/revoke'"); app.at("/account/manage/tokens/:token-id/revoke") .get(frontend::account::manage::tokens::revoke::get); - log::info!("mounting '/indexer'"); - app.at("/indexer").get(frontend::indexer::get); log::info!("mounting '/assets/*path'"); app.at("/assets").serve_dir(frontend_config.assets.path)?; @@ -244,7 +242,12 @@ async fn run() -> Result<(), Error> { #[cfg(feature = "frontend")] let frontend_config = config.frontend.clone(); - let state: Arc = Arc::new(config.try_into()?); + let state:config::State = config.try_into()?; + + let database = &state.db; + state.search.index_all(database).await?; + + let state = Arc::new(state); log::info!("starting Alexandrie (version: {})", build::short()); From 19e1be47c2ceee3077a68b73219cb0ae0b6575f0 Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Thu, 18 May 2023 22:12:19 +0200 Subject: [PATCH 08/22] Run fmt (cherry picked from commit 2759c5322e26c956a7606da11fbaff332ce02655) --- crates/alexandrie/src/api/crates/suggest.rs | 2 +- crates/alexandrie/src/frontend/search.rs | 12 +++---- crates/alexandrie/src/fts/document.rs | 6 ++-- crates/alexandrie/src/fts/index.rs | 37 +++++++++++++-------- crates/alexandrie/src/main.rs | 2 +- 5 files changed, 34 insertions(+), 25 deletions(-) diff --git a/crates/alexandrie/src/api/crates/suggest.rs b/crates/alexandrie/src/api/crates/suggest.rs index f0ecf5d1..93b367a5 100644 --- a/crates/alexandrie/src/api/crates/suggest.rs +++ b/crates/alexandrie/src/api/crates/suggest.rs @@ -8,8 +8,8 @@ use tide::Request; use alexandrie_index::Indexer; use crate::error::{AlexError, Error}; -use crate::State; use crate::utils; +use crate::State; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct APIResponse { diff --git a/crates/alexandrie/src/frontend/search.rs b/crates/alexandrie/src/frontend/search.rs index b259b49c..5dd4683c 100644 --- a/crates/alexandrie/src/frontend/search.rs +++ b/crates/alexandrie/src/frontend/search.rs @@ -7,14 +7,14 @@ use tide::Request; use alexandrie_index::Indexer; -use crate::db::DATETIME_FORMAT; use crate::db::models::Crate; use crate::db::schema::*; +use crate::db::DATETIME_FORMAT; use crate::error::Error; use crate::frontend::helpers; -use crate::State; use crate::utils; use crate::utils::auth::AuthExt; +use crate::State; const RESULT_PER_PAGE: i64 = 15; @@ -48,10 +48,10 @@ pub(crate) async fn get(req: Request) -> tide::Result { let page_count = (count / RESULT_PER_PAGE as usize + if count > 0 && count % RESULT_PER_PAGE as usize == 0 { - 0 - } else { - 1 - }) as u32; + 0 + } else { + 1 + }) as u32; let transaction = repo.transaction(move |conn| { let state = req.state(); diff --git a/crates/alexandrie/src/fts/document.rs b/crates/alexandrie/src/fts/document.rs index 8899c7a5..0d52bf97 100644 --- a/crates/alexandrie/src/fts/document.rs +++ b/crates/alexandrie/src/fts/document.rs @@ -108,16 +108,14 @@ impl TantivyDocument { } if let Some(field) = keyword_field { - self - .keywords + self.keywords .clone() .into_iter() .for_each(|v| document.add_text(field, v)) } if let Some(field) = category_field { - self - .categories + self.categories .clone() .into_iter() .for_each(|v| document.add_text(field, v)) diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index 8af21dca..2e59a3f8 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -3,7 +3,6 @@ use std::num::NonZeroUsize; use std::sync::RwLock; use log::{debug, error, info, warn}; -use tantivy::{Index as TantivyIndex, IndexReader, IndexWriter, Opstamp, ReloadPolicy, TantivyError, Term}; use tantivy::collector::{Count, TopDocs}; use tantivy::directory::MmapDirectory; use tantivy::query::QueryParser; @@ -12,6 +11,9 @@ use tantivy::tokenizer::{ Language, LowerCaser, RawTokenizer, SimpleTokenizer, StopWordFilter, TextAnalyzer, TokenizerManager, }; +use tantivy::{ + Index as TantivyIndex, IndexReader, IndexWriter, Opstamp, ReloadPolicy, TantivyError, Term, +}; use tantivy_analysis_contrib::commons::EdgeNgramTokenFilter; use crate::config::SearchConfig; @@ -23,7 +25,6 @@ use crate::db::schema::*; use crate::error::Error; use crate::fts::TantivyDocument; - const NUMBER_RESULT_PER_PAGE: i64 = 1000; type CrateKeywordCategory = (Vec, Vec<(i64, String)>, Vec<(i64, String)>); @@ -141,7 +142,10 @@ impl TryFrom for Tantivy { // Get an index writer with 50MB of heap let index_writer = RwLock::new(index.writer(50_000_000)?); - let index_reader = index.reader_builder().reload_policy(ReloadPolicy::OnCommit).try_into()?; + let index_reader = index + .reader_builder() + .reload_policy(ReloadPolicy::OnCommit) + .try_into()?; Ok(Self { index_reader, @@ -163,7 +167,8 @@ impl Tantivy { let document = document.try_into(&self.schema)?; if let Some(field) = self.schema.get_field(super::ID_FIELD_NAME) { let term = Term::from_field_i64(field, id); - let index_writer = self.index_writer + let index_writer = self + .index_writer .read() .map_err(|error| Error::PoisonedError(error.to_string()))?; index_writer.delete_term(term); @@ -176,7 +181,8 @@ impl Tantivy { } pub fn delete_all_documents(&self) -> Result { - let index_writer = self.index_writer + let index_writer = self + .index_writer .read() .map_err(|error| Error::PoisonedError(error.to_string()))?; Ok(index_writer.delete_all_documents()?) @@ -184,7 +190,8 @@ impl Tantivy { /// Commit all pending changes inside the index. pub fn commit(&self) -> Result { - let mut index_writer = self.index_writer + let mut index_writer = self + .index_writer .write() .map_err(|error| Error::PoisonedError(error.to_string()))?; Ok(index_writer.commit()?) @@ -313,7 +320,7 @@ impl Tantivy { Ok((count, results)) } - pub async fn index_all(&self, repo:&Database) -> Result<(), Error> { + pub async fn index_all(&self, repo: &Database) -> Result<(), Error> { info!("Index all crates"); self.delete_all_documents()?; self.commit()?; @@ -323,7 +330,10 @@ impl Tantivy { 'indexing: loop { let result: Result, Error> = repo .run(move |conn| { - debug!("Querying crates from {start} to {}", start + NUMBER_RESULT_PER_PAGE); + debug!( + "Querying crates from {start} to {}", + start + NUMBER_RESULT_PER_PAGE + ); let krates = crates::table .order_by(crates::id.asc()) .limit(NUMBER_RESULT_PER_PAGE) @@ -359,7 +369,8 @@ impl Tantivy { .load::<(i64, String)>(conn)?; Ok(Some((krates, keywords, categories))) - }).await; + }) + .await; let result = result?; @@ -402,10 +413,10 @@ impl Tantivy { if let Err(error) = self.create_or_update(krate.id, doc) { warn!( - "Can't convert crate '{}' ({}) into Tantivy document : {error}", - krate.id, - krate.name.clone() - ); + "Can't convert crate '{}' ({}) into Tantivy document : {error}", + krate.id, + krate.name.clone() + ); } count_crate += 1; diff --git a/crates/alexandrie/src/main.rs b/crates/alexandrie/src/main.rs index af94e6af..4a71d3b2 100644 --- a/crates/alexandrie/src/main.rs +++ b/crates/alexandrie/src/main.rs @@ -242,7 +242,7 @@ async fn run() -> Result<(), Error> { #[cfg(feature = "frontend")] let frontend_config = config.frontend.clone(); - let state:config::State = config.try_into()?; + let state: config::State = config.try_into()?; let database = &state.db; state.search.index_all(database).await?; From 495ebf7de69d3399ef30f90b3981866eb6c898bd Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Thu, 18 May 2023 22:33:18 +0200 Subject: [PATCH 09/22] Improve merging crates with keywords and categories (cherry picked from commit 3aad76790f56f1c4ff89d54d14e1ff169b00e690) --- crates/alexandrie/src/fts/index.rs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index 2e59a3f8..864001b4 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -2,6 +2,7 @@ use std::convert::TryFrom; use std::num::NonZeroUsize; use std::sync::RwLock; +use diesel::prelude::*; use log::{debug, error, info, warn}; use tantivy::collector::{Count, TopDocs}; use tantivy::directory::MmapDirectory; @@ -17,11 +18,9 @@ use tantivy::{ use tantivy_analysis_contrib::commons::EdgeNgramTokenFilter; use crate::config::SearchConfig; -use crate::db::Database; -use diesel::prelude::*; - use crate::db::models::Crate; use crate::db::schema::*; +use crate::db::Database; use crate::error::Error; use crate::fts::TantivyDocument; @@ -376,8 +375,8 @@ impl Tantivy { if let Some((krates, keywords, categories)) = result { start = start + krates.len() as i64; - let mut keywords_iterator = keywords.into_iter(); - let mut categories_iterator = categories.into_iter(); + let mut keywords_iterator = keywords.into_iter().peekable(); + let mut categories_iterator = categories.into_iter().peekable(); let mut current_keyword: Option<(i64, String)> = keywords_iterator.next(); let mut current_category: Option<(i64, String)> = categories_iterator.next(); @@ -393,19 +392,23 @@ impl Tantivy { doc.set_description(description.clone()); } - // Add all keywords + // Skip keywords that might be orphan and add keywords that match ids while current_keyword.is_some() - && current_keyword.as_ref().unwrap().0 == krate.id + && current_keyword.as_ref().unwrap().0 <= krate.id { - doc.add_keyword(current_keyword.unwrap().1); + if current_keyword.as_ref().unwrap().0 == krate.id { + doc.add_keyword(current_keyword.unwrap().1); + } current_keyword = keywords_iterator.next(); } - // Add all cateogries + // Skip keywords that might be orphan and add keywords that match ids while current_category.is_some() - && current_category.as_ref().unwrap().0 == krate.id + && current_category.as_ref().unwrap().0 <= krate.id { - doc.add_category(current_category.unwrap().1); + if current_category.as_ref().unwrap().0 == krate.id { + doc.add_keyword(current_category.unwrap().1); + } current_category = categories_iterator.next(); } From d09b235727889d9aa17e179e566573116c1cb47b Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Mon, 22 May 2023 22:07:56 +0200 Subject: [PATCH 10/22] Fix and handle publish (cherry picked from commit 3054040c8b7521c6cfe9c509c7037abfa1a3d9f1) --- .gitignore | 1 + crates/alexandrie/src/api/crates/publish.rs | 29 +++++++++++++++---- crates/alexandrie/src/fts/document.rs | 32 +++++++++++++++++++++ crates/alexandrie/src/fts/index.rs | 30 +++++++------------ crates/alexandrie/src/main.rs | 6 ++-- 5 files changed, 71 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index 10d8bc34..281fbff1 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ # Rust compilation folder /target +crates/*/target # hidden folders from code editors /.vscode diff --git a/crates/alexandrie/src/api/crates/publish.rs b/crates/alexandrie/src/api/crates/publish.rs index c4607fc1..b6250db0 100644 --- a/crates/alexandrie/src/api/crates/publish.rs +++ b/crates/alexandrie/src/api/crates/publish.rs @@ -4,12 +4,12 @@ use std::path::PathBuf; use std::pin::pin; use async_std::io::prelude::*; - use byteorder::{LittleEndian, ReadBytesExt}; use chrono::Utc; use diesel::dsl as sql; use diesel::prelude::*; use flate2::read::GzDecoder; +use log::warn; use ring::digest as hasher; use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; @@ -26,6 +26,7 @@ use crate::db::schema::*; use crate::db::Connection; use crate::db::DATETIME_FORMAT; use crate::error::{AlexError, Error}; +use crate::fts::TantivyDocument; use crate::utils; use crate::State; @@ -70,7 +71,7 @@ struct CrateMetaDependency { fn link_keywords( conn: &mut Connection, crate_id: i64, - keywords: Option>, + keywords: &Option>, ) -> Result<(), Error> { diesel::delete(crate_keywords::table.filter(crate_keywords::crate_id.eq(crate_id))) .execute(conn)?; @@ -116,7 +117,7 @@ fn link_keywords( fn link_categories( conn: &mut Connection, crate_id: i64, - categories: Option>, + categories: &Option>, ) -> Result<(), Error> { diesel::delete(crate_categories::table.filter(crate_categories::crate_id.eq(crate_id))) .execute(conn)?; @@ -354,10 +355,10 @@ pub(crate) async fn put(mut req: Request) -> tide::Result { }; //? Update keywords. - link_keywords(conn, krate.id, metadata.keywords)?; + link_keywords(conn, krate.id, &metadata.keywords)?; //? Update categories. - link_categories(conn, krate.id, metadata.categories)?; + link_categories(conn, krate.id, &metadata.categories)?; //? Update badges. link_badges(conn, krate.id, metadata.badges)?; @@ -396,6 +397,24 @@ pub(crate) async fn put(mut req: Request) -> tide::Result { .storage .store_crate(&crate_desc.name, crate_desc.vers.clone(), crate_bytes)?; + let id = krate.id; + let name = krate.name.clone(); + + // Index into full text index + let mut document: TantivyDocument = krate.into(); + if let Some(keywords) = metadata.keywords { + document.add_all_keywords(keywords); + } + if let Some(categories) = metadata.categories { + document.add_all_categories(categories); + } + + if let Err(error) = state.search.create_or_update(document) { + warn!("Can't convert crate '{id}' ({name}) into Tantivy document : {error}"); + } else { + state.search.commit()?; + } + //? Store the crate's readme. if let Some(rendered) = rendered_readme { state diff --git a/crates/alexandrie/src/fts/document.rs b/crates/alexandrie/src/fts/document.rs index 0d52bf97..9b4149b1 100644 --- a/crates/alexandrie/src/fts/document.rs +++ b/crates/alexandrie/src/fts/document.rs @@ -1,5 +1,6 @@ use std::fmt::Formatter; +use crate::db::models::Crate; use tantivy::schema::Schema; use tantivy::Document; @@ -47,6 +48,19 @@ impl std::fmt::Display for TantivyDocument { } } +impl From for TantivyDocument { + fn from(value: Crate) -> Self { + Self { + id: value.id, + name: value.name, + description: value.description, + readme: None, + keywords: vec![], + categories: vec![], + } + } +} + impl TantivyDocument { pub fn new(id: i64, name: String) -> Self { Self { @@ -124,6 +138,10 @@ impl TantivyDocument { Ok(document) } + pub fn id(&self) -> i64 { + self.id + } + /// Set crate's description pub fn set_description(&mut self, description: String) { self.description = Some(description); @@ -139,8 +157,22 @@ impl TantivyDocument { self.keywords.push(keyword); } + /// Add all keywords + pub fn add_all_keywords(&mut self, keywords: Vec) { + for keyword in keywords { + self.add_keyword(keyword); + } + } + /// Add new crate's category pub fn add_category(&mut self, category: String) { self.categories.push(category); } + + /// Add all keywords + pub fn add_all_categories(&mut self, categories: Vec) { + for category in categories { + self.add_category(category); + } + } } diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index 864001b4..b9b204e3 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -162,7 +162,8 @@ impl Tantivy { /// Method that create or update a document in Tantivy index. As there is no update, we need /// to first delete the document then create a new document. - pub fn create_or_update(&self, id: i64, document: TantivyDocument) -> Result<(), Error> { + pub fn create_or_update(&self, document: TantivyDocument) -> Result<(), Error> { + let id = document.id(); let document = document.try_into(&self.schema)?; if let Some(field) = self.schema.get_field(super::ID_FIELD_NAME) { let term = Term::from_field_i64(field, id); @@ -384,29 +385,22 @@ impl Tantivy { for krate in krates.into_iter() { debug!("crate {:?}", krate); // Create a document with database ID and crate name - let mut doc: TantivyDocument = - TantivyDocument::new(krate.id, krate.name.clone()); + let id = krate.id; + let name = krate.name.clone(); - // If there is some description, then set it - if let Some(description) = krate.description.as_ref() { - doc.set_description(description.clone()); - } + let mut doc: TantivyDocument = krate.into(); // Skip keywords that might be orphan and add keywords that match ids - while current_keyword.is_some() - && current_keyword.as_ref().unwrap().0 <= krate.id - { - if current_keyword.as_ref().unwrap().0 == krate.id { + while current_keyword.is_some() && current_keyword.as_ref().unwrap().0 <= id { + if current_keyword.as_ref().unwrap().0 == id { doc.add_keyword(current_keyword.unwrap().1); } current_keyword = keywords_iterator.next(); } // Skip keywords that might be orphan and add keywords that match ids - while current_category.is_some() - && current_category.as_ref().unwrap().0 <= krate.id - { - if current_category.as_ref().unwrap().0 == krate.id { + while current_category.is_some() && current_category.as_ref().unwrap().0 <= id { + if current_category.as_ref().unwrap().0 == id { doc.add_keyword(current_category.unwrap().1); } current_category = categories_iterator.next(); @@ -414,11 +408,9 @@ impl Tantivy { // TODO get README - if let Err(error) = self.create_or_update(krate.id, doc) { + if let Err(error) = self.create_or_update(doc) { warn!( - "Can't convert crate '{}' ({}) into Tantivy document : {error}", - krate.id, - krate.name.clone() + "Can't convert crate '{id}' ({name}) into Tantivy document : {error}" ); } count_crate += 1; diff --git a/crates/alexandrie/src/main.rs b/crates/alexandrie/src/main.rs index 4a71d3b2..4da89d0d 100644 --- a/crates/alexandrie/src/main.rs +++ b/crates/alexandrie/src/main.rs @@ -244,9 +244,6 @@ async fn run() -> Result<(), Error> { let state: config::State = config.try_into()?; - let database = &state.db; - state.search.index_all(database).await?; - let state = Arc::new(state); log::info!("starting Alexandrie (version: {})", build::short()); @@ -256,6 +253,9 @@ async fn run() -> Result<(), Error> { state.db.run(|conn| conn.run_pending_migrations(db::MIGRATIONS).map(|_| ())).await .expect("migration execution error"); + let database = &state.db; + state.search.index_all(database).await?; + let mut app = tide::with_state(Arc::clone(&state)); log::info!("setting up request logger middleware"); From 097afff0176f693342fee390b93f51d48b32f3b2 Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Sat, 3 Jun 2023 11:03:28 +0200 Subject: [PATCH 11/22] Fix e2e test --- docker/mysql/alexandrie.toml | 3 +++ docker/postgres/alexandrie.toml | 3 +++ docker/sqlite/alexandrie.toml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/docker/mysql/alexandrie.toml b/docker/mysql/alexandrie.toml index af326f60..af17c303 100644 --- a/docker/mysql/alexandrie.toml +++ b/docker/mysql/alexandrie.toml @@ -53,3 +53,6 @@ path = "syntect/dumps/syntaxes.dump" type = "dump" path = "syntect/dumps/themes.dump" theme_name = "frontier-contrast" + +[search] +directory = "appdata/tantivy" \ No newline at end of file diff --git a/docker/postgres/alexandrie.toml b/docker/postgres/alexandrie.toml index 44d98ad1..3a6d39c1 100644 --- a/docker/postgres/alexandrie.toml +++ b/docker/postgres/alexandrie.toml @@ -51,3 +51,6 @@ path = "syntect/dumps/syntaxes.dump" type = "dump" path = "syntect/dumps/themes.dump" theme_name = "frontier-contrast" + +[search] +directory = "appdata/tantivy" \ No newline at end of file diff --git a/docker/sqlite/alexandrie.toml b/docker/sqlite/alexandrie.toml index 1c82fe19..a20661b7 100644 --- a/docker/sqlite/alexandrie.toml +++ b/docker/sqlite/alexandrie.toml @@ -51,3 +51,6 @@ path = "syntect/dumps/syntaxes.dump" type = "dump" path = "syntect/dumps/themes.dump" theme_name = "frontier-contrast" + +[search] +directory = "appdata/tantivy" \ No newline at end of file From 494ecb6ae46fe3e30d6f1966b0b8e7a092ede87f Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Sun, 11 Jun 2023 15:07:30 +0200 Subject: [PATCH 12/22] Tantivy 0.20 --- Cargo.lock | 944 ++++++++++++++------------ crates/alexandrie/Cargo.toml | 4 +- crates/alexandrie/src/fts/document.rs | 69 +- crates/alexandrie/src/fts/index.rs | 47 +- 4 files changed, 560 insertions(+), 504 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 738f2f70..f1995bd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,20 +64,20 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.6" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ - "getrandom 0.2.9", + "cfg-if 1.0.0", "once_cell", "version_check", ] [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] @@ -122,8 +122,8 @@ dependencies = [ "tar", "thiserror", "tide", - "time 0.3.20", - "toml 0.7.3", + "time 0.3.22", + "toml", "url", ] @@ -173,6 +173,12 @@ dependencies = [ "url", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -184,9 +190,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e579a7752471abc2a8268df8b20005e3eadd975f585398f17efcfd8d4927371" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" dependencies = [ "anstyle", "anstyle-parse", @@ -223,9 +229,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcd8291a340dd8ac70e18878bc4501dd7b4ff970cfa21c207d36ece51ea88fd" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -233,9 +239,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" [[package]] name = "arc-swap" @@ -483,7 +489,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -529,15 +535,15 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "bigdecimal" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aaf33151a6429fe9211d1b276eafdf70cdff28b071e76c0b0e1503221ea3744" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" dependencies = [ "num-bigint", "num-integer", @@ -560,6 +566,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6776fc96284a0bb647b615056fc496d1fe1644a7ab01829818a6d91cae888b84" + [[package]] name = "bitpacking" version = "0.8.4" @@ -630,9 +642,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byteorder" @@ -687,13 +699,13 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.24" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", "serde", "time 0.1.45", @@ -712,41 +724,31 @@ dependencies = [ [[package]] name = "clap" -version = "4.2.2" +version = "4.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b802d85aaf3a1cdb02b224ba472ebdea62014fccfcb269b95a4d76443b5ee5a" +checksum = "ca8f255e4b8027970e78db75e78831229c9815fdbfa67eb1a1b777a62e24b4a0" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.2.2" +version = "4.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14a1a858f532119338887a4b8e1af9c60de8249cd7bafd68036a489e261e37b6" +checksum = "acd4f3c17c83b0ba34ffbc4f8bbd74f079413f747f84a6f89292f138057e36ab" dependencies = [ "anstream", "anstyle", - "bitflags", + "bitflags 1.3.2", "clap_lex", "strsim", ] [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" - -[[package]] -name = "codespan-reporting" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] name = "colorchoice" @@ -819,9 +821,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" dependencies = [ "libc", ] @@ -930,16 +932,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "ctor" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" -dependencies = [ - "quote", - "syn 1.0.109", -] - [[package]] name = "ctr" version = "0.6.0" @@ -949,50 +941,6 @@ dependencies = [ "cipher", ] -[[package]] -name = "cxx" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.15", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - [[package]] name = "darling" version = "0.14.4" @@ -1061,12 +1009,12 @@ dependencies = [ [[package]] name = "diesel" -version = "2.0.3" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4391a22b19c916e50bec4d6140f29bdda3e3bb187223fe6e3ea0b6e4d1021c04" +checksum = "f7a532c1f99a0f596f6960a60d1e119e91582b24b39e2d83a190e61262c3ef0c" dependencies = [ "bigdecimal", - "bitflags", + "bitflags 2.3.1", "byteorder", "chrono", "diesel_derives", @@ -1079,32 +1027,42 @@ dependencies = [ "percent-encoding", "pq-sys", "r2d2", + "time 0.3.22", "url", ] [[package]] name = "diesel_derives" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad74fdcf086be3d4fdd142f67937678fe60ed431c3b2f08599e7687269410c4" +checksum = "74398b79d81e52e130d991afeed9c86034bb1b7735f46d2f5bf7deb261d80303" dependencies = [ - "proc-macro-error", + "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] name = "diesel_migrations" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9ae22beef5e9d6fab9225ddb073c1c6c1a7a6ded5019d5da11d1e5c5adc34e2" +checksum = "6036b3f0120c5961381b570ee20a02432d7e2d27ea60de9578799cf9156914ac" dependencies = [ "diesel", "migrations_internals", "migrations_macros", ] +[[package]] +name = "diesel_table_macro_syntax" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" +dependencies = [ + "syn 2.0.18", +] + [[package]] name = "digest" version = "0.9.0" @@ -1116,9 +1074,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common", @@ -1172,6 +1130,15 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "erased-serde" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2b0c2380453a92ea8b6c8e5f64ecaafccddde8ceab55ff7a8ac1029f894569" +dependencies = [ + "serde", +] + [[package]] name = "errno" version = "0.3.1" @@ -1216,20 +1183,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25c7df09945d65ea8d70b3321547ed414bbc540aad5bac6883d021b970f35b04" -[[package]] -name = "fastfield_codecs" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374a3a53c1bd5fb31b10084229290eafb0a05f260ec90f1f726afffda4877a8a" -dependencies = [ - "fastdivide", - "itertools", - "log", - "ownedbytes", - "tantivy-bitpacker", - "tantivy-common", -] - [[package]] name = "fastrand" version = "1.9.0" @@ -1253,9 +1206,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", "miniz_oxide", @@ -1284,21 +1237,21 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] [[package]] -name = "fs2" -version = "0.4.3" +name = "fs4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +checksum = "7672706608ecb74ab2e055c68327ffc25ae4cac1e12349204fd5fb0f3487cce2" dependencies = [ - "libc", - "winapi", + "rustix", + "windows-sys 0.48.0", ] [[package]] @@ -1388,7 +1341,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -1466,9 +1419,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -1493,7 +1446,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf7f68c2995f392c49fffb4f95ae2c873297830eb25c6bc4c114ce8f4562acc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", "libgit2-sys", "log", @@ -1516,9 +1469,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" dependencies = [ "bytes", "fnv", @@ -1535,9 +1488,9 @@ dependencies = [ [[package]] name = "handlebars" -version = "4.3.6" +version = "4.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "035ef95d03713f2c347a72547b7cd38cbc9af7cd51e6099fb62d586d4a6dee3a" +checksum = "83c3372087601b532857d332f5957cbae686da52bb7810bf038c3e3c3cc2fa0d" dependencies = [ "log", "pest", @@ -1553,6 +1506,12 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ "ahash", ] @@ -1741,9 +1700,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.23.2" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" dependencies = [ "http", "hyper", @@ -1767,9 +1726,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1781,12 +1740,11 @@ dependencies = [ [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] [[package]] @@ -1797,9 +1755,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1812,7 +1770,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", ] [[package]] @@ -1835,9 +1793,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.1", "libc", @@ -1879,9 +1837,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] @@ -1909,9 +1867,9 @@ checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" [[package]] name = "libc" -version = "0.2.141" +version = "0.2.146" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" [[package]] name = "libgit2-sys" @@ -1929,9 +1887,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.25.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" +checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" dependencies = [ "pkg-config", "vcpkg", @@ -1953,9 +1911,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" dependencies = [ "cc", "libc", @@ -1972,15 +1930,6 @@ dependencies = [ "safemem", ] -[[package]] -name = "link-cplusplus" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] - [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1989,15 +1938,15 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -2005,11 +1954,10 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" dependencies = [ - "cfg-if 1.0.0", "value-bag", ] @@ -2029,18 +1977,18 @@ dependencies = [ [[package]] name = "lru" -version = "0.7.8" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +checksum = "03f1160296536f10c833a82dca22267d5486734230d47bf00bf435885814ba1e" dependencies = [ - "hashbrown", + "hashbrown 0.13.2", ] [[package]] name = "lz4_flex" -version = "0.9.5" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a8cbbb2831780bc3b9c15a41f5b49222ef756b6730a95f3decfdd15903eb5a3" +checksum = "8b8c72594ac26bfd34f2d99dfced2edfaddfe8a476e3ff2ca0eb293d925c4f83" [[package]] name = "mac" @@ -2106,9 +2054,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" -version = "0.5.10" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" dependencies = [ "libc", ] @@ -2124,19 +2072,19 @@ dependencies = [ [[package]] name = "migrations_internals" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c493c09323068c01e54c685f7da41a9ccf9219735c3766fbfd6099806ea08fbc" +checksum = "0f23f71580015254b020e856feac3df5878c2c7a8812297edd6c0a485ac9dada" dependencies = [ "serde", - "toml 0.5.11", + "toml", ] [[package]] name = "migrations_macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8ff27a350511de30cdabb77147501c36ef02e0451d957abea2f30caffb2b58" +checksum = "cce3325ac70e67bbab5bd837a31cae01f1a6db64e0e744a33cb03a543469ef08" dependencies = [ "migrations_internals", "proc-macro2", @@ -2151,33 +2099,29 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "murmurhash32" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d736ff882f0e85fe9689fb23db229616c4c00aee2b3ac282f666d8f20eb25d4a" -dependencies = [ - "byteorder", -] +checksum = "d9380db4c04d219ac5c51d14996bbf2c2e9a15229771b53f8671eb6c83cf44df" [[package]] name = "mysqlclient-sys" @@ -2284,13 +2228,13 @@ dependencies = [ [[package]] name = "oauth2" -version = "4.3.0" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeaf26a72311c087f8c5ba617c96fac67a5c04f430e716ac8d8ab2de62e23368" +checksum = "09a6e2a2b13a56ebeabba9142f911745be6456163fd6c3d361274ebcd891a80c" dependencies = [ "base64 0.13.1", "chrono", - "getrandom 0.2.9", + "getrandom 0.2.10", "http", "rand 0.8.5", "reqwest", @@ -2304,9 +2248,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "oneshot" @@ -2323,7 +2267,7 @@ version = "6.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", "once_cell", "onig_sys", @@ -2351,7 +2295,7 @@ version = "0.10.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if 1.0.0", "foreign-types", "libc", @@ -2368,7 +2312,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -2397,9 +2341,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "ownedbytes" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e957eaa64a299f39755416e5b3128c505e9d63a91d0453771ad2ccd3907f8db" +checksum = "c718e498b20704d5fb5d51d07f414a22f61c19254c1708e117b93fd76860739c" dependencies = [ "stable_deref_trait", ] @@ -2422,28 +2366,28 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.3.5", "smallvec", - "windows-sys 0.45.0", + "windows-targets", ] [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.5.7" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1403e8401ad5dedea73c626b99758535b342502f8d1e361f4a2dd952749122" +checksum = "e68e84bfb01f0507134eac1e9b410a12ba379d064eab48c50ba4ce329a527b70" dependencies = [ "thiserror", "ucd-trie", @@ -2451,9 +2395,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.5.7" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be99c4c1d2fc2769b1d00239431d711d08f6efedcecb8b6e30707160aee99c15" +checksum = "6b79d4c71c865a25a4322296122e3924d30bc8ee0834c8bfc8b95f7f054afbfb" dependencies = [ "pest", "pest_generator", @@ -2461,22 +2405,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.5.7" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56094789873daa36164de2e822b3888c6ae4b4f9da555a1103587658c805b1e" +checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] name = "pest_meta" -version = "2.5.7" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6733073c7cff3d8459fda0e42f13a047870242aed8b509fe98000928975f359e" +checksum = "745a452f8eb71e39ffd8ee32b3c5f51d03845f99786fa9b68db6ff509c505411" dependencies = [ "once_cell", "pest", @@ -2523,22 +2467,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] @@ -2561,9 +2505,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "plist" @@ -2571,22 +2515,22 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bd9647b268a3d3e14ff09c23201133a62589c658db02bb7388c7246aafe0590" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", "indexmap", "line-wrap", "quick-xml", "serde", - "time 0.3.20", + "time 0.3.22", ] [[package]] name = "polling" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be1c66a6add46bff50935c313dae30a5030cf8385c5206e8a95e9e9def974aa" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", - "bitflags", + "bitflags 1.3.2", "cfg-if 1.0.0", "concurrent-queue", "libc", @@ -2614,9 +2558,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "pq-sys" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b845d6d8ec554f972a2c5298aad68953fd64e7441e846075450b44656a016d1" +checksum = "31c0052426df997c0cbd30789eb44ca097e3541717a7b8fa36b1c464ee7edebd" dependencies = [ "vcpkg", ] @@ -2627,30 +2571,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro-hack" version = "0.5.20+deprecated" @@ -2659,20 +2579,20 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" dependencies = [ "unicode-ident", ] [[package]] name = "pulldown-cmark" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d9cc634bc78768157b5cbfe988ffcd1dcba95cd2b2f03a88316c08c6d00ed63" +checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" dependencies = [ - "bitflags", + "bitflags 1.3.2", "getopts", "memchr", "unicase", @@ -2689,9 +2609,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -2766,7 +2686,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.9", + "getrandom 0.2.10", ] [[package]] @@ -2806,7 +2726,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2815,7 +2735,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2824,20 +2744,20 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.9", + "getrandom 0.2.10", "redox_syscall 0.2.16", "thiserror", ] [[package]] name = "regex" -version = "1.7.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.2", ] [[package]] @@ -2846,7 +2766,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] @@ -2855,13 +2775,19 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" + [[package]] name = "reqwest" -version = "0.11.16" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", "bytes", "encoding_rs", "futures-core", @@ -3036,11 +2962,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.11" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -3050,14 +2976,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.8" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" dependencies = [ "log", "ring", + "rustls-webpki", "sct", - "webpki", ] [[package]] @@ -3066,7 +2992,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", ] [[package]] @@ -3126,12 +3062,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "scratch" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" - [[package]] name = "sct" version = "0.7.0" @@ -3144,11 +3074,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.8.2" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -3157,9 +3087,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" dependencies = [ "core-foundation-sys", "libc", @@ -3191,22 +3121,31 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", +] + +[[package]] +name = "serde_fmt" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d4ddca14104cd60529e8c7f7ba71a2c8acd8f7f5cfcdc2faf97eeb7c3010a4" +dependencies = [ + "serde", ] [[package]] @@ -3242,9 +3181,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +checksum = "93107647184f6027e3b7dcb2e11034cf95ffa1e3a682c67951963ac69c1c007d" dependencies = [ "serde", ] @@ -3297,7 +3236,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.6", + "digest 0.10.7", ] [[package]] @@ -3340,6 +3279,15 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +[[package]] +name = "sketches-ddsketch" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" +dependencies = [ + "serde", +] + [[package]] name = "slab" version = "0.4.8" @@ -3399,7 +3347,7 @@ dependencies = [ "slog", "term", "thread_local", - "time 0.3.20", + "time 0.3.22", ] [[package]] @@ -3528,9 +3476,71 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "sval" -version = "1.0.0-alpha.5" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45f6ee7c7b87caf59549e9fe45d6a69c75c8019e79e212a835c5da0e92f0ba08" +checksum = "0e6aa16ce8d9e472e21a528a52c875a76a49190f3968f2ec7e9b550ccc28b410" + +[[package]] +name = "sval_buffer" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "905c4373621186ee9637464b0aaa026389ea9e7f841f2225f160a32ba5d5bac4" +dependencies = [ + "sval", + "sval_ref", +] + +[[package]] +name = "sval_dynamic" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad6b4988322c5f22859a6a7649fa1249aa3dd01514caf8ed57d83735f997bb8b" +dependencies = [ + "sval", +] + +[[package]] +name = "sval_fmt" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d3ccd10346f925c2fbd97b75e8573b38e34431bfba04cc531cd23aad0fbabb8" +dependencies = [ + "itoa", + "ryu", + "sval", +] + +[[package]] +name = "sval_json" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a12e1488defd6344e23243c17ea4a1b185c547968749e8a281373fde0bde2d5" +dependencies = [ + "itoa", + "ryu", + "sval", +] + +[[package]] +name = "sval_ref" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b797fc4b284dd0e45f7ec424479e604ea5be9bb191a1ef4e96c20c7685649938" +dependencies = [ + "sval", +] + +[[package]] +name = "sval_serde" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "810fa9a837e67a23e0efa7536250fc4d24043306cc1efd076f1943ba2fc2e62d" +dependencies = [ + "serde", + "sval", + "sval_buffer", + "sval_fmt", +] [[package]] name = "syn" @@ -3545,9 +3555,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.15" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ "proc-macro2", "quote", @@ -3561,14 +3571,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6c454c27d9d7d9a84c7803aaa3c50cd088d2906fe3c6e42da3209aa623576a8" dependencies = [ "bincode", - "bitflags", + "bitflags 1.3.2", "flate2", "fnv", "lazy_static", "once_cell", "onig", "plist", - "regex-syntax", + "regex-syntax 0.6.29", "serde", "serde_derive", "serde_json", @@ -3592,14 +3602,14 @@ checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" [[package]] name = "tantivy" -version = "0.19.2" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb26a6b22c84d8be41d99a14016d6f04d30d8d31a2ea411a8ab553af5cc490d" +checksum = "e71e08fe5b8242d2d6b8d047ed18c7df0db79d2162c8be29945f7c15cce26a8e" dependencies = [ "aho-corasick", "arc-swap", "async-trait", - "base64 0.13.1", + "base64 0.21.2", "bitpacking", "byteorder", "census", @@ -3608,8 +3618,7 @@ dependencies = [ "downcast-rs", "fail", "fastdivide", - "fastfield_codecs", - "fs2", + "fs4", "htmlescape", "itertools", "levenshtein_automata", @@ -3622,52 +3631,78 @@ dependencies = [ "num_cpus", "once_cell", "oneshot", - "ownedbytes", "rayon", "regex", "rust-stemmers", "rustc-hash", "serde", "serde_json", + "sketches-ddsketch", "smallvec", - "stable_deref_trait", "tantivy-bitpacker", + "tantivy-columnar", "tantivy-common", "tantivy-fst", "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", "tempfile", "thiserror", - "time 0.3.20", + "time 0.3.22", "uuid", "winapi", ] [[package]] name = "tantivy-analysis-contrib" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4240ab300b15d81dca023ad3e27b7b6b45522340594634f0c7e1dfbaec82c53b" +checksum = "f1b2d2ea57f12d9c7a1a33ec5420555a4adb70b8989ba2480dc9dc00c82adfba" dependencies = [ "derive_builder", "either", "fst", + "rustc-hash", "tantivy", + "thiserror", ] [[package]] name = "tantivy-bitpacker" -version = "0.3.0" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16099e96f0ede682084469b80d6909dc170aa2b11d2a45538b5b36b2a90090b9" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e71a0c95b82d4292b097a09b989a6380d28c3a86800c841a2d03bae1fc8b9fa6" +checksum = "56e32b024b26eab93eb8648faf08004356bf9d47376557ee4409f4b210163656" +dependencies = [ + "fastdivide", + "fnv", + "itertools", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] [[package]] name = "tantivy-common" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14fef4182bb60df9a4b92cd8ecab39ba2e50a05542934af17eef1f49660705cb" +checksum = "e7d12fdd6ec0f7e0962f129c03c696a85ec567734950cbb2b89af4a293ce342f" dependencies = [ + "async-trait", "byteorder", "ownedbytes", + "serde", + "time 0.3.22", ] [[package]] @@ -3677,21 +3712,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc3c506b1a8443a3a65352df6382a1fb6a7afe1a02e871cee0d25e2c3d5f3944" dependencies = [ "byteorder", - "regex-syntax", + "regex-syntax 0.6.29", "utf8-ranges", ] [[package]] name = "tantivy-query-grammar" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343e3ada4c1c480953f6960f8a21ce9c76611480ffdd4f4e230fdddce0fc5331" +checksum = "106d8f78ad1da4f0fdd526a0760c326c0573510d4dedabeb1962d35a35879797" dependencies = [ "combine", "once_cell", "regex", ] +[[package]] +name = "tantivy-sstable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda34243d3ee64bd8f9ba74a3b0d05f4d07beff7767a727212e9b5a19c13dde7" +dependencies = [ + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b9e9470301b026ad3b95f79a791a2a3ee81f3ab16fbe412a9dd81ff834acf5" +dependencies = [ + "murmurhash32", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bee3519354cbe6bfff4bf6be9cf3e2dfcd0a6ea748a42312fbf1242c0d66fc6" +dependencies = [ + "serde", +] + [[package]] name = "tar" version = "0.4.38" @@ -3705,15 +3770,16 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.5.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ + "autocfg", "cfg-if 1.0.0", "fastrand", "redox_syscall 0.3.5", "rustix", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -3738,15 +3804,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] - [[package]] name = "thiserror" version = "1.0.40" @@ -3764,7 +3821,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -3827,23 +3884,23 @@ dependencies = [ [[package]] name = "time" -version = "0.3.20" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" dependencies = [ "itoa", "libc", "num_threads", "serde", "time-core", - "time-macros 0.2.8", + "time-macros 0.2.9", ] [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" @@ -3857,9 +3914,9 @@ dependencies = [ [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" dependencies = [ "time-core", ] @@ -3894,9 +3951,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.27.0" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", "bytes", @@ -3907,18 +3964,18 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -3933,20 +3990,19 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.23.4" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", "tokio", - "webpki", ] [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", @@ -3958,18 +4014,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "toml" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" +checksum = "d6135d499e69981f9ff0ef2167955a5333c35e36f6937d382974566b3d5b94ec" dependencies = [ "serde", "serde_spanned", @@ -3979,18 +4026,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.8" +version = "0.19.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" +checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" dependencies = [ "indexmap", "serde", @@ -4025,14 +4072,14 @@ checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -4102,9 +4149,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-normalization" @@ -4139,9 +4186,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", "idna", @@ -4169,11 +4216,11 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.3.2" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" +checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" dependencies = [ - "getrandom 0.2.9", + "getrandom 0.2.10", "serde", ] @@ -4185,13 +4232,38 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.0.0-alpha.9" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4d330786735ea358f3bc09eea4caa098569c1c93f342d9aca0514915022fe7e" +dependencies = [ + "value-bag-serde1", + "value-bag-sval2", +] + +[[package]] +name = "value-bag-serde1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" +checksum = "4735c95b4cca1447b448e2e2e87e98d7e7498f4da27e355cf7af02204521001d" +dependencies = [ + "erased-serde", + "serde", + "serde_fmt", +] + +[[package]] +name = "value-bag-sval2" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859cb4f0ce7da6a118b559ba74b0e63bf569bea867c20ba457a6b1c886a04e97" dependencies = [ - "ctor", "sval", - "version_check", + "sval_buffer", + "sval_dynamic", + "sval_fmt", + "sval_json", + "sval_ref", + "sval_serde", ] [[package]] @@ -4252,9 +4324,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -4262,24 +4334,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -4289,9 +4361,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4299,28 +4371,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" dependencies = [ "js-sys", "wasm-bindgen", @@ -4382,7 +4454,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets 0.48.0", + "windows-targets", ] [[package]] @@ -4400,37 +4472,13 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-targets", ] [[package]] @@ -4534,9 +4582,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.1" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8970b36c66498d8ff1d66685dc86b91b29db0c7739899012f63a63814b4b28" +checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" dependencies = [ "memchr", ] @@ -4579,3 +4627,33 @@ name = "zeroize" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" + +[[package]] +name = "zstd" +version = "0.12.3+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "6.0.5+zstd.1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.8+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +dependencies = [ + "cc", + "libc", + "pkg-config", +] diff --git a/crates/alexandrie/Cargo.toml b/crates/alexandrie/Cargo.toml index 9c500f5d..7e6dd783 100644 --- a/crates/alexandrie/Cargo.toml +++ b/crates/alexandrie/Cargo.toml @@ -51,8 +51,8 @@ diesel = { version = "2.0.3", features = ["r2d2", "chrono"] } diesel_migrations = "2.0.0" # Text indexation and search -tantivy = "0.19" -tantivy-analysis-contrib = { version = "0.8", default-features = false, features = ["commons"] } +tantivy = "0.20" +tantivy-analysis-contrib = { version = "0.9", default-features = false, features = ["commons"] } # async primitives async-std = { version = "1.12.0", features = ["attributes", "tokio1"] } diff --git a/crates/alexandrie/src/fts/document.rs b/crates/alexandrie/src/fts/document.rs index 9b4149b1..6617e478 100644 --- a/crates/alexandrie/src/fts/document.rs +++ b/crates/alexandrie/src/fts/document.rs @@ -78,62 +78,39 @@ impl TantivyDocument { // tuple and Document are in another crate :-( let mut document = Document::new(); - let id_field = schema.get_field(super::ID_FIELD_NAME); - let name_field = schema.get_field(super::NAME_FIELD_NAME); - let name_full_field = schema.get_field(super::NAME_FIELD_NAME_FULL); - let name_prefix_field = schema.get_field(super::NAME_FIELD_PREFIX_NAME); - let description_field = schema.get_field(super::DESCRIPTION_FIELD_NAME); - let readme_field = schema.get_field(super::README_FIELD_NAME); - let category_field = schema.get_field(super::CATEGORY_FIELD_NAME); - let keyword_field = schema.get_field(super::KEYWORD_FIELD_NAME); - - // None of the fields should be `None`. - // But we check that anyway. - if id_field.is_none() { - return Err(Error::MissingField(super::ID_FIELD_NAME)); - } - if name_field.is_none() { - return Err(Error::MissingField(super::NAME_FIELD_NAME)); - } - if name_full_field.is_none() { - return Err(Error::MissingField(super::NAME_FIELD_NAME_FULL)); - } - if name_prefix_field.is_none() { - return Err(Error::MissingField(super::NAME_FIELD_PREFIX_NAME)); - } - - document.add_i64(id_field.unwrap(), self.id); - document.add_text(name_field.unwrap(), &self.name); - document.add_text(name_full_field.unwrap(), &self.name); - document.add_text(name_prefix_field.unwrap(), self.name); + let id_field = schema.get_field(super::ID_FIELD_NAME)?; + let name_field = schema.get_field(super::NAME_FIELD_NAME)?; + let name_full_field = schema.get_field(super::NAME_FIELD_NAME_FULL)?; + let name_prefix_field = schema.get_field(super::NAME_FIELD_PREFIX_NAME)?; + let category_field = schema.get_field(super::CATEGORY_FIELD_NAME)?; + let keyword_field = schema.get_field(super::KEYWORD_FIELD_NAME)?; + + document.add_i64(id_field, self.id); + document.add_text(name_field, &self.name); + document.add_text(name_full_field, &self.name); + document.add_text(name_prefix_field, self.name); // For the following fields we will not fail if they are not in schema // but TODO add warn if let Some(description) = &self.description { - if let Some(field) = description_field { - document.add_text(field, description) - } + let description_field = schema.get_field(super::DESCRIPTION_FIELD_NAME)?; + document.add_text(description_field, description); } if let Some(readme) = &self.readme { - if let Some(field) = readme_field { - document.add_text(field, readme) - } + let readme_field = schema.get_field(super::README_FIELD_NAME)?; + document.add_text(readme_field, readme); } - if let Some(field) = keyword_field { - self.keywords - .clone() - .into_iter() - .for_each(|v| document.add_text(field, v)) - } + self.keywords + .clone() + .into_iter() + .for_each(|v| document.add_text(keyword_field, v)); - if let Some(field) = category_field { - self.categories - .clone() - .into_iter() - .for_each(|v| document.add_text(field, v)) - } + self.categories + .clone() + .into_iter() + .for_each(|v| document.add_text(category_field, v)); Ok(document) } diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index b9b204e3..af128ab0 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -3,7 +3,7 @@ use std::num::NonZeroUsize; use std::sync::RwLock; use diesel::prelude::*; -use log::{debug, error, info, warn}; +use log::{debug, info, warn}; use tantivy::collector::{Count, TopDocs}; use tantivy::directory::MmapDirectory; use tantivy::query::QueryParser; @@ -107,15 +107,19 @@ impl TryFrom for Tantivy { // A filter that lowercase words let stop_words = StopWordFilter::new(Language::English).ok_or(Self::Error::EmptyStopWord)?; - let analyzer = TextAnalyzer::from(SimpleTokenizer) + let analyzer = TextAnalyzer::builder(SimpleTokenizer::default()) .filter(stop_words) - .filter(LowerCaser); + .filter(LowerCaser) + .build(); - let analyzer_full = TextAnalyzer::from(RawTokenizer).filter(LowerCaser); + let analyzer_full = TextAnalyzer::builder(RawTokenizer::default()) + .filter(LowerCaser) + .build(); - let analyzer_prefix = TextAnalyzer::from(SimpleTokenizer) + let analyzer_prefix = TextAnalyzer::builder(SimpleTokenizer::default()) .filter(LowerCaser) - .filter(EdgeNgramTokenFilter::new(NonZeroUsize::new(1).unwrap(), None, false).unwrap()); + .filter(EdgeNgramTokenFilter::new(NonZeroUsize::new(1).unwrap(), None, false).unwrap()) + .build(); let index = TantivyIndex::open_or_create(directory, schema.clone())?; // Register analyzer @@ -135,7 +139,9 @@ impl TryFrom for Tantivy { search_tokenizer_manager.register(analyzer_name_full, analyzer_full); search_tokenizer_manager.register( analyzer_prefix_name, - TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser), + TextAnalyzer::builder(SimpleTokenizer::default()) + .filter(LowerCaser) + .build(), ); // Get an index writer with 50MB of heap @@ -165,17 +171,14 @@ impl Tantivy { pub fn create_or_update(&self, document: TantivyDocument) -> Result<(), Error> { let id = document.id(); let document = document.try_into(&self.schema)?; - if let Some(field) = self.schema.get_field(super::ID_FIELD_NAME) { - let term = Term::from_field_i64(field, id); - let index_writer = self - .index_writer - .read() - .map_err(|error| Error::PoisonedError(error.to_string()))?; - index_writer.delete_term(term); - index_writer.add_document(document)?; - } else { - error!("There is no field {} in schema", super::ID_FIELD_NAME); - } + let field = self.schema.get_field(super::ID_FIELD_NAME)?; + let term = Term::from_field_i64(field, id); + let index_writer = self + .index_writer + .read() + .map_err(|error| Error::PoisonedError(error.to_string()))?; + index_writer.delete_term(term); + index_writer.add_document(document)?; Ok(()) } @@ -239,8 +242,7 @@ impl Tantivy { } x.cloned() }) - .map(|v| v.and_then(|i| i.as_text().map(|t| t.to_owned()))) - .flatten() + .filter_map(|v| v.and_then(|i| i.as_text().map(|t| t.to_owned()))) .collect(); Ok(results) @@ -293,7 +295,7 @@ impl Tantivy { let results = results .into_iter() - .map(|(score, doc_address)| { + .filter_map(|(score, doc_address)| { let retrieve_doc = match searcher.doc(doc_address) { Ok(retrieve_doc) => retrieve_doc, Err(error) => { @@ -314,7 +316,6 @@ impl Tantivy { None } }) - .flatten() .collect(); Ok((count, results)) @@ -375,7 +376,7 @@ impl Tantivy { let result = result?; if let Some((krates, keywords, categories)) = result { - start = start + krates.len() as i64; + start += krates.len() as i64; let mut keywords_iterator = keywords.into_iter().peekable(); let mut categories_iterator = categories.into_iter().peekable(); From c6ec4f4232832b05192c8cc89851c264eb88262c Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Sun, 11 Jun 2023 16:17:34 +0200 Subject: [PATCH 13/22] Use tantivy on search endpoint --- crates/alexandrie/src/api/crates/search.rs | 64 +++++++------------ crates/alexandrie/src/fts/index.rs | 8 ++- .../src/programmatic-api/crates/search/get.md | 4 +- 3 files changed, 29 insertions(+), 47 deletions(-) diff --git a/crates/alexandrie/src/api/crates/search.rs b/crates/alexandrie/src/api/crates/search.rs index 6c988b48..ce06bced 100644 --- a/crates/alexandrie/src/api/crates/search.rs +++ b/crates/alexandrie/src/api/crates/search.rs @@ -1,6 +1,5 @@ -use std::num::NonZeroU32; +use std::num::NonZeroUsize; -use diesel::dsl as sql; use diesel::prelude::*; use semver::Version; use serde::{Deserialize, Serialize}; @@ -15,6 +14,10 @@ use crate::error::{AlexError, Error}; use crate::utils; use crate::State; +/// Default number of result per page +/// Perhaps should make this configurable in toml. +const DEFAULT_PER_PAGE: usize = 15; + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct SearchResponse { pub crates: Vec, @@ -35,14 +38,14 @@ struct SearchResult { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct SearchMeta { - pub total: i64, + pub total: usize, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct QueryParams { pub q: String, - pub per_page: Option, - pub page: Option, + pub per_page: Option, + pub page: Option, } /// Route to search through crates (used by `cargo search`). @@ -53,50 +56,27 @@ pub(crate) async fn get(req: Request) -> tide::Result { missing_params: &["q"], })?; let state = req.state().clone(); - let db = &state.db; - - //? Fetch the latest index changes. - // state.index.refresh()?; - let name = utils::canonical_name(params.q.as_str()); + let query = params.q; + let per_page = params.per_page.map(|v| v.get()).unwrap_or(DEFAULT_PER_PAGE); + let page = params.page.map(|v| v.get()).unwrap_or(1) - 1; - //? Build the search pattern. - let name_pattern = format!("%{0}%", name.replace('\\', "\\\\").replace('%', "\\%")); + let searcher = &state.search; + // Run query on tantivy and get total and matching ids + // Perhaps should use suggest method as it allow to deal with "starts with", but I don't think + // that's what is expected. + let (total, ids) = searcher.search(&query, page * per_page, per_page)?; + let db = &state.db; let transaction = db.transaction(move |conn| { let state = req.state(); - //? Limit the result count depending on parameters. - let results = match (params.per_page, params.page) { - (Some(per_page), Some(page)) => { - //? Get search results for the given page number and entries per page. - crates::table - .filter(crates::canon_name.like(name_pattern.as_str())) - .limit(i64::from(per_page.get())) - .offset(i64::from((page.get() - 1) * per_page.get())) - .load::(conn)? - } - (Some(per_page), None) => { - //? Get the first page of search results with the given entries per page. - crates::table - .filter(crates::canon_name.like(name_pattern.as_str())) - .limit(i64::from(per_page.get())) - .load::(conn)? - } - _ => { - //? Get ALL the crates (might be too much, tbh). - crates::table - .filter(crates::canon_name.like(name_pattern.as_str())) - .load::(conn)? - } - }; - - //? Fetch the total result count. - let total = crates::table - .select(sql::count(crates::id)) - .filter(crates::canon_name.like(name_pattern.as_str())) - .first::(conn)?; + // Get crate from database + let results = crates::table + .filter(crates::id.eq_any(ids)) + .load::(conn)?; + // Fetch missing informations from index let crates = results .into_iter() .map(|krate| { diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index af128ab0..cd2c5562 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -249,12 +249,14 @@ impl Tantivy { } /// Search documents. Return document count & database IDs. - pub fn search( + pub fn search>( &self, - query: String, + query: Q, offset: usize, limit: usize, ) -> Result<(usize, Vec), TantivyError> { + let query = query.as_ref(); + let searcher = self.index_reader.searcher(); let id = self.schema.get_field(super::ID_FIELD_NAME).unwrap(); @@ -284,7 +286,7 @@ impl Tantivy { query_parser.set_field_boost(description, 0.2); query_parser.set_field_boost(readme, 0.2); - let query = query_parser.parse_query(&query)?; + let query = query_parser.parse_query(query)?; info!("Query offset={} query limit={}", offset, limit); diff --git a/docs/src/programmatic-api/crates/search/get.md b/docs/src/programmatic-api/crates/search/get.md index 82e59e3b..f6479616 100644 --- a/docs/src/programmatic-api/crates/search/get.md +++ b/docs/src/programmatic-api/crates/search/get.md @@ -15,8 +15,8 @@ HTTP Query Parameters This endpoint accepts the following query parameters: - **(required)** `q`: The query string for the search (like `serde json` to possibly find `serde_json`). -- `page`: The non-zero page number to retrive (defaults to `1`), ignored if `per_page` is missing. -- `per_page`: The non-zero number of results per page (infinite by default). +- `page`: The non-zero page number to retrive (defaults to `1`). +- `per_page`: The non-zero number of results per page (default to `15`). Responses --------- From 58db05ae968ac2fba6a0659d7c9c243d537dea49 Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Sun, 11 Jun 2023 16:35:30 +0200 Subject: [PATCH 14/22] Improvement --- crates/alexandrie/src/api/crates/search.rs | 6 +----- crates/alexandrie/src/frontend/search.rs | 18 ++++++++---------- crates/alexandrie/src/fts/mod.rs | 5 +++++ crates/alexandrie/src/main.rs | 4 +++- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/crates/alexandrie/src/api/crates/search.rs b/crates/alexandrie/src/api/crates/search.rs index ce06bced..effd541f 100644 --- a/crates/alexandrie/src/api/crates/search.rs +++ b/crates/alexandrie/src/api/crates/search.rs @@ -14,10 +14,6 @@ use crate::error::{AlexError, Error}; use crate::utils; use crate::State; -/// Default number of result per page -/// Perhaps should make this configurable in toml. -const DEFAULT_PER_PAGE: usize = 15; - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct SearchResponse { pub crates: Vec, @@ -58,7 +54,7 @@ pub(crate) async fn get(req: Request) -> tide::Result { let state = req.state().clone(); let query = params.q; - let per_page = params.per_page.map(|v| v.get()).unwrap_or(DEFAULT_PER_PAGE); + let per_page = params.per_page.map(|v| v.get()).unwrap_or(crate::fts::DEFAULT_RESULT_PER_PAGE); let page = params.page.map(|v| v.get()).unwrap_or(1) - 1; let searcher = &state.search; diff --git a/crates/alexandrie/src/frontend/search.rs b/crates/alexandrie/src/frontend/search.rs index 5dd4683c..642de0f1 100644 --- a/crates/alexandrie/src/frontend/search.rs +++ b/crates/alexandrie/src/frontend/search.rs @@ -1,4 +1,4 @@ -use std::num::NonZeroU32; +use std::num::NonZeroUsize; use diesel::prelude::*; use json::json; @@ -16,12 +16,10 @@ use crate::utils; use crate::utils::auth::AuthExt; use crate::State; -const RESULT_PER_PAGE: i64 = 15; - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct SearchParams { pub q: String, - pub page: Option, + pub page: Option, } /// Route to search through crates (used by `cargo search`) using tantivy index @@ -30,7 +28,7 @@ pub(crate) async fn get(req: Request) -> tide::Result { let searched_text = params.q.clone(); let page_number = params.page.map_or_else(|| 1, |page| page.get()); - let offset = (page_number - 1) * RESULT_PER_PAGE as u32; + let offset = (page_number - 1) * crate::fts::DEFAULT_RESULT_PER_PAGE; let user = req.get_author(); if req.state().is_login_required() && user.is_none() { @@ -42,16 +40,16 @@ pub(crate) async fn get(req: Request) -> tide::Result { let (count, results) = state.search.search( searched_text.clone(), - offset as usize, - RESULT_PER_PAGE as usize, + offset, + crate::fts::DEFAULT_RESULT_PER_PAGE, )?; - let page_count = (count / RESULT_PER_PAGE as usize - + if count > 0 && count % RESULT_PER_PAGE as usize == 0 { + let page_count = count / crate::fts::DEFAULT_RESULT_PER_PAGE + + if count > 0 && count % crate::fts::DEFAULT_RESULT_PER_PAGE == 0 { 0 } else { 1 - }) as u32; + }; let transaction = repo.transaction(move |conn| { let state = req.state(); diff --git a/crates/alexandrie/src/fts/mod.rs b/crates/alexandrie/src/fts/mod.rs index 19bf37bf..b6d33468 100644 --- a/crates/alexandrie/src/fts/mod.rs +++ b/crates/alexandrie/src/fts/mod.rs @@ -5,6 +5,11 @@ mod index; pub(crate) use document::TantivyDocument; pub(crate) use index::Tantivy; + +/// Default number of result per page +/// Perhaps should make this configurable in toml. +pub const DEFAULT_RESULT_PER_PAGE: usize = 15; + /// Database ID. const ID_FIELD_NAME: &str = "id"; /// Tokenized version of crate's name. diff --git a/crates/alexandrie/src/main.rs b/crates/alexandrie/src/main.rs index 4da89d0d..c372b949 100644 --- a/crates/alexandrie/src/main.rs +++ b/crates/alexandrie/src/main.rs @@ -57,7 +57,9 @@ pub mod utils; /// Frontend endpoints definitions. #[cfg(feature = "frontend")] pub mod frontend; -mod fts; + +/// Full text search +pub mod fts; use crate::config::Config; use crate::error::Error; From c5752f2024a4ffed2467f3673aa516a8c88c7ecd Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Sun, 11 Jun 2023 17:17:29 +0200 Subject: [PATCH 15/22] Fix search sort --- crates/alexandrie/src/api/crates/search.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/alexandrie/src/api/crates/search.rs b/crates/alexandrie/src/api/crates/search.rs index effd541f..b8374ad0 100644 --- a/crates/alexandrie/src/api/crates/search.rs +++ b/crates/alexandrie/src/api/crates/search.rs @@ -54,7 +54,10 @@ pub(crate) async fn get(req: Request) -> tide::Result { let state = req.state().clone(); let query = params.q; - let per_page = params.per_page.map(|v| v.get()).unwrap_or(crate::fts::DEFAULT_RESULT_PER_PAGE); + let per_page = params + .per_page + .map(|v| v.get()) + .unwrap_or(crate::fts::DEFAULT_RESULT_PER_PAGE); let page = params.page.map(|v| v.get()).unwrap_or(1) - 1; let searcher = &state.search; @@ -68,12 +71,21 @@ pub(crate) async fn get(req: Request) -> tide::Result { let state = req.state(); // Get crate from database - let results = crates::table - .filter(crates::id.eq_any(ids)) + let mut tmp = crates::table + .filter(crates::id.eq_any(&ids)) .load::(conn)?; + // Sort database result by relevance since we lost ordering... + let mut sorted: Vec = Vec::with_capacity(tmp.len()); + for id in ids { + if let Some(idx) = tmp.iter().position(|krate| krate.id == id) { + let krate = tmp.remove(idx); + sorted.push(krate); + } + } + // Fetch missing informations from index - let crates = results + let crates = sorted .into_iter() .map(|krate| { let latest = state.index.latest_record(krate.name.as_str())?; From 43b3c32a66c0a94f90daaf4d82630843a9121fea Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Sun, 11 Jun 2023 17:42:41 +0200 Subject: [PATCH 16/22] Fix fmt --- crates/alexandrie/src/fts/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/alexandrie/src/fts/mod.rs b/crates/alexandrie/src/fts/mod.rs index b6d33468..c4f96729 100644 --- a/crates/alexandrie/src/fts/mod.rs +++ b/crates/alexandrie/src/fts/mod.rs @@ -5,7 +5,6 @@ mod index; pub(crate) use document::TantivyDocument; pub(crate) use index::Tantivy; - /// Default number of result per page /// Perhaps should make this configurable in toml. pub const DEFAULT_RESULT_PER_PAGE: usize = 15; From 5a998b9a3a6098276039f6d4af1f937eae19ee5f Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Mon, 12 Jun 2023 20:53:48 +0200 Subject: [PATCH 17/22] Apply suggestions from code review Co-authored-by: Nicolas Polomack --- alexandrie.toml | 2 +- crates/alexandrie/src/config/mod.rs | 2 +- docker/mysql/alexandrie.toml | 2 +- docker/postgres/alexandrie.toml | 2 +- docker/sqlite/alexandrie.toml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/alexandrie.toml b/alexandrie.toml index b323db29..47fd777f 100644 --- a/alexandrie.toml +++ b/alexandrie.toml @@ -78,4 +78,4 @@ path = "syntect/dumps/themes.dump" theme_name = "frontier-contrast" [search] -directory = "/tmp/tantivy" \ No newline at end of file +path = "/tmp/tantivy" \ No newline at end of file diff --git a/crates/alexandrie/src/config/mod.rs b/crates/alexandrie/src/config/mod.rs index f55dd019..71976a04 100644 --- a/crates/alexandrie/src/config/mod.rs +++ b/crates/alexandrie/src/config/mod.rs @@ -38,7 +38,7 @@ pub struct GeneralConfig { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SearchConfig { /// Path to the directory where Tantivy will store its index. - pub directory: String, + pub path: String, } /// The application configuration struct. diff --git a/docker/mysql/alexandrie.toml b/docker/mysql/alexandrie.toml index af17c303..898633cd 100644 --- a/docker/mysql/alexandrie.toml +++ b/docker/mysql/alexandrie.toml @@ -55,4 +55,4 @@ path = "syntect/dumps/themes.dump" theme_name = "frontier-contrast" [search] -directory = "appdata/tantivy" \ No newline at end of file +path = "appdata/tantivy" \ No newline at end of file diff --git a/docker/postgres/alexandrie.toml b/docker/postgres/alexandrie.toml index 3a6d39c1..5ff086fb 100644 --- a/docker/postgres/alexandrie.toml +++ b/docker/postgres/alexandrie.toml @@ -53,4 +53,4 @@ path = "syntect/dumps/themes.dump" theme_name = "frontier-contrast" [search] -directory = "appdata/tantivy" \ No newline at end of file +path = "appdata/tantivy" \ No newline at end of file diff --git a/docker/sqlite/alexandrie.toml b/docker/sqlite/alexandrie.toml index a20661b7..6624b63d 100644 --- a/docker/sqlite/alexandrie.toml +++ b/docker/sqlite/alexandrie.toml @@ -53,4 +53,4 @@ path = "syntect/dumps/themes.dump" theme_name = "frontier-contrast" [search] -directory = "appdata/tantivy" \ No newline at end of file +path = "appdata/tantivy" \ No newline at end of file From e3ce284f6273d7813cf9278c7640e4bccf56dea3 Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:20:05 +0200 Subject: [PATCH 18/22] Fix following review --- crates/alexandrie/src/api/crates/search.rs | 17 ++++++++--------- crates/alexandrie/src/frontend/search.rs | 10 ++++------ crates/alexandrie/src/fts/index.rs | 2 +- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/crates/alexandrie/src/api/crates/search.rs b/crates/alexandrie/src/api/crates/search.rs index b8374ad0..af18da9f 100644 --- a/crates/alexandrie/src/api/crates/search.rs +++ b/crates/alexandrie/src/api/crates/search.rs @@ -71,21 +71,20 @@ pub(crate) async fn get(req: Request) -> tide::Result { let state = req.state(); // Get crate from database - let mut tmp = crates::table + let mut crates = crates::table .filter(crates::id.eq_any(&ids)) .load::(conn)?; // Sort database result by relevance since we lost ordering... - let mut sorted: Vec = Vec::with_capacity(tmp.len()); - for id in ids { - if let Some(idx) = tmp.iter().position(|krate| krate.id == id) { - let krate = tmp.remove(idx); - sorted.push(krate); - } - } + // (the `unwrap_or` call should be unreachable, but if it is reached, it would sort the crate towards the end) + crates.sort_unstable_by_key(|krate| { + ids.iter() + .position(|id| *id == krate.id) + .unwrap_or(ids.len()) + }); // Fetch missing informations from index - let crates = sorted + let crates = crates .into_iter() .map(|krate| { let latest = state.index.latest_record(krate.name.as_str())?; diff --git a/crates/alexandrie/src/frontend/search.rs b/crates/alexandrie/src/frontend/search.rs index 642de0f1..8814417d 100644 --- a/crates/alexandrie/src/frontend/search.rs +++ b/crates/alexandrie/src/frontend/search.rs @@ -59,17 +59,15 @@ pub(crate) async fn get(req: Request) -> tide::Result { .map(|v| { let krate = crates::table .filter(crates::id.eq(v)) - .first::(conn) - .unwrap(); + .first::(conn)?; let keywords = crate_keywords::table .inner_join(keywords::table) .select(keywords::name) .filter(crate_keywords::crate_id.eq(krate.id)) - .load::(conn) - .unwrap(); - (krate, keywords) + .load::(conn)?; + Ok((krate, keywords)) }) - .collect(); + .collect::>()?; let encoded_q = percent_encoding::percent_encode( params.q.as_bytes(), diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index cd2c5562..d0595610 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -51,7 +51,7 @@ impl TryFrom for Tantivy { let analyzer_prefix_name = "alexandrie_prefix"; // Create index directory - let path = search.directory.as_str(); + let path = search.path.as_str(); std::fs::create_dir_all(path)?; let directory = MmapDirectory::open(path)?; From 48ab92a05d0aa4db66eb995bd59e0c4e3d4c2f9b Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:30:20 +0200 Subject: [PATCH 19/22] Remove readme from tantivy --- crates/alexandrie/src/fts/document.rs | 19 ------------------- crates/alexandrie/src/fts/index.rs | 5 +---- crates/alexandrie/src/fts/mod.rs | 1 - 3 files changed, 1 insertion(+), 24 deletions(-) diff --git a/crates/alexandrie/src/fts/document.rs b/crates/alexandrie/src/fts/document.rs index 6617e478..d3af7d12 100644 --- a/crates/alexandrie/src/fts/document.rs +++ b/crates/alexandrie/src/fts/document.rs @@ -12,7 +12,6 @@ pub struct TantivyDocument { id: i64, name: String, description: Option, - readme: Option, keywords: Vec, categories: Vec, } @@ -23,12 +22,6 @@ impl std::fmt::Display for TantivyDocument { if let Some(description) = &self.description { write!(f, ", '{}'", description)?; } - // Don't write the README, it might be big, and - // log will be unreadable. Just say that it has - // a README. - if self.readme.is_some() { - write!(f, ", crate has README",)?; - } if self.keywords.is_empty() { write!(f, ", no keyword")?; @@ -54,7 +47,6 @@ impl From for TantivyDocument { id: value.id, name: value.name, description: value.description, - readme: None, keywords: vec![], categories: vec![], } @@ -67,7 +59,6 @@ impl TantivyDocument { id, name, description: None, - readme: None, keywords: Vec::with_capacity(5), categories: Vec::with_capacity(5), } @@ -97,11 +88,6 @@ impl TantivyDocument { document.add_text(description_field, description); } - if let Some(readme) = &self.readme { - let readme_field = schema.get_field(super::README_FIELD_NAME)?; - document.add_text(readme_field, readme); - } - self.keywords .clone() .into_iter() @@ -124,11 +110,6 @@ impl TantivyDocument { self.description = Some(description); } - /// Set crate's README - pub fn set_readme(&mut self, readme: String) { - self.readme = Some(readme); - } - /// Add new crate's keyword pub fn add_keyword(&mut self, keyword: String) { self.keywords.push(keyword); diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index d0595610..789c04f3 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -97,7 +97,6 @@ impl TryFrom for Tantivy { schema_builder.add_text_field(super::NAME_FIELD_NAME_FULL, options_full.clone()); schema_builder.add_text_field(super::NAME_FIELD_PREFIX_NAME, options_prefixes); schema_builder.add_text_field(super::DESCRIPTION_FIELD_NAME, options.clone()); - schema_builder.add_text_field(super::README_FIELD_NAME, options.clone()); schema_builder.add_text_field(super::CATEGORY_FIELD_NAME, options_full); schema_builder.add_text_field(super::KEYWORD_FIELD_NAME, options); let schema = schema_builder.build(); @@ -266,13 +265,12 @@ impl Tantivy { .schema .get_field(super::DESCRIPTION_FIELD_NAME) .unwrap(); - let readme = self.schema.get_field(super::README_FIELD_NAME).unwrap(); let categories = self.schema.get_field(super::CATEGORY_FIELD_NAME).unwrap(); let keywords = self.schema.get_field(super::KEYWORD_FIELD_NAME).unwrap(); let mut query_parser = QueryParser::for_index( searcher.index(), - vec![name, name_full, description, readme, categories, keywords], + vec![name, name_full, description, categories, keywords], ); // Exact matches (on name_full) have a big boost @@ -284,7 +282,6 @@ impl Tantivy { query_parser.set_field_boost(keywords, 0.5); // description & readme are full text they got a lower boost (if there is a match, that might not be relevant) query_parser.set_field_boost(description, 0.2); - query_parser.set_field_boost(readme, 0.2); let query = query_parser.parse_query(query)?; diff --git a/crates/alexandrie/src/fts/mod.rs b/crates/alexandrie/src/fts/mod.rs index c4f96729..26d42ba9 100644 --- a/crates/alexandrie/src/fts/mod.rs +++ b/crates/alexandrie/src/fts/mod.rs @@ -23,6 +23,5 @@ const NAME_FIELD_NAME_FULL: &str = "name.full"; /// contains word's prefixes to do "search as you type". const NAME_FIELD_PREFIX_NAME: &str = "name.prefix"; const DESCRIPTION_FIELD_NAME: &str = "description"; -const README_FIELD_NAME: &str = "readme"; const CATEGORY_FIELD_NAME: &str = "category"; const KEYWORD_FIELD_NAME: &str = "keyword"; From 1e7f80edc5a0d8142f9fa986d4344ac3848659c9 Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:36:50 +0200 Subject: [PATCH 20/22] Handle empty query --- crates/alexandrie/src/fts/index.rs | 40 ++++++++++++++++-------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index 789c04f3..09dc985e 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -6,7 +6,7 @@ use diesel::prelude::*; use log::{debug, info, warn}; use tantivy::collector::{Count, TopDocs}; use tantivy::directory::MmapDirectory; -use tantivy::query::QueryParser; +use tantivy::query::{AllQuery, QueryParser}; use tantivy::schema::{NumericOptions, Schema, TextFieldIndexing, TextOptions}; use tantivy::tokenizer::{ Language, LowerCaser, RawTokenizer, SimpleTokenizer, StopWordFilter, TextAnalyzer, @@ -254,7 +254,7 @@ impl Tantivy { offset: usize, limit: usize, ) -> Result<(usize, Vec), TantivyError> { - let query = query.as_ref(); + let query = query.as_ref().trim(); let searcher = self.index_reader.searcher(); @@ -268,22 +268,26 @@ impl Tantivy { let categories = self.schema.get_field(super::CATEGORY_FIELD_NAME).unwrap(); let keywords = self.schema.get_field(super::KEYWORD_FIELD_NAME).unwrap(); - let mut query_parser = QueryParser::for_index( - searcher.index(), - vec![name, name_full, description, categories, keywords], - ); - - // Exact matches (on name_full) have a big boost - query_parser.set_field_boost(name_full, 10.0); - query_parser.set_field_boost(name, 5.0); - // Categories shouldn't be free (there is a list) so a nice boost - query_parser.set_field_boost(categories, 1.0); - // Keywords are free - query_parser.set_field_boost(keywords, 0.5); - // description & readme are full text they got a lower boost (if there is a match, that might not be relevant) - query_parser.set_field_boost(description, 0.2); - - let query = query_parser.parse_query(query)?; + let query = if query.is_empty() { + Box::new(AllQuery) + } else { + let mut query_parser = QueryParser::for_index( + searcher.index(), + vec![name, name_full, description, categories, keywords], + ); + + // Exact matches (on name_full) have a big boost + query_parser.set_field_boost(name_full, 10.0); + query_parser.set_field_boost(name, 5.0); + // Categories shouldn't be free (there is a list) so a nice boost + query_parser.set_field_boost(categories, 1.0); + // Keywords are free + query_parser.set_field_boost(keywords, 0.5); + // description & readme are full text they got a lower boost (if there is a match, that might not be relevant) + query_parser.set_field_boost(description, 0.2); + + query_parser.parse_query(query)? + }; info!("Query offset={} query limit={}", offset, limit); From 14ff5a90876a7905e6d64a4bf994d5e97c40d4f6 Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:42:38 +0200 Subject: [PATCH 21/22] Apply suggestions from code review Co-authored-by: Nicolas Polomack --- crates/alexandrie/src/fts/index.rs | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs index 09dc985e..a6f8ae64 100644 --- a/crates/alexandrie/src/fts/index.rs +++ b/crates/alexandrie/src/fts/index.rs @@ -395,18 +395,30 @@ impl Tantivy { let mut doc: TantivyDocument = krate.into(); // Skip keywords that might be orphan and add keywords that match ids - while current_keyword.is_some() && current_keyword.as_ref().unwrap().0 <= id { - if current_keyword.as_ref().unwrap().0 == id { - doc.add_keyword(current_keyword.unwrap().1); + while let Some((crate_id, keyword)) = current_keyword { + if crate_id > id { + current_keyword = Some((crate_id, keyword)); + break; } + + if crate_id == id { + doc.add_keyword(keyword); + } + current_keyword = keywords_iterator.next(); } - // Skip keywords that might be orphan and add keywords that match ids - while current_category.is_some() && current_category.as_ref().unwrap().0 <= id { - if current_category.as_ref().unwrap().0 == id { - doc.add_keyword(current_category.unwrap().1); + // Skip categories that might be orphan and add categories that match ids + while let Some((crate_id, category)) = current_category { + if crate_id > id { + current_category = Some((crate_id, category)); + break; } + + if crate_id == id { + doc.add_category(category); + } + current_category = categories_iterator.next(); } From aa296f1f6f22b46cdeafebbbad8cfcc3127b0711 Mon Sep 17 00:00:00 2001 From: Dalvany <9901407+Dalvany@users.noreply.github.com> Date: Tue, 27 Jun 2023 19:58:34 +0200 Subject: [PATCH 22/22] Update Cargo.lock Resolve conflict on Cargo.lock. Ran cargo update. --- Cargo.lock | 306 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 177 insertions(+), 129 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1995bd0..44ca7b7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,7 +123,7 @@ dependencies = [ "thiserror", "tide", "time 0.3.22", - "toml", + "toml 0.7.5", "url", ] @@ -205,15 +205,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" [[package]] name = "anstyle-parse" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" dependencies = [ "utf8parse", ] @@ -263,9 +263,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrayvec" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "async-attributes" @@ -489,7 +489,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -568,9 +568,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.1" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6776fc96284a0bb647b615056fc496d1fe1644a7ab01829818a6d91cae888b84" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" [[package]] name = "bitpacking" @@ -724,18 +724,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.3" +version = "4.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8f255e4b8027970e78db75e78831229c9815fdbfa67eb1a1b777a62e24b4a0" +checksum = "d9394150f5b4273a1763355bd1c2ec54cc5a2593f790587bcd6b2c947cfa9211" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.3.3" +version = "4.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acd4f3c17c83b0ba34ffbc4f8bbd74f079413f747f84a6f89292f138057e36ab" +checksum = "9a78fbdd3cc2914ddf37ba444114bc7765bbdcb55ec9cbe6fa054f0137400717" dependencies = [ "anstream", "anstyle", @@ -821,9 +821,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" dependencies = [ "libc", ] @@ -866,9 +866,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.14" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if 1.0.0", @@ -879,9 +879,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if 1.0.0", ] @@ -1014,7 +1014,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7a532c1f99a0f596f6960a60d1e119e91582b24b39e2d83a190e61262c3ef0c" dependencies = [ "bigdecimal", - "bitflags 2.3.1", + "bitflags 2.3.3", "byteorder", "chrono", "diesel_derives", @@ -1040,7 +1040,7 @@ dependencies = [ "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1060,7 +1060,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" dependencies = [ - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1130,6 +1130,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "equivalent" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" + [[package]] name = "erased-serde" version = "0.3.25" @@ -1341,7 +1347,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1376,9 +1382,9 @@ dependencies = [ [[package]] name = "generator" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3e123d9ae7c02966b4d892e550bdc32164f05853cd40ab570650ad600596a8a" +checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" dependencies = [ "cc", "libc", @@ -1469,9 +1475,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ "bytes", "fnv", @@ -1479,7 +1485,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -1516,6 +1522,12 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1676,9 +1688,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -1773,6 +1785,16 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "infer" version = "0.2.3" @@ -1804,9 +1826,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.7.2" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "is-terminal" @@ -1820,6 +1842,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -1837,9 +1868,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.63" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -1867,9 +1898,9 @@ checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" [[package]] name = "libc" -version = "0.2.146" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libgit2-sys" @@ -2063,9 +2094,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] @@ -2077,7 +2108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f23f71580015254b020e856feac3df5878c2c7a8812297edd6c0a485ac9dada" dependencies = [ "serde", - "toml", + "toml 0.7.5", ] [[package]] @@ -2184,7 +2215,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" dependencies = [ - "arrayvec 0.7.2", + "arrayvec 0.7.4", "itoa", ] @@ -2241,7 +2272,7 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", - "sha2 0.10.6", + "sha2 0.10.7", "thiserror", "url", ] @@ -2312,7 +2343,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -2385,9 +2416,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e68e84bfb01f0507134eac1e9b410a12ba379d064eab48c50ba4ce329a527b70" +checksum = "f73935e4d55e2abf7f130186537b19e7a4abc886a0252380b59248af473a3fc9" dependencies = [ "thiserror", "ucd-trie", @@ -2395,9 +2426,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b79d4c71c865a25a4322296122e3924d30bc8ee0834c8bfc8b95f7f054afbfb" +checksum = "aef623c9bbfa0eedf5a0efba11a5ee83209c326653ca31ff019bec3a95bfff2b" dependencies = [ "pest", "pest_generator", @@ -2405,26 +2436,26 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" +checksum = "b3e8cba4ec22bada7fc55ffe51e2deb6a0e0db2d0b7ab0b103acc80d2510c190" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] name = "pest_meta" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745a452f8eb71e39ffd8ee32b3c5f51d03845f99786fa9b68db6ff509c505411" +checksum = "a01f71cb40bd8bb94232df14b946909e14660e33fc05db3e50ae2a82d7ea0ca0" dependencies = [ "once_cell", "pest", - "sha2 0.10.6", + "sha2 0.10.7", ] [[package]] @@ -2482,7 +2513,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -2516,7 +2547,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bd9647b268a3d3e14ff09c23201133a62589c658db02bb7388c7246aafe0590" dependencies = [ "base64 0.21.2", - "indexmap", + "indexmap 1.9.3", "line-wrap", "quick-xml", "serde", @@ -2579,9 +2610,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" dependencies = [ "unicode-ident", ] @@ -2962,9 +2993,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.19" +version = "0.37.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" dependencies = [ "bitflags 1.3.2", "errno", @@ -2976,9 +3007,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" +checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" dependencies = [ "log", "ring", @@ -3136,7 +3167,7 @@ checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -3150,9 +3181,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" dependencies = [ "itoa", "ryu", @@ -3181,9 +3212,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93107647184f6027e3b7dcb2e11034cf95ffa1e3a682c67951963ac69c1c007d" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" dependencies = [ "serde", ] @@ -3230,15 +3261,24 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if 1.0.0", "cpufeatures", "digest 0.10.7", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.1.0" @@ -3476,15 +3516,15 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "sval" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e6aa16ce8d9e472e21a528a52c875a76a49190f3968f2ec7e9b550ccc28b410" +checksum = "8b031320a434d3e9477ccf9b5756d57d4272937b8d22cb88af80b7633a1b78b1" [[package]] name = "sval_buffer" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "905c4373621186ee9637464b0aaa026389ea9e7f841f2225f160a32ba5d5bac4" +checksum = "6bf7e9412af26b342f3f2cc5cc4122b0105e9d16eb76046cd14ed10106cf6028" dependencies = [ "sval", "sval_ref", @@ -3492,18 +3532,18 @@ dependencies = [ [[package]] name = "sval_dynamic" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad6b4988322c5f22859a6a7649fa1249aa3dd01514caf8ed57d83735f997bb8b" +checksum = "a0ef628e8a77a46ed3338db8d1b08af77495123cc229453084e47cd716d403cf" dependencies = [ "sval", ] [[package]] name = "sval_fmt" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d3ccd10346f925c2fbd97b75e8573b38e34431bfba04cc531cd23aad0fbabb8" +checksum = "7dc09e9364c2045ab5fa38f7b04d077b3359d30c4c2b3ec4bae67a358bd64326" dependencies = [ "itoa", "ryu", @@ -3512,9 +3552,9 @@ dependencies = [ [[package]] name = "sval_json" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a12e1488defd6344e23243c17ea4a1b185c547968749e8a281373fde0bde2d5" +checksum = "ada6f627e38cbb8860283649509d87bc4a5771141daa41c78fd31f2b9485888d" dependencies = [ "itoa", "ryu", @@ -3523,18 +3563,18 @@ dependencies = [ [[package]] name = "sval_ref" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b797fc4b284dd0e45f7ec424479e604ea5be9bb191a1ef4e96c20c7685649938" +checksum = "703ca1942a984bd0d9b5a4c0a65ab8b4b794038d080af4eb303c71bc6bf22d7c" dependencies = [ "sval", ] [[package]] name = "sval_serde" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "810fa9a837e67a23e0efa7536250fc4d24043306cc1efd076f1943ba2fc2e62d" +checksum = "830926cd0581f7c3e5d51efae4d35c6b6fc4db583842652891ba2f1bed8db046" dependencies = [ "serde", "sval", @@ -3555,9 +3595,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "2efbeae7acf4eabd6bcdcbd11c92f45231ddda7539edc7806bd1a04a03b24616" dependencies = [ "proc-macro2", "quote", @@ -3602,9 +3642,9 @@ checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" [[package]] name = "tantivy" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e71e08fe5b8242d2d6b8d047ed18c7df0db79d2162c8be29945f7c15cce26a8e" +checksum = "aec540e9cebc88f523f67f596dee213e491f0c55961de013566f267a0c31f5e9" dependencies = [ "aho-corasick", "arc-swap", @@ -3750,9 +3790,9 @@ dependencies = [ [[package]] name = "tantivy-tokenizer-api" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bee3519354cbe6bfff4bf6be9cf3e2dfcd0a6ea748a42312fbf1242c0d66fc6" +checksum = "64186801b6e06b3a1c4275e23b517835ff4ecbb707318b838dc9de457c062200" dependencies = [ "serde", ] @@ -3821,7 +3861,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -3975,7 +4015,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -4014,9 +4054,18 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.4" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6135d499e69981f9ff0ef2167955a5333c35e36f6937d382974566b3d5b94ec" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "toml" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebafdf5ad1220cb59e7d17cf4d2c72015297b75b19a10472f99b89225089240" dependencies = [ "serde", "serde_spanned", @@ -4026,20 +4075,20 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.10" +version = "0.19.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" +checksum = "266f016b7f039eec8a1a80dfe6156b633d208b9fccca5e4db1d6775b0c4e34a7" dependencies = [ - "indexmap", + "indexmap 2.0.0", "serde", "serde_spanned", "toml_datetime", @@ -4066,13 +4115,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -4216,9 +4265,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.3.3" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" +checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be" dependencies = [ "getrandom 0.2.10", "serde", @@ -4232,9 +4281,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4d330786735ea358f3bc09eea4caa098569c1c93f342d9aca0514915022fe7e" +checksum = "d92ccd67fb88503048c01b59152a04effd0782d035a83a6d256ce6085f08f4a3" dependencies = [ "value-bag-serde1", "value-bag-sval2", @@ -4242,9 +4291,9 @@ dependencies = [ [[package]] name = "value-bag-serde1" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4735c95b4cca1447b448e2e2e87e98d7e7498f4da27e355cf7af02204521001d" +checksum = "b0b9f3feef403a50d4d67e9741a6d8fc688bcbb4e4f31bd4aab72cc690284394" dependencies = [ "erased-serde", "serde", @@ -4253,9 +4302,9 @@ dependencies = [ [[package]] name = "value-bag-sval2" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859cb4f0ce7da6a118b559ba74b0e63bf569bea867c20ba457a6b1c886a04e97" +checksum = "30b24f4146b6f3361e91cbf527d1fb35e9376c3c0cef72ca5ec5af6d640fad7d" dependencies = [ "sval", "sval_buffer", @@ -4296,11 +4345,10 @@ dependencies = [ [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] @@ -4324,9 +4372,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -4334,24 +4382,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.36" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -4361,9 +4409,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4371,28 +4419,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.63" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -4582,9 +4630,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" +checksum = "ca0ace3845f0d96209f0375e6d367e3eb87eb65d27d445bdc9f1843a26f39448" dependencies = [ "memchr", ]