diff --git a/.gitignore b/.gitignore index 10d8bc34..281fbff1 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ # Rust compilation folder /target +crates/*/target # hidden folders from code editors /.vscode diff --git a/Cargo.lock b/Cargo.lock index 73e31b35..44ca7b7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,11 +62,22 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] @@ -106,11 +117,13 @@ dependencies = [ "slog-scope", "slog-stdlog", "slog-term", + "tantivy", + "tantivy-analysis-contrib", "tar", "thiserror", "tide", - "time 0.3.20", - "toml 0.7.3", + "time 0.3.22", + "toml 0.7.5", "url", ] @@ -160,6 +173,12 @@ dependencies = [ "url", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -171,9 +190,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e579a7752471abc2a8268df8b20005e3eadd975f585398f17efcfd8d4927371" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" dependencies = [ "anstyle", "anstyle-parse", @@ -186,15 +205,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" [[package]] name = "anstyle-parse" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" dependencies = [ "utf8parse", ] @@ -210,9 +229,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcd8291a340dd8ac70e18878bc4501dd7b4ff970cfa21c207d36ece51ea88fd" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -220,9 +239,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" [[package]] name = "arc-swap" @@ -244,9 +263,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrayvec" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "async-attributes" @@ -470,7 +489,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.22", ] [[package]] @@ -516,15 +535,15 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "bigdecimal" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aaf33151a6429fe9211d1b276eafdf70cdff28b071e76c0b0e1503221ea3744" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" dependencies = [ "num-bigint", "num-integer", @@ -547,6 +566,21 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + +[[package]] +name = "bitpacking" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c7d2ac73c167c06af4a5f37e6e59d84148d57ccbe4480b76f0273eefea82d7" +dependencies = [ + "crunchy", +] + [[package]] name = "blake3" version = "0.3.8" @@ -608,9 +642,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byteorder" @@ -645,6 +679,12 @@ dependencies = [ "jobserver", ] +[[package]] +name = "census" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fafee10a5dd1cffcb5cc560e0d0df8803d7355a2b12272e3557dee57314cb6e" + [[package]] name = "cfg-if" version = "0.1.10" @@ -659,13 +699,13 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.24" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", "serde", "time 0.1.45", @@ -684,41 +724,31 @@ dependencies = [ [[package]] name = "clap" -version = "4.2.2" +version = "4.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b802d85aaf3a1cdb02b224ba472ebdea62014fccfcb269b95a4d76443b5ee5a" +checksum = "d9394150f5b4273a1763355bd1c2ec54cc5a2593f790587bcd6b2c947cfa9211" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.2.2" +version = "4.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14a1a858f532119338887a4b8e1af9c60de8249cd7bafd68036a489e261e37b6" +checksum = "9a78fbdd3cc2914ddf37ba444114bc7765bbdcb55ec9cbe6fa054f0137400717" dependencies = [ "anstream", "anstyle", - "bitflags", + "bitflags 1.3.2", "clap_lex", "strsim", ] [[package]] name = "clap_lex" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" - -[[package]] -name = "codespan-reporting" -version = "0.11.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] name = "colorchoice" @@ -726,6 +756,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.2.0" @@ -782,9 +821,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" dependencies = [ "libc", ] @@ -814,15 +853,45 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -863,16 +932,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "ctor" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" -dependencies = [ - "quote", - "syn 1.0.109", -] - [[package]] name = "ctr" version = "0.6.0" @@ -883,57 +942,79 @@ dependencies = [ ] [[package]] -name = "cxx" -version = "1.0.94" +name = "darling" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", + "darling_core", + "darling_macro", ] [[package]] -name = "cxx-build" -version = "1.0.94" +name = "darling_core" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" dependencies = [ - "cc", - "codespan-reporting", - "once_cell", + "fnv", + "ident_case", "proc-macro2", "quote", - "scratch", - "syn 2.0.15", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", ] [[package]] -name = "cxxbridge-flags" -version = "1.0.94" +name = "derive_builder" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro", +] [[package]] -name = "cxxbridge-macro" -version = "1.0.94" +name = "derive_builder_core" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" dependencies = [ + "darling", "proc-macro2", "quote", - "syn 2.0.15", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", ] [[package]] name = "diesel" -version = "2.0.3" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4391a22b19c916e50bec4d6140f29bdda3e3bb187223fe6e3ea0b6e4d1021c04" +checksum = "f7a532c1f99a0f596f6960a60d1e119e91582b24b39e2d83a190e61262c3ef0c" dependencies = [ "bigdecimal", - "bitflags", + "bitflags 2.3.3", "byteorder", "chrono", "diesel_derives", @@ -946,32 +1027,42 @@ dependencies = [ "percent-encoding", "pq-sys", "r2d2", + "time 0.3.22", "url", ] [[package]] name = "diesel_derives" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad74fdcf086be3d4fdd142f67937678fe60ed431c3b2f08599e7687269410c4" +checksum = "74398b79d81e52e130d991afeed9c86034bb1b7735f46d2f5bf7deb261d80303" dependencies = [ - "proc-macro-error", + "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.22", ] [[package]] name = "diesel_migrations" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9ae22beef5e9d6fab9225ddb073c1c6c1a7a6ded5019d5da11d1e5c5adc34e2" +checksum = "6036b3f0120c5961381b570ee20a02432d7e2d27ea60de9578799cf9156914ac" dependencies = [ "diesel", "migrations_internals", "migrations_macros", ] +[[package]] +name = "diesel_table_macro_syntax" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" +dependencies = [ + "syn 2.0.22", +] + [[package]] name = "digest" version = "0.9.0" @@ -983,9 +1074,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common", @@ -1018,6 +1109,18 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "downcast-rs" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + [[package]] name = "encoding_rs" version = "0.8.32" @@ -1027,6 +1130,21 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "equivalent" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" + +[[package]] +name = "erased-serde" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2b0c2380453a92ea8b6c8e5f64ecaafccddde8ceab55ff7a8ac1029f894569" +dependencies = [ + "serde", +] + [[package]] name = "errno" version = "0.3.1" @@ -1054,6 +1172,23 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "fail" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c" +dependencies = [ + "log", + "once_cell", + "rand 0.8.5", +] + +[[package]] +name = "fastdivide" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25c7df09945d65ea8d70b3321547ed414bbc540aad5bac6883d021b970f35b04" + [[package]] name = "fastrand" version = "1.9.0" @@ -1077,9 +1212,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", "miniz_oxide", @@ -1108,13 +1243,29 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] +[[package]] +name = "fs4" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7672706608ecb74ab2e055c68327ffc25ae4cac1e12349204fd5fb0f3487cce2" +dependencies = [ + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" + [[package]] name = "futf" version = "0.1.5" @@ -1196,7 +1347,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.22", ] [[package]] @@ -1229,6 +1380,19 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" +dependencies = [ + "cc", + "libc", + "log", + "rustversion", + "windows", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1261,9 +1425,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -1288,7 +1452,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf7f68c2995f392c49fffb4f95ae2c873297830eb25c6bc4c114ce8f4562acc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", "libgit2-sys", "log", @@ -1311,9 +1475,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ "bytes", "fnv", @@ -1321,7 +1485,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -1330,9 +1494,9 @@ dependencies = [ [[package]] name = "handlebars" -version = "4.3.6" +version = "4.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "035ef95d03713f2c347a72547b7cd38cbc9af7cd51e6099fb62d586d4a6dee3a" +checksum = "83c3372087601b532857d332f5957cbae686da52bb7810bf038c3e3c3cc2fa0d" dependencies = [ "log", "pest", @@ -1349,6 +1513,21 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1433,6 +1612,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + [[package]] name = "http" version = "0.2.9" @@ -1503,9 +1688,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -1527,9 +1712,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.23.2" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" dependencies = [ "http", "hyper", @@ -1553,9 +1738,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1567,19 +1752,24 @@ dependencies = [ [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1592,7 +1782,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", ] [[package]] @@ -1608,13 +1808,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.1", "libc", @@ -1623,9 +1826,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.7.2" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "is-terminal" @@ -1639,6 +1842,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -1656,9 +1868,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -1678,11 +1890,17 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + [[package]] name = "libc" -version = "0.2.141" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libgit2-sys" @@ -1700,9 +1918,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.25.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" +checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" dependencies = [ "pkg-config", "vcpkg", @@ -1724,9 +1942,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" dependencies = [ "cc", "libc", @@ -1743,15 +1961,6 @@ dependencies = [ "safemem", ] -[[package]] -name = "link-cplusplus" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] - [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1760,15 +1969,15 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -1776,14 +1985,42 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" dependencies = [ - "cfg-if 1.0.0", "value-bag", ] +[[package]] +name = "loom" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5" +dependencies = [ + "cfg-if 1.0.0", + "generator", + "pin-utils", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03f1160296536f10c833a82dca22267d5486734230d47bf00bf435885814ba1e" +dependencies = [ + "hashbrown 0.13.2", +] + +[[package]] +name = "lz4_flex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b8c72594ac26bfd34f2d99dfced2edfaddfe8a476e3ff2ca0eb293d925c4f83" + [[package]] name = "mac" version = "0.1.1" @@ -1810,6 +2047,15 @@ dependencies = [ "tendril", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + [[package]] name = "md-5" version = "0.9.1" @@ -1822,26 +2068,54 @@ dependencies = [ ] [[package]] -name = "memchr" -version = "2.5.0" +name = "measure_time" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56220900f1a0923789ecd6bf25fbae8af3b2f1ff3e9e297fc9b6b8674dd4d852" +dependencies = [ + "instant", + "log", +] + +[[package]] +name = "memchr" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "migrations_internals" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c493c09323068c01e54c685f7da41a9ccf9219735c3766fbfd6099806ea08fbc" +checksum = "0f23f71580015254b020e856feac3df5878c2c7a8812297edd6c0a485ac9dada" dependencies = [ "serde", - "toml 0.5.11", + "toml 0.7.5", ] [[package]] name = "migrations_macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8ff27a350511de30cdabb77147501c36ef02e0451d957abea2f30caffb2b58" +checksum = "cce3325ac70e67bbab5bd837a31cae01f1a6db64e0e744a33cb03a543469ef08" dependencies = [ "migrations_internals", "proc-macro2", @@ -1856,25 +2130,30 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] +[[package]] +name = "murmurhash32" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9380db4c04d219ac5c51d14996bbf2c2e9a15229771b53f8671eb6c83cf44df" + [[package]] name = "mysqlclient-sys" version = "0.2.5" @@ -1909,6 +2188,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.3" @@ -1926,7 +2215,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" dependencies = [ - "arrayvec 0.7.2", + "arrayvec 0.7.4", "itoa", ] @@ -1970,29 +2259,38 @@ dependencies = [ [[package]] name = "oauth2" -version = "4.3.0" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeaf26a72311c087f8c5ba617c96fac67a5c04f430e716ac8d8ab2de62e23368" +checksum = "09a6e2a2b13a56ebeabba9142f911745be6456163fd6c3d361274ebcd891a80c" dependencies = [ "base64 0.13.1", "chrono", - "getrandom 0.2.9", + "getrandom 0.2.10", "http", "rand 0.8.5", "reqwest", "serde", "serde_json", "serde_path_to_error", - "sha2 0.10.6", + "sha2 0.10.7", "thiserror", "url", ] [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "oneshot" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc22d22931513428ea6cc089e942d38600e3d00976eef8c86de6b8a3aadec6eb" +dependencies = [ + "loom", +] [[package]] name = "onig" @@ -2000,7 +2298,7 @@ version = "6.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", "once_cell", "onig_sys", @@ -2028,7 +2326,7 @@ version = "0.10.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if 1.0.0", "foreign-types", "libc", @@ -2045,7 +2343,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.22", ] [[package]] @@ -2066,6 +2364,21 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "ownedbytes" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c718e498b20704d5fb5d51d07f414a22f61c19254c1708e117b93fd76860739c" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "parking" version = "2.1.0" @@ -2084,28 +2397,28 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.3.5", "smallvec", - "windows-sys 0.45.0", + "windows-targets", ] [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.5.7" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1403e8401ad5dedea73c626b99758535b342502f8d1e361f4a2dd952749122" +checksum = "f73935e4d55e2abf7f130186537b19e7a4abc886a0252380b59248af473a3fc9" dependencies = [ "thiserror", "ucd-trie", @@ -2113,9 +2426,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.5.7" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be99c4c1d2fc2769b1d00239431d711d08f6efedcecb8b6e30707160aee99c15" +checksum = "aef623c9bbfa0eedf5a0efba11a5ee83209c326653ca31ff019bec3a95bfff2b" dependencies = [ "pest", "pest_generator", @@ -2123,26 +2436,26 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.5.7" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56094789873daa36164de2e822b3888c6ae4b4f9da555a1103587658c805b1e" +checksum = "b3e8cba4ec22bada7fc55ffe51e2deb6a0e0db2d0b7ab0b103acc80d2510c190" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.22", ] [[package]] name = "pest_meta" -version = "2.5.7" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6733073c7cff3d8459fda0e42f13a047870242aed8b509fe98000928975f359e" +checksum = "a01f71cb40bd8bb94232df14b946909e14660e33fc05db3e50ae2a82d7ea0ca0" dependencies = [ "once_cell", "pest", - "sha2 0.10.6", + "sha2 0.10.7", ] [[package]] @@ -2185,22 +2498,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.22", ] [[package]] @@ -2223,9 +2536,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "plist" @@ -2233,22 +2546,22 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bd9647b268a3d3e14ff09c23201133a62589c658db02bb7388c7246aafe0590" dependencies = [ - "base64 0.21.0", - "indexmap", + "base64 0.21.2", + "indexmap 1.9.3", "line-wrap", "quick-xml", "serde", - "time 0.3.20", + "time 0.3.22", ] [[package]] name = "polling" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be1c66a6add46bff50935c313dae30a5030cf8385c5206e8a95e9e9def974aa" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", - "bitflags", + "bitflags 1.3.2", "cfg-if 1.0.0", "concurrent-queue", "libc", @@ -2276,9 +2589,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "pq-sys" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b845d6d8ec554f972a2c5298aad68953fd64e7441e846075450b44656a016d1" +checksum = "31c0052426df997c0cbd30789eb44ca097e3541717a7b8fa36b1c464ee7edebd" dependencies = [ "vcpkg", ] @@ -2289,30 +2602,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro-hack" version = "0.5.20+deprecated" @@ -2321,20 +2610,20 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" dependencies = [ "unicode-ident", ] [[package]] name = "pulldown-cmark" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d9cc634bc78768157b5cbfe988ffcd1dcba95cd2b2f03a88316c08c6d00ed63" +checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" dependencies = [ - "bitflags", + "bitflags 1.3.2", "getopts", "memchr", "unicase", @@ -2351,9 +2640,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -2428,7 +2717,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.9", + "getrandom 0.2.10", ] [[package]] @@ -2440,13 +2729,35 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2455,7 +2766,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2464,20 +2775,29 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.9", + "getrandom 0.2.10", "redox_syscall 0.2.16", "thiserror", ] [[package]] name = "regex" -version = "1.7.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.2", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -2486,13 +2806,19 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" + [[package]] name = "reqwest" -version = "0.11.16" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", "bytes", "encoding_rs", "futures-core", @@ -2631,6 +2957,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.2.3" @@ -2651,11 +2993,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.11" +version = "0.37.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -2665,14 +3007,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.8" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" dependencies = [ "log", "ring", + "rustls-webpki", "sct", - "webpki", ] [[package]] @@ -2681,7 +3023,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", ] [[package]] @@ -2730,16 +3082,16 @@ dependencies = [ ] [[package]] -name = "scopeguard" -version = "1.1.0" +name = "scoped-tls" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" [[package]] -name = "scratch" -version = "1.0.5" +name = "scopeguard" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "sct" @@ -2753,11 +3105,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.8.2" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -2766,9 +3118,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" dependencies = [ "core-foundation-sys", "libc", @@ -2800,29 +3152,38 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.22", +] + +[[package]] +name = "serde_fmt" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d4ddca14104cd60529e8c7f7ba71a2c8acd8f7f5cfcdc2faf97eeb7c3010a4" +dependencies = [ + "serde", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" dependencies = [ "itoa", "ryu", @@ -2851,9 +3212,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" dependencies = [ "serde", ] @@ -2900,13 +3261,22 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.6", + "digest 0.10.7", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", ] [[package]] @@ -2949,6 +3319,15 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +[[package]] +name = "sketches-ddsketch" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" +dependencies = [ + "serde", +] + [[package]] name = "slab" version = "0.4.8" @@ -3008,7 +3387,7 @@ dependencies = [ "slog", "term", "thread_local", - "time 0.3.20", + "time 0.3.22", ] [[package]] @@ -3033,6 +3412,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "standback" version = "0.2.17" @@ -3131,9 +3516,71 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "sval" -version = "1.0.0-alpha.5" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45f6ee7c7b87caf59549e9fe45d6a69c75c8019e79e212a835c5da0e92f0ba08" +checksum = "8b031320a434d3e9477ccf9b5756d57d4272937b8d22cb88af80b7633a1b78b1" + +[[package]] +name = "sval_buffer" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf7e9412af26b342f3f2cc5cc4122b0105e9d16eb76046cd14ed10106cf6028" +dependencies = [ + "sval", + "sval_ref", +] + +[[package]] +name = "sval_dynamic" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0ef628e8a77a46ed3338db8d1b08af77495123cc229453084e47cd716d403cf" +dependencies = [ + "sval", +] + +[[package]] +name = "sval_fmt" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dc09e9364c2045ab5fa38f7b04d077b3359d30c4c2b3ec4bae67a358bd64326" +dependencies = [ + "itoa", + "ryu", + "sval", +] + +[[package]] +name = "sval_json" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada6f627e38cbb8860283649509d87bc4a5771141daa41c78fd31f2b9485888d" +dependencies = [ + "itoa", + "ryu", + "sval", +] + +[[package]] +name = "sval_ref" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703ca1942a984bd0d9b5a4c0a65ab8b4b794038d080af4eb303c71bc6bf22d7c" +dependencies = [ + "sval", +] + +[[package]] +name = "sval_serde" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830926cd0581f7c3e5d51efae4d35c6b6fc4db583842652891ba2f1bed8db046" +dependencies = [ + "serde", + "sval", + "sval_buffer", + "sval_fmt", +] [[package]] name = "syn" @@ -3148,9 +3595,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.15" +version = "2.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "2efbeae7acf4eabd6bcdcbd11c92f45231ddda7539edc7806bd1a04a03b24616" dependencies = [ "proc-macro2", "quote", @@ -3164,14 +3611,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6c454c27d9d7d9a84c7803aaa3c50cd088d2906fe3c6e42da3209aa623576a8" dependencies = [ "bincode", - "bitflags", + "bitflags 1.3.2", "flate2", "fnv", "lazy_static", "once_cell", "onig", "plist", - "regex-syntax", + "regex-syntax 0.6.29", "serde", "serde_derive", "serde_json", @@ -3193,6 +3640,163 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" +[[package]] +name = "tantivy" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aec540e9cebc88f523f67f596dee213e491f0c55961de013566f267a0c31f5e9" +dependencies = [ + "aho-corasick", + "arc-swap", + "async-trait", + "base64 0.21.2", + "bitpacking", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "downcast-rs", + "fail", + "fastdivide", + "fs4", + "htmlescape", + "itertools", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "murmurhash32", + "num_cpus", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror", + "time 0.3.22", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-analysis-contrib" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1b2d2ea57f12d9c7a1a33ec5420555a4adb70b8989ba2480dc9dc00c82adfba" +dependencies = [ + "derive_builder", + "either", + "fst", + "rustc-hash", + "tantivy", + "thiserror", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16099e96f0ede682084469b80d6909dc170aa2b11d2a45538b5b36b2a90090b9" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e32b024b26eab93eb8648faf08004356bf9d47376557ee4409f4b210163656" +dependencies = [ + "fastdivide", + "fnv", + "itertools", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7d12fdd6ec0f7e0962f129c03c696a85ec567734950cbb2b89af4a293ce342f" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time 0.3.22", +] + +[[package]] +name = "tantivy-fst" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc3c506b1a8443a3a65352df6382a1fb6a7afe1a02e871cee0d25e2c3d5f3944" +dependencies = [ + "byteorder", + "regex-syntax 0.6.29", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106d8f78ad1da4f0fdd526a0760c326c0573510d4dedabeb1962d35a35879797" +dependencies = [ + "combine", + "once_cell", + "regex", +] + +[[package]] +name = "tantivy-sstable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda34243d3ee64bd8f9ba74a3b0d05f4d07beff7767a727212e9b5a19c13dde7" +dependencies = [ + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b9e9470301b026ad3b95f79a791a2a3ee81f3ab16fbe412a9dd81ff834acf5" +dependencies = [ + "murmurhash32", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64186801b6e06b3a1c4275e23b517835ff4ecbb707318b838dc9de457c062200" +dependencies = [ + "serde", +] + [[package]] name = "tar" version = "0.4.38" @@ -3206,15 +3810,16 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.5.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ + "autocfg", "cfg-if 1.0.0", "fastrand", "redox_syscall 0.3.5", "rustix", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -3239,15 +3844,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] - [[package]] name = "thiserror" version = "1.0.40" @@ -3265,7 +3861,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.22", ] [[package]] @@ -3328,23 +3924,23 @@ dependencies = [ [[package]] name = "time" -version = "0.3.20" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" dependencies = [ "itoa", "libc", "num_threads", "serde", "time-core", - "time-macros 0.2.8", + "time-macros 0.2.9", ] [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" @@ -3358,9 +3954,9 @@ dependencies = [ [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" dependencies = [ "time-core", ] @@ -3395,9 +3991,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.27.0" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", "bytes", @@ -3408,18 +4004,18 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.22", ] [[package]] @@ -3434,20 +4030,19 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.23.4" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", "tokio", - "webpki", ] [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", @@ -3468,9 +4063,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" +checksum = "1ebafdf5ad1220cb59e7d17cf4d2c72015297b75b19a10472f99b89225089240" dependencies = [ "serde", "serde_spanned", @@ -3480,20 +4075,20 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.8" +version = "0.19.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" +checksum = "266f016b7f039eec8a1a80dfe6156b633d208b9fccca5e4db1d6775b0c4e34a7" dependencies = [ - "indexmap", + "indexmap 2.0.0", "serde", "serde_spanned", "toml_datetime", @@ -3514,16 +4109,58 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if 1.0.0", "pin-project-lite 0.2.9", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.22", +] + [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -3561,9 +4198,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-normalization" @@ -3598,9 +4235,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", "idna", @@ -3614,21 +4251,68 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + [[package]] name = "utf8parse" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be" +dependencies = [ + "getrandom 0.2.10", + "serde", +] + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "value-bag" -version = "1.0.0-alpha.9" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" +checksum = "d92ccd67fb88503048c01b59152a04effd0782d035a83a6d256ce6085f08f4a3" +dependencies = [ + "value-bag-serde1", + "value-bag-sval2", +] + +[[package]] +name = "value-bag-serde1" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0b9f3feef403a50d4d67e9741a6d8fc688bcbb4e4f31bd4aab72cc690284394" +dependencies = [ + "erased-serde", + "serde", + "serde_fmt", +] + +[[package]] +name = "value-bag-sval2" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b24f4146b6f3361e91cbf527d1fb35e9376c3c0cef72ca5ec5af6d640fad7d" dependencies = [ - "ctor", "sval", - "version_check", + "sval_buffer", + "sval_dynamic", + "sval_fmt", + "sval_json", + "sval_ref", + "sval_serde", ] [[package]] @@ -3661,11 +4345,10 @@ dependencies = [ [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] @@ -3689,9 +4372,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -3699,24 +4382,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.22", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3726,9 +4409,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3736,28 +4419,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.22", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -3819,7 +4502,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets 0.48.0", + "windows-targets", ] [[package]] @@ -3837,37 +4520,13 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-targets", ] [[package]] @@ -3971,9 +4630,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.1" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8970b36c66498d8ff1d66685dc86b91b29db0c7739899012f63a63814b4b28" +checksum = "ca0ace3845f0d96209f0375e6d367e3eb87eb65d27d445bdc9f1843a26f39448" dependencies = [ "memchr", ] @@ -4016,3 +4675,33 @@ name = "zeroize" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" + +[[package]] +name = "zstd" +version = "0.12.3+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "6.0.5+zstd.1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.8+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +dependencies = [ + "cc", + "libc", + "pkg-config", +] diff --git a/alexandrie.toml b/alexandrie.toml index b28f4caf..47fd777f 100644 --- a/alexandrie.toml +++ b/alexandrie.toml @@ -76,3 +76,6 @@ path = "syntect/dumps/syntaxes.dump" type = "dump" path = "syntect/dumps/themes.dump" theme_name = "frontier-contrast" + +[search] +path = "/tmp/tantivy" \ No newline at end of file diff --git a/crates/alexandrie/Cargo.toml b/crates/alexandrie/Cargo.toml index 5fa1249c..7e6dd783 100644 --- a/crates/alexandrie/Cargo.toml +++ b/crates/alexandrie/Cargo.toml @@ -50,6 +50,10 @@ percent-encoding = "2.2.0" diesel = { version = "2.0.3", features = ["r2d2", "chrono"] } diesel_migrations = "2.0.0" +# Text indexation and search +tantivy = "0.20" +tantivy-analysis-contrib = { version = "0.9", default-features = false, features = ["commons"] } + # async primitives async-std = { version = "1.12.0", features = ["attributes", "tokio1"] } diff --git a/crates/alexandrie/src/api/crates/publish.rs b/crates/alexandrie/src/api/crates/publish.rs index c4607fc1..b6250db0 100644 --- a/crates/alexandrie/src/api/crates/publish.rs +++ b/crates/alexandrie/src/api/crates/publish.rs @@ -4,12 +4,12 @@ use std::path::PathBuf; use std::pin::pin; use async_std::io::prelude::*; - use byteorder::{LittleEndian, ReadBytesExt}; use chrono::Utc; use diesel::dsl as sql; use diesel::prelude::*; use flate2::read::GzDecoder; +use log::warn; use ring::digest as hasher; use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; @@ -26,6 +26,7 @@ use crate::db::schema::*; use crate::db::Connection; use crate::db::DATETIME_FORMAT; use crate::error::{AlexError, Error}; +use crate::fts::TantivyDocument; use crate::utils; use crate::State; @@ -70,7 +71,7 @@ struct CrateMetaDependency { fn link_keywords( conn: &mut Connection, crate_id: i64, - keywords: Option>, + keywords: &Option>, ) -> Result<(), Error> { diesel::delete(crate_keywords::table.filter(crate_keywords::crate_id.eq(crate_id))) .execute(conn)?; @@ -116,7 +117,7 @@ fn link_keywords( fn link_categories( conn: &mut Connection, crate_id: i64, - categories: Option>, + categories: &Option>, ) -> Result<(), Error> { diesel::delete(crate_categories::table.filter(crate_categories::crate_id.eq(crate_id))) .execute(conn)?; @@ -354,10 +355,10 @@ pub(crate) async fn put(mut req: Request) -> tide::Result { }; //? Update keywords. - link_keywords(conn, krate.id, metadata.keywords)?; + link_keywords(conn, krate.id, &metadata.keywords)?; //? Update categories. - link_categories(conn, krate.id, metadata.categories)?; + link_categories(conn, krate.id, &metadata.categories)?; //? Update badges. link_badges(conn, krate.id, metadata.badges)?; @@ -396,6 +397,24 @@ pub(crate) async fn put(mut req: Request) -> tide::Result { .storage .store_crate(&crate_desc.name, crate_desc.vers.clone(), crate_bytes)?; + let id = krate.id; + let name = krate.name.clone(); + + // Index into full text index + let mut document: TantivyDocument = krate.into(); + if let Some(keywords) = metadata.keywords { + document.add_all_keywords(keywords); + } + if let Some(categories) = metadata.categories { + document.add_all_categories(categories); + } + + if let Err(error) = state.search.create_or_update(document) { + warn!("Can't convert crate '{id}' ({name}) into Tantivy document : {error}"); + } else { + state.search.commit()?; + } + //? Store the crate's readme. if let Some(rendered) = rendered_readme { state diff --git a/crates/alexandrie/src/api/crates/search.rs b/crates/alexandrie/src/api/crates/search.rs index 6c988b48..af18da9f 100644 --- a/crates/alexandrie/src/api/crates/search.rs +++ b/crates/alexandrie/src/api/crates/search.rs @@ -1,6 +1,5 @@ -use std::num::NonZeroU32; +use std::num::NonZeroUsize; -use diesel::dsl as sql; use diesel::prelude::*; use semver::Version; use serde::{Deserialize, Serialize}; @@ -35,14 +34,14 @@ struct SearchResult { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct SearchMeta { - pub total: i64, + pub total: usize, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct QueryParams { pub q: String, - pub per_page: Option, - pub page: Option, + pub per_page: Option, + pub page: Option, } /// Route to search through crates (used by `cargo search`). @@ -53,51 +52,39 @@ pub(crate) async fn get(req: Request) -> tide::Result { missing_params: &["q"], })?; let state = req.state().clone(); - let db = &state.db; - - //? Fetch the latest index changes. - // state.index.refresh()?; - let name = utils::canonical_name(params.q.as_str()); + let query = params.q; + let per_page = params + .per_page + .map(|v| v.get()) + .unwrap_or(crate::fts::DEFAULT_RESULT_PER_PAGE); + let page = params.page.map(|v| v.get()).unwrap_or(1) - 1; - //? Build the search pattern. - let name_pattern = format!("%{0}%", name.replace('\\', "\\\\").replace('%', "\\%")); + let searcher = &state.search; + // Run query on tantivy and get total and matching ids + // Perhaps should use suggest method as it allow to deal with "starts with", but I don't think + // that's what is expected. + let (total, ids) = searcher.search(&query, page * per_page, per_page)?; + let db = &state.db; let transaction = db.transaction(move |conn| { let state = req.state(); - //? Limit the result count depending on parameters. - let results = match (params.per_page, params.page) { - (Some(per_page), Some(page)) => { - //? Get search results for the given page number and entries per page. - crates::table - .filter(crates::canon_name.like(name_pattern.as_str())) - .limit(i64::from(per_page.get())) - .offset(i64::from((page.get() - 1) * per_page.get())) - .load::(conn)? - } - (Some(per_page), None) => { - //? Get the first page of search results with the given entries per page. - crates::table - .filter(crates::canon_name.like(name_pattern.as_str())) - .limit(i64::from(per_page.get())) - .load::(conn)? - } - _ => { - //? Get ALL the crates (might be too much, tbh). - crates::table - .filter(crates::canon_name.like(name_pattern.as_str())) - .load::(conn)? - } - }; - - //? Fetch the total result count. - let total = crates::table - .select(sql::count(crates::id)) - .filter(crates::canon_name.like(name_pattern.as_str())) - .first::(conn)?; - - let crates = results + // Get crate from database + let mut crates = crates::table + .filter(crates::id.eq_any(&ids)) + .load::(conn)?; + + // Sort database result by relevance since we lost ordering... + // (the `unwrap_or` call should be unreachable, but if it is reached, it would sort the crate towards the end) + crates.sort_unstable_by_key(|krate| { + ids.iter() + .position(|id| *id == krate.id) + .unwrap_or(ids.len()) + }); + + // Fetch missing informations from index + let crates = crates .into_iter() .map(|krate| { let latest = state.index.latest_record(krate.name.as_str())?; diff --git a/crates/alexandrie/src/api/crates/suggest.rs b/crates/alexandrie/src/api/crates/suggest.rs index 9f6e4712..93b367a5 100644 --- a/crates/alexandrie/src/api/crates/suggest.rs +++ b/crates/alexandrie/src/api/crates/suggest.rs @@ -1,14 +1,12 @@ use std::num::NonZeroU32; -use diesel::prelude::*; +use log::info; use semver::Version; use serde::{Deserialize, Serialize}; use tide::Request; use alexandrie_index::Indexer; -use crate::db::models::Crate; -use crate::db::schema::*; use crate::error::{AlexError, Error}; use crate::utils; use crate::State; @@ -37,37 +35,21 @@ pub(crate) async fn get(req: Request) -> tide::Result { .map_err(|_| AlexError::MissingQueryParams { missing_params: &["q"], })?; - let state = req.state().clone(); - let db = &state.db; - - //? Fetch the latest index changes. - // state.index.refresh()?; let name = utils::canonical_name(params.q); + let limit = params.limit.map_or(10, |limit| limit.get() as usize); - //? Build the search pattern. - let name_pattern = format!("%{0}%", name.replace('\\', "\\\\").replace('%', "\\%")); - - //? Limit the result count depending on parameters. - let limit = params.limit.map_or(10, |limit| i64::from(limit.get())); - - //? Fetch results. - let results = db - .run(move |conn| { - crates::table - .filter(crates::canon_name.like(name_pattern.as_str())) - .limit(limit) - .load::(conn) - }) - .await?; + info!("Suggester : {} & {}", name, limit); + let state = req.state().clone(); + let index = &state.index; - //? Fetch version information about these crates. + let results = state.search.suggest(name, limit)?; let suggestions = results .into_iter() .map(|krate| { - let latest = state.index.latest_record(krate.name.as_str())?; + let latest = index.latest_record(krate.to_lowercase().as_str())?; Ok(Suggestion { - name: krate.name, + name: krate, vers: latest.vers, }) }) diff --git a/crates/alexandrie/src/config/mod.rs b/crates/alexandrie/src/config/mod.rs index 642ef246..71976a04 100644 --- a/crates/alexandrie/src/config/mod.rs +++ b/crates/alexandrie/src/config/mod.rs @@ -19,6 +19,8 @@ use crate::db::Database; #[cfg(feature = "frontend")] pub use crate::config::frontend::*; +use crate::error::Error; +use crate::fts::Tantivy; use self::database::DatabaseConfig; @@ -32,6 +34,13 @@ pub struct GeneralConfig { max_crate_size: Option, } +/// Configuration for search index. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SearchConfig { + /// Path to the directory where Tantivy will store its index. + pub path: String, +} + /// The application configuration struct. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Config { @@ -45,6 +54,8 @@ pub struct Config { pub database: DatabaseConfig, /// The syntax-highlighting configuration. pub syntect: SyntectConfig, + /// Search config + pub search: SearchConfig, /// The frontend configuration. #[cfg(feature = "frontend")] pub frontend: FrontendConfig, @@ -68,6 +79,8 @@ pub struct State { pub db: Database, /// The syntect configuration. pub syntect: SyntectState, + /// Search config + pub search: Tantivy, /// The frontend configured state. #[cfg(feature = "frontend")] pub frontend: FrontendState, @@ -81,17 +94,20 @@ impl From for GeneralState { } } -impl From for State { - fn from(config: Config) -> State { - State { +impl TryFrom for State { + type Error = Error; + + fn try_from(config: Config) -> Result { + Ok(State { general: config.general.into(), index: config.index.into(), storage: config.storage.into(), db: Database::new(&config.database), syntect: config.syntect.into(), + search: config.search.try_into()?, #[cfg(feature = "frontend")] frontend: config.frontend.into(), - } + }) } } diff --git a/crates/alexandrie/src/error.rs b/crates/alexandrie/src/error.rs index f4aa52f4..44cfc3df 100644 --- a/crates/alexandrie/src/error.rs +++ b/crates/alexandrie/src/error.rs @@ -6,6 +6,8 @@ use hex::FromHexError as HexError; use io::Error as IOError; use json::Error as JSONError; use semver::{Error as SemverError, Version}; +use tantivy::directory::error::OpenDirectoryError; +use tantivy::TantivyError; use thiserror::Error; use toml::de::Error as TOMLError; @@ -49,6 +51,22 @@ pub enum Error { /// Storage-specific errors. #[error("{0}")] StorageError(#[from] StorageError), + /// Tantivy's errors. + #[error("{0}")] + TantivyError(#[from] TantivyError), + /// Open directory error + #[error("{0}")] + OpenDirectoryError(#[from] OpenDirectoryError), + /// Empty stop words. + #[error("Empty stop word filter")] + EmptyStopWord, + /// Tantivy's index is poisoned + #[error("Tantivy's index is poisoned: {0}")] + PoisonedError(String), + /// Missing id field or on of nae's field in index schema + /// Should never happen... + #[error("Missing {0} in Tantivy's schema")] + MissingField(&'static str), } /// The Error type for Alexandrie's own errors. diff --git a/crates/alexandrie/src/frontend/search.rs b/crates/alexandrie/src/frontend/search.rs index f4f8004c..8814417d 100644 --- a/crates/alexandrie/src/frontend/search.rs +++ b/crates/alexandrie/src/frontend/search.rs @@ -1,6 +1,5 @@ -use std::num::NonZeroU32; +use std::num::NonZeroUsize; -use diesel::dsl as sql; use diesel::prelude::*; use json::json; use serde::{Deserialize, Serialize}; @@ -20,74 +19,75 @@ use crate::State; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct SearchParams { pub q: String, - pub page: Option, + pub page: Option, } +/// Route to search through crates (used by `cargo search`) using tantivy index pub(crate) async fn get(req: Request) -> tide::Result { - let params = req.query::()?; - let searched_text = utils::canonical_name(params.q.as_str()); - let q = format!( - "%{0}%", - searched_text.replace('\\', "\\\\").replace('%', "\\%") - ); - + let params = req.query::().unwrap(); + let searched_text = params.q.clone(); let page_number = params.page.map_or_else(|| 1, |page| page.get()); + let offset = (page_number - 1) * crate::fts::DEFAULT_RESULT_PER_PAGE; + let user = req.get_author(); if req.state().is_login_required() && user.is_none() { return Ok(utils::response::redirect("/account/login")); } let state = req.state().clone(); - let db = &state.db; + let repo = &state.db; - let transaction = db.transaction(move |conn| { - let state = req.state(); + let (count, results) = state.search.search( + searched_text.clone(), + offset, + crate::fts::DEFAULT_RESULT_PER_PAGE, + )?; - //? Get the total count of search results. - let total_results = crates::table - .select(sql::count(crates::id)) - .filter(crates::canon_name.like(q.as_str())) - .first::(conn)?; + let page_count = count / crate::fts::DEFAULT_RESULT_PER_PAGE + + if count > 0 && count % crate::fts::DEFAULT_RESULT_PER_PAGE == 0 { + 0 + } else { + 1 + }; - //? Get the search results for the given page number. - let results: Vec = crates::table - .filter(crates::canon_name.like(q.as_str())) - .limit(15) - .offset(15 * i64::from(page_number - 1)) - .load(conn)?; + let transaction = repo.transaction(move |conn| { + let state = req.state(); - let results = results + let results: Vec<(Crate, Vec)> = results .into_iter() - .map(|result| { + .map(|v| { + let krate = crates::table + .filter(crates::id.eq(v)) + .first::(conn)?; let keywords = crate_keywords::table .inner_join(keywords::table) .select(keywords::name) - .filter(crate_keywords::crate_id.eq(result.id)) + .filter(crate_keywords::crate_id.eq(krate.id)) .load::(conn)?; - Ok((result, keywords)) + Ok((krate, keywords)) }) - .collect::)>, Error>>()?; - - //? Make page number starts counting from 1 (instead of 0). - let page_count = (total_results / 15 - + if total_results > 0 && total_results % 15 == 0 { - 0 - } else { - 1 - }) as u32; + .collect::>()?; let encoded_q = percent_encoding::percent_encode( params.q.as_bytes(), percent_encoding::NON_ALPHANUMERIC, ); let next_page = if page_number < page_count { - Some(format!("/search?q={0}&page={1}", encoded_q, page_number + 1)) + Some(format!( + "/search?q={0}&page={1}", + encoded_q, + page_number + 1 + )) } else { None }; let prev_page = if page_number > 1 { - Some(format!("/search?q={0}&page={1}", encoded_q, page_number - 1)) + Some(format!( + "/search?q={0}&page={1}", + encoded_q, + page_number - 1 + )) } else { None }; @@ -96,8 +96,8 @@ pub(crate) async fn get(req: Request) -> tide::Result { let context = json!({ "user": user, "instance": &state.frontend.config, - "searched_text": params.q, - "total_results": total_results, + "searched_text": searched_text, + "total_results": count, "pagination": { "current": page_number, "total_count": page_count, diff --git a/crates/alexandrie/src/fts/document.rs b/crates/alexandrie/src/fts/document.rs new file mode 100644 index 00000000..d3af7d12 --- /dev/null +++ b/crates/alexandrie/src/fts/document.rs @@ -0,0 +1,136 @@ +use std::fmt::Formatter; + +use crate::db::models::Crate; +use tantivy::schema::Schema; +use tantivy::Document; + +use crate::error::Error; + +/// Represent a crate. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TantivyDocument { + id: i64, + name: String, + description: Option, + keywords: Vec, + categories: Vec, +} + +impl std::fmt::Display for TantivyDocument { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "id: {}, name: {}", self.id, self.name)?; + if let Some(description) = &self.description { + write!(f, ", '{}'", description)?; + } + + if self.keywords.is_empty() { + write!(f, ", no keyword")?; + } else { + let keywords = self.keywords.join(", "); + write!(f, ", keywords : {keywords}")?; + } + + if self.categories.is_empty() { + write!(f, ", no categories")?; + } else { + let categories = self.categories.join(", "); + write!(f, ", categories : {categories}")?; + } + + Ok(()) + } +} + +impl From for TantivyDocument { + fn from(value: Crate) -> Self { + Self { + id: value.id, + name: value.name, + description: value.description, + keywords: vec![], + categories: vec![], + } + } +} + +impl TantivyDocument { + pub fn new(id: i64, name: String) -> Self { + Self { + id, + name, + description: None, + keywords: Vec::with_capacity(5), + categories: Vec::with_capacity(5), + } + } + + pub fn try_into(self, schema: &Schema) -> Result { + // Can't implement From because I need to use a tuple to hold schema and both + // tuple and Document are in another crate :-( + let mut document = Document::new(); + + let id_field = schema.get_field(super::ID_FIELD_NAME)?; + let name_field = schema.get_field(super::NAME_FIELD_NAME)?; + let name_full_field = schema.get_field(super::NAME_FIELD_NAME_FULL)?; + let name_prefix_field = schema.get_field(super::NAME_FIELD_PREFIX_NAME)?; + let category_field = schema.get_field(super::CATEGORY_FIELD_NAME)?; + let keyword_field = schema.get_field(super::KEYWORD_FIELD_NAME)?; + + document.add_i64(id_field, self.id); + document.add_text(name_field, &self.name); + document.add_text(name_full_field, &self.name); + document.add_text(name_prefix_field, self.name); + + // For the following fields we will not fail if they are not in schema + // but TODO add warn + if let Some(description) = &self.description { + let description_field = schema.get_field(super::DESCRIPTION_FIELD_NAME)?; + document.add_text(description_field, description); + } + + self.keywords + .clone() + .into_iter() + .for_each(|v| document.add_text(keyword_field, v)); + + self.categories + .clone() + .into_iter() + .for_each(|v| document.add_text(category_field, v)); + + Ok(document) + } + + pub fn id(&self) -> i64 { + self.id + } + + /// Set crate's description + pub fn set_description(&mut self, description: String) { + self.description = Some(description); + } + + /// Add new crate's keyword + pub fn add_keyword(&mut self, keyword: String) { + self.keywords.push(keyword); + } + + /// Add all keywords + pub fn add_all_keywords(&mut self, keywords: Vec) { + for keyword in keywords { + self.add_keyword(keyword); + } + } + + /// Add new crate's category + pub fn add_category(&mut self, category: String) { + self.categories.push(category); + } + + /// Add all keywords + pub fn add_all_categories(&mut self, categories: Vec) { + for category in categories { + self.add_category(category); + } + } +} diff --git a/crates/alexandrie/src/fts/index.rs b/crates/alexandrie/src/fts/index.rs new file mode 100644 index 00000000..a6f8ae64 --- /dev/null +++ b/crates/alexandrie/src/fts/index.rs @@ -0,0 +1,447 @@ +use std::convert::TryFrom; +use std::num::NonZeroUsize; +use std::sync::RwLock; + +use diesel::prelude::*; +use log::{debug, info, warn}; +use tantivy::collector::{Count, TopDocs}; +use tantivy::directory::MmapDirectory; +use tantivy::query::{AllQuery, QueryParser}; +use tantivy::schema::{NumericOptions, Schema, TextFieldIndexing, TextOptions}; +use tantivy::tokenizer::{ + Language, LowerCaser, RawTokenizer, SimpleTokenizer, StopWordFilter, TextAnalyzer, + TokenizerManager, +}; +use tantivy::{ + Index as TantivyIndex, IndexReader, IndexWriter, Opstamp, ReloadPolicy, TantivyError, Term, +}; +use tantivy_analysis_contrib::commons::EdgeNgramTokenFilter; + +use crate::config::SearchConfig; +use crate::db::models::Crate; +use crate::db::schema::*; +use crate::db::Database; +use crate::error::Error; +use crate::fts::TantivyDocument; + +const NUMBER_RESULT_PER_PAGE: i64 = 1000; + +type CrateKeywordCategory = (Vec, Vec<(i64, String)>, Vec<(i64, String)>); + +/// Helper for using Tantivy +pub struct Tantivy { + index_reader: IndexReader, + /// There can only be one index writer at a time (see https://tantivy-search.github.io/examples/basic_search.html) + /// so we keep only one here. It has its own pool. + index_writer: RwLock, + /// Index schema + schema: Schema, + /// Search tokenizer manager + search_tokenizer_manager: TokenizerManager, +} + +impl TryFrom for Tantivy { + type Error = crate::error::Error; + + fn try_from(search: SearchConfig) -> Result { + let analyzer_name = "alexandrie"; + // Not tokenized + let analyzer_name_full = "alexandrie_full"; + // Prefix + let analyzer_prefix_name = "alexandrie_prefix"; + + // Create index directory + let path = search.path.as_str(); + std::fs::create_dir_all(path)?; + let directory = MmapDirectory::open(path)?; + + // Index options for all fields (perhaps keywords and category could have another analysis & options) + let options = TextOptions::default() + .set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer(analyzer_name) + .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions), + ) + .set_stored(); + + let options_full = TextOptions::default() + .set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer(analyzer_name_full) + .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions), + ) + .set_stored(); + + let options_prefixes = TextOptions::default() + .set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer(analyzer_prefix_name) + .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions), + ) + .set_stored(); + + let id_options = NumericOptions::default().set_stored().set_indexed(); + + // Schema of a document, we index and store (though storing isn't really necessary): + // * name : crate's name + // * description: crate's description + // * categories: crate's categories + // * keywords: crate's keywords + let mut schema_builder = Schema::builder(); + // Easier with i64 than u64 because IDs are i64 i db module. + schema_builder.add_i64_field(super::ID_FIELD_NAME, id_options); + + // For these fields, we could avoid storing original data if we get things in the database right + // after searching indices + schema_builder.add_text_field(super::NAME_FIELD_NAME, options.clone()); + schema_builder.add_text_field(super::NAME_FIELD_NAME_FULL, options_full.clone()); + schema_builder.add_text_field(super::NAME_FIELD_PREFIX_NAME, options_prefixes); + schema_builder.add_text_field(super::DESCRIPTION_FIELD_NAME, options.clone()); + schema_builder.add_text_field(super::CATEGORY_FIELD_NAME, options_full); + schema_builder.add_text_field(super::KEYWORD_FIELD_NAME, options); + let schema = schema_builder.build(); + + // Analysis a tokenizer that tokenizes on non-alphanumeric characters + // A filter that removes common english words (the, a, ...etc) + // A filter that lowercase words + let stop_words = + StopWordFilter::new(Language::English).ok_or(Self::Error::EmptyStopWord)?; + let analyzer = TextAnalyzer::builder(SimpleTokenizer::default()) + .filter(stop_words) + .filter(LowerCaser) + .build(); + + let analyzer_full = TextAnalyzer::builder(RawTokenizer::default()) + .filter(LowerCaser) + .build(); + + let analyzer_prefix = TextAnalyzer::builder(SimpleTokenizer::default()) + .filter(LowerCaser) + .filter(EdgeNgramTokenFilter::new(NonZeroUsize::new(1).unwrap(), None, false).unwrap()) + .build(); + + let index = TantivyIndex::open_or_create(directory, schema.clone())?; + // Register analyzer + index.tokenizers().register(analyzer_name, analyzer.clone()); + index + .tokenizers() + .register(analyzer_name_full, analyzer_full.clone()); + index + .tokenizers() + .register(analyzer_prefix_name, analyzer_prefix); + + // Create an analyzer manager for search: on name prefix we do not want to apply + // the edge ngram filter: more efficient & less noise + // We need to have the other tokenizer registered so that search work on any field properly + let search_tokenizer_manager = TokenizerManager::new(); + search_tokenizer_manager.register(analyzer_name, analyzer); + search_tokenizer_manager.register(analyzer_name_full, analyzer_full); + search_tokenizer_manager.register( + analyzer_prefix_name, + TextAnalyzer::builder(SimpleTokenizer::default()) + .filter(LowerCaser) + .build(), + ); + + // Get an index writer with 50MB of heap + let index_writer = RwLock::new(index.writer(50_000_000)?); + + let index_reader = index + .reader_builder() + .reload_policy(ReloadPolicy::OnCommit) + .try_into()?; + + Ok(Self { + index_reader, + index_writer, + schema, + search_tokenizer_manager, + }) + } +} + +impl Tantivy { + pub fn schema(&self) -> &Schema { + &self.schema + } + + /// Method that create or update a document in Tantivy index. As there is no update, we need + /// to first delete the document then create a new document. + pub fn create_or_update(&self, document: TantivyDocument) -> Result<(), Error> { + let id = document.id(); + let document = document.try_into(&self.schema)?; + let field = self.schema.get_field(super::ID_FIELD_NAME)?; + let term = Term::from_field_i64(field, id); + let index_writer = self + .index_writer + .read() + .map_err(|error| Error::PoisonedError(error.to_string()))?; + index_writer.delete_term(term); + index_writer.add_document(document)?; + + Ok(()) + } + + pub fn delete_all_documents(&self) -> Result { + let index_writer = self + .index_writer + .read() + .map_err(|error| Error::PoisonedError(error.to_string()))?; + Ok(index_writer.delete_all_documents()?) + } + + /// Commit all pending changes inside the index. + pub fn commit(&self) -> Result { + let mut index_writer = self + .index_writer + .write() + .map_err(|error| Error::PoisonedError(error.to_string()))?; + Ok(index_writer.commit()?) + } + + /// Search document by default through all crate's name index. This allows having search + /// as you type (using prefixes) while increasing relevance when there's a matching word + /// or if the whole text matches a crate's name (using the other crate's name indices). + pub fn suggest(&self, query: String, limit: usize) -> Result, TantivyError> { + let searcher = self.index_reader.searcher(); + + let name = self.schema.get_field(super::NAME_FIELD_NAME).unwrap(); + let name_full = self.schema.get_field(super::NAME_FIELD_NAME_FULL).unwrap(); + let name_prefix = self + .schema + .get_field(super::NAME_FIELD_PREFIX_NAME) + .unwrap(); + + let mut query_parser = QueryParser::new( + self.schema.clone(), + vec![name_full, name_prefix], + self.search_tokenizer_manager.clone(), + ); + + query_parser.set_field_boost(name_full, 10.0); + query_parser.set_field_boost(name, 5.0); + query_parser.set_field_boost(name_prefix, 1.0); + + let query = query_parser.parse_query(&query)?; + + info!("Query : {:?}", query); + + let results = searcher.search(&query, &TopDocs::with_limit(limit))?; + + info!("Result : {:?}", results); + + let results = results + .into_iter() + .map(|(score, doc_address)| { + let retrieve_doc = searcher.doc(doc_address).unwrap(); + + let x = retrieve_doc.get_all(name).next(); + if let Some(n) = x { + info!("Score : {} / Crate : {:?}", score, n); + } + x.cloned() + }) + .filter_map(|v| v.and_then(|i| i.as_text().map(|t| t.to_owned()))) + .collect(); + + Ok(results) + } + + /// Search documents. Return document count & database IDs. + pub fn search>( + &self, + query: Q, + offset: usize, + limit: usize, + ) -> Result<(usize, Vec), TantivyError> { + let query = query.as_ref().trim(); + + let searcher = self.index_reader.searcher(); + + let id = self.schema.get_field(super::ID_FIELD_NAME).unwrap(); + let name = self.schema.get_field(super::NAME_FIELD_NAME).unwrap(); + let name_full = self.schema.get_field(super::NAME_FIELD_NAME_FULL).unwrap(); + let description = self + .schema + .get_field(super::DESCRIPTION_FIELD_NAME) + .unwrap(); + let categories = self.schema.get_field(super::CATEGORY_FIELD_NAME).unwrap(); + let keywords = self.schema.get_field(super::KEYWORD_FIELD_NAME).unwrap(); + + let query = if query.is_empty() { + Box::new(AllQuery) + } else { + let mut query_parser = QueryParser::for_index( + searcher.index(), + vec![name, name_full, description, categories, keywords], + ); + + // Exact matches (on name_full) have a big boost + query_parser.set_field_boost(name_full, 10.0); + query_parser.set_field_boost(name, 5.0); + // Categories shouldn't be free (there is a list) so a nice boost + query_parser.set_field_boost(categories, 1.0); + // Keywords are free + query_parser.set_field_boost(keywords, 0.5); + // description & readme are full text they got a lower boost (if there is a match, that might not be relevant) + query_parser.set_field_boost(description, 0.2); + + query_parser.parse_query(query)? + }; + + info!("Query offset={} query limit={}", offset, limit); + + let (count, results) = searcher.search( + &query, + &(Count, TopDocs::with_limit(limit).and_offset(offset)), + )?; + + let results = results + .into_iter() + .filter_map(|(score, doc_address)| { + let retrieve_doc = match searcher.doc(doc_address) { + Ok(retrieve_doc) => retrieve_doc, + Err(error) => { + warn!("Could not find document {doc_address:?} : {error}"); + return None; + } + }; + + if let Some(name) = retrieve_doc.get_all(name).next() { + info!("Score : {} / Crate : {:?}", score, name); + } + + let mut field = retrieve_doc.get_all(id); + if let Some(x) = field.next() { + x.as_i64() + } else { + warn!("Could not find field id"); + None + } + }) + .collect(); + + Ok((count, results)) + } + + pub async fn index_all(&self, repo: &Database) -> Result<(), Error> { + info!("Index all crates"); + self.delete_all_documents()?; + self.commit()?; + let mut start: i64 = 0; + let mut count_crate = 0; + + 'indexing: loop { + let result: Result, Error> = repo + .run(move |conn| { + debug!( + "Querying crates from {start} to {}", + start + NUMBER_RESULT_PER_PAGE + ); + let krates = crates::table + .order_by(crates::id.asc()) + .limit(NUMBER_RESULT_PER_PAGE) + .offset(start) + .load::(conn)?; + + debug!("{} crates fetched", krates.len()); + + if krates.is_empty() { + return Ok(None); + } + + let ids = krates + .clone() + .into_iter() + .map(|c| c.id) + .collect::>(); + + debug!("Crates {:?}", ids); + + let keywords = keywords::table + .inner_join(crate_keywords::table) + .select((crate_keywords::crate_id, keywords::name)) + .filter(crate_keywords::crate_id.eq_any(&ids)) + .order_by(crate_keywords::crate_id.asc()) + .load::<(i64, String)>(conn)?; + + let categories = categories::table + .inner_join(crate_categories::table) + .select((crate_categories::crate_id, categories::name)) + .filter(crate_categories::crate_id.eq_any(&ids)) + .order_by(crate_categories::crate_id.asc()) + .load::<(i64, String)>(conn)?; + + Ok(Some((krates, keywords, categories))) + }) + .await; + + let result = result?; + + if let Some((krates, keywords, categories)) = result { + start += krates.len() as i64; + let mut keywords_iterator = keywords.into_iter().peekable(); + let mut categories_iterator = categories.into_iter().peekable(); + + let mut current_keyword: Option<(i64, String)> = keywords_iterator.next(); + let mut current_category: Option<(i64, String)> = categories_iterator.next(); + + for krate in krates.into_iter() { + debug!("crate {:?}", krate); + // Create a document with database ID and crate name + let id = krate.id; + let name = krate.name.clone(); + + let mut doc: TantivyDocument = krate.into(); + + // Skip keywords that might be orphan and add keywords that match ids + while let Some((crate_id, keyword)) = current_keyword { + if crate_id > id { + current_keyword = Some((crate_id, keyword)); + break; + } + + if crate_id == id { + doc.add_keyword(keyword); + } + + current_keyword = keywords_iterator.next(); + } + + // Skip categories that might be orphan and add categories that match ids + while let Some((crate_id, category)) = current_category { + if crate_id > id { + current_category = Some((crate_id, category)); + break; + } + + if crate_id == id { + doc.add_category(category); + } + + current_category = categories_iterator.next(); + } + + // TODO get README + + if let Err(error) = self.create_or_update(doc) { + warn!( + "Can't convert crate '{id}' ({name}) into Tantivy document : {error}" + ); + } + count_crate += 1; + + if count_crate % 1000 == 0 { + info!("{} crates indexed", count_crate); + } + } + } else { + info!("End indexing {start} crates"); + self.commit()?; + break 'indexing; + } + } + + Ok(()) + } +} diff --git a/crates/alexandrie/src/fts/mod.rs b/crates/alexandrie/src/fts/mod.rs new file mode 100644 index 00000000..26d42ba9 --- /dev/null +++ b/crates/alexandrie/src/fts/mod.rs @@ -0,0 +1,27 @@ +//! Full-text search module +mod document; +mod index; + +pub(crate) use document::TantivyDocument; +pub(crate) use index::Tantivy; + +/// Default number of result per page +/// Perhaps should make this configurable in toml. +pub const DEFAULT_RESULT_PER_PAGE: usize = 15; + +/// Database ID. +const ID_FIELD_NAME: &str = "id"; +/// Tokenized version of crate's name. +const NAME_FIELD_NAME: &str = "name"; +/// Another index of crate's name, this one +/// isn't tokenized. So it's an exact match +/// but case-insensitive. It's here to improve +/// results relevance. +const NAME_FIELD_NAME_FULL: &str = "name.full"; +/// A third index for crate's name. This one is for +/// suggestion in the search bar. It's tokenized and +/// contains word's prefixes to do "search as you type". +const NAME_FIELD_PREFIX_NAME: &str = "name.prefix"; +const DESCRIPTION_FIELD_NAME: &str = "description"; +const CATEGORY_FIELD_NAME: &str = "category"; +const KEYWORD_FIELD_NAME: &str = "keyword"; diff --git a/crates/alexandrie/src/main.rs b/crates/alexandrie/src/main.rs index 71340839..c372b949 100644 --- a/crates/alexandrie/src/main.rs +++ b/crates/alexandrie/src/main.rs @@ -58,6 +58,9 @@ pub mod utils; #[cfg(feature = "frontend")] pub mod frontend; +/// Full text search +pub mod fts; + use crate::config::Config; use crate::error::Error; use crate::utils::build; @@ -241,7 +244,9 @@ async fn run() -> Result<(), Error> { #[cfg(feature = "frontend")] let frontend_config = config.frontend.clone(); - let state: Arc = Arc::new(config.into()); + let state: config::State = config.try_into()?; + + let state = Arc::new(state); log::info!("starting Alexandrie (version: {})", build::short()); @@ -250,6 +255,9 @@ async fn run() -> Result<(), Error> { state.db.run(|conn| conn.run_pending_migrations(db::MIGRATIONS).map(|_| ())).await .expect("migration execution error"); + let database = &state.db; + state.search.index_all(database).await?; + let mut app = tide::with_state(Arc::clone(&state)); log::info!("setting up request logger middleware"); diff --git a/docker/mysql/alexandrie.toml b/docker/mysql/alexandrie.toml index af326f60..898633cd 100644 --- a/docker/mysql/alexandrie.toml +++ b/docker/mysql/alexandrie.toml @@ -53,3 +53,6 @@ path = "syntect/dumps/syntaxes.dump" type = "dump" path = "syntect/dumps/themes.dump" theme_name = "frontier-contrast" + +[search] +path = "appdata/tantivy" \ No newline at end of file diff --git a/docker/postgres/alexandrie.toml b/docker/postgres/alexandrie.toml index 44d98ad1..5ff086fb 100644 --- a/docker/postgres/alexandrie.toml +++ b/docker/postgres/alexandrie.toml @@ -51,3 +51,6 @@ path = "syntect/dumps/syntaxes.dump" type = "dump" path = "syntect/dumps/themes.dump" theme_name = "frontier-contrast" + +[search] +path = "appdata/tantivy" \ No newline at end of file diff --git a/docker/sqlite/alexandrie.toml b/docker/sqlite/alexandrie.toml index 1c82fe19..6624b63d 100644 --- a/docker/sqlite/alexandrie.toml +++ b/docker/sqlite/alexandrie.toml @@ -51,3 +51,6 @@ path = "syntect/dumps/syntaxes.dump" type = "dump" path = "syntect/dumps/themes.dump" theme_name = "frontier-contrast" + +[search] +path = "appdata/tantivy" \ No newline at end of file diff --git a/docs/src/programmatic-api/crates/search/get.md b/docs/src/programmatic-api/crates/search/get.md index 82e59e3b..f6479616 100644 --- a/docs/src/programmatic-api/crates/search/get.md +++ b/docs/src/programmatic-api/crates/search/get.md @@ -15,8 +15,8 @@ HTTP Query Parameters This endpoint accepts the following query parameters: - **(required)** `q`: The query string for the search (like `serde json` to possibly find `serde_json`). -- `page`: The non-zero page number to retrive (defaults to `1`), ignored if `per_page` is missing. -- `per_page`: The non-zero number of results per page (infinite by default). +- `page`: The non-zero page number to retrive (defaults to `1`). +- `per_page`: The non-zero number of results per page (default to `15`). Responses ---------