diff --git a/.gitignore b/.gitignore index cc51f186d23..8275b6c2017 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ # will have compiled files and executables **/target/** **/proptest-regressions +**/perf.data* +**/flamegraph.svg # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html diff --git a/Cargo.lock b/Cargo.lock index bb51f597386..e1b8c4f0399 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,9 +134,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47594e438a243791dba58124b6669561f5baa14cb12046641d8008bf035e5a25" +checksum = "f523b4e98ba6897ae90994bc18423d9877c54f9047b06a00ddc8122a957b1c70" dependencies = [ "async-trait", "axum-core", @@ -163,9 +163,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a671c9ae99531afdd5d3ee8340b8da547779430689947144c140fc74a740244" +checksum = "d3ddbd16eabff8b45f21b98671fddcc93daaa7ac4c84f8473693437226040de5" dependencies = [ "async-trait", "bytes", @@ -385,6 +385,21 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chitchat" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41474c726dd5fe2e90b4653fb0fb932b5469986e7f8da76a304df6ea7f9e1a9c" +dependencies = [ + "anyhow", + "bytes", + "rand", + "serde", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "chrono" version = "0.4.19" @@ -396,7 +411,7 @@ dependencies = [ "num-integer", "num-traits", "serde", - "time 0.1.44", + "time 0.1.43", "wasm-bindgen", "winapi 0.3.9", ] @@ -431,20 +446,29 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.6" +version = "3.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8c93436c21e4698bacadf42917db28b23017027a4deccb35dbe47a7e7840123" +checksum = "3124f3f75ce09e22d1410043e1e24f2ecc44fad3afe4f08408f1f7663d68da2b" dependencies = [ "atty", "bitflags", + "clap_lex", "indexmap", - "os_str_bytes", "strsim", "termcolor", "textwrap 0.15.0", "yaml-rust", ] +[[package]] +name = "clap_lex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "cmake" version = "0.1.48" @@ -476,22 +500,21 @@ dependencies = [ [[package]] name = "console-api" -version = "0.1.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc347c19eb5b940f396ac155822caee6662f850d97306890ac3773ed76c90c5a" +checksum = "24cb05777feccbb2642d4f2df44d0505601a2cd88ca517d8c913f263a5a8dc8b" dependencies = [ - "prost 0.9.0", - "prost-types 0.9.0", - "tonic 0.6.2", - "tonic-build 0.6.2", + "prost", + "prost-types", + "tonic", "tracing-core", ] [[package]] name = "console-subscriber" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "565a7dfea2d10dd0e5c57cc394d5d441b1910960d8c9211ed14135e0e6ec3a20" +checksum = "f8bd6b23feb4180ccf20cefca9357262818443aafb7159b5e503d170d442a872" dependencies = [ "console-api", "crossbeam-channel", @@ -499,13 +522,13 @@ dependencies = [ "futures", "hdrhistogram", "humantime", - "prost-types 0.9.0", + "prost-types", "serde", "serde_json", "thread_local", "tokio", "tokio-stream", - "tonic 0.6.2", + "tonic", "tracing", "tracing-core", "tracing-subscriber", @@ -769,7 +792,7 @@ dependencies = [ "pq-sys", "r2d2", "serde_json", - "time 0.1.44", + "time 0.1.43", "uuid 0.6.5", ] @@ -972,9 +995,9 @@ checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" [[package]] name = "flate2" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" dependencies = [ "cfg-if 1.0.0", "crc32fast", @@ -1001,7 +1024,7 @@ dependencies = [ "futures-sink", "nanorand", "pin-project", - "spin 0.9.2", + "spin 0.9.3", ] [[package]] @@ -1037,9 +1060,9 @@ dependencies = [ [[package]] name = "fragile" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da1b8f89c5b5a5b7e59405cfcf0bb9588e5ed19f0b57a4cd542bbba3f164a6d" +checksum = "e9d758e60b45e8d749c89c1b389ad8aee550f86aa12e2b9298b546dda7a82ab1" [[package]] name = "fs2" @@ -1165,14 +1188,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if 1.0.0", "js-sys", "libc", - "wasi 0.10.0+wasi-snapshot-preview1", + "wasi 0.10.2+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -1195,9 +1218,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "h2" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" dependencies = [ "bytes", "fnv", @@ -1208,7 +1231,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.6.9", + "tokio-util 0.7.1", "tracing", ] @@ -1265,15 +1288,6 @@ dependencies = [ "http", ] -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.4.0" @@ -1361,9 +1375,9 @@ checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" [[package]] name = "httparse" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" +checksum = "6330e8a36bd8c859f3fa6d9382911fbb7147ec39807f63b923933a247240b9ba" [[package]] name = "httpdate" @@ -1462,9 +1476,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg", "hashbrown", @@ -1477,6 +1491,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -1497,9 +1514,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" [[package]] name = "ipnetwork" @@ -1542,9 +1559,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" dependencies = [ "wasm-bindgen", ] @@ -1557,9 +1574,9 @@ checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" [[package]] name = "json_comments" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab25c689752fbb49903458495d3e71853229e262a2e6136325359e0705606483" +checksum = "41ee439ee368ba4a77ac70d04f14015415af8600d6c894dc1f11bd79758c57d5" [[package]] name = "lazy_static" @@ -1581,9 +1598,9 @@ checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" [[package]] name = "libc" -version = "0.2.121" +version = "0.2.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" +checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50" [[package]] name = "libloading" @@ -1635,10 +1652,11 @@ checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] name = "lock_api" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ + "autocfg", "scopeguard", ] @@ -1666,9 +1684,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.7.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb87f3080f6d1d69e8c564c0fcfde1d7aa8cc451ce40cae89479111f03bc0eb" +checksum = "32613e41de4c47ab04970c348ca7ae7382cf116625755af070b008a15516a889" dependencies = [ "hashbrown", ] @@ -1728,10 +1746,11 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "measure_time" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f07966480d8562b3622f51df0b4e3fe6ea7ddb3b48b19b0f44ef863c455bdf9" +checksum = "56220900f1a0923789ecd6bf25fbae8af3b2f1ff3e9e297fc9b6b8674dd4d852" dependencies = [ + "instant", "log", ] @@ -1804,12 +1823,11 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.4.4" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" dependencies = [ "adler", - "autocfg", ] [[package]] @@ -2129,9 +2147,6 @@ name = "os_str_bytes" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" -dependencies = [ - "memchr", -] [[package]] name = "output_vt100" @@ -2177,7 +2192,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ "lock_api", - "parking_lot_core 0.9.1", + "parking_lot_core 0.9.2", ] [[package]] @@ -2196,9 +2211,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" dependencies = [ "cfg-if 1.0.0", "libc", @@ -2263,9 +2278,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "plotters" @@ -2342,9 +2357,9 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c038cb5319b9c704bf9c227c261d275bfec0ad438118a2787ce47944fb228b" +checksum = "c89f989ac94207d048d92db058e4f6ec7342b0971fc58d1271ca148b799b3563" dependencies = [ "ansi_term", "ctor", @@ -2404,9 +2419,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" dependencies = [ "unicode-xid", ] @@ -2464,61 +2479,31 @@ dependencies = [ [[package]] name = "prost" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" -dependencies = [ - "bytes", - "prost-derive 0.9.0", -] - -[[package]] -name = "prost" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bd5316aa8f5c82add416dfbc25116b84b748a21153f512917e8143640a71bbd" +checksum = "a07b0857a71a8cb765763950499cae2413c3f9cede1133478c43600d9e146890" dependencies = [ "bytes", - "prost-derive 0.10.0", + "prost-derive", ] [[package]] name = "prost-build" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" -dependencies = [ - "bytes", - "heck 0.3.3", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prost 0.9.0", - "prost-types 0.9.0", - "regex", - "tempfile", - "which", -] - -[[package]] -name = "prost-build" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328f9f29b82409216decb172d81e936415d21245befa79cd34c3f29d87d1c50b" +checksum = "120fbe7988713f39d780a58cf1a7ef0d7ef66c6d87e5aa3438940c05357929f4" dependencies = [ "bytes", "cfg-if 1.0.0", "cmake", - "heck 0.4.0", + "heck", "itertools", "lazy_static", "log", "multimap", "petgraph", - "prost 0.10.0", - "prost-types 0.10.0", + "prost", + "prost-types", "regex", "tempfile", "which", @@ -2526,22 +2511,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "prost-derive" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df35198f0777b75e9ff669737c6da5136b59dba33cf5a010a6d1cc4d56defc6f" +checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" dependencies = [ "anyhow", "itertools", @@ -2552,22 +2524,12 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" -dependencies = [ - "bytes", - "prost 0.9.0", -] - -[[package]] -name = "prost-types" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "926681c118ae6e512a3ccefd4abbe5521a14f4cc1e207356d4d00c0b7f2006fd" +checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" dependencies = [ "bytes", - "prost 0.10.0", + "prost", ] [[package]] @@ -2632,7 +2594,7 @@ dependencies = [ "atty", "byte-unit", "chrono", - "clap 3.1.6", + "clap 3.1.10", "colored", "console-subscriber", "futures", @@ -2667,7 +2629,7 @@ dependencies = [ "thousands", "time 0.3.9", "tokio", - "tokio-util 0.7.0", + "tokio-util 0.7.1", "toml", "tracing", "tracing-opentelemetry", @@ -2680,12 +2642,12 @@ version = "0.2.1" dependencies = [ "anyhow", "async-trait", + "chitchat", "flume", "itertools", "quickwit-common", "quickwit-config", "quickwit-proto", - "scuttlebutt", "serde", "serde_json", "tempfile", @@ -2761,7 +2723,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "tokio-util 0.7.0", + "tokio-util 0.7.1", "tracing", "uuid 0.8.2", ] @@ -2892,11 +2854,11 @@ dependencies = [ name = "quickwit-proto" version = "0.2.1" dependencies = [ - "prost 0.10.0", - "prost-build 0.10.0", + "prost", + "prost-build", "serde", - "tonic 0.7.0", - "tonic-build 0.7.0", + "tonic", + "tonic-build", ] [[package]] @@ -2956,7 +2918,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "tokio-util 0.7.0", + "tokio-util 0.7.1", "tracing", "tracing-futures", "tracing-opentelemetry", @@ -2970,6 +2932,7 @@ dependencies = [ "assert-json-diff", "async-trait", "bytes", + "chitchat", "futures", "futures-util", "hyper", @@ -2991,7 +2954,6 @@ dependencies = [ "quickwit-search", "quickwit-storage", "quickwit-telemetry", - "scuttlebutt", "serde", "serde_json", "serde_qs", @@ -3030,7 +2992,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", - "tokio-util 0.7.0", + "tokio-util 0.7.1", "tracing", "tracing-subscriber", ] @@ -3054,9 +3016,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ "proc-macro2", ] @@ -3123,9 +3085,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +checksum = "fd249e82c21598a9a426a4e00dd7adc1d640b22445ec8545feef801d1a74c221" dependencies = [ "autocfg", "crossbeam-deque", @@ -3135,14 +3097,13 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +checksum = "9f51245e1e62e1f1629cbfec37b5793bbabcaeb90f30e94d2ba03564687353e4" dependencies = [ "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "lazy_static", "num_cpus", ] @@ -3179,18 +3140,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", "redox_syscall", @@ -3272,7 +3233,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.22.2", + "webpki-roots 0.22.3", "winreg", ] @@ -3567,20 +3528,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "scuttlebutt" -version = "0.1.0" -source = "git+https://github.com/quickwit-oss/scuttlebutt?rev=fb799bd#fb799bd19643403389976d52c1b885da2f7b1b95" -dependencies = [ - "anyhow", - "bytes", - "rand", - "serde", - "tokio", - "tokio-stream", - "tracing", -] - [[package]] name = "security-framework" version = "2.6.1" @@ -3806,9 +3753,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "smallvec" @@ -3834,9 +3781,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "spin" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" +checksum = "c530c2b0d0bf8b69304b39fe2001993e267461948b890cd037d8ad4293fa1a0d" dependencies = [ "lock_api", ] @@ -3925,9 +3872,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" +checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" dependencies = [ "proc-macro2", "quote", @@ -3942,9 +3889,9 @@ checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" [[package]] name = "tabled" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97353139cc8a4e3f726e54f78081883d1c133b8e1834f55f168122f99f989717" +checksum = "d15827061abcf689257b1841c8e2732b1dfcc3ef825b24ce6c606e1e9e1a7bde" dependencies = [ "papergrid", "tabled_derive", @@ -4150,12 +4097,11 @@ dependencies = [ [[package]] name = "time" -version = "0.1.44" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" dependencies = [ "libc", - "wasi 0.10.0+wasi-snapshot-preview1", "winapi 0.3.9", ] @@ -4339,66 +4285,35 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64910e1b9c1901aaf5375561e35b9c057d95ff41a44ede043a03e09279eabaf1" +checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" dependencies = [ "bytes", "futures-core", "futures-io", "futures-sink", "futures-util", - "log", "pin-project-lite", "slab", "tokio", + "tracing", ] [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde", ] [[package]] name = "tonic" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a" -dependencies = [ - "async-stream", - "async-trait", - "base64", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost 0.9.0", - "prost-derive 0.9.0", - "tokio", - "tokio-stream", - "tokio-util 0.6.9", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", -] - -[[package]] -name = "tonic" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1a361140b1af3f548e0a5105126b3fc737542f6cd4947b66419c80be07db22" +checksum = "30fb54bf1e446f44d870d260d99957e7d11fb9d0a0f5bd1a662ad1411cc103f9" dependencies = [ "async-stream", "async-trait", @@ -4414,11 +4329,11 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost 0.10.0", - "prost-derive 0.10.0", + "prost", + "prost-derive", "tokio", "tokio-stream", - "tokio-util 0.7.0", + "tokio-util 0.7.1", "tower", "tower-layer", "tower-service", @@ -4426,18 +4341,6 @@ dependencies = [ "tracing-futures", ] -[[package]] -name = "tonic-build" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757" -dependencies = [ - "proc-macro2", - "prost-build 0.9.0", - "quote", - "syn", -] - [[package]] name = "tonic-build" version = "0.7.0" @@ -4446,7 +4349,7 @@ checksum = "4d17087af5c80e5d5fc8ba9878e60258065a0a757e35efe7a05b7904bece1943" dependencies = [ "prettyplease", "proc-macro2", - "prost-build 0.10.0", + "prost-build", "quote", "syn", ] @@ -4465,7 +4368,7 @@ dependencies = [ "rand", "slab", "tokio", - "tokio-util 0.7.0", + "tokio-util 0.7.1", "tower-layer", "tower-service", "tracing", @@ -4504,9 +4407,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.32" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" dependencies = [ "cfg-if 1.0.0", "log", @@ -4528,9 +4431,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" dependencies = [ "lazy_static", "valuable", @@ -4572,9 +4475,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0ab7bdc962035a87fba73f3acca9b8a8d0034c2e6f60b84aeaaddddc155dce" +checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" dependencies = [ "ansi_term", "lazy_static", @@ -4697,12 +4600,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-segmentation" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" - [[package]] name = "unicode-width" version = "0.1.9" @@ -4770,9 +4667,9 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "utf8-ranges" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ae116fef2b7fea257ed6440d3cfcff7f190865f170cdad00bb6465bf18ecba" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" [[package]] name = "utf8-width" @@ -4879,9 +4776,9 @@ dependencies = [ [[package]] name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasi" @@ -4891,9 +4788,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -4901,9 +4798,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" dependencies = [ "bumpalo", "lazy_static", @@ -4916,9 +4813,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" +checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -4928,9 +4825,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4938,9 +4835,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" dependencies = [ "proc-macro2", "quote", @@ -4951,15 +4848,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" [[package]] name = "web-sys" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" dependencies = [ "js-sys", "wasm-bindgen", @@ -4996,9 +4893,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552ceb903e957524388c4d3475725ff2c8b7960922063af6ce53c9a43da07449" +checksum = "44d8de8415c823c8abd270ad483c6feeac771fad964890779f9a8cb24fbbc1bf" dependencies = [ "webpki 0.22.0", ] @@ -5059,9 +4956,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -5072,33 +4969,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" +checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" [[package]] name = "windows_i686_gnu" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" +checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" [[package]] name = "windows_i686_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" [[package]] name = "windows_x86_64_gnu" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" [[package]] name = "windows_x86_64_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" [[package]] name = "winreg" diff --git a/quickwit-cluster/Cargo.toml b/quickwit-cluster/Cargo.toml index dfdc74dd9d9..25f57fc5f26 100644 --- a/quickwit-cluster/Cargo.toml +++ b/quickwit-cluster/Cargo.toml @@ -23,7 +23,7 @@ tokio-stream = { version = "0.1", features = [ "sync" ] } tracing = "0.1.29" uuid = "0.8" quickwit-proto = { version = "0.2.1", path = "../quickwit-proto"} -scuttlebutt = { git = "https://github.com/quickwit-oss/scuttlebutt", version = "0.1.0", rev="fb799bd" } +chitchat = "0.2" [dev-dependencies] itertools = '0.10' diff --git a/quickwit-cluster/src/cluster.rs b/quickwit-cluster/src/cluster.rs index 90908661eec..6f844eb4371 100644 --- a/quickwit-cluster/src/cluster.rs +++ b/quickwit-cluster/src/cluster.rs @@ -22,9 +22,9 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; +use chitchat::server::ChitchatServer; +use chitchat::{FailureDetectorConfig, NodeId, SerializableClusterState}; use quickwit_common::net::get_socket_addr; -use scuttlebutt::server::ScuttleServer; -use scuttlebutt::{FailureDetectorConfig, NodeId, SerializableClusterState}; use serde::{Deserialize, Serialize}; use tokio::sync::watch; use tokio::time::timeout; @@ -102,7 +102,7 @@ pub struct Cluster { pub listen_addr: SocketAddr, /// The actual cluster that implement the SWIM protocol. - scuttlebutt_server: ScuttleServer, + chitchat_server: ChitchatServer, /// A receiver(channel) for exchanging members in a cluster. members: watch::Receiver>, @@ -126,14 +126,16 @@ impl Cluster { failure_detector_config: FailureDetectorConfig, ) -> ClusterResult { info!(member=?me, listen_addr=?listen_addr, "Create new cluster."); - let scuttlebutt_server = ScuttleServer::spawn( + let chitchat_server = ChitchatServer::spawn( NodeId::from(me.clone()), seed_nodes, listen_addr.to_string(), + // make `cluster_id` configurable + "quickwit-cluster-id".to_string(), vec![(GRPC_ADDRESS_KEY, grpc_addr)], failure_detector_config, ); - let scuttlebutt = scuttlebutt_server.scuttlebutt(); + let chitchat = chitchat_server.chitchat(); let (members_sender, members_receiver) = watch::channel(Vec::new()); @@ -141,7 +143,7 @@ impl Cluster { let cluster = Cluster { node_id: me.internal_id(), listen_addr, - scuttlebutt_server, + chitchat_server, members: members_receiver, stop: Arc::new(AtomicBool::new(false)), }; @@ -155,7 +157,7 @@ impl Cluster { // Prepare to start a task that will monitor cluster events. let task_stop = cluster.stop.clone(); tokio::task::spawn(async move { - let mut node_change_receiver = scuttlebutt.lock().await.live_nodes_watcher(); + let mut node_change_receiver = chitchat.lock().await.live_nodes_watcher(); while let Some(members_set) = node_change_receiver.next().await { let mut members = members_set @@ -196,11 +198,11 @@ impl Cluster { &self, members: &[Member], ) -> anyhow::Result> { - let scuttlebutt = self.scuttlebutt_server.scuttlebutt(); - let scuttlebutt_guard = scuttlebutt.lock().await; + let chitchat = self.chitchat_server.chitchat(); + let chitchat_guard = chitchat.lock().await; let mut grpc_addresses = vec![]; for member in members { - let node_state = scuttlebutt_guard + let node_state = chitchat_guard .node_state(&NodeId::from(member.clone())) .unwrap(); @@ -222,26 +224,26 @@ impl Cluster { /// Set a key-value pair on the cluster node's state. pub async fn set_key_value(&self, key: K, value: V) { - let scuttlebutt = self.scuttlebutt_server.scuttlebutt(); - let mut scuttlebutt_guard = scuttlebutt.lock().await; - scuttlebutt_guard.self_node_state().set(key, value); + let chitchat = self.chitchat_server.chitchat(); + let mut chitchat_guard = chitchat.lock().await; + chitchat_guard.self_node_state().set(key, value); } /// Leave the cluster. pub async fn leave(&self) { info!(self_addr = ?self.listen_addr, "Leaving the cluster."); - // TODO: implements leave/join on ScuttleButt - // https://github.com/quickwit-oss/scuttlebutt/issues/30 + // TODO: implements leave/join on Chitchat + // https://github.com/quickwit-oss/chitchat/issues/30 self.stop.store(true, Ordering::Relaxed); } pub async fn state(&self) -> ClusterState { - let scuttlebutt = self.scuttlebutt_server.scuttlebutt(); - let scuttlebutt_guard = scuttlebutt.lock().await; + let chitchat = self.chitchat_server.chitchat(); + let chitchat_guard = chitchat.lock().await; - let cluster_state = scuttlebutt_guard.cluster_state(); - let live_nodes = scuttlebutt_guard.live_nodes().cloned().collect::>(); - let dead_nodes = scuttlebutt_guard.dead_nodes().cloned().collect::>(); + let cluster_state = chitchat_guard.cluster_state(); + let live_nodes = chitchat_guard.live_nodes().cloned().collect::>(); + let dead_nodes = chitchat_guard.dead_nodes().cloned().collect::>(); ClusterState { state: SerializableClusterState::from(cluster_state), live_nodes, @@ -252,7 +254,7 @@ impl Cluster { /// Leave the cluster. pub async fn shutdown(self) { info!(self_addr = ?self.listen_addr, "Shutting down the cluster."); - let result = self.scuttlebutt_server.shutdown().await; + let result = self.chitchat_server.shutdown().await; if let Err(error) = result { error!(self_addr = ?self.listen_addr, error = ?error, "Error while shuting down."); } @@ -288,7 +290,7 @@ pub struct ClusterState { dead_nodes: Vec, } -/// Compute the gRPC port from the scuttlebutt listen address for tests. +/// Compute the gRPC port from the chitchat listen address for tests. pub fn grpc_addr_from_listen_addr_for_test(listen_addr: SocketAddr) -> SocketAddr { SocketAddr::new(listen_addr.ip(), listen_addr.port() + 1) } diff --git a/quickwit-cluster/src/lib.rs b/quickwit-cluster/src/lib.rs index 24e3262725e..e17f7367bb4 100644 --- a/quickwit-cluster/src/lib.rs +++ b/quickwit-cluster/src/lib.rs @@ -23,8 +23,8 @@ mod service; use std::sync::Arc; +use chitchat::FailureDetectorConfig; use quickwit_config::QuickwitConfig; -use scuttlebutt::FailureDetectorConfig; pub use crate::cluster::{ create_cluster_for_test, grpc_addr_from_listen_addr_for_test, Cluster, Member, diff --git a/quickwit-serve/Cargo.toml b/quickwit-serve/Cargo.toml index eb7d0857471..b77349682e2 100644 --- a/quickwit-serve/Cargo.toml +++ b/quickwit-serve/Cargo.toml @@ -38,7 +38,7 @@ opentelemetry = "0.17" tracing-opentelemetry = "0.17" prometheus = "0.13" once_cell = '1' -scuttlebutt = { git = "https://github.com/quickwit-oss/scuttlebutt", version = "0.1.0", rev="fb799bd" } +chitchat = "0.2" quickwit-actors = {version="0.2.1", path="../quickwit-actors"} [dev-dependencies]