diff --git a/Cargo.lock b/Cargo.lock index 9d60e508d3b7d..751a3688c8dff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,7 +45,7 @@ dependencies = [ "getrandom", "once_cell", "version_check", - "zerocopy 0.7.31", + "zerocopy", ] [[package]] @@ -170,40 +170,40 @@ dependencies = [ [[package]] name = "apache-avro" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0fdddc3fdac97394ffcc5c89c634faa9c1c166ced54189af34e407c97b6ee7" +version = "0.16.0" +source = "git+https://github.com/risingwavelabs/avro?rev=d0846a16ce813a225af04ade35b3b8117b137a29#d0846a16ce813a225af04ade35b3b8117b137a29" dependencies = [ - "apache-avro-derive", - "byteorder", + "bzip2", + "crc32fast", "digest", "lazy_static", - "libflate 1.4.0", + "libflate", "log", "num-bigint", "quad-rand", "rand", - "regex", + "regex-lite", "serde", "serde_json", + "snap", "strum", "strum_macros", "thiserror", - "typed-builder 0.14.0", + "typed-builder 0.16.2", "uuid", - "zerocopy 0.6.6", + "xz2", + "zstd 0.12.4", ] [[package]] name = "apache-avro" -version = "0.16.0" -source = "git+https://github.com/risingwavelabs/avro?rev=d0846a16ce813a225af04ade35b3b8117b137a29#d0846a16ce813a225af04ade35b3b8117b137a29" +version = "0.17.0" +source = "git+https://github.com/icelake-io/avro.git?branch=icelake-dev#4b828e9283e7248fd3ca42f5b590c2160b201785" dependencies = [ - "bzip2", - "crc32fast", + "apache-avro-derive", + "bigdecimal 0.4.2", "digest", - "lazy_static", - "libflate 2.0.0", + "libflate", "log", "num-bigint", "quad-rand", @@ -211,27 +211,23 @@ dependencies = [ "regex-lite", "serde", "serde_json", - "snap", "strum", "strum_macros", "thiserror", - "typed-builder 0.16.2", + "typed-builder 0.18.0", "uuid", - "xz2", - "zstd 0.12.4", ] [[package]] name = "apache-avro-derive" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6686cd705badba064ec2322b9c3d72f5c70db8394e486bbb56e84fbdb3fa158c" +version = "0.17.0" +source = "git+https://github.com/icelake-io/avro.git?branch=icelake-dev#4b828e9283e7248fd3ca42f5b590c2160b201785" dependencies = [ "darling 0.20.3", "proc-macro2", "quote", "serde_json", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -806,7 +802,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -855,7 +851,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -872,7 +868,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -905,7 +901,7 @@ dependencies = [ "derive_utils", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -1689,7 +1685,7 @@ dependencies = [ "proc-macro-crate 2.0.0", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", "syn_derive", ] @@ -1978,7 +1974,7 @@ checksum = "bc7cb2538d4ecc42b6c3b57a83094d8c69894e74468d18cd045a09fdea807358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2095,7 +2091,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2530,7 +2526,7 @@ dependencies = [ "proc-macro-error 1.0.4", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2744,7 +2740,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30d2b3721e861707777e3195b0158f950ae6dc4a27e4d02ff9f67e3eb3de199e" dependencies = [ "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2772,7 +2768,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2799,7 +2795,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2816,7 +2812,7 @@ checksum = "2fa16a70dd58129e4dfffdff535fb1bce66673f7bbeec4a5a1765a504e1ccd84" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2888,7 +2884,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2921,7 +2917,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3214,8 +3210,9 @@ dependencies = [ [[package]] name = "delta_btree_map" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ + "educe 0.5.7", "enum-as-inner", ] @@ -3361,7 +3358,7 @@ checksum = "9abcad25e9720609ccb3dcdb795d845e37d8ce34183330a9f48b03a1a71c8e21" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3590,7 +3587,7 @@ dependencies = [ "enum-ordinalize 4.3.0", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3667,7 +3664,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3692,22 +3689,22 @@ dependencies = [ [[package]] name = "enum-iterator" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7add3873b5dd076766ee79c8e406ad1a472c385476b9e38849f8eec24f1be689" +checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" +checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3720,7 +3717,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3740,7 +3737,7 @@ checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3760,7 +3757,7 @@ checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3801,6 +3798,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "escape8259" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba4f4911e3666fcd7826997b4745c8224295a6f3072f1418c3067b97a67557ee" +dependencies = [ + "rustversion", +] + [[package]] name = "etcd-client" version = "0.12.1" @@ -4238,7 +4244,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -4250,7 +4256,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -4262,7 +4268,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -4350,7 +4356,7 @@ checksum = "5df2c13d48c8cb8a3ec093ede6f0f4482f327d7bb781120c5fb483ef0f17e758" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -4420,7 +4426,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -5055,10 +5061,10 @@ dependencies = [ [[package]] name = "icelake" version = "0.0.10" -source = "git+https://github.com/icelake-io/icelake?rev=3f7b53ba5b563524212c25810345d1314678e7fc#3f7b53ba5b563524212c25810345d1314678e7fc" +source = "git+https://github.com/icelake-io/icelake?rev=32c0bbf242f5c47b1e743f10577012fe7436c770#32c0bbf242f5c47b1e743f10577012fe7436c770" dependencies = [ "anyhow", - "apache-avro 0.15.0", + "apache-avro 0.17.0", "arrow-arith 49.0.0", "arrow-array 49.0.0", "arrow-buffer 49.0.0", @@ -5077,6 +5083,7 @@ dependencies = [ "faster-hex", "futures", "itertools 0.11.0", + "lazy_static", "log", "murmur3", "once_cell", @@ -5193,7 +5200,7 @@ checksum = "ce243b1bfa62ffc028f1cc3b6034ec63d649f3031bc8a4fbbb004e1ac17d1f68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -5550,17 +5557,6 @@ version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" -[[package]] -name = "libflate" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18" -dependencies = [ - "adler32", - "crc32fast", - "libflate_lz77 1.2.0", -] - [[package]] name = "libflate" version = "2.0.0" @@ -5571,16 +5567,7 @@ dependencies = [ "core2", "crc32fast", "dary_heap", - "libflate_lz77 2.0.0", -] - -[[package]] -name = "libflate_lz77" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf" -dependencies = [ - "rle-decode-fast", + "libflate_lz77", ] [[package]] @@ -5643,6 +5630,18 @@ dependencies = [ "threadpool", ] +[[package]] +name = "libtest-mimic" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f0f4c6f44ecfd52e8b443f2ad18f2b996540135771561283c2352ce56a1c70b" +dependencies = [ + "clap", + "escape8259", + "termcolor", + "threadpool", +] + [[package]] name = "libz-sys" version = "1.1.12" @@ -5690,7 +5689,7 @@ checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "local_stats_alloc" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "workspace-hack", ] @@ -5956,7 +5955,7 @@ dependencies = [ "proc-macro2", "prost-build 0.12.1", "quote", - "syn 2.0.37", + "syn 2.0.48", "tonic-build", ] @@ -6167,9 +6166,9 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "multimap" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70db9248a93dc36a36d9a47898caa007a32755c7ad140ec64eeeb50d5a730631" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" dependencies = [ "serde", ] @@ -6193,7 +6192,7 @@ dependencies = [ "proc-macro-error 1.0.4", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", "termcolor", "thiserror", ] @@ -6722,7 +6721,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -6921,7 +6920,7 @@ dependencies = [ "proc-macro-error 1.0.4", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -7130,7 +7129,7 @@ dependencies = [ "regex", "regex-syntax 0.7.5", "structmeta", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -7251,7 +7250,7 @@ dependencies = [ [[package]] name = "pgwire" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "auto_enums", @@ -7262,6 +7261,7 @@ dependencies = [ "madsim-tokio", "openssl", "panic-message", + "parking_lot 0.12.1", "risingwave_common", "risingwave_sqlparser", "tempfile", @@ -7329,7 +7329,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -7460,7 +7460,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -7608,7 +7608,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -7697,9 +7697,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -7830,7 +7830,7 @@ dependencies = [ "prost 0.12.1", "prost-types 0.12.1", "regex", - "syn 2.0.37", + "syn 2.0.48", "tempfile", "which", ] @@ -7858,7 +7858,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -7867,7 +7867,7 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -8062,9 +8062,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -8244,7 +8244,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -8469,7 +8469,7 @@ dependencies = [ [[package]] name = "risedev" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "chrono", @@ -8498,7 +8498,7 @@ dependencies = [ [[package]] name = "risedev-config" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "clap", @@ -8511,19 +8511,19 @@ dependencies = [ [[package]] name = "risingwave-fields-derive" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "expect-test", "indoc", "prettyplease 0.2.15", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] name = "risingwave_backup" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "async-trait", @@ -8545,7 +8545,7 @@ dependencies = [ [[package]] name = "risingwave_batch" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "assert_matches", @@ -8575,6 +8575,7 @@ dependencies = [ "risingwave_rpc_client", "risingwave_source", "risingwave_storage", + "rw_futures_util", "scopeguard", "serde_json", "task_stats_alloc", @@ -8591,7 +8592,7 @@ dependencies = [ [[package]] name = "risingwave_bench" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "async-trait", "aws-config", @@ -8603,6 +8604,7 @@ dependencies = [ "bytesize", "clap", "futures", + "futures-async-stream", "hdrhistogram", "itertools 0.12.0", "libc", @@ -8613,9 +8615,13 @@ dependencies = [ "prometheus", "rand", "risingwave_common", + "risingwave_connector", + "risingwave_pb", "risingwave_rt", "risingwave_storage", + "risingwave_stream", "serde", + "serde_yaml", "tokio-stream", "toml 0.8.2", "tracing", @@ -8625,7 +8631,7 @@ dependencies = [ [[package]] name = "risingwave_cmd" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "clap", "madsim-tokio", @@ -8646,7 +8652,7 @@ dependencies = [ [[package]] name = "risingwave_cmd_all" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "clap", @@ -8678,7 +8684,7 @@ dependencies = [ [[package]] name = "risingwave_common" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "arc-swap", @@ -8753,6 +8759,7 @@ dependencies = [ "risingwave_pb", "rust_decimal", "rusty-fork", + "rw_futures_util", "ryu", "serde", "serde_bytes", @@ -8784,7 +8791,7 @@ dependencies = [ [[package]] name = "risingwave_common_heap_profiling" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "chrono", @@ -8799,7 +8806,7 @@ dependencies = [ [[package]] name = "risingwave_common_proc_macro" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "bae", "proc-macro-error 1.0.4", @@ -8810,7 +8817,7 @@ dependencies = [ [[package]] name = "risingwave_common_service" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "async-trait", "futures", @@ -8831,7 +8838,7 @@ dependencies = [ [[package]] name = "risingwave_compaction_test" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "async-trait", @@ -8858,7 +8865,7 @@ dependencies = [ [[package]] name = "risingwave_compactor" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "async-trait", "await-tree", @@ -8880,7 +8887,7 @@ dependencies = [ [[package]] name = "risingwave_compute" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "async-trait", @@ -8923,7 +8930,7 @@ dependencies = [ [[package]] name = "risingwave_connector" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "apache-avro 0.16.0", @@ -8997,6 +9004,7 @@ dependencies = [ "risingwave_pb", "risingwave_rpc_client", "rust_decimal", + "rw_futures_util", "serde", "serde_derive", "serde_json", @@ -9027,7 +9035,7 @@ dependencies = [ [[package]] name = "risingwave_ctl" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "bytes", @@ -9062,7 +9070,7 @@ dependencies = [ [[package]] name = "risingwave_e2e_extended_mode_test" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "chrono", @@ -9077,7 +9085,7 @@ dependencies = [ [[package]] name = "risingwave_error" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "bincode 1.3.3", "bytes", @@ -9092,7 +9100,7 @@ dependencies = [ [[package]] name = "risingwave_expr" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "arrow-array 49.0.0", @@ -9132,10 +9140,11 @@ dependencies = [ [[package]] name = "risingwave_expr_impl" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "aho-corasick", "anyhow", + "arrow-schema 49.0.0", "async-trait", "auto_enums", "chrono", @@ -9145,6 +9154,7 @@ dependencies = [ "futures-async-stream", "futures-util", "hex", + "icelake", "itertools 0.12.0", "jsonbb", "madsim-tokio", @@ -9174,12 +9184,12 @@ dependencies = [ "itertools 0.12.0", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] name = "risingwave_frontend" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "arc-swap", @@ -9237,6 +9247,7 @@ dependencies = [ "risingwave_storage", "risingwave_udf", "risingwave_variables", + "rw_futures_util", "serde", "serde_json", "sha2", @@ -9252,7 +9263,7 @@ dependencies = [ [[package]] name = "risingwave_hummock_sdk" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "bytes", "easy-ext", @@ -9268,7 +9279,7 @@ dependencies = [ [[package]] name = "risingwave_hummock_test" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "async-trait", "bytes", @@ -9300,7 +9311,7 @@ dependencies = [ [[package]] name = "risingwave_hummock_trace" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "async-trait", "bincode 2.0.0-rc.3", @@ -9355,6 +9366,7 @@ dependencies = [ "risingwave_object_store", "risingwave_pb", "risingwave_storage", + "rw_futures_util", "serde", "serde_json", "thiserror", @@ -9364,7 +9376,7 @@ dependencies = [ [[package]] name = "risingwave_mem_table_spill_test" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "async-trait", "bytes", @@ -9380,7 +9392,7 @@ dependencies = [ [[package]] name = "risingwave_meta" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "arc-swap", @@ -9432,6 +9444,7 @@ dependencies = [ "risingwave_rpc_client", "risingwave_sqlparser", "risingwave_test_runner", + "rw_futures_util", "scopeguard", "sea-orm", "serde", @@ -9452,7 +9465,7 @@ dependencies = [ [[package]] name = "risingwave_meta_model_migration" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "async-std", "sea-orm-migration", @@ -9461,8 +9474,9 @@ dependencies = [ [[package]] name = "risingwave_meta_model_v2" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ + "risingwave_common", "risingwave_hummock_sdk", "risingwave_pb", "sea-orm", @@ -9472,7 +9486,7 @@ dependencies = [ [[package]] name = "risingwave_meta_node" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "clap", @@ -9503,7 +9517,7 @@ dependencies = [ [[package]] name = "risingwave_meta_service" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "async-trait", @@ -9529,7 +9543,7 @@ dependencies = [ [[package]] name = "risingwave_object_store" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "async-trait", "await-tree", @@ -9563,7 +9577,7 @@ dependencies = [ [[package]] name = "risingwave_pb" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "enum-as-inner", "fs-err", @@ -9583,12 +9597,12 @@ dependencies = [ [[package]] name = "risingwave_planner_test" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "expect-test", "itertools 0.12.0", - "libtest-mimic", + "libtest-mimic 0.7.0", "madsim-tokio", "paste", "risingwave_expr_impl", @@ -9605,7 +9619,7 @@ dependencies = [ [[package]] name = "risingwave_regress_test" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "clap", @@ -9619,7 +9633,7 @@ dependencies = [ [[package]] name = "risingwave_rpc_client" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "async-trait", @@ -9638,6 +9652,7 @@ dependencies = [ "risingwave_error", "risingwave_hummock_sdk", "risingwave_pb", + "rw_futures_util", "static_assertions", "thiserror", "thiserror-ext", @@ -9651,7 +9666,7 @@ dependencies = [ [[package]] name = "risingwave_rt" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "await-tree", "console", @@ -9731,7 +9746,7 @@ dependencies = [ [[package]] name = "risingwave_source" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "assert_matches", @@ -9746,6 +9761,7 @@ dependencies = [ "risingwave_common", "risingwave_connector", "risingwave_pb", + "rw_futures_util", "tempfile", "tracing", "workspace-hack", @@ -9753,7 +9769,7 @@ dependencies = [ [[package]] name = "risingwave_sqlparser" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "itertools 0.12.0", "matches", @@ -9780,14 +9796,14 @@ dependencies = [ [[package]] name = "risingwave_sqlsmith" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "chrono", "clap", "expect-test", "itertools 0.12.0", - "libtest-mimic", + "libtest-mimic 0.7.0", "madsim-tokio", "rand", "rand_chacha", @@ -9807,7 +9823,7 @@ dependencies = [ [[package]] name = "risingwave_state_cleaning_test" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "clap", @@ -9827,7 +9843,7 @@ dependencies = [ [[package]] name = "risingwave_storage" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "arc-swap", @@ -9893,7 +9909,7 @@ dependencies = [ [[package]] name = "risingwave_stream" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "anyhow", "assert_matches", @@ -9921,7 +9937,7 @@ dependencies = [ "madsim-tonic", "maplit", "memcomparable", - "multimap 0.9.0", + "multimap 0.10.0", "parking_lot 0.12.1", "pin-project", "prometheus", @@ -9937,6 +9953,7 @@ dependencies = [ "risingwave_rpc_client", "risingwave_source", "risingwave_storage", + "rw_futures_util", "serde", "serde_json", "serde_yaml", @@ -9955,7 +9972,7 @@ dependencies = [ [[package]] name = "risingwave_test_runner" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "fail", "sync-point", @@ -9984,7 +10001,7 @@ dependencies = [ [[package]] name = "risingwave_variables" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "chrono", "workspace-hack", @@ -10086,7 +10103,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.37", + "syn 2.0.48", "walkdir", ] @@ -10253,6 +10270,15 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "rw_futures_util" +version = "0.0.0" +dependencies = [ + "futures", + "pin-project-lite", + "tokio", +] + [[package]] name = "ryu" version = "1.0.15" @@ -10345,7 +10371,7 @@ dependencies = [ "proc-macro-error 1.0.4", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -10403,7 +10429,7 @@ dependencies = [ "proc-macro2", "quote", "sea-bae", - "syn 2.0.37", + "syn 2.0.48", "unicode-ident", ] @@ -10586,9 +10612,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] @@ -10635,13 +10661,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -10657,9 +10683,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -10702,7 +10728,7 @@ checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -10752,7 +10778,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -10790,7 +10816,7 @@ checksum = "b93fb4adc70021ac1b47f7d45e8cc4169baaa7ea58483bc5b721d19a26202212" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -11032,9 +11058,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "2593d31f82ead8df961d8bd23a64c2ccf2eb5dd34b0a34bfb4dd54011c72009e" dependencies = [ "serde", ] @@ -11175,7 +11201,7 @@ dependencies = [ "glob", "humantime", "itertools 0.11.0", - "libtest-mimic", + "libtest-mimic 0.6.1", "md-5", "owo-colors", "regex", @@ -11466,7 +11492,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -11477,7 +11503,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -11491,15 +11517,15 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.25.2" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8d03b598d3d0fff69bf533ee3ef19b8eeb342729596df84bcc7e1f96ec4059" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck 0.4.1", "proc-macro2", "quote", "rustversion", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -11564,9 +11590,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -11593,7 +11619,7 @@ dependencies = [ "proc-macro-error 1.0.4", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -11698,9 +11724,9 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] @@ -11724,18 +11750,18 @@ dependencies = [ "either", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -11910,7 +11936,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -12154,7 +12180,7 @@ dependencies = [ "proc-macro2", "prost-build 0.12.1", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -12235,7 +12261,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -12374,22 +12400,20 @@ dependencies = [ [[package]] name = "typed-builder" -version = "0.14.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cba322cb9b7bc6ca048de49e83918223f35e7a86311267013afff257004870" +checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", + "typed-builder-macro 0.16.2", ] [[package]] name = "typed-builder" -version = "0.16.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" +checksum = "e47c0496149861b7c95198088cbf36645016b1a0734cf350c50e2a38e070f38a" dependencies = [ - "typed-builder-macro", + "typed-builder-macro 0.18.0", ] [[package]] @@ -12400,7 +12424,18 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", +] + +[[package]] +name = "typed-builder-macro" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982ee4197351b5c9782847ef5ec1fdcaf50503fb19d68f9771adae314e72b492" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", ] [[package]] @@ -12557,11 +12592,12 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "8.2.5" +version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e7dc29b3c54a2ea67ef4f953d5ec0c4085035c0ae2d325be1c0d2144bd9f16" +checksum = "ec0d895592fa7710eba03fe072e614e3dc6a61ab76ae7ae10d2eb4a7ed5b00ca" dependencies = [ "anyhow", + "cfg-if", "rustversion", "time", ] @@ -12688,7 +12724,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -12722,7 +12758,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -12852,7 +12888,7 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", "wasmtime-component-util", "wasmtime-wit-bindgen", "wit-parser", @@ -13044,7 +13080,7 @@ checksum = "f55c5565959287c21dd0f4277ae3518dd2ae62679f655ee2dbc4396e19d210db" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -13211,7 +13247,7 @@ dependencies = [ "proc-macro2", "quote", "shellexpand", - "syn 2.0.37", + "syn 2.0.48", "witx", ] @@ -13223,7 +13259,7 @@ checksum = "31ae1ec11a17ea481539ee9a5719a278c9790d974060fbf71db4b2c05378780b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", "wiggle-generate", ] @@ -13548,11 +13584,11 @@ dependencies = [ [[package]] name = "with_options" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -13569,7 +13605,7 @@ dependencies = [ [[package]] name = "workspace-config" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "log", "openssl-sys", @@ -13580,7 +13616,7 @@ dependencies = [ [[package]] name = "workspace-hack" -version = "1.5.0-alpha" +version = "1.7.0-alpha" dependencies = [ "ahash 0.8.6", "allocator-api2", @@ -13595,6 +13631,7 @@ dependencies = [ "aws-smithy-types", "axum", "base64 0.21.4", + "bigdecimal 0.4.2", "bit-vec", "bitflags 2.4.0", "byteorder", @@ -13617,6 +13654,7 @@ dependencies = [ "futures", "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -13695,7 +13733,7 @@ dependencies = [ "strum", "subtle", "syn 1.0.109", - "syn 2.0.37", + "syn 2.0.48", "time", "time-macros", "tinyvec", @@ -13716,6 +13754,8 @@ dependencies = [ "uuid", "whoami", "zeroize", + "zstd 0.13.0", + "zstd-safe 7.0.0", "zstd-sys", ] @@ -13807,34 +13847,13 @@ version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a599daf1b507819c1121f0bf87fa37eb19daac6aff3aefefd4e6e2e0f2020fc" -[[package]] -name = "zerocopy" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6" -dependencies = [ - "byteorder", - "zerocopy-derive 0.6.6", -] - [[package]] name = "zerocopy" version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" dependencies = [ - "zerocopy-derive 0.7.31", -] - -[[package]] -name = "zerocopy-derive" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", + "zerocopy-derive", ] [[package]] @@ -13845,7 +13864,7 @@ checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b09dd5def90b2..7bd67bc583745 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ members = [ "src/tests/sqlsmith", "src/tests/state_cleaning_test", "src/utils/delta_btree_map", + "src/utils/futures_util", "src/utils/local_stats_alloc", "src/utils/pgwire", "src/utils/runtime", @@ -63,7 +64,7 @@ exclude = ["e2e_test/udf/wasm", "lints"] resolver = "2" [workspace.package] -version = "1.5.0-alpha" +version = "1.7.0-alpha" edition = "2021" homepage = "https://github.com/risingwavelabs/risingwave" keywords = ["sql", "database", "streaming"] @@ -120,7 +121,7 @@ tonic = { package = "madsim-tonic", version = "0.4.1" } tonic-build = { package = "madsim-tonic-build", version = "0.4.2" } otlp-embedded = { git = "https://github.com/risingwavelabs/otlp-embedded", rev = "58c1f003484449d7c6dd693b348bf19dd44889cb" } prost = { version = "0.12" } -icelake = { git = "https://github.com/icelake-io/icelake", rev = "3f7b53ba5b563524212c25810345d1314678e7fc", features = [ +icelake = { git = "https://github.com/icelake-io/icelake", rev = "32c0bbf242f5c47b1e743f10577012fe7436c770", features = [ "prometheus", ] } arrow-array = "49" @@ -187,6 +188,7 @@ risingwave_udf = { path = "./src/expr/udf" } risingwave_variables = { path = "./src/utils/variables" } risingwave_java_binding = { path = "./src/java_binding" } risingwave_jni_core = { path = "src/jni_core" } +rw_futures_util = { path = "src/utils/futures_util" } tokio-util = "0.7" [workspace.lints.rust] diff --git a/Makefile.toml b/Makefile.toml index 347f2234e5fda..c0135dc84c048 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -245,7 +245,7 @@ do fi done -if (( "$(du -sk ${PREFIX_LOG} | cut -f1)" > 2000 )) ; then +if (( "$(du -sk ${PREFIX_LOG} | cut -f1)" > 3000 )) ; then echo "$(tput setaf 1)ERROR: log size is significantly large ($(du -sh ${PREFIX_LOG} | cut -f1)).$(tput sgr0) Please disable unnecessary logs." exit 1 fi diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh index 64144d051ad58..ec04a1d6863cf 100755 --- a/ci/scripts/e2e-source-test.sh +++ b/ci/scripts/e2e-source-test.sh @@ -5,6 +5,15 @@ set -euo pipefail source ci/scripts/common.sh +# Arguments: +# $1: subject name +# $2: schema file path +function register_schema_registry() { + curl -X POST http://message_queue:8081/subjects/$1/versions \ + -H ‘Content-Type: application/vnd.schemaregistry.v1+json’ \ + --data-binary @<(jq -n --arg schema “$(cat $2)” ‘{schemaType: “PROTOBUF”, schema: $schema}’) +} + # prepare environment export CONNECTOR_LIBS_PATH="./connector-node/libs" @@ -115,12 +124,13 @@ export RISINGWAVE_CI=true RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ cargo make ci-start ci-1cn-1fe python3 -m pip install requests protobuf confluent-kafka -python3 e2e_test/schema_registry/pb.py "message_queue:29092" "http://message_queue:8081" "sr_pb_test" 20 +python3 e2e_test/schema_registry/pb.py "message_queue:29092" "http://message_queue:8081" "sr_pb_test" 20 user echo "make sure google/protobuf/source_context.proto is NOT in schema registry" curl --silent 'http://message_queue:8081/subjects'; echo # curl --silent --head -X GET 'http://message_queue:8081/subjects/google%2Fprotobuf%2Fsource_context.proto/versions' | grep 404 curl --silent 'http://message_queue:8081/subjects' | grep -v 'google/protobuf/source_context.proto' sqllogictest -p 4566 -d dev './e2e_test/schema_registry/pb.slt' +sqllogictest -p 4566 -d dev './e2e_test/schema_registry/alter_sr.slt' echo "--- Kill cluster" cargo make ci-kill diff --git a/ci/scripts/gen-integration-test-yaml.py b/ci/scripts/gen-integration-test-yaml.py index f2cf770be8ece..8451290a93c93 100644 --- a/ci/scripts/gen-integration-test-yaml.py +++ b/ci/scripts/gen-integration-test-yaml.py @@ -37,8 +37,8 @@ 'starrocks-sink': ['json'], 'deltalake-sink': ['json'], 'pinot-sink': ['json'], + 'presto-trino': ['json'], 'client-library': ['none'], - } def gen_pipeline_steps(): diff --git a/ci/scripts/run-backfill-tests.sh b/ci/scripts/run-backfill-tests.sh index 46bad790d21f7..f26a983d67050 100755 --- a/ci/scripts/run-backfill-tests.sh +++ b/ci/scripts/run-backfill-tests.sh @@ -200,12 +200,82 @@ test_sink_backfill_recovery() { wait } +test_no_shuffle_backfill_runtime() { + echo "--- e2e, test_no_shuffle_backfill_runtime" + cargo make ci-start $CLUSTER_PROFILE + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_table.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt' + + # Provide updates ... + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt' 2>&1 1>out.log & + + # ... and concurrently create mv. + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_no_shuffle_mv.slt' + + wait + + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows.slt' + + cargo make kill + cargo make wait-processes-exit +} + +test_arrangement_backfill_runtime() { + echo "--- e2e, test_arrangement_backfill_runtime" + cargo make ci-start $CLUSTER_PROFILE + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_table.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt' + + # Provide updates ... + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt' 2>&1 1>out.log & + + # ... and concurrently create mv. + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt' + + wait + + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows.slt' + + cargo make kill + cargo make wait-processes-exit +} + +test_no_shuffle_backfill_snapshot_only_runtime() { + echo "--- e2e, test_no_shuffle_backfill_snapshot_only_runtime" + cargo make ci-start $CLUSTER_PROFILE + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_table.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_no_shuffle_mv.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows.slt' + + cargo make kill + cargo make wait-processes-exit +} + +test_arrangement_backfill_snapshot_only_runtime() { + echo "--- e2e, test_arrangement_backfill_snapshot_only_runtime" + cargo make ci-start $CLUSTER_PROFILE + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_table.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows.slt' + + cargo make kill + cargo make wait-processes-exit +} + main() { set -euo pipefail test_snapshot_and_upstream_read test_backfill_tombstone test_replication_with_column_pruning test_sink_backfill_recovery + test_no_shuffle_backfill_runtime + test_arrangement_backfill_runtime + test_no_shuffle_backfill_snapshot_only_runtime + test_arrangement_backfill_snapshot_only_runtime } main diff --git a/ci/scripts/run-unit-test.sh b/ci/scripts/run-unit-test.sh index 6f2093060f370..8fe4b150bf866 100755 --- a/ci/scripts/run-unit-test.sh +++ b/ci/scripts/run-unit-test.sh @@ -10,10 +10,6 @@ cd ${REPO_ROOT}/src/expr/udf/python python3 -m pytest cd ${REPO_ROOT} -echo "+++ Run unit tests with coverage" +echo "+++ Run unit tests" # use tee to disable progress bar -NEXTEST_PROFILE=ci cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints,sync_point --workspace --exclude risingwave_simulation - -echo "--- Codecov upload coverage reports" -curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov -./codecov -t "$CODECOV_TOKEN" -s . -F rust +NEXTEST_PROFILE=ci cargo nextest run --features failpoints,sync_point --workspace --exclude risingwave_simulation diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml index eac8de1ef49f9..0e8e015dc68d6 100644 --- a/ci/workflows/pull-request.yml +++ b/ci/workflows/pull-request.yml @@ -96,7 +96,7 @@ steps: config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 15 + timeout_in_minutes: 16 retry: *auto-retry - label: "end-to-end test (parallel)" diff --git a/dashboard/components/CatalogModal.tsx b/dashboard/components/CatalogModal.tsx new file mode 100644 index 0000000000000..cf6a2f8cc9e0d --- /dev/null +++ b/dashboard/components/CatalogModal.tsx @@ -0,0 +1,87 @@ +/* + * Copyright 2024 RisingWave Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { + Button, + Modal, + ModalBody, + ModalCloseButton, + ModalContent, + ModalFooter, + ModalHeader, + ModalOverlay, +} from "@chakra-ui/react" + +import Link from "next/link" +import { parseAsInteger, useQueryState } from "nuqs" +import { + Relation, + relationIsStreamingJob, + relationTypeTitleCase, +} from "../pages/api/streaming" +import { ReactJson } from "./Relations" + +export function useCatalogModal(relationList: Relation[] | undefined) { + const [modalId, setModalId] = useQueryState("modalId", parseAsInteger) + const modalData = relationList?.find((r) => r.id === modalId) + + return [modalData, setModalId] as const +} + +export function CatalogModal({ + modalData, + onClose, +}: { + modalData: Relation | undefined + onClose: () => void +}) { + return ( + + + + + Catalog of {modalData && relationTypeTitleCase(modalData)}{" "} + {modalData?.id} - {modalData?.name} + + + + {modalData && ( + + )} + + + + {modalData && relationIsStreamingJob(modalData) && ( + + )} + + + + + ) +} diff --git a/dashboard/components/FragmentDependencyGraph.tsx b/dashboard/components/FragmentDependencyGraph.tsx index 553c40ec53f92..7b3417507efcf 100644 --- a/dashboard/components/FragmentDependencyGraph.tsx +++ b/dashboard/components/FragmentDependencyGraph.tsx @@ -3,18 +3,18 @@ import * as d3 from "d3" import { Dag, DagLink, DagNode, zherebko } from "d3-dag" import { cloneDeep } from "lodash" import { useCallback, useEffect, useRef, useState } from "react" -import { Position } from "../lib/layout" +import { Enter, FragmentBox, Position } from "../lib/layout" const nodeRadius = 5 const edgeRadius = 12 export default function FragmentDependencyGraph({ - mvDependency, + fragmentDependency, svgWidth, selectedId, onSelectedIdChange, }: { - mvDependency: Dag + fragmentDependency: Dag svgWidth: number selectedId: string | undefined onSelectedIdChange: (id: string) => void | undefined @@ -24,21 +24,21 @@ export default function FragmentDependencyGraph({ const MARGIN_X = 10 const MARGIN_Y = 2 - const mvDependencyDagCallback = useCallback(() => { + const fragmentDependencyDagCallback = useCallback(() => { const layout = zherebko().nodeSize([ nodeRadius * 2, (nodeRadius + edgeRadius) * 2, nodeRadius, ]) - const dag = cloneDeep(mvDependency) + const dag = cloneDeep(fragmentDependency) const { width, height } = layout(dag) return { width, height, dag } - }, [mvDependency]) + }, [fragmentDependency]) - const mvDependencyDag = mvDependencyDagCallback() + const fragmentDependencyDag = fragmentDependencyDagCallback() useEffect(() => { - const { width, height, dag } = mvDependencyDag + const { width, height, dag } = fragmentDependencyDag // This code only handles rendering @@ -53,25 +53,27 @@ export default function FragmentDependencyGraph({ .x(({ x }) => x + MARGIN_X) .y(({ y }) => y) - const isSelected = (d: any) => d.data.id === selectedId + const isSelected = (d: DagNode) => d.data.id === selectedId const edgeSelection = svgSelection .select(".edges") - .selectAll(".edge") + .selectAll(".edge") .data(dag.links()) - const applyEdge = (sel: any) => + type EdgeSelection = typeof edgeSelection + + const applyEdge = (sel: EdgeSelection) => sel .attr("d", ({ points }: DagLink) => line(points)) .attr("fill", "none") - .attr("stroke-width", (d: any) => + .attr("stroke-width", (d) => isSelected(d.source) || isSelected(d.target) ? 2 : 1 ) - .attr("stroke", (d: any) => + .attr("stroke", (d) => isSelected(d.source) || isSelected(d.target) ? theme.colors.blue["500"] : theme.colors.gray["300"] ) - const createEdge = (sel: any) => + const createEdge = (sel: Enter) => sel.append("path").attr("class", "edge").call(applyEdge) edgeSelection.exit().remove() edgeSelection.enter().call(createEdge) @@ -80,19 +82,18 @@ export default function FragmentDependencyGraph({ // Select nodes const nodeSelection = svgSelection .select(".nodes") - .selectAll(".node") + .selectAll(".node") .data(dag.descendants()) - const applyNode = (sel: any) => + type NodeSelection = typeof nodeSelection + + const applyNode = (sel: NodeSelection) => sel - .attr( - "transform", - ({ x, y }: Position) => `translate(${x + MARGIN_X}, ${y})` - ) - .attr("fill", (d: any) => + .attr("transform", (d) => `translate(${d.x! + MARGIN_X}, ${d.y})`) + .attr("fill", (d) => isSelected(d) ? theme.colors.blue["500"] : theme.colors.gray["500"] ) - const createNode = (sel: any) => + const createNode = (sel: Enter) => sel .append("circle") .attr("class", "node") @@ -105,22 +106,23 @@ export default function FragmentDependencyGraph({ // Add text to nodes const labelSelection = svgSelection .select(".labels") - .selectAll(".label") + .selectAll(".label") .data(dag.descendants()) + type LabelSelection = typeof labelSelection - const applyLabel = (sel: any) => + const applyLabel = (sel: LabelSelection) => sel - .text((d: any) => d.data.name) + .text((d) => d.data.name) .attr("x", svgWidth - MARGIN_X) .attr("font-family", "inherit") .attr("text-anchor", "end") .attr("alignment-baseline", "middle") - .attr("y", (d: any) => d.y) - .attr("fill", (d: any) => + .attr("y", (d) => d.y!) + .attr("fill", (d) => isSelected(d) ? theme.colors.black["500"] : theme.colors.gray["500"] ) .attr("font-weight", "600") - const createLabel = (sel: any) => + const createLabel = (sel: Enter) => sel.append("text").attr("class", "label").call(applyLabel) labelSelection.exit().remove() labelSelection.enter().call(createLabel) @@ -129,11 +131,12 @@ export default function FragmentDependencyGraph({ // Add overlays const overlaySelection = svgSelection .select(".overlays") - .selectAll(".overlay") + .selectAll(".overlay") .data(dag.descendants()) + type OverlaySelection = typeof overlaySelection const STROKE_WIDTH = 3 - const applyOverlay = (sel: any) => + const applyOverlay = (sel: OverlaySelection) => sel .attr("x", STROKE_WIDTH) .attr( @@ -143,20 +146,13 @@ export default function FragmentDependencyGraph({ .attr("width", svgWidth - STROKE_WIDTH * 2) .attr( "y", - (d: any) => d.y - nodeRadius - edgeRadius + MARGIN_Y + STROKE_WIDTH + (d) => d.y! - nodeRadius - edgeRadius + MARGIN_Y + STROKE_WIDTH ) .attr("rx", 5) .attr("fill", theme.colors.gray["500"]) .attr("opacity", 0) .style("cursor", "pointer") - const createOverlay = ( - sel: d3.Selection< - d3.EnterElement, - DagNode, - d3.BaseType, - unknown - > - ) => + const createOverlay = (sel: Enter) => sel .append("rect") .attr("class", "overlay") @@ -187,7 +183,7 @@ export default function FragmentDependencyGraph({ }) .on("click", function (d, i) { if (onSelectedIdChange) { - onSelectedIdChange((i.data as any).id) + onSelectedIdChange(i.data.id) } }) @@ -196,7 +192,13 @@ export default function FragmentDependencyGraph({ overlaySelection.call(applyOverlay) setSvgHeight(`${height}px`) - }, [mvDependency, selectedId, svgWidth, onSelectedIdChange, mvDependencyDag]) + }, [ + fragmentDependency, + selectedId, + svgWidth, + onSelectedIdChange, + fragmentDependencyDag, + ]) return ( diff --git a/dashboard/components/FragmentGraph.tsx b/dashboard/components/FragmentGraph.tsx index 875d92baa2b6b..72184d1b2a8bc 100644 --- a/dashboard/components/FragmentGraph.tsx +++ b/dashboard/components/FragmentGraph.tsx @@ -17,11 +17,12 @@ import { cloneDeep } from "lodash" import { Fragment, useCallback, useEffect, useRef, useState } from "react" import { Edge, + Enter, FragmentBox, FragmentBoxPosition, Position, - generateBoxEdges, - layout, + generateFragmentEdges, + layoutItem, } from "../lib/layout" import { PlanNodeDatum } from "../pages/fragment_graph" import { StreamNode } from "../proto/gen/stream_plan" @@ -36,10 +37,6 @@ type FragmentLayout = { actorIds: string[] } & Position -type Enter = Type extends d3.Selection - ? d3.Selection - : never - function treeLayoutFlip( root: d3.HierarchyNode, { dx, dy }: { dx: number; dy: number } @@ -145,7 +142,7 @@ export default function FragmentGraph({ includedFragmentIds.add(fragmentId) } - const fragmentLayout = layout( + const fragmentLayout = layoutItem( fragmentDependencyDag.map(({ width: _1, height: _2, id, ...data }) => { const { width, height } = layoutFragmentResult.get(id)! return { width, height, id, ...data } @@ -170,7 +167,7 @@ export default function FragmentGraph({ svgHeight = Math.max(svgHeight, y + height + 50) svgWidth = Math.max(svgWidth, x + width) }) - const edges = generateBoxEdges(fragmentLayout) + const edges = generateFragmentEdges(fragmentLayout) return { layoutResult, diff --git a/dashboard/components/RelationDependencyGraph.tsx b/dashboard/components/RelationDependencyGraph.tsx index 99d40ca2615fd..0f677101cce17 100644 --- a/dashboard/components/RelationDependencyGraph.tsx +++ b/dashboard/components/RelationDependencyGraph.tsx @@ -19,15 +19,23 @@ import { theme } from "@chakra-ui/react" import * as d3 from "d3" import { useCallback, useEffect, useRef } from "react" import { - FragmentPoint, - FragmentPointPosition, + Enter, Position, - flipLayoutPoint, - generatePointEdges, + RelationPoint, + RelationPointPosition, + flipLayoutRelation, + generateRelationEdges, } from "../lib/layout" +import { + Relation, + relationIsStreamingJob, + relationType, + relationTypeTitleCase, +} from "../pages/api/streaming" +import { CatalogModal, useCatalogModal } from "./CatalogModal" function boundBox( - fragmentPosition: FragmentPointPosition[], + relationPosition: RelationPointPosition[], nodeRadius: number ): { width: number @@ -35,7 +43,7 @@ function boundBox( } { let width = 0 let height = 0 - for (const { x, y } of fragmentPosition) { + for (const { x, y } of relationPosition) { width = Math.max(width, x + nodeRadius) height = Math.max(height, y + nodeRadius) } @@ -43,21 +51,25 @@ function boundBox( } const layerMargin = 50 -const rowMargin = 200 -const nodeRadius = 10 -const layoutMargin = 100 +const rowMargin = 50 +export const nodeRadius = 12 +const layoutMargin = 50 export default function RelationDependencyGraph({ nodes, selectedId, + setSelectedId, }: { - nodes: FragmentPoint[] - selectedId?: string + nodes: RelationPoint[] + selectedId: string | undefined + setSelectedId: (id: string) => void }) { - const svgRef = useRef() + const [modalData, setModalId] = useCatalogModal(nodes.map((n) => n.relation)) + + const svgRef = useRef(null) const layoutMapCallback = useCallback(() => { - const layoutMap = flipLayoutPoint( + const layoutMap = flipLayoutRelation( nodes, layerMargin, rowMargin, @@ -68,9 +80,9 @@ export default function RelationDependencyGraph({ x: x + layoutMargin, y: y + layoutMargin, ...data, - } as FragmentPointPosition) + } as RelationPointPosition) ) - const links = generatePointEdges(layoutMap) + const links = generateRelationEdges(layoutMap) const { width, height } = boundBox(layoutMap, nodeRadius) return { layoutMap, @@ -96,29 +108,30 @@ export default function RelationDependencyGraph({ const edgeSelection = svgSelection .select(".edges") - .selectAll(".edge") + .selectAll(".edge") .data(links) + type EdgeSelection = typeof edgeSelection const isSelected = (id: string) => id === selectedId - const applyEdge = (sel: any) => + const applyEdge = (sel: EdgeSelection) => sel - .attr("d", ({ points }: any) => line(points)) + .attr("d", ({ points }) => line(points)) .attr("fill", "none") .attr("stroke-width", 1) - .attr("stroke-width", (d: any) => - isSelected(d.source) || isSelected(d.target) ? 2 : 1 + .attr("stroke-width", (d) => + isSelected(d.source) || isSelected(d.target) ? 4 : 2 ) - .attr("opacity", (d: any) => + .attr("opacity", (d) => isSelected(d.source) || isSelected(d.target) ? 1 : 0.5 ) - .attr("stroke", (d: any) => + .attr("stroke", (d) => isSelected(d.source) || isSelected(d.target) ? theme.colors.blue["500"] : theme.colors.gray["300"] ) - const createEdge = (sel: any) => + const createEdge = (sel: Enter) => sel.append("path").attr("class", "edge").call(applyEdge) edgeSelection.exit().remove() edgeSelection.enter().call(createEdge) @@ -127,21 +140,23 @@ export default function RelationDependencyGraph({ const applyNode = (g: NodeSelection) => { g.attr("transform", ({ x, y }) => `translate(${x},${y})`) + // Circle let circle = g.select("circle") if (circle.empty()) { circle = g.append("circle") } - circle - .attr("r", nodeRadius) - .style("cursor", "pointer") - .attr("fill", ({ id }) => - isSelected(id) ? theme.colors.blue["500"] : theme.colors.gray["500"] - ) + circle.attr("r", nodeRadius).attr("fill", ({ id, relation }) => { + const weight = relationIsStreamingJob(relation) ? "500" : "400" + return isSelected(id) + ? theme.colors.blue[weight] + : theme.colors.gray[weight] + }) - let text = g.select("text") + // Relation name + let text = g.select(".text") if (text.empty()) { - text = g.append("text") + text = g.append("text").attr("class", "text") } text @@ -150,24 +165,66 @@ export default function RelationDependencyGraph({ .attr("font-family", "inherit") .attr("text-anchor", "middle") .attr("dy", nodeRadius * 2) - .attr("fill", "black") .attr("font-size", 12) .attr("transform", "rotate(-8)") + // Relation type + let typeText = g.select(".type") + if (typeText.empty()) { + typeText = g.append("text").attr("class", "type") + } + + const relationTypeAbbr = (relation: Relation) => { + const type = relationType(relation) + if (type === "SINK") { + return "K" + } else { + return type.charAt(0) + } + } + + typeText + .attr("fill", "white") + .text(({ relation }) => `${relationTypeAbbr(relation)}`) + .attr("font-family", "inherit") + .attr("text-anchor", "middle") + .attr("dy", nodeRadius * 0.5) + .attr("font-size", 16) + .attr("font-weight", "bold") + + // Relation type tooltip + let typeTooltip = g.select("title") + if (typeTooltip.empty()) { + typeTooltip = g.append("title") + } + + typeTooltip.text( + ({ relation }) => + `${relation.name} (${relationTypeTitleCase(relation)})` + ) + + // Relation modal + g.style("cursor", "pointer").on("click", (_, { relation, id }) => { + setSelectedId(id) + setModalId(relation.id) + }) + return g } - const createNode = (sel: any) => + const createNode = (sel: Enter) => sel.append("g").attr("class", "node").call(applyNode) const g = svgSelection.select(".boxes") - const nodeSelection = g.selectAll(".node").data(layoutMap) + const nodeSelection = g + .selectAll(".node") + .data(layoutMap) type NodeSelection = typeof nodeSelection nodeSelection.enter().call(createNode) nodeSelection.call(applyNode) nodeSelection.exit().remove() - }, [layoutMap, links, selectedId]) + }, [layoutMap, links, selectedId, setModalId, setSelectedId]) return ( <> @@ -175,6 +232,7 @@ export default function RelationDependencyGraph({ + setModalId(null)} /> ) } diff --git a/dashboard/components/Relations.tsx b/dashboard/components/Relations.tsx index c16a70e8c6fa2..0422eaa2531fa 100644 --- a/dashboard/components/Relations.tsx +++ b/dashboard/components/Relations.tsx @@ -18,13 +18,6 @@ import { Box, Button, - Modal, - ModalBody, - ModalCloseButton, - ModalContent, - ModalFooter, - ModalHeader, - ModalOverlay, Table, TableContainer, Tbody, @@ -37,7 +30,6 @@ import loadable from "@loadable/component" import Head from "next/head" import Link from "next/link" -import { parseAsInteger, useQueryState } from "nuqs" import { Fragment } from "react" import Title from "../components/Title" import extractColumnInfo from "../lib/extractInfo" @@ -48,8 +40,9 @@ import { Source as RwSource, Table as RwTable, } from "../proto/gen/catalog" +import { CatalogModal, useCatalogModal } from "./CatalogModal" -const ReactJson = loadable(() => import("react-json-view")) +export const ReactJson = loadable(() => import("react-json-view")) export type Column = { name: string @@ -122,40 +115,10 @@ export function Relations( extraColumns: Column[] ) { const { response: relationList } = useFetch(getRelations) + const [modalData, setModalId] = useCatalogModal(relationList) - const [modalId, setModalId] = useQueryState("id", parseAsInteger) - const modalData = relationList?.find((r) => r.id === modalId) - - const catalogModal = ( - setModalId(null)} - size="3xl" - > - - - - Catalog of {modalData?.id} - {modalData?.name} - - - - {modalData && ( - - )} - - - - - - - + const modal = ( + setModalId(null)} /> ) const table = ( @@ -214,7 +177,7 @@ export function Relations( {title} - {catalogModal} + {modal} {table} ) diff --git a/dashboard/lib/layout.ts b/dashboard/lib/layout.ts index 1182976dfe8cb..924374341daa8 100644 --- a/dashboard/lib/layout.ts +++ b/dashboard/lib/layout.ts @@ -15,10 +15,20 @@ * */ -import { cloneDeep, max } from "lodash" +import { max } from "lodash" +import { Relation } from "../pages/api/streaming" import { TableFragments_Fragment } from "../proto/gen/meta" import { GraphNode } from "./algo" +export type Enter = Type extends d3.Selection< + any, + infer B, + infer C, + infer D +> + ? d3.Selection + : never + interface DagNode { node: GraphNode temp: boolean @@ -210,16 +220,16 @@ function dagLayout(nodes: GraphNode[]) { } /** - * @param fragments - * @returns Layer and row of the fragment + * @param items + * @returns Layer and row of the item */ -function gridLayout( - fragments: Array -): Map { - // turn FragmentBox to GraphNode - let idToBox = new Map() - for (let fragment of fragments) { - idToBox.set(fragment.id, fragment) +function gridLayout( + items: Array +): Map { + // turn item to GraphNode + let idToItem = new Map() + for (let item of items) { + idToItem.set(item.id, item) } let nodeToId = new Map() @@ -232,23 +242,23 @@ function gridLayout( let newNode = { nextNodes: new Array(), } - let ab = idToBox.get(id) - if (ab === undefined) { + let item = idToItem.get(id) + if (item === undefined) { throw Error(`no such id ${id}`) } - for (let id of ab.parentIds) { + for (let id of item.parentIds) { getNode(id).nextNodes.push(newNode) } idToNode.set(id, newNode) nodeToId.set(newNode, id) return newNode } - for (let fragment of fragments) { - getNode(fragment.id) + for (let item of items) { + getNode(item.id) } // run daglayout on GraphNode - let rtn = new Map() + let rtn = new Map() let allNodes = new Array() for (let _n of nodeToId.keys()) { allNodes.push(_n) @@ -257,33 +267,34 @@ function gridLayout( for (let item of resultMap) { let id = nodeToId.get(item[0]) if (!id) { - throw Error(`no corresponding fragment id of node ${item[0]}`) + throw Error(`no corresponding item of node ${item[0]}`) } - let fb = idToBox.get(id) + let fb = idToItem.get(id) if (!fb) { - throw Error(`fragment id ${id} is not present in idToBox`) + throw Error(`item id ${id} is not present in idToBox`) } rtn.set(fb, item[1]) } return rtn } -export interface FragmentBox { +export interface LayoutItemBase { id: string - name: string - order: number // preference order, fragment box with larger order will be placed at right + order: number // preference order, item with larger order will be placed at right or down width: number height: number parentIds: string[] +} + +export type FragmentBox = LayoutItemBase & { + name: string externalParentIds: string[] fragment?: TableFragments_Fragment } -export interface FragmentPoint { - id: string +export type RelationPoint = LayoutItemBase & { name: string - order: number // preference order, fragment box with larger order will be placed at right - parentIds: string[] + relation: Relation } export interface Position { @@ -292,7 +303,7 @@ export interface Position { } export type FragmentBoxPosition = FragmentBox & Position -export type FragmentPointPosition = FragmentPoint & Position +export type RelationPointPosition = RelationPoint & Position export interface Edge { points: Array @@ -301,15 +312,15 @@ export interface Edge { } /** - * @param fragments + * @param items * @returns the coordination of the top-left corner of the fragment box */ -export function layout( - fragments: Array, +export function layoutItem( + items: Array, layerMargin: number, rowMargin: number -): FragmentBoxPosition[] { - let layoutMap = gridLayout(fragments) +): (I & Position)[] { + let layoutMap = gridLayout(items) let layerRequiredWidth = new Map() let rowRequiredHeight = new Map() let maxLayer = 0, @@ -373,7 +384,7 @@ export function layout( getCumulativeMargin(i, rowMargin, rowCumulativeHeight, rowRequiredHeight) } - let rtn: Array = [] + let rtn: Array = [] for (let [data, [layer, row]] of layoutMap) { let x = layerCumulativeWidth.get(layer) @@ -391,39 +402,13 @@ export function layout( return rtn } -export function flipLayout( - fragments: Array, - layerMargin: number, - rowMargin: number -): FragmentBoxPosition[] { - const fragments_ = cloneDeep(fragments) - for (let fragment of fragments_) { - ;[fragment.width, fragment.height] = [fragment.height, fragment.width] - } - const fragmentPosition = layout(fragments_, rowMargin, layerMargin) - return fragmentPosition.map(({ x, y, ...data }) => ({ - x: y, - y: x, - ...data, - })) -} - -export function layoutPoint( - fragments: Array, +function layoutRelation( + relations: Array, layerMargin: number, rowMargin: number, nodeRadius: number -): FragmentPointPosition[] { - const fragmentBoxes: Array = [] - for (let { ...others } of fragments) { - fragmentBoxes.push({ - width: nodeRadius * 2, - height: nodeRadius * 2, - externalParentIds: [], // we don't care about external parent for point layout - ...others, - }) - } - const result = layout(fragmentBoxes, layerMargin, rowMargin) +): RelationPointPosition[] { + const result = layoutItem(relations, layerMargin, rowMargin) return result.map(({ x, y, ...data }) => ({ x: x + nodeRadius, y: y + nodeRadius, @@ -431,14 +416,14 @@ export function layoutPoint( })) } -export function flipLayoutPoint( - fragments: Array, +export function flipLayoutRelation( + relations: Array, layerMargin: number, rowMargin: number, nodeRadius: number -): FragmentPointPosition[] { - const fragmentPosition = layoutPoint( - fragments, +): RelationPointPosition[] { + const fragmentPosition = layoutRelation( + relations, rowMargin, layerMargin, nodeRadius @@ -450,21 +435,23 @@ export function flipLayoutPoint( })) } -export function generatePointEdges(layoutMap: FragmentPointPosition[]): Edge[] { +export function generateRelationEdges( + layoutMap: RelationPointPosition[] +): Edge[] { const links = [] - const fragmentMap = new Map() + const relationMap = new Map() for (const x of layoutMap) { - fragmentMap.set(x.id, x) + relationMap.set(x.id, x) } - for (const fragment of layoutMap) { - for (const parentId of fragment.parentIds) { - const parentFragment = fragmentMap.get(parentId)! + for (const relation of layoutMap) { + for (const parentId of relation.parentIds) { + const parentRelation = relationMap.get(parentId)! links.push({ points: [ - { x: fragment.x, y: fragment.y }, - { x: parentFragment.x, y: parentFragment.y }, + { x: relation.x, y: relation.y }, + { x: parentRelation.x, y: parentRelation.y }, ], - source: fragment.id, + source: relation.id, target: parentId, }) } @@ -472,7 +459,9 @@ export function generatePointEdges(layoutMap: FragmentPointPosition[]): Edge[] { return links } -export function generateBoxEdges(layoutMap: FragmentBoxPosition[]): Edge[] { +export function generateFragmentEdges( + layoutMap: FragmentBoxPosition[] +): Edge[] { const links = [] const fragmentMap = new Map() for (const x of layoutMap) { diff --git a/dashboard/pages/api/streaming.ts b/dashboard/pages/api/streaming.ts index a77a165357b9f..13fa8716f821a 100644 --- a/dashboard/pages/api/streaming.ts +++ b/dashboard/pages/api/streaming.ts @@ -45,8 +45,26 @@ export interface StreamingJob extends Relation { dependentRelations: number[] } +export function relationType(x: Relation) { + if ((x as Table).tableType !== undefined) { + return (x as Table).tableType + } else if ((x as Sink).sinkFromName !== undefined) { + return "SINK" + } else if ((x as Source).info !== undefined) { + return "SOURCE" + } else { + return "UNKNOWN" + } +} +export type RelationType = ReturnType + +export function relationTypeTitleCase(x: Relation) { + return _.startCase(_.toLower(relationType(x))) +} + export function relationIsStreamingJob(x: Relation): x is StreamingJob { - return (x as StreamingJob).dependentRelations !== undefined + const type = relationType(x) + return type !== "UNKNOWN" && type !== "SOURCE" && type !== "INTERNAL" } export async function getStreamingJobs() { diff --git a/dashboard/pages/dependency_graph.tsx b/dashboard/pages/dependency_graph.tsx index fb29f57b11bb5..a4c13a94df169 100644 --- a/dashboard/pages/dependency_graph.tsx +++ b/dashboard/pages/dependency_graph.tsx @@ -20,15 +20,17 @@ import { reverse, sortBy } from "lodash" import Head from "next/head" import { parseAsInteger, useQueryState } from "nuqs" import { Fragment, useCallback } from "react" -import RelationDependencyGraph from "../components/RelationDependencyGraph" +import RelationDependencyGraph, { + nodeRadius, +} from "../components/RelationDependencyGraph" import Title from "../components/Title" -import { FragmentPoint } from "../lib/layout" +import { RelationPoint } from "../lib/layout" import useFetch from "./api/fetch" import { Relation, getRelations, relationIsStreamingJob } from "./api/streaming" const SIDEBAR_WIDTH = "200px" -function buildDependencyAsEdges(list: Relation[]): FragmentPoint[] { +function buildDependencyAsEdges(list: Relation[]): RelationPoint[] { const edges = [] const relationSet = new Set(list.map((r) => r.id)) for (const r of reverse(sortBy(list, "id"))) { @@ -41,24 +43,27 @@ function buildDependencyAsEdges(list: Relation[]): FragmentPoint[] { .map((r) => r.toString()) : [], order: r.id, + width: nodeRadius * 2, + height: nodeRadius * 2, + relation: r, }) } return edges } export default function StreamingGraph() { - const { response: streamingJobList } = useFetch(getRelations) + const { response: relationList } = useFetch(getRelations) const [selectedId, setSelectedId] = useQueryState("id", parseAsInteger) - const mvDependencyCallback = useCallback(() => { - if (streamingJobList) { - return buildDependencyAsEdges(streamingJobList) + const relationDependencyCallback = useCallback(() => { + if (relationList) { + return buildDependencyAsEdges(relationList) } else { return undefined } - }, [streamingJobList]) + }, [relationList]) - const mvDependency = mvDependencyCallback() + const relationDependency = relationDependencyCallback() const retVal = ( @@ -77,7 +82,7 @@ export default function StreamingGraph() { - {streamingJobList?.map((r) => { + {relationList?.map((r) => { const match = selectedId === r.id return (