diff --git a/Cargo.lock b/Cargo.lock index 8950c51de77c..fb63420db1f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,7 +45,7 @@ dependencies = [ "getrandom", "once_cell", "version_check", - "zerocopy 0.7.31", + "zerocopy", ] [[package]] @@ -170,40 +170,40 @@ dependencies = [ [[package]] name = "apache-avro" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0fdddc3fdac97394ffcc5c89c634faa9c1c166ced54189af34e407c97b6ee7" +version = "0.16.0" +source = "git+https://github.com/risingwavelabs/avro?rev=d0846a16ce813a225af04ade35b3b8117b137a29#d0846a16ce813a225af04ade35b3b8117b137a29" dependencies = [ - "apache-avro-derive", - "byteorder", + "bzip2", + "crc32fast", "digest", "lazy_static", - "libflate 1.4.0", + "libflate", "log", "num-bigint", "quad-rand", "rand", - "regex", + "regex-lite", "serde", "serde_json", + "snap", "strum", "strum_macros", "thiserror", - "typed-builder 0.14.0", + "typed-builder 0.16.2", "uuid", - "zerocopy 0.6.6", + "xz2", + "zstd 0.12.4", ] [[package]] name = "apache-avro" -version = "0.16.0" -source = "git+https://github.com/risingwavelabs/avro?rev=d0846a16ce813a225af04ade35b3b8117b137a29#d0846a16ce813a225af04ade35b3b8117b137a29" +version = "0.17.0" +source = "git+https://github.com/icelake-io/avro.git?rev=4866a4ad0ed5d6af7160c9b52af898ab6d0551f9#4866a4ad0ed5d6af7160c9b52af898ab6d0551f9" dependencies = [ - "bzip2", - "crc32fast", + "apache-avro-derive", + "bigdecimal 0.4.2", "digest", - "lazy_static", - "libflate 2.0.0", + "libflate", "log", "num-bigint", "quad-rand", @@ -211,27 +211,23 @@ dependencies = [ "regex-lite", "serde", "serde_json", - "snap", "strum", "strum_macros", "thiserror", - "typed-builder 0.16.2", + "typed-builder 0.18.0", "uuid", - "xz2", - "zstd 0.12.4", ] [[package]] name = "apache-avro-derive" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6686cd705badba064ec2322b9c3d72f5c70db8394e486bbb56e84fbdb3fa158c" +version = "0.17.0" +source = "git+https://github.com/icelake-io/avro.git?rev=4866a4ad0ed5d6af7160c9b52af898ab6d0551f9#4866a4ad0ed5d6af7160c9b52af898ab6d0551f9" dependencies = [ "darling 0.20.3", "proc-macro2", "quote", "serde_json", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -806,7 +802,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -855,7 +851,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -872,7 +868,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -905,7 +901,7 @@ dependencies = [ "derive_utils", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -1689,7 +1685,7 @@ dependencies = [ "proc-macro-crate 2.0.0", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", "syn_derive", ] @@ -1978,7 +1974,7 @@ checksum = "bc7cb2538d4ecc42b6c3b57a83094d8c69894e74468d18cd045a09fdea807358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2095,7 +2091,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2530,7 +2526,7 @@ dependencies = [ "proc-macro-error 1.0.4", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2744,7 +2740,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30d2b3721e861707777e3195b0158f950ae6dc4a27e4d02ff9f67e3eb3de199e" dependencies = [ "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2772,7 +2768,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2799,7 +2795,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2816,7 +2812,7 @@ checksum = "2fa16a70dd58129e4dfffdff535fb1bce66673f7bbeec4a5a1765a504e1ccd84" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2888,7 +2884,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -2921,7 +2917,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3361,7 +3357,7 @@ checksum = "9abcad25e9720609ccb3dcdb795d845e37d8ce34183330a9f48b03a1a71c8e21" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3590,7 +3586,7 @@ dependencies = [ "enum-ordinalize 4.3.0", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3667,7 +3663,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3707,7 +3703,7 @@ checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3720,7 +3716,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3740,7 +3736,7 @@ checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -3760,7 +3756,7 @@ checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -4238,7 +4234,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -4250,7 +4246,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -4262,7 +4258,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -4350,7 +4346,7 @@ checksum = "5df2c13d48c8cb8a3ec093ede6f0f4482f327d7bb781120c5fb483ef0f17e758" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -4420,7 +4416,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -5055,10 +5051,10 @@ dependencies = [ [[package]] name = "icelake" version = "0.0.10" -source = "git+https://github.com/icelake-io/icelake?rev=3f7b53ba5b563524212c25810345d1314678e7fc#3f7b53ba5b563524212c25810345d1314678e7fc" +source = "git+https://github.com/icelake-io/icelake?rev=b4f4ca3c6d29092bd331925ead0bcceaa38bdd57#b4f4ca3c6d29092bd331925ead0bcceaa38bdd57" dependencies = [ "anyhow", - "apache-avro 0.15.0", + "apache-avro 0.17.0", "arrow-arith 49.0.0", "arrow-array 49.0.0", "arrow-buffer 49.0.0", @@ -5077,6 +5073,7 @@ dependencies = [ "faster-hex", "futures", "itertools 0.11.0", + "lazy_static", "log", "murmur3", "once_cell", @@ -5193,7 +5190,7 @@ checksum = "ce243b1bfa62ffc028f1cc3b6034ec63d649f3031bc8a4fbbb004e1ac17d1f68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -5550,17 +5547,6 @@ version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" -[[package]] -name = "libflate" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18" -dependencies = [ - "adler32", - "crc32fast", - "libflate_lz77 1.2.0", -] - [[package]] name = "libflate" version = "2.0.0" @@ -5571,16 +5557,7 @@ dependencies = [ "core2", "crc32fast", "dary_heap", - "libflate_lz77 2.0.0", -] - -[[package]] -name = "libflate_lz77" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf" -dependencies = [ - "rle-decode-fast", + "libflate_lz77", ] [[package]] @@ -5956,7 +5933,7 @@ dependencies = [ "proc-macro2", "prost-build 0.12.1", "quote", - "syn 2.0.37", + "syn 2.0.48", "tonic-build", ] @@ -6193,7 +6170,7 @@ dependencies = [ "proc-macro-error 1.0.4", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", "termcolor", "thiserror", ] @@ -6722,7 +6699,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -6921,7 +6898,7 @@ dependencies = [ "proc-macro-error 1.0.4", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -7130,7 +7107,7 @@ dependencies = [ "regex", "regex-syntax 0.7.5", "structmeta", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -7329,7 +7306,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -7460,7 +7437,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -7608,7 +7585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -7697,9 +7674,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -7821,7 +7798,7 @@ checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" dependencies = [ "bytes", "heck 0.4.1", - "itertools 0.11.0", + "itertools 0.10.5", "log", "multimap 0.8.3", "once_cell", @@ -7830,7 +7807,7 @@ dependencies = [ "prost 0.12.1", "prost-types 0.12.1", "regex", - "syn 2.0.37", + "syn 2.0.48", "tempfile", "which", ] @@ -7855,10 +7832,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.10.5", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -7867,7 +7844,7 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -8062,9 +8039,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -8244,7 +8221,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -8518,7 +8495,7 @@ dependencies = [ "prettyplease 0.2.15", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -9136,6 +9113,7 @@ version = "1.7.0-alpha" dependencies = [ "aho-corasick", "anyhow", + "arrow-schema 49.0.0", "async-trait", "auto_enums", "chrono", @@ -9145,6 +9123,7 @@ dependencies = [ "futures-async-stream", "futures-util", "hex", + "icelake", "itertools 0.12.0", "jsonbb", "madsim-tokio", @@ -9174,7 +9153,7 @@ dependencies = [ "itertools 0.12.0", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -10084,7 +10063,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.37", + "syn 2.0.48", "walkdir", ] @@ -10343,7 +10322,7 @@ dependencies = [ "proc-macro-error 1.0.4", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -10401,7 +10380,7 @@ dependencies = [ "proc-macro2", "quote", "sea-bae", - "syn 2.0.37", + "syn 2.0.48", "unicode-ident", ] @@ -10584,9 +10563,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] @@ -10633,13 +10612,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -10655,9 +10634,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -10700,7 +10679,7 @@ checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -10750,7 +10729,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -10788,7 +10767,7 @@ checksum = "b93fb4adc70021ac1b47f7d45e8cc4169baaa7ea58483bc5b721d19a26202212" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -11464,7 +11443,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -11475,7 +11454,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -11489,15 +11468,15 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.25.2" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8d03b598d3d0fff69bf533ee3ef19b8eeb342729596df84bcc7e1f96ec4059" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck 0.4.1", "proc-macro2", "quote", "rustversion", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -11562,9 +11541,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -11591,7 +11570,7 @@ dependencies = [ "proc-macro-error 1.0.4", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -11696,9 +11675,9 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] @@ -11722,18 +11701,18 @@ dependencies = [ "either", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -11908,7 +11887,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -12152,7 +12131,7 @@ dependencies = [ "proc-macro2", "prost-build 0.12.1", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -12233,7 +12212,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -12372,22 +12351,20 @@ dependencies = [ [[package]] name = "typed-builder" -version = "0.14.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cba322cb9b7bc6ca048de49e83918223f35e7a86311267013afff257004870" +checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", + "typed-builder-macro 0.16.2", ] [[package]] name = "typed-builder" -version = "0.16.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" +checksum = "e47c0496149861b7c95198088cbf36645016b1a0734cf350c50e2a38e070f38a" dependencies = [ - "typed-builder-macro", + "typed-builder-macro 0.18.0", ] [[package]] @@ -12398,7 +12375,18 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", +] + +[[package]] +name = "typed-builder-macro" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982ee4197351b5c9782847ef5ec1fdcaf50503fb19d68f9771adae314e72b492" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", ] [[package]] @@ -12686,7 +12674,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -12720,7 +12708,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -12850,7 +12838,7 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", "wasmtime-component-util", "wasmtime-wit-bindgen", "wit-parser", @@ -13042,7 +13030,7 @@ checksum = "f55c5565959287c21dd0f4277ae3518dd2ae62679f655ee2dbc4396e19d210db" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -13209,7 +13197,7 @@ dependencies = [ "proc-macro2", "quote", "shellexpand", - "syn 2.0.37", + "syn 2.0.48", "witx", ] @@ -13221,7 +13209,7 @@ checksum = "31ae1ec11a17ea481539ee9a5719a278c9790d974060fbf71db4b2c05378780b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", "wiggle-generate", ] @@ -13550,7 +13538,7 @@ version = "1.7.0-alpha" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] @@ -13593,6 +13581,7 @@ dependencies = [ "aws-smithy-types", "axum", "base64 0.21.4", + "bigdecimal 0.4.2", "bit-vec", "bitflags 2.4.0", "byteorder", @@ -13628,7 +13617,7 @@ dependencies = [ "hyper", "indexmap 1.9.3", "indexmap 2.0.0", - "itertools 0.11.0", + "itertools 0.10.5", "jni", "lazy_static", "lexical-core", @@ -13693,7 +13682,7 @@ dependencies = [ "strum", "subtle", "syn 1.0.109", - "syn 2.0.37", + "syn 2.0.48", "time", "time-macros", "tinyvec", @@ -13714,6 +13703,8 @@ dependencies = [ "uuid", "whoami", "zeroize", + "zstd 0.13.0", + "zstd-safe 7.0.0", "zstd-sys", ] @@ -13805,34 +13796,13 @@ version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a599daf1b507819c1121f0bf87fa37eb19daac6aff3aefefd4e6e2e0f2020fc" -[[package]] -name = "zerocopy" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6" -dependencies = [ - "byteorder", - "zerocopy-derive 0.6.6", -] - [[package]] name = "zerocopy" version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" dependencies = [ - "zerocopy-derive 0.7.31", -] - -[[package]] -name = "zerocopy-derive" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", + "zerocopy-derive", ] [[package]] @@ -13843,7 +13813,7 @@ checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 945091f38437..d3fec07a9c3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,7 +120,7 @@ tonic = { package = "madsim-tonic", version = "0.4.1" } tonic-build = { package = "madsim-tonic-build", version = "0.4.2" } otlp-embedded = { git = "https://github.com/risingwavelabs/otlp-embedded", rev = "58c1f003484449d7c6dd693b348bf19dd44889cb" } prost = { version = "0.12" } -icelake = { git = "https://github.com/icelake-io/icelake", rev = "3f7b53ba5b563524212c25810345d1314678e7fc", features = [ +icelake = { git = "https://github.com/icelake-io/icelake", rev = "b4f4ca3c6d29092bd331925ead0bcceaa38bdd57", features = [ "prometheus", ] } arrow-array = "49" diff --git a/proto/expr.proto b/proto/expr.proto index e02f7e45bb19..9c6dd8e59fbf 100644 --- a/proto/expr.proto +++ b/proto/expr.proto @@ -275,6 +275,9 @@ message ExprNode { PG_GET_INDEXDEF = 2400; COL_DESCRIPTION = 2401; PG_GET_VIEWDEF = 2402; + + // EXTERNAL + ICEBERG_TRANSFORM = 2201; } // Only use this field for function call. For other types of expression, it should be UNSPECIFIED. Type function_type = 1; diff --git a/src/expr/impl/Cargo.toml b/src/expr/impl/Cargo.toml index bd24448ebbd6..bee2e4eb10c0 100644 --- a/src/expr/impl/Cargo.toml +++ b/src/expr/impl/Cargo.toml @@ -18,6 +18,7 @@ normal = ["workspace-hack", "ctor"] [dependencies] aho-corasick = "1" anyhow = "1" +arrow-schema = { workspace = true } async-trait = "0.1" auto_enums = "0.8" chrono = { version = "0.4", default-features = false, features = [ @@ -28,6 +29,7 @@ fancy-regex = "0.13" futures-async-stream = { workspace = true } futures-util = "0.3" hex = "0.4" +icelake = { workspace = true } itertools = "0.12" jsonbb = "0.1.2" md5 = "0.7" diff --git a/src/expr/impl/src/scalar/array.rs b/src/expr/impl/src/scalar/array.rs index 792ebcb0346f..aaefd17bba07 100644 --- a/src/expr/impl/src/scalar/array.rs +++ b/src/expr/impl/src/scalar/array.rs @@ -27,3 +27,36 @@ fn array(row: impl Row, ctx: &Context) -> ListValue { fn row_(row: impl Row) -> StructValue { StructValue::new(row.iter().map(|d| d.to_owned_datum()).collect()) } + +#[cfg(test)] +mod tests { + use risingwave_common::array::DataChunk; + use risingwave_common::row::Row; + use risingwave_common::test_prelude::DataChunkTestExt; + use risingwave_common::types::ToOwnedDatum; + use risingwave_common::util::iter_util::ZipEqDebug; + use risingwave_expr::expr::build_from_pretty; + + #[tokio::test] + async fn test_row_expr() { + let expr = build_from_pretty("(row:struct $0:int4 $1:int4 $2:int4)"); + let (input, expected) = DataChunk::from_pretty( + "i i i + 1 2 3 (1,2,3) + 4 2 1 (4,2,1) + 9 1 3 (9,1,3) + 1 1 1 (1,1,1)", + ) + .split_column_at(3); + + // test eval + let output = expr.eval(&input).await.unwrap(); + assert_eq!(&output, expected.column_at(0)); + + // test eval_row + for (row, expected) in input.rows().zip_eq_debug(expected.rows()) { + let result = expr.eval_row(&row.to_owned_row()).await.unwrap(); + assert_eq!(result, expected.datum_at(0).to_owned_datum()); + } + } +} diff --git a/src/expr/impl/src/scalar/external/iceberg.rs b/src/expr/impl/src/scalar/external/iceberg.rs new file mode 100644 index 000000000000..3973efee559d --- /dev/null +++ b/src/expr/impl/src/scalar/external/iceberg.rs @@ -0,0 +1,197 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contains the expression for computing the iceberg partition value. +//! spec ref: +use std::fmt::Formatter; +use std::str::FromStr; +use std::sync::Arc; + +use anyhow::anyhow; +use arrow_schema::DataType as ArrowDataType; +use icelake::types::{ + create_transform_function, Any as IcelakeDataType, BoxedTransformFunction, Transform, +}; +use risingwave_common::array::{ArrayRef, DataChunk}; +use risingwave_common::ensure; +use risingwave_common::row::OwnedRow; +use risingwave_common::types::{DataType, Datum}; +use risingwave_expr::expr::BoxedExpression; +use risingwave_expr::{build_function, ExprError, Result}; + +pub struct IcebergTransform { + child: BoxedExpression, + transform: BoxedTransformFunction, + return_type: DataType, +} + +impl std::fmt::Debug for IcebergTransform { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("IcebergTransform") + .field("child", &self.child) + .field("return_type", &self.return_type) + .finish() + } +} + +#[async_trait::async_trait] +impl risingwave_expr::expr::Expression for IcebergTransform { + fn return_type(&self) -> DataType { + self.return_type.clone() + } + + async fn eval(&self, data_chunk: &DataChunk) -> Result { + // Get the child array + let array = self.child.eval(data_chunk).await?; + // Convert to arrow array + let arrow_array = array.as_ref().try_into().unwrap(); + // Transform + let res_array = self.transform.transform(arrow_array).unwrap(); + // Convert back to array ref and return it + Ok(Arc::new((&res_array).try_into().unwrap())) + } + + async fn eval_row(&self, _row: &OwnedRow) -> Result { + Err(ExprError::Internal(anyhow!( + "eval_row in iceberg_transform is not supported yet" + ))) + } +} + +#[build_function("iceberg_transform(varchar, any) -> any", type_infer = "panic")] +fn build(return_type: DataType, mut children: Vec) -> Result { + let transform_type = { + let datum = children[0].eval_const()?.unwrap(); + let str = datum.as_utf8(); + Transform::from_str(str).map_err(|_| ExprError::InvalidParam { + name: "transform type in icberg_transform", + reason: format!("Fail to parse {str} as iceberg transform type").into(), + })? + }; + + // For Identity and Void transform, we will use `InputRef` and const null in frontend, + // so it should not reach here. + assert!(!matches!( + transform_type, + Transform::Identity | Transform::Void + )); + + // Check type: + // 1. input type can be transform successfully + // 2. return type is the same as the result type + let input_type = IcelakeDataType::try_from(ArrowDataType::try_from(children[1].return_type())?) + .map_err(|err| ExprError::InvalidParam { + name: "input type in iceberg_transform", + reason: format!("Failed to convert input type to icelake type, got error: {err}",) + .into(), + })?; + let expect_res_type = transform_type.result_type(&input_type).map_err( + |err| ExprError::InvalidParam { + name: "input type in iceberg_transform", + reason: format!( + "Failed to get result type for transform type {:?} and input type {:?}, got error: {}", + transform_type, input_type, err + ) + .into() + })?; + let actual_res_type = IcelakeDataType::try_from(ArrowDataType::try_from(return_type.clone())?) + .map_err(|err| ExprError::InvalidParam { + name: "return type in iceberg_transform", + reason: format!("Failed to convert return type to icelake type, got error: {err}",) + .into(), + })?; + ensure!( + expect_res_type == actual_res_type, + ExprError::InvalidParam { + name: "return type in iceberg_transform", + reason: format!( + "Expect return type {:?} but got {:?}", + expect_res_type, actual_res_type + ) + .into() + } + ); + + Ok(Box::new(IcebergTransform { + child: children.remove(1), + transform: create_transform_function(&transform_type) + .map_err(|err| ExprError::Internal(err.into()))?, + return_type, + })) +} + +#[cfg(test)] +mod test { + use risingwave_common::array::{DataChunk, DataChunkTestExt}; + use risingwave_expr::expr::build_from_pretty; + + #[tokio::test] + async fn test_bucket() { + let (input, expected) = DataChunk::from_pretty( + "i i + 34 1373", + ) + .split_column_at(1); + let expr = build_from_pretty("(iceberg_transform:int4 bucket[2017]:varchar $0:int)"); + let res = expr.eval(&input).await.unwrap(); + assert_eq!(res, *expected.column_at(0)); + } + + #[tokio::test] + async fn test_truncate() { + let (input, expected) = DataChunk::from_pretty( + "T T + iceberg ice + risingwave ris + delta del", + ) + .split_column_at(1); + let expr = build_from_pretty("(iceberg_transform:varchar truncate[3]:varchar $0:varchar)"); + let res = expr.eval(&input).await.unwrap(); + assert_eq!(res, *expected.column_at(0)); + } + + #[tokio::test] + async fn test_year_month_day_hour() { + let (input, expected) = DataChunk::from_pretty( + "TZ i i i i + 1970-01-01T00:00:00.000000000+00:00 0 0 0 0 + 1971-02-01T01:00:00.000000000+00:00 1 13 396 9505 + 1972-03-01T02:00:00.000000000+00:00 2 26 790 18962 + 1970-05-01T06:00:00.000000000+00:00 0 4 120 2886 + 1970-06-01T07:00:00.000000000+00:00 0 5 151 3631", + ) + .split_column_at(1); + + // year + let expr = build_from_pretty("(iceberg_transform:int4 year:varchar $0:timestamptz)"); + let res = expr.eval(&input).await.unwrap(); + assert_eq!(res, *expected.column_at(0)); + + // month + let expr = build_from_pretty("(iceberg_transform:int4 month:varchar $0:timestamptz)"); + let res = expr.eval(&input).await.unwrap(); + assert_eq!(res, *expected.column_at(1)); + + // day + let expr = build_from_pretty("(iceberg_transform:int4 day:varchar $0:timestamptz)"); + let res = expr.eval(&input).await.unwrap(); + assert_eq!(res, *expected.column_at(2)); + + // hour + let expr = build_from_pretty("(iceberg_transform:int4 hour:varchar $0:timestamptz)"); + let res = expr.eval(&input).await.unwrap(); + assert_eq!(res, *expected.column_at(3)); + } +} diff --git a/src/expr/impl/src/scalar/external/mod.rs b/src/expr/impl/src/scalar/external/mod.rs new file mode 100644 index 000000000000..fee743b9a9d7 --- /dev/null +++ b/src/expr/impl/src/scalar/external/mod.rs @@ -0,0 +1,16 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contain the expression for external system. Such as partition compute expression for external sink. +pub mod iceberg; diff --git a/src/expr/impl/src/scalar/mod.rs b/src/expr/impl/src/scalar/mod.rs index c76a7b48663e..bc925e3f2831 100644 --- a/src/expr/impl/src/scalar/mod.rs +++ b/src/expr/impl/src/scalar/mod.rs @@ -78,6 +78,7 @@ mod to_char; mod to_jsonb; mod vnode; pub use to_jsonb::*; +mod external; mod to_timestamp; mod translate; mod trigonometric; diff --git a/src/frontend/src/expr/pure.rs b/src/frontend/src/expr/pure.rs index 08b6bb47fa04..7e7378a65752 100644 --- a/src/frontend/src/expr/pure.rs +++ b/src/frontend/src/expr/pure.rs @@ -225,7 +225,8 @@ impl ExprVisitor for ImpureAnalyzer { | expr_node::Type::Greatest | expr_node::Type::Least | expr_node::Type::ConvertFrom - | expr_node::Type::ConvertTo => + | expr_node::Type::ConvertTo + | expr_node::Type::IcebergTransform => // expression output is deterministic(same result for the same input) { func_call diff --git a/src/workspace-hack/Cargo.toml b/src/workspace-hack/Cargo.toml index c1bf5e9370c8..6b5996fe81d5 100644 --- a/src/workspace-hack/Cargo.toml +++ b/src/workspace-hack/Cargo.toml @@ -30,6 +30,7 @@ aws-smithy-runtime = { version = "1", default-features = false, features = ["cli aws-smithy-types = { version = "1", default-features = false, features = ["byte-stream-poll-next", "http-body-0-4-x", "hyper-0-14-x", "rt-tokio"] } axum = { version = "0.6" } base64 = { version = "0.21", features = ["alloc"] } +bigdecimal = { version = "0.4" } bit-vec = { version = "0.6" } bitflags = { version = "2", default-features = false, features = ["serde", "std"] } byteorder = { version = "1" } @@ -64,7 +65,7 @@ hmac = { version = "0.12", default-features = false, features = ["reset"] } hyper = { version = "0.14", features = ["full"] } indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1", default-features = false, features = ["serde", "std"] } indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2", features = ["serde"] } -itertools = { version = "0.11" } +itertools = { version = "0.10" } jni = { version = "0.21", features = ["invocation"] } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } lexical-core = { version = "0.8", features = ["format"] } @@ -148,6 +149,8 @@ url = { version = "2", features = ["serde"] } uuid = { version = "1", features = ["fast-rng", "serde", "v4"] } whoami = { version = "1" } zeroize = { version = "1" } +zstd = { version = "0.13" } +zstd-safe = { version = "7", default-features = false, features = ["arrays", "legacy", "std", "zdict_builder"] } zstd-sys = { version = "2", default-features = false, features = ["legacy", "std", "zdict_builder"] } [build-dependencies] @@ -166,7 +169,7 @@ frunk_core = { version = "0.4", default-features = false, features = ["std"] } generic-array = { version = "0.14", default-features = false, features = ["more_lengths", "zeroize"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["nightly", "raw"] } indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2", features = ["serde"] } -itertools = { version = "0.11" } +itertools = { version = "0.10" } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits"] } log = { version = "0.4", default-features = false, features = ["kv_unstable", "std"] }