From bb0745a3935cab24a5a097a0b153a9d5c488b0a7 Mon Sep 17 00:00:00 2001 From: Runji Wang Date: Wed, 31 Jan 2024 16:45:17 +0800 Subject: [PATCH 1/4] bump rquickjs Signed-off-by: Runji Wang --- Cargo.lock | 13 +++++++------ Cargo.toml | 2 -- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b73909cc979..a09d519868fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10090,26 +10090,27 @@ dependencies = [ [[package]] name = "rquickjs" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0db265d331ae1b1a9fdb68466a8359bc9dcc5e78a9c323f790322f8442e005ac" +checksum = "511ee02660e19cf451ebcbda6151a4023044293066e9ccd2b0498047203f6ab1" dependencies = [ "rquickjs-core", ] [[package]] name = "rquickjs-core" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e51f2fc99917699385bfa290b776e712e414b222d7c2a9b2cd67b8e93585f3" +checksum = "c10ce0bc11e249e738aadb8119e9a7619125cb866208236d88d2ee3dcdb7a67d" dependencies = [ "rquickjs-sys", ] [[package]] name = "rquickjs-sys" -version = "0.4.2" -source = "git+https://github.com/DelSkayn/rquickjs.git?rev=60696e8#60696e88dfb903d8f5cd81b2667fb3f64f9e9f67" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86030d52fc20e68115f93738c2cb960d6f900f64d99e1013ea0e170381eb0a72" dependencies = [ "cc", ] diff --git a/Cargo.toml b/Cargo.toml index b887bb254da6..02ca4fd6b0c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -282,8 +282,6 @@ tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = futures-timer = { git = "https://github.com/madsim-rs/futures-timer.git", rev = "05b33b4" } # patch: unlimit 4MB message size for grpc client etcd-client = { git = "https://github.com/risingwavelabs/etcd-client.git", rev = "4e84d40" } -# need binding on aarch64-unknown-linux-gnu, waiting for new release -rquickjs-sys = { git = "https://github.com/DelSkayn/rquickjs.git", rev = "60696e8" } [workspace.metadata.dylint] libraries = [{ path = "./lints" }] From fab45720a405e29b142e7514da56bf5350d5a44f Mon Sep 17 00:00:00 2001 From: Runji Wang Date: Wed, 31 Jan 2024 19:03:14 +0800 Subject: [PATCH 2/4] support inlined rust udf Signed-off-by: Runji Wang --- Cargo.lock | 166 ++++++++++---------- Cargo.toml | 4 +- e2e_test/udf/wasm_udf.slt | 21 +++ src/expr/core/src/expr/expr_udf.rs | 2 +- src/frontend/src/handler/create_function.rs | 70 +++++++-- src/sqlparser/src/ast/mod.rs | 18 +++ 6 files changed, 180 insertions(+), 101 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a09d519868fb..89b8d35e5887 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -630,7 +630,8 @@ dependencies = [ [[package]] name = "arrow-udf-js" version = "0.1.0" -source = "git+https://github.com/risingwavelabs/arrow-udf.git?rev=7ba1c22#7ba1c226fa2f7418a217ee064a19b90efeb7143c" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ce46976744232876b82f2017171f1b8abe930aab2916d475e05bb00d25e51e" dependencies = [ "anyhow", "arrow-array 50.0.0", @@ -641,9 +642,9 @@ dependencies = [ [[package]] name = "arrow-udf-wasm" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c45450e29b016fdc0ccbb22408daceead464a2178f7f8408886c7ca0c9e1aed" +checksum = "70b965912f2c5514ce243a384e9b4e61544932acd28f126ad1adb13da04fa247" dependencies = [ "anyhow", "arrow-array 50.0.0", @@ -652,6 +653,7 @@ dependencies = [ "base64 0.21.4", "genawaiter", "lazy_static", + "tempfile", "wasmtime", "wasmtime-wasi", ] @@ -2388,18 +2390,18 @@ dependencies = [ [[package]] name = "cranelift-bforest" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c22542c0b95bd3302f7ed6839869c561f2324bac2fd5e7e99f5cfa65fdc8b92" +checksum = "d819feeda4c420a18f1e28236ca0ce1177b22bf7c8a44ddee92dfe40de15bcf0" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b3db903ef2e9c8a4de2ea6db5db052c7857282952f9df604aa55d169e6000d8" +checksum = "e9b8d03d5bdbca7e5f72b0e0a0f69933ed1f09e24be6c075aa6fe3f802b0cc0c" dependencies = [ "bumpalo", "cranelift-bforest", @@ -2418,33 +2420,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6590feb5a1d6438f974bf6a5ac4dddf69fca14e1f07f3265d880f69e61a94463" +checksum = "a3fd3664e38e51649b17dc30cfdd561273fe2f590dcd013fb75d9eabc6272dfb" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7239038c56fafe77fddc8788fc8533dd6c474dc5bdc5637216404f41ba807330" +checksum = "4b031ec5e605828975952622b5a77d49126f20ffe88d33719a0af66b23a0fc36" [[package]] name = "cranelift-control" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dc9c595341404d381d27a3d950160856b35b402275f0c3990cd1ad683c8053" +checksum = "fada054d017cf2ed8f7ed2336e0517fc1b19e6825be1790de9eb00c94788362b" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44e3ee532fc4776c69bcedf7e62f9632cbb3f35776fa9a525cdade3195baa3f7" +checksum = "177b6f94ae8de6348eb45bf977c79ab9e3c40fc3ac8cb7ed8109560ea39bee7d" dependencies = [ "serde", "serde_derive", @@ -2452,9 +2454,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a612c94d09e653662ec37681dc2d6fd2b9856e6df7147be0afc9aabb0abf19df" +checksum = "ebebd23a69a23e3ddea78e98ff3a2de222e88c8e045d81ef4a72f042e0d79dbd" dependencies = [ "cranelift-codegen", "log", @@ -2464,15 +2466,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85db9830abeb1170b7d29b536ffd55af1d4d26ac8a77570b5d1aca003bf225cc" +checksum = "1571bfc14df8966d12c6121b5325026591a4b4009e22fea0fe3765ab7cd33b96" [[package]] name = "cranelift-native" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301ef0edafeaeda5771a5d2db64ac53e1818ae3111220a185677025fe91db4a1" +checksum = "35a69c37e0c10b46fe5527f2397ac821046efbf5f7ec112c8b84df25712f465b" dependencies = [ "cranelift-codegen", "libc", @@ -2481,9 +2483,9 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380f0abe8264e4570ac615fc31cef32a3b90a77f7eb97b08331f9dd357b1f500" +checksum = "9b3fef8bbceb8cb56d3f1778b0418d75c5cf12ec571a35fc01eb41abb0227a25" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -12728,9 +12730,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi-cap-std-sync" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "154528979a211aa28d969846e883df75705809ed9bcc70aba61460683ea7355b" +checksum = "db014d2ced91f17d1f1a8f2b76d6ea8d731bc1dbc8c2bbaec689d6a242568e5d" dependencies = [ "anyhow", "async-trait", @@ -12746,14 +12748,14 @@ dependencies = [ "system-interface", "tracing", "wasi-common", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasi-common" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d888b611fee7d273dd057dc009d2dd3132736f36710ffd65657ac83628d1e3b" +checksum = "449d17849e3c83a931374442fe2deee4d6bd1ebf469719ef44192e9e82e19c89" dependencies = [ "anyhow", "bitflags 2.4.0", @@ -12766,7 +12768,7 @@ dependencies = [ "tracing", "wasmtime", "wiggle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -12879,9 +12881,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8e539fded2495422ea3c4dfa7beeddba45904eece182cf315294009e1a323bf" +checksum = "910fabce77e660f0e0e41cfd5f69fc8bf020a025f059718846e918db7177f469" dependencies = [ "anyhow", "async-trait", @@ -12913,23 +12915,23 @@ dependencies = [ "wasmtime-runtime", "wasmtime-winch", "wat", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-asm-macros" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "660ba9143e15a2acd921820df221b73aee256bd3ca2d208d73d8adc9587ccbb9" +checksum = "37288142e9b4a61655a3bcbdc7316c2e4bb9e776b10ce3dd758f8186b4469572" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-cache" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3ce373743892002f9391c6741ef0cb0335b55ec899d874f311222b7e36f4594" +checksum = "45cbd74a636f09d2108f9405c79857f061e19323e4abeed22e837cfe7b08a22b" dependencies = [ "anyhow", "base64 0.21.4", @@ -12941,15 +12943,15 @@ dependencies = [ "serde_derive", "sha2", "toml 0.5.11", - "windows-sys 0.48.0", + "windows-sys 0.52.0", "zstd 0.11.2+zstd.1.5.2", ] [[package]] name = "wasmtime-component-macro" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ef32643324e564e1c359e9044daa06cbf90d7e2d6c99a738d17a12959f01a5" +checksum = "ad63de18eb42e586386b6091f787c82707cbd5ac5e9343216dba1976190cd03a" dependencies = [ "anyhow", "proc-macro2", @@ -12962,15 +12964,15 @@ dependencies = [ [[package]] name = "wasmtime-component-util" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c87d06c18d21a4818f354c00a85f4ebc62b2270961cd022968452b0e4dbed9d" +checksum = "7e0a160c0c44369aa4bee6d311a8e4366943bab1651040cc8b0fcec2c9eb8906" [[package]] name = "wasmtime-cranelift" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d648c8b4064a7911093b02237cd5569f71ca171d3a0a486bf80600b19e1cba2" +checksum = "3734cc01b7cd37bc62fdbcd9529ca9547440052d4b3886cfdec3b8081a5d3647" dependencies = [ "anyhow", "cfg-if", @@ -12993,9 +12995,9 @@ dependencies = [ [[package]] name = "wasmtime-cranelift-shared" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290a89027688782da8ff60b12bb95695494b1874e0d0ba2ba387d23dace6d70c" +checksum = "e0eb33cd30c47844aa228d4d0030587e65c1108343f311fe9f7248b5bd9cb65c" dependencies = [ "anyhow", "cranelift-codegen", @@ -13009,9 +13011,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61eb64fb3e0da883e2df4a13a81d6282e072336e6cb6295021d0f7ab2e352754" +checksum = "9a3a056b041fdea604f0972e2fae97958e7748d629a55180228348baefdfc217" dependencies = [ "anyhow", "cranelift-entity", @@ -13032,9 +13034,9 @@ dependencies = [ [[package]] name = "wasmtime-fiber" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecf1d3a838b0956b71ad3f8cb80069a228339775bf02dd35d86a5a68bbe443" +checksum = "43987d0977c07f15c3608c2f255870c127ffd19e35eeedb1ac1dccedf9932a42" dependencies = [ "anyhow", "cc", @@ -13042,14 +13044,14 @@ dependencies = [ "rustix 0.38.28", "wasmtime-asm-macros", "wasmtime-versioned-export-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-jit" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f485336add49267d8859e8f8084d2d4b9a4b1564496b6f30ba5b168d50c10ceb" +checksum = "9b3e48395ac672b386ed588d97a9612aa13a345008f26466f0dfb2a91628aa9f" dependencies = [ "addr2line", "anyhow", @@ -13069,14 +13071,14 @@ dependencies = [ "wasmtime-jit-debug", "wasmtime-jit-icache-coherence", "wasmtime-runtime", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-jit-debug" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e119affec40edb2fab9044f188759a00c2df9c3017278d047012a2de1efb4f" +checksum = "dd21fd0f5ca68681d3d5b636eea00f182d0f9d764144469e9257fd7e3f55ae0e" dependencies = [ "object", "once_cell", @@ -13086,20 +13088,20 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b6d197fcc34ad32ed440e1f9552fd57d1f377d9699d31dee1b5b457322c1f8a" +checksum = "bdc26415bb89e9ccd3bdc498fef63aabf665c4c0dd710c107691deb9694955da" dependencies = [ "cfg-if", "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-runtime" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "794b2bb19b99ef8322ff0dd9fe1ba7e19c41036dfb260b3f99ecce128c42ff92" +checksum = "0abddaf17912aabaf39be0802d5eba9a002e956e902d1ebd438a2fe1c88769a2" dependencies = [ "anyhow", "cc", @@ -13122,14 +13124,14 @@ dependencies = [ "wasmtime-jit-debug", "wasmtime-versioned-export-macros", "wasmtime-wmemcheck", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-types" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d995db8bb56f2cd8d2dc0ed5ffab94ffb435283b0fe6747f80f7aab40b2d06a1" +checksum = "b35a95cdc1433729085beab42c0a5c742b431f25b17c40d7718e46df63d5ffc7" dependencies = [ "cranelift-entity", "serde", @@ -13140,9 +13142,9 @@ dependencies = [ [[package]] name = "wasmtime-versioned-export-macros" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c5565959287c21dd0f4277ae3518dd2ae62679f655ee2dbc4396e19d210db" +checksum = "fad322733fe67e45743784d8b1df452bcb54f581572a4f1a646a4332deecbcc2" dependencies = [ "proc-macro2", "quote", @@ -13151,9 +13153,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccd8370078149d49a3a47e93741553fd79b700421464b6a27ca32718192ab130" +checksum = "902cc299b73655c36679b77efdfce4bb5971992f1a4a8a436dd3809a6848ff0e" dependencies = [ "anyhow", "async-trait", @@ -13181,14 +13183,14 @@ dependencies = [ "wasi-common", "wasmtime", "wiggle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-winch" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6f945ff9bad96e0a69973d74f193c19f627c8adbf250e7cb73ae7564b6cc8a" +checksum = "9e63aeca929f84560eec52c5af43bf5d623b92683b0195d9fb06da8ed860e092" dependencies = [ "anyhow", "cranelift-codegen", @@ -13203,9 +13205,9 @@ dependencies = [ [[package]] name = "wasmtime-wit-bindgen" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f328b2d4a690270324756e886ed5be3a4da4c00be0eea48253f4595ad068062b" +checksum = "41e5675998fdc74495afdd90ad2bd221206a258075b23048af0535a969b07893" dependencies = [ "anyhow", "heck 0.4.1", @@ -13215,9 +13217,9 @@ dependencies = [ [[package]] name = "wasmtime-wmemcheck" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67761d8f8c0b3c13a5d34356274b10a40baba67fe9cfabbfc379a8b414e45de2" +checksum = "b20a19e10d8cb50b45412fb21192982b7ce85c0122dc33bb71f1813e25dc6e52" [[package]] name = "wast" @@ -13289,9 +13291,9 @@ dependencies = [ [[package]] name = "wiggle" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0afb26cd3269289bb314a361ff0a6685e5ce793b62181a9fe3f81ace15051697" +checksum = "737728db69a7657a5f6a7bac445c02d8564d603d62c46c95edf928554e67d072" dependencies = [ "anyhow", "async-trait", @@ -13304,9 +13306,9 @@ dependencies = [ [[package]] name = "wiggle-generate" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef2868fed7584d2b552fa317104858ded80021d23b073b2d682d3c932a027bd" +checksum = "2460c7163b79ffefd9a564eaeab0a5b0e84bb91afdfeeb84d36f304ddbe08982" dependencies = [ "anyhow", "heck 0.4.1", @@ -13319,9 +13321,9 @@ dependencies = [ [[package]] name = "wiggle-macro" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31ae1ec11a17ea481539ee9a5719a278c9790d974060fbf71db4b2c05378780b" +checksum = "fa8d8412375ba8325d61fbae56dead51dabfaec85d620ce36427922fb9cece83" dependencies = [ "proc-macro2", "quote", @@ -13362,9 +13364,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e58c236a6abdd9ab454552b4f29e16cfa837a86897c1503313b2e62e7609ec" +checksum = "9d2b346bad5397b219b4ff0a8fa7230936061ff07c61f05d589d8d81e06fb7b2" dependencies = [ "anyhow", "cranelift-codegen", diff --git a/Cargo.toml b/Cargo.toml index 02ca4fd6b0c5..43818107d0ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -133,8 +133,8 @@ arrow-flight = "50" arrow-select = "50" arrow-ord = "50" arrow-row = "50" -arrow-udf-js = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "7ba1c22" } -arrow-udf-wasm = "0.1" +arrow-udf-js = "0.1" +arrow-udf-wasm = { version = "0.1.1", features = ["build"] } arrow-array-deltalake = { package = "arrow-array", version = "48.0.1" } arrow-buffer-deltalake = { package = "arrow-buffer", version = "48.0.1" } arrow-cast-deltalake = { package = "arrow-cast", version = "48.0.1" } diff --git a/e2e_test/udf/wasm_udf.slt b/e2e_test/udf/wasm_udf.slt index d182e8ff0f99..c2c97a029a1a 100644 --- a/e2e_test/udf/wasm_udf.slt +++ b/e2e_test/udf/wasm_udf.slt @@ -91,3 +91,24 @@ drop function jsonb_access; statement ok drop function series; + +# inlined rust function +statement ok +create function gcd(int, int) returns int language rust as $$ + fn gcd(mut a: i32, mut b: i32) -> i32 { + while b != 0 { + let t = b; + b = a % b; + a = t; + } + a + } +$$; + +query I +select gcd(25, 15); +---- +5 + +statement ok +drop function gcd; diff --git a/src/expr/core/src/expr/expr_udf.rs b/src/expr/core/src/expr/expr_udf.rs index 260e5bb7a998..3532f15677b3 100644 --- a/src/expr/core/src/expr/expr_udf.rs +++ b/src/expr/core/src/expr/expr_udf.rs @@ -194,7 +194,7 @@ impl Build for UserDefinedFunction { let identifier = udf.get_identifier()?; let imp = match udf.language.as_str() { - "wasm" => { + "wasm" | "rust" => { let link = udf.get_link()?; // Use `block_in_place` as an escape hatch to run async code here in sync context. // Calling `block_on` directly will panic. diff --git a/src/frontend/src/handler/create_function.rs b/src/frontend/src/handler/create_function.rs index 10a7fab06267..e1d20be7ff8c 100644 --- a/src/frontend/src/handler/create_function.rs +++ b/src/frontend/src/handler/create_function.rs @@ -24,9 +24,7 @@ use risingwave_expr::expr::get_or_create_wasm_runtime; use risingwave_object_store::object::{build_remote_object_store, ObjectStoreConfig}; use risingwave_pb::catalog::function::{Kind, ScalarFunction, TableFunction}; use risingwave_pb::catalog::Function; -use risingwave_sqlparser::ast::{ - CreateFunctionBody, FunctionDefinition, ObjectName, OperateFunctionArg, -}; +use risingwave_sqlparser::ast::{CreateFunctionBody, ObjectName, OperateFunctionArg}; use risingwave_storage::monitor::ObjectStoreMetrics; use risingwave_udf::ArrowFlightUdfClient; @@ -54,7 +52,7 @@ pub async fn handle_create_function( Some(lang) => { let lang = lang.real_value().to_lowercase(); match &*lang { - "python" | "java" | "wasm" | "javascript" => lang, + "python" | "java" | "wasm" | "rust" | "javascript" => lang, _ => { return Err(ErrorCode::InvalidParameterValue(format!( "language {} is not supported", @@ -134,12 +132,12 @@ pub async fn handle_create_function( ) .into()); }; - let Some(FunctionDefinition::SingleQuotedDef(id)) = params.as_ else { + let Some(as_) = params.as_ else { return Err( ErrorCode::InvalidParameterValue("AS must be specified".to_string()).into(), ); }; - identifier = id; + identifier = as_.into_string(); // check UDF server { @@ -173,16 +171,56 @@ pub async fn handle_create_function( } "javascript" => { identifier = function_name.to_string(); - body = Some(match params.as_ { - Some(FunctionDefinition::SingleQuotedDef(s)) => s, - Some(FunctionDefinition::DoubleDollarDef(s)) => s, - _ => { - return Err(ErrorCode::InvalidParameterValue( - "AS must be specified".to_string(), - ) - .into()) - } - }); + body = Some( + params + .as_ + .ok_or_else(|| ErrorCode::InvalidParameterValue("AS must be specified".into()))? + .into_string(), + ); + } + "rust" => { + identifier = wasm_identifier( + &function_name, + &arg_types, + &return_type, + matches!(kind, Kind::Table(_)), + ); + if params.using.is_some() { + return Err(ErrorCode::InvalidParameterValue( + "USING is not supported for rust function".to_string(), + ) + .into()); + } + let function_body = params + .as_ + .ok_or_else(|| ErrorCode::InvalidParameterValue("AS must be specified".into()))? + .into_string(); + let script = format!("#[arrow_udf::function(\"{identifier}\")]\n{function_body}"); + body = Some(function_body.clone()); + + let wasm_binary = + tokio::task::spawn_blocking(move || arrow_udf_wasm::build("", &script)) + .await? + .context("failed to build rust function")?; + + // below is the same as `wasm` language + let runtime = arrow_udf_wasm::Runtime::new(&wasm_binary)?; + check_wasm_function(&runtime, &identifier)?; + + let system_params = session.env().meta_client().get_system_params().await?; + let object_name = format!("{:?}.wasm", md5::compute(&wasm_binary)); + upload_wasm_binary( + system_params.wasm_storage_url(), + &object_name, + wasm_binary.into(), + ) + .await?; + + link = Some(format!( + "{}/{}", + system_params.wasm_storage_url(), + object_name + )); } "wasm" => { identifier = wasm_identifier( diff --git a/src/sqlparser/src/ast/mod.rs b/src/sqlparser/src/ast/mod.rs index f49d2fb8af15..a5ebb66955f4 100644 --- a/src/sqlparser/src/ast/mod.rs +++ b/src/sqlparser/src/ast/mod.rs @@ -2642,6 +2642,24 @@ impl fmt::Display for FunctionDefinition { } } +impl FunctionDefinition { + /// Returns the function definition as a string slice. + pub fn as_str(&self) -> &str { + match self { + FunctionDefinition::SingleQuotedDef(s) => s, + FunctionDefinition::DoubleDollarDef(s) => s, + } + } + + /// Returns the function definition as a string. + pub fn into_string(self) -> String { + match self { + FunctionDefinition::SingleQuotedDef(s) => s, + FunctionDefinition::DoubleDollarDef(s) => s, + } + } +} + /// Return types of a function. #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] From 71e3d5cae2f63f7a1b7aecfa664aeb8f221fb61b Mon Sep 17 00:00:00 2001 From: Runji Wang Date: Sun, 4 Feb 2024 13:42:02 +0800 Subject: [PATCH 3/4] auto install wasm32-wasi target Signed-off-by: Runji Wang --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- src/frontend/src/handler/create_function.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 89b8d35e5887..473d2826e113 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -642,9 +642,9 @@ dependencies = [ [[package]] name = "arrow-udf-wasm" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70b965912f2c5514ce243a384e9b4e61544932acd28f126ad1adb13da04fa247" +checksum = "3ccaec984501c42dab31a0363e4c8b11e1dc2d870d7e3af6868b8ffbd704ce0a" dependencies = [ "anyhow", "arrow-array 50.0.0", diff --git a/Cargo.toml b/Cargo.toml index 43818107d0ff..c8c10fac1d6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,7 +134,7 @@ arrow-select = "50" arrow-ord = "50" arrow-row = "50" arrow-udf-js = "0.1" -arrow-udf-wasm = { version = "0.1.1", features = ["build"] } +arrow-udf-wasm = { version = "0.1.2", features = ["build"] } arrow-array-deltalake = { package = "arrow-array", version = "48.0.1" } arrow-buffer-deltalake = { package = "arrow-buffer", version = "48.0.1" } arrow-cast-deltalake = { package = "arrow-cast", version = "48.0.1" } diff --git a/src/frontend/src/handler/create_function.rs b/src/frontend/src/handler/create_function.rs index e1d20be7ff8c..0a8329e54be0 100644 --- a/src/frontend/src/handler/create_function.rs +++ b/src/frontend/src/handler/create_function.rs @@ -199,7 +199,7 @@ pub async fn handle_create_function( body = Some(function_body.clone()); let wasm_binary = - tokio::task::spawn_blocking(move || arrow_udf_wasm::build("", &script)) + tokio::task::spawn_blocking(move || arrow_udf_wasm::build::build("", &script)) .await? .context("failed to build rust function")?; From 2414228beea266391895a51c55faa2db2045fc81 Mon Sep 17 00:00:00 2001 From: Runji Wang Date: Sun, 4 Feb 2024 13:45:42 +0800 Subject: [PATCH 4/4] add wasm32-wasi target to dockerfile Signed-off-by: Runji Wang --- ci/Dockerfile | 1 + ci/build-ci-image.sh | 2 +- ci/docker-compose.yml | 10 +++++----- ci/scripts/build-other.sh | 1 - docker/Dockerfile | 3 ++- docker/Dockerfile.hdfs | 3 ++- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ci/Dockerfile b/ci/Dockerfile index b45222bb20e6..3d14a61dfcfb 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -41,6 +41,7 @@ RUN curl -sSL https://install.python-poetry.org | python3 - # add required rustup components RUN rustup component add rustfmt llvm-tools-preview clippy +RUN rustup target add wasm32-wasi ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh index c684724bd153..cfddc603ace7 100755 --- a/ci/build-ci-image.sh +++ b/ci/build-ci-image.sh @@ -10,7 +10,7 @@ cat ../rust-toolchain # shellcheck disable=SC2155 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml -export BUILD_ENV_VERSION=v20240124_1 +export BUILD_ENV_VERSION=v20240204 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}" diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index 9047fda1737e..0a81a772b1ea 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -71,7 +71,7 @@ services: retries: 5 source-test-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204 depends_on: - mysql - db @@ -81,7 +81,7 @@ services: - ..:/risingwave sink-test-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204 depends_on: - mysql - db @@ -93,12 +93,12 @@ services: - ..:/risingwave rw-build-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204 volumes: - ..:/risingwave ci-flamegraph-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204 # NOTE(kwannoel): This is used in order to permit # syscalls for `nperf` (perf_event_open), # so it can do CPU profiling. @@ -109,7 +109,7 @@ services: - ..:/risingwave regress-test-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204 depends_on: db: condition: service_healthy diff --git a/ci/scripts/build-other.sh b/ci/scripts/build-other.sh index e303a5f5160c..2311e5164fe7 100755 --- a/ci/scripts/build-other.sh +++ b/ci/scripts/build-other.sh @@ -8,7 +8,6 @@ source ci/scripts/common.sh echo "--- Build Rust UDF" cd e2e_test/udf/wasm -rustup target add wasm32-wasi cargo build --release cd ../../.. diff --git a/docker/Dockerfile b/docker/Dockerfile index 58e93de54bce..ed24e05ede2d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,7 +33,8 @@ COPY rust-toolchain rust-toolchain RUN rustup self update \ && rustup set profile minimal \ && rustup show \ - && rustup component add rustfmt + && rustup component add rustfmt \ + && rustup target add wasm32-wasi RUN cargo install flamegraph # TODO: cargo-chef doesn't work well now, because we update Cargo.lock very often. diff --git a/docker/Dockerfile.hdfs b/docker/Dockerfile.hdfs index e8dd1988bd6f..98cb5a566d92 100644 --- a/docker/Dockerfile.hdfs +++ b/docker/Dockerfile.hdfs @@ -34,7 +34,8 @@ RUN unzip dashboard-artifact.zip && mv risingwave-dashboard-artifact /risingwave RUN rustup self update \ && rustup set profile minimal \ && rustup show \ - && rustup component add rustfmt + && rustup component add rustfmt \ + && rustup target add wasm32-wasi RUN cargo fetch