From 7eb9581f56b04bd8a715491d182eccb892190002 Mon Sep 17 00:00:00 2001 From: xxchan Date: Thu, 26 Oct 2023 14:33:42 +0800 Subject: [PATCH] Squashed commit of the following: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit b52a0046fa0b4d2e5ce9df6f0dbb8ff9b5877f84 Author: xxchan Date: Thu Oct 26 14:25:18 2023 +0800 update arrow-ipc commit e94feeb86ea0eb9f6ddfd24eaa8c14d62850496a Author: xxchan Date: Thu Oct 26 06:21:34 2023 +0000 Fix "cargo-hakari" commit 08a56017eee2bd7426a22b402df7f82387a2b555 Merge: 56e6fc4899 942e99dc74 Author: xxchan Date: Thu Oct 26 14:19:34 2023 +0800 Merge branch 'main' into xxchan/wasm-udf commit 942e99dc74228659ff106589634ddc6196009191 Author: Yufan Song <33971064+yufansong@users.noreply.github.com> Date: Wed Oct 25 22:10:31 2023 -0700 fix(nats-connector): change stream into optional string, add replace stream name logic (#13024) commit 90fb4a3478ec90c1497732af3e75afea6d22ae17 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu Oct 26 04:25:11 2023 +0000 chore(deps): Bump comfy-table from 7.0.1 to 7.1.0 (#13049) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> commit b724be78a46b15d397ac509d79aabf6365d687cf Author: jinser Date: Thu Oct 26 00:26:15 2023 +0800 feat: add `comment on` clause support (#12849) Co-authored-by: Richard Chien Co-authored-by: August commit 7f791d65f10ff304e10ba3a6d58bb6bb3a33bad9 Author: August Date: Wed Oct 25 20:29:16 2023 +0800 feat: move model_v2 and model_migration into a separate crates (#13058) commit 7f82929b38131f43b1290e51b0722c972fa3b6c2 Author: Noel Kwan <47273164+kwannoel@users.noreply.github.com> Date: Wed Oct 25 16:57:45 2023 +0800 fix(meta): persist internal tables of `CREATE TABLE` (#13039) commit 09a67abbefb5c7a0c53596e0a5cf557a2fae3664 Author: Noel Kwan <47273164+kwannoel@users.noreply.github.com> Date: Wed Oct 25 16:49:08 2023 +0800 fix: `WAIT` should return error if timeout (#13045) commit e48547dac8c7083a13d9537f9f16e4cc855bc4e7 Author: Runji Wang Date: Wed Oct 25 16:41:16 2023 +0800 refactor(type): switch jsonb to flat representation (#12952) Signed-off-by: Runji Wang commit 56e6fc4899423db6ab0cf9ee39c233a6a54e3956 Author: xxchan Date: Wed Oct 25 15:33:36 2023 +0800 fix merge issue commit c64436155d235b01582bb6b0c1f1e7bf7aeb802b Merge: fcd69926a7 2d428b153e Author: xxchan Date: Wed Oct 25 15:23:44 2023 +0800 Merge remote-tracking branch 'origin/main' into xxchan/wasm-udf commit fcd69926a76217c219b56b715ba6e7cc98de6aed Author: xxchan Date: Wed Oct 25 14:28:53 2023 +0800 fix s3 stuck commit 21e974075ff784b1fac82642d92e6855ef1e41d5 Author: xxchan Date: Wed Oct 25 12:47:24 2023 +0800 Revert "fix s3 stuck (why?)" This reverts commit f19a6b49a5a106151a6adb25ee1b8916477650f1. commit f19a6b49a5a106151a6adb25ee1b8916477650f1 Author: xxchan Date: Wed Sep 13 14:32:28 2023 +0800 fix s3 stuck (why?) commit 019f30983d93c592e08ad6317302ffb0ff0c9895 Author: xxchan Date: Tue Sep 12 15:29:52 2023 +0800 ON_ERROR_STOP=1 commit 6e4ee3cd94afbf603215d21fb6fe2a12ac36909f Author: xxchan Date: Tue Sep 12 15:09:58 2023 +0800 generate-config commit b63a1c32546ad0e6145500101456cf84c284120b Merge: 2b0cc96995 53611bfbbc Author: xxchan Date: Tue Sep 12 14:53:10 2023 +0800 Merge remote-tracking branch 'origin/main' into xxchan/wasm-udf commit 2b0cc9699545c6afd60fe97e269423ed7798b807 Author: xxchan Date: Sat Sep 9 23:49:43 2023 +0800 fix conflicts commit 6b13fe3e64677e60cdd9be3cc17b31d9638b0b1b Author: xxchan Date: Sat Sep 9 23:35:50 2023 +0800 update system param default commit a27394357b8475f4623d08b6da505b9d4bb3343f Merge: cc34bfe12d f649aa6e4b Author: xxchan Date: Sat Sep 9 23:33:38 2023 +0800 Merge remote-tracking branch 'origin/main' into xxchan/wasm-udf commit cc34bfe12dac9e7537992529198f1684630fd4b8 Author: xxchan Date: Tue Aug 1 17:47:42 2023 +0200 use count_char as the example commit f913f63e21d06d624ace0b576deb5c2d1b31d31d Merge: 53bf8e0e03 2637dbde5b Author: xxchan Date: Tue Aug 1 17:22:13 2023 +0200 Merge branch 'main' into xxchan/wasm-udf commit 53bf8e0e0373b41c5786023b868301460636c77c Author: xxchan Date: Mon Jul 31 14:20:07 2023 +0200 minor update commit 70cee4229408b6a494d612f7b3f92e31365078a4 Author: xxchan Date: Mon Jul 17 14:53:29 2023 +0200 fix arrow_schema into -> try_into commit a7d172d7786879b28ebeb007c512a1c429843a4d Author: xxchan Date: Fri Jul 14 16:31:20 2023 +0200 buf format commit 43a32902161d4115f07fe0dfbcf2f0a335718ad1 Author: xxchan Date: Thu Jul 13 23:04:16 2023 +0200 add tinygo example & turn on wasi support commit 61a49986f310e3a995d0368d817e77a26c683ac8 Author: xxchan Date: Wed Jul 12 11:40:56 2023 +0200 cleanup commit 165d4d9d8b1545aaca631cbb21c6f3f14ef975f6 Author: xxchan Date: Wed Jul 12 11:02:44 2023 +0200 use object store to store wasm commit 88979e4a53efb69e7a5fb7afb70ace014cdc9465 Author: xxchan Date: Tue Jul 11 15:32:52 2023 +0200 add wasm_storage_url system param commit a8973204254e44d49d515bbf4f6ec2cf6ff03e60 Author: xxchan Date: Thu Jul 6 20:04:45 2023 +0200 Load compiled wasm module in expr 🚀🚀🚀 commit 63b35238aa7b993d0f75621137d567b283a5ae61 Author: xxchan Date: Sun Jul 2 19:27:22 2023 +0200 it works (although very slow) --- .licenserc.yaml | 2 +- Cargo.lock | 1150 ++++++++++++++++- Cargo.toml | 6 +- .../batch/catalog/pg_description.slt.part | 81 +- e2e_test/ddl/show.slt | 56 +- e2e_test/extended_mode/basic.slt | 15 +- proto/catalog.proto | 17 + proto/ddl_service.proto | 10 + proto/expr.proto | 25 + proto/meta.proto | 1 + proto/plan_common.proto | 3 + src/common/Cargo.toml | 1 + src/common/src/array/jsonb_array.rs | 128 +- src/common/src/array/proto_reader.rs | 4 +- src/common/src/array/value_reader.rs | 15 +- src/common/src/catalog/column.rs | 8 + src/common/src/catalog/mod.rs | 2 + src/common/src/catalog/test_utils.rs | 1 + src/common/src/config.rs | 4 + src/common/src/system_param/mod.rs | 1 + src/common/src/system_param/reader.rs | 4 + src/common/src/test_utils/rand_array.rs | 2 +- src/common/src/types/jsonb.rs | 261 ++-- src/common/src/types/mod.rs | 4 +- src/compute/tests/integration_tests.rs | 1 + src/config/example.toml | 1 + src/connector/src/common.rs | 13 +- src/connector/src/parser/avro/util.rs | 1 + src/connector/src/parser/protobuf/parser.rs | 1 + src/connector/src/source/manager.rs | 1 + src/connector/src/source/nats/mod.rs | 3 + .../src/source/nats/source/reader.rs | 6 +- src/expr/core/Cargo.toml | 3 + src/expr/core/src/error.rs | 6 +- src/expr/core/src/expr/expr_udf.rs | 81 +- .../core/src/table_function/user_defined.rs | 2 +- src/expr/impl/Cargo.toml | 1 + src/expr/impl/benches/expr.rs | 31 +- src/expr/impl/src/aggregate/jsonb_agg.rs | 150 ++- src/expr/impl/src/scalar/jsonb_concat.rs | 28 +- src/expr/macro/src/gen.rs | 86 +- src/expr/macro/src/lib.rs | 2 + src/expr/macro/src/parse.rs | 30 +- src/frontend/Cargo.toml | 1 + src/frontend/src/binder/expr/mod.rs | 2 + src/frontend/src/catalog/catalog_service.rs | 10 +- src/frontend/src/catalog/function_catalog.rs | 4 + .../src/catalog/system_catalog/mod.rs | 5 + .../pg_catalog/pg_description.rs | 39 +- .../catalog/system_catalog/rw_catalog/mod.rs | 2 + .../rw_catalog/rw_description.rs | 84 ++ src/frontend/src/catalog/table_catalog.rs | 8 + .../src/expr/user_defined_function.rs | 3 + src/frontend/src/handler/comment.rs | 87 ++ src/frontend/src/handler/create_function.rs | 132 +- src/frontend/src/handler/create_source.rs | 4 + src/frontend/src/handler/create_table.rs | 1 + src/frontend/src/handler/describe.rs | 77 +- src/frontend/src/handler/mod.rs | 6 + src/frontend/src/handler/util.rs | 1 + .../optimizer/plan_node/stream_materialize.rs | 1 + src/frontend/src/optimizer/plan_node/utils.rs | 1 + src/frontend/src/session.rs | 5 + src/frontend/src/test_utils.rs | 6 +- src/meta/Cargo.toml | 3 +- src/meta/model_v2/Cargo.toml | 26 + src/meta/model_v2/migration/Cargo.toml | 22 + .../{src => }/model_v2/migration/README.md | 0 .../{src => }/model_v2/migration/src/lib.rs | 0 .../migration/src/m20230908_072257_init.rs | 0 .../migration/src/m20231008_020431_hummock.rs | 0 .../{src => }/model_v2/migration/src/main.rs | 2 +- .../{src/model_v2 => model_v2/src}/README.md | 2 +- .../{src/model_v2 => model_v2/src}/actor.rs | 2 +- .../{src/model_v2 => model_v2/src}/cluster.rs | 0 .../src}/compaction_config.rs | 0 .../src}/compaction_status.rs | 0 .../src}/compaction_task.rs | 0 .../model_v2 => model_v2/src}/connection.rs | 2 +- .../model_v2 => model_v2/src}/database.rs | 13 +- .../model_v2 => model_v2/src}/fragment.rs | 2 +- .../model_v2 => model_v2/src}/function.rs | 2 +- .../src}/hummock_pinned_snapshot.rs | 0 .../src}/hummock_pinned_version.rs | 0 .../src}/hummock_version_delta.rs | 0 .../src}/hummock_version_stats.rs | 0 .../{src/model_v2 => model_v2/src}/index.rs | 2 +- .../model_v2/mod.rs => model_v2/src/lib.rs} | 2 - .../{src/model_v2 => model_v2/src}/object.rs | 2 +- .../src}/object_dependency.rs | 2 +- .../{src/model_v2 => model_v2/src}/prelude.rs | 0 .../{src/model_v2 => model_v2/src}/schema.rs | 13 +- .../{src/model_v2 => model_v2/src}/sink.rs | 2 +- .../{src/model_v2 => model_v2/src}/source.rs | 2 +- .../src}/system_parameter.rs | 0 .../{src/model_v2 => model_v2/src}/table.rs | 2 +- .../{src/model_v2 => model_v2/src}/user.rs | 2 +- .../src}/user_privilege.rs | 2 +- .../{src/model_v2 => model_v2/src}/view.rs | 2 +- src/meta/model_v2/src/worker.rs | 128 ++ .../src}/worker_property.rs | 2 +- src/meta/node/Cargo.toml | 2 +- src/meta/node/src/server.rs | 2 +- src/meta/service/Cargo.toml | 1 + src/meta/service/src/ddl_service.rs | 28 +- src/meta/service/src/telemetry_service.rs | 2 +- src/meta/src/backup_restore/restore.rs | 1 + src/meta/src/controller/catalog.rs | 14 +- src/meta/src/controller/cluster.rs | 64 +- src/meta/src/controller/mod.rs | 27 +- src/meta/src/controller/system_param.rs | 4 +- src/meta/src/controller/utils.rs | 14 +- src/meta/src/lib.rs | 1 - src/meta/src/manager/catalog/mod.rs | 78 +- src/meta/src/manager/env.rs | 2 +- src/meta/src/manager/mod.rs | 3 +- src/meta/src/model_v2/ext/hummock.rs | 61 - src/meta/src/model_v2/ext/mod.rs | 16 - src/meta/src/model_v2/migration/Cargo.toml | 17 - src/meta/src/model_v2/trx.rs | 276 ---- src/meta/src/model_v2/worker.rs | 67 - src/meta/src/rpc/ddl_controller.rs | 14 +- src/object_store/src/object/mod.rs | 7 +- src/prost/build.rs | 3 + src/prost/src/lib.rs | 19 + src/rpc_client/src/meta_client.rs | 12 +- src/source/src/source_desc.rs | 1 + src/sqlparser/src/ast/mod.rs | 4 + src/sqlparser/src/keywords.rs | 1 + src/sqlparser/src/parser.rs | 18 +- src/storage/hummock_trace/Cargo.toml | 2 +- src/storage/src/filter_key_extractor.rs | 1 + .../src/delete_range_runner.rs | 1 + src/udf/Cargo.toml | 8 + src/udf/src/lib.rs | 4 + src/udf/src/wasm.rs | 307 +++++ src/udf/wit/udf.wit | 49 + src/udf/wit_example/.gitignore | 4 + src/udf/wit_example/README.md | 54 + src/udf/wit_example/build.sh | 95 ++ src/udf/wit_example/create.sh | 27 + src/udf/wit_example/rust/Cargo.toml | 25 + src/udf/wit_example/rust/src/lib.rs | 118 ++ src/udf/wit_example/tinygo/go.mod | 19 + src/udf/wit_example/tinygo/go.sum | 30 + src/udf/wit_example/tinygo/my_udf.go | 89 ++ src/utils/pgwire/src/pg_protocol.rs | 2 +- src/utils/pgwire/src/pg_response.rs | 1 + src/utils/runtime/src/logger.rs | 5 +- src/workspace-hack/Cargo.toml | 16 +- 150 files changed, 3503 insertions(+), 1079 deletions(-) create mode 100644 src/frontend/src/catalog/system_catalog/rw_catalog/rw_description.rs create mode 100644 src/frontend/src/handler/comment.rs create mode 100644 src/meta/model_v2/Cargo.toml create mode 100644 src/meta/model_v2/migration/Cargo.toml rename src/meta/{src => }/model_v2/migration/README.md (100%) rename src/meta/{src => }/model_v2/migration/src/lib.rs (100%) rename src/meta/{src => }/model_v2/migration/src/m20230908_072257_init.rs (100%) rename src/meta/{src => }/model_v2/migration/src/m20231008_020431_hummock.rs (100%) rename src/meta/{src => }/model_v2/migration/src/main.rs (52%) rename src/meta/{src/model_v2 => model_v2/src}/README.md (93%) rename src/meta/{src/model_v2 => model_v2/src}/actor.rs (97%) rename src/meta/{src/model_v2 => model_v2/src}/cluster.rs (100%) rename src/meta/{src/model_v2 => model_v2/src}/compaction_config.rs (100%) rename src/meta/{src/model_v2 => model_v2/src}/compaction_status.rs (100%) rename src/meta/{src/model_v2 => model_v2/src}/compaction_task.rs (100%) rename src/meta/{src/model_v2 => model_v2/src}/connection.rs (97%) rename src/meta/{src/model_v2 => model_v2/src}/database.rs (81%) rename src/meta/{src/model_v2 => model_v2/src}/fragment.rs (98%) rename src/meta/{src/model_v2 => model_v2/src}/function.rs (97%) rename src/meta/{src/model_v2 => model_v2/src}/hummock_pinned_snapshot.rs (100%) rename src/meta/{src/model_v2 => model_v2/src}/hummock_pinned_version.rs (100%) rename src/meta/{src/model_v2 => model_v2/src}/hummock_version_delta.rs (100%) rename src/meta/{src/model_v2 => model_v2/src}/hummock_version_stats.rs (100%) rename src/meta/{src/model_v2 => model_v2/src}/index.rs (96%) rename src/meta/{src/model_v2/mod.rs => model_v2/src/lib.rs} (99%) rename src/meta/{src/model_v2 => model_v2/src}/object.rs (98%) rename src/meta/{src/model_v2 => model_v2/src}/object_dependency.rs (97%) rename src/meta/{src/model_v2 => model_v2/src}/prelude.rs (100%) rename src/meta/{src/model_v2 => model_v2/src}/schema.rs (81%) rename src/meta/{src/model_v2 => model_v2/src}/sink.rs (99%) rename src/meta/{src/model_v2 => model_v2/src}/source.rs (99%) rename src/meta/{src/model_v2 => model_v2/src}/system_parameter.rs (100%) rename src/meta/{src/model_v2 => model_v2/src}/table.rs (99%) rename src/meta/{src/model_v2 => model_v2/src}/user.rs (97%) rename src/meta/{src/model_v2 => model_v2/src}/user_privilege.rs (97%) rename src/meta/{src/model_v2 => model_v2/src}/view.rs (97%) create mode 100644 src/meta/model_v2/src/worker.rs rename src/meta/{src/model_v2 => model_v2/src}/worker_property.rs (97%) delete mode 100644 src/meta/src/model_v2/ext/hummock.rs delete mode 100644 src/meta/src/model_v2/ext/mod.rs delete mode 100644 src/meta/src/model_v2/migration/Cargo.toml delete mode 100644 src/meta/src/model_v2/trx.rs delete mode 100644 src/meta/src/model_v2/worker.rs create mode 100644 src/udf/src/wasm.rs create mode 100644 src/udf/wit/udf.wit create mode 100644 src/udf/wit_example/.gitignore create mode 100644 src/udf/wit_example/README.md create mode 100755 src/udf/wit_example/build.sh create mode 100755 src/udf/wit_example/create.sh create mode 100644 src/udf/wit_example/rust/Cargo.toml create mode 100644 src/udf/wit_example/rust/src/lib.rs create mode 100644 src/udf/wit_example/tinygo/go.mod create mode 100644 src/udf/wit_example/tinygo/go.sum create mode 100644 src/udf/wit_example/tinygo/my_udf.go diff --git a/.licenserc.yaml b/.licenserc.yaml index c1745a4d1ad74..7b49108b6b2f3 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -17,6 +17,6 @@ header: - "**/*.d.ts" - "src/sqlparser/**/*.rs" - "java/connector-node/risingwave-source-cdc/src/main/java/com/risingwave/connector/cdc/debezium/internal/*.java" - - "src/meta/src/model_v2/migration/**/*.rs" + - "src/meta/model_v2/migration/**/*.rs" comment: on-failure diff --git a/Cargo.lock b/Cargo.lock index 3c9dbe9602508..93dd074beda3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,13 +2,22 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +dependencies = [ + "gimli 0.27.3", +] + [[package]] name = "addr2line" version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ - "gimli", + "gimli 0.28.0", ] [[package]] @@ -83,6 +92,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -229,6 +244,12 @@ dependencies = [ "syn 2.0.37", ] +[[package]] +name = "arbitrary" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2e1373abdaa212b704512ec2bd8b26bd0b7d5c3f70117411a5d9a451383c859" + [[package]] name = "arc-swap" version = "1.6.0" @@ -1136,12 +1157,12 @@ version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ - "addr2line", + "addr2line 0.21.0", "cc", "cfg-if", "libc", "miniz_oxide", - "object", + "object 0.32.1", "rustc-demangle", ] @@ -1534,6 +1555,69 @@ dependencies = [ "serde", ] +[[package]] +name = "cap-fs-ext" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58bc48200a1a0fa6fba138b1802ad7def18ec1cdd92f7b2a04e21f1bd887f7b9" +dependencies = [ + "cap-primitives", + "cap-std", + "io-lifetimes 1.0.11", + "windows-sys 0.48.0", +] + +[[package]] +name = "cap-primitives" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4b6df5b295dca8d56f35560be8c391d59f0420f72e546997154e24e765e6451" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes 1.0.11", + "ipnet", + "maybe-owned", + "rustix 0.37.23", + "windows-sys 0.48.0", + "winx 0.35.1", +] + +[[package]] +name = "cap-rand" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d25555efacb0b5244cf1d35833d55d21abc916fff0eaad254b8e2453ea9b8ab" +dependencies = [ + "ambient-authority", + "rand", +] + +[[package]] +name = "cap-std" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3373a62accd150b4fcba056d4c5f3b552127f0ec86d3c8c102d60b978174a012" +dependencies = [ + "cap-primitives", + "io-extras", + "io-lifetimes 1.0.11", + "rustix 0.37.23", +] + +[[package]] +name = "cap-time-ext" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e95002993b7baee6b66c8950470e59e5226a23b3af39fc59c47fe416dd39821a" +dependencies = [ + "cap-primitives", + "once_cell", + "rustix 0.37.23", + "winx 0.35.1", +] + [[package]] name = "cargo-platform" version = "0.1.3" @@ -1825,13 +1909,13 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.0.1" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b" +checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" dependencies = [ - "crossterm 0.26.1", - "strum 0.24.1", - "strum_macros 0.24.3", + "crossterm 0.27.0", + "strum 0.25.0", + "strum_macros 0.25.2", "unicode-width", ] @@ -1953,6 +2037,15 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "cpp_demangle" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaa953eaad386a53111e47172c2fedba671e5684c8dd601a5f474f4f118710f" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpp_demangle" version = "0.4.3" @@ -1971,6 +2064,114 @@ dependencies = [ "libc", ] +[[package]] +name = "cranelift-bforest" +version = "0.97.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aae6f552c4c0ccfb30b9559b77bc985a387d998e1736cbbe6b14c903f3656cf" +dependencies = [ + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen" +version = "0.97.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95551de96900cefae691ce895ff2abc691ae3a0b97911a76b45faf99e432937b" +dependencies = [ + "bumpalo", + "cranelift-bforest", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-control", + "cranelift-entity", + "cranelift-isle", + "gimli 0.27.3", + "hashbrown 0.13.2", + "log", + "regalloc2", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.97.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36a3ad7b2bb03de3383f258b00ca29d80234bebd5130cb6ef3bae37ada5baab0" +dependencies = [ + "cranelift-codegen-shared", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.97.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915918fee4142c85fb04bafe0bcd697e2fd6c15a260301ea6f8d2ea332a30e86" + +[[package]] +name = "cranelift-control" +version = "0.97.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e447d548cd7f4fcb87fbd10edbd66a4f77966d17785ed50a08c8f3835483c8" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "cranelift-entity" +version = "0.97.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d8ab3352a1e5966968d7ab424bd3de8e6b58314760745c3817c2eec3fa2f918" +dependencies = [ + "serde", +] + +[[package]] +name = "cranelift-frontend" +version = "0.97.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bffa38431f7554aa1594f122263b87c9e04abc55c9f42b81d37342ac44f79f0" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.97.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cef66a71c77938148b72bf006892c89d6be9274a08f7e669ff15a56145d701" + +[[package]] +name = "cranelift-native" +version = "0.97.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f33c7e5eb446e162d2d10b17fe68e1f091020cc2e4e38b5501c21099600b0a1b" +dependencies = [ + "cranelift-codegen", + "libc", + "target-lexicon", +] + +[[package]] +name = "cranelift-wasm" +version = "0.97.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "632f7b64fa6a8c5b980eb6a17ef22089e15cb9f779f1ed3bd3072beab0686c09" +dependencies = [ + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "itertools 0.10.5", + "log", + "smallvec", + "wasmparser 0.107.0", + "wasmtime-types", +] + [[package]] name = "crc" version = "3.0.1" @@ -2099,7 +2300,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", + "memoffset 0.9.0", "scopeguard", ] @@ -2140,17 +2341,14 @@ dependencies = [ [[package]] name = "crossterm" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" +checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "crossterm_winapi", "libc", - "mio", "parking_lot 0.12.1", - "signal-hook", - "signal-hook-mio", "winapi", ] @@ -2546,6 +2744,47 @@ dependencies = [ "subtle", ] +[[package]] +name = "directories-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dissimilar" version = "1.0.7" @@ -2889,6 +3128,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +[[package]] +name = "fd-lock" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b0377f1edc77dbd1118507bc7a66e4ab64d2b90c66f90726dc801e73a8c68f9" +dependencies = [ + "cfg-if", + "rustix 0.38.11", + "windows-sys 0.48.0", +] + [[package]] name = "fiat-crypto" version = "0.2.1" @@ -2904,6 +3154,16 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "file-per-thread-logger" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a3cc21c33af89af0930c8cae4ade5e6fdc17b5d2c97b3d2e2edb67a1cf683f3" +dependencies = [ + "env_logger", + "log", +] + [[package]] name = "findshlibs" version = "0.10.2" @@ -3051,7 +3311,7 @@ dependencies = [ "foyer-common", "foyer-workspace-hack", "itertools 0.11.0", - "memoffset", + "memoffset 0.9.0", "parking_lot 0.12.1", "paste", "tracing", @@ -3076,7 +3336,7 @@ dependencies = [ "itertools 0.11.0", "libc", "madsim-tokio", - "memoffset", + "memoffset 0.9.0", "nix 0.27.1", "parking_lot 0.12.1", "paste", @@ -3180,6 +3440,17 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" +[[package]] +name = "fs-set-times" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d167b646a876ba8fda6b50ac645cfd96242553cbaf0ca4fccaa39afcbf0801f" +dependencies = [ + "io-lifetimes 1.0.11", + "rustix 0.38.11", + "windows-sys 0.48.0", +] + [[package]] name = "fs2" version = "0.4.3" @@ -3363,6 +3634,19 @@ dependencies = [ "byteorder", ] +[[package]] +name = "fxprof-processed-profile" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d12c0aed7f1e24276a241aadc4cb8ea9f83000f34bc062b7cc2d51e3b0fabd" +dependencies = [ + "bitflags 2.4.0", + "debugid", + "fxhash", + "serde", + "serde_json", +] + [[package]] name = "generator" version = "0.7.5" @@ -3398,6 +3682,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +dependencies = [ + "fallible-iterator", + "indexmap 1.9.3", + "stable_deref_trait", +] + [[package]] name = "gimli" version = "0.28.0" @@ -3896,6 +4191,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + [[package]] name = "ident_case" version = "1.0.1" @@ -4013,6 +4314,16 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +[[package]] +name = "io-extras" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fde93d48f0d9277f977a333eca8313695ddd5301dc96f7e02aeddcb0dd99096f" +dependencies = [ + "io-lifetimes 1.0.11", + "windows-sys 0.48.0", +] + [[package]] name = "io-lifetimes" version = "1.0.11" @@ -4024,6 +4335,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "io-lifetimes" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffb4def18c48926ccac55c1223e02865ce1a821751a95920448662696e7472c" + [[package]] name = "ipnet" version = "2.8.0" @@ -4065,6 +4382,26 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "ittapi" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a5c0b993601cad796222ea076565c5d9f337d35592f8622c753724f06d7271" +dependencies = [ + "anyhow", + "ittapi-sys", + "log", +] + +[[package]] +name = "ittapi-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7b5e473765060536a660eed127f758cf1a810c73e49063264959c60d1727d9" +dependencies = [ + "cc", +] + [[package]] name = "java-locator" version = "0.1.5" @@ -4117,6 +4454,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonbb" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44376417b2ff0cd879b5c84976fa9e0855c316321b4e0502e33e52963bf84f74" +dependencies = [ + "bytes", + "serde", + "serde_json", + "smallvec", +] + [[package]] name = "jsonschema-transpiler" version = "1.10.0" @@ -4179,6 +4528,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + [[package]] name = "lexical" version = "6.1.1" @@ -4460,6 +4815,15 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "mach2" version = "0.4.1" @@ -4646,6 +5010,12 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + [[package]] name = "md-5" version = "0.10.5" @@ -4680,12 +5050,30 @@ dependencies = [ ] [[package]] -name = "memmap2" -version = "0.5.10" +name = "memfd" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" dependencies = [ - "libc", + "rustix 0.38.11", +] + +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", ] [[package]] @@ -4767,15 +5155,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "model_migration" -version = "0.1.0" -dependencies = [ - "async-std", - "sea-orm-migration", - "uuid", -] - [[package]] name = "moka" version = "0.12.0" @@ -5225,6 +5604,18 @@ dependencies = [ "url", ] +[[package]] +name = "object" +version = "0.30.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385" +dependencies = [ + "crc32fast", + "hashbrown 0.13.2", + "indexmap 1.9.3", + "memchr", +] + [[package]] name = "object" version = "0.32.1" @@ -6430,6 +6821,15 @@ dependencies = [ "autotools", ] +[[package]] +name = "psm" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +dependencies = [ + "cc", +] + [[package]] name = "ptr_meta" version = "0.1.4" @@ -6450,6 +6850,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "pulldown-cmark" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffade02495f22453cd593159ea2f59827aae7f53fa8323f756799b670881dcf8" +dependencies = [ + "bitflags 1.3.2", + "memchr", + "unicase", +] + [[package]] name = "pulldown-cmark" version = "0.9.3" @@ -6684,6 +7095,30 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall 0.2.16", + "thiserror", +] + +[[package]] +name = "regalloc2" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad156d539c879b7a24a363a2016d77961786e71f48f2e2fc8302a92abd2429a6" +dependencies = [ + "hashbrown 0.13.2", + "log", + "rustc-hash", + "slice-group-by", + "smallvec", +] + [[package]] name = "regex" version = "1.10.0" @@ -7073,6 +7508,7 @@ dependencies = [ "hytra", "itertools 0.11.0", "itoa", + "jsonbb", "libc", "lru 0.7.6", "mach2", @@ -7450,6 +7886,7 @@ dependencies = [ "futures-util", "hex", "itertools 0.11.0", + "jsonbb", "madsim-tokio", "md5", "num-traits", @@ -7489,6 +7926,7 @@ dependencies = [ "async-recursion", "async-trait", "auto_enums", + "base64 0.21.4", "bk-tree", "bytes", "clap", @@ -7600,7 +8038,7 @@ dependencies = [ "bytes", "futures", "futures-async-stream", - "itertools 0.10.5", + "itertools 0.11.0", "madsim-tokio", "mockall", "parking_lot 0.12.1", @@ -7681,7 +8119,6 @@ dependencies = [ "maplit", "memcomparable", "mime_guess", - "model_migration", "num-integer", "num-traits", "parking_lot 0.12.1", @@ -7695,6 +8132,8 @@ dependencies = [ "risingwave_common_heap_profiling", "risingwave_connector", "risingwave_hummock_sdk", + "risingwave_meta_model_migration", + "risingwave_meta_model_v2", "risingwave_object_store", "risingwave_pb", "risingwave_rpc_client", @@ -7716,6 +8155,25 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "risingwave_meta_model_migration" +version = "1.3.0-alpha" +dependencies = [ + "async-std", + "sea-orm-migration", + "uuid", +] + +[[package]] +name = "risingwave_meta_model_v2" +version = "1.3.0-alpha" +dependencies = [ + "risingwave_pb", + "sea-orm", + "serde", + "serde_json", +] + [[package]] name = "risingwave_meta_node" version = "1.3.0-alpha" @@ -7728,13 +8186,13 @@ dependencies = [ "madsim-etcd-client", "madsim-tokio", "madsim-tonic", - "model_migration", "prometheus-http-query", "regex", "risingwave_common", "risingwave_common_heap_profiling", "risingwave_common_service", "risingwave_meta", + "risingwave_meta_model_migration", "risingwave_meta_service", "risingwave_pb", "risingwave_rpc_client", @@ -7758,6 +8216,7 @@ dependencies = [ "risingwave_common", "risingwave_connector", "risingwave_meta", + "risingwave_meta_model_v2", "risingwave_pb", "sea-orm", "sync-point", @@ -8182,14 +8641,22 @@ version = "0.1.0" dependencies = [ "arrow-array", "arrow-flight", + "arrow-ipc", "arrow-schema", "arrow-select", + "base64 0.21.4", + "bytes", "cfg-or-panic", "futures-util", + "itertools 0.11.0", "madsim-tokio", "madsim-tonic", + "risingwave_object_store", "static_assertions", "thiserror", + "tracing", + "wasmtime", + "wasmtime-wasi", ] [[package]] @@ -8321,7 +8788,7 @@ checksum = "6da3636faa25820d8648e0e31c5d519bbb01f72fdf57131f0f5f7da5fed36eab" dependencies = [ "bitflags 1.3.2", "errno", - "io-lifetimes", + "io-lifetimes 1.0.11", "libc", "linux-raw-sys 0.1.4", "windows-sys 0.45.0", @@ -8335,9 +8802,11 @@ checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" dependencies = [ "bitflags 1.3.2", "errno", - "io-lifetimes", + "io-lifetimes 1.0.11", + "itoa", "libc", "linux-raw-sys 0.3.8", + "once_cell", "windows-sys 0.48.0", ] @@ -9004,6 +9473,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +[[package]] +name = "shellexpand" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" +dependencies = [ + "dirs", +] + [[package]] name = "shlex" version = "1.2.0" @@ -9123,7 +9601,7 @@ dependencies = [ "cargo_metadata", "error-chain", "glob", - "pulldown-cmark", + "pulldown-cmark 0.9.3", "tempfile", "walkdir", ] @@ -9153,6 +9631,12 @@ dependencies = [ "parking_lot 0.11.2", ] +[[package]] +name = "slice-group-by" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" + [[package]] name = "smallbitset" version = "0.7.1" @@ -9232,6 +9716,12 @@ dependencies = [ "der", ] +[[package]] +name = "sptr" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" + [[package]] name = "sqlformat" version = "0.2.2" @@ -9625,7 +10115,7 @@ version = "12.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "691e53bdc0702aba3a5abc2cffff89346fcbd4050748883c7e2f714b33a69045" dependencies = [ - "cpp_demangle", + "cpp_demangle 0.4.3", "rustc-demangle", "symbolic-common", ] @@ -9682,6 +10172,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "system-interface" +version = "0.25.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10081a99cbecbc363d381b9503563785f0b02735fccbb0d4c1a2cb3d39f7e7fe" +dependencies = [ + "bitflags 2.4.0", + "cap-fs-ext", + "cap-std", + "fd-lock", + "io-lifetimes 2.0.2", + "rustix 0.38.11", + "windows-sys 0.48.0", + "winx 0.36.2", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -9694,6 +10200,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "target-lexicon" +version = "0.12.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" + [[package]] name = "task_stats_alloc" version = "0.1.11" @@ -10463,6 +10975,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + [[package]] name = "unicode_categories" version = "0.1.1" @@ -10615,6 +11133,50 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi-cap-std-sync" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2fe3aaf51c1e1a04a490e89f0a9cab789d21a496c0ce398d49a24f8df883a58" +dependencies = [ + "anyhow", + "async-trait", + "cap-fs-ext", + "cap-rand", + "cap-std", + "cap-time-ext", + "fs-set-times", + "io-extras", + "io-lifetimes 1.0.11", + "is-terminal", + "once_cell", + "rustix 0.37.23", + "system-interface", + "tracing", + "wasi-common", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasi-common" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e74e9a2c8bfda59870a8bff38a31b9ba80b6fdb7abdfd2487177b85537d2e8a8" +dependencies = [ + "anyhow", + "bitflags 1.3.2", + "cap-rand", + "cap-std", + "io-extras", + "log", + "rustix 0.37.23", + "thiserror", + "tracing", + "wasmtime", + "wiggle", + "windows-sys 0.48.0", +] + [[package]] name = "wasm-bindgen" version = "0.2.87" @@ -10681,6 +11243,24 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +[[package]] +name = "wasm-encoder" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18c41dbd92eaebf3612a39be316540b8377c871cb9bde6b064af962984912881" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-encoder" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca90ba1b5b0a70d3d49473c5579951f3bddc78d47b59256d2f9d4922b150aca" +dependencies = [ + "leb128", +] + [[package]] name = "wasm-streams" version = "0.3.0" @@ -10694,6 +11274,370 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.107.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29e3ac9b780c7dda0cac7a52a5d6d2d6707cc6e3451c9db209b6c758f40d7acb" +dependencies = [ + "indexmap 1.9.3", + "semver", +] + +[[package]] +name = "wasmparser" +version = "0.115.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e06c0641a4add879ba71ccb3a1e4278fd546f76f1eafb21d8f7b07733b547cd5" +dependencies = [ + "indexmap 2.0.0", + "semver", +] + +[[package]] +name = "wasmprinter" +version = "0.2.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e74458a9bc5cc9c7108abfa0fe4dc88d5abf1f3baf194df3264985f17d559b5e" +dependencies = [ + "anyhow", + "wasmparser 0.115.0", +] + +[[package]] +name = "wasmtime" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc104ced94ff0a6981bde77a0bc29aab4af279914a4143b8d1af9fd4b2c9d41" +dependencies = [ + "anyhow", + "async-trait", + "bincode 1.3.3", + "bumpalo", + "cfg-if", + "encoding_rs", + "fxprof-processed-profile", + "indexmap 1.9.3", + "libc", + "log", + "object 0.30.4", + "once_cell", + "paste", + "psm", + "rayon", + "serde", + "serde_json", + "target-lexicon", + "wasmparser 0.107.0", + "wasmtime-cache", + "wasmtime-component-macro", + "wasmtime-component-util", + "wasmtime-cranelift", + "wasmtime-environ", + "wasmtime-fiber", + "wasmtime-jit", + "wasmtime-runtime", + "wasmtime-winch", + "wat", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-asm-macros" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b28e5661a9b5f7610a62ab3c69222fa161f7bd31d04529e856461d8c3e706b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "wasmtime-cache" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f58ddfe801df3886feaf466d883ea37e941bcc6d841b9f644a08c7acabfe7f8" +dependencies = [ + "anyhow", + "base64 0.21.4", + "bincode 1.3.3", + "directories-next", + "file-per-thread-logger", + "log", + "rustix 0.37.23", + "serde", + "sha2", + "toml 0.5.11", + "windows-sys 0.48.0", + "zstd 0.11.2+zstd.1.5.2", +] + +[[package]] +name = "wasmtime-component-macro" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39725d9633fb064bd3a6d83c5ea5077289256de0862d3d96295822edb13419c0" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn 1.0.109", + "wasmtime-component-util", + "wasmtime-wit-bindgen", + "wit-parser", +] + +[[package]] +name = "wasmtime-component-util" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1153feafc824f95dc69472cb89a3396b3b05381f781a7508b01840f9df7b1a51" + +[[package]] +name = "wasmtime-cranelift" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc1e39ce9aa0fa0b319541ed423960b06cfa7343eca1574f811ea34275739c2" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-control", + "cranelift-entity", + "cranelift-frontend", + "cranelift-native", + "cranelift-wasm", + "gimli 0.27.3", + "log", + "object 0.30.4", + "target-lexicon", + "thiserror", + "wasmparser 0.107.0", + "wasmtime-cranelift-shared", + "wasmtime-environ", +] + +[[package]] +name = "wasmtime-cranelift-shared" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd32739326690e51c76551d7cbf29d371e7de4dc7b37d2d503be314ab5b7d04" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-control", + "cranelift-native", + "gimli 0.27.3", + "object 0.30.4", + "target-lexicon", + "wasmtime-environ", +] + +[[package]] +name = "wasmtime-environ" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b60e4ae5c9ae81750d8bc59110bf25444aa1d9266c19999c3b64b801db3c73" +dependencies = [ + "anyhow", + "cranelift-entity", + "gimli 0.27.3", + "indexmap 1.9.3", + "log", + "object 0.30.4", + "serde", + "target-lexicon", + "thiserror", + "wasm-encoder 0.29.0", + "wasmparser 0.107.0", + "wasmprinter", + "wasmtime-component-util", + "wasmtime-types", +] + +[[package]] +name = "wasmtime-fiber" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd40c8d869916ee6b1f3fcf1858c52041445475ca8550aee81c684c0eb530ca" +dependencies = [ + "cc", + "cfg-if", + "rustix 0.37.23", + "wasmtime-asm-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-jit" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655b23a10eddfe7814feb548a466f3f25aa4bb4f43098a147305c544a2de28e1" +dependencies = [ + "addr2line 0.19.0", + "anyhow", + "bincode 1.3.3", + "cfg-if", + "cpp_demangle 0.3.5", + "gimli 0.27.3", + "ittapi", + "log", + "object 0.30.4", + "rustc-demangle", + "rustix 0.37.23", + "serde", + "target-lexicon", + "wasmtime-environ", + "wasmtime-jit-debug", + "wasmtime-jit-icache-coherence", + "wasmtime-runtime", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-jit-debug" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46b7e98979a69d3df093076bde8431204e3c96a770e8d216fea365c627d88a4" +dependencies = [ + "object 0.30.4", + "once_cell", + "rustix 0.37.23", +] + +[[package]] +name = "wasmtime-jit-icache-coherence" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb1e7c68ede63dc7a98c3e473162954e224951854e229c8b4e74697fe17dbdd" +dependencies = [ + "cfg-if", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-runtime" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843e33bf9e0f0c57902c87a1dea1389cc23865c65f007214318dbdfcb3fd4ae5" +dependencies = [ + "anyhow", + "cc", + "cfg-if", + "encoding_rs", + "indexmap 1.9.3", + "libc", + "log", + "mach", + "memfd", + "memoffset 0.8.0", + "paste", + "rand", + "rustix 0.37.23", + "sptr", + "wasmtime-asm-macros", + "wasmtime-environ", + "wasmtime-fiber", + "wasmtime-jit-debug", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-types" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7473a07bebd85671bada453123e3d465c8e0a59668ff79f5004076e6a2235ef5" +dependencies = [ + "cranelift-entity", + "serde", + "thiserror", + "wasmparser 0.107.0", +] + +[[package]] +name = "wasmtime-wasi" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aff7b3b3272ad5b4ba63c9aac6248da6f06a8227d0c0d6017d89225d794e966c" +dependencies = [ + "anyhow", + "async-trait", + "bitflags 1.3.2", + "cap-fs-ext", + "cap-rand", + "cap-std", + "cap-time-ext", + "fs-set-times", + "io-extras", + "libc", + "rustix 0.37.23", + "system-interface", + "thiserror", + "tracing", + "wasi-cap-std-sync", + "wasi-common", + "wasmtime", + "wiggle", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-winch" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "351c9d4e60658dd0cf616c12c5508f86cc2cefcc0cff307eed0a31b23d3c0b70" +dependencies = [ + "anyhow", + "cranelift-codegen", + "gimli 0.27.3", + "object 0.30.4", + "target-lexicon", + "wasmparser 0.107.0", + "wasmtime-cranelift-shared", + "wasmtime-environ", + "winch-codegen", +] + +[[package]] +name = "wasmtime-wit-bindgen" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f114407efbd09e4ef67053b6ae54c16455a821ef2f6096597fcba83b7625e59c" +dependencies = [ + "anyhow", + "heck 0.4.1", + "wit-parser", +] + +[[package]] +name = "wast" +version = "35.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ef140f1b49946586078353a453a1d28ba90adfc54dde75710bc1931de204d68" +dependencies = [ + "leb128", +] + +[[package]] +name = "wast" +version = "66.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93cb43b0ac6dd156f2c375735ccfd72b012a7c0a6e6d09503499b8d3cb6e6072" +dependencies = [ + "leb128", + "memchr", + "unicode-width", + "wasm-encoder 0.35.0", +] + +[[package]] +name = "wat" +version = "1.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e367582095d2903caeeea9acbb140e1db9c7677001efa4347c3687fd34fe7072" +dependencies = [ + "wast 66.0.2", +] + [[package]] name = "web-sys" version = "0.3.64" @@ -10751,6 +11695,48 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wiggle" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e63f150c6e39ef29a58139564c5ed7a0ef34d6df8a8eecd4233af85a576968d9" +dependencies = [ + "anyhow", + "async-trait", + "bitflags 1.3.2", + "thiserror", + "tracing", + "wasmtime", + "wiggle-macro", +] + +[[package]] +name = "wiggle-generate" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f31e961fb0a5ad3ff10689c85f327f4abf10b4cac033b9d7372ccbb106aea24" +dependencies = [ + "anyhow", + "heck 0.4.1", + "proc-macro2", + "quote", + "shellexpand", + "syn 1.0.109", + "witx", +] + +[[package]] +name = "wiggle-macro" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a28ae3d6b90f212beca7fab5910d0a3b1a171290c06eaa81bb39f41e6f74589" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "wiggle-generate", +] + [[package]] name = "winapi" version = "0.3.9" @@ -10782,6 +11768,22 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winch-codegen" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1bf2ac354be169bb201de7867b84f45d91d0ef812f67f11c33f74a7f5a24e56" +dependencies = [ + "anyhow", + "cranelift-codegen", + "gimli 0.27.3", + "regalloc2", + "smallvec", + "target-lexicon", + "wasmparser 0.107.0", + "wasmtime-environ", +] + [[package]] name = "windows" version = "0.48.0" @@ -10942,6 +11944,55 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winx" +version = "0.35.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c52a121f0fbf9320d5f2a9a5d82f6cb7557eda5e8b47fc3e7f359ec866ae960" +dependencies = [ + "bitflags 1.3.2", + "io-lifetimes 1.0.11", + "windows-sys 0.48.0", +] + +[[package]] +name = "winx" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357bb8e2932df531f83b052264b050b81ba0df90ee5a59b2d1d3949f344f81e5" +dependencies = [ + "bitflags 2.4.0", + "windows-sys 0.48.0", +] + +[[package]] +name = "wit-parser" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6daec9f093dbaea0e94043eeb92ece327bbbe70c86b1f41aca9bbfefd7f050f0" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 1.9.3", + "log", + "pulldown-cmark 0.8.0", + "semver", + "unicode-xid", + "url", +] + +[[package]] +name = "witx" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e366f27a5cabcddb2706a78296a40b8fcc451e1a6aba2fc1d94b4a01bdaaef4b" +dependencies = [ + "anyhow", + "log", + "thiserror", + "wast 35.0.2", +] + [[package]] name = "workspace-config" version = "1.3.0-alpha" @@ -10975,6 +12026,7 @@ dependencies = [ "clap", "clap_builder", "combine", + "crc32fast", "crossbeam-epoch", "crossbeam-queue", "crossbeam-utils", @@ -10982,6 +12034,7 @@ dependencies = [ "digest", "either", "fail", + "fallible-iterator", "fixedbitset", "flate2", "frunk_core", @@ -10993,6 +12046,7 @@ dependencies = [ "futures-task", "futures-util", "hashbrown 0.12.3", + "hashbrown 0.13.2", "hashbrown 0.14.0", "hyper", "indexmap 1.9.3", @@ -11042,6 +12096,8 @@ dependencies = [ "reqwest", "ring", "rust_decimal", + "rustc-hash", + "rustix 0.38.11", "rustls 0.21.7", "scopeguard", "sea-orm", @@ -11059,6 +12115,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", + "stable_deref_trait", "strum 0.25.0", "subtle", "syn 1.0.109", @@ -11167,6 +12224,15 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe 5.0.2+zstd.1.5.2", +] + [[package]] name = "zstd" version = "0.12.4" @@ -11185,6 +12251,16 @@ dependencies = [ "zstd-safe 7.0.0", ] +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + [[package]] name = "zstd-safe" version = "6.0.6" diff --git a/Cargo.toml b/Cargo.toml index f0dd2d0443b9e..cddb598e6f0dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,9 +19,10 @@ members = [ "src/java_binding", "src/jni_core", "src/meta", + "src/meta/model_v2", + "src/meta/model_v2/migration", "src/meta/node", "src/meta/service", - "src/meta/src/model_v2/migration", "src/object_store", "src/prost", "src/prost/helpers", @@ -119,6 +120,7 @@ arrow-buffer = "48" arrow-flight = "48" arrow-select = "48" arrow-ord = "48" +arrow-ipc = "48" tikv-jemalloc-ctl = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9" } tikv-jemallocator = { git = "https://github.com/risingwavelabs/jemallocator.git", features = [ "profiling", @@ -143,6 +145,8 @@ risingwave_hummock_test = { path = "./src/storage/hummock_test" } risingwave_hummock_trace = { path = "./src/storage/hummock_trace" } risingwave_meta = { path = "./src/meta" } risingwave_meta_service = { path = "./src/meta/service" } +risingwave_meta_model_migration = { path = "src/meta/model_v2/migration" } +risingwave_meta_model_v2 = { path = "./src/meta/model_v2" } risingwave_meta_node = { path = "./src/meta/node" } risingwave_object_store = { path = "./src/object_store" } risingwave_pb = { path = "./src/prost" } diff --git a/e2e_test/batch/catalog/pg_description.slt.part b/e2e_test/batch/catalog/pg_description.slt.part index 41ade5ffbb9b3..44ba006ea4367 100644 --- a/e2e_test/batch/catalog/pg_description.slt.part +++ b/e2e_test/batch/catalog/pg_description.slt.part @@ -1,18 +1,65 @@ -query IIITT -SELECT d.*, c.relname FROM pg_catalog.pg_description d join pg_catalog.pg_class c on d.objoid = c.oid ORDER BY d.objoid limit 15; +statement ok +create table t(a int, b text, c date); + +statement ok +comment on table t is 'Lorem ipsum'; + +statement ok +comment on column t.a is 'Praesent elementum'; + +statement ok +comment on column public.t.c is 'Nullam ultricies'; + +statement ok +comment on column public.t._row_id is 'facilisis enim'; + +query TIIIT +SELECT + c.relname, + ( + SELECT relname FROM pg_catalog.pg_class WHERE oid = d.classoid + ) AS classoid, + d.objsubid, + d.description +FROM + pg_catalog.pg_description d + JOIN pg_catalog.pg_class c + ON d.objoid = c.oid +ORDER BY d.objsubid; ---- -1 NULL 0 NULL pg_type -2 NULL 0 NULL pg_namespace -3 NULL 0 NULL pg_cast -4 NULL 0 NULL pg_matviews -5 NULL 0 NULL pg_user -6 NULL 0 NULL pg_class -7 NULL 0 NULL pg_index -8 NULL 0 NULL pg_opclass -9 NULL 0 NULL pg_collation -10 NULL 0 NULL pg_am -11 NULL 0 NULL pg_operator -12 NULL 0 NULL pg_views -13 NULL 0 NULL pg_attribute -14 NULL 0 NULL pg_database -15 NULL 0 NULL pg_description \ No newline at end of file +t rw_tables -1 facilisis enim +t rw_tables 0 Lorem ipsum +t rw_tables 1 Praesent elementum +t rw_tables 3 Nullam ultricies + +statement ok +comment on table public.t is NULL; + +statement ok +comment on column t._row_id is NULL; + +statement ok +comment on column t.c is ''; + +statement ok +comment on column public.t.b is 'Vivamus fermentum'; + +query TIIIT +SELECT + c.relname, + ( + SELECT relname FROM pg_catalog.pg_class WHERE oid = d.classoid + ) AS classoid, + d.objsubid, + d.description +FROM + pg_catalog.pg_description d + JOIN pg_catalog.pg_class c + ON d.objoid = c.oid +ORDER BY d.objsubid; +---- +t rw_tables 1 Praesent elementum +t rw_tables 2 Vivamus fermentum + +statement ok +drop table t; diff --git a/e2e_test/ddl/show.slt b/e2e_test/ddl/show.slt index 5ae7575668645..9586731207fa7 100644 --- a/e2e_test/ddl/show.slt +++ b/e2e_test/ddl/show.slt @@ -7,15 +7,28 @@ create materialized view mv3 as select sum(v1) as sum_v1 from t3; statement ok create view v3 as select sum(v2) as sum_v2 from t3; -query TTT +statement ok +comment on table t3 is 'volutpat vitae'; + +statement ok +comment on column t3.v1 is 'turpis vehicula'; + +statement ok +comment on column t3.v2 is 'Lorem ipsum dolor sit amet'; + +statement ok +comment on column public.t3._row_id is 'consectetur adipiscing elit'; + +query TTTT describe t3; ---- -v1 integer false -v2 integer false -v3 integer false -_row_id serial true -primary key _row_id NULL -distribution key _row_id NULL +v1 integer false turpis vehicula +v2 integer false Lorem ipsum dolor sit amet +v3 integer false NULL +_row_id serial true consectetur adipiscing elit +primary key _row_id NULL NULL +distribution key _row_id NULL NULL +table description t3 NULL volutpat vitae query TTT show columns from t3; @@ -33,16 +46,29 @@ show indexes from t3; ---- idx1 t3 v1 ASC, v2 ASC v3 v1 -query TTT +statement ok +comment on table public.t3 is 'consectetur turpis'; + +statement ok +comment on column t3.v1 is 'Nemo enim ipsam'; + +statement ok +comment on column t3.v2 is ''; + +statement ok +comment on column t3._row_id is NULL; + +query TTTT describe t3; ---- -v1 integer false -v2 integer false -v3 integer false -_row_id serial true -primary key _row_id NULL -distribution key _row_id NULL -idx1 index(v1 ASC, v2 ASC) include(v3) distributed by(v1) NULL +v1 integer false Nemo enim ipsam +v2 integer false NULL +v3 integer false NULL +_row_id serial true NULL +primary key _row_id NULL NULL +distribution key _row_id NULL NULL +idx1 index(v1 ASC, v2 ASC) include(v3) distributed by(v1) NULL NULL +table description t3 NULL consectetur turpis query TT show create index idx1; diff --git a/e2e_test/extended_mode/basic.slt b/e2e_test/extended_mode/basic.slt index 7869494979e47..054dae2f6a234 100644 --- a/e2e_test/extended_mode/basic.slt +++ b/e2e_test/extended_mode/basic.slt @@ -39,15 +39,16 @@ values(round(42.4382)); statement ok create table t3 (v1 int, v2 int, v3 int); -query TTT +query TTTT describe t3; ---- -v1 integer false -v2 integer false -v3 integer false -_row_id serial true -primary key _row_id NULL -distribution key _row_id NULL +v1 integer false NULL +v2 integer false NULL +v3 integer false NULL +_row_id serial true NULL +primary key _row_id NULL NULL +distribution key _row_id NULL NULL +table description t3 NULL NULL query TTT show columns from t3; diff --git a/proto/catalog.proto b/proto/catalog.proto index 2d4d51b5692b3..12c72726ad57b 100644 --- a/proto/catalog.proto +++ b/proto/catalog.proto @@ -193,6 +193,7 @@ message Function { repeated data.DataType arg_types = 5; data.DataType return_type = 6; string language = 7; + // external function service only string link = 8; string identifier = 10; @@ -204,6 +205,11 @@ message Function { message ScalarFunction {} message TableFunction {} message AggregateFunction {} + + oneof extra { + expr.ExternalUdfExtra external = 14; + expr.WasmUdfExtra wasm = 15; + } } // See `TableCatalog` struct in frontend crate for more information. @@ -280,6 +286,9 @@ message Table { CreateType create_type = 32; + // This field is used to store the description set by the `comment on` clause. + optional string description = 33; + // Per-table catalog version, used by schema change. `None` for internal tables and tests. // Not to be confused with the global catalog version for notification service. TableVersion version = 100; @@ -317,3 +326,11 @@ message Database { string name = 2; uint32 owner = 3; } + +message Comment { + uint32 table_id = 1; + uint32 schema_id = 2; + uint32 database_id = 3; + optional uint32 column_index = 4; + optional string description = 5; +} diff --git a/proto/ddl_service.proto b/proto/ddl_service.proto index 1efc933a7d033..b3c7f17509f8c 100644 --- a/proto/ddl_service.proto +++ b/proto/ddl_service.proto @@ -318,6 +318,15 @@ message WaitRequest {} message WaitResponse {} +message CommentOnRequest { + catalog.Comment comment = 1; +} + +message CommentOnResponse { + common.Status status = 1; + uint64 version = 2; +} + service DdlService { rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse); rpc DropDatabase(DropDatabaseRequest) returns (DropDatabaseResponse); @@ -348,4 +357,5 @@ service DdlService { rpc DropConnection(DropConnectionRequest) returns (DropConnectionResponse); rpc GetTables(GetTablesRequest) returns (GetTablesResponse); rpc Wait(WaitRequest) returns (WaitResponse); + rpc CommentOn(CommentOnRequest) returns (CommentOnResponse); } diff --git a/proto/expr.proto b/proto/expr.proto index 2f252d67c8400..745231a9cd2b5 100644 --- a/proto/expr.proto +++ b/proto/expr.proto @@ -233,6 +233,7 @@ message ExprNode { COL_DESCRIPTION = 2100; CAST_REGCLASS = 2101; } + // Only use this field for function call. For other types of expression, it should be UNSPECIFIED. Type function_type = 1; data.DataType return_type = 3; oneof rex_node { @@ -417,15 +418,39 @@ message WindowFunction { WindowFrame frame = 5; } +// Note: due to historic reasons, UserDefinedFunction is a oneof variant parallel to FunctionCall, +// while UserDefinedTableFunction is embedded as a field in TableFunction. + message UserDefinedFunction { repeated ExprNode children = 1; string name = 2; repeated data.DataType arg_types = 3; string language = 4; + // external function service only string link = 5; + // An unique identifier for the function. Different kinds of UDF may handle this field differently. + // - For external UDF, it's the name of the function in the external function service. + // It doesn't need to be unique across different external function servers. + // - For wasm UDF, it's the name of the function stored in remote object store, and needs to be globally unique. string identifier = 6; + + oneof extra { + ExternalUdfExtra external = 7; + WasmUdfExtra wasm = 8; + } +} + +// extra information for external functions +message ExternalUdfExtra {} + +// extra information for wasm functions +message WasmUdfExtra { + // We store the url of the remote object store here, as it's inconvenient to pass it to build expr. + // Maybe we can deprecate it in the future. + string wasm_storage_url = 1; } +// Additional information for user defined table functions. message UserDefinedTableFunction { repeated data.DataType arg_types = 3; string language = 4; diff --git a/proto/meta.proto b/proto/meta.proto index f2375eed7653a..f9947d278b468 100644 --- a/proto/meta.proto +++ b/proto/meta.proto @@ -514,6 +514,7 @@ message SystemParams { optional uint32 parallel_compact_size_mb = 11; optional uint32 max_concurrent_creating_streaming_jobs = 12; optional bool pause_on_next_bootstrap = 13; + optional string wasm_storage_url = 14; } message GetSystemParamsRequest {} diff --git a/proto/plan_common.proto b/proto/plan_common.proto index d4c7a2e04f138..afea3aff14bc7 100644 --- a/proto/plan_common.proto +++ b/proto/plan_common.proto @@ -37,6 +37,9 @@ message ColumnDesc { GeneratedColumnDesc generated_column = 6; DefaultColumnDesc default_column = 7; } + + // This field is used to store the description set by the `comment on` clause. + optional string description = 8; } message ColumnCatalog { diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml index ddd1fe5a33cdb..168ba836d4c1b 100644 --- a/src/common/Cargo.toml +++ b/src/common/Cargo.toml @@ -49,6 +49,7 @@ hyper = "0.14" hytra = { workspace = true } itertools = "0.11" itoa = "1.0" +jsonbb = "0.1" lru = { git = "https://github.com/risingwavelabs/lru-rs.git", rev = "cb2d7c7" } memcomparable = { version = "0.2", features = ["decimal"] } num-integer = "0.1" diff --git a/src/common/src/array/jsonb_array.rs b/src/common/src/array/jsonb_array.rs index 0e9ba7c48511d..3c4ca23fff04e 100644 --- a/src/common/src/array/jsonb_array.rs +++ b/src/common/src/array/jsonb_array.rs @@ -12,36 +12,35 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::mem::size_of; - use risingwave_pb::data::{PbArray, PbArrayType}; -use serde_json::Value; -use super::{Array, ArrayBuilder}; +use super::{Array, ArrayBuilder, ArrayImpl, ArrayResult}; use crate::buffer::{Bitmap, BitmapBuilder}; use crate::estimate_size::EstimateSize; -use crate::types::{DataType, JsonbRef, JsonbVal, F32, F64}; -use crate::util::iter_util::ZipEqFast; +use crate::types::{DataType, JsonbRef, JsonbVal, Scalar}; #[derive(Debug)] pub struct JsonbArrayBuilder { bitmap: BitmapBuilder, - data: Vec, + builder: jsonbb::Builder, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct JsonbArray { bitmap: Bitmap, - data: Vec, + /// Elements are stored as a single JSONB array value. + data: jsonbb::Value, } impl ArrayBuilder for JsonbArrayBuilder { type ArrayType = JsonbArray; fn new(capacity: usize) -> Self { + let mut builder = jsonbb::Builder::with_capacity(capacity); + builder.begin_array(); Self { bitmap: BitmapBuilder::with_capacity(capacity), - data: Vec::with_capacity(capacity), + builder, } } @@ -54,13 +53,15 @@ impl ArrayBuilder for JsonbArrayBuilder { match value { Some(x) => { self.bitmap.append_n(n, true); - self.data - .extend(std::iter::repeat(x).take(n).map(|x| x.0.clone())); + for _ in 0..n { + self.builder.add_value(x.0); + } } None => { self.bitmap.append_n(n, false); - self.data - .extend(std::iter::repeat(*JsonbVal::dummy().0).take(n)); + for _ in 0..n { + self.builder.add_null(); + } } } } @@ -69,29 +70,44 @@ impl ArrayBuilder for JsonbArrayBuilder { for bit in other.bitmap.iter() { self.bitmap.append(bit); } - self.data.extend_from_slice(&other.data); + for value in other.data.as_array().unwrap().iter() { + self.builder.add_value(value); + } } fn pop(&mut self) -> Option<()> { - self.data.pop().map(|_| self.bitmap.pop().unwrap()) + self.bitmap.pop()?; + self.builder.pop(); + Some(()) } fn len(&self) -> usize { self.bitmap.len() } - fn finish(self) -> Self::ArrayType { + fn finish(mut self) -> Self::ArrayType { + self.builder.end_array(); Self::ArrayType { bitmap: self.bitmap.finish(), - data: self.data, + data: self.builder.finish(), } } } -impl JsonbArrayBuilder { - pub fn append_move(&mut self, value: JsonbVal) { - self.bitmap.append(true); - self.data.push(*value.0); +impl JsonbArray { + /// Loads a `JsonbArray` from a protobuf array. + /// + /// See also `JsonbArray::to_protobuf`. + pub fn from_protobuf(array: &PbArray) -> ArrayResult { + ensure!( + array.values.len() == 1, + "Must have exactly 1 buffer in a jsonb array" + ); + let arr = JsonbArray { + bitmap: array.get_null_bitmap()?.into(), + data: jsonbb::Value::from_bytes(&array.values[0].body), + }; + Ok(arr.into()) } } @@ -101,52 +117,23 @@ impl Array for JsonbArray { type RefItem<'a> = JsonbRef<'a>; unsafe fn raw_value_at_unchecked(&self, idx: usize) -> Self::RefItem<'_> { - JsonbRef(self.data.get_unchecked(idx)) + JsonbRef(self.data.as_array().unwrap().get(idx).unwrap()) } fn len(&self) -> usize { - self.data.len() + self.bitmap.len() } fn to_protobuf(&self) -> PbArray { - // The memory layout contains `serde_json::Value` trees, but in protobuf we transmit this as - // variable length bytes in value encoding. That is, one buffer of length n+1 containing - // start and end offsets into the 2nd buffer containing all value bytes concatenated. - use risingwave_pb::common::buffer::CompressionType; use risingwave_pb::common::Buffer; - let mut offset_buffer = - Vec::::with_capacity((1 + self.data.len()) * std::mem::size_of::()); - let mut data_buffer = Vec::::with_capacity(self.data.len()); - - let mut offset = 0; - for (v, not_null) in self.data.iter().zip_eq_fast(self.null_bitmap().iter()) { - if !not_null { - continue; - } - let d = JsonbRef(v).value_serialize(); - offset_buffer.extend_from_slice(&(offset as u64).to_be_bytes()); - data_buffer.extend_from_slice(&d); - offset += d.len(); - } - offset_buffer.extend_from_slice(&(offset as u64).to_be_bytes()); - - let values = vec![ - Buffer { - compression: CompressionType::None as i32, - body: offset_buffer, - }, - Buffer { - compression: CompressionType::None as i32, - body: data_buffer, - }, - ]; - - let null_bitmap = self.null_bitmap().to_protobuf(); PbArray { - null_bitmap: Some(null_bitmap), - values, + null_bitmap: Some(self.null_bitmap().to_protobuf()), + values: vec![Buffer { + compression: CompressionType::None as i32, + body: self.data.as_bytes().to_vec(), + }], array_type: PbArrayType::Jsonb as i32, struct_array_data: None, list_array_data: None, @@ -176,7 +163,7 @@ impl FromIterator> for JsonbArray { let mut builder = ::Builder::new(iter.size_hint().0); for i in iter { match i { - Some(x) => builder.append_move(x), + Some(x) => builder.append(Some(x.as_scalar_ref())), None => builder.append(None), } } @@ -190,31 +177,8 @@ impl FromIterator for JsonbArray { } } -// TODO: We need to fix this later. impl EstimateSize for JsonbArray { fn estimated_heap_size(&self) -> usize { - self.bitmap.estimated_heap_size() + self.data.capacity() * size_of::() - } -} - -impl From for Value { - fn from(v: F32) -> Value { - serde_json::Number::from_f64(v.0 as f64) - .expect("todo: convert Inf/NaN to jsonb") - .into() - } -} - -impl From for Value { - fn from(v: F64) -> Value { - serde_json::Number::from_f64(v.0) - .expect("todo: convert Inf/NaN to jsonb") - .into() - } -} - -impl From> for Value { - fn from(v: JsonbRef<'_>) -> Value { - v.0.clone() + self.bitmap.estimated_heap_size() + self.data.capacity() } } diff --git a/src/common/src/array/proto_reader.rs b/src/common/src/array/proto_reader.rs index 55d505343dadd..4ca6bf7b70d05 100644 --- a/src/common/src/array/proto_reader.rs +++ b/src/common/src/array/proto_reader.rs @@ -52,9 +52,7 @@ impl ArrayImpl { PbArrayType::Timestamp => read_timestamp_array(array, cardinality)?, PbArrayType::Timestamptz => read_timestamptz_array(array, cardinality)?, PbArrayType::Interval => read_interval_array(array, cardinality)?, - PbArrayType::Jsonb => { - read_string_array::(array, cardinality)? - } + PbArrayType::Jsonb => JsonbArray::from_protobuf(array)?, PbArrayType::Struct => StructArray::from_protobuf(array)?, PbArrayType::List => ListArray::from_protobuf(array)?, PbArrayType::Unspecified => unreachable!(), diff --git a/src/common/src/array/value_reader.rs b/src/common/src/array/value_reader.rs index 96ed7c31b88aa..45db47f23242b 100644 --- a/src/common/src/array/value_reader.rs +++ b/src/common/src/array/value_reader.rs @@ -19,8 +19,7 @@ use byteorder::{BigEndian, ReadBytesExt}; use super::ArrayResult; use crate::array::{ - ArrayBuilder, BytesArrayBuilder, JsonbArrayBuilder, PrimitiveArrayItemType, Serial, - Utf8ArrayBuilder, + ArrayBuilder, BytesArrayBuilder, PrimitiveArrayItemType, Serial, Utf8ArrayBuilder, }; use crate::types::{Decimal, F32, F64}; @@ -89,15 +88,3 @@ impl VarSizedValueReader for BytesValueReader { Ok(()) } } - -pub struct JsonbValueReader; - -impl VarSizedValueReader for JsonbValueReader { - fn read(buf: &[u8], builder: &mut JsonbArrayBuilder) -> ArrayResult<()> { - let Some(v) = super::JsonbVal::value_deserialize(buf) else { - bail!("failed to read jsonb from bytes"); - }; - builder.append_move(v); - Ok(()) - } -} diff --git a/src/common/src/catalog/column.rs b/src/common/src/catalog/column.rs index cde16ef8d7652..b70084fbf864a 100644 --- a/src/common/src/catalog/column.rs +++ b/src/common/src/catalog/column.rs @@ -101,6 +101,7 @@ pub struct ColumnDesc { pub field_descs: Vec, pub type_name: String, pub generated_or_default_column: Option, + pub description: Option, } impl ColumnDesc { @@ -112,6 +113,7 @@ impl ColumnDesc { field_descs: vec![], type_name: String::new(), generated_or_default_column: None, + description: None, } } @@ -129,6 +131,7 @@ impl ColumnDesc { .collect_vec(), type_name: self.type_name.clone(), generated_or_default_column: self.generated_or_default_column.clone(), + description: self.description.clone(), } } @@ -172,6 +175,7 @@ impl ColumnDesc { field_descs: vec![], type_name: "".to_string(), generated_or_default_column: None, + description: None, } } @@ -192,6 +196,7 @@ impl ColumnDesc { field_descs: fields, type_name: type_name.to_string(), generated_or_default_column: None, + description: None, } } @@ -206,6 +211,7 @@ impl ColumnDesc { .map(Self::from_field_without_column_id) .collect_vec(), type_name: field.type_name.clone(), + description: None, generated_or_default_column: None, } } @@ -243,6 +249,7 @@ impl From for ColumnDesc { type_name: prost.type_name, field_descs, generated_or_default_column: prost.generated_or_default_column, + description: prost.description.clone(), } } } @@ -262,6 +269,7 @@ impl From<&ColumnDesc> for PbColumnDesc { field_descs: c.field_descs.iter().map(ColumnDesc::to_protobuf).collect(), type_name: c.type_name.clone(), generated_or_default_column: c.generated_or_default_column.clone(), + description: c.description.clone(), } } } diff --git a/src/common/src/catalog/mod.rs b/src/common/src/catalog/mod.rs index e83b4aa638907..1a46cdcf4057a 100644 --- a/src/common/src/catalog/mod.rs +++ b/src/common/src/catalog/mod.rs @@ -110,6 +110,7 @@ pub fn row_id_column_desc() -> ColumnDesc { field_descs: vec![], type_name: "".to_string(), generated_or_default_column: None, + description: None, } } @@ -131,6 +132,7 @@ pub fn offset_column_desc() -> ColumnDesc { field_descs: vec![], type_name: "".to_string(), generated_or_default_column: None, + description: None, } } diff --git a/src/common/src/catalog/test_utils.rs b/src/common/src/catalog/test_utils.rs index 2cce9b79b346e..6b524edb92430 100644 --- a/src/common/src/catalog/test_utils.rs +++ b/src/common/src/catalog/test_utils.rs @@ -57,6 +57,7 @@ impl ColumnDescTestExt for ColumnDesc { type_name: type_name.to_string(), field_descs: fields, generated_or_default_column: None, + description: None, } } } diff --git a/src/common/src/config.rs b/src/common/src/config.rs index 9e515ba471967..4188c3faa2b18 100644 --- a/src/common/src/config.rs +++ b/src/common/src/config.rs @@ -844,6 +844,9 @@ pub struct SystemConfig { /// Whether to pause all data sources on next bootstrap. #[serde(default = "default::system::pause_on_next_bootstrap")] pub pause_on_next_bootstrap: Option, + + #[serde(default = "default::system::wasm_storage_url")] + pub wasm_storage_url: Option, } impl SystemConfig { @@ -863,6 +866,7 @@ impl SystemConfig { max_concurrent_creating_streaming_jobs: self.max_concurrent_creating_streaming_jobs, pause_on_next_bootstrap: self.pause_on_next_bootstrap, telemetry_enabled: None, // deprecated + wasm_storage_url: self.wasm_storage_url, } } } diff --git a/src/common/src/system_param/mod.rs b/src/common/src/system_param/mod.rs index 0b35d53ae7e8c..ce29b7ab6a4d1 100644 --- a/src/common/src/system_param/mod.rs +++ b/src/common/src/system_param/mod.rs @@ -55,6 +55,7 @@ macro_rules! for_all_params { { backup_storage_directory, String, Some("backup".to_string()), true }, { max_concurrent_creating_streaming_jobs, u32, Some(1_u32), true }, { pause_on_next_bootstrap, bool, Some(false), true }, + { wasm_storage_url, String, Some("fs://@/tmp/risingwave".to_string()), false }, } }; } diff --git a/src/common/src/system_param/reader.rs b/src/common/src/system_param/reader.rs index 643905c89f919..8333e224a04a9 100644 --- a/src/common/src/system_param/reader.rs +++ b/src/common/src/system_param/reader.rs @@ -80,6 +80,10 @@ impl SystemParamsReader { self.prost.pause_on_next_bootstrap.unwrap_or(false) } + pub fn wasm_storage_url(&self) -> &str { + self.prost.wasm_storage_url.as_ref().unwrap() + } + pub fn to_kv(&self) -> Vec<(String, String)> { system_params_to_kv(&self.prost).unwrap() } diff --git a/src/common/src/test_utils/rand_array.rs b/src/common/src/test_utils/rand_array.rs index 70d0cb73d4dfa..f2dd8ad42854b 100644 --- a/src/common/src/test_utils/rand_array.rs +++ b/src/common/src/test_utils/rand_array.rs @@ -135,7 +135,7 @@ impl RandValue for Int256 { impl RandValue for JsonbVal { fn rand_value(_rand: &mut R) -> Self { - JsonbVal::dummy() + JsonbVal::null() } } diff --git a/src/common/src/types/jsonb.rs b/src/common/src/types/jsonb.rs index 7f4c002037060..590b693e47891 100644 --- a/src/common/src/types/jsonb.rs +++ b/src/common/src/types/jsonb.rs @@ -15,23 +15,21 @@ use std::fmt; use std::hash::Hash; -use postgres_types::{FromSql as _, ToSql as _, Type}; -use serde_json::Value; +use bytes::Buf; +use jsonbb::{Value, ValueRef}; use crate::estimate_size::EstimateSize; -use crate::types::{Scalar, ScalarRef}; +use crate::types::{Scalar, ScalarRef, F32, F64}; -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JsonbVal(pub(crate) Box); // The `Box` is just to keep `size_of::` smaller. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct JsonbVal(pub(crate) Value); -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct JsonbRef<'a>(pub(crate) &'a Value); +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct JsonbRef<'a>(pub(crate) ValueRef<'a>); impl EstimateSize for JsonbVal { fn estimated_heap_size(&self) -> usize { - // https://github.com/risingwavelabs/risingwave/issues/8957 - // FIXME: correctly handle jsonb size - 0 + self.0.capacity() } } @@ -63,7 +61,7 @@ impl<'a> ScalarRef<'a> for JsonbRef<'a> { type ScalarType = JsonbVal; fn to_owned_scalar(&self) -> Self::ScalarType { - JsonbVal(self.0.clone().into()) + JsonbVal(self.0.into()) } fn hash_scalar(&self, state: &mut H) { @@ -71,22 +69,6 @@ impl<'a> ScalarRef<'a> for JsonbRef<'a> { } } -impl Hash for JsonbRef<'_> { - fn hash(&self, state: &mut H) { - // We do not intend to support hashing `jsonb` type. - // Before #7981 is done, we do not panic but just hash its string representation. - // Note that `serde_json` without feature `preserve_order` uses `BTreeMap` for json object. - // So its string form always have keys sorted. - self.0.to_string().hash(state) - } -} - -impl Hash for JsonbVal { - fn hash(&self, state: &mut H) { - self.0.to_string().hash(state) - } -} - impl PartialOrd for JsonbVal { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) @@ -160,9 +142,7 @@ impl crate::types::to_binary::ToBinary for JsonbRef<'_> { &self, _ty: &crate::types::DataType, ) -> crate::error::Result> { - let mut output = bytes::BytesMut::new(); - self.0.to_sql(&Type::JSONB, &mut output).unwrap(); - Ok(Some(output.freeze())) + Ok(Some(self.value_serialize().into())) } } @@ -170,43 +150,130 @@ impl std::str::FromStr for JsonbVal { type Err = ::Err; fn from_str(s: &str) -> Result { - let v: Value = s.parse()?; - Ok(Self(v.into())) + Ok(Self(s.parse()?)) } } impl JsonbVal { - /// Constructs a value without specific meaning. Usually used as a lightweight placeholder. - pub fn dummy() -> Self { - Self(Value::Null.into()) + /// Returns a jsonb `null`. + pub fn null() -> Self { + Self(Value::null()) + } + + /// Returns an empty array `[]`. + pub fn empty_array() -> Self { + Self(Value::array([])) + } + + /// Returns an empty array `{}`. + pub fn empty_object() -> Self { + Self(Value::object([])) } + /// Deserialize from a memcomparable encoding. pub fn memcmp_deserialize( deserializer: &mut memcomparable::Deserializer, ) -> memcomparable::Result { - let v: Value = ::deserialize(deserializer)? + let v = ::deserialize(deserializer)? .parse() .map_err(|_| memcomparable::Error::Message("invalid json".into()))?; - Ok(Self(v.into())) + Ok(Self(v)) + } + + /// Deserialize from a pgwire "BINARY" encoding. + pub fn value_deserialize(mut buf: &[u8]) -> Option { + if buf.is_empty() || buf.get_u8() != 1 { + return None; + } + Value::from_text(buf).ok().map(Self) + } + + /// Convert the value to a [`serde_json::Value`]. + pub fn take(self) -> serde_json::Value { + self.0.into() + } +} + +impl From for JsonbVal { + fn from(v: serde_json::Value) -> Self { + Self(v.into()) + } +} + +impl From for JsonbVal { + fn from(v: bool) -> Self { + Self(v.into()) + } +} + +impl From for JsonbVal { + fn from(v: i16) -> Self { + Self(v.into()) } +} - pub fn value_deserialize(buf: &[u8]) -> Option { - let v = Value::from_sql(&Type::JSONB, buf).ok()?; - Some(Self(v.into())) +impl From for JsonbVal { + fn from(v: i32) -> Self { + Self(v.into()) } +} - pub fn take(mut self) -> Value { - self.0.take() +impl From for JsonbVal { + fn from(v: i64) -> Self { + Self(v.into()) } +} - pub fn as_serde_mut(&mut self) -> &mut Value { - &mut self.0 +impl From for JsonbVal { + fn from(v: F32) -> Self { + if v.0 == f32::INFINITY { + Self("Infinity".into()) + } else if v.0 == f32::NEG_INFINITY { + Self("-Infinity".into()) + } else if v.0.is_nan() { + Self("NaN".into()) + } else { + Self(v.0.into()) + } + } +} + +// NOTE: Infinite or NaN values are not JSON numbers. They are stored as strings in Postgres. +impl From for JsonbVal { + fn from(v: F64) -> Self { + if v.0 == f64::INFINITY { + Self("Infinity".into()) + } else if v.0 == f64::NEG_INFINITY { + Self("-Infinity".into()) + } else if v.0.is_nan() { + Self("NaN".into()) + } else { + Self(v.0.into()) + } + } +} + +impl From<&str> for JsonbVal { + fn from(v: &str) -> Self { + Self(v.into()) + } +} + +impl From> for JsonbVal { + fn from(v: JsonbRef<'_>) -> Self { + Self(v.0.to_owned()) } } impl From for JsonbVal { fn from(v: Value) -> Self { - Self(v.into()) + Self(v) + } +} + +impl<'a> From> for ValueRef<'a> { + fn from(v: JsonbRef<'a>) -> Self { + v.0 } } @@ -221,49 +288,52 @@ impl<'a> JsonbRef<'a> { serde::Serialize::serialize(&s, serializer) } + /// Serialize to a pgwire "BINARY" encoding. pub fn value_serialize(&self) -> Vec { + use std::io::Write; // Reuse the pgwire "BINARY" encoding for jsonb type. // It is not truly binary, but one byte of version `1u8` followed by string form. // This version number helps us maintain compatibility when we switch to more efficient // encoding later. - let mut output = bytes::BytesMut::new(); - self.0.to_sql(&Type::JSONB, &mut output).unwrap(); - output.freeze().into() + let mut buf = Vec::with_capacity(self.0.capacity()); + buf.push(1); + write!(&mut buf, "{}", self.0).unwrap(); + buf } + /// Returns true if this is a jsonb `null`. pub fn is_jsonb_null(&self) -> bool { - matches!(self.0, Value::Null) + self.0.as_null().is_some() } + /// Returns the type name of this jsonb. + /// + /// Possible values are: `null`, `boolean`, `number`, `string`, `array`, `object`. pub fn type_name(&self) -> &'static str { match self.0 { - Value::Null => "null", - Value::Bool(_) => "boolean", - Value::Number(_) => "number", - Value::String(_) => "string", - Value::Array(_) => "array", - Value::Object(_) => "object", + ValueRef::Null => "null", + ValueRef::Bool(_) => "boolean", + ValueRef::Number(_) => "number", + ValueRef::String(_) => "string", + ValueRef::Array(_) => "array", + ValueRef::Object(_) => "object", } } + /// Returns the length of this json array. pub fn array_len(&self) -> Result { - match self.0 { - Value::Array(v) => Ok(v.len()), - _ => Err(format!( - "cannot get array length of a jsonb {}", - self.type_name() - )), - } + let array = self + .0 + .as_array() + .ok_or_else(|| format!("cannot get array length of a jsonb {}", self.type_name()))?; + Ok(array.len()) } + /// If the JSON is a boolean, returns the associated bool. pub fn as_bool(&self) -> Result { - match self.0 { - Value::Bool(v) => Ok(*v), - _ => Err(format!( - "cannot cast jsonb {} to type boolean", - self.type_name() - )), - } + self.0 + .as_bool() + .ok_or_else(|| format!("cannot cast jsonb {} to type boolean", self.type_name())) } /// Attempt to read jsonb as a JSON number. @@ -271,13 +341,11 @@ impl<'a> JsonbRef<'a> { /// According to RFC 8259, only number within IEEE 754 binary64 (double precision) has good /// interoperability. We do not support arbitrary precision like PostgreSQL `numeric` right now. pub fn as_number(&self) -> Result { - match self.0 { - Value::Number(v) => v.as_f64().ok_or_else(|| "jsonb number out of range".into()), - _ => Err(format!( - "cannot cast jsonb {} to type number", - self.type_name() - )), - } + self.0 + .as_number() + .ok_or_else(|| format!("cannot cast jsonb {} to type number", self.type_name()))? + .as_f64() + .ok_or_else(|| "jsonb number out of range".into()) } /// This is part of the `->>` or `#>>` syntax to access a child as string. @@ -291,9 +359,9 @@ impl<'a> JsonbRef<'a> { /// * Jsonb string is displayed with quotes but treated as its inner value here. pub fn force_str(&self, writer: &mut W) -> std::fmt::Result { match self.0 { - Value::String(v) => writer.write_str(v), - Value::Null => Ok(()), - Value::Bool(_) | Value::Number(_) | Value::Array(_) | Value::Object(_) => { + ValueRef::String(v) => writer.write_str(v), + ValueRef::Null => Ok(()), + ValueRef::Bool(_) | ValueRef::Number(_) | ValueRef::Array(_) | ValueRef::Object(_) => { use crate::types::to_text::ToText as _; self.write_with_type(&crate::types::DataType::Jsonb, writer) } @@ -316,38 +384,33 @@ impl<'a> JsonbRef<'a> { /// Returns an iterator over the elements if this is an array. pub fn array_elements(self) -> Result>, String> { - match &self.0 { - Value::Array(array) => Ok(array.iter().map(Self)), - _ => Err(format!( - "cannot extract elements from a jsonb {}", - self.type_name() - )), - } + let array = self + .0 + .as_array() + .ok_or_else(|| format!("cannot extract elements from a jsonb {}", self.type_name()))?; + Ok(array.iter().map(Self)) } /// Returns an iterator over the keys if this is an object. pub fn object_keys(self) -> Result, String> { - match &self.0 { - Value::Object(object) => Ok(object.keys().map(|s| s.as_str())), - _ => Err(format!( + let object = self.0.as_object().ok_or_else(|| { + format!( "cannot call jsonb_object_keys on a jsonb {}", self.type_name() - )), - } + ) + })?; + Ok(object.keys()) } /// Returns an iterator over the key-value pairs if this is an object. pub fn object_key_values( self, ) -> Result)>, String> { - match &self.0 { - Value::Object(object) => Ok(object.iter().map(|(k, v)| (k.as_str(), Self(v)))), - _ => Err(format!("cannot deconstruct a jsonb {}", self.type_name())), - } - } - - pub fn value(&self) -> &'a Value { - self.0 + let object = self + .0 + .as_object() + .ok_or_else(|| format!("cannot deconstruct a jsonb {}", self.type_name()))?; + Ok(object.iter().map(|(k, v)| (k, Self(v)))) } } diff --git a/src/common/src/types/mod.rs b/src/common/src/types/mod.rs index 83d281c5238e6..386f63280a557 100644 --- a/src/common/src/types/mod.rs +++ b/src/common/src/types/mod.rs @@ -416,7 +416,7 @@ impl DataType { DataType::Timestamptz => ScalarImpl::Timestamptz(Timestamptz::MIN), DataType::Decimal => ScalarImpl::Decimal(Decimal::NegativeInf), DataType::Interval => ScalarImpl::Interval(Interval::MIN), - DataType::Jsonb => ScalarImpl::Jsonb(JsonbVal::dummy()), // NOT `min` #7981 + DataType::Jsonb => ScalarImpl::Jsonb(JsonbVal::null()), // NOT `min` #7981 DataType::Struct(data_types) => ScalarImpl::Struct(StructValue::new( data_types .types() @@ -1303,7 +1303,7 @@ mod tests { ScalarImpl::Interval(Interval::from_month_day_usec(2, 3, 3333)), DataType::Interval, ), - DataTypeName::Jsonb => (ScalarImpl::Jsonb(JsonbVal::dummy()), DataType::Jsonb), + DataTypeName::Jsonb => (ScalarImpl::Jsonb(JsonbVal::null()), DataType::Jsonb), DataTypeName::Struct => ( ScalarImpl::Struct(StructValue::new(vec![ ScalarImpl::Int64(233).into(), diff --git a/src/compute/tests/integration_tests.rs b/src/compute/tests/integration_tests.rs index 6d7e93365c275..078dfa05aa448 100644 --- a/src/compute/tests/integration_tests.rs +++ b/src/compute/tests/integration_tests.rs @@ -159,6 +159,7 @@ async fn test_table_materialize() -> StreamResult<()> { field_descs: vec![], type_name: "".to_string(), generated_or_default_column: None, + description: None, }) .collect_vec(); let (barrier_tx, barrier_rx) = unbounded_channel(); diff --git a/src/config/example.toml b/src/config/example.toml index 141078ddf8a54..3af89710d47c6 100644 --- a/src/config/example.toml +++ b/src/config/example.toml @@ -166,3 +166,4 @@ backup_storage_url = "memory" backup_storage_directory = "backup" max_concurrent_creating_streaming_jobs = 1 pause_on_next_bootstrap = false +wasm_storage_url = "fs://@/tmp/risingwave" diff --git a/src/connector/src/common.rs b/src/connector/src/common.rs index 2af396f5c33b4..c9a55dfe15086 100644 --- a/src/connector/src/common.rs +++ b/src/connector/src/common.rs @@ -417,8 +417,6 @@ pub struct UpsertMessage<'a> { pub struct NatsCommon { #[serde(rename = "server_url")] pub server_url: String, - #[serde(rename = "stream")] - pub stream: String, #[serde(rename = "subject")] pub subject: String, #[serde(rename = "connect_mode")] @@ -505,14 +503,18 @@ impl NatsCommon { pub(crate) async fn build_consumer( &self, + stream: String, split_id: String, start_sequence: NatsOffset, ) -> anyhow::Result< async_nats::jetstream::consumer::Consumer, > { let context = self.build_context().await?; - let stream = self.build_or_get_stream(context.clone()).await?; - let subject_name = self.subject.replace(',', "-"); + let stream = self.build_or_get_stream(context.clone(), stream).await?; + let subject_name = self + .subject + .replace(',', "-") + .replace(['.', '>', '*', ' ', '\t'], "_"); let name = format!("risingwave-consumer-{}-{}", subject_name, split_id); let mut config = jetstream::consumer::pull::Config { ack_policy: jetstream::consumer::AckPolicy::None, @@ -545,10 +547,11 @@ impl NatsCommon { pub(crate) async fn build_or_get_stream( &self, jetstream: jetstream::Context, + stream: String, ) -> anyhow::Result { let subjects: Vec = self.subject.split(',').map(|s| s.to_string()).collect(); let mut config = jetstream::stream::Config { - name: self.stream.clone(), + name: stream, max_bytes: 1000000, subjects, ..Default::default() diff --git a/src/connector/src/parser/avro/util.rs b/src/connector/src/parser/avro/util.rs index 917b23f1af6d0..e1b63962bf23c 100644 --- a/src/connector/src/parser/avro/util.rs +++ b/src/connector/src/parser/avro/util.rs @@ -57,6 +57,7 @@ fn avro_field_to_column_desc( field_descs: vec_column, type_name: schema_name.to_string(), generated_or_default_column: None, + description: None, }) } _ => { diff --git a/src/connector/src/parser/protobuf/parser.rs b/src/connector/src/parser/protobuf/parser.rs index 71d122470902e..0ca0235254000 100644 --- a/src/connector/src/parser/protobuf/parser.rs +++ b/src/connector/src/parser/protobuf/parser.rs @@ -199,6 +199,7 @@ impl ProtobufParserConfig { field_descs, type_name: m.full_name().to_string(), generated_or_default_column: None, + description: None, }) } else { *index += 1; diff --git a/src/connector/src/source/manager.rs b/src/connector/src/source/manager.rs index bdd72a090f9e8..8624550274299 100644 --- a/src/connector/src/source/manager.rs +++ b/src/connector/src/source/manager.rs @@ -123,6 +123,7 @@ impl From<&SourceColumnDesc> for ColumnDesc { field_descs: s.fields.clone(), type_name: "".to_string(), generated_or_default_column: None, + description: None, } } } diff --git a/src/connector/src/source/nats/mod.rs b/src/connector/src/source/nats/mod.rs index 3e8cc57bc1da8..f209086693fd3 100644 --- a/src/connector/src/source/nats/mod.rs +++ b/src/connector/src/source/nats/mod.rs @@ -35,6 +35,9 @@ pub struct NatsProperties { #[serde(rename = "scan.startup.timestamp_millis")] pub start_time: Option, + + #[serde(rename = "stream")] + pub stream: String, } impl NatsProperties {} diff --git a/src/connector/src/source/nats/source/reader.rs b/src/connector/src/source/nats/source/reader.rs index 6e22748bcf468..7f9a5718f95d4 100644 --- a/src/connector/src/source/nats/source/reader.rs +++ b/src/connector/src/source/nats/source/reader.rs @@ -77,7 +77,11 @@ impl SplitReader for NatsSplitReader { let consumer = properties .common - .build_consumer(split_id.to_string(), start_position.clone()) + .build_consumer( + properties.stream.clone(), + split_id.to_string(), + start_position.clone(), + ) .await?; Ok(Self { consumer, diff --git a/src/expr/core/Cargo.toml b/src/expr/core/Cargo.toml index ada9a3639525c..9801c2767c18b 100644 --- a/src/expr/core/Cargo.toml +++ b/src/expr/core/Cargo.toml @@ -44,6 +44,9 @@ risingwave_udf = { workspace = true } smallvec = "1" static_assertions = "1" thiserror = "1" +tokio = { version = "0.2", package = "madsim-tokio", features = [ + "rt-multi-thread", +] } tracing = "0.1" [target.'cfg(not(madsim))'.dependencies] diff --git a/src/expr/core/src/error.rs b/src/expr/core/src/error.rs index 7204b851e5b2b..ffb0e004c29e8 100644 --- a/src/expr/core/src/error.rs +++ b/src/expr/core/src/error.rs @@ -80,7 +80,9 @@ pub enum ExprError { Internal(#[from] anyhow::Error), #[error("UDF error: {0}")] - Udf(#[from] risingwave_udf::Error), + ExternalUdf(#[from] risingwave_udf::Error), + #[error("UDF error: {0}")] + WasmUdf(#[from] risingwave_udf::wasm::WasmUdfError), #[error("not a constant")] NotConstant, @@ -98,7 +100,7 @@ pub enum ExprError { InvalidState(String), } -static_assertions::const_assert_eq!(std::mem::size_of::(), 40); +static_assertions::const_assert_eq!(std::mem::size_of::(), 48); impl From for RwError { fn from(s: ExprError) -> Self { diff --git a/src/expr/core/src/expr/expr_udf.rs b/src/expr/core/src/expr/expr_udf.rs index a11af2434b4f9..9d694f4c4fda1 100644 --- a/src/expr/core/src/expr/expr_udf.rs +++ b/src/expr/core/src/expr/expr_udf.rs @@ -22,8 +22,11 @@ use cfg_or_panic::cfg_or_panic; use risingwave_common::array::{ArrayRef, DataChunk}; use risingwave_common::row::OwnedRow; use risingwave_common::types::{DataType, Datum}; -use risingwave_pb::expr::ExprNode; +use risingwave_pb::expr::user_defined_function::PbExtra; +use risingwave_pb::expr::{ExprNode, PbExternalUdfExtra, PbWasmUdfExtra}; +use risingwave_udf::wasm::{InstantiatedComponent, WasmEngine}; use risingwave_udf::ArrowFlightUdfClient; +use tracing::Instrument; use super::{BoxedExpression, Build}; use crate::expr::Expression; @@ -35,11 +38,36 @@ pub struct UdfExpression { arg_types: Vec, return_type: DataType, arg_schema: Arc, - client: Arc, - identifier: String, + imp: UdfImpl, span: await_tree::Span, } +enum UdfImpl { + External { + client: Arc, + identifier: String, + }, + Wasm { + component: InstantiatedComponent, + }, +} + +impl std::fmt::Debug for UdfImpl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::External { client, identifier } => f + .debug_struct("External") + .field("client", client) + .field("identifier", identifier) + .finish(), + Self::Wasm { component: _ } => f + .debug_struct("Wasm") + // .field("component", component) + .finish(), + } + } +} + #[async_trait::async_trait] impl Expression for UdfExpression { fn return_type(&self) -> DataType { @@ -98,11 +126,20 @@ impl UdfExpression { ) .expect("failed to build record batch"); - let output = self - .client - .call(&self.identifier, input) - .instrument_await(self.span.clone()) - .await?; + let output: arrow_array::RecordBatch = match &self.imp { + UdfImpl::Wasm { component } => { + component + .eval(input) + .instrument_await(self.span.clone()) + .await? + } + UdfImpl::External { client, identifier } => { + client + .call(identifier, input) + .instrument_await(self.span.clone()) + .await? + } + }; if output.num_rows() != vis.count_ones() { bail!( "UDF returned {} rows, but expected {}", @@ -110,7 +147,6 @@ impl UdfExpression { vis.len(), ); } - let data_chunk = DataChunk::try_from(&output).expect("failed to convert UDF output to DataChunk"); let output = data_chunk.uncompact(vis.clone()); @@ -139,8 +175,26 @@ impl Build for UdfExpression { let return_type = DataType::from(prost.get_return_type().unwrap()); let udf = prost.get_rex_node().unwrap().as_udf().unwrap(); - // connect to UDF service - let client = get_or_create_client(&udf.link)?; + let imp = match &udf.extra { + None | Some(PbExtra::External(PbExternalUdfExtra {})) => UdfImpl::External { + client: get_or_create_flight_client(&udf.link)?, + identifier: udf.identifier.clone(), + }, + Some(PbExtra::Wasm(PbWasmUdfExtra { wasm_storage_url })) => { + let wasm_engine = WasmEngine::get_or_create(); + // Use `block_in_place` as an escape hatch to run async code here in sync context. + // Calling `block_on` directly will panic. + let component = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on({ + wasm_engine + .load_component(wasm_storage_url, &udf.identifier) + .instrument(tracing::info_span!("load_component", %udf.identifier)) + }) + })?; + + UdfImpl::Wasm { component } + } + }; let arg_schema = Arc::new(Schema::new( udf.arg_types @@ -162,8 +216,7 @@ impl Build for UdfExpression { arg_types: udf.arg_types.iter().map(|t| t.into()).collect(), return_type, arg_schema, - client, - identifier: udf.identifier.clone(), + imp, span: format!("expr_udf_call ({})", udf.identifier).into(), }) } @@ -173,7 +226,7 @@ impl Build for UdfExpression { /// Get or create a client for the given UDF service. /// /// There is a global cache for clients, so that we can reuse the same client for the same service. -pub(crate) fn get_or_create_client(link: &str) -> Result> { +pub(crate) fn get_or_create_flight_client(link: &str) -> Result> { static CLIENTS: LazyLock>>> = LazyLock::new(Default::default); let mut clients = CLIENTS.lock().unwrap(); diff --git a/src/expr/core/src/table_function/user_defined.rs b/src/expr/core/src/table_function/user_defined.rs index 60fde34f9df1f..88d3252338fed 100644 --- a/src/expr/core/src/table_function/user_defined.rs +++ b/src/expr/core/src/table_function/user_defined.rs @@ -147,7 +147,7 @@ pub fn new_user_defined(prost: &PbTableFunction, chunk_size: usize) -> Result()?, )); // connect to UDF service - let client = crate::expr::expr_udf::get_or_create_client(&udtf.link)?; + let client = crate::expr::expr_udf::get_or_create_flight_client(&udtf.link)?; Ok(UserDefinedTableFunction { children: prost.args.iter().map(expr_build_from_prost).try_collect()?, diff --git a/src/expr/impl/Cargo.toml b/src/expr/impl/Cargo.toml index 81cd685c4dc27..cc0229f83ebab 100644 --- a/src/expr/impl/Cargo.toml +++ b/src/expr/impl/Cargo.toml @@ -29,6 +29,7 @@ futures-async-stream = { workspace = true } futures-util = "0.3" hex = "0.4" itertools = "0.11" +jsonbb = "0.1" md5 = "0.7" num-traits = "0.2" regex = "1" diff --git a/src/expr/impl/benches/expr.rs b/src/expr/impl/benches/expr.rs index 1e84d8d8e4825..010508c8de45e 100644 --- a/src/expr/impl/benches/expr.rs +++ b/src/expr/impl/benches/expr.rs @@ -170,10 +170,7 @@ fn bench_expr(c: &mut Criterion) { // 25: serial array SerialArray::from_iter((1..=CHUNK_SIZE).map(|i| Serial::from(i as i64))).into_ref(), // 26: jsonb array - JsonbArray::from_iter( - (1..=CHUNK_SIZE).map(|i| JsonbVal::from(serde_json::Value::Number(i.into()))), - ) - .into_ref(), + JsonbArray::from_iter((1..=CHUNK_SIZE).map(|i| JsonbVal::from(i as i64))).into_ref(), // 27: int256 array Int256Array::from_iter((1..=CHUNK_SIZE).map(|_| Int256::from(1))).into_ref(), // 28: extract field for interval @@ -279,16 +276,16 @@ fn bench_expr(c: &mut Criterion) { 'sig: for sig in sigs { if (sig.inputs_type.iter()) .chain([&sig.ret_type]) - .any(|t| !t.is_exact()) + .any(|t| !t.is_exact() || t.as_exact().is_array()) { - // TODO: support struct and list + // TODO: support struct and array println!("todo: {sig:?}"); continue; } if [ - "date_trunc(varchar, timestamptz) -> timestamptz", - "to_timestamp1(varchar, varchar) -> timestamptz", - "to_char(timestamptz, varchar) -> varchar", + "date_trunc(character varying, timestamp with time zone) -> timestamp with time zone", + "to_timestamp1(character varying, character varying) -> timestamp with time zone", + "to_char(timestamp with time zone, character varying) -> character varying", ] .contains(&format!("{sig:?}").as_str()) { @@ -376,6 +373,13 @@ fn bench_expr(c: &mut Criterion) { args: match sig.inputs_type.as_slice() { [] => AggArgs::None, [t] => AggArgs::Unary(t.as_exact().clone(), input_index_for_type(t.as_exact())), + [t1, t2] => AggArgs::Binary( + [t1.as_exact().clone(), t2.as_exact().clone()], + [ + input_index_for_type(t1.as_exact()), + input_index_for_type(t2.as_exact()), + ], + ), _ => { println!("todo: {sig:?}"); continue; @@ -393,6 +397,15 @@ fn bench_expr(c: &mut Criterion) { continue; } }; + let input = match sig.inputs_type.as_slice() { + [] => input.project(&[]), + [t] => input.project(&[input_index_for_type(t.as_exact())]), + [t1, t2] => input.project(&[ + input_index_for_type(t1.as_exact()), + input_index_for_type(t2.as_exact()), + ]), + _ => unreachable!(), + }; c.bench_function(&format!("{sig:?}"), |bencher| { bencher .to_async(FuturesExecutor) diff --git a/src/expr/impl/src/aggregate/jsonb_agg.rs b/src/expr/impl/src/aggregate/jsonb_agg.rs index 8385e2c6a060b..96f5e50da85e3 100644 --- a/src/expr/impl/src/aggregate/jsonb_agg.rs +++ b/src/expr/impl/src/aggregate/jsonb_agg.rs @@ -12,22 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::types::JsonbVal; +use risingwave_common::estimate_size::EstimateSize; +use risingwave_common::types::{JsonbRef, JsonbVal, ScalarImpl, F32, F64}; +use risingwave_expr::aggregate::AggStateDyn; use risingwave_expr::{aggregate, ExprError, Result}; -use serde_json::Value; #[aggregate("jsonb_agg(boolean) -> jsonb")] #[aggregate("jsonb_agg(*int) -> jsonb")] #[aggregate("jsonb_agg(*float) -> jsonb")] #[aggregate("jsonb_agg(varchar) -> jsonb")] #[aggregate("jsonb_agg(jsonb) -> jsonb")] -fn jsonb_agg(state: Option, input: Option>) -> JsonbVal { - let mut jsonb = state.unwrap_or_else(|| Value::Array(Vec::with_capacity(1)).into()); - match jsonb.as_serde_mut() { - Value::Array(a) => a.push(input.map_or(Value::Null, Into::into)), - _ => unreachable!("invalid jsonb state"), - }; - jsonb +fn jsonb_agg(state: &mut JsonbArrayState, input: Option) { + match input { + Some(input) => input.add_to(&mut state.0), + None => state.0.add_null(), + } } #[aggregate("jsonb_object_agg(varchar, boolean) -> jsonb")] @@ -36,15 +35,130 @@ fn jsonb_agg(state: Option, input: Option>) -> JsonbV #[aggregate("jsonb_object_agg(varchar, varchar) -> jsonb")] #[aggregate("jsonb_object_agg(varchar, jsonb) -> jsonb")] fn jsonb_object_agg( - state: Option, + state: &mut JsonbObjectState, key: Option<&str>, - value: Option>, -) -> Result { + value: Option, +) -> Result<()> { let key = key.ok_or(ExprError::FieldNameNull)?; - let mut jsonb = state.unwrap_or_else(|| Value::Object(Default::default()).into()); - match jsonb.as_serde_mut() { - Value::Object(map) => map.insert(key.into(), value.map_or(Value::Null, Into::into)), - _ => unreachable!("invalid jsonb state"), - }; - Ok(jsonb) + state.0.add_string(key); + match value { + Some(value) => value.add_to(&mut state.0), + None => state.0.add_null(), + } + Ok(()) +} + +#[derive(Debug)] +struct JsonbArrayState(jsonbb::Builder); + +impl EstimateSize for JsonbArrayState { + fn estimated_heap_size(&self) -> usize { + self.0.capacity() + } +} + +impl AggStateDyn for JsonbArrayState {} + +/// Creates an initial state. +impl Default for JsonbArrayState { + fn default() -> Self { + let mut builder = jsonbb::Builder::default(); + builder.begin_array(); + Self(builder) + } +} + +/// Finishes aggregation and returns the result. +impl From<&JsonbArrayState> for ScalarImpl { + fn from(builder: &JsonbArrayState) -> Self { + // TODO: avoid clone + let mut builder = builder.0.clone(); + builder.end_array(); + let jsonb: JsonbVal = builder.finish().into(); + jsonb.into() + } +} + +#[derive(Debug)] +struct JsonbObjectState(jsonbb::Builder); + +impl EstimateSize for JsonbObjectState { + fn estimated_heap_size(&self) -> usize { + self.0.capacity() + } +} + +impl AggStateDyn for JsonbObjectState {} + +/// Creates an initial state. +impl Default for JsonbObjectState { + fn default() -> Self { + let mut builder = jsonbb::Builder::default(); + builder.begin_object(); + Self(builder) + } +} + +/// Finishes aggregation and returns the result. +impl From<&JsonbObjectState> for ScalarImpl { + fn from(builder: &JsonbObjectState) -> Self { + // TODO: avoid clone + let mut builder = builder.0.clone(); + builder.end_object(); + let jsonb: JsonbVal = builder.finish().into(); + jsonb.into() + } +} + +/// Values that can be converted to JSON. +trait ToJson { + fn add_to(self, builder: &mut jsonbb::Builder); +} + +impl ToJson for bool { + fn add_to(self, builder: &mut jsonbb::Builder) { + builder.add_bool(self); + } +} + +impl ToJson for i16 { + fn add_to(self, builder: &mut jsonbb::Builder) { + builder.add_i64(self as _); + } +} + +impl ToJson for i32 { + fn add_to(self, builder: &mut jsonbb::Builder) { + builder.add_i64(self as _); + } +} + +impl ToJson for i64 { + fn add_to(self, builder: &mut jsonbb::Builder) { + builder.add_i64(self); + } +} + +impl ToJson for F32 { + fn add_to(self, builder: &mut jsonbb::Builder) { + builder.add_f64(self.0 as f64); + } +} + +impl ToJson for F64 { + fn add_to(self, builder: &mut jsonbb::Builder) { + builder.add_f64(self.0); + } +} + +impl ToJson for &str { + fn add_to(self, builder: &mut jsonbb::Builder) { + builder.add_string(self); + } +} + +impl ToJson for JsonbRef<'_> { + fn add_to(self, builder: &mut jsonbb::Builder) { + builder.add_value(self.into()); + } } diff --git a/src/expr/impl/src/scalar/jsonb_concat.rs b/src/expr/impl/src/scalar/jsonb_concat.rs index 6277db8d5b981..db469457bb135 100644 --- a/src/expr/impl/src/scalar/jsonb_concat.rs +++ b/src/expr/impl/src/scalar/jsonb_concat.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use jsonbb::{Value, ValueRef}; use risingwave_common::types::{JsonbRef, JsonbVal}; use risingwave_expr::function; -use serde_json::{json, Value}; /// Concatenates the two jsonbs. /// @@ -59,43 +59,35 @@ use serde_json::{json, Value}; /// ``` #[function("jsonb_cat(jsonb, jsonb) -> jsonb")] pub fn jsonb_cat(left: JsonbRef<'_>, right: JsonbRef<'_>) -> JsonbVal { - let left_val = left.value().clone(); - let right_val = right.value().clone(); - match (left_val, right_val) { + match (left.into(), right.into()) { // left and right are object based. // This would have left:{'a':1}, right:{'b':2} -> {'a':1,'b':2} - (Value::Object(mut left_map), Value::Object(right_map)) => { - left_map.extend(right_map); - JsonbVal::from(Value::Object(left_map)) + (ValueRef::Object(left), ValueRef::Object(right)) => { + JsonbVal::from(Value::object(left.iter().chain(right.iter()))) } // left and right are array-based. // This would merge both arrays into one array. // This would have left:[1,2], right:[3,4] -> [1,2,3,4] - (Value::Array(mut left_arr), Value::Array(right_arr)) => { - left_arr.extend(right_arr); - JsonbVal::from(Value::Array(left_arr)) + (ValueRef::Array(left), ValueRef::Array(right)) => { + JsonbVal::from(Value::array(left.iter().chain(right.iter()))) } // One operand is an array, and the other is a single element. // This would insert the non-array value as another element into the array // Eg left:[1,2] right: {'a':1} -> [1,2,{'a':1}] - (Value::Array(mut left_arr), single_val) => { - left_arr.push(single_val); - JsonbVal::from(Value::Array(left_arr)) - } + (ValueRef::Array(left), value) => JsonbVal::from(Value::array(left.iter().chain([value]))), // One operand is an array, and the other is a single element. // This would insert the non-array value as another element into the array // Eg left:{'a':1} right:[1,2] -> [{'a':1},1,2] - (single_val, Value::Array(mut right_arr)) => { - right_arr.insert(0, single_val); - JsonbVal::from(Value::Array(right_arr)) + (value, ValueRef::Array(right)) => { + JsonbVal::from(Value::array([value].into_iter().chain(right.iter()))) } // Both are non-array inputs. // Both elements would be placed together in an array // Eg left:1 right: 2 -> [1,2] - (left, right) => JsonbVal::from(json!([left, right])), + (left, right) => JsonbVal::from(Value::array([left, right])), } } diff --git a/src/expr/macro/src/gen.rs b/src/expr/macro/src/gen.rs index 9155853df5b7b..454d2a3169137 100644 --- a/src/expr/macro/src/gen.rs +++ b/src/expr/macro/src/gen.rs @@ -579,9 +579,13 @@ impl FunctionAttr { /// Generate build function for aggregate function. fn generate_agg_build_fn(&self, user_fn: &AggregateFnOrImpl) -> Result { - let state_type: TokenStream2 = match &self.state { - Some(state) if state == "ref" => types::ref_type(&self.ret).parse().unwrap(), - Some(state) if state != "ref" => types::owned_type(state).parse().unwrap(), + // If the first argument of the aggregate function is of type `&mut T`, + // we assume it is a user defined state type. + let custom_state = user_fn.accumulate().first_mut_ref_arg.as_ref(); + let state_type: TokenStream2 = match (custom_state, &self.state) { + (Some(s), _) => s.parse().unwrap(), + (_, Some(state)) if state == "ref" => types::ref_type(&self.ret).parse().unwrap(), + (_, Some(state)) if state != "ref" => types::owned_type(state).parse().unwrap(), _ => types::owned_type(&self.ret).parse().unwrap(), }; let let_arrays = self @@ -603,24 +607,37 @@ impl FunctionAttr { quote! { let #v = unsafe { #a.value_at_unchecked(row_id) }; } }) .collect_vec(); - let let_state = match &self.state { - Some(s) if s == "ref" => { - quote! { state0.as_ref().map(|x| x.as_scalar_ref_impl().try_into().unwrap()) } - } - _ => quote! { state0.take().map(|s| s.try_into().unwrap()) }, + let downcast_state = if custom_state.is_some() { + quote! { let mut state: &mut #state_type = state0.downcast_mut(); } + } else if let Some(s) = &self.state && s == "ref" { + quote! { let mut state: Option<#state_type> = state0.as_datum_mut().as_ref().map(|x| x.as_scalar_ref_impl().try_into().unwrap()); } + } else { + quote! { let mut state: Option<#state_type> = state0.as_datum_mut().take().map(|s| s.try_into().unwrap()); } }; - let assign_state = match &self.state { - Some(s) if s == "ref" => quote! { state.map(|x| x.to_owned_scalar().into()) }, - _ => quote! { state.map(|s| s.into()) }, + let restore_state = if custom_state.is_some() { + quote! {} + } else if let Some(s) = &self.state && s == "ref" { + quote! { *state0.as_datum_mut() = state.map(|x| x.to_owned_scalar().into()); } + } else { + quote! { *state0.as_datum_mut() = state.map(|s| s.into()); } }; - let create_state = self.init_state.as_ref().map(|state| { + let create_state = if custom_state.is_some() { + quote! { + fn create_state(&self) -> AggregateState { + AggregateState::Any(Box::<#state_type>::default()) + } + } + } else if let Some(state) = &self.init_state { let state: TokenStream2 = state.parse().unwrap(); quote! { fn create_state(&self) -> AggregateState { AggregateState::Datum(Some(#state.into())) } } - }); + } else { + // by default: `AggregateState::Datum(None)` + quote! {} + }; let args = (0..self.args.len()).map(|i| format_ident!("v{i}")); let args = quote! { #(#args,)* }; let panic_on_retract = { @@ -703,17 +720,23 @@ impl FunctionAttr { _ => todo!("multiple arguments are not supported for non-option function"), } } - let get_result = match user_fn { - AggregateFnOrImpl::Impl(impl_) if impl_.finalize.is_some() => { - quote! { - let state = match state { - Some(s) => s.as_scalar_ref_impl().try_into().unwrap(), - None => return Ok(None), - }; - Ok(Some(self.function.finalize(state).into())) - } + let update_state = if custom_state.is_some() { + quote! { _ = #next_state; } + } else { + quote! { state = #next_state; } + }; + let get_result = if custom_state.is_some() { + quote! { Ok(Some(state.downcast_ref::<#state_type>().into())) } + } else if let AggregateFnOrImpl::Impl(impl_) = user_fn && impl_.finalize.is_some() { + quote! { + let state = match state.as_datum() { + Some(s) => s.as_scalar_ref_impl().try_into().unwrap(), + None => return Ok(None), + }; + Ok(Some(self.function.finalize(state).into())) } - _ => quote! { Ok(state.clone()) }, + } else { + quote! { Ok(state.as_datum().clone()) } }; let function_field = match user_fn { AggregateFnOrImpl::Fn(_) => quote! {}, @@ -768,27 +791,25 @@ impl FunctionAttr { async fn update(&self, state0: &mut AggregateState, input: &StreamChunk) -> Result<()> { #(#let_arrays)* - let state0 = state0.as_datum_mut(); - let mut state: Option<#state_type> = #let_state; + #downcast_state for row_id in input.visibility().iter_ones() { let op = unsafe { *input.ops().get_unchecked(row_id) }; #(#let_values)* - state = #next_state; + #update_state } - *state0 = #assign_state; + #restore_state Ok(()) } async fn update_range(&self, state0: &mut AggregateState, input: &StreamChunk, range: Range) -> Result<()> { assert!(range.end <= input.capacity()); #(#let_arrays)* - let state0 = state0.as_datum_mut(); - let mut state: Option<#state_type> = #let_state; + #downcast_state if input.is_compacted() { for row_id in range { let op = unsafe { *input.ops().get_unchecked(row_id) }; #(#let_values)* - state = #next_state; + #update_state } } else { for row_id in input.visibility().iter_ones() { @@ -799,15 +820,14 @@ impl FunctionAttr { } let op = unsafe { *input.ops().get_unchecked(row_id) }; #(#let_values)* - state = #next_state; + #update_state } } - *state0 = #assign_state; + #restore_state Ok(()) } async fn get_result(&self, state: &AggregateState) -> Result { - let state = state.as_datum(); #get_result } } diff --git a/src/expr/macro/src/lib.rs b/src/expr/macro/src/lib.rs index 363fc958b557d..50a99cf3fda22 100644 --- a/src/expr/macro/src/lib.rs +++ b/src/expr/macro/src/lib.rs @@ -522,6 +522,8 @@ struct UserFunctionAttr { retract: bool, /// The argument type are `Option`s. arg_option: bool, + /// If the first argument type is `&mut T`, then `Some(T)`. + first_mut_ref_arg: Option, /// The return type kind. return_type_kind: ReturnTypeKind, /// The kind of inner type `T` in `impl Iterator` diff --git a/src/expr/macro/src/parse.rs b/src/expr/macro/src/parse.rs index 24cc6942afcee..8e2e8c6d0b2f1 100644 --- a/src/expr/macro/src/parse.rs +++ b/src/expr/macro/src/parse.rs @@ -123,6 +123,7 @@ impl From<&syn::Signature> for UserFunctionAttr { context: sig.inputs.iter().any(arg_is_context), retract: last_arg_is_retract(sig), arg_option: args_contain_option(sig), + first_mut_ref_arg: first_mut_ref_arg(sig), return_type_kind, iterator_item_kind, core_return_type, @@ -223,18 +224,15 @@ fn last_arg_is_retract(sig: &syn::Signature) -> bool { /// Check if any argument is `Option`. fn args_contain_option(sig: &syn::Signature) -> bool { - if sig.inputs.is_empty() { - return false; - } for arg in &sig.inputs { let syn::FnArg::Typed(arg) = arg else { - return false; + continue; }; let syn::Type::Path(path) = arg.ty.as_ref() else { - return false; + continue; }; let Some(seg) = path.path.segments.last() else { - return false; + continue; }; if seg.ident == "Option" { return true; @@ -243,6 +241,26 @@ fn args_contain_option(sig: &syn::Signature) -> bool { false } +/// Returns `T` if the first argument (except `self`) is `&mut T`. +fn first_mut_ref_arg(sig: &syn::Signature) -> Option { + let arg = match sig.inputs.first()? { + syn::FnArg::Typed(arg) => arg, + syn::FnArg::Receiver(_) => match sig.inputs.iter().nth(1)? { + syn::FnArg::Typed(arg) => arg, + _ => return None, + }, + }; + let syn::Type::Reference(syn::TypeReference { + elem, + mutability: Some(_), + .. + }) = arg.ty.as_ref() + else { + return None; + }; + Some(elem.to_token_stream().to_string()) +} + /// Check the return type. fn check_type(ty: &syn::Type) -> (ReturnTypeKind, &syn::Type) { if let Some(inner) = strip_outer_type(ty, "Result") { diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml index 37f9f6326faea..513f81234a3ce 100644 --- a/src/frontend/Cargo.toml +++ b/src/frontend/Cargo.toml @@ -21,6 +21,7 @@ arrow-schema = { workspace = true } async-recursion = "1.0.5" async-trait = "0.1" auto_enums = { version = "0.8", features = ["futures03"] } +base64 = "0.21" bk-tree = "0.5.0" bytes = "1" clap = { version = "4", features = ["derive"] } diff --git a/src/frontend/src/binder/expr/mod.rs b/src/frontend/src/binder/expr/mod.rs index 6da590c2d315d..221056f3a4822 100644 --- a/src/frontend/src/binder/expr/mod.rs +++ b/src/frontend/src/binder/expr/mod.rs @@ -576,6 +576,7 @@ pub fn bind_struct_field(column_def: &StructField) -> Result { field_descs: vec![], type_name: "".to_string(), generated_or_default_column: None, + description: None, }) }) .collect::>>()? @@ -589,6 +590,7 @@ pub fn bind_struct_field(column_def: &StructField) -> Result { field_descs, type_name: "".to_string(), generated_or_default_column: None, + description: None, }) } diff --git a/src/frontend/src/catalog/catalog_service.rs b/src/frontend/src/catalog/catalog_service.rs index 8eb6b9e3e4485..be85293acd27f 100644 --- a/src/frontend/src/catalog/catalog_service.rs +++ b/src/frontend/src/catalog/catalog_service.rs @@ -21,7 +21,8 @@ use risingwave_common::error::ErrorCode::InternalError; use risingwave_common::error::{Result, RwError}; use risingwave_common::util::column_index_mapping::ColIndexMapping; use risingwave_pb::catalog::{ - PbCreateType, PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbTable, PbView, + PbComment, PbCreateType, PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbTable, + PbView, }; use risingwave_pb::ddl_service::alter_relation_name_request::Relation; use risingwave_pb::ddl_service::create_connection_request; @@ -111,6 +112,8 @@ pub trait CatalogWriter: Send + Sync { connection: create_connection_request::Payload, ) -> Result<()>; + async fn comment_on(&self, comment: PbComment) -> Result<()>; + async fn drop_table( &self, source_id: Option, @@ -282,6 +285,11 @@ impl CatalogWriter for CatalogWriterImpl { self.wait_version(version).await } + async fn comment_on(&self, comment: PbComment) -> Result<()> { + let version = self.meta_client.comment_on(comment).await?; + self.wait_version(version).await + } + async fn drop_table( &self, source_id: Option, diff --git a/src/frontend/src/catalog/function_catalog.rs b/src/frontend/src/catalog/function_catalog.rs index e56a36d85f8c2..492b017f19621 100644 --- a/src/frontend/src/catalog/function_catalog.rs +++ b/src/frontend/src/catalog/function_catalog.rs @@ -17,6 +17,7 @@ use risingwave_common::catalog::FunctionId; use risingwave_common::types::DataType; use risingwave_pb::catalog::function::PbKind; use risingwave_pb::catalog::PbFunction; +use risingwave_pb::expr::user_defined_function::PbExtra; use crate::catalog::OwnedByUserCatalog; @@ -31,6 +32,8 @@ pub struct FunctionCatalog { pub language: String, pub identifier: String, pub link: String, + // for backward compatibility, newly added fields should be optional + pub extra: Option, } #[derive(Clone, Display, PartialEq, Eq, Hash, Debug)] @@ -64,6 +67,7 @@ impl From<&PbFunction> for FunctionCatalog { language: prost.language.clone(), identifier: prost.identifier.clone(), link: prost.link.clone(), + extra: prost.extra.clone().map(Into::into), } } } diff --git a/src/frontend/src/catalog/system_catalog/mod.rs b/src/frontend/src/catalog/system_catalog/mod.rs index 4cd271f0495b9..d64db79b8ced1 100644 --- a/src/frontend/src/catalog/system_catalog/mod.rs +++ b/src/frontend/src/catalog/system_catalog/mod.rs @@ -57,6 +57,9 @@ pub struct SystemTableCatalog { // owner of table, should always be default super user, keep it for compatibility. pub owner: u32, + + /// description of table, set by `comment on`. + pub description: Option, } impl SystemTableCatalog { @@ -165,6 +168,7 @@ impl From<&BuiltinTable> for SystemTableCatalog { .collect(), pk: val.pk.to_vec(), owner: DEFAULT_SUPER_USER_ID, + description: None, } } } @@ -412,6 +416,7 @@ prepare_sys_catalog! { { BuiltinCatalog::Table(&RW_HUMMOCK_BRANCHED_OBJECTS), read_hummock_branched_objects await }, { BuiltinCatalog::Table(&RW_HUMMOCK_COMPACTION_GROUP_CONFIGS), read_hummock_compaction_group_configs await }, { BuiltinCatalog::Table(&RW_HUMMOCK_META_CONFIGS), read_hummock_meta_configs await}, + { BuiltinCatalog::Table(&RW_DESCRIPTION), read_rw_description }, } #[cfg(test)] diff --git a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_description.rs b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_description.rs index 8bc91f4a10bb9..593522ceda705 100644 --- a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_description.rs +++ b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_description.rs @@ -21,22 +21,25 @@ use crate::catalog::system_catalog::BuiltinView; /// The catalog `pg_description` stores description. /// Ref: [`https://www.postgresql.org/docs/current/catalog-pg-description.html`] -pub static PG_DESCRIPTION: LazyLock = LazyLock::new(|| { - BuiltinView { - name: "pg_description", - schema: PG_CATALOG_SCHEMA_NAME, - columns: &[ - (DataType::Int32, "objoid"), - (DataType::Int32, "classoid"), - (DataType::Int32, "objsubid"), - (DataType::Varchar, "description"), - ], - sql: "SELECT id AS objoid, NULL::integer AS classoid, 0 AS objsubid, NULL AS description FROM rw_catalog.rw_tables \ - UNION ALL SELECT id AS objoid, NULL::integer AS classoid, 0 AS objsubid, NULL AS description FROM rw_catalog.rw_materialized_views \ - UNION ALL SELECT id AS objoid, NULL::integer AS classoid, 0 AS objsubid, NULL AS description FROM rw_catalog.rw_views \ - UNION ALL SELECT id AS objoid, NULL::integer AS classoid, 0 AS objsubid, NULL AS description FROM rw_catalog.rw_indexes \ - UNION ALL SELECT id AS objoid, NULL::integer AS classoid, 0 AS objsubid, NULL AS description FROM rw_catalog.rw_sources \ - UNION ALL SELECT id AS objoid, NULL::integer AS classoid, 0 AS objsubid, NULL AS description FROM rw_catalog.rw_system_tables\ - ".into(), - } +pub static PG_DESCRIPTION: LazyLock = LazyLock::new(|| BuiltinView { + name: "pg_description", + schema: PG_CATALOG_SCHEMA_NAME, + columns: &[ + (DataType::Int32, "objoid"), + (DataType::Int32, "classoid"), + (DataType::Int32, "objsubid"), + (DataType::Varchar, "description"), + ], + // objsubid = 0 => _row_id (hidden column) + // objsubid is NULL => table self + sql: "SELECT objoid, \ + classoid, \ + CASE \ + WHEN objsubid = 0 THEN -1 \ + WHEN objsubid IS NULL THEN 0 \ + ELSE objsubid \ + END AS objsubid, \ + description FROM rw_catalog.rw_description \ + WHERE description IS NOT NULL;" + .into(), }); diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/mod.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/mod.rs index 9f89c9eed5e81..517f40b460e94 100644 --- a/src/frontend/src/catalog/system_catalog/rw_catalog/mod.rs +++ b/src/frontend/src/catalog/system_catalog/rw_catalog/mod.rs @@ -17,6 +17,7 @@ mod rw_columns; mod rw_connections; mod rw_databases; mod rw_ddl_progress; +mod rw_description; mod rw_fragments; mod rw_functions; mod rw_hummock_branched_objects; @@ -50,6 +51,7 @@ pub use rw_columns::*; pub use rw_connections::*; pub use rw_databases::*; pub use rw_ddl_progress::*; +pub use rw_description::*; pub use rw_fragments::*; pub use rw_functions::*; pub use rw_hummock_branched_objects::*; diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_description.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_description.rs new file mode 100644 index 0000000000000..370dec33a2a2a --- /dev/null +++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_description.rs @@ -0,0 +1,84 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::iter; + +use risingwave_common::catalog::RW_CATALOG_SCHEMA_NAME; +use risingwave_common::error::Result; +use risingwave_common::row::OwnedRow; +use risingwave_common::types::{DataType, ScalarImpl}; + +use crate::catalog::system_catalog::{BuiltinTable, SysCatalogReaderImpl}; + +pub const RW_DESCRIPTION: BuiltinTable = BuiltinTable { + name: "rw_description", + schema: RW_CATALOG_SCHEMA_NAME, + columns: &[ + // table_id, view_id, function_id, etc. + (DataType::Int32, "objoid"), + // rw_tables, rw_views, rw_functions, etc. + (DataType::Int32, "classoid"), + // If `objoid` is `table_id`, then non-null `objsubid` is column number. + (DataType::Int32, "objsubid"), + (DataType::Varchar, "description"), + ], + pk: &[0, 1, 2], +}; + +impl SysCatalogReaderImpl { + pub fn read_rw_description(&self) -> Result> { + let build_row = + |table_id, catalog_id, index: Option, description: Option>| { + OwnedRow::new(vec![ + Some(ScalarImpl::Int32(table_id)), + Some(ScalarImpl::Int32(catalog_id)), + index.map(ScalarImpl::Int32), + description.map(ScalarImpl::Utf8), + ]) + }; + + let reader = self.catalog_reader.read_guard(); + let rw_catalog = + reader.get_schema_by_name(&self.auth_context.database, RW_CATALOG_SCHEMA_NAME)?; + let schemas = reader + .iter_schemas(&self.auth_context.database)? + .filter(|schema| schema.id() != rw_catalog.id()); + + let rw_tables_id: i32 = rw_catalog + .get_system_table_by_name("rw_tables") + .map(|st| st.id.table_id) + .unwrap_or_default() as _; + + Ok(schemas + .flat_map(|schema| { + schema.iter_table().flat_map(|table| { + iter::once(build_row( + table.id.table_id as _, + rw_tables_id, + None, + table.description.as_deref().map(Into::into), + )) + .chain(table.columns.iter().map(|col| { + build_row( + table.id.table_id as _, + rw_tables_id, + Some(col.column_id().get_id() as _), + col.column_desc.description.as_deref().map(Into::into), + ) + })) + }) + }) + .collect()) + } +} diff --git a/src/frontend/src/catalog/table_catalog.rs b/src/frontend/src/catalog/table_catalog.rs index b0f9088132f59..750a06da7d231 100644 --- a/src/frontend/src/catalog/table_catalog.rs +++ b/src/frontend/src/catalog/table_catalog.rs @@ -152,6 +152,9 @@ pub struct TableCatalog { /// Indicate whether to create table in background or foreground. pub create_type: CreateType, + + /// description of table, set by `comment on`. + pub description: Option, } // How the stream job was created will determine @@ -438,6 +441,7 @@ impl TableCatalog { cleaned_by_watermark: self.cleaned_by_watermark, stream_job_status: PbStreamJobStatus::Creating.into(), create_type: self.create_type.to_prost().into(), + description: self.description.clone(), } } @@ -551,6 +555,7 @@ impl From for TableCatalog { initialized_at_epoch: tb.initialized_at_epoch.map(Epoch::from), cleaned_by_watermark: matches!(tb.cleaned_by_watermark, true), create_type: CreateType::from_prost(create_type), + description: tb.description, } } } @@ -643,6 +648,7 @@ mod tests { cleaned_by_watermark: false, stream_job_status: PbStreamJobStatus::Creating.into(), create_type: PbCreateType::Foreground.into(), + description: Some("description".to_string()), } .into(); @@ -668,6 +674,7 @@ mod tests { ColumnDesc::new_atomic(DataType::Varchar, "zipcode", 3), ], type_name: ".test.Country".to_string(), + description: None, generated_or_default_column: None, }, is_hidden: false @@ -698,6 +705,7 @@ mod tests { initialized_at_epoch: None, cleaned_by_watermark: false, create_type: CreateType::Foreground, + description: Some("description".to_string()) } ); assert_eq!(table, TableCatalog::from(table.to_prost(0, 0))); diff --git a/src/frontend/src/expr/user_defined_function.rs b/src/frontend/src/expr/user_defined_function.rs index 1c9d06320ba15..131a5e34e1531 100644 --- a/src/frontend/src/expr/user_defined_function.rs +++ b/src/frontend/src/expr/user_defined_function.rs @@ -56,6 +56,7 @@ impl UserDefinedFunction { language: udf.get_language().clone(), identifier: udf.get_identifier().clone(), link: udf.get_link().clone(), + extra: udf.extra.clone(), }; Ok(Self { @@ -88,6 +89,8 @@ impl Expr for UserDefinedFunction { language: self.catalog.language.clone(), identifier: self.catalog.identifier.clone(), link: self.catalog.link.clone(), + + extra: self.catalog.extra.clone(), })), } } diff --git a/src/frontend/src/handler/comment.rs b/src/frontend/src/handler/comment.rs new file mode 100644 index 0000000000000..b0ff42a790346 --- /dev/null +++ b/src/frontend/src/handler/comment.rs @@ -0,0 +1,87 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use pgwire::pg_response::{PgResponse, StatementType}; +use risingwave_common::error::{ErrorCode, Result}; +use risingwave_pb::catalog::PbComment; +use risingwave_sqlparser::ast::{CommentObject, ObjectName}; + +use super::{HandlerArgs, RwPgResponse}; +use crate::Binder; + +pub async fn handle_comment( + handler_args: HandlerArgs, + object_type: CommentObject, + object_name: ObjectName, + comment: Option, +) -> Result { + let session = handler_args.session; + let comment = comment.filter(|s| !s.is_empty()); + + let comment = { + let mut binder = Binder::new_for_ddl(&session); + // only `Column` and `Table` object are now supported + match object_type { + CommentObject::Column => { + let [tab @ .., col] = object_name.0.as_slice() else { + return Err(ErrorCode::BindError(format!( + "Invalid column: {}", + object_name.real_value() + )) + .into()); + }; + + let (schema, table) = Binder::resolve_schema_qualified_name( + session.database(), + ObjectName(tab.to_vec()), + )?; + + let (database_id, schema_id) = + session.get_database_and_schema_id_for_create(schema)?; + let table = binder.bind_table(None, &table, None)?; + binder.bind_columns_to_context(col.real_value(), table.table_catalog.columns)?; + + let column = binder.bind_column(object_name.0.as_slice())?; + + PbComment { + table_id: table.table_id.into(), + schema_id, + database_id, + column_index: column.as_input_ref().map(|input_ref| input_ref.index as _), + description: comment, + } + } + CommentObject::Table => { + let (schema, table) = + Binder::resolve_schema_qualified_name(session.database(), object_name)?; + let (database_id, schema_id) = + session.get_database_and_schema_id_for_create(schema)?; + let table = binder.bind_table(None, &table, None)?; + + PbComment { + table_id: table.table_id.into(), + schema_id, + database_id, + column_index: None, + description: comment, + } + } + } + }; + + let catalog_writer = session.catalog_writer()?; + catalog_writer.comment_on(comment).await?; + + Ok(PgResponse::empty_result(StatementType::COMMENT)) +} diff --git a/src/frontend/src/handler/create_function.rs b/src/frontend/src/handler/create_function.rs index 9d9db08204e49..a57b9d26ddf59 100644 --- a/src/frontend/src/handler/create_function.rs +++ b/src/frontend/src/handler/create_function.rs @@ -18,12 +18,15 @@ use itertools::Itertools; use pgwire::pg_response::StatementType; use risingwave_common::catalog::FunctionId; use risingwave_common::types::DataType; -use risingwave_pb::catalog::function::{Kind, ScalarFunction, TableFunction}; +use risingwave_pb::catalog::function::{Kind, PbExtra, ScalarFunction, TableFunction}; use risingwave_pb::catalog::Function; +use risingwave_pb::expr::{PbExternalUdfExtra, PbWasmUdfExtra}; use risingwave_sqlparser::ast::{ CreateFunctionBody, FunctionDefinition, ObjectName, OperateFunctionArg, }; +use risingwave_udf::wasm::WasmEngine; use risingwave_udf::ArrowFlightUdfClient; +use tracing::Instrument; use super::*; use crate::catalog::CatalogError; @@ -56,7 +59,7 @@ pub async fn handle_create_function( Some(lang) => { let lang = lang.real_value().to_lowercase(); match &*lang { - "python" | "java" => lang, + "python" | "java" | "wasm_v1" => lang, _ => { return Err(ErrorCode::InvalidParameterValue(format!( "language {} is not supported", @@ -70,12 +73,6 @@ pub async fn handle_create_function( // correct protocol. None => "".to_string(), }; - let Some(FunctionDefinition::SingleQuotedDef(identifier)) = params.as_ else { - return Err(ErrorCode::InvalidParameterValue("AS must be specified".to_string()).into()); - }; - let Some(CreateFunctionUsing::Link(link)) = params.using else { - return Err(ErrorCode::InvalidParameterValue("USING must be specified".to_string()).into()); - }; let return_type; let kind = match returns { Some(CreateFunctionReturns::Value(data_type)) => { @@ -88,14 +85,10 @@ pub async fn handle_create_function( return_type = bind_data_type(&columns[0].data_type)?; } else { // return type is a struct for multiple columns - let datatypes = columns - .iter() - .map(|c| bind_data_type(&c.data_type)) - .collect::>>()?; - let names = columns - .iter() - .map(|c| c.name.real_value()) - .collect::>(); + let it = columns + .into_iter() + .map(|c| bind_data_type(&c.data_type).map(|ty| (ty, c.name.real_value()))); + let (datatypes, names) = itertools::process_results(it, |it| it.unzip())?; return_type = DataType::new_struct(datatypes, names); } Kind::Table(TableFunction {}) @@ -108,6 +101,10 @@ pub async fn handle_create_function( } }; + let Some(using) = params.using else { + return Err(ErrorCode::InvalidParameterValue("USING must be specified".to_string()).into()); + }; + let mut arg_types = vec![]; for arg in args.unwrap_or_default() { arg_types.push(bind_data_type(&arg.data_type)?); @@ -119,7 +116,7 @@ pub async fn handle_create_function( let (schema_name, function_name) = Binder::resolve_schema_qualified_name(db_name, name)?; let (database_id, schema_id) = session.get_database_and_schema_id_for_create(schema_name)?; - // check if function exists + // check if the function exists in the catalog if (session.env().catalog_reader().read_guard()) .get_schema_by_id(&database_id, &schema_id)? .get_function_by_name_args(&function_name, &arg_types) @@ -132,32 +129,80 @@ pub async fn handle_create_function( return Err(CatalogError::Duplicated("function", name).into()); } - // check the service - let client = ArrowFlightUdfClient::connect(&link) - .await - .map_err(|e| anyhow!(e))?; - /// A helper function to create a unnamed field from data type. - fn to_field(data_type: arrow_schema::DataType) -> arrow_schema::Field { - arrow_schema::Field::new("", data_type, true) - } - let args = arrow_schema::Schema::new( - arg_types - .iter() - .map::, _>(|t| Ok(to_field(t.try_into()?))) - .try_collect::<_, Fields, _>()?, - ); - let returns = arrow_schema::Schema::new(match kind { - Kind::Scalar(_) => vec![to_field(return_type.clone().try_into()?)], - Kind::Table(_) => vec![ - arrow_schema::Field::new("row_index", arrow_schema::DataType::Int32, true), - to_field(return_type.clone().try_into()?), - ], - _ => unreachable!(), - }); - client - .check(&identifier, &args, &returns) - .await - .map_err(|e| anyhow!(e))?; + let link; + let identifier; + + // judge the type of the UDF, and do some type-specific checks correspondingly. + let extra = match using { + CreateFunctionUsing::Link(l) => { + let Some(FunctionDefinition::SingleQuotedDef(id)) = params.as_ else { + return Err(ErrorCode::InvalidParameterValue( + "AS must be specified for USING link".to_string(), + ) + .into()); + }; + identifier = id; + link = l; + + // check UDF server + { + let client = ArrowFlightUdfClient::connect(&link) + .await + .map_err(|e| anyhow!(e))?; + /// A helper function to create a unnamed field from data type. + fn to_field(data_type: arrow_schema::DataType) -> arrow_schema::Field { + arrow_schema::Field::new("", data_type, true) + } + let args = arrow_schema::Schema::new( + arg_types + .iter() + .map::, _>(|t| Ok(to_field(t.try_into()?))) + .try_collect::<_, Fields, _>()?, + ); + let returns = arrow_schema::Schema::new(match kind { + Kind::Scalar(_) => vec![to_field(return_type.clone().try_into()?)], + Kind::Table(_) => vec![ + arrow_schema::Field::new("row_index", arrow_schema::DataType::Int32, true), + to_field(return_type.clone().try_into()?), + ], + _ => unreachable!(), + }); + client + .check(&identifier, &args, &returns) + .await + .map_err(|e| anyhow!(e))?; + } + + PbExtra::External(PbExternalUdfExtra {}) + } + CreateFunctionUsing::Base64(module) => { + link = String::new(); + identifier = format!("{}.{}.{}", database_id, schema_id, function_name); + if language != "wasm_v1" { + return Err(ErrorCode::InvalidParameterValue( + "LANGUAGE should be wasm_v1 for USING base64".to_string(), + ) + .into()); + } + + use base64::prelude::{Engine, BASE64_STANDARD}; + let module = BASE64_STANDARD.decode(module).map_err(|e| anyhow!(e))?; + + let system_params = session.env().meta_client().get_system_params().await?; + let wasm_storage_url = system_params.wasm_storage_url(); + + let wasm_engine = WasmEngine::get_or_create(); + wasm_engine + .compile_and_upload_component(module, wasm_storage_url, &identifier) + .instrument(tracing::info_span!("compile_and_upload_component", %identifier)) + .await + .map_err(|e| anyhow!(e))?; + + PbExtra::Wasm(PbWasmUdfExtra { + wasm_storage_url: wasm_storage_url.to_string(), + }) + } + }; let function = Function { id: FunctionId::placeholder().0, @@ -171,6 +216,7 @@ pub async fn handle_create_function( identifier, link, owner: session.user_id(), + extra: Some(extra), }; let catalog_writer = session.catalog_writer()?; diff --git a/src/frontend/src/handler/create_source.rs b/src/frontend/src/handler/create_source.rs index 6f7de61285cb0..0c2398a608eb8 100644 --- a/src/frontend/src/handler/create_source.rs +++ b/src/frontend/src/handler/create_source.rs @@ -630,6 +630,7 @@ pub(crate) async fn try_bind_columns_from_source( field_descs: vec![], type_name: "".to_string(), generated_or_default_column: None, + description: None, }, is_hidden: false, }, @@ -641,6 +642,7 @@ pub(crate) async fn try_bind_columns_from_source( field_descs: vec![], type_name: "".to_string(), generated_or_default_column: None, + description: None, }, is_hidden: false, }, @@ -775,6 +777,7 @@ fn check_and_add_timestamp_column( field_descs: vec![], type_name: "".to_string(), generated_or_default_column: None, + description: None, }, is_hidden: true, @@ -792,6 +795,7 @@ fn add_upsert_default_key_column(columns: &mut Vec) { field_descs: vec![], type_name: "".to_string(), generated_or_default_column: None, + description: None, }, is_hidden: true, }; diff --git a/src/frontend/src/handler/create_table.rs b/src/frontend/src/handler/create_table.rs index 19d9a2f25c4b8..bb02797c21395 100644 --- a/src/frontend/src/handler/create_table.rs +++ b/src/frontend/src/handler/create_table.rs @@ -188,6 +188,7 @@ pub fn bind_sql_columns(column_defs: &[ColumnDef]) -> Result> field_descs, type_name: "".to_string(), generated_or_default_column: None, + description: None, }, is_hidden: false, }); diff --git a/src/frontend/src/handler/describe.rs b/src/frontend/src/handler/describe.rs index 4100b9a20be02..cef7af9dbd324 100644 --- a/src/frontend/src/handler/describe.rs +++ b/src/frontend/src/handler/describe.rs @@ -12,20 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; +use std::fmt::Display; use itertools::Itertools; use pgwire::pg_field_descriptor::PgFieldDescriptor; use pgwire::pg_response::{PgResponse, StatementType}; use pgwire::types::Row; -use risingwave_common::catalog::{ColumnCatalog, ColumnDesc}; use risingwave_common::error::Result; use risingwave_common::types::DataType; use risingwave_sqlparser::ast::{display_comma_separated, ObjectName}; use super::RwPgResponse; use crate::binder::{Binder, Relation}; -use crate::catalog::{CatalogError, IndexCatalog}; +use crate::catalog::CatalogError; use crate::handler::util::col_descs_to_rows; use crate::handler::HandlerArgs; @@ -34,12 +33,9 @@ pub fn handle_describe(handler_args: HandlerArgs, table_name: ObjectName) -> Res let mut binder = Binder::new_for_system(&session); let relation = binder.bind_relation_by_name(table_name.clone(), None, false)?; // For Source, it doesn't have table catalog so use get source to get column descs. - let (columns, pk_columns, dist_columns, indices): ( - Vec, - Vec, - Vec, - Vec>, - ) = match relation { + + // Vec, Vec, Vec, Vec>, String, Option + let (columns, pk_columns, dist_columns, indices, relname, description) = match relation { Relation::Source(s) => { let pk_column_catalogs = s .catalog @@ -55,7 +51,14 @@ pub fn handle_describe(handler_args: HandlerArgs, table_name: ObjectName) -> Res .unwrap() }) .collect_vec(); - (s.catalog.columns, pk_column_catalogs, vec![], vec![]) + ( + s.catalog.columns, + pk_column_catalogs, + vec![], + vec![], + s.catalog.name, + None, // Description + ) } Relation::BaseTable(t) => { let pk_column_catalogs = t @@ -75,6 +78,8 @@ pub fn handle_describe(handler_args: HandlerArgs, table_name: ObjectName) -> Res pk_column_catalogs, dist_columns, t.table_indexes, + t.table_catalog.name, + t.table_catalog.description, ) } Relation::SystemTable(t) => { @@ -89,6 +94,8 @@ pub fn handle_describe(handler_args: HandlerArgs, table_name: ObjectName) -> Res pk_column_catalogs, vec![], vec![], + t.sys_table_catalog.name.clone(), + None, // Description ) } _ => { @@ -99,18 +106,23 @@ pub fn handle_describe(handler_args: HandlerArgs, table_name: ObjectName) -> Res // Convert all column descs to rows let mut rows = col_descs_to_rows(columns); + fn concat(display_elems: impl IntoIterator) -> String + where + T: Display, + { + format!( + "{}", + display_comma_separated(&display_elems.into_iter().collect::>()) + ) + } + // Convert primary key to rows if !pk_columns.is_empty() { rows.push(Row::new(vec![ Some("primary key".into()), - Some( - format!( - "{}", - display_comma_separated(&pk_columns.into_iter().map(|x| x.name).collect_vec()), - ) - .into(), - ), - None, + Some(concat(pk_columns.iter().map(|x| &x.name)).into()), + None, // Is Hidden + None, // Description ])); } @@ -118,14 +130,9 @@ pub fn handle_describe(handler_args: HandlerArgs, table_name: ObjectName) -> Res if !dist_columns.is_empty() { rows.push(Row::new(vec![ Some("distribution key".into()), - Some( - display_comma_separated( - &dist_columns.into_iter().map(|col| col.name).collect_vec(), - ) - .to_string() - .into(), - ), - None, + Some(concat(dist_columns.iter().map(|x| &x.name)).into()), + None, // Is Hidden + None, // Description ])); } @@ -155,10 +162,22 @@ pub fn handle_describe(handler_args: HandlerArgs, table_name: ObjectName) -> Res .into(), ) }, + // Is Hidden + None, + // Description + // TODO: index description None, ]) })); + rows.push(Row::new(vec![ + Some("table description".into()), + Some(relname.into()), + None, // Is Hidden + description.map(Into::into), // Description + ])); + + // TODO: table name and description as title of response // TODO: recover the original user statement Ok(PgResponse::builder(StatementType::DESCRIBE) .values( @@ -179,6 +198,11 @@ pub fn handle_describe(handler_args: HandlerArgs, table_name: ObjectName) -> Res DataType::Varchar.to_oid(), DataType::Varchar.type_len(), ), + PgFieldDescriptor::new( + "Description".to_owned(), + DataType::Varchar.to_oid(), + DataType::Varchar.type_len(), + ), ], ) .into()) @@ -233,6 +257,7 @@ mod tests { "primary key".into() => "v3".into(), "distribution key".into() => "v3".into(), "idx1".into() => "index(v1 DESC, v2 ASC, v3 ASC) include(v4) distributed by(v1)".into(), + "table description".into() => "t".into(), }; assert_eq!(columns, expected_columns); diff --git a/src/frontend/src/handler/mod.rs b/src/frontend/src/handler/mod.rs index 174ed23e03ec5..8275551fbc4a6 100644 --- a/src/frontend/src/handler/mod.rs +++ b/src/frontend/src/handler/mod.rs @@ -39,6 +39,7 @@ mod alter_system; mod alter_table_column; pub mod alter_user; pub mod cancel_job; +mod comment; pub mod create_connection; mod create_database; pub mod create_function; @@ -525,6 +526,11 @@ pub async fn handle( session, } => transaction::handle_set(handler_args, modes, snapshot, session).await, Statement::CancelJobs(jobs) => handle_cancel(handler_args, jobs).await, + Statement::Comment { + object_type, + object_name, + comment, + } => comment::handle_comment(handler_args, object_type, object_name, comment).await, _ => Err( ErrorCode::NotImplemented(format!("Unhandled statement: {}", stmt), None.into()).into(), ), diff --git a/src/frontend/src/handler/util.rs b/src/frontend/src/handler/util.rs index 66494be928d42..7c1eca3fa9bbf 100644 --- a/src/frontend/src/handler/util.rs +++ b/src/frontend/src/handler/util.rs @@ -187,6 +187,7 @@ pub fn col_descs_to_rows(columns: Vec) -> Vec { Some(c.name.into()), Some(type_name.into()), Some(col.is_hidden.to_string().into()), + c.description.map(Into::into), ]) }) .collect_vec() diff --git a/src/frontend/src/optimizer/plan_node/stream_materialize.rs b/src/frontend/src/optimizer/plan_node/stream_materialize.rs index 9c87f1a34abbd..d5435e9beb397 100644 --- a/src/frontend/src/optimizer/plan_node/stream_materialize.rs +++ b/src/frontend/src/optimizer/plan_node/stream_materialize.rs @@ -250,6 +250,7 @@ impl StreamMaterialize { initialized_at_epoch: None, cleaned_by_watermark: false, create_type: CreateType::Foreground, // Will be updated in the handler itself. + description: None, }) } diff --git a/src/frontend/src/optimizer/plan_node/utils.rs b/src/frontend/src/optimizer/plan_node/utils.rs index f167d73c53a46..f05a8be162554 100644 --- a/src/frontend/src/optimizer/plan_node/utils.rs +++ b/src/frontend/src/optimizer/plan_node/utils.rs @@ -180,6 +180,7 @@ impl TableCatalogBuilder { // NOTE(kwannoel): This may not match the create type of the materialized table. // It should be ignored for internal tables. create_type: CreateType::Foreground, + description: None, } } diff --git a/src/frontend/src/session.rs b/src/frontend/src/session.rs index 67eac0df34d05..0385bd52690ef 100644 --- a/src/frontend/src/session.rs +++ b/src/frontend/src/session.rs @@ -1148,6 +1148,11 @@ fn infer(bound: Option, stmt: Statement) -> Result Ok(vec![PgFieldDescriptor::new( "QUERY PLAN".to_owned(), diff --git a/src/frontend/src/test_utils.rs b/src/frontend/src/test_utils.rs index cf915ae35713d..6cca805f3caae 100644 --- a/src/frontend/src/test_utils.rs +++ b/src/frontend/src/test_utils.rs @@ -32,7 +32,7 @@ use risingwave_common::util::column_index_mapping::ColIndexMapping; use risingwave_pb::backup_service::MetaSnapshotMetadata; use risingwave_pb::catalog::table::OptionalAssociatedSourceId; use risingwave_pb::catalog::{ - PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbTable, PbView, Table, + PbComment, PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbTable, PbView, Table, }; use risingwave_pb::ddl_service::{create_connection_request, DdlProgress}; use risingwave_pb::hummock::write_limits::WriteLimit; @@ -318,6 +318,10 @@ impl CatalogWriter for MockCatalogWriter { unreachable!() } + async fn comment_on(&self, _comment: PbComment) -> Result<()> { + unreachable!() + } + async fn drop_table( &self, source_id: Option, diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml index 3e96dfcc7be2f..f37c909546594 100644 --- a/src/meta/Cargo.toml +++ b/src/meta/Cargo.toml @@ -36,7 +36,6 @@ hyper = "0.14" itertools = "0.11" memcomparable = { version = "0.2" } mime_guess = "2" -model_migration = { path = "src/model_v2/migration" } num-integer = "0.1" num-traits = "0.2" parking_lot = { version = "0.12", features = ["arc_lock"] } @@ -50,6 +49,8 @@ risingwave_common = { workspace = true } risingwave_common_heap_profiling = { workspace = true } risingwave_connector = { workspace = true } risingwave_hummock_sdk = { workspace = true } +risingwave_meta_model_migration = { workspace = true } +risingwave_meta_model_v2 = { workspace = true } risingwave_object_store = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } diff --git a/src/meta/model_v2/Cargo.toml b/src/meta/model_v2/Cargo.toml new file mode 100644 index 0000000000000..1d9992da8a832 --- /dev/null +++ b/src/meta/model_v2/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "risingwave_meta_model_v2" +version = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +keywords = { workspace = true } +license = { workspace = true } +repository = { workspace = true } + +[package.metadata.cargo-machete] +ignored = ["workspace-hack"] + +[package.metadata.cargo-udeps.ignore] +normal = ["workspace-hack"] + +[dependencies] +risingwave_pb = { workspace = true } +sea-orm = { version = "0.12.0", features = [ + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "runtime-tokio-native-tls", + "macros", +] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/src/meta/model_v2/migration/Cargo.toml b/src/meta/model_v2/migration/Cargo.toml new file mode 100644 index 0000000000000..4745125140a22 --- /dev/null +++ b/src/meta/model_v2/migration/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "risingwave_meta_model_migration" +version = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +keywords = { workspace = true } +license = { workspace = true } +repository = { workspace = true } + +[package.metadata.cargo-machete] +ignored = ["workspace-hack"] + +[package.metadata.cargo-udeps.ignore] +normal = ["workspace-hack"] + +[dependencies] +async-std = { version = "1", features = ["attributes", "tokio1"] } +uuid = { version = "1", features = ["v4"] } + +[dependencies.sea-orm-migration] +version = "0.12.0" +features = ["sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-native-tls", "with-uuid"] diff --git a/src/meta/src/model_v2/migration/README.md b/src/meta/model_v2/migration/README.md similarity index 100% rename from src/meta/src/model_v2/migration/README.md rename to src/meta/model_v2/migration/README.md diff --git a/src/meta/src/model_v2/migration/src/lib.rs b/src/meta/model_v2/migration/src/lib.rs similarity index 100% rename from src/meta/src/model_v2/migration/src/lib.rs rename to src/meta/model_v2/migration/src/lib.rs diff --git a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/model_v2/migration/src/m20230908_072257_init.rs similarity index 100% rename from src/meta/src/model_v2/migration/src/m20230908_072257_init.rs rename to src/meta/model_v2/migration/src/m20230908_072257_init.rs diff --git a/src/meta/src/model_v2/migration/src/m20231008_020431_hummock.rs b/src/meta/model_v2/migration/src/m20231008_020431_hummock.rs similarity index 100% rename from src/meta/src/model_v2/migration/src/m20231008_020431_hummock.rs rename to src/meta/model_v2/migration/src/m20231008_020431_hummock.rs diff --git a/src/meta/src/model_v2/migration/src/main.rs b/src/meta/model_v2/migration/src/main.rs similarity index 52% rename from src/meta/src/model_v2/migration/src/main.rs rename to src/meta/model_v2/migration/src/main.rs index 9354e45ecd198..9be884a68a11d 100644 --- a/src/meta/src/model_v2/migration/src/main.rs +++ b/src/meta/model_v2/migration/src/main.rs @@ -2,5 +2,5 @@ use sea_orm_migration::prelude::*; #[async_std::main] async fn main() { - cli::run_cli(model_migration::Migrator).await; + cli::run_cli(risingwave_meta_model_migration::Migrator).await; } diff --git a/src/meta/src/model_v2/README.md b/src/meta/model_v2/src/README.md similarity index 93% rename from src/meta/src/model_v2/README.md rename to src/meta/model_v2/src/README.md index 25c22a4f566e1..48095d3e6d67f 100644 --- a/src/meta/src/model_v2/README.md +++ b/src/meta/model_v2/src/README.md @@ -1,6 +1,6 @@ # How to define changes between versions and generate migration and model files -- Generate a new migration file and apply it to the database, check [migration](./migration/README.md) for more details. Let's take a local PG database as an example(`postgres://postgres:@localhost:5432/postgres`): +- Generate a new migration file and apply it to the database, check [migration](../migration/README.md) for more details. Let's take a local PG database as an example(`postgres://postgres:@localhost:5432/postgres`): ```sh export DATABASE_URL=postgres://postgres:@localhost:5432/postgres; cargo run -- generate MIGRATION_NAME diff --git a/src/meta/src/model_v2/actor.rs b/src/meta/model_v2/src/actor.rs similarity index 97% rename from src/meta/src/model_v2/actor.rs rename to src/meta/model_v2/src/actor.rs index 8fecb3046b1bc..79a70e3f65e95 100644 --- a/src/meta/src/model_v2/actor.rs +++ b/src/meta/model_v2/src/actor.rs @@ -14,7 +14,7 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::I32Array; +use crate::I32Array; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "actor")] diff --git a/src/meta/src/model_v2/cluster.rs b/src/meta/model_v2/src/cluster.rs similarity index 100% rename from src/meta/src/model_v2/cluster.rs rename to src/meta/model_v2/src/cluster.rs diff --git a/src/meta/src/model_v2/compaction_config.rs b/src/meta/model_v2/src/compaction_config.rs similarity index 100% rename from src/meta/src/model_v2/compaction_config.rs rename to src/meta/model_v2/src/compaction_config.rs diff --git a/src/meta/src/model_v2/compaction_status.rs b/src/meta/model_v2/src/compaction_status.rs similarity index 100% rename from src/meta/src/model_v2/compaction_status.rs rename to src/meta/model_v2/src/compaction_status.rs diff --git a/src/meta/src/model_v2/compaction_task.rs b/src/meta/model_v2/src/compaction_task.rs similarity index 100% rename from src/meta/src/model_v2/compaction_task.rs rename to src/meta/model_v2/src/compaction_task.rs diff --git a/src/meta/src/model_v2/connection.rs b/src/meta/model_v2/src/connection.rs similarity index 97% rename from src/meta/src/model_v2/connection.rs rename to src/meta/model_v2/src/connection.rs index 0096603c843a3..8cff6b2a6025b 100644 --- a/src/meta/src/model_v2/connection.rs +++ b/src/meta/model_v2/src/connection.rs @@ -17,7 +17,7 @@ use risingwave_pb::catalog::PbConnection; use sea_orm::entity::prelude::*; use sea_orm::ActiveValue; -use crate::model_v2::{ConnectionId, PrivateLinkService}; +use crate::{ConnectionId, PrivateLinkService}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "connection")] diff --git a/src/meta/src/model_v2/database.rs b/src/meta/model_v2/src/database.rs similarity index 81% rename from src/meta/src/model_v2/database.rs rename to src/meta/model_v2/src/database.rs index 909c12eceac5a..95ff3a8aee8e6 100644 --- a/src/meta/src/model_v2/database.rs +++ b/src/meta/model_v2/src/database.rs @@ -12,9 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use risingwave_pb::catalog::PbDatabase; use sea_orm::entity::prelude::*; +use sea_orm::ActiveValue; -use crate::model_v2::DatabaseId; +use crate::DatabaseId; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "database")] @@ -44,3 +46,12 @@ impl Related for Entity { } impl ActiveModelBehavior for ActiveModel {} + +impl From for ActiveModel { + fn from(db: PbDatabase) -> Self { + Self { + database_id: ActiveValue::Set(db.id), + name: ActiveValue::Set(db.name), + } + } +} diff --git a/src/meta/src/model_v2/fragment.rs b/src/meta/model_v2/src/fragment.rs similarity index 98% rename from src/meta/src/model_v2/fragment.rs rename to src/meta/model_v2/src/fragment.rs index 9263dd99eabb8..c590a58da771e 100644 --- a/src/meta/src/model_v2/fragment.rs +++ b/src/meta/model_v2/src/fragment.rs @@ -14,7 +14,7 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::I32Array; +use crate::I32Array; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "fragment")] diff --git a/src/meta/src/model_v2/function.rs b/src/meta/model_v2/src/function.rs similarity index 97% rename from src/meta/src/model_v2/function.rs rename to src/meta/model_v2/src/function.rs index 663f8e2284fd7..4126dddc0f5ee 100644 --- a/src/meta/src/model_v2/function.rs +++ b/src/meta/model_v2/src/function.rs @@ -17,7 +17,7 @@ use risingwave_pb::catalog::PbFunction; use sea_orm::entity::prelude::*; use sea_orm::ActiveValue; -use crate::model_v2::{DataType, DataTypeArray, FunctionId}; +use crate::{DataType, DataTypeArray, FunctionId}; #[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] #[sea_orm(rs_type = "String", db_type = "String(None)")] diff --git a/src/meta/src/model_v2/hummock_pinned_snapshot.rs b/src/meta/model_v2/src/hummock_pinned_snapshot.rs similarity index 100% rename from src/meta/src/model_v2/hummock_pinned_snapshot.rs rename to src/meta/model_v2/src/hummock_pinned_snapshot.rs diff --git a/src/meta/src/model_v2/hummock_pinned_version.rs b/src/meta/model_v2/src/hummock_pinned_version.rs similarity index 100% rename from src/meta/src/model_v2/hummock_pinned_version.rs rename to src/meta/model_v2/src/hummock_pinned_version.rs diff --git a/src/meta/src/model_v2/hummock_version_delta.rs b/src/meta/model_v2/src/hummock_version_delta.rs similarity index 100% rename from src/meta/src/model_v2/hummock_version_delta.rs rename to src/meta/model_v2/src/hummock_version_delta.rs diff --git a/src/meta/src/model_v2/hummock_version_stats.rs b/src/meta/model_v2/src/hummock_version_stats.rs similarity index 100% rename from src/meta/src/model_v2/hummock_version_stats.rs rename to src/meta/model_v2/src/hummock_version_stats.rs diff --git a/src/meta/src/model_v2/index.rs b/src/meta/model_v2/src/index.rs similarity index 96% rename from src/meta/src/model_v2/index.rs rename to src/meta/model_v2/src/index.rs index 3b80632e2cfc3..c85a896914240 100644 --- a/src/meta/src/model_v2/index.rs +++ b/src/meta/model_v2/src/index.rs @@ -14,7 +14,7 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::{ExprNodeArray, I32Array, IndexId, JobStatus, TableId}; +use crate::{ExprNodeArray, I32Array, IndexId, JobStatus, TableId}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "index")] diff --git a/src/meta/src/model_v2/mod.rs b/src/meta/model_v2/src/lib.rs similarity index 99% rename from src/meta/src/model_v2/mod.rs rename to src/meta/model_v2/src/lib.rs index 1c2f928063fff..5fe23bcaa280c 100644 --- a/src/meta/src/model_v2/mod.rs +++ b/src/meta/model_v2/src/lib.rs @@ -27,7 +27,6 @@ pub mod compaction_status; pub mod compaction_task; pub mod connection; pub mod database; -pub mod ext; pub mod fragment; pub mod function; pub mod hummock_pinned_snapshot; @@ -42,7 +41,6 @@ pub mod sink; pub mod source; pub mod system_parameter; pub mod table; -pub mod trx; pub mod user; pub mod user_privilege; pub mod view; diff --git a/src/meta/src/model_v2/object.rs b/src/meta/model_v2/src/object.rs similarity index 98% rename from src/meta/src/model_v2/object.rs rename to src/meta/model_v2/src/object.rs index 5048f93a483d9..39506777068a3 100644 --- a/src/meta/src/model_v2/object.rs +++ b/src/meta/model_v2/src/object.rs @@ -14,7 +14,7 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::{DatabaseId, ObjectId, SchemaId, UserId}; +use crate::{DatabaseId, ObjectId, SchemaId, UserId}; #[derive(Clone, Debug, PartialEq, Eq, Copy, EnumIter, DeriveActiveEnum)] #[sea_orm(rs_type = "String", db_type = "String(None)")] diff --git a/src/meta/src/model_v2/object_dependency.rs b/src/meta/model_v2/src/object_dependency.rs similarity index 97% rename from src/meta/src/model_v2/object_dependency.rs rename to src/meta/model_v2/src/object_dependency.rs index 53800112a7370..52ca229c6997a 100644 --- a/src/meta/src/model_v2/object_dependency.rs +++ b/src/meta/model_v2/src/object_dependency.rs @@ -14,7 +14,7 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::{ObjectId, UserId}; +use crate::{ObjectId, UserId}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "object_dependency")] diff --git a/src/meta/src/model_v2/prelude.rs b/src/meta/model_v2/src/prelude.rs similarity index 100% rename from src/meta/src/model_v2/prelude.rs rename to src/meta/model_v2/src/prelude.rs diff --git a/src/meta/src/model_v2/schema.rs b/src/meta/model_v2/src/schema.rs similarity index 81% rename from src/meta/src/model_v2/schema.rs rename to src/meta/model_v2/src/schema.rs index 2c28665fd06f0..0af2d7fc020c9 100644 --- a/src/meta/src/model_v2/schema.rs +++ b/src/meta/model_v2/src/schema.rs @@ -12,9 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use risingwave_pb::catalog::PbSchema; use sea_orm::entity::prelude::*; +use sea_orm::ActiveValue; -use crate::model_v2::SchemaId; +use crate::SchemaId; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "schema")] @@ -43,3 +45,12 @@ impl Related for Entity { } impl ActiveModelBehavior for ActiveModel {} + +impl From for ActiveModel { + fn from(schema: PbSchema) -> Self { + Self { + schema_id: ActiveValue::Set(schema.id), + name: ActiveValue::Set(schema.name), + } + } +} diff --git a/src/meta/src/model_v2/sink.rs b/src/meta/model_v2/src/sink.rs similarity index 99% rename from src/meta/src/model_v2/sink.rs rename to src/meta/model_v2/src/sink.rs index bef46f1d7195f..21ac172246703 100644 --- a/src/meta/src/model_v2/sink.rs +++ b/src/meta/model_v2/src/sink.rs @@ -15,7 +15,7 @@ use risingwave_pb::catalog::PbSinkType; use sea_orm::entity::prelude::*; -use crate::model_v2::{ +use crate::{ ColumnCatalogArray, ColumnOrderArray, ConnectionId, I32Array, JobStatus, Property, SinkFormatDesc, SinkId, }; diff --git a/src/meta/src/model_v2/source.rs b/src/meta/model_v2/src/source.rs similarity index 99% rename from src/meta/src/model_v2/source.rs rename to src/meta/model_v2/src/source.rs index 2ad1de7914d96..620d002c27b55 100644 --- a/src/meta/src/model_v2/source.rs +++ b/src/meta/model_v2/src/source.rs @@ -14,7 +14,7 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::{ +use crate::{ ColumnCatalogArray, ConnectionId, I32Array, Property, SourceId, StreamSourceInfo, TableId, WatermarkDescArray, }; diff --git a/src/meta/src/model_v2/system_parameter.rs b/src/meta/model_v2/src/system_parameter.rs similarity index 100% rename from src/meta/src/model_v2/system_parameter.rs rename to src/meta/model_v2/src/system_parameter.rs diff --git a/src/meta/src/model_v2/table.rs b/src/meta/model_v2/src/table.rs similarity index 99% rename from src/meta/src/model_v2/table.rs rename to src/meta/model_v2/src/table.rs index 08caee7009f8f..a335f41023442 100644 --- a/src/meta/src/model_v2/table.rs +++ b/src/meta/model_v2/src/table.rs @@ -16,7 +16,7 @@ use risingwave_pb::catalog::table::PbTableType; use risingwave_pb::catalog::PbHandleConflictBehavior; use sea_orm::entity::prelude::*; -use crate::model_v2::{ +use crate::{ Cardinality, ColumnCatalogArray, ColumnOrderArray, CreateType, I32Array, JobStatus, Property, SourceId, TableId, TableVersion, }; diff --git a/src/meta/src/model_v2/user.rs b/src/meta/model_v2/src/user.rs similarity index 97% rename from src/meta/src/model_v2/user.rs rename to src/meta/model_v2/src/user.rs index 0e7ab4dd17876..e9cd36f75fb43 100644 --- a/src/meta/src/model_v2/user.rs +++ b/src/meta/model_v2/src/user.rs @@ -14,7 +14,7 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::UserId; +use crate::UserId; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "user")] diff --git a/src/meta/src/model_v2/user_privilege.rs b/src/meta/model_v2/src/user_privilege.rs similarity index 97% rename from src/meta/src/model_v2/user_privilege.rs rename to src/meta/model_v2/src/user_privilege.rs index 335f716cec1c8..7e12af225ed02 100644 --- a/src/meta/src/model_v2/user_privilege.rs +++ b/src/meta/model_v2/src/user_privilege.rs @@ -14,7 +14,7 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::{ObjectId, UserId}; +use crate::{ObjectId, UserId}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "user_privilege")] diff --git a/src/meta/src/model_v2/view.rs b/src/meta/model_v2/src/view.rs similarity index 97% rename from src/meta/src/model_v2/view.rs rename to src/meta/model_v2/src/view.rs index 8f7d22408d3f2..0de9ea64a616e 100644 --- a/src/meta/src/model_v2/view.rs +++ b/src/meta/model_v2/src/view.rs @@ -16,7 +16,7 @@ use risingwave_pb::catalog::PbView; use sea_orm::entity::prelude::*; use sea_orm::ActiveValue; -use crate::model_v2::{FieldArray, Property, ViewId}; +use crate::{FieldArray, Property, ViewId}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "view")] diff --git a/src/meta/model_v2/src/worker.rs b/src/meta/model_v2/src/worker.rs new file mode 100644 index 0000000000000..d164fba62b41e --- /dev/null +++ b/src/meta/model_v2/src/worker.rs @@ -0,0 +1,128 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_pb::common::worker_node::PbState; +use risingwave_pb::common::{PbWorkerNode, PbWorkerType}; +use sea_orm::entity::prelude::*; +use sea_orm::ActiveValue; + +use crate::{TransactionId, WorkerId}; + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "String(None)")] +pub enum WorkerType { + #[sea_orm(string_value = "FRONTEND")] + Frontend, + #[sea_orm(string_value = "COMPUTE_NODE")] + ComputeNode, + #[sea_orm(string_value = "RISE_CTL")] + RiseCtl, + #[sea_orm(string_value = "COMPACTOR")] + Compactor, + #[sea_orm(string_value = "META")] + Meta, +} + +impl From for WorkerType { + fn from(worker_type: PbWorkerType) -> Self { + match worker_type { + PbWorkerType::Unspecified => unreachable!("unspecified worker type"), + PbWorkerType::Frontend => Self::Frontend, + PbWorkerType::ComputeNode => Self::ComputeNode, + PbWorkerType::RiseCtl => Self::RiseCtl, + PbWorkerType::Compactor => Self::Compactor, + PbWorkerType::Meta => Self::Meta, + } + } +} + +impl From for PbWorkerType { + fn from(worker_type: WorkerType) -> Self { + match worker_type { + WorkerType::Frontend => Self::Frontend, + WorkerType::ComputeNode => Self::ComputeNode, + WorkerType::RiseCtl => Self::RiseCtl, + WorkerType::Compactor => Self::Compactor, + WorkerType::Meta => Self::Meta, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "String(None)")] +pub enum WorkerStatus { + #[sea_orm(string_value = "STARTING")] + Starting, + #[sea_orm(string_value = "RUNNING")] + Running, +} + +impl From for WorkerStatus { + fn from(state: PbState) -> Self { + match state { + PbState::Unspecified => unreachable!("unspecified worker status"), + PbState::Starting => Self::Starting, + PbState::Running => Self::Running, + } + } +} + +impl From for PbState { + fn from(status: WorkerStatus) -> Self { + match status { + WorkerStatus::Starting => Self::Starting, + WorkerStatus::Running => Self::Running, + } + } +} + +impl From<&PbWorkerNode> for ActiveModel { + fn from(worker: &PbWorkerNode) -> Self { + let host = worker.host.clone().unwrap(); + Self { + worker_id: ActiveValue::Set(worker.id), + worker_type: ActiveValue::Set(worker.r#type().into()), + host: ActiveValue::Set(host.host), + port: ActiveValue::Set(host.port), + status: ActiveValue::Set(worker.state().into()), + ..Default::default() + } + } +} + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "worker")] +pub struct Model { + #[sea_orm(primary_key)] + pub worker_id: WorkerId, + pub worker_type: WorkerType, + pub host: String, + pub port: i32, + pub status: WorkerStatus, + pub transaction_id: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::worker_property::Entity")] + WorkerProperty, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::WorkerProperty.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/worker_property.rs b/src/meta/model_v2/src/worker_property.rs similarity index 97% rename from src/meta/src/model_v2/worker_property.rs rename to src/meta/model_v2/src/worker_property.rs index 8521cbed15ce2..0512ea97e5be3 100644 --- a/src/meta/src/model_v2/worker_property.rs +++ b/src/meta/model_v2/src/worker_property.rs @@ -14,7 +14,7 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::{I32Array, WorkerId}; +use crate::{I32Array, WorkerId}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "worker_property")] diff --git a/src/meta/node/Cargo.toml b/src/meta/node/Cargo.toml index 8c2a5aeadbe41..84793a74591c8 100644 --- a/src/meta/node/Cargo.toml +++ b/src/meta/node/Cargo.toml @@ -20,13 +20,13 @@ either = "1" etcd-client = { workspace = true } futures = { version = "0.3", default-features = false, features = ["alloc"] } itertools = "0.11" -model_migration = { path = "../src/model_v2/migration" } prometheus-http-query = "0.7" regex = "1" risingwave_common = { workspace = true } risingwave_common_heap_profiling = { workspace = true } risingwave_common_service = { workspace = true } risingwave_meta = { workspace = true } +risingwave_meta_model_migration = { workspace = true } risingwave_meta_service = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } diff --git a/src/meta/node/src/server.rs b/src/meta/node/src/server.rs index d922f1c37e033..d8d8525aca235 100644 --- a/src/meta/node/src/server.rs +++ b/src/meta/node/src/server.rs @@ -19,7 +19,6 @@ use either::Either; use etcd_client::ConnectOptions; use futures::future::join_all; use itertools::Itertools; -use model_migration::{Migrator, MigratorTrait}; use regex::Regex; use risingwave_common::monitor::connection::{RouterExt, TcpConfig}; use risingwave_common::telemetry::manager::TelemetryManager; @@ -28,6 +27,7 @@ use risingwave_common_service::metrics_manager::MetricsManager; use risingwave_common_service::tracing::TracingExtractLayer; use risingwave_meta::rpc::intercept::MetricsMiddlewareLayer; use risingwave_meta::rpc::ElectionClientRef; +use risingwave_meta_model_migration::{Migrator, MigratorTrait}; use risingwave_meta_service::backup_service::BackupServiceImpl; use risingwave_meta_service::cloud_service::CloudServiceImpl; use risingwave_meta_service::cluster_service::ClusterServiceImpl; diff --git a/src/meta/service/Cargo.toml b/src/meta/service/Cargo.toml index 1760ccd56a85a..87b293f64a5e6 100644 --- a/src/meta/service/Cargo.toml +++ b/src/meta/service/Cargo.toml @@ -23,6 +23,7 @@ regex = "1" risingwave_common = { workspace = true } risingwave_connector = { workspace = true } risingwave_meta = { workspace = true } +risingwave_meta_model_v2 = { workspace = true } risingwave_pb = { workspace = true } sea-orm = { version = "0.12.0", features = [ "sqlx-mysql", diff --git a/src/meta/service/src/ddl_service.rs b/src/meta/service/src/ddl_service.rs index 061ff93589163..6f08ebfb18d17 100644 --- a/src/meta/service/src/ddl_service.rs +++ b/src/meta/service/src/ddl_service.rs @@ -25,7 +25,7 @@ use risingwave_pb::catalog::connection::private_link_service::{ use risingwave_pb::catalog::connection::PbPrivateLinkService; use risingwave_pb::catalog::source::OptionalAssociatedTableId; use risingwave_pb::catalog::table::OptionalAssociatedSourceId; -use risingwave_pb::catalog::{connection, Connection, CreateType, PbSource, PbTable}; +use risingwave_pb::catalog::{connection, Comment, Connection, CreateType, PbSource, PbTable}; use risingwave_pb::ddl_service::ddl_service_server::DdlService; use risingwave_pb::ddl_service::drop_table_request::PbSourceId; use risingwave_pb::ddl_service::*; @@ -717,6 +717,30 @@ impl DdlService for DdlServiceImpl { })) } + async fn comment_on( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let comment = req.get_comment()?.clone(); + + let version = self + .ddl_controller + .run_command(DdlCommand::CommentOn(Comment { + table_id: comment.table_id, + schema_id: comment.schema_id, + database_id: comment.database_id, + column_index: comment.column_index, + description: comment.description, + })) + .await?; + + Ok(Response::new(CommentOnResponse { + status: None, + version, + })) + } + #[cfg_attr(coverage, coverage(off))] async fn get_tables( &self, @@ -734,7 +758,7 @@ impl DdlService for DdlServiceImpl { } async fn wait(&self, _request: Request) -> Result, Status> { - self.ddl_controller.wait().await; + self.ddl_controller.wait().await?; Ok(Response::new(WaitResponse {})) } } diff --git a/src/meta/service/src/telemetry_service.rs b/src/meta/service/src/telemetry_service.rs index 7c413406f13e5..42200e10a4eeb 100644 --- a/src/meta/service/src/telemetry_service.rs +++ b/src/meta/service/src/telemetry_service.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use risingwave_meta_model_v2::prelude::Cluster; use risingwave_pb::meta::telemetry_info_service_server::TelemetryInfoService; use risingwave_pb::meta::{GetTelemetryInfoRequest, TelemetryInfoResponse}; use sea_orm::EntityTrait; @@ -19,7 +20,6 @@ use tonic::{Request, Response, Status}; use crate::controller::SqlMetaStore; use crate::model::ClusterId; -use crate::model_v2::prelude::Cluster; use crate::storage::MetaStoreRef; use crate::MetaResult; diff --git a/src/meta/src/backup_restore/restore.rs b/src/meta/src/backup_restore/restore.rs index ab4696e62f9bd..47d14330f5fb2 100644 --- a/src/meta/src/backup_restore/restore.rs +++ b/src/meta/src/backup_restore/restore.rs @@ -320,6 +320,7 @@ mod tests { SystemParams { state_store: Some("state_store".to_string()), data_directory: Some("data_directory".to_string()), + wasm_storage_url: None, ..SystemConfig::default().into_init_system_params() } } diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index cb37307384aa2..daaa9b684850c 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -17,6 +17,13 @@ use std::iter; use itertools::Itertools; use risingwave_common::bail; use risingwave_common::catalog::{DEFAULT_SCHEMA_NAME, SYSTEM_SCHEMAS}; +use risingwave_meta_model_v2::object::ObjectType; +use risingwave_meta_model_v2::prelude::*; +use risingwave_meta_model_v2::{ + connection, database, function, index, object, object_dependency, schema, sink, source, table, + view, ConnectionId, DatabaseId, FunctionId, ObjectId, PrivateLinkService, SchemaId, SourceId, + TableId, UserId, +}; use risingwave_pb::catalog::{ PbConnection, PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbTable, PbView, }; @@ -40,13 +47,6 @@ use crate::controller::utils::{ }; use crate::controller::ObjectModel; use crate::manager::{MetaSrvEnv, NotificationVersion}; -use crate::model_v2::object::ObjectType; -use crate::model_v2::prelude::*; -use crate::model_v2::{ - connection, database, function, index, object, object_dependency, schema, sink, source, table, - view, ConnectionId, DatabaseId, FunctionId, ObjectId, PrivateLinkService, SchemaId, SourceId, - TableId, UserId, -}; use crate::rpc::ddl_controller::DropMode; use crate::{MetaError, MetaResult}; diff --git a/src/meta/src/controller/cluster.rs b/src/meta/src/controller/cluster.rs index ca29380a49fca..392a0def5d53f 100644 --- a/src/meta/src/controller/cluster.rs +++ b/src/meta/src/controller/cluster.rs @@ -22,6 +22,9 @@ use std::time::{Duration, SystemTime}; use itertools::Itertools; use risingwave_common::hash::ParallelUnitId; use risingwave_hummock_sdk::HummockSstableObjectId; +use risingwave_meta_model_v2::prelude::{Worker, WorkerProperty}; +use risingwave_meta_model_v2::worker::{WorkerStatus, WorkerType}; +use risingwave_meta_model_v2::{worker, worker_property, I32Array, TransactionId, WorkerId}; use risingwave_pb::common::worker_node::{PbProperty, PbState}; use risingwave_pb::common::{ HostAddress, ParallelUnit, PbHostAddress, PbParallelUnit, PbWorkerNode, PbWorkerType, @@ -39,10 +42,7 @@ use tokio::sync::oneshot::Sender; use tokio::sync::{RwLock, RwLockReadGuard}; use tokio::task::JoinHandle; -use crate::manager::prelude::{Worker, WorkerProperty}; use crate::manager::{LocalNotification, MetaSrvEnv, WorkerKey}; -use crate::model_v2::worker::{WorkerStatus, WorkerType}; -use crate::model_v2::{worker, worker_property, I32Array, TransactionId, WorkerId}; use crate::{MetaError, MetaResult}; pub type ClusterControllerRef = Arc; @@ -89,64 +89,6 @@ impl From for PbWorkerNode { } } -impl From for WorkerType { - fn from(worker_type: PbWorkerType) -> Self { - match worker_type { - PbWorkerType::Unspecified => unreachable!("unspecified worker type"), - PbWorkerType::Frontend => Self::Frontend, - PbWorkerType::ComputeNode => Self::ComputeNode, - PbWorkerType::RiseCtl => Self::RiseCtl, - PbWorkerType::Compactor => Self::Compactor, - PbWorkerType::Meta => Self::Meta, - } - } -} - -impl From for PbWorkerType { - fn from(worker_type: WorkerType) -> Self { - match worker_type { - WorkerType::Frontend => Self::Frontend, - WorkerType::ComputeNode => Self::ComputeNode, - WorkerType::RiseCtl => Self::RiseCtl, - WorkerType::Compactor => Self::Compactor, - WorkerType::Meta => Self::Meta, - } - } -} - -impl From for WorkerStatus { - fn from(state: PbState) -> Self { - match state { - PbState::Unspecified => unreachable!("unspecified worker status"), - PbState::Starting => Self::Starting, - PbState::Running => Self::Running, - } - } -} - -impl From for PbState { - fn from(status: WorkerStatus) -> Self { - match status { - WorkerStatus::Starting => Self::Starting, - WorkerStatus::Running => Self::Running, - } - } -} - -impl From<&PbWorkerNode> for worker::ActiveModel { - fn from(worker: &PbWorkerNode) -> Self { - let host = worker.host.clone().unwrap(); - Self { - worker_id: ActiveValue::Set(worker.id), - worker_type: ActiveValue::Set(worker.r#type().into()), - host: ActiveValue::Set(host.host), - port: ActiveValue::Set(host.port), - status: ActiveValue::Set(worker.state().into()), - ..Default::default() - } - } -} - impl ClusterController { pub async fn new(env: MetaSrvEnv, max_heartbeat_interval: Duration) -> MetaResult { let meta_store = env diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs index 07793e30a17fe..c69c615165d11 100644 --- a/src/meta/src/controller/mod.rs +++ b/src/meta/src/controller/mod.rs @@ -14,6 +14,9 @@ use anyhow::anyhow; use risingwave_common::util::epoch::Epoch; +use risingwave_meta_model_v2::{ + connection, database, index, object, schema, sink, source, table, view, +}; use risingwave_pb::catalog::connection::PbInfo as PbConnectionInfo; use risingwave_pb::catalog::source::PbOptionalAssociatedTableId; use risingwave_pb::catalog::table::{PbOptionalAssociatedSourceId, PbTableType}; @@ -21,9 +24,8 @@ use risingwave_pb::catalog::{ PbConnection, PbCreateType, PbDatabase, PbHandleConflictBehavior, PbIndex, PbSchema, PbSink, PbSinkType, PbSource, PbStreamJobStatus, PbTable, PbView, }; -use sea_orm::{ActiveValue, DatabaseConnection, ModelTrait}; +use sea_orm::{DatabaseConnection, ModelTrait}; -use crate::model_v2::{connection, database, index, object, schema, sink, source, table, view}; use crate::MetaError; #[allow(dead_code)] @@ -56,7 +58,7 @@ impl SqlMetaStore { #[cfg(any(test, feature = "test"))] #[cfg(not(madsim))] pub async fn for_test() -> Self { - use model_migration::{Migrator, MigratorTrait}; + use risingwave_meta_model_migration::{Migrator, MigratorTrait}; let conn = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); Migrator::up(&conn, None).await.unwrap(); Self { conn } @@ -75,24 +77,6 @@ impl From> for PbDatabase { } } -impl From for database::ActiveModel { - fn from(db: PbDatabase) -> Self { - Self { - database_id: ActiveValue::Set(db.id), - name: ActiveValue::Set(db.name), - } - } -} - -impl From for schema::ActiveModel { - fn from(schema: PbSchema) -> Self { - Self { - schema_id: ActiveValue::Set(schema.id), - name: ActiveValue::Set(schema.name), - } - } -} - impl From> for PbSchema { fn from(value: ObjectModel) -> Self { Self { @@ -147,6 +131,7 @@ impl From> for PbTable { .0 .optional_associated_source_id .map(PbOptionalAssociatedSourceId::AssociatedSourceId), + description: None, } } } diff --git a/src/meta/src/controller/system_param.rs b/src/meta/src/controller/system_param.rs index 0656da5ea9a46..5c9761a9a119d 100644 --- a/src/meta/src/controller/system_param.rs +++ b/src/meta/src/controller/system_param.rs @@ -21,6 +21,8 @@ use risingwave_common::system_param::{ check_missing_params, derive_missing_fields, set_system_param, }; use risingwave_common::{for_all_params, key_of}; +use risingwave_meta_model_v2::prelude::SystemParameter; +use risingwave_meta_model_v2::system_parameter; use risingwave_pb::meta::subscribe_response::{Info, Operation}; use risingwave_pb::meta::PbSystemParams; use sea_orm::{ActiveModelTrait, ActiveValue, DatabaseConnection, EntityTrait, TransactionTrait}; @@ -31,8 +33,6 @@ use tracing::info; use crate::controller::SqlMetaStore; use crate::manager::{LocalNotification, NotificationManagerRef}; -use crate::model_v2::prelude::SystemParameter; -use crate::model_v2::system_parameter; use crate::{MetaError, MetaResult}; pub type SystemParamsControllerRef = Arc; diff --git a/src/meta/src/controller/utils.rs b/src/meta/src/controller/utils.rs index d36918db3820d..2dbd89ac92423 100644 --- a/src/meta/src/controller/utils.rs +++ b/src/meta/src/controller/utils.rs @@ -13,7 +13,13 @@ // limitations under the License. use anyhow::anyhow; -use model_migration::WithQuery; +use risingwave_meta_model_migration::WithQuery; +use risingwave_meta_model_v2::object::ObjectType; +use risingwave_meta_model_v2::prelude::*; +use risingwave_meta_model_v2::{ + connection, function, index, object, object_dependency, schema, sink, source, table, view, + DataTypeArray, DatabaseId, ObjectId, SchemaId, UserId, +}; use risingwave_pb::catalog::{PbConnection, PbFunction}; use sea_orm::sea_query::{ Alias, CommonTableExpression, Expr, Query, QueryStatementBuilder, SelectStatement, UnionType, @@ -24,12 +30,6 @@ use sea_orm::{ Order, PaginatorTrait, QueryFilter, QuerySelect, RelationTrait, Statement, }; -use crate::model_v2::object::ObjectType; -use crate::model_v2::prelude::*; -use crate::model_v2::{ - connection, function, index, object, object_dependency, schema, sink, source, table, view, - DataTypeArray, DatabaseId, ObjectId, SchemaId, UserId, -}; use crate::{MetaError, MetaResult}; /// This function will construct a query using recursive cte to find all objects[(id, `obj_type`)] that are used by the given object. diff --git a/src/meta/src/lib.rs b/src/meta/src/lib.rs index f549578f079c6..95b4ce7ead72d 100644 --- a/src/meta/src/lib.rs +++ b/src/meta/src/lib.rs @@ -42,7 +42,6 @@ pub mod error; pub mod hummock; pub mod manager; pub mod model; -pub mod model_v2; pub mod rpc; pub mod serving; pub mod storage; diff --git a/src/meta/src/manager/catalog/mod.rs b/src/meta/src/manager/catalog/mod.rs index f988646428aac..d50c8bc5d31e3 100644 --- a/src/meta/src/manager/catalog/mod.rs +++ b/src/meta/src/manager/catalog/mod.rs @@ -34,8 +34,8 @@ use risingwave_common::catalog::{ use risingwave_common::{bail, ensure}; use risingwave_pb::catalog::table::{OptionalAssociatedSourceId, TableType}; use risingwave_pb::catalog::{ - Connection, CreateType, Database, Function, Index, PbStreamJobStatus, Schema, Sink, Source, - StreamJobStatus, Table, View, + Comment, Connection, CreateType, Database, Function, Index, PbStreamJobStatus, Schema, Sink, + Source, StreamJobStatus, Table, View, }; use risingwave_pb::meta::subscribe_response::{Info, Operation}; use risingwave_pb::user::grant_privilege::{ActionWithGrantOption, Object}; @@ -585,6 +585,7 @@ impl CatalogManager { #[cfg(not(test))] user_core.ensure_user_id(function.owner)?; + tracing::debug!("create function: {:?}", function); let mut functions = BTreeMapTransaction::new(&mut database_core.functions); functions.insert(function.id, function.clone()); commit_meta!(self, functions)?; @@ -647,7 +648,8 @@ impl CatalogManager { self.start_create_table_procedure_with_source(source, table) .await } else { - self.start_create_table_procedure(table, vec![]).await + self.start_create_table_procedure(table, internal_tables) + .await } } } @@ -765,7 +767,9 @@ impl CatalogManager { /// 2. Not belonging to a background stream job. /// Clean up these hanging tables by the id. pub async fn clean_dirty_tables(&self, fragment_manager: FragmentManagerRef) -> MetaResult<()> { - let creating_tables: Vec = self.list_persisted_creating_tables().await; + let core = &mut *self.core.lock().await; + let database_core = &mut core.database; + let creating_tables: Vec
= database_core.list_persisted_creating_tables(); tracing::debug!( "creating_tables ids: {:#?}", creating_tables.iter().map(|t| t.id).collect_vec() @@ -839,14 +843,13 @@ impl CatalogManager { } } - let core = &mut *self.core.lock().await; - let database_core = &mut core.database; let tables = &mut database_core.tables; let mut tables = BTreeMapTransaction::new(tables); for table in &tables_to_clean { - tracing::debug!("cleaning table_id: {}", table.id); - let table = tables.remove(table.id); - assert!(table.is_some()) + let table_id = table.id; + tracing::debug!("cleaning table_id: {}", table_id); + let table = tables.remove(table_id); + assert!(table.is_some(), "table_id {} missing", table_id) } commit_meta!(self, tables)?; @@ -929,14 +932,8 @@ impl CatalogManager { ); return Ok(()); }; - table - }; - - tracing::trace!("cleanup tables for {}", table.id); - { - let core = &mut self.core.lock().await; - let database_core = &mut core.database; + tracing::trace!("cleanup tables for {}", table.id); let mut table_ids = vec![table.id]; table_ids.extend(internal_table_ids); @@ -944,10 +941,11 @@ impl CatalogManager { let mut tables = BTreeMapTransaction::new(tables); for table_id in table_ids { let res = tables.remove(table_id); - assert!(res.is_some()); + assert!(res.is_some(), "table_id {} missing", table_id); } commit_meta!(self, tables)?; - } + table + }; { let core = &mut self.core.lock().await; @@ -1984,9 +1982,7 @@ impl CatalogManager { let table_key = (table.database_id, table.schema_id, table.name.clone()); assert!( !database_core.sources.contains_key(&source.id) - && !database_core.tables.contains_key(&table.id) - && database_core.has_in_progress_creation(&source_key) - && database_core.has_in_progress_creation(&table_key), + && !database_core.tables.contains_key(&table.id), "table and source must be in creating procedure" ); @@ -2370,6 +2366,46 @@ impl CatalogManager { Ok(()) } + pub async fn comment_on(&self, comment: Comment) -> MetaResult { + let core = &mut *self.core.lock().await; + let database_core = &mut core.database; + + database_core.ensure_database_id(comment.database_id)?; + database_core.ensure_schema_id(comment.schema_id)?; + database_core.ensure_table_id(comment.table_id)?; + + let mut tables = BTreeMapTransaction::new(&mut database_core.tables); + + // unwrap is safe because the table id was ensured before + let mut table = tables.get_mut(comment.table_id).unwrap(); + if let Some(col_idx) = comment.column_index { + let column = table + .columns + .get_mut(col_idx as usize) + .ok_or_else(|| MetaError::catalog_id_not_found("column", col_idx))?; + let column_desc = column.column_desc.as_mut().ok_or_else(|| { + anyhow!( + "column desc at index {} for table id {} not found", + col_idx, + comment.table_id + ) + })?; + column_desc.description = comment.description; + } else { + table.description = comment.description; + } + + let new_table = table.clone(); + + commit_meta!(self, tables)?; + + let version = self + .notify_frontend_relation_info(Operation::Update, RelationInfo::Table(new_table)) + .await; + + Ok(version) + } + pub async fn list_connections(&self) -> Vec { self.core.lock().await.database.list_connections() } diff --git a/src/meta/src/manager/env.rs b/src/meta/src/manager/env.rs index 16a4bcb248b23..28d8200c73ea5 100644 --- a/src/meta/src/manager/env.rs +++ b/src/meta/src/manager/env.rs @@ -16,6 +16,7 @@ use std::ops::Deref; use std::sync::Arc; use risingwave_common::config::{CompactionConfig, DefaultParallelism}; +use risingwave_meta_model_v2::prelude::Cluster; use risingwave_pb::meta::SystemParams; use risingwave_rpc_client::{ConnectorClient, StreamClientPool, StreamClientPoolRef}; use sea_orm::EntityTrait; @@ -28,7 +29,6 @@ use crate::manager::{ NotificationManagerRef, }; use crate::model::ClusterId; -use crate::model_v2::prelude::Cluster; use crate::storage::MetaStoreRef; #[cfg(any(test, feature = "test"))] use crate::storage::{MemStore, MetaStoreBoxExt}; diff --git a/src/meta/src/manager/mod.rs b/src/meta/src/manager/mod.rs index 35642ed0ec143..e7e5208856bc3 100644 --- a/src/meta/src/manager/mod.rs +++ b/src/meta/src/manager/mod.rs @@ -28,7 +28,6 @@ pub use env::{MetaSrvEnv, *}; pub use id::*; pub use idle::*; pub use notification::{LocalNotification, MessageStatus, NotificationManagerRef, *}; +pub use risingwave_meta_model_v2::prelude; pub use streaming_job::*; pub use system_param::*; - -pub use super::model_v2::prelude; diff --git a/src/meta/src/model_v2/ext/hummock.rs b/src/meta/src/model_v2/ext/hummock.rs deleted file mode 100644 index 77111e2e7d202..0000000000000 --- a/src/meta/src/model_v2/ext/hummock.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2023 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use risingwave_pb::hummock::HummockPinnedVersion; -use sea_orm::sea_query::OnConflict; -use sea_orm::ActiveValue::{Set, Unchanged}; -use sea_orm::EntityTrait; - -use crate::model::{MetadataModelResult, Transactional}; -use crate::model_v2::hummock_pinned_version; -use crate::model_v2::trx::Transaction; - -#[async_trait::async_trait] -impl Transactional for HummockPinnedVersion { - async fn upsert_in_transaction( - &self, - trx: &mut crate::model_v2::trx::Transaction, - ) -> MetadataModelResult<()> { - // TODO: error type conversion - // TODO: integer type conversion - let m = hummock_pinned_version::ActiveModel { - context_id: Unchanged(self.context_id.try_into().unwrap()), - min_pinned_id: Set(self.min_pinned_id.try_into().unwrap()), - }; - hummock_pinned_version::Entity::insert(m) - .on_conflict( - OnConflict::column(hummock_pinned_version::Column::ContextId) - .update_columns([hummock_pinned_version::Column::MinPinnedId]) - .to_owned(), - ) - .exec(trx) - .await - .unwrap(); - Ok(()) - } - - async fn delete_in_transaction( - &self, - trx: &mut crate::model_v2::trx::Transaction, - ) -> MetadataModelResult<()> { - // TODO: error type conversion - // TODO: integer type conversion - let id: i32 = self.context_id.try_into().unwrap(); - hummock_pinned_version::Entity::delete_by_id(id) - .exec(trx) - .await - .unwrap(); - Ok(()) - } -} diff --git a/src/meta/src/model_v2/ext/mod.rs b/src/meta/src/model_v2/ext/mod.rs deleted file mode 100644 index 47a5ce8623dc4..0000000000000 --- a/src/meta/src/model_v2/ext/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2023 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod hummock; -pub use hummock::*; diff --git a/src/meta/src/model_v2/migration/Cargo.toml b/src/meta/src/model_v2/migration/Cargo.toml deleted file mode 100644 index d5d51d77da909..0000000000000 --- a/src/meta/src/model_v2/migration/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "model_migration" -version = "0.1.0" -edition = "2021" -publish = false - -[lib] -name = "model_migration" -path = "src/lib.rs" - -[dependencies] -async-std = { version = "1", features = ["attributes", "tokio1"] } -uuid = { version = "1", features = ["v4"] } - -[dependencies.sea-orm-migration] -version = "0.12.0" -features = ["sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-native-tls", "with-uuid"] diff --git a/src/meta/src/model_v2/trx.rs b/src/meta/src/model_v2/trx.rs deleted file mode 100644 index 4bfe6d0261de4..0000000000000 --- a/src/meta/src/model_v2/trx.rs +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright 2023 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub type Transaction = sea_orm::DatabaseTransaction; - -#[cfg(not(madsim))] -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use risingwave_pb::hummock::HummockPinnedVersion; - use sea_orm::{EntityTrait, TransactionTrait}; - - use crate::controller::SqlMetaStore; - use crate::model::{BTreeMapTransaction, ValTransaction, VarTransaction}; - use crate::model_v2::hummock_pinned_version::Model as HummockPinnedVersionModel; - use crate::model_v2::prelude::HummockPinnedVersion as HummockPinnedVersionEntity; - use crate::model_v2::trx::Transaction; - - #[tokio::test] - async fn test_simple_var_transaction_commit() { - let store = SqlMetaStore::for_test().await; - let db = &store.conn; - let mut kv = HummockPinnedVersion { - context_id: 1, - min_pinned_id: 2, - }; - let mut num_txn = VarTransaction::<'_, Transaction, _>::new(&mut kv); - num_txn.min_pinned_id = 3; - assert_eq!(num_txn.min_pinned_id, 3); - let mut txn = db.begin().await.unwrap(); - num_txn.apply_to_txn(&mut txn).await.unwrap(); - txn.commit().await.unwrap(); - let db_val = HummockPinnedVersionEntity::find_by_id(1) - .one(db) - .await - .unwrap() - .unwrap(); - assert_eq!(db_val.min_pinned_id, 3); - num_txn.commit(); - assert_eq!(kv.min_pinned_id, 3); - } - - #[test] - fn test_simple_var_transaction_abort() { - let mut kv = HummockPinnedVersion { - context_id: 1, - min_pinned_id: 11, - }; - let mut num_txn = VarTransaction::<'_, Transaction, _>::new(&mut kv); - num_txn.min_pinned_id = 2; - num_txn.abort(); - assert_eq!(11, kv.min_pinned_id); - } - - #[tokio::test] - async fn test_tree_map_transaction_commit() { - let mut map: BTreeMap = BTreeMap::new(); - // to remove - map.insert( - 1, - HummockPinnedVersion { - context_id: 1, - min_pinned_id: 11, - }, - ); - // to-remove-after-modify - map.insert( - 2, - HummockPinnedVersion { - context_id: 2, - min_pinned_id: 22, - }, - ); - // first - map.insert( - 3, - HummockPinnedVersion { - context_id: 3, - min_pinned_id: 33, - }, - ); - - let mut map_copy = map.clone(); - let mut map_txn = BTreeMapTransaction::new(&mut map); - map_txn.remove(1); - map_txn.insert( - 2, - HummockPinnedVersion { - context_id: 2, - min_pinned_id: 0, - }, - ); - map_txn.remove(2); - // first - map_txn.insert( - 3, - HummockPinnedVersion { - context_id: 3, - min_pinned_id: 333, - }, - ); - // second - map_txn.insert( - 4, - HummockPinnedVersion { - context_id: 4, - min_pinned_id: 44, - }, - ); - assert_eq!( - &HummockPinnedVersion { - context_id: 4, - min_pinned_id: 44 - }, - map_txn.get(&4).unwrap() - ); - // third - map_txn.insert( - 5, - HummockPinnedVersion { - context_id: 5, - min_pinned_id: 55, - }, - ); - assert_eq!( - &HummockPinnedVersion { - context_id: 5, - min_pinned_id: 55 - }, - map_txn.get(&5).unwrap() - ); - - let mut third_entry = map_txn.get_mut(5).unwrap(); - third_entry.min_pinned_id = 555; - assert_eq!( - &HummockPinnedVersion { - context_id: 5, - min_pinned_id: 555 - }, - map_txn.get(&5).unwrap() - ); - - let store = SqlMetaStore::for_test().await; - let db = &store.conn; - let mut txn = db.begin().await.unwrap(); - map_txn.apply_to_txn(&mut txn).await.unwrap(); - txn.commit().await.unwrap(); - - let db_rows: Vec = - HummockPinnedVersionEntity::find().all(db).await.unwrap(); - assert_eq!(db_rows.len(), 3); - assert_eq!( - 1, - db_rows - .iter() - .filter(|m| m.context_id == 3 && m.min_pinned_id == 333) - .count() - ); - assert_eq!( - 1, - db_rows - .iter() - .filter(|m| m.context_id == 4 && m.min_pinned_id == 44) - .count() - ); - assert_eq!( - 1, - db_rows - .iter() - .filter(|m| m.context_id == 5 && m.min_pinned_id == 555) - .count() - ); - map_txn.commit(); - - // replay the change to local copy and compare - map_copy.remove(&1).unwrap(); - map_copy.insert( - 2, - HummockPinnedVersion { - context_id: 2, - min_pinned_id: 22, - }, - ); - map_copy.remove(&2).unwrap(); - map_copy.insert( - 3, - HummockPinnedVersion { - context_id: 3, - min_pinned_id: 333, - }, - ); - map_copy.insert( - 4, - HummockPinnedVersion { - context_id: 4, - min_pinned_id: 44, - }, - ); - map_copy.insert( - 5, - HummockPinnedVersion { - context_id: 5, - min_pinned_id: 555, - }, - ); - assert_eq!(map_copy, map); - } - - #[tokio::test] - async fn test_tree_map_entry_update_transaction_commit() { - let mut map: BTreeMap = BTreeMap::new(); - map.insert( - 1, - HummockPinnedVersion { - context_id: 1, - min_pinned_id: 11, - }, - ); - - let mut map_txn = BTreeMapTransaction::new(&mut map); - let mut first_entry_txn = map_txn.new_entry_txn(1).unwrap(); - first_entry_txn.min_pinned_id = 111; - - let store = SqlMetaStore::for_test().await; - let db = &store.conn; - let mut txn = db.begin().await.unwrap(); - first_entry_txn.apply_to_txn(&mut txn).await.unwrap(); - txn.commit().await.unwrap(); - first_entry_txn.commit(); - - let db_rows: Vec = - HummockPinnedVersionEntity::find().all(db).await.unwrap(); - assert_eq!(db_rows.len(), 1); - assert_eq!( - 1, - db_rows - .iter() - .filter(|m| m.context_id == 1 && m.min_pinned_id == 111) - .count() - ); - assert_eq!(111, map.get(&1).unwrap().min_pinned_id); - } - - #[tokio::test] - async fn test_tree_map_entry_insert_transaction_commit() { - let mut map: BTreeMap = BTreeMap::new(); - - let mut map_txn = BTreeMapTransaction::new(&mut map); - let first_entry_txn = map_txn.new_entry_insert_txn( - 1, - HummockPinnedVersion { - context_id: 1, - min_pinned_id: 11, - }, - ); - let store = SqlMetaStore::for_test().await; - let db = &store.conn; - let mut txn = db.begin().await.unwrap(); - first_entry_txn.apply_to_txn(&mut txn).await.unwrap(); - txn.commit().await.unwrap(); - first_entry_txn.commit(); - assert_eq!(11, map.get(&1).unwrap().min_pinned_id); - } -} diff --git a/src/meta/src/model_v2/worker.rs b/src/meta/src/model_v2/worker.rs deleted file mode 100644 index 08cdb2be34da1..0000000000000 --- a/src/meta/src/model_v2/worker.rs +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2023 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use sea_orm::entity::prelude::*; - -use crate::model_v2::{TransactionId, WorkerId}; - -#[derive(Clone, Debug, Hash, PartialEq, Eq, EnumIter, DeriveActiveEnum)] -#[sea_orm(rs_type = "String", db_type = "String(None)")] -pub enum WorkerType { - #[sea_orm(string_value = "FRONTEND")] - Frontend, - #[sea_orm(string_value = "COMPUTE_NODE")] - ComputeNode, - #[sea_orm(string_value = "RISE_CTL")] - RiseCtl, - #[sea_orm(string_value = "COMPACTOR")] - Compactor, - #[sea_orm(string_value = "META")] - Meta, -} - -#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] -#[sea_orm(rs_type = "String", db_type = "String(None)")] -pub enum WorkerStatus { - #[sea_orm(string_value = "STARTING")] - Starting, - #[sea_orm(string_value = "RUNNING")] - Running, -} - -#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] -#[sea_orm(table_name = "worker")] -pub struct Model { - #[sea_orm(primary_key)] - pub worker_id: WorkerId, - pub worker_type: WorkerType, - pub host: String, - pub port: i32, - pub status: WorkerStatus, - pub transaction_id: Option, -} - -#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation { - #[sea_orm(has_many = "super::worker_property::Entity")] - WorkerProperty, -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::WorkerProperty.def() - } -} - -impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/rpc/ddl_controller.rs b/src/meta/src/rpc/ddl_controller.rs index 36615bd93b757..8f6e7c0be6915 100644 --- a/src/meta/src/rpc/ddl_controller.rs +++ b/src/meta/src/rpc/ddl_controller.rs @@ -24,7 +24,7 @@ use risingwave_common::util::column_index_mapping::ColIndexMapping; use risingwave_common::util::epoch::Epoch; use risingwave_pb::catalog::connection::private_link_service::PbPrivateLinkProvider; use risingwave_pb::catalog::{ - connection, Connection, CreateType, Database, Function, Schema, Source, Table, View, + connection, Comment, Connection, CreateType, Database, Function, Schema, Source, Table, View, }; use risingwave_pb::ddl_service::alter_relation_name_request::Relation; use risingwave_pb::ddl_service::DdlProgress; @@ -103,6 +103,7 @@ pub enum DdlCommand { AlterSourceColumn(Source), CreateConnection(Connection), DropConnection(ConnectionId), + CommentOn(Comment), } #[derive(Clone)] @@ -260,6 +261,7 @@ impl DdlController { ctrl.drop_connection(connection_id).await } DdlCommand::AlterSourceColumn(source) => ctrl.alter_source_column(source).await, + DdlCommand::CommentOn(comment) => ctrl.comment_on(comment).await, } } .in_current_span(); @@ -431,6 +433,7 @@ impl DdlController { let env = StreamEnvironment::from_protobuf(fragment_graph.get_env().unwrap()); + // Persist tables tracing::debug!(id = stream_job.id(), "preparing stream job"); let fragment_graph = self .prepare_stream_job(&mut stream_job, fragment_graph) @@ -1097,7 +1100,7 @@ impl DdlController { } } - pub async fn wait(&self) { + pub async fn wait(&self) -> MetaResult<()> { for _ in 0..30 * 60 { if self .catalog_manager @@ -1105,9 +1108,14 @@ impl DdlController { .await .is_empty() { - break; + return Ok(()); } sleep(Duration::from_secs(1)).await; } + Err(MetaError::cancelled("timeout".into())) + } + + async fn comment_on(&self, comment: Comment) -> MetaResult { + self.catalog_manager.comment_on(comment).await } } diff --git a/src/object_store/src/object/mod.rs b/src/object_store/src/object/mod.rs index 96e58397dfa82..38b826920d158 100644 --- a/src/object_store/src/object/mod.rs +++ b/src/object_store/src/object/mod.rs @@ -46,7 +46,7 @@ pub trait ObjectRangeBounds = RangeBounds + Clone + Send + Sync + std::fm /// Partitions a set of given paths into two vectors. The first vector contains all local paths, and /// the second contains all remote paths. -pub fn partition_object_store_paths(paths: &[String]) -> Vec { +fn partition_object_store_paths(paths: &[String]) -> Vec { // ToDo: Currently the result is a copy of the input. Would it be worth it to use an in-place // partition instead? let mut vec_rem = vec![]; @@ -785,6 +785,11 @@ impl MonitoredObjectStore { } } +/// Creates a new [`ObjectStore`] from the given `url`. Credentials are configured via environment +/// variables. +/// +/// # Panics +/// If the `url` is invalid. Therefore, it is only suitable to be used during startup. pub async fn parse_remote_object_store( url: &str, metrics: Arc, diff --git a/src/prost/build.rs b/src/prost/build.rs index 5722a04767962..12476f60b9ac0 100644 --- a/src/prost/build.rs +++ b/src/prost/build.rs @@ -101,6 +101,9 @@ fn main() -> Result<(), Box> { .type_attribute("data.Datum", "#[derive(Eq, Hash)]") .type_attribute("expr.FunctionCall", "#[derive(Eq, Hash)]") .type_attribute("expr.UserDefinedFunction", "#[derive(Eq, Hash)]") + .type_attribute("expr.UserDefinedFunction.extra", "#[derive(Eq, Hash)]") + .type_attribute("expr.ExternalUdfExtra", "#[derive(Eq, Hash)]") + .type_attribute("expr.WasmUdfExtra", "#[derive(Eq, Hash)]") .type_attribute( "plan_common.ColumnDesc.generated_or_default_column", "#[derive(Eq, Hash)]", diff --git a/src/prost/src/lib.rs b/src/prost/src/lib.rs index a5e0cf0e82664..b89a471c223de 100644 --- a/src/prost/src/lib.rs +++ b/src/prost/src/lib.rs @@ -166,6 +166,25 @@ impl FromStr for crate::expr::table_function::PbType { } } +// They are the same oneof, but different types +impl From for expr::user_defined_function::Extra { + fn from(value: catalog::function::Extra) -> Self { + match value { + catalog::function::Extra::External(v) => Self::External(v), + catalog::function::Extra::Wasm(v) => Self::Wasm(v), + } + } +} + +impl From for catalog::function::Extra { + fn from(value: expr::user_defined_function::Extra) -> Self { + match value { + expr::user_defined_function::Extra::External(v) => Self::External(v), + expr::user_defined_function::Extra::Wasm(v) => Self::Wasm(v), + } + } +} + #[cfg(test)] mod tests { use crate::data::{data_type, DataType}; diff --git a/src/rpc_client/src/meta_client.rs b/src/rpc_client/src/meta_client.rs index 95b746ea33e6c..b8603fbe46e62 100644 --- a/src/rpc_client/src/meta_client.rs +++ b/src/rpc_client/src/meta_client.rs @@ -40,7 +40,8 @@ use risingwave_hummock_sdk::{ use risingwave_pb::backup_service::backup_service_client::BackupServiceClient; use risingwave_pb::backup_service::*; use risingwave_pb::catalog::{ - Connection, PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbTable, PbView, Table, + Connection, PbComment, PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbTable, + PbView, Table, }; use risingwave_pb::cloud_service::cloud_service_client::CloudServiceClient; use risingwave_pb::cloud_service::*; @@ -407,6 +408,14 @@ impl MetaClient { Ok((resp.table_id.into(), resp.version)) } + pub async fn comment_on(&self, comment: PbComment) -> Result { + let request = CommentOnRequest { + comment: Some(comment), + }; + let resp = self.inner.comment_on(request).await?; + Ok(resp.version) + } + pub async fn alter_relation_name( &self, relation: Relation, @@ -1724,6 +1733,7 @@ macro_rules! for_all_meta_rpc { ,{ ddl_client, create_connection, CreateConnectionRequest, CreateConnectionResponse } ,{ ddl_client, list_connections, ListConnectionsRequest, ListConnectionsResponse } ,{ ddl_client, drop_connection, DropConnectionRequest, DropConnectionResponse } + ,{ ddl_client, comment_on, CommentOnRequest, CommentOnResponse } ,{ ddl_client, get_tables, GetTablesRequest, GetTablesResponse } ,{ ddl_client, wait, WaitRequest, WaitResponse } ,{ hummock_client, unpin_version_before, UnpinVersionBeforeRequest, UnpinVersionBeforeResponse } diff --git a/src/source/src/source_desc.rs b/src/source/src/source_desc.rs index 161bbc41ceb63..e6646c03282a0 100644 --- a/src/source/src/source_desc.rs +++ b/src/source/src/source_desc.rs @@ -197,6 +197,7 @@ pub mod test_utils { field_descs: vec![], type_name: "".to_string(), generated_or_default_column: None, + description: None, } .to_protobuf(), ), diff --git a/src/sqlparser/src/ast/mod.rs b/src/sqlparser/src/ast/mod.rs index 5d802bae99cdc..9470724f9a0c7 100644 --- a/src/sqlparser/src/ast/mod.rs +++ b/src/sqlparser/src/ast/mod.rs @@ -2576,6 +2576,7 @@ impl fmt::Display for CreateFunctionBody { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum CreateFunctionUsing { Link(String), + Base64(String), } impl fmt::Display for CreateFunctionUsing { @@ -2583,6 +2584,9 @@ impl fmt::Display for CreateFunctionUsing { write!(f, "USING ")?; match self { CreateFunctionUsing::Link(uri) => write!(f, "LINK '{uri}'"), + CreateFunctionUsing::Base64(s) => { + write!(f, "BASE64 '{s}'") + } } } } diff --git a/src/sqlparser/src/keywords.rs b/src/sqlparser/src/keywords.rs index 4188f06f76ae3..338fffc174bb1 100644 --- a/src/sqlparser/src/keywords.rs +++ b/src/sqlparser/src/keywords.rs @@ -92,6 +92,7 @@ define_keywords!( ATOMIC, AUTHORIZATION, AVG, + BASE64, BEGIN, BEGIN_FRAME, BEGIN_PARTITION, diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs index 5cc094a204268..a6a08ab089ae9 100644 --- a/src/sqlparser/src/parser.rs +++ b/src/sqlparser/src/parser.rs @@ -2303,16 +2303,18 @@ impl Parser { } fn parse_create_function_using(&mut self) -> Result { - let keyword = self.expect_one_of_keywords(&[Keyword::LINK])?; - - let uri = self.parse_literal_string()?; + let keyword = self.expect_one_of_keywords(&[Keyword::LINK, Keyword::BASE64])?; match keyword { - Keyword::LINK => Ok(CreateFunctionUsing::Link(uri)), - _ => self.expected( - "LINK, got {:?}", - TokenWithLocation::wrap(Token::make_keyword(format!("{keyword:?}").as_str())), - ), + Keyword::LINK => { + let uri = self.parse_literal_string()?; + Ok(CreateFunctionUsing::Link(uri)) + } + Keyword::BASE64 => { + let base64 = self.parse_literal_string()?; + Ok(CreateFunctionUsing::Base64(base64)) + } + _ => unreachable!("{}", keyword), } } diff --git a/src/storage/hummock_trace/Cargo.toml b/src/storage/hummock_trace/Cargo.toml index 150b35b79cda0..316447fb41934 100644 --- a/src/storage/hummock_trace/Cargo.toml +++ b/src/storage/hummock_trace/Cargo.toml @@ -25,7 +25,7 @@ tokio = { version = "0.2", package = "madsim-tokio" } tracing = "0.1" [dev-dependencies] -itertools = "0.10.5" +itertools = "0.11" mockall = "0.11.4" [lints] diff --git a/src/storage/src/filter_key_extractor.rs b/src/storage/src/filter_key_extractor.rs index b5a79a6f6b42f..6538042566537 100644 --- a/src/storage/src/filter_key_extractor.rs +++ b/src/storage/src/filter_key_extractor.rs @@ -551,6 +551,7 @@ mod tests { cleaned_by_watermark: false, stream_job_status: PbStreamJobStatus::Created.into(), create_type: PbCreateType::Foreground.into(), + description: None, } } diff --git a/src/tests/compaction_test/src/delete_range_runner.rs b/src/tests/compaction_test/src/delete_range_runner.rs index 346cf2fe6acf8..db64dc6334c04 100644 --- a/src/tests/compaction_test/src/delete_range_runner.rs +++ b/src/tests/compaction_test/src/delete_range_runner.rs @@ -154,6 +154,7 @@ async fn compaction_test( cleaned_by_watermark: false, stream_job_status: PbStreamJobStatus::Created.into(), create_type: PbCreateType::Foreground.into(), + description: None, }; let mut delete_range_table = delete_key_table.clone(); delete_range_table.id = 2; diff --git a/src/udf/Cargo.toml b/src/udf/Cargo.toml index 2d13f39bdddc4..0f8e80c42612f 100644 --- a/src/udf/Cargo.toml +++ b/src/udf/Cargo.toml @@ -13,14 +13,22 @@ normal = ["workspace-hack"] [dependencies] arrow-array = { workspace = true } arrow-flight = { workspace = true } +arrow-ipc = { workspace = true } arrow-schema = { workspace = true } arrow-select = { workspace = true } +base64 = "0.21" +bytes = "1.4" cfg-or-panic = "0.2" futures-util = "0.3.28" +itertools = "0.11" +risingwave_object_store = { workspace = true } static_assertions = "1" thiserror = "1" tokio = { version = "0.2", package = "madsim-tokio", features = ["rt", "macros"] } tonic = { workspace = true } +tracing = "0.1" +wasmtime = { version = "10", features = ["component-model"] } +wasmtime-wasi = { version = "10" } [lints] workspace = true diff --git a/src/udf/src/lib.rs b/src/udf/src/lib.rs index 513551a9108af..f303d56a850ef 100644 --- a/src/udf/src/lib.rs +++ b/src/udf/src/lib.rs @@ -12,8 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![feature(lazy_cell)] +#![feature(lint_reasons)] + mod error; mod external; pub use error::{Error, Result}; pub use external::ArrowFlightUdfClient; +pub mod wasm; diff --git a/src/udf/src/wasm.rs b/src/udf/src/wasm.rs new file mode 100644 index 0000000000000..06c84c9baf7db --- /dev/null +++ b/src/udf/src/wasm.rs @@ -0,0 +1,307 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![expect(dead_code)] + +use std::sync::Arc; + +use bytes::Bytes; +use itertools::Itertools; +use risingwave_object_store::object::object_metrics::ObjectStoreMetrics; +use risingwave_object_store::object::{parse_remote_object_store, ObjectStoreImpl}; +use tokio::sync::Mutex; +use tracing::debug; +use wasmtime::component::{Component, Linker}; +use wasmtime::{Config, Store, WasmBacktraceDetails}; + +pub mod component { + mod bindgen { + wasmtime::component::bindgen!({ + world: "udf", + path: "wit/udf.wit", + async: true // required for wasi + }); + } + pub use bindgen::{EvalErrno, RecordBatch as WasmRecordBatch, Schema, Udf}; +} + +/// Host state +/// +/// Currently this is only a placeholder. No states. +struct WasmState { + wasi_ctx: wasmtime_wasi::preview2::WasiCtx, + table: wasmtime_wasi::preview2::Table, +} + +impl WasmState { + pub fn try_new() -> WasmUdfResult { + let mut table = wasmtime_wasi::preview2::Table::new(); + + let wasi_ctx = wasmtime_wasi::preview2::WasiCtxBuilder::new() + // Note: panic message is printed, and only available in WASI. + // TODO: redirect to tracing to make it clear it's from WASM. + .inherit_stdout() + .inherit_stderr() + .build(&mut table)?; + Ok(Self { wasi_ctx, table }) + } +} + +impl wasmtime_wasi::preview2::WasiView for WasmState { + fn table(&self) -> &wasmtime_wasi::preview2::Table { + &self.table + } + + fn table_mut(&mut self) -> &mut wasmtime_wasi::preview2::Table { + &mut self.table + } + + fn ctx(&self) -> &wasmtime_wasi::preview2::WasiCtx { + &self.wasi_ctx + } + + fn ctx_mut(&mut self) -> &mut wasmtime_wasi::preview2::WasiCtx { + &mut self.wasi_ctx + } +} + +type ArrowResult = std::result::Result; +type WasmtimeResult = std::result::Result; + +pub struct InstantiatedComponent { + store: Arc>>, + bindings: component::Udf, + #[expect(dead_code)] + instance: wasmtime::component::Instance, +} + +use convert::*; +mod convert { + use super::*; + + pub fn to_wasm_batch( + batch: arrow_array::RecordBatch, + ) -> WasmUdfResult { + let mut buf = vec![]; + { + let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut buf, &batch.schema())?; + writer.write(&batch)?; + writer.finish()?; + } + Ok(buf) + } + + pub fn from_wasm_batch( + batch: &component::WasmRecordBatch, + ) -> WasmUdfResult> + '_> { + let reader = arrow_ipc::reader::StreamReader::try_new(&batch[..], None).unwrap(); + + Ok(reader) + } + + // pub fn from_wasm_schema(schema: &component::Schema) -> WasmUdfResult + // { } +} + +impl InstantiatedComponent { + pub async fn eval( + &self, + input: arrow_array::RecordBatch, + ) -> WasmUdfResult { + // let input_schema = self.bindings.call_input_schema(&mut self.store)?; + // let output_schema = self.bindings.call_output_schema(&mut self.store)?; + + let input = to_wasm_batch(input)?; + // TODO: Use tokio Mutex to use it across the await here. Does it make sense? + let result = self + .bindings + .call_eval(&mut *self.store.lock().await, &input) + .await??; + let result = from_wasm_batch(&result)?; + let Some((record_batch,)) = result.collect_tuple() else { + return Err(WasmUdfError::Encoding( + "should return only one record batch in IPC buffer".to_string(), + )); + }; + Ok(record_batch?) + } +} + +/// The interface to interact with the wasm engine. +/// +/// It can be safely shared across threads and is a cheap cloneable handle to the actual engine. +#[derive(Clone)] +pub struct WasmEngine { + engine: wasmtime::Engine, +} + +impl WasmEngine { + #[expect(clippy::new_without_default)] + pub fn new() -> Self { + // Is this expensive? + let mut config = Config::new(); + config + .wasm_component_model(true) + // required for wasi + .async_support(true) + .wasm_backtrace(true) + .wasm_backtrace_details(WasmBacktraceDetails::Enable); + + Self { + engine: wasmtime::Engine::new(&config).expect("failed to create wasm engine"), + } + } + + pub fn get_or_create() -> Self { + use std::sync::LazyLock; + static WASM_ENGINE: LazyLock = LazyLock::new(WasmEngine::new); + WASM_ENGINE.clone() + } + + pub async fn compile_and_upload_component( + &self, + binary: Vec, + wasm_storage_url: &str, + identifier: &str, + ) -> WasmUdfResult<()> { + let object_store = get_wasm_storage(wasm_storage_url).await?; + let binary: Bytes = binary.into(); + object_store + .upload(&raw_path(identifier), binary.clone()) + .await?; + + // This is expensive. + let component = Component::from_binary(&self.engine, &binary[..])?; + tracing::info!("wasm component loaded"); + + // This function is similar to the Engine::precompile_module method where it produces an + // artifact of Wasmtime which is suitable to later pass into Module::deserialize. If a + // module is never instantiated then it’s recommended to use Engine::precompile_module + // instead of this method, but if a module is both instantiated and serialized then this + // method can be useful to get the serialized version without compiling twice. + let serialized = component.serialize()?; + debug!( + "compile component, size: {} -> {}", + binary.len(), + serialized.len() + ); + + // check the component can be instantiated + let mut linker = Linker::new(&self.engine); + // A Store is intended to be a short-lived object in a program. No form of GC is + // implemented at this time so once an instance is created within a Store it will not be + // deallocated until the Store itself is dropped. This makes Store unsuitable for + // creating an unbounded number of instances in it because Store will never release this + // memory. It's recommended to have a Store correspond roughly to the lifetime of a + // "main instance" that an embedding is interested in executing. + + // So creating a Store is cheap? + + let mut store = Store::new(&self.engine, WasmState::try_new()?); + wasmtime_wasi::preview2::wasi::command::add_to_linker(&mut linker)?; + let (_bindings, _instance) = + component::Udf::instantiate_async(&mut store, &component, &linker).await?; + + object_store + .upload(&compiled_path(identifier), serialized.into()) + .await?; + + tracing::debug!( + path = compiled_path(identifier), + "wasm component compiled and uploaded", + ); + + Ok(()) + } + + pub async fn load_component( + &self, + wasm_storage_url: &str, + identifier: &str, + ) -> WasmUdfResult { + let object_store = get_wasm_storage(wasm_storage_url).await?; + let serialized_component = object_store.read(&compiled_path(identifier), ..).await?; + + // This is fast. + let component = unsafe { + // safety: it's serialized by ourself + // https://docs.rs/wasmtime/latest/wasmtime/struct.Module.html#unsafety-1 + Component::deserialize(&self.engine, serialized_component)? + }; + + let mut linker = Linker::new(&self.engine); + let mut store = Store::new(&self.engine, WasmState::try_new()?); + wasmtime_wasi::preview2::wasi::command::add_to_linker(&mut linker)?; + let (bindings, instance) = + component::Udf::instantiate_async(&mut store, &component, &linker).await?; + + Ok(InstantiatedComponent { + store: Arc::new(Mutex::new(store)), + bindings, + instance, + }) + } +} + +pub type WasmUdfResult = std::result::Result; + +#[derive(thiserror::Error, Debug)] +pub enum WasmUdfError { + #[error("wasm error: {0:#}")] + Wasmtime(#[from] wasmtime::Error), + #[error("arrow error: {0}")] + Arrow(#[from] arrow_schema::ArrowError), + #[error("eval error: {0}")] + Eval(#[from] component::EvalErrno), + #[error("{0}")] + Encoding(String), + #[error("object store error: {0}")] + ObjectStore(#[from] Box), + #[error("object store error: {0}")] + ObjectStore1(String), +} + +const WASM_RAW_MODULE_DIR: &str = "wasm/raw"; +const WASM_COMPILED_MODULE_DIR: &str = "wasm/compiled"; + +fn raw_path(identifier: &str) -> String { + format!("{}/{}", WASM_RAW_MODULE_DIR, identifier) +} + +fn compiled_path(identifier: &str) -> String { + format!("{}/{}", WASM_COMPILED_MODULE_DIR, identifier) +} + +async fn get_wasm_storage(wasm_storage_url: &str) -> WasmUdfResult { + if wasm_storage_url.starts_with("memory") { + // because we create a new store every time... + return Err(WasmUdfError::ObjectStore1( + "memory storage is not supported".to_string(), + )); + } + // Note: it will panic if the url is invalid. We did a validation on meta startup. + let object_store = parse_remote_object_store( + wasm_storage_url, + Arc::new(ObjectStoreMetrics::unused()), + "Wasm Engine", + ) + .await; + Ok(object_store) +} + +impl From for WasmUdfError { + fn from(e: risingwave_object_store::object::ObjectError) -> Self { + WasmUdfError::ObjectStore(Box::new(e)) + } +} diff --git a/src/udf/wit/udf.wit b/src/udf/wit/udf.wit new file mode 100644 index 0000000000000..22b629b8d8cbc --- /dev/null +++ b/src/udf/wit/udf.wit @@ -0,0 +1,49 @@ +package risingwave:udf // Maybe we can use a different package name? udf as a world name is enough. + +world udf { + use types.{schema, record-batch, eval-errno} + + // TODO: is schema needed? since record-batch already contains schema. + export input-schema: func() -> schema + export output-schema: func() -> schema + + // export init: func(inputs: list) -> result<_, init-errno> + export eval: func(batch: record-batch) -> result +} + +interface types { + // in Arrow IPC Streaming Format + type record-batch = list + + enum data-type { + dt-i16, + dt-i32, + dt-i64, + dt-bool, + dt-string, + } + + type schema = list + + record field { + name: string, + data-type: data-type, + } + + enum init-errno { + invalid-params, + } + + enum eval-errno { + numeric-overflow, + division-by-zero, + } + + union scalar { + s16, + s32, + s64, + bool, + string, + } +} diff --git a/src/udf/wit_example/.gitignore b/src/udf/wit_example/.gitignore new file mode 100644 index 0000000000000..cb53a1ef0fa8d --- /dev/null +++ b/src/udf/wit_example/.gitignore @@ -0,0 +1,4 @@ +*.sql +*.wasm +Cargo.lock +tinygo/gen diff --git a/src/udf/wit_example/README.md b/src/udf/wit_example/README.md new file mode 100644 index 0000000000000..39ae9984dd87a --- /dev/null +++ b/src/udf/wit_example/README.md @@ -0,0 +1,54 @@ +# WASM UDF examples + +TODO: +- [ ] error handing +- [ ] schema validation + +## Required tools + +- [wasm-tools](https://github.com/bytecodealliance/wasm-tools): to create WASM component from WASM module. +- [wit-bindgen](https://github.com/bytecodealliance/wit-bindgen) CLI: to generate guest code from WIT file. (Not required for Rust guest) + +``` +cargo install wasm-tools@1.0.35 +cargo install wit-bindgen-cli@0.8.0 +``` + +> **Note** +> +> WASM component model IS NOT stable and may change. Please use the version specified above. + +## Examples for different guest languages + +Refer to each language's directory for an example. Some additional notes are listed below. +Generally you will just need to copy the `wit` directory and follow the examples. + +It's not guaranteed to work if you used different versions of toolchains and project dependencies. + +### Rust + +nothing special + +### Golang + +#### TinyGo + +[TinyGo](https://tinygo.org/getting-started/install/) is an alternative Go compiler for small devices. It also supports WASM. + +tested under +``` +> tinygo version +tinygo version 0.28.1 darwin/amd64 (using go version go1.20.2 and LLVM version 15.0.0) +``` + +- TinyGo cannot compile the lz4 package ([Add support for reading Go assembly files by aykevl · Pull Request #3103 · tinygo-org/tinygo](https://github.com/tinygo-org/tinygo/pull/3103)), which is used by Arrow. Can workaround by using the forked version of arrow, which removed lz4. + +``` +replace github.com/apache/arrow/go/v13 => github.com/xxchan/arrow/go/v13 v13.0.0-20230713134335-45002b4934f9 +``` + +#### TODO: Go 1.21 + +(requires full wasi_snapshot_preview1) +- [Go 1.21 Release Notes - The Go Programming Language](https://tip.golang.org/doc/go1.21) +- [all: add GOOS=wasip1 GOARCH=wasm port · Issue #58141 · golang/go](https://github.com/golang/go/issues/58141) diff --git a/src/udf/wit_example/build.sh b/src/udf/wit_example/build.sh new file mode 100755 index 0000000000000..01b70932cc7a3 --- /dev/null +++ b/src/udf/wit_example/build.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +set -e + +# usage: ./build.sh --lang [rust|go] [--rebuild] + +while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + --lang) + lang="$2" + shift # past argument + shift # past value + ;; + --rebuild) + rebuild="true" + shift # past argument + ;; + *) # unknown option + shift # past argument + ;; + esac +done + +if [ -z "$lang" ]; then + echo "Please specify --lang [rust|go]" + exit 1 +fi + +if [ "$(wasm-tools -V)" != "wasm-tools 1.0.35" ]; then + echo "wasm-tools 1.0.35 is required" + exit 1 +fi + +path=$(dirname "$0") + +function build_rust() { + echo "--- Build Rust guest component" + + cd "$path/rust" + + rustup target add wasm32-wasi + + profile=release + if [ "$profile" == "dev" ]; then + target_dir=debug + else + target_dir=$profile + fi + + cargo build --target=wasm32-wasi --profile "${profile}" + mv ./target/wasm32-wasi/"${target_dir}"/my_udf.wasm ../my_udf.rust.wasm + + cd .. +} + +function build_go() { + echo "--- Build TinyGo guest component" + # Note: TinyGo will rebuild the whole binary every time and it's slow. + + cd "$path/tinygo" + go generate # generate bindings for Go + tinygo build -target=wasi -o my_udf.go.wasm my_udf.go + wasm-tools component embed ../../wit my_udf.go.wasm -o my_udf.go.wasm + + mv ./my_udf.go.wasm .. + cd .. +} + +# if the file "my_udf.$lang.wasm" does not exist, or --rebuild is specified, rebuild it +if [ ! -f "my_udf.$lang.wasm" ] || [ "$rebuild" == "true" ]; then + if [ "$lang" == "rust" ]; then + build_rust + elif [ "$lang" == "go" ]; then + build_go + else + echo "Unknown language: $lang" + exit 1 + fi +else + echo "my_udf.$lang.wasm exists, skip building" +fi + + + +# (WASI adaptor) if file not found, download from +if [ ! -f wasi_snapshot_preview1.reactor.wasm ]; then + wget https://github.com/bytecodealliance/wasmtime/releases/download/v10.0.1/wasi_snapshot_preview1.reactor.wasm +fi + +echo wasm-tools component new "my_udf.$lang.wasm" -o my_udf.component.wasm +wasm-tools component new "my_udf.$lang.wasm" -o my_udf.component.wasm \ + --adapt wasi_snapshot_preview1=./wasi_snapshot_preview1.reactor.wasm +wasm-tools validate my_udf.component.wasm --features component-model diff --git a/src/udf/wit_example/create.sh b/src/udf/wit_example/create.sh new file mode 100755 index 0000000000000..f55e94683a2c5 --- /dev/null +++ b/src/udf/wit_example/create.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +path=$(dirname "$0") +cd "$path" + +if [ ! -f "./my_udf.component.wasm" ]; then + echo "my_udf.component.wasm not found, please run ./build.sh first" + exit 1 +fi + +echo "size of wasm: $(stat -f "%z" my_udf.component.wasm) bytes" +encoded=$(base64 -i my_udf.component.wasm) +echo "size of encoded wasm: ${#encoded} bytes" +# debug: 23557258 +# release: 12457072 + +psql -h localhost -p 4566 -d dev -U root -c "DROP FUNCTION IF EXISTS count_char;" +sql="CREATE FUNCTION count_char (s varchar, c varchar) RETURNS BIGINT LANGUAGE wasm_v1 USING BASE64 '$encoded';" +echo "$sql" > create.sql +psql -h localhost -p 4566 -d dev -U root -v "ON_ERROR_STOP=1" -f ./create.sql + +# test +psql -h localhost -p 4566 -d dev -U root -c "SELECT count_char('aabca', 'a');" +psql -h localhost -p 4566 -d dev -U root -c "SELECT count_char('aabca', 'b');" +psql -h localhost -p 4566 -d dev -U root -c "SELECT count_char('aabca', 'd');" diff --git a/src/udf/wit_example/rust/Cargo.toml b/src/udf/wit_example/rust/Cargo.toml new file mode 100644 index 0000000000000..76a25356a885a --- /dev/null +++ b/src/udf/wit_example/rust/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "my_udf" +version = "0.1.0" +edition = "2021" +homepage = "https://github.com/risingwavelabs/risingwave" +license = "Apache-2.0" +repository = "https://github.com/risingwavelabs/risingwave" + +[workspace] + +[dependencies] +wit-bindgen = "0.8" +arrow-array = "43" +arrow-schema = "43" +arrow-ipc = "43" +arrow = "43" +arrow-data = "43" +arrow-buffer = "43" + + +[lib] +crate-type = ["cdylib"] + +[profile.release] +debug = 1 \ No newline at end of file diff --git a/src/udf/wit_example/rust/src/lib.rs b/src/udf/wit_example/rust/src/lib.rs new file mode 100644 index 0000000000000..2c90529aea8b1 --- /dev/null +++ b/src/udf/wit_example/rust/src/lib.rs @@ -0,0 +1,118 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use arrow_array::Array; + +use crate::risingwave::udf::types::{DataType, Field}; + +wit_bindgen::generate!({ + // optional, since there's only one world. We make it explicit here. + world: "udf", + // path is relative to Cargo.toml + path:"../../wit" +}); + +// Define a custom type and implement the generated `Udf` trait for it which +// represents implementing all the necesssary exported interfaces for this +// component. +/// User defined function tou count number of specified characters. +/// Ref +struct CountChar; + +export_udf!(CountChar); + +fn count_char(s: &str, char: &str) -> i64 { + let mut count = 0; + let char = char.bytes().next().unwrap(); + + for c in s.bytes() { + if c == char { + count += 1; + } + } + count +} + +impl Udf for CountChar { + fn eval(batch: RecordBatch) -> Result { + // Read data from IPC buffer + let batch = arrow_ipc::reader::StreamReader::try_new(batch.as_slice(), None).unwrap(); + + // Do UDF computation (for each batch, for each row, do scalar -> scalar) + let mut ret = arrow_array::builder::Int64Builder::new(); + for batch in batch { + let batch = batch.unwrap(); + for i in 0..batch.num_rows() { + let s = batch + .column(0) + .as_any() + .downcast_ref::() + .expect( + format!( + "expected StringArray, got {:?}", + batch.column(0).data_type() + ) + .as_str(), + ) + .value(i); + let c = batch + .column(1) + .as_any() + .downcast_ref::() + .expect( + format!( + "expected StringArray, got {:?}", + batch.column(1).data_type() + ) + .as_str(), + ) + .value(i); + ret.append_value(count_char(s, c)); + } + } + + // Write data to IPC buffer + let mut buf = vec![]; + { + let array = ret.finish(); + let schema = arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "result", + arrow_schema::DataType::Int64, + false, + )]); + let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut buf, &schema).unwrap(); + let batch = + arrow_array::RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + } + Ok(buf) + } + + fn input_schema() -> Schema { + vec![Field { + name: "input".to_string(), + data_type: DataType::DtString, + }] + } + + fn output_schema() -> Schema { + vec![Field { + name: "result".to_string(), + data_type: DataType::DtI64, + }] + } +} diff --git a/src/udf/wit_example/tinygo/go.mod b/src/udf/wit_example/tinygo/go.mod new file mode 100644 index 0000000000000..9e2110de9ad42 --- /dev/null +++ b/src/udf/wit_example/tinygo/go.mod @@ -0,0 +1,19 @@ +module github.com/my_account/my_udf + +go 1.20 + +require github.com/apache/arrow/go/v13 v13.0.0-20230712165359-085a0baf7868 + +require ( + github.com/goccy/go-json v0.10.0 // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/tools v0.6.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect +) + +replace github.com/apache/arrow/go/v13 => github.com/xxchan/arrow/go/v13 v13.0.0-20230713134335-45002b4934f9 diff --git a/src/udf/wit_example/tinygo/go.sum b/src/udf/wit_example/tinygo/go.sum new file mode 100644 index 0000000000000..347e7e70feda6 --- /dev/null +++ b/src/udf/wit_example/tinygo/go.sum @@ -0,0 +1,30 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= +github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= +github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/xxchan/arrow/go/v13 v13.0.0-20230713134335-45002b4934f9 h1:gsKJOVHt6vUME28wICV67tIIGAxpIsVJN3Ip3IuxNcU= +github.com/xxchan/arrow/go/v13 v13.0.0-20230713134335-45002b4934f9/go.mod h1:t8LhfrObSFaJZGpfHtA6BSRnP0sl0yaK/rDHBEdvWBU= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/src/udf/wit_example/tinygo/my_udf.go b/src/udf/wit_example/tinygo/my_udf.go new file mode 100644 index 0000000000000..4fc14a463d6c6 --- /dev/null +++ b/src/udf/wit_example/tinygo/my_udf.go @@ -0,0 +1,89 @@ +package main + +import ( + "bytes" + + "github.com/apache/arrow/go/v13/arrow" + "github.com/apache/arrow/go/v13/arrow/array" + "github.com/apache/arrow/go/v13/arrow/ipc" + "github.com/apache/arrow/go/v13/arrow/memory" + gen "github.com/my_account/my_udf/gen" +) + +type MyUdf struct { +} + +func init() { + a := MyUdf{} + gen.SetUdf(a) +} + +func (u MyUdf) InputSchema() gen.RisingwaveUdfTypesSchema { + return gen.RisingwaveUdfTypesSchema{} +} +func (u MyUdf) OutputSchema() gen.RisingwaveUdfTypesSchema { + return gen.RisingwaveUdfTypesSchema{} +} +func (u MyUdf) Eval(batch []uint8) gen.Result[[]uint8, gen.RisingwaveUdfTypesEvalErrno] { + reader, err := ipc.NewReader(bytes.NewReader(batch)) + if err != nil { + panic(err) + } + builder := array.NewBooleanBuilder(memory.NewGoAllocator()) + for reader.Next() { + rec := reader.Record() + col := rec.Column(0).(*array.Int64).Int64Values() + + for i := 0; i < int(rec.NumRows()); i++ { + builder.Append(col[i] > 0) + } + } + arr := builder.NewArray() + schema := arrow.NewSchema( + []arrow.Field{ + {Name: "result", Type: &arrow.BooleanType{}}, + }, + nil, + ) + record := array.NewRecord(schema, []arrow.Array{arr}, int64(arr.Len())) + + buffer := new(bytes.Buffer) + writer := ipc.NewWriter(buffer, ipc.WithSchema(record.Schema())) + writer.Write(record) + + return gen.Result[[]uint8, gen.RisingwaveUdfTypesEvalErrno]{ + Kind: gen.Ok, + Val: buffer.Bytes(), + Err: gen.RisingwaveUdfTypesEvalErrno{}, + } +} + +//go:generate wit-bindgen tiny-go ../../wit --out-dir=gen +func main() { + // Just for testing + + builder := array.NewInt64Builder(memory.NewGoAllocator()) + builder.Append(-1) + builder.Append(0) + builder.Append(1) + arr := builder.NewArray() + record := array.NewRecord( + arrow.NewSchema( + []arrow.Field{ + {Name: "input", Type: &arrow.Int64Type{}}, + }, + nil, + ), + []arrow.Array{arr}, + int64(arr.Len()), + ) + + buffer := new(bytes.Buffer) + writer := ipc.NewWriter(buffer, ipc.WithSchema(record.Schema())) + writer.Write(record) + + udf := MyUdf{} + + result := udf.Eval(buffer.Bytes()) + println("result:", result.Kind) +} diff --git a/src/utils/pgwire/src/pg_protocol.rs b/src/utils/pgwire/src/pg_protocol.rs index ff705025a0d64..d2b6770211c09 100644 --- a/src/utils/pgwire/src/pg_protocol.rs +++ b/src/utils/pgwire/src/pg_protocol.rs @@ -52,7 +52,7 @@ static RW_QUERY_LOG_TRUNCATE_LEN: LazyLock = Ok(len) if len.parse::().is_ok() => len.parse::().unwrap(), _ => { if cfg!(debug_assertions) { - usize::MAX + 65536 } else { 1024 } diff --git a/src/utils/pgwire/src/pg_response.rs b/src/utils/pgwire/src/pg_response.rs index eeec929732f50..802f651ce4298 100644 --- a/src/utils/pgwire/src/pg_response.rs +++ b/src/utils/pgwire/src/pg_response.rs @@ -53,6 +53,7 @@ pub enum StatementType { CREATE_INDEX, CREATE_FUNCTION, CREATE_CONNECTION, + COMMENT, DESCRIBE, GRANT_PRIVILEGE, DROP_TABLE, diff --git a/src/utils/runtime/src/logger.rs b/src/utils/runtime/src/logger.rs index 916dd93d7a32b..a86c585c0a3fd 100644 --- a/src/utils/runtime/src/logger.rs +++ b/src/utils/runtime/src/logger.rs @@ -156,7 +156,6 @@ pub fn init_risingwave_logger(settings: LoggerSettings) { .with_target("aws_sdk_ec2", Level::INFO) .with_target("aws_sdk_s3", Level::INFO) .with_target("aws_config", Level::WARN) - // Only enable WARN and ERROR for 3rd-party crates .with_target("aws_endpoint", Level::WARN) .with_target("aws_credential_types::cache::lazy_caching", Level::WARN) .with_target("hyper", Level::WARN) @@ -166,7 +165,9 @@ pub fn init_risingwave_logger(settings: LoggerSettings) { .with_target("isahc", Level::WARN) .with_target("console_subscriber", Level::WARN) .with_target("reqwest", Level::WARN) - .with_target("sled", Level::INFO); + .with_target("sled", Level::INFO) + .with_target("cranelift", Level::INFO) + .with_target("wasmtime", Level::INFO); // For all other crates, apply default level depending on the deployment and `debug_assertions` flag. let default_level = match deployment { diff --git a/src/workspace-hack/Cargo.toml b/src/workspace-hack/Cargo.toml index 67b218c787652..d21f92780d053 100644 --- a/src/workspace-hack/Cargo.toml +++ b/src/workspace-hack/Cargo.toml @@ -34,6 +34,7 @@ chrono = { version = "0.4", features = ["serde"] } clap = { version = "4", features = ["cargo", "derive", "env"] } clap_builder = { version = "4", default-features = false, features = ["cargo", "color", "env", "help", "std", "suggestions", "usage"] } combine = { version = "4", features = ["tokio"] } +crc32fast = { version = "1" } crossbeam-epoch = { version = "0.9" } crossbeam-queue = { version = "0.3" } crossbeam-utils = { version = "0.8" } @@ -41,6 +42,7 @@ deranged = { version = "0.3", default-features = false, features = ["powerfmt", digest = { version = "0.10", features = ["mac", "oid", "std"] } either = { version = "1", features = ["serde"] } fail = { version = "0.5", default-features = false, features = ["failpoints"] } +fallible-iterator = { version = "0.2" } fixedbitset = { version = "0.4" } flate2 = { version = "1", features = ["zlib"] } frunk_core = { version = "0.4", default-features = false, features = ["std"] } @@ -52,9 +54,10 @@ futures-sink = { version = "0.3" } futures-task = { version = "0.3" } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["nightly", "raw"] } +hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13", features = ["raw"] } hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12", features = ["nightly", "raw"] } hyper = { version = "0.14", features = ["full"] } -indexmap = { version = "1", default-features = false, features = ["serde", "std"] } +indexmap = { version = "1", default-features = false, features = ["serde-1", "std"] } itertools-93f6ce9d446188ac = { package = "itertools", version = "0.10" } itertools-a6292c17cd707f01 = { package = "itertools", version = "0.11" } jni = { version = "0.21", features = ["invocation"] } @@ -100,6 +103,8 @@ regex-syntax = { version = "0.8" } reqwest = { version = "0.11", features = ["blocking", "json", "rustls-tls"] } ring = { version = "0.16", features = ["std"] } rust_decimal = { version = "1", features = ["db-postgres", "maths"] } +rustc-hash = { version = "1" } +rustix = { version = "0.38", features = ["fs"] } rustls = { version = "0.21" } scopeguard = { version = "1" } sea-orm = { version = "0.12", features = ["runtime-tokio-native-tls", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite"] } @@ -117,6 +122,7 @@ sqlx-core = { version = "0.7", features = ["_rt-tokio", "_tls-native-tls", "bigd sqlx-mysql = { version = "0.7", default-features = false, features = ["bigdecimal", "chrono", "json", "rust_decimal", "time", "uuid"] } sqlx-postgres = { version = "0.7", default-features = false, features = ["bigdecimal", "chrono", "json", "rust_decimal", "time", "uuid"] } sqlx-sqlite = { version = "0.7", default-features = false, features = ["chrono", "json", "time", "uuid"] } +stable_deref_trait = { version = "1" } strum = { version = "0.25", features = ["derive"] } subtle = { version = "2" } time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } @@ -152,6 +158,8 @@ either = { version = "1", features = ["serde"] } fixedbitset = { version = "0.4" } frunk_core = { version = "0.4", default-features = false, features = ["std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["nightly", "raw"] } +hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12", features = ["nightly", "raw"] } +indexmap = { version = "1", default-features = false, features = ["serde-1", "std"] } itertools-93f6ce9d446188ac = { package = "itertools", version = "0.10" } itertools-a6292c17cd707f01 = { package = "itertools", version = "0.11" } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } @@ -174,13 +182,19 @@ rand_core = { version = "0.6", default-features = false, features = ["std"] } regex = { version = "1" } regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } regex-syntax = { version = "0.8" } +rustc-hash = { version = "1" } +rustix = { version = "0.38", features = ["fs"] } serde = { version = "1", features = ["alloc", "derive", "rc"] } serde_json = { version = "1", features = ["alloc", "raw_value"] } syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit", "visit-mut"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] } +tinyvec = { version = "1", features = ["alloc", "grab_spare_slice", "rustc_1_55"] } toml_datetime = { version = "0.6", default-features = false, features = ["serde"] } toml_edit = { version = "0.19", features = ["serde"] } +unicode-bidi = { version = "0.3" } +unicode-normalization = { version = "0.1" } +url = { version = "2", features = ["serde"] } ### END HAKARI SECTION