diff --git a/Cargo.lock b/Cargo.lock index f02546622eff..98467d29c021 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,9 +113,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -151,12 +151,12 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -981,7 +981,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.0.0", + "fastrand 2.0.1", "http 0.2.9", "hyper", "time", @@ -1032,7 +1032,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "fastrand 2.0.0", + "fastrand 2.0.1", "http 0.2.9", "percent-encoding", "tracing", @@ -1057,7 +1057,7 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "fastrand 2.0.0", + "fastrand 2.0.1", "http 0.2.9", "regex", "tracing", @@ -1260,7 +1260,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "fastrand 2.0.0", + "fastrand 2.0.1", "http 0.2.9", "http-body", "hyper", @@ -1866,7 +1866,7 @@ checksum = "6ffc30dee200c20b4dcb80572226f42658e1d9c4b668656d7cc59c33d50e396e" dependencies = [ "cap-primitives", "cap-std", - "rustix 0.38.28", + "rustix 0.38.31", "smallvec", ] @@ -1882,7 +1882,7 @@ dependencies = [ "io-lifetimes 2.0.3", "ipnet", "maybe-owned", - "rustix 0.38.28", + "rustix 0.38.31", "windows-sys 0.48.0", "winx", ] @@ -1906,7 +1906,7 @@ dependencies = [ "cap-primitives", "io-extras", "io-lifetimes 2.0.3", - "rustix 0.38.28", + "rustix 0.38.31", ] [[package]] @@ -1917,7 +1917,7 @@ checksum = "f8f52b3c8f4abfe3252fd0a071f3004aaa3b18936ec97bdbd8763ce03aff6247" dependencies = [ "cap-primitives", "once_cell", - "rustix 0.38.28", + "rustix 0.38.31", "winx", ] @@ -2076,9 +2076,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.4" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" +checksum = "c918d541ef2913577a0f9566e9ce27cb35b6df072075769e0b26cb5a554520da" dependencies = [ "clap_builder", "clap_derive", @@ -2086,21 +2086,21 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.4" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" +checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.0", ] [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" dependencies = [ "heck 0.4.1", "proc-macro2", @@ -2110,9 +2110,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "clickhouse" @@ -2523,9 +2523,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -2859,7 +2859,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 1.0.109", ] @@ -2873,7 +2873,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 1.0.109", ] @@ -2887,7 +2887,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 2.0.48", ] @@ -3596,9 +3596,9 @@ dependencies = [ [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" dependencies = [ "serde", ] @@ -3916,9 +3916,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fd-lock" @@ -3927,7 +3927,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b93f7a0db71c99f68398f80653ed05afb0b00e062e1a20c7ff849c4edfabbbcc" dependencies = [ "cfg-if", - "rustix 0.38.28", + "rustix 0.38.31", "windows-sys 0.52.0", ] @@ -4291,7 +4291,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "033b337d725b97690d86893f9de22b67b80dcc4e9ad815f348254c38119db8fb" dependencies = [ "io-lifetimes 2.0.3", - "rustix 0.38.28", + "rustix 0.38.31", "windows-sys 0.52.0", ] @@ -5278,7 +5278,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.28", + "rustix 0.38.31", "windows-sys 0.48.0", ] @@ -5557,9 +5557,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.150" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libflate" @@ -6039,9 +6039,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.6.3" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memcomparable" @@ -6061,7 +6061,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" dependencies = [ - "rustix 0.38.28", + "rustix 0.38.31", ] [[package]] @@ -7751,7 +7751,7 @@ dependencies = [ "hex", "lazy_static", "procfs-core", - "rustix 0.38.28", + "rustix 0.38.31", ] [[package]] @@ -7896,9 +7896,9 @@ dependencies = [ [[package]] name = "prost-reflect" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057237efdb71cf4b3f9396302a3d6599a92fa94063ba537b66130980ea9909f3" +checksum = "9ae9372e3227f3685376a0836e5c248611eafc95a0be900d44bc6cdf225b700f" dependencies = [ "once_cell", "prost 0.12.1", @@ -9007,6 +9007,7 @@ dependencies = [ "jsonschema-transpiler", "madsim-rdkafka", "madsim-tokio", + "madsim-tonic", "maplit", "moka", "mysql_async", @@ -9053,7 +9054,6 @@ dependencies = [ "tokio-retry", "tokio-stream", "tokio-util", - "tonic 0.10.2", "tracing", "tracing-subscriber", "tracing-test", @@ -9145,6 +9145,7 @@ dependencies = [ name = "risingwave_error" version = "1.7.0-alpha" dependencies = [ + "anyhow", "bincode 1.3.3", "bytes", "easy-ext", @@ -9552,6 +9553,7 @@ dependencies = [ name = "risingwave_meta_model_v2" version = "1.7.0-alpha" dependencies = [ + "prost 0.12.1", "risingwave_common", "risingwave_hummock_sdk", "risingwave_pb", @@ -10280,9 +10282,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ "bitflags 2.4.0", "errno", @@ -10460,9 +10462,9 @@ dependencies = [ [[package]] name = "sea-orm" -version = "0.12.2" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61f6c7daef05dde3476d97001e11fca7a52b655aa3bf4fd610ab2da1176a2ed5" +checksum = "6632f499b80cc6aaa781b302e4c9fae663e0e3dcf2640e9d80034d5b10731efe" dependencies = [ "async-stream", "async-trait", @@ -10488,9 +10490,9 @@ dependencies = [ [[package]] name = "sea-orm-cli" -version = "0.12.2" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e3f0ff2fa5672e2e7314d107c6498a18e469beeb340a0ed84e3075fce73c2cd" +checksum = "465ea2308d4716837e9af4a2cff8e14c28135867a580bb93e9e03d408a3a6afb" dependencies = [ "chrono", "clap", @@ -10505,9 +10507,9 @@ dependencies = [ [[package]] name = "sea-orm-macros" -version = "0.12.2" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd90e73d5f5b184bad525767da29fbfec132b4e62ebd6f60d2f2737ec6468f62" +checksum = "ec13bfb4c4aef208f68dbea970dd40d13830c868aa8dcb4e106b956e6bb4f2fa" dependencies = [ "heck 0.4.1", "proc-macro2", @@ -10519,9 +10521,9 @@ dependencies = [ [[package]] name = "sea-orm-migration" -version = "0.12.2" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21f673fcefb3a7e7b89a12b6c0e854ec0be14367635ac3435369c8ad7f11e09e" +checksum = "ac734b6e5610c2764056cc8495fbc293cd1c8ebe084fdfb74c3b0cdaaff9bb92" dependencies = [ "async-trait", "clap", @@ -10536,9 +10538,9 @@ dependencies = [ [[package]] name = "sea-query" -version = "0.30.1" +version = "0.30.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28c05a5bf6403834be253489bbe95fa9b1e5486bc843b61f60d26b5c9c1e244b" +checksum = "4166a1e072292d46dc91f31617c2a1cdaf55a8be4b5c9f4bf2ba248e3ac4999b" dependencies = [ "bigdecimal 0.3.1", "chrono", @@ -10583,9 +10585,9 @@ dependencies = [ [[package]] name = "sea-schema" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cd9561232bd1b82ea748b581f15909d11de0db6563ddcf28c5d908aee8282f1" +checksum = "30d148608012d25222442d1ebbfafd1228dbc5221baf4ec35596494e27a2394e" dependencies = [ "futures", "sea-query", @@ -10696,9 +10698,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] @@ -10745,9 +10747,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", @@ -10767,9 +10769,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ "itoa", "ryu", @@ -11567,6 +11569,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" + [[package]] name = "structmeta" version = "0.3.0" @@ -11760,7 +11768,7 @@ dependencies = [ "cap-std", "fd-lock", "io-lifetimes 2.0.3", - "rustix 0.38.28", + "rustix 0.38.31", "windows-sys 0.48.0", "winx", ] @@ -11793,14 +11801,13 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.9.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", - "fastrand 2.0.0", - "redox_syscall 0.4.1", - "rustix 0.38.28", + "fastrand 2.0.1", + "rustix 0.38.31", "windows-sys 0.52.0", ] @@ -12772,7 +12779,7 @@ dependencies = [ "io-extras", "io-lifetimes 2.0.3", "once_cell", - "rustix 0.38.28", + "rustix 0.38.31", "system-interface", "tracing", "wasi-common", @@ -12791,7 +12798,7 @@ dependencies = [ "cap-std", "io-extras", "log", - "rustix 0.38.28", + "rustix 0.38.31", "thiserror", "tracing", "wasmtime", @@ -12966,7 +12973,7 @@ dependencies = [ "bincode 1.3.3", "directories-next", "log", - "rustix 0.38.28", + "rustix 0.38.31", "serde", "serde_derive", "sha2", @@ -13069,7 +13076,7 @@ dependencies = [ "anyhow", "cc", "cfg-if", - "rustix 0.38.28", + "rustix 0.38.31", "wasmtime-asm-macros", "wasmtime-versioned-export-macros", "windows-sys 0.52.0", @@ -13091,7 +13098,7 @@ dependencies = [ "log", "object", "rustc-demangle", - "rustix 0.38.28", + "rustix 0.38.31", "serde", "serde_derive", "target-lexicon", @@ -13110,7 +13117,7 @@ checksum = "dd21fd0f5ca68681d3d5b636eea00f182d0f9d764144469e9257fd7e3f55ae0e" dependencies = [ "object", "once_cell", - "rustix 0.38.28", + "rustix 0.38.31", "wasmtime-versioned-export-macros", ] @@ -13143,7 +13150,7 @@ dependencies = [ "memoffset", "paste", "psm", - "rustix 0.38.28", + "rustix 0.38.31", "sptr", "wasm-encoder", "wasmtime-asm-macros", @@ -13201,7 +13208,7 @@ dependencies = [ "libc", "log", "once_cell", - "rustix 0.38.28", + "rustix 0.38.31", "system-interface", "thiserror", "tokio", @@ -13304,7 +13311,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.28", + "rustix 0.38.31", ] [[package]] @@ -13813,7 +13820,7 @@ dependencies = [ "ring 0.16.20", "rust_decimal", "rustc-hash", - "rustix 0.38.28", + "rustix 0.38.31", "scopeguard", "sea-orm", "sea-query", diff --git a/ci/Dockerfile b/ci/Dockerfile index 3d14a61dfcfb..dec0ab4d598e 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -29,6 +29,7 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-mo ENV PATH /root/.cargo/bin/:$PATH RUN rustup show +RUN rustup default `rustup show active-toolchain | awk '{print $1}'` RUN curl -sSL "https://github.com/bufbuild/buf/releases/download/v1.4.0/buf-$(uname -s)-$(uname -m).tar.gz" | \ tar -xvzf - -C /usr/local --strip-components 1 @@ -48,7 +49,7 @@ ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse # install build tools RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \ - cargo-make@0.36.10 \ + cargo-make@0.37.9 \ sqllogictest-bin@0.19.1 \ sccache@0.7.4 \ && cargo cache -a \ diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh index cfddc603ace7..ed40458fcdf8 100755 --- a/ci/build-ci-image.sh +++ b/ci/build-ci-image.sh @@ -10,7 +10,7 @@ cat ../rust-toolchain # shellcheck disable=SC2155 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml -export BUILD_ENV_VERSION=v20240204 +export BUILD_ENV_VERSION=v20240213 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}" diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index 6450e7baf4c6..4a9f2970b84c 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -71,7 +71,7 @@ services: retries: 5 source-test-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213 depends_on: - mysql - db @@ -81,7 +81,7 @@ services: - ..:/risingwave sink-test-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213 depends_on: - mysql - db @@ -93,12 +93,12 @@ services: - ..:/risingwave rw-build-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213 volumes: - ..:/risingwave ci-flamegraph-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213 # NOTE(kwannoel): This is used in order to permit # syscalls for `nperf` (perf_event_open), # so it can do CPU profiling. @@ -109,7 +109,7 @@ services: - ..:/risingwave regress-test-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213 depends_on: db: condition: service_healthy diff --git a/ci/scripts/sql/nexmark/q14.drop.sql b/ci/scripts/sql/nexmark/q14.drop.sql index d716883571b1..6cd0e12d317a 100644 --- a/ci/scripts/sql/nexmark/q14.drop.sql +++ b/ci/scripts/sql/nexmark/q14.drop.sql @@ -1,3 +1,4 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile DROP SINK nexmark_q14; +DROP FUNCTION count_char; \ No newline at end of file diff --git a/ci/scripts/sql/nexmark/q14.sql b/ci/scripts/sql/nexmark/q14.sql index c64b503fae6a..c5c174e3579c 100644 --- a/ci/scripts/sql/nexmark/q14.sql +++ b/ci/scripts/sql/nexmark/q14.sql @@ -1,5 +1,16 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile + +CREATE FUNCTION count_char(s varchar, c varchar) RETURNS int LANGUAGE javascript AS $$ + var count = 0; + for (var cc of s) { + if (cc === c) { + count++; + } + } + return count; +$$; + CREATE SINK nexmark_q14 AS SELECT auction, bidder, @@ -15,11 +26,8 @@ SELECT auction, THEN 'nightTime' ELSE 'otherTime' END AS bidTimeType, - date_time - -- extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java - -- count_char(extra, 'c') AS c_counts + date_time, + count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000 diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml index 75783ceeaf53..c48de6df64f1 100644 --- a/ci/workflows/pull-request.yml +++ b/ci/workflows/pull-request.yml @@ -96,7 +96,7 @@ steps: config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 16 + timeout_in_minutes: 17 retry: *auto-retry - label: "end-to-end test (parallel)" diff --git a/e2e_test/batch/catalog/pg_database.slt.part b/e2e_test/batch/catalog/pg_database.slt.part index 8b38ebac23e4..a2b610b678af 100644 --- a/e2e_test/batch/catalog/pg_database.slt.part +++ b/e2e_test/batch/catalog/pg_database.slt.part @@ -1,4 +1,4 @@ query ITIITT -SELECT oid, datname, encoding, datcollate, datctype, datistemplate, datallowconn, datconnlimit, dattablespace FROM pg_catalog.pg_database where oid = 0; +SELECT datname, encoding, datcollate, datctype, datistemplate, datallowconn, datconnlimit, dattablespace FROM pg_catalog.pg_database where datname = 'dev'; ---- -0 dev 6 C C f t -1 1663 +dev 6 C C f t -1 1663 diff --git a/e2e_test/source/nexmark_endless_sinks/nexmark_endless_part3.slt b/e2e_test/source/nexmark_endless_sinks/nexmark_endless_part3.slt index 461a5adf736e..93cdb9882ebd 100644 --- a/e2e_test/source/nexmark_endless_sinks/nexmark_endless_part3.slt +++ b/e2e_test/source/nexmark_endless_sinks/nexmark_endless_part3.slt @@ -16,6 +16,9 @@ drop sink nexmark_q10; statement ok drop sink nexmark_q14; +statement ok +drop function count_char; + statement ok drop sink nexmark_q15; diff --git a/e2e_test/streaming/nexmark/drop_views.slt.part b/e2e_test/streaming/nexmark/drop_views.slt.part index a0084ebeff6c..ccd5a522d5aa 100644 --- a/e2e_test/streaming/nexmark/drop_views.slt.part +++ b/e2e_test/streaming/nexmark/drop_views.slt.part @@ -34,6 +34,9 @@ drop materialized view nexmark_q10; statement ok drop materialized view nexmark_q14; +statement ok +drop function count_char; + statement ok drop materialized view nexmark_q15; diff --git a/e2e_test/streaming/nexmark/q14.slt.part b/e2e_test/streaming/nexmark/q14.slt.part index c500c4d62c6f..6e9de9bf85d8 100644 --- a/e2e_test/streaming/nexmark/q14.slt.part +++ b/e2e_test/streaming/nexmark/q14.slt.part @@ -1,15 +1,15 @@ query IIRTT rowsort SELECT * FROM nexmark_q14; ---- -1000 1001 12696452.316 nightTime 2015-07-15 00:00:11.003 -1000 1001 1753269.004 nightTime 2015-07-15 00:00:14.004 -1000 1001 2196534.628 nightTime 2015-07-15 00:00:04.001 -1000 1001 26038438.152 nightTime 2015-07-15 00:00:13.003 -1000 1001 39654430.240 nightTime 2015-07-15 00:00:04.001 -1000 1001 9503126.184 nightTime 2015-07-15 00:00:22.005 -1000 1006 1301754.200 nightTime 2015-07-15 00:00:12.003 -1006 1001 19666916.800 nightTime 2015-07-15 00:00:16.004 -1006 1001 3099951.044 nightTime 2015-07-15 00:00:16.004 -1008 1002 26289625.456 nightTime 2015-07-15 00:00:04.001 -1012 1001 3067656.208 nightTime 2015-07-15 00:00:20.005 -1012 1001 3299152.624 nightTime 2015-07-15 00:00:20.005 +1000 1001 12696452.316 nightTime 2015-07-15 00:00:11.003 1 +1000 1001 1753269.004 nightTime 2015-07-15 00:00:14.004 0 +1000 1001 2196534.628 nightTime 2015-07-15 00:00:04.001 3 +1000 1001 26038438.152 nightTime 2015-07-15 00:00:13.003 3 +1000 1001 39654430.240 nightTime 2015-07-15 00:00:04.001 2 +1000 1001 9503126.184 nightTime 2015-07-15 00:00:22.005 1 +1000 1006 1301754.200 nightTime 2015-07-15 00:00:12.003 2 +1006 1001 19666916.800 nightTime 2015-07-15 00:00:16.004 4 +1006 1001 3099951.044 nightTime 2015-07-15 00:00:16.004 3 +1008 1002 26289625.456 nightTime 2015-07-15 00:00:04.001 4 +1012 1001 3067656.208 nightTime 2015-07-15 00:00:20.005 3 +1012 1001 3299152.624 nightTime 2015-07-15 00:00:20.005 3 diff --git a/e2e_test/streaming/nexmark/sinks/q14.slt.part b/e2e_test/streaming/nexmark/sinks/q14.slt.part index 77047c7f3c81..cd9c18aad1bb 100644 --- a/e2e_test/streaming/nexmark/sinks/q14.slt.part +++ b/e2e_test/streaming/nexmark/sinks/q14.slt.part @@ -1,3 +1,14 @@ +statement ok +CREATE FUNCTION count_char(s varchar, c varchar) RETURNS int LANGUAGE javascript AS $$ + var count = 0; + for (var cc of s) { + if (cc === c) { + count++; + } + } + return count; +$$; + statement ok CREATE SINK nexmark_q14 AS SELECT @@ -15,11 +26,8 @@ SELECT THEN 'nightTime' ELSE 'otherTime' END AS bidTimeType, - date_time - -- extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java - -- count_char(extra, 'c') AS c_counts + date_time, + count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000 WITH ( connector = 'blackhole', type = 'append-only', force_append_only = 'true'); diff --git a/e2e_test/streaming/nexmark/views/q14.slt.part b/e2e_test/streaming/nexmark/views/q14.slt.part index 1182afb1e198..7f4029c8defe 100644 --- a/e2e_test/streaming/nexmark/views/q14.slt.part +++ b/e2e_test/streaming/nexmark/views/q14.slt.part @@ -1,3 +1,14 @@ +statement ok +CREATE FUNCTION count_char(s varchar, c varchar) RETURNS int LANGUAGE javascript AS $$ + var count = 0; + for (var cc of s) { + if (cc === c) { + count++; + } + } + return count; +$$; + statement ok CREATE MATERIALIZED VIEW nexmark_q14 AS SELECT @@ -15,10 +26,7 @@ SELECT THEN 'nightTime' ELSE 'otherTime' END AS bidTimeType, - date_time - -- extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java - -- count_char(extra, 'c') AS c_counts + date_time, + count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000; diff --git a/e2e_test/udf/sql_udf.slt b/e2e_test/udf/sql_udf.slt index bfae0f704588..c33daba9d7cc 100644 --- a/e2e_test/udf/sql_udf.slt +++ b/e2e_test/udf/sql_udf.slt @@ -274,9 +274,9 @@ select type_match(114514); ---- $1 + 114514 + $1 -################################################################################# -# Invalid definition (and maybe not yet supported features 🤪) / use case tests # -################################################################################# +################################################## +# Invalid definition tests when creating sql udf # +################################################## # Named sql udf with invalid parameter in body definition # Will be rejected at creation time @@ -306,14 +306,6 @@ create function fib(INT) returns int else fib($1 - 1) + fib($1 - 2) end;'; -# The execution will eventually exceed the pre-defined max stack depth -# statement error function fib calling stack depth limit exceeded -# select fib(100); - -# Currently create a materialized view with a recursive sql udf will be rejected -# statement error function fib calling stack depth limit exceeded -# create materialized view foo_mv as select fib(100); - # Calling a non-existent function statement error failed to conduct semantic check, please see if you are calling non-existent functions create function non_exist(INT) returns int language sql as 'select yo(114514)'; @@ -326,6 +318,20 @@ create function type_mismatch(INT) returns varchar language sql as 'select $1 + statement error Expected an expression:, found: EOF at the end create function add_error(INT, INT) returns int language sql as $$select $1 + $2 +$$; +###################################################################### +# Not yet supported features 🤪 (and potential basic use case tests) # +###################################################################### + +# 1. Recursion Support for SQL UDF + +# The execution will eventually exceed the pre-defined max stack depth +# statement error function fib calling stack depth limit exceeded +# select fib(100); + +# Currently create a materialized view with a recursive sql udf will be rejected +# statement error function fib calling stack depth limit exceeded +# create materialized view foo_mv as select fib(100); + # Rejected deep calling stack # statement error function recursive calling stack depth limit exceeded # select recursive(1, 1); @@ -342,6 +348,116 @@ create function add_error(INT, INT) returns int language sql as $$select $1 + $2 # statement error function fib calling stack depth limit exceeded # create materialized view bar_mv as select fib(c1) from t1; +# 2. Select from table inside SQL UDF (potential concerns: naming conflict) + +# statment ok +# create funciton from_table_single_row() returns int language sql as 'select c1 from t1'; + +# Do NOT need explicit `select ... from ...` +# query I +# select from_table_single_row(); +# ---- +# 1 + +# statement ok +# create function from_table_single_row_1() returns int language sql as 'select c1 from t1 order by c1 desc'; + +# Need explict `select ... from ...` +# query I +# select * from from_table_single_row_1(); +# ---- +# 5 + +# Should add parser support +# statement ok +# create function from_table_multiple_rows() returns setof int language sql as 'select c1 from t1'; + +# P.S. postgres shadows the `c1` parameter by the actual column `c1` to resolve the naming conflict +# statement ok +# create function from_table_conflict(c1 INT) returns int language sql as 'select c1 from t1'; + +# 3. Output multiple columns / expressions from a single SQL UDF + +# Parser support is needed +# statement ok +# create function out_parameters_without_name(out INT, out INT) language sql as 'select 1919810, 114514'; + +# query II +# select out_parameters_without_name(); +# ---- +# 1919810 114514 + +# query II +# select * out_parameters_without_name(); +# ---- +# 1919810 114514 + +# statement error non-existent column +# select a, b from out_parameters_without_name(); +# ---- +# 1919810 114514 + +# statement ok +# create function out_parameters_with_name(out a INT, out b INT) language sql as 'select 1919810, 114514'; + +# query II +# select out_parameters_with_name(); +# ---- +# 1919810 114514 + +# query II +# select a, b from out_parameters_with_name(); +# ---- +# 1919810 114514 + +# statement ok +# create function multiple_cols() returns setof record as 'select c1, c2, c3 from t2' language sql; + +# query III +# select * from multiple_cols() as t(c1 INT, c2 FLOAT, c3 INT); +# ---- +# corresponding results +# may need to order the sequence + +# 4. Polymorphic arguments + +# statement ok +# create function is_greater(anyelement, anyelement) returns boolean language sql as 'select $1 > $2'; + +# query I +# select is_greater(1, 2); +# ---- +# f + +# query I +# select is_greater(3.14, 2.95); +# ---- +# t + +# 5. Overloading functions + +# statement ok +# create function overload(c1 INT) returns int language sql as 'select c1'; + +# statement ok +# create function overload(c1 VARCHAR) returns varchar language sql as 'select c1'; + +# query I +# select overload(114514), overload('114514'); +# ---- +# 114514 114514 + +# statement error naming conflict with first overload +# create function overload(c1 INT, out VARCHAR) + +# This definition will cause ambiguity with the second overload during runtime if `c2` is not specified +# statement ok +# create function overload(c1 VARCHAR, c2 VARCHAR default '114514', out VARCHAR, out VARCHAR) language sql as 'select c1, c2'; + +# statement error can not choose a best candidate function, need explicit type cast +# query TT +# select overload('114514'); + ################################################## # Clean up the funtions / mock tables at the end # ################################################## diff --git a/e2e_test/udf/wasm/src/lib.rs b/e2e_test/udf/wasm/src/lib.rs index 522dd471daf5..556022f9c6e1 100644 --- a/e2e_test/udf/wasm/src/lib.rs +++ b/e2e_test/udf/wasm/src/lib.rs @@ -1,6 +1,11 @@ use arrow_udf::function; use rust_decimal::Decimal; +#[function("count_char(varchar, varchar) -> int")] +fn count_char(s: &str, c: &str) -> i32 { + s.matches(c).count() as i32 +} + #[function("int_42() -> int")] fn int_42() -> i32 { 42 diff --git a/integration_tests/feature-store/server/Cargo.toml b/integration_tests/feature-store/server/Cargo.toml index ce43a3f8af50..e84500f5f792 100644 --- a/integration_tests/feature-store/server/Cargo.toml +++ b/integration_tests/feature-store/server/Cargo.toml @@ -11,17 +11,17 @@ edition = "2021" [dependencies] sqlx = { version = "0.7", features = ["runtime-tokio-native-tls", "postgres"] } tokio = { version = "1", features = ["full"] } -tonic = "0.10.2" +tonic = "0.11.0" reqwest = { version = "0.11", features = ["blocking"] } rdkafka = { version = "0.34", features = ["cmake-build"] } serde_json = "1.0" prost = "0.12" clap = "4.4.6" tokio-postgres = "0.7.10" -tonic-build = "0.10.2" +tonic-build = "0.11.0" [build-dependencies] -tonic-build = "0.10.2" +tonic-build = "0.11.0" [[bin]] name = "server" diff --git a/integration_tests/feature-store/simulator/Cargo.toml b/integration_tests/feature-store/simulator/Cargo.toml index 03264cde563b..1058ac26ed55 100644 --- a/integration_tests/feature-store/simulator/Cargo.toml +++ b/integration_tests/feature-store/simulator/Cargo.toml @@ -10,7 +10,7 @@ edition = "2021" [dependencies] tokio = { version = "1", features=["macros","rt", "rt-multi-thread"]} -tonic = "0.10.2" +tonic = "0.11.0" reqwest = { version = "0.11"} serde_json = "1.0" serde_derive = "1.0" diff --git a/lints/Cargo.lock b/lints/Cargo.lock index 5251d536cd1a..650983eb008f 100644 --- a/lints/Cargo.lock +++ b/lints/Cargo.lock @@ -535,9 +535,9 @@ checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libgit2-sys" -version = "0.16.1+1.7.1" +version = "0.16.2+1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2a2bb3680b094add03bb3732ec520ece34da31a8cd2d633d1389d0f0fb60d0c" +checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" dependencies = [ "cc", "libc", diff --git a/proto/expr.proto b/proto/expr.proto index 14f9eb8c102c..4c9be4d15ea2 100644 --- a/proto/expr.proto +++ b/proto/expr.proto @@ -283,6 +283,9 @@ message ExprNode { PG_GET_INDEXDEF = 2400; COL_DESCRIPTION = 2401; PG_GET_VIEWDEF = 2402; + PG_GET_USERBYID = 2403; + PG_INDEXES_SIZE = 2404; + PG_RELATION_SIZE = 2405; // EXTERNAL ICEBERG_TRANSFORM = 2201; diff --git a/scripts/install/install-risingwave.sh b/scripts/install/install-risingwave.sh index 7547f574ebe4..87b3682a4660 100755 --- a/scripts/install/install-risingwave.sh +++ b/scripts/install/install-risingwave.sh @@ -1,19 +1,41 @@ #!/bin/sh -e -OS=$(uname -s) -ARCH=$(uname -m) -VERSION=$(curl -s https://api.github.com/repos/risingwavelabs/risingwave/releases/latest | grep '.tag_name' | sed -E -n 's/.*(v[0-9]+.[0-9]+.[0-9])\",/\1/p') +if [ -z "${OS}" ]; then + OS=$(uname -s) +fi +if [ -z "${ARCH}" ]; then + ARCH=$(uname -m) +fi +STATE_STORE_PATH="${HOME}/.risingwave/state_store" +META_STORE_PATH="${HOME}/.risingwave/meta_store" + +VERSION="v1.7.0-single-node-2" +HOMEBREW_VERSION="1.7-single-node" +# TODO(kwannoel): re-enable it once we have stable release in latest for single node mode. +#VERSION=$(curl -s https://api.github.com/repos/risingwavelabs/risingwave/releases/latest \ +# | grep '.tag_name' \ +# | sed -E -n 's/.*(v[0-9]+.[0-9]+.[0-9])\",/\1/p') BASE_URL="https://github.com/risingwavelabs/risingwave/releases/download" -# TODO(kwannoel): Add support for other OS and architectures. -if [[ "${OS}" == "Linux" && "${ARCH}" == "x86_64" || "${ARCH}" == "amd64" ]]; then - BASE_ARCHIVE_NAME="risingwave-${VERSION}-x86_64-unknown-linux-all-in-one" - ARCHIVE_NAME="${BASE_ARCHIVE_NAME}.tar.gz" - URL="${BASE_URL}/${VERSION}/${ARCHIVE_NAME}" - USE_BREW=0 -elif [[ "${OS}" == "Darwin" ]] && [[ "${ARCH}" == "x86_64" || "${ARCH}" == "amd64" || "${ARCH}" == "aarch64" || "${ARCH}" == "arm64" ]]; then - USE_BREW=1 -else +if [ "${OS}" = "Linux" ]; then + if [ "${ARCH}" = "x86_64" ] || [ "${ARCH}" = "amd64" ]; then + BASE_ARCHIVE_NAME="risingwave-${VERSION}-x86_64-unknown-linux-all-in-one" + ARCHIVE_NAME="${BASE_ARCHIVE_NAME}.tar.gz" + URL="${BASE_URL}/${VERSION}/${ARCHIVE_NAME}" + USE_BREW=0 + elif [ "${ARCH}" = "arm64" ] || [ "${ARCH}" = "aarch64" ]; then + BASE_ARCHIVE_NAME="risingwave-${VERSION}-aarch64-unknown-linux-all-in-one" + ARCHIVE_NAME="${BASE_ARCHIVE_NAME}.tar.gz" + URL="${BASE_URL}/${VERSION}/${ARCHIVE_NAME}" + USE_BREW=0 + fi +elif [ "${OS}" = "Darwin" ]; then + if [ "${ARCH}" = "x86_64" ] || [ "${ARCH}" = "amd64" ] || [ "${ARCH}" = "aarch64" ] || [ "${ARCH}" = "arm64" ]; then + USE_BREW=1 + fi +fi + +if [ -z "$USE_BREW" ]; then echo echo "Unsupported OS or Architecture: ${OS}-${ARCH}" echo @@ -27,28 +49,70 @@ else fi ############# Setup data directories +echo echo "Setting up data directories." -mkdir -p "${HOME}/.risingwave/data/state_store" -mkdir -p "${HOME}/.risingwave/data/meta_store" +echo "- ${STATE_STORE_PATH}" +echo "- ${META_STORE_PATH}" +mkdir -p "${STATE_STORE_PATH}" +mkdir -p "${META_STORE_PATH}" +echo ############# BREW INSTALL -if [[ "${USE_BREW}" -eq 1 ]]; then - echo "Installing RisingWave using Homebrew." +if [ "${USE_BREW}" -eq 1 ]; then + echo "Installing RisingWave@${HOMEBREW_VERSION} using Homebrew." brew tap risingwavelabs/risingwave - brew install risingwave - echo "Successfully installed RisingWave using Homebrew." + brew install risingwave@${HOMEBREW_VERSION} + echo "Successfully installed RisingWave@${HOMEBREW_VERSION} using Homebrew." echo echo "You can run it as:" - echo " risingwave standalone" + echo + echo " risingwave >risingwave.log 2>&1 &" + echo + echo + echo "You can attach a psql client to the standalone server using:" + echo + echo " psql -h localhost -p 4566 -d dev -U root" + echo + echo + echo "To start a fresh cluster, you can just delete the data directory contents:" + echo + echo " rm -r ~/.risingwave/state_store/*" + echo " rm -r ~/.risingwave/meta_store/*" + echo + echo + echo "To view available options, run:" + echo + echo " risingwave single-node --help" + echo + echo exit 0 fi ############# BINARY INSTALL echo -echo "Downloading ${URL} into ${PWD}." +echo "Downloading RisingWave@${VERSION} from ${URL} into ${PWD}." echo curl -L "${URL}" | tar -zx || exit 1 chmod +x risingwave echo echo "Successfully downloaded the RisingWave binary, you can run it as:" -echo " ./risingwave standalone" \ No newline at end of file +echo +echo " ./risingwave >risingwave.log 2>&1 &" +echo +echo +echo "You can connect a psql client to the standalone server using:" +echo +echo " psql -h localhost -p 4566 -d dev -U root" +echo +echo +echo "To start a fresh cluster, you can just delete the data directory contents:" +echo +echo " rm -r ~/.risingwave/state_store/*" +echo " rm -r ~/.risingwave/meta_store/*" +echo +echo +echo "To view other available options, run:" +echo +echo " ./risingwave single-node --help" +echo +# TODO(kwannoel): Include link to our docs. \ No newline at end of file diff --git a/src/common/src/session_config/sink_decouple.rs b/src/common/src/session_config/sink_decouple.rs index fbb7f684ccab..19f2a8ec8156 100644 --- a/src/common/src/session_config/sink_decouple.rs +++ b/src/common/src/session_config/sink_decouple.rs @@ -38,12 +38,16 @@ impl FromStr for SinkDecouple { } } -impl ToString for SinkDecouple { - fn to_string(&self) -> String { - match self { - Self::Default => "default".to_string(), - Self::Enable => "enable".to_string(), - Self::Disable => "disable".to_string(), - } +impl std::fmt::Display for SinkDecouple { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + Self::Default => "default", + Self::Enable => "enable", + Self::Disable => "disable", + } + ) } } diff --git a/src/connector/Cargo.toml b/src/connector/Cargo.toml index 83f359a12b20..f73bd5f51c3e 100644 --- a/src/connector/Cargo.toml +++ b/src/connector/Cargo.toml @@ -87,7 +87,7 @@ parking_lot = "0.12" paste = "1" prometheus = { version = "0.13", features = ["process"] } prost = { version = "0.12", features = ["no-recursion-limit"] } -prost-reflect = "0.12" +prost-reflect = "0.13" prost-types = "0.12" protobuf-native = "0.2.1" pulsar = { version = "6.1", default-features = false, features = [ @@ -139,7 +139,7 @@ tokio-postgres = "0.7" tokio-retry = "0.3" tokio-stream = "0.1" tokio-util = { version = "0.7", features = ["codec", "io"] } -tonic = "0.10.2" +tonic = { workspace = true } tracing = "0.1" url = "2" urlencoding = "2" diff --git a/src/connector/src/common.rs b/src/connector/src/common.rs index 88f6c9f9ea31..418155250e74 100644 --- a/src/connector/src/common.rs +++ b/src/connector/src/common.rs @@ -259,6 +259,10 @@ pub struct RdKafkaPropertiesCommon { #[serde(rename = "properties.client.id")] #[serde_as(as = "Option")] pub client_id: Option, + + #[serde(rename = "properties.enable.ssl.certificate.verification")] + #[serde_as(as = "Option")] + pub enable_ssl_certificate_verification: Option, } impl RdKafkaPropertiesCommon { @@ -275,6 +279,9 @@ impl RdKafkaPropertiesCommon { if let Some(v) = self.client_id.as_ref() { c.set("client.id", v); } + if let Some(v) = self.enable_ssl_certificate_verification { + c.set("enable.ssl.certificate.verification", v.to_string()); + } } } diff --git a/src/connector/src/error.rs b/src/connector/src/error.rs index d25658dfaf5e..4cf36e9859d3 100644 --- a/src/connector/src/error.rs +++ b/src/connector/src/error.rs @@ -12,50 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use thiserror::Error; +use risingwave_common::error::v2::def_anyhow_newtype; -#[derive(Error, Debug)] -pub enum ConnectorError { - #[error("Parse error: {0}")] - Parse(&'static str), +def_anyhow_newtype! { + pub ConnectorError, - #[error("Invalid parameter {name}: {reason}")] - InvalidParam { name: &'static str, reason: String }, - - #[error("Kafka error: {0}")] - Kafka(#[from] rdkafka::error::KafkaError), - - #[error("Config error: {0}")] - Config( - #[source] - #[backtrace] - anyhow::Error, - ), - - #[error("Connection error: {0}")] - Connection( - #[source] - #[backtrace] - anyhow::Error, - ), - - #[error("MySQL error: {0}")] - MySql(#[from] mysql_async::Error), - - #[error("Postgres error: {0}")] - Postgres(#[from] tokio_postgres::Error), - - #[error("Pulsar error: {0}")] - Pulsar( - #[source] - #[backtrace] - anyhow::Error, - ), - - #[error(transparent)] - Internal( - #[from] - #[backtrace] - anyhow::Error, - ), + // TODO(error-handling): Remove implicit contexts below and specify ad-hoc context for each conversion. + mysql_async::Error => "MySQL error", + tokio_postgres::Error => "Postgres error", } + +pub type ConnectorResult = Result; diff --git a/src/connector/src/sink/doris.rs b/src/connector/src/sink/doris.rs index caf478934b3d..64ab8121aaaa 100644 --- a/src/connector/src/sink/doris.rs +++ b/src/connector/src/sink/doris.rs @@ -171,7 +171,7 @@ impl DorisSink { Ok(doris_data_type.contains("DATETIME")) } risingwave_common::types::DataType::Timestamptz => Err(SinkError::Doris( - "doris can not support Timestamptz".to_string(), + "TIMESTAMP WITH TIMEZONE is not supported for Doris sink as Doris doesn't store time values with timezone information. Please convert to TIMESTAMP first.".to_string(), )), risingwave_common::types::DataType::Interval => Err(SinkError::Doris( "doris can not support Interval".to_string(), diff --git a/src/connector/src/sink/doris_starrocks_connector.rs b/src/connector/src/sink/doris_starrocks_connector.rs index ce019dd18600..8a77f1a13cf1 100644 --- a/src/connector/src/sink/doris_starrocks_connector.rs +++ b/src/connector/src/sink/doris_starrocks_connector.rs @@ -141,6 +141,14 @@ impl HeaderBuilder { self } + pub fn set_partial_update(mut self, partial_update: Option) -> Self { + self.header.insert( + "partial_update".to_string(), + partial_update.unwrap_or_else(|| "false".to_string()), + ); + self + } + pub fn build(self) -> HashMap { self.header } diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index 22c41a18c002..eb5c7b129385 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -98,6 +98,24 @@ impl JsonEncoder { } } + pub fn new_with_starrocks( + schema: Schema, + col_indices: Option>, + timestamp_handling_mode: TimestampHandlingMode, + map: HashMap, + ) -> Self { + Self { + schema, + col_indices, + time_handling_mode: TimeHandlingMode::Milli, + date_handling_mode: DateHandlingMode::String, + timestamp_handling_mode, + timestamptz_handling_mode: TimestamptzHandlingMode::UtcWithoutSuffix, + custom_json_type: CustomJsonType::StarRocks(map), + kafka_connect: None, + } + } + pub fn with_kafka_connect(self, kafka_connect: KafkaConnectParams) -> Self { Self { kafka_connect: Some(Arc::new(kafka_connect)), @@ -205,7 +223,7 @@ fn datum_to_json_object( json!(v) } (DataType::Decimal, ScalarRefImpl::Decimal(mut v)) => match custom_json_type { - CustomJsonType::Doris(map) => { + CustomJsonType::Doris(map) | CustomJsonType::StarRocks(map) => { if !matches!(v, Decimal::Normalized(_)) { return Err(ArrayError::internal( "doris/starrocks can't support decimal Inf, -Inf, Nan".to_string(), @@ -272,8 +290,10 @@ fn datum_to_json_object( json!(v.as_iso_8601()) } (DataType::Jsonb, ScalarRefImpl::Jsonb(jsonb_ref)) => match custom_json_type { - CustomJsonType::Es => JsonbVal::from(jsonb_ref).take(), - CustomJsonType::Doris(_) | CustomJsonType::None => json!(jsonb_ref.to_string()), + CustomJsonType::Es | CustomJsonType::StarRocks(_) => JsonbVal::from(jsonb_ref).take(), + CustomJsonType::Doris(_) | CustomJsonType::None => { + json!(jsonb_ref.to_string()) + } }, (DataType::List(datatype), ScalarRefImpl::List(list_ref)) => { let elems = list_ref.iter(); @@ -317,6 +337,11 @@ fn datum_to_json_object( serde_json::to_string(&map).context("failed to serialize into JSON")?, ) } + CustomJsonType::StarRocks(_) => { + return Err(ArrayError::internal( + "starrocks can't support struct".to_string(), + )); + } CustomJsonType::Es | CustomJsonType::None => { let mut map = Map::with_capacity(st.len()); for (sub_datum_ref, sub_field) in struct_ref.iter_fields_ref().zip_eq_debug( diff --git a/src/connector/src/sink/encoder/mod.rs b/src/connector/src/sink/encoder/mod.rs index 83c28ce4b4a5..3c76803c8e0f 100644 --- a/src/connector/src/sink/encoder/mod.rs +++ b/src/connector/src/sink/encoder/mod.rs @@ -142,6 +142,8 @@ pub enum CustomJsonType { Doris(HashMap), // Es's json need jsonb is struct Es, + // starrocks' need jsonb is struct + StarRocks(HashMap), None, } diff --git a/src/connector/src/sink/starrocks.rs b/src/connector/src/sink/starrocks.rs index 4c9460abc431..11594133695d 100644 --- a/src/connector/src/sink/starrocks.rs +++ b/src/connector/src/sink/starrocks.rs @@ -69,6 +69,8 @@ pub struct StarrocksCommon { /// The StarRocks table you want to sink data to. #[serde(rename = "starrocks.table")] pub table: String, + #[serde(rename = "starrocks.partial_update")] + pub partial_update: Option, } #[serde_as] @@ -126,8 +128,8 @@ impl StarrocksSink { starrocks_columns_desc: HashMap, ) -> Result<()> { let rw_fields_name = self.schema.fields(); - if rw_fields_name.len().ne(&starrocks_columns_desc.len()) { - return Err(SinkError::Starrocks("The length of the RisingWave column must be equal to the length of the starrocks column".to_string())); + if rw_fields_name.len() > starrocks_columns_desc.len() { + return Err(SinkError::Starrocks("The length of the RisingWave column must be equal or less to the length of the starrocks column".to_string())); } for i in rw_fields_name { @@ -179,7 +181,7 @@ impl StarrocksSink { Ok(starrocks_data_type.contains("datetime")) } risingwave_common::types::DataType::Timestamptz => Err(SinkError::Starrocks( - "starrocks can not support Timestamptz".to_string(), + "TIMESTAMP WITH TIMEZONE is not supported for Starrocks sink as Starrocks doesn't store time values with timezone information. Please convert to TIMESTAMP first.".to_string(), )), risingwave_common::types::DataType::Interval => Err(SinkError::Starrocks( "starrocks can not support Interval".to_string(), @@ -194,9 +196,7 @@ impl StarrocksSink { risingwave_common::types::DataType::Bytea => Err(SinkError::Starrocks( "starrocks can not support Bytea".to_string(), )), - risingwave_common::types::DataType::Jsonb => Err(SinkError::Starrocks( - "starrocks can not support import json".to_string(), - )), + risingwave_common::types::DataType::Jsonb => Ok(starrocks_data_type.contains("json")), risingwave_common::types::DataType::Serial => { Ok(starrocks_data_type.contains("bigint")) } @@ -337,18 +337,18 @@ impl StarrocksSinkWriter { decimal_map.insert(name.to_string(), (length, scale)); } } + let mut fields_name = schema.names_str(); + if !is_append_only { + fields_name.push(STARROCKS_DELETE_SIGN); + }; - let builder = HeaderBuilder::new() + let header = HeaderBuilder::new() .add_common_header() .set_user_password(config.common.user.clone(), config.common.password.clone()) - .add_json_format(); - let header = if !is_append_only { - let mut fields_name = schema.names_str(); - fields_name.push(STARROCKS_DELETE_SIGN); - builder.set_columns_name(fields_name).build() - } else { - builder.build() - }; + .add_json_format() + .set_partial_update(config.common.partial_update.clone()) + .set_columns_name(fields_name) + .build(); let starrocks_insert_builder = InserterInnerBuilder::new( format!("http://{}:{}", config.common.host, config.common.http_port), @@ -363,7 +363,7 @@ impl StarrocksSinkWriter { inserter_innet_builder: starrocks_insert_builder, is_append_only, client: None, - row_encoder: JsonEncoder::new_with_doris( + row_encoder: JsonEncoder::new_with_starrocks( schema, None, TimestampHandlingMode::String, diff --git a/src/connector/src/source/cdc/external/mock_external_table.rs b/src/connector/src/source/cdc/external/mock_external_table.rs index 2c8ee00f67af..d7c39e2a9aa8 100644 --- a/src/connector/src/source/cdc/external/mock_external_table.rs +++ b/src/connector/src/source/cdc/external/mock_external_table.rs @@ -19,10 +19,9 @@ use futures_async_stream::try_stream; use risingwave_common::row::OwnedRow; use risingwave_common::types::ScalarImpl; -use crate::error::ConnectorError; +use crate::error::{ConnectorError, ConnectorResult}; use crate::source::cdc::external::{ - CdcOffset, CdcOffsetParseFunc, ConnectorResult, ExternalTableReader, MySqlOffset, - SchemaTableName, + CdcOffset, CdcOffsetParseFunc, ExternalTableReader, MySqlOffset, SchemaTableName, }; #[derive(Debug)] diff --git a/src/connector/src/source/cdc/external/mod.rs b/src/connector/src/source/cdc/external/mod.rs index 78c6c714e2bc..f281d1ecea58 100644 --- a/src/connector/src/source/cdc/external/mod.rs +++ b/src/connector/src/source/cdc/external/mod.rs @@ -17,7 +17,7 @@ mod postgres; use std::collections::HashMap; -use anyhow::{anyhow, Context}; +use anyhow::Context; use futures::stream::BoxStream; use futures::{pin_mut, StreamExt}; use futures_async_stream::try_stream; @@ -32,13 +32,11 @@ use risingwave_common::types::DataType; use risingwave_common::util::iter_util::ZipEqFast; use serde_derive::{Deserialize, Serialize}; -use crate::error::ConnectorError; +use crate::error::{ConnectorError, ConnectorResult}; use crate::parser::mysql_row_to_owned_row; use crate::source::cdc::external::mock_external_table::MockExternalTableReader; use crate::source::cdc::external::postgres::{PostgresExternalTableReader, PostgresOffset}; -pub type ConnectorResult = std::result::Result; - #[derive(Debug)] pub enum CdcTableType { Undefined, @@ -77,10 +75,7 @@ impl CdcTableType { Self::Postgres => Ok(ExternalTableReaderImpl::Postgres( PostgresExternalTableReader::new(with_properties, schema).await?, )), - _ => bail!(ConnectorError::Config(anyhow!( - "invalid external table type: {:?}", - *self - ))), + _ => bail!("invalid external table type: {:?}", *self), } } } @@ -405,19 +400,11 @@ impl MySqlExternalTableReader { DataType::Date => Value::from(value.into_date().0), DataType::Time => Value::from(value.into_time().0), DataType::Timestamp => Value::from(value.into_timestamp().0), - _ => { - return Err(ConnectorError::Internal(anyhow!( - "unsupported primary key data type: {}", - ty - ))) - } + _ => bail!("unsupported primary key data type: {}", ty), }; - Ok((pk.clone(), val)) + ConnectorResult::Ok((pk.clone(), val)) } else { - Err(ConnectorError::Internal(anyhow!( - "primary key {} cannot be null", - pk - ))) + bail!("primary key {} cannot be null", pk); } }) .try_collect()?; diff --git a/src/connector/src/source/cdc/external/postgres.rs b/src/connector/src/source/cdc/external/postgres.rs index f8f0c9d40234..bd8a0b51c04e 100644 --- a/src/connector/src/source/cdc/external/postgres.rs +++ b/src/connector/src/source/cdc/external/postgres.rs @@ -28,11 +28,11 @@ use thiserror_ext::AsReport; use tokio_postgres::types::PgLsn; use tokio_postgres::NoTls; -use crate::error::ConnectorError; +use crate::error::{ConnectorError, ConnectorResult}; use crate::parser::postgres_row_to_owned_row; use crate::source::cdc::external::{ - CdcOffset, CdcOffsetParseFunc, ConnectorResult, DebeziumOffset, ExternalTableConfig, - ExternalTableReader, SchemaTableName, + CdcOffset, CdcOffsetParseFunc, DebeziumOffset, ExternalTableConfig, ExternalTableReader, + SchemaTableName, }; #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] diff --git a/src/connector/src/source/pulsar/source/reader.rs b/src/connector/src/source/pulsar/source/reader.rs index 7181710b7086..139af839bd16 100644 --- a/src/connector/src/source/pulsar/source/reader.rs +++ b/src/connector/src/source/pulsar/source/reader.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::time::{SystemTime, UNIX_EPOCH}; -use anyhow::{anyhow, Context}; +use anyhow::Context; use arrow_array::{Int32Array, Int64Array, RecordBatch}; use async_trait::async_trait; use futures::StreamExt; @@ -32,7 +32,6 @@ use risingwave_common::array::{DataChunk, StreamChunk}; use risingwave_common::catalog::ROWID_PREFIX; use risingwave_common::{bail, ensure}; -use crate::error::ConnectorError; use crate::parser::ParserConfig; use crate::source::pulsar::split::PulsarSplit; use crate::source::pulsar::{PulsarEnumeratorOffset, PulsarProperties}; @@ -398,10 +397,11 @@ impl PulsarIcebergReader { fn build_iceberg_configs(&self) -> anyhow::Result> { let mut iceberg_configs = HashMap::new(); - let bucket = - self.props.iceberg_bucket.as_ref().ok_or_else(|| { - ConnectorError::Pulsar(anyhow!("Iceberg bucket is not configured")) - })?; + let bucket = self + .props + .iceberg_bucket + .as_ref() + .context("Iceberg bucket is not configured")?; iceberg_configs.insert(CATALOG_TYPE.to_string(), "storage".to_string()); iceberg_configs.insert(CATALOG_NAME.to_string(), "pulsar".to_string()); diff --git a/src/connector/src/source/pulsar/topic.rs b/src/connector/src/source/pulsar/topic.rs index 8c6ec41022b0..4512662d6252 100644 --- a/src/connector/src/source/pulsar/topic.rs +++ b/src/connector/src/source/pulsar/topic.rs @@ -32,9 +32,10 @@ pub struct Topic { pub partition_index: Option, } -impl ToString for Topic { - fn to_string(&self) -> String { - format!( +impl std::fmt::Display for Topic { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, "{}://{}/{}/{}", self.domain, self.tenant, self.namespace, self.topic ) diff --git a/src/connector/src/source/reader/fs_reader.rs b/src/connector/src/source/reader/fs_reader.rs index 5b199c8d8955..f64a9def6aab 100644 --- a/src/connector/src/source/reader/fs_reader.rs +++ b/src/connector/src/source/reader/fs_reader.rs @@ -72,7 +72,7 @@ impl FsSourceReader { .with_context(|| { format!("Failed to find column id: {} in source: {:?}", id, self) }) - .map(|col| col.clone()) + .cloned() }) .try_collect() } diff --git a/src/connector/with_options_sink.yaml b/src/connector/with_options_sink.yaml index b2090d1dbf4b..74cb5c21e9c7 100644 --- a/src/connector/with_options_sink.yaml +++ b/src/connector/with_options_sink.yaml @@ -252,6 +252,9 @@ KafkaConfig: field_type: String comments: Client identifier required: false + - name: properties.enable.ssl.certificate.verification + field_type: bool + required: false - name: properties.allow.auto.create.topics field_type: bool comments: Allow automatic topic creation on the broker when subscribing to or assigning non-existent topics. @@ -483,6 +486,9 @@ StarrocksConfig: field_type: String comments: The StarRocks table you want to sink data to. required: true + - name: starrocks.partial_update + field_type: String + required: false - name: r#type field_type: String required: true diff --git a/src/connector/with_options_source.yaml b/src/connector/with_options_source.yaml index 3ef323aa8857..2d811ce639c9 100644 --- a/src/connector/with_options_source.yaml +++ b/src/connector/with_options_source.yaml @@ -143,6 +143,9 @@ KafkaProperties: field_type: String comments: Client identifier required: false + - name: properties.enable.ssl.certificate.verification + field_type: bool + required: false - name: properties.queued.min.messages field_type: usize comments: Minimum number of messages per topic+partition librdkafka tries to maintain in the local consumer queue. diff --git a/src/error/Cargo.toml b/src/error/Cargo.toml index 13bb50a37185..4a99711db6c4 100644 --- a/src/error/Cargo.toml +++ b/src/error/Cargo.toml @@ -8,6 +8,7 @@ license = { workspace = true } repository = { workspace = true } [dependencies] +anyhow = "1" bincode = "1" bytes = "1" easy-ext = "1" diff --git a/src/error/src/anyhow.rs b/src/error/src/anyhow.rs new file mode 100644 index 000000000000..0acadddd88fa --- /dev/null +++ b/src/error/src/anyhow.rs @@ -0,0 +1,146 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Define a newtype wrapper around [`anyhow::Error`]. +/// +/// # Usage +/// +/// ```ignore +/// def_anyhow_newtype! { +/// /// Documentation for the newtype. +/// #[derive(..)] +/// pub MyError, +/// +/// // Default context messages for each source error type goes below. +/// mysql::Error => "failed to interact with MySQL", +/// postgres::Error => "failed to interact with PostgreSQL", +/// opendal::Error => transparent, // if it's believed to be self-explanatory +/// // and any context is not necessary +/// } +/// ``` +/// +/// # Construction +/// +/// Unlike [`anyhow::Error`], the newtype **CANNOT** be converted from any error +/// types implicitly. Instead, it can only be converted from [`anyhow::Error`] +/// by default. +/// +/// - Users are encouraged to use [`anyhow::Context`] to attach detailed +/// information to the source error and make it an [`anyhow::Error`] before +/// converting it to the newtype. +/// +/// - Otherwise, specify the default context for each source error type as shown +/// in the example above, which will be expanded into a `From` implementation +/// from the source error type to the newtype. This should **NOT** be preferred +/// in most cases since it's less informative than the ad-hoc context provided +/// with [`anyhow::Context`] at the call site, but it could still be useful +/// during refactoring, or if the source error type is believed to be +/// self-explanatory. +/// +/// To construct a new error from scratch, one can still use macros like +/// `anyhow::anyhow!` or `risingwave_common::bail!`. Since `bail!` and `?` +/// already imply an `into()` call, developers do not need to care about the +/// type conversion most of the time. +/// +/// ## Example +/// +/// ```ignore +/// fn read_offset_from_mysql() -> Result { +/// .. +/// } +/// fn parse_offset(offset: &str) -> Result { +/// .. +/// } +/// +/// fn work() -> Result<(), MyError> { +/// // `mysql::Error` can be converted to `MyError` implicitly with `?` +/// // as the default context is provided in the definition. +/// let offset = read_offset_from_mysql()?; +/// +/// // Instead, `ParseIntError` cannot be directly converted to `MyError` +/// // with `?`, so the caller must attach context explicitly. +/// // +/// // This makes sense as the semantics of the integer ("offset") holds +/// // important information and are not implied by the error type itself. +/// let offset = parse_offset(&offset).context("failed to parse offset")?; +/// +/// if offset < 0 { +/// // Construct a new error with `bail!` macro. +/// bail!("offset `{}` must be non-negative", offset); +/// } +/// } +/// ``` +/// +/// # Discussion +/// +/// - What's the purpose of the newtype? +/// * It is to provide extra type information for errors, which makes it +/// clearer to identify which module or crate the error comes from when +/// it is passed around. +/// * It enforces the developer to attach context (explicitly or by default) +/// when doing type conversion, which makes the error more informative. +/// +/// - Is the effect essentially the same as `thiserror`? +/// * Yes, but we're here intentionally making the error type less actionable +/// to make it informative with no fear. +/// * To elaborate, consider the following `thiserror` example: +/// ```ignore +/// #[derive(thiserror::Error, Debug)] +/// pub enum MyError { +/// #[error("failed to interact with MySQL")] +/// MySql(#[from] mysql::Error), +/// #[error(transparent)] +/// Other(#[from] anyhow::Error), +/// } +/// ``` +/// This gives the caller an illusion that all errors related to MySQL are +/// under the `MySql` variant, which is not true as one could attach context +/// to an `mysql::Error` with [`anyhow::Context`] and make it go into the +/// `Other` variant. +/// +/// By doing type erasure with `anyhow`, we're making it clear that the +/// error is not actionable so such confusion is avoided. +#[macro_export] +macro_rules! def_anyhow_newtype { + (@from $error:ident transparent) => { + Self(::anyhow::Error::new($error)) + }; + (@from $error:ident $context:literal) => { + Self(::anyhow::Error::new($error).context($context)) + }; + + ( + $(#[$attr:meta])* $vis:vis $name:ident + $(, $from:ty => $context:tt)* $(,)? + ) => { + #[derive(::thiserror::Error, ::std::fmt::Debug)] + #[error(transparent)] + $(#[$attr])* $vis struct $name(#[from] #[backtrace] ::anyhow::Error); + + impl $name { + /// Unwrap the newtype to get the inner [`anyhow::Error`]. + pub fn into_inner(self) -> ::anyhow::Error { + self.0 + } + } + + $( + impl From<$from> for $name { + fn from(error: $from) -> Self { + def_anyhow_newtype!(@from error $context) + } + } + )* + }; +} diff --git a/src/error/src/lib.rs b/src/error/src/lib.rs index ccfec0cfbcc1..d3364485e8f2 100644 --- a/src/error/src/lib.rs +++ b/src/error/src/lib.rs @@ -21,4 +21,5 @@ #![feature(register_tool)] #![register_tool(rw)] +pub mod anyhow; pub mod tonic; diff --git a/src/expr/core/src/expr/build.rs b/src/expr/core/src/expr/build.rs index 51e9b87de19f..91ebe67f479a 100644 --- a/src/expr/core/src/expr/build.rs +++ b/src/expr/core/src/expr/build.rs @@ -131,6 +131,7 @@ pub(crate) trait Build: Expression + Sized { /// Build the expression `Self` from protobuf for test, where each child is built with /// [`build_from_prost`]. + #[cfg(test)] fn build_for_test(prost: &ExprNode) -> Result { Self::build(prost, build_from_prost) } diff --git a/src/expr/core/src/expr/expr_udf.rs b/src/expr/core/src/expr/expr_udf.rs index 3532f15677b3..b162691896a4 100644 --- a/src/expr/core/src/expr/expr_udf.rs +++ b/src/expr/core/src/expr/expr_udf.rs @@ -71,7 +71,6 @@ impl Expression for UserDefinedFunction { self.return_type.clone() } - #[cfg_or_panic(not(madsim))] async fn eval(&self, input: &DataChunk) -> Result { let vis = input.visibility(); let mut columns = Vec::with_capacity(self.children.len()); @@ -82,7 +81,6 @@ impl Expression for UserDefinedFunction { self.eval_inner(columns, vis).await } - #[cfg_or_panic(not(madsim))] async fn eval_row(&self, input: &OwnedRow) -> Result { let mut columns = Vec::with_capacity(self.children.len()); for child in &self.children { @@ -183,7 +181,6 @@ impl UserDefinedFunction { } } -#[cfg_or_panic(not(madsim))] impl Build for UserDefinedFunction { fn build( prost: &ExprNode, @@ -194,6 +191,7 @@ impl Build for UserDefinedFunction { let identifier = udf.get_identifier()?; let imp = match udf.language.as_str() { + #[cfg(not(madsim))] "wasm" | "rust" => { let link = udf.get_link()?; // Use `block_in_place` as an escape hatch to run async code here in sync context. @@ -218,10 +216,13 @@ impl Build for UserDefinedFunction { )?; UdfImpl::JavaScript(rt) } + #[cfg(not(madsim))] _ => { let link = udf.get_link()?; UdfImpl::External(get_or_create_flight_client(link)?) } + #[cfg(madsim)] + l => panic!("UDF language {l:?} is not supported on madsim"), }; let arg_schema = Arc::new(Schema::new( diff --git a/src/expr/macro/src/parse.rs b/src/expr/macro/src/parse.rs index d5b1ddf57fff..30976e14f192 100644 --- a/src/expr/macro/src/parse.rs +++ b/src/expr/macro/src/parse.rs @@ -280,9 +280,7 @@ fn strip_outer_type<'a>(ty: &'a syn::Type, type_: &str) -> Option<&'a syn::Type> let syn::Type::Path(path) = ty else { return None; }; - let Some(seg) = path.path.segments.last() else { - return None; - }; + let seg = path.path.segments.last()?; if seg.ident != type_ { return None; } diff --git a/src/frontend/planner_test/tests/testdata/input/nexmark.yaml b/src/frontend/planner_test/tests/testdata/input/nexmark.yaml index 9cc9ee747e33..c96bacc3a41b 100644 --- a/src/frontend/planner_test/tests/testdata/input/nexmark.yaml +++ b/src/frontend/planner_test/tests/testdata/input/nexmark.yaml @@ -336,8 +336,7 @@ END AS bidTimeType, date_time, extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java + -- ignore UDF in planner test -- count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000; diff --git a/src/frontend/planner_test/tests/testdata/input/nexmark_source.yaml b/src/frontend/planner_test/tests/testdata/input/nexmark_source.yaml index a88a5d138a61..9cda437345fe 100644 --- a/src/frontend/planner_test/tests/testdata/input/nexmark_source.yaml +++ b/src/frontend/planner_test/tests/testdata/input/nexmark_source.yaml @@ -338,8 +338,7 @@ END AS bidTimeType, date_time, extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java + -- ignore UDF in planner test -- count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000; diff --git a/src/frontend/planner_test/tests/testdata/input/nexmark_temporal_filter.yaml b/src/frontend/planner_test/tests/testdata/input/nexmark_temporal_filter.yaml index 10f3bafc559d..7ae9091a83a3 100644 --- a/src/frontend/planner_test/tests/testdata/input/nexmark_temporal_filter.yaml +++ b/src/frontend/planner_test/tests/testdata/input/nexmark_temporal_filter.yaml @@ -321,8 +321,7 @@ END AS bidTimeType, date_time, extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java + -- ignore UDF in planner test -- count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000; diff --git a/src/frontend/planner_test/tests/testdata/input/nexmark_watermark.yaml b/src/frontend/planner_test/tests/testdata/input/nexmark_watermark.yaml index e78287613d5e..65845ce054aa 100644 --- a/src/frontend/planner_test/tests/testdata/input/nexmark_watermark.yaml +++ b/src/frontend/planner_test/tests/testdata/input/nexmark_watermark.yaml @@ -305,8 +305,7 @@ END AS bidTimeType, date_time, extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java + -- ignore UDF in planner test -- count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000; diff --git a/src/frontend/planner_test/tests/testdata/input/topn.yaml b/src/frontend/planner_test/tests/testdata/input/topn.yaml new file mode 100644 index 000000000000..676ac327efc9 --- /dev/null +++ b/src/frontend/planner_test/tests/testdata/input/topn.yaml @@ -0,0 +1,18 @@ +- sql: | + CREATE TABLE t1 (pk int, a int, b int, c bigint, d int); + CREATE MATERIALIZED VIEW t1_mv AS SELECT SUM(a) as sa, SUM(b) as sb, SUM(c) as sc, SUM(d) as sd FROM t1; + SELECT sa, count(*) as cnt2 FROM t1_mv GROUP BY sa ORDER BY cnt2 DESC LIMIT 50 OFFSET 50; + expected_outputs: + - batch_plan +- sql: | + CREATE TABLE t1 (pk int, a int, b int, c bigint, d int); + CREATE MATERIALIZED VIEW t1_mv AS SELECT * from t1; + SELECT * FROM t1_mv ORDER BY a DESC LIMIT 50 OFFSET 50; + expected_outputs: + - batch_plan +- sql: | + CREATE TABLE t1 (pk int, a int, b int, c bigint, d int); + CREATE MATERIALIZED VIEW t1_mv AS SELECT * from t1 order by a desc; + SELECT * FROM t1_mv ORDER BY a DESC LIMIT 50 OFFSET 50; + expected_outputs: + - batch_plan diff --git a/src/frontend/planner_test/tests/testdata/output/nexmark.yaml b/src/frontend/planner_test/tests/testdata/output/nexmark.yaml index ca19eb4dade1..59bd43f020b1 100644 --- a/src/frontend/planner_test/tests/testdata/output/nexmark.yaml +++ b/src/frontend/planner_test/tests/testdata/output/nexmark.yaml @@ -1185,8 +1185,7 @@ END AS bidTimeType, date_time, extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java + -- ignore UDF in planner test -- count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000; diff --git a/src/frontend/planner_test/tests/testdata/output/nexmark_source.yaml b/src/frontend/planner_test/tests/testdata/output/nexmark_source.yaml index c60d6e61acae..32c30f2e76d4 100644 --- a/src/frontend/planner_test/tests/testdata/output/nexmark_source.yaml +++ b/src/frontend/planner_test/tests/testdata/output/nexmark_source.yaml @@ -1003,8 +1003,7 @@ END AS bidTimeType, date_time, extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java + -- ignore UDF in planner test -- count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000; diff --git a/src/frontend/planner_test/tests/testdata/output/nexmark_temporal_filter.yaml b/src/frontend/planner_test/tests/testdata/output/nexmark_temporal_filter.yaml index d1ed31b042f8..f77e975780c8 100644 --- a/src/frontend/planner_test/tests/testdata/output/nexmark_temporal_filter.yaml +++ b/src/frontend/planner_test/tests/testdata/output/nexmark_temporal_filter.yaml @@ -957,8 +957,7 @@ END AS bidTimeType, date_time, extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java + -- ignore UDF in planner test -- count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000; diff --git a/src/frontend/planner_test/tests/testdata/output/nexmark_watermark.yaml b/src/frontend/planner_test/tests/testdata/output/nexmark_watermark.yaml index dfaba49beb63..12593221419d 100644 --- a/src/frontend/planner_test/tests/testdata/output/nexmark_watermark.yaml +++ b/src/frontend/planner_test/tests/testdata/output/nexmark_watermark.yaml @@ -1208,8 +1208,7 @@ END AS bidTimeType, date_time, extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java + -- ignore UDF in planner test -- count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000; diff --git a/src/frontend/planner_test/tests/testdata/output/pg_catalog.yaml b/src/frontend/planner_test/tests/testdata/output/pg_catalog.yaml index 7842e311e47a..b369b85de69f 100644 --- a/src/frontend/planner_test/tests/testdata/output/pg_catalog.yaml +++ b/src/frontend/planner_test/tests/testdata/output/pg_catalog.yaml @@ -204,20 +204,11 @@ - sql: | select pg_catalog.pg_get_userbyid(1) logical_plan: |- - LogicalProject { exprs: [rw_users.name] } - └─LogicalApply { type: LeftOuter, on: true, correlated_id: 1, max_one_row: true } - ├─LogicalValues { rows: [[]], schema: Schema { fields: [] } } - └─LogicalProject { exprs: [rw_users.name] } - └─LogicalFilter { predicate: (1:Int32 = rw_users.id) } - └─LogicalShare { id: 2 } - └─LogicalProject { exprs: [rw_users.id, rw_users.name, rw_users.create_db, rw_users.is_super, '********':Varchar] } - └─LogicalSysScan { table: rw_users, columns: [rw_users.id, rw_users.name, rw_users.is_super, rw_users.create_db, rw_users.create_user, rw_users.can_login] } + LogicalProject { exprs: [PgGetUserbyid(1:Int32) as $expr1] } + └─LogicalValues { rows: [[]], schema: Schema { fields: [] } } batch_plan: |- - BatchNestedLoopJoin { type: LeftOuter, predicate: true, output: all } - ├─BatchValues { rows: [[]] } - └─BatchProject { exprs: [rw_users.name] } - └─BatchFilter { predicate: (1:Int32 = rw_users.id) } - └─BatchScan { table: rw_users, columns: [rw_users.name, rw_users.id], distribution: Single } + BatchProject { exprs: [PgGetUserbyid(1:Int32) as $expr1] } + └─BatchValues { rows: [[]] } - sql: | select 'pg_namespace'::regclass logical_plan: |- diff --git a/src/frontend/planner_test/tests/testdata/output/subquery.yaml b/src/frontend/planner_test/tests/testdata/output/subquery.yaml index 6dda70d6f98e..6196244e193a 100644 --- a/src/frontend/planner_test/tests/testdata/output/subquery.yaml +++ b/src/frontend/planner_test/tests/testdata/output/subquery.yaml @@ -194,31 +194,21 @@ - sql: | SELECT (SELECT pg_catalog.pg_get_userbyid(1)) logical_plan: |- - LogicalProject { exprs: [rw_users.name] } + LogicalProject { exprs: [$expr1] } └─LogicalApply { type: LeftOuter, on: true, correlated_id: 1, max_one_row: true } ├─LogicalValues { rows: [[]], schema: Schema { fields: [] } } - └─LogicalProject { exprs: [rw_users.name] } - └─LogicalApply { type: LeftOuter, on: true, correlated_id: 2, max_one_row: true } - ├─LogicalValues { rows: [[]], schema: Schema { fields: [] } } - └─LogicalProject { exprs: [rw_users.name] } - └─LogicalFilter { predicate: (1:Int32 = rw_users.id) } - └─LogicalShare { id: 2 } - └─LogicalProject { exprs: [rw_users.id, rw_users.name, rw_users.create_db, rw_users.is_super, '********':Varchar] } - └─LogicalSysScan { table: rw_users, columns: [rw_users.id, rw_users.name, rw_users.is_super, rw_users.create_db, rw_users.create_user, rw_users.can_login] } + └─LogicalProject { exprs: [PgGetUserbyid(1:Int32) as $expr1] } + └─LogicalValues { rows: [[]], schema: Schema { fields: [] } } optimized_logical_plan_for_batch: |- LogicalJoin { type: LeftOuter, on: true, output: all } ├─LogicalValues { rows: [[]], schema: Schema { fields: [] } } - └─LogicalJoin { type: LeftOuter, on: true, output: all } - ├─LogicalValues { rows: [[]], schema: Schema { fields: [] } } - └─LogicalSysScan { table: rw_users, output_columns: [rw_users.name], required_columns: [rw_users.name, rw_users.id], predicate: (1:Int32 = rw_users.id) } + └─LogicalProject { exprs: [PgGetUserbyid(1:Int32) as $expr1] } + └─LogicalValues { rows: [[]], schema: Schema { fields: [] } } batch_plan: |- BatchNestedLoopJoin { type: LeftOuter, predicate: true, output: all } ├─BatchValues { rows: [[]] } - └─BatchNestedLoopJoin { type: LeftOuter, predicate: true, output: all } - ├─BatchValues { rows: [[]] } - └─BatchProject { exprs: [rw_users.name] } - └─BatchFilter { predicate: (1:Int32 = rw_users.id) } - └─BatchScan { table: rw_users, columns: [rw_users.name, rw_users.id], distribution: Single } + └─BatchProject { exprs: [PgGetUserbyid(1:Int32) as $expr1] } + └─BatchValues { rows: [[]] } - sql: | SELECT n.nspname as "Schema", c.relname as "Name", @@ -233,73 +223,62 @@ AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 1,2; logical_plan: |- - LogicalProject { exprs: [rw_schemas.name, rw_tables.name, Case(($expr1 = 'r':Varchar), 'table':Varchar, ($expr1 = 'v':Varchar), 'view':Varchar, ($expr1 = 'm':Varchar), 'materialized view':Varchar, ($expr1 = 'i':Varchar), 'index':Varchar, ($expr1 = 'S':Varchar), 'sequence':Varchar, ($expr1 = 's':Varchar), 'special':Varchar, ($expr1 = 't':Varchar), 'TOAST table':Varchar, ($expr1 = 'f':Varchar), 'foreign table':Varchar, ($expr1 = 'p':Varchar), 'partitioned table':Varchar, ($expr1 = 'I':Varchar), 'partitioned index':Varchar) as $expr3, rw_users.name] } - └─LogicalApply { type: LeftOuter, on: true, correlated_id: 1, max_one_row: true } - ├─LogicalFilter { predicate: In($expr1, 'r':Varchar, 'p':Varchar, 'v':Varchar, 'm':Varchar, 'S':Varchar, 'f':Varchar, '':Varchar) AND (rw_schemas.name <> 'pg_catalog':Varchar) AND Not(RegexpEq(rw_schemas.name, '^pg_toast':Varchar)) AND (rw_schemas.name <> 'information_schema':Varchar) } - │ └─LogicalJoin { type: LeftOuter, on: (rw_schemas.id = rw_tables.schema_id), output: all } - │ ├─LogicalShare { id: 16 } - │ │ └─LogicalProject { exprs: [rw_tables.id, rw_tables.name, rw_tables.schema_id, rw_tables.owner, 'p':Varchar, Case(('table':Varchar = 'table':Varchar), 'r':Varchar, ('table':Varchar = 'system table':Varchar), 'r':Varchar, ('table':Varchar = 'index':Varchar), 'i':Varchar, ('table':Varchar = 'view':Varchar), 'v':Varchar, ('table':Varchar = 'materialized view':Varchar), 'm':Varchar) as $expr1, 0:Int32, 0:Int32, Array as $expr2] } - │ │ └─LogicalShare { id: 14 } - │ │ └─LogicalUnion { all: true } - │ │ ├─LogicalUnion { all: true } - │ │ │ ├─LogicalUnion { all: true } - │ │ │ │ ├─LogicalUnion { all: true } - │ │ │ │ │ ├─LogicalUnion { all: true } - │ │ │ │ │ │ ├─LogicalUnion { all: true } - │ │ │ │ │ │ │ ├─LogicalProject { exprs: [rw_tables.id, rw_tables.name, 'table':Varchar, rw_tables.schema_id, rw_tables.owner, rw_tables.definition, rw_tables.acl] } - │ │ │ │ │ │ │ │ └─LogicalSysScan { table: rw_tables, columns: [rw_tables.id, rw_tables.name, rw_tables.schema_id, rw_tables.owner, rw_tables.definition, rw_tables.acl, rw_tables.initialized_at, rw_tables.created_at, rw_tables.initialized_at_cluster_version, rw_tables.created_at_cluster_version] } - │ │ │ │ │ │ │ └─LogicalProject { exprs: [rw_system_tables.id, rw_system_tables.name, 'system table':Varchar, rw_system_tables.schema_id, rw_system_tables.owner, rw_system_tables.definition, rw_system_tables.acl] } - │ │ │ │ │ │ │ └─LogicalSysScan { table: rw_system_tables, columns: [rw_system_tables.id, rw_system_tables.name, rw_system_tables.schema_id, rw_system_tables.owner, rw_system_tables.definition, rw_system_tables.acl] } - │ │ │ │ │ │ └─LogicalProject { exprs: [rw_sources.id, rw_sources.name, 'source':Varchar, rw_sources.schema_id, rw_sources.owner, rw_sources.definition, rw_sources.acl] } - │ │ │ │ │ │ └─LogicalSysScan { table: rw_sources, columns: [rw_sources.id, rw_sources.name, rw_sources.schema_id, rw_sources.owner, rw_sources.connector, rw_sources.columns, rw_sources.format, rw_sources.row_encode, rw_sources.append_only, rw_sources.connection_id, rw_sources.definition, rw_sources.acl, rw_sources.initialized_at, rw_sources.created_at, rw_sources.initialized_at_cluster_version, rw_sources.created_at_cluster_version] } - │ │ │ │ │ └─LogicalProject { exprs: [rw_indexes.id, rw_indexes.name, 'index':Varchar, rw_indexes.schema_id, rw_indexes.owner, rw_indexes.definition, rw_indexes.acl] } - │ │ │ │ │ └─LogicalSysScan { table: rw_indexes, columns: [rw_indexes.id, rw_indexes.name, rw_indexes.primary_table_id, rw_indexes.indkey, rw_indexes.schema_id, rw_indexes.owner, rw_indexes.definition, rw_indexes.acl, rw_indexes.initialized_at, rw_indexes.created_at, rw_indexes.initialized_at_cluster_version, rw_indexes.created_at_cluster_version] } - │ │ │ │ └─LogicalProject { exprs: [rw_sinks.id, rw_sinks.name, 'sink':Varchar, rw_sinks.schema_id, rw_sinks.owner, rw_sinks.definition, rw_sinks.acl] } - │ │ │ │ └─LogicalSysScan { table: rw_sinks, columns: [rw_sinks.id, rw_sinks.name, rw_sinks.schema_id, rw_sinks.owner, rw_sinks.connector, rw_sinks.sink_type, rw_sinks.connection_id, rw_sinks.definition, rw_sinks.acl, rw_sinks.initialized_at, rw_sinks.created_at, rw_sinks.initialized_at_cluster_version, rw_sinks.created_at_cluster_version] } - │ │ │ └─LogicalProject { exprs: [rw_materialized_views.id, rw_materialized_views.name, 'materialized view':Varchar, rw_materialized_views.schema_id, rw_materialized_views.owner, rw_materialized_views.definition, rw_materialized_views.acl] } - │ │ │ └─LogicalSysScan { table: rw_materialized_views, columns: [rw_materialized_views.id, rw_materialized_views.name, rw_materialized_views.schema_id, rw_materialized_views.owner, rw_materialized_views.definition, rw_materialized_views.acl, rw_materialized_views.initialized_at, rw_materialized_views.created_at, rw_materialized_views.initialized_at_cluster_version, rw_materialized_views.created_at_cluster_version] } - │ │ └─LogicalProject { exprs: [rw_views.id, rw_views.name, 'view':Varchar, rw_views.schema_id, rw_views.owner, rw_views.definition, rw_views.acl] } - │ │ └─LogicalSysScan { table: rw_views, columns: [rw_views.id, rw_views.name, rw_views.schema_id, rw_views.owner, rw_views.definition, rw_views.acl] } - │ └─LogicalShare { id: 18 } - │ └─LogicalProject { exprs: [rw_schemas.id, rw_schemas.name, rw_schemas.owner, rw_schemas.acl] } - │ └─LogicalSysScan { table: rw_schemas, columns: [rw_schemas.id, rw_schemas.name, rw_schemas.owner, rw_schemas.acl] } - └─LogicalProject { exprs: [rw_users.name] } - └─LogicalFilter { predicate: (CorrelatedInputRef { index: 3, correlated_id: 1 } = rw_users.id) } - └─LogicalShare { id: 22 } - └─LogicalProject { exprs: [rw_users.id, rw_users.name, rw_users.create_db, rw_users.is_super, '********':Varchar] } - └─LogicalSysScan { table: rw_users, columns: [rw_users.id, rw_users.name, rw_users.is_super, rw_users.create_db, rw_users.create_user, rw_users.can_login] } + LogicalProject { exprs: [rw_schemas.name, rw_tables.name, Case(($expr1 = 'r':Varchar), 'table':Varchar, ($expr1 = 'v':Varchar), 'view':Varchar, ($expr1 = 'm':Varchar), 'materialized view':Varchar, ($expr1 = 'i':Varchar), 'index':Varchar, ($expr1 = 'S':Varchar), 'sequence':Varchar, ($expr1 = 's':Varchar), 'special':Varchar, ($expr1 = 't':Varchar), 'TOAST table':Varchar, ($expr1 = 'f':Varchar), 'foreign table':Varchar, ($expr1 = 'p':Varchar), 'partitioned table':Varchar, ($expr1 = 'I':Varchar), 'partitioned index':Varchar) as $expr3, PgGetUserbyid(rw_tables.owner) as $expr4] } + └─LogicalFilter { predicate: In($expr1, 'r':Varchar, 'p':Varchar, 'v':Varchar, 'm':Varchar, 'S':Varchar, 'f':Varchar, '':Varchar) AND (rw_schemas.name <> 'pg_catalog':Varchar) AND Not(RegexpEq(rw_schemas.name, '^pg_toast':Varchar)) AND (rw_schemas.name <> 'information_schema':Varchar) } + └─LogicalJoin { type: LeftOuter, on: (rw_schemas.id = rw_tables.schema_id), output: all } + ├─LogicalShare { id: 16 } + │ └─LogicalProject { exprs: [rw_tables.id, rw_tables.name, rw_tables.schema_id, rw_tables.owner, 'p':Varchar, Case(('table':Varchar = 'table':Varchar), 'r':Varchar, ('table':Varchar = 'system table':Varchar), 'r':Varchar, ('table':Varchar = 'index':Varchar), 'i':Varchar, ('table':Varchar = 'view':Varchar), 'v':Varchar, ('table':Varchar = 'materialized view':Varchar), 'm':Varchar) as $expr1, 0:Int32, 0:Int32, Array as $expr2] } + │ └─LogicalShare { id: 14 } + │ └─LogicalUnion { all: true } + │ ├─LogicalUnion { all: true } + │ │ ├─LogicalUnion { all: true } + │ │ │ ├─LogicalUnion { all: true } + │ │ │ │ ├─LogicalUnion { all: true } + │ │ │ │ │ ├─LogicalUnion { all: true } + │ │ │ │ │ │ ├─LogicalProject { exprs: [rw_tables.id, rw_tables.name, 'table':Varchar, rw_tables.schema_id, rw_tables.owner, rw_tables.definition, rw_tables.acl] } + │ │ │ │ │ │ │ └─LogicalSysScan { table: rw_tables, columns: [rw_tables.id, rw_tables.name, rw_tables.schema_id, rw_tables.owner, rw_tables.definition, rw_tables.acl, rw_tables.initialized_at, rw_tables.created_at, rw_tables.initialized_at_cluster_version, rw_tables.created_at_cluster_version] } + │ │ │ │ │ │ └─LogicalProject { exprs: [rw_system_tables.id, rw_system_tables.name, 'system table':Varchar, rw_system_tables.schema_id, rw_system_tables.owner, rw_system_tables.definition, rw_system_tables.acl] } + │ │ │ │ │ │ └─LogicalSysScan { table: rw_system_tables, columns: [rw_system_tables.id, rw_system_tables.name, rw_system_tables.schema_id, rw_system_tables.owner, rw_system_tables.definition, rw_system_tables.acl] } + │ │ │ │ │ └─LogicalProject { exprs: [rw_sources.id, rw_sources.name, 'source':Varchar, rw_sources.schema_id, rw_sources.owner, rw_sources.definition, rw_sources.acl] } + │ │ │ │ │ └─LogicalSysScan { table: rw_sources, columns: [rw_sources.id, rw_sources.name, rw_sources.schema_id, rw_sources.owner, rw_sources.connector, rw_sources.columns, rw_sources.format, rw_sources.row_encode, rw_sources.append_only, rw_sources.connection_id, rw_sources.definition, rw_sources.acl, rw_sources.initialized_at, rw_sources.created_at, rw_sources.initialized_at_cluster_version, rw_sources.created_at_cluster_version] } + │ │ │ │ └─LogicalProject { exprs: [rw_indexes.id, rw_indexes.name, 'index':Varchar, rw_indexes.schema_id, rw_indexes.owner, rw_indexes.definition, rw_indexes.acl] } + │ │ │ │ └─LogicalSysScan { table: rw_indexes, columns: [rw_indexes.id, rw_indexes.name, rw_indexes.primary_table_id, rw_indexes.indkey, rw_indexes.schema_id, rw_indexes.owner, rw_indexes.definition, rw_indexes.acl, rw_indexes.initialized_at, rw_indexes.created_at, rw_indexes.initialized_at_cluster_version, rw_indexes.created_at_cluster_version] } + │ │ │ └─LogicalProject { exprs: [rw_sinks.id, rw_sinks.name, 'sink':Varchar, rw_sinks.schema_id, rw_sinks.owner, rw_sinks.definition, rw_sinks.acl] } + │ │ │ └─LogicalSysScan { table: rw_sinks, columns: [rw_sinks.id, rw_sinks.name, rw_sinks.schema_id, rw_sinks.owner, rw_sinks.connector, rw_sinks.sink_type, rw_sinks.connection_id, rw_sinks.definition, rw_sinks.acl, rw_sinks.initialized_at, rw_sinks.created_at, rw_sinks.initialized_at_cluster_version, rw_sinks.created_at_cluster_version] } + │ │ └─LogicalProject { exprs: [rw_materialized_views.id, rw_materialized_views.name, 'materialized view':Varchar, rw_materialized_views.schema_id, rw_materialized_views.owner, rw_materialized_views.definition, rw_materialized_views.acl] } + │ │ └─LogicalSysScan { table: rw_materialized_views, columns: [rw_materialized_views.id, rw_materialized_views.name, rw_materialized_views.schema_id, rw_materialized_views.owner, rw_materialized_views.definition, rw_materialized_views.acl, rw_materialized_views.initialized_at, rw_materialized_views.created_at, rw_materialized_views.initialized_at_cluster_version, rw_materialized_views.created_at_cluster_version] } + │ └─LogicalProject { exprs: [rw_views.id, rw_views.name, 'view':Varchar, rw_views.schema_id, rw_views.owner, rw_views.definition, rw_views.acl] } + │ └─LogicalSysScan { table: rw_views, columns: [rw_views.id, rw_views.name, rw_views.schema_id, rw_views.owner, rw_views.definition, rw_views.acl] } + └─LogicalShare { id: 18 } + └─LogicalProject { exprs: [rw_schemas.id, rw_schemas.name, rw_schemas.owner, rw_schemas.acl] } + └─LogicalSysScan { table: rw_schemas, columns: [rw_schemas.id, rw_schemas.name, rw_schemas.owner, rw_schemas.acl] } batch_plan: |- BatchExchange { order: [rw_schemas.name ASC, rw_tables.name ASC], dist: Single } - └─BatchProject { exprs: [rw_schemas.name, rw_tables.name, Case(($expr1 = 'r':Varchar), 'table':Varchar, ($expr1 = 'v':Varchar), 'view':Varchar, ($expr1 = 'm':Varchar), 'materialized view':Varchar, ($expr1 = 'i':Varchar), 'index':Varchar, ($expr1 = 'S':Varchar), 'sequence':Varchar, ($expr1 = 's':Varchar), 'special':Varchar, ($expr1 = 't':Varchar), 'TOAST table':Varchar, ($expr1 = 'f':Varchar), 'foreign table':Varchar, ($expr1 = 'p':Varchar), 'partitioned table':Varchar, ($expr1 = 'I':Varchar), 'partitioned index':Varchar) as $expr2, rw_users.name] } - └─BatchProject { exprs: [rw_tables.name, Case(('table':Varchar = 'table':Varchar), 'r':Varchar, ('table':Varchar = 'system table':Varchar), 'r':Varchar, ('table':Varchar = 'index':Varchar), 'i':Varchar, ('table':Varchar = 'view':Varchar), 'v':Varchar, ('table':Varchar = 'materialized view':Varchar), 'm':Varchar) as $expr1, rw_schemas.name, rw_users.name] } + └─BatchProject { exprs: [rw_schemas.name, rw_tables.name, Case(($expr1 = 'r':Varchar), 'table':Varchar, ($expr1 = 'v':Varchar), 'view':Varchar, ($expr1 = 'm':Varchar), 'materialized view':Varchar, ($expr1 = 'i':Varchar), 'index':Varchar, ($expr1 = 'S':Varchar), 'sequence':Varchar, ($expr1 = 's':Varchar), 'special':Varchar, ($expr1 = 't':Varchar), 'TOAST table':Varchar, ($expr1 = 'f':Varchar), 'foreign table':Varchar, ($expr1 = 'p':Varchar), 'partitioned table':Varchar, ($expr1 = 'I':Varchar), 'partitioned index':Varchar) as $expr2, PgGetUserbyid(rw_tables.owner) as $expr3] } + └─BatchProject { exprs: [rw_tables.name, rw_tables.owner, Case(('table':Varchar = 'table':Varchar), 'r':Varchar, ('table':Varchar = 'system table':Varchar), 'r':Varchar, ('table':Varchar = 'index':Varchar), 'i':Varchar, ('table':Varchar = 'view':Varchar), 'v':Varchar, ('table':Varchar = 'materialized view':Varchar), 'm':Varchar) as $expr1, rw_schemas.name] } └─BatchSort { order: [rw_schemas.name ASC, rw_tables.name ASC] } - └─BatchHashJoin { type: LeftOuter, predicate: rw_tables.owner = rw_users.id, output: all } - ├─BatchExchange { order: [], dist: HashShard(rw_tables.owner) } - │ └─BatchHashJoin { type: Inner, predicate: rw_tables.schema_id = rw_schemas.id, output: all } - │ ├─BatchExchange { order: [], dist: HashShard(rw_tables.schema_id) } - │ │ └─BatchUnion { all: true } - │ │ ├─BatchProject { exprs: [rw_tables.name, 'table':Varchar, rw_tables.schema_id, rw_tables.owner] } - │ │ │ └─BatchScan { table: rw_tables, columns: [rw_tables.name, rw_tables.schema_id, rw_tables.owner], distribution: Single } - │ │ ├─BatchProject { exprs: [rw_system_tables.name, 'system table':Varchar, rw_system_tables.schema_id, rw_system_tables.owner] } - │ │ │ └─BatchScan { table: rw_system_tables, columns: [rw_system_tables.name, rw_system_tables.schema_id, rw_system_tables.owner], distribution: Single } - │ │ ├─BatchProject { exprs: [rw_sources.name, 'source':Varchar, rw_sources.schema_id, rw_sources.owner] } - │ │ │ └─BatchFilter { predicate: null:Boolean } - │ │ │ └─BatchScan { table: rw_sources, columns: [rw_sources.name, rw_sources.schema_id, rw_sources.owner], distribution: Single } - │ │ ├─BatchProject { exprs: [rw_indexes.name, 'index':Varchar, rw_indexes.schema_id, rw_indexes.owner] } - │ │ │ └─BatchValues { rows: [] } - │ │ ├─BatchProject { exprs: [rw_sinks.name, 'sink':Varchar, rw_sinks.schema_id, rw_sinks.owner] } - │ │ │ └─BatchFilter { predicate: null:Boolean } - │ │ │ └─BatchScan { table: rw_sinks, columns: [rw_sinks.name, rw_sinks.schema_id, rw_sinks.owner], distribution: Single } - │ │ ├─BatchProject { exprs: [rw_materialized_views.name, 'materialized view':Varchar, rw_materialized_views.schema_id, rw_materialized_views.owner] } - │ │ │ └─BatchScan { table: rw_materialized_views, columns: [rw_materialized_views.name, rw_materialized_views.schema_id, rw_materialized_views.owner], distribution: Single } - │ │ └─BatchProject { exprs: [rw_views.name, 'view':Varchar, rw_views.schema_id, rw_views.owner] } - │ │ └─BatchScan { table: rw_views, columns: [rw_views.name, rw_views.schema_id, rw_views.owner], distribution: Single } - │ └─BatchExchange { order: [], dist: HashShard(rw_schemas.id) } - │ └─BatchFilter { predicate: (rw_schemas.name <> 'pg_catalog':Varchar) AND Not(RegexpEq(rw_schemas.name, '^pg_toast':Varchar)) AND (rw_schemas.name <> 'information_schema':Varchar) } - │ └─BatchScan { table: rw_schemas, columns: [rw_schemas.id, rw_schemas.name], distribution: Single } - └─BatchExchange { order: [], dist: HashShard(rw_users.id) } - └─BatchProject { exprs: [rw_users.name, rw_users.id] } - └─BatchScan { table: rw_users, columns: [rw_users.id, rw_users.name], distribution: Single } + └─BatchHashJoin { type: Inner, predicate: rw_tables.schema_id = rw_schemas.id, output: all } + ├─BatchExchange { order: [], dist: HashShard(rw_tables.schema_id) } + │ └─BatchUnion { all: true } + │ ├─BatchProject { exprs: [rw_tables.name, 'table':Varchar, rw_tables.schema_id, rw_tables.owner] } + │ │ └─BatchScan { table: rw_tables, columns: [rw_tables.name, rw_tables.schema_id, rw_tables.owner], distribution: Single } + │ ├─BatchProject { exprs: [rw_system_tables.name, 'system table':Varchar, rw_system_tables.schema_id, rw_system_tables.owner] } + │ │ └─BatchScan { table: rw_system_tables, columns: [rw_system_tables.name, rw_system_tables.schema_id, rw_system_tables.owner], distribution: Single } + │ ├─BatchProject { exprs: [rw_sources.name, 'source':Varchar, rw_sources.schema_id, rw_sources.owner] } + │ │ └─BatchFilter { predicate: null:Boolean } + │ │ └─BatchScan { table: rw_sources, columns: [rw_sources.name, rw_sources.schema_id, rw_sources.owner], distribution: Single } + │ ├─BatchProject { exprs: [rw_indexes.name, 'index':Varchar, rw_indexes.schema_id, rw_indexes.owner] } + │ │ └─BatchValues { rows: [] } + │ ├─BatchProject { exprs: [rw_sinks.name, 'sink':Varchar, rw_sinks.schema_id, rw_sinks.owner] } + │ │ └─BatchFilter { predicate: null:Boolean } + │ │ └─BatchScan { table: rw_sinks, columns: [rw_sinks.name, rw_sinks.schema_id, rw_sinks.owner], distribution: Single } + │ ├─BatchProject { exprs: [rw_materialized_views.name, 'materialized view':Varchar, rw_materialized_views.schema_id, rw_materialized_views.owner] } + │ │ └─BatchScan { table: rw_materialized_views, columns: [rw_materialized_views.name, rw_materialized_views.schema_id, rw_materialized_views.owner], distribution: Single } + │ └─BatchProject { exprs: [rw_views.name, 'view':Varchar, rw_views.schema_id, rw_views.owner] } + │ └─BatchScan { table: rw_views, columns: [rw_views.name, rw_views.schema_id, rw_views.owner], distribution: Single } + └─BatchExchange { order: [], dist: HashShard(rw_schemas.id) } + └─BatchFilter { predicate: (rw_schemas.name <> 'pg_catalog':Varchar) AND Not(RegexpEq(rw_schemas.name, '^pg_toast':Varchar)) AND (rw_schemas.name <> 'information_schema':Varchar) } + └─BatchScan { table: rw_schemas, columns: [rw_schemas.id, rw_schemas.name], distribution: Single } - sql: | create table auction (date_time date); select * from hop( auction, auction.date_time, INTERVAL '1', INTERVAL '3600' ) AS hop_1 diff --git a/src/frontend/planner_test/tests/testdata/output/topn.yaml b/src/frontend/planner_test/tests/testdata/output/topn.yaml new file mode 100644 index 000000000000..d3bc247fb11d --- /dev/null +++ b/src/frontend/planner_test/tests/testdata/output/topn.yaml @@ -0,0 +1,28 @@ +# This file is automatically generated. See `src/frontend/planner_test/README.md` for more information. +- sql: | + CREATE TABLE t1 (pk int, a int, b int, c bigint, d int); + CREATE MATERIALIZED VIEW t1_mv AS SELECT SUM(a) as sa, SUM(b) as sb, SUM(c) as sc, SUM(d) as sd FROM t1; + SELECT sa, count(*) as cnt2 FROM t1_mv GROUP BY sa ORDER BY cnt2 DESC LIMIT 50 OFFSET 50; + batch_plan: |- + BatchExchange { order: [count DESC], dist: Single } + └─BatchTopN { order: [count DESC], limit: 50, offset: 50 } + └─BatchSimpleAgg { aggs: [internal_last_seen_value(t1_mv.sa), count] } + └─BatchScan { table: t1_mv, columns: [t1_mv.sa], distribution: Single } +- sql: | + CREATE TABLE t1 (pk int, a int, b int, c bigint, d int); + CREATE MATERIALIZED VIEW t1_mv AS SELECT * from t1; + SELECT * FROM t1_mv ORDER BY a DESC LIMIT 50 OFFSET 50; + batch_plan: |- + BatchTopN { order: [t1_mv.a DESC], limit: 50, offset: 50 } + └─BatchExchange { order: [], dist: Single } + └─BatchTopN { order: [t1_mv.a DESC], limit: 100, offset: 0 } + └─BatchScan { table: t1_mv, columns: [t1_mv.pk, t1_mv.a, t1_mv.b, t1_mv.c, t1_mv.d], distribution: SomeShard } +- sql: | + CREATE TABLE t1 (pk int, a int, b int, c bigint, d int); + CREATE MATERIALIZED VIEW t1_mv AS SELECT * from t1 order by a desc; + SELECT * FROM t1_mv ORDER BY a DESC LIMIT 50 OFFSET 50; + batch_plan: |- + BatchTopN { order: [t1_mv.a DESC], limit: 50, offset: 50 } + └─BatchExchange { order: [], dist: Single } + └─BatchLimit { limit: 100, offset: 0 } + └─BatchScan { table: t1_mv, columns: [t1_mv.pk, t1_mv.a, t1_mv.b, t1_mv.c, t1_mv.d], limit: 100, distribution: SomeShard } diff --git a/src/frontend/src/binder/expr/function.rs b/src/frontend/src/binder/expr/function.rs index 477c493ef2ab..b787632846e9 100644 --- a/src/frontend/src/binder/expr/function.rs +++ b/src/frontend/src/binder/expr/function.rs @@ -37,11 +37,11 @@ use risingwave_sqlparser::parser::ParserError; use thiserror_ext::AsReport; use crate::binder::bind_context::Clause; -use crate::binder::{Binder, BoundQuery, BoundSetExpr, UdfContext}; +use crate::binder::{Binder, UdfContext}; use crate::error::{ErrorCode, Result, RwError}; use crate::expr::{ AggCall, Expr, ExprImpl, ExprType, FunctionCall, FunctionCallWithLambda, Literal, Now, OrderBy, - Subquery, SubqueryKind, TableFunction, TableFunctionType, UserDefinedFunction, WindowFunction, + TableFunction, TableFunctionType, UserDefinedFunction, WindowFunction, }; use crate::utils::Condition; @@ -1230,112 +1230,27 @@ impl Binder { ("current_role", current_user()), ("current_user", current_user()), ("user", current_user()), - ("pg_get_userbyid", guard_by_len(1, raw(|binder, inputs|{ - let input = &inputs[0]; - let bound_query = binder.bind_get_user_by_id_select(input)?; - Ok(ExprImpl::Subquery(Box::new(Subquery::new( - BoundQuery { - body: BoundSetExpr::Select(Box::new(bound_query)), - order: vec![], - limit: None, - offset: None, - with_ties: false, - extra_order_exprs: vec![], - }, - SubqueryKind::Scalar, - )))) - } - ))), + ("pg_get_userbyid", raw_call(ExprType::PgGetUserbyid)), ("pg_get_indexdef", raw_call(ExprType::PgGetIndexdef)), ("pg_get_viewdef", raw_call(ExprType::PgGetViewdef)), - ("pg_relation_size", dispatch_by_len(vec![ - (1, raw(|binder, inputs|{ - let table_name = &inputs[0]; - let bound_query = binder.bind_get_table_size_select("pg_relation_size", table_name)?; - Ok(ExprImpl::Subquery(Box::new(Subquery::new( - BoundQuery { - body: BoundSetExpr::Select(Box::new(bound_query)), - order: vec![], - limit: None, - offset: None, - with_ties: false, - extra_order_exprs: vec![], - }, - SubqueryKind::Scalar, - )))) - })), - (2, raw(|binder, inputs|{ - let table_name = &inputs[0]; - match inputs[1].as_literal() { - Some(literal) if literal.return_type() == DataType::Varchar => { - match literal - .get_data() - .as_ref() - .expect("ExprImpl value is a Literal but cannot get ref to data") - .as_utf8() - .as_ref() { - "main" => { - let bound_query = binder.bind_get_table_size_select("pg_relation_size", table_name)?; - Ok(ExprImpl::Subquery(Box::new(Subquery::new( - BoundQuery { - body: BoundSetExpr::Select(Box::new(bound_query)), - order: vec![], - limit: None, - offset: None, - with_ties: false, - extra_order_exprs: vec![], - }, - SubqueryKind::Scalar, - )))) - }, - // These options are invalid in RW so we return 0 value as the result - "fsm"|"vm"|"init" => { - Ok(ExprImpl::literal_int(0)) - }, - _ => Err(ErrorCode::InvalidInputSyntax( - "invalid fork name. Valid fork names are \"main\", \"fsm\", \"vm\", and \"init\"".into()).into()) - } - }, - _ => Err(ErrorCode::ExprError( - "The 2nd argument of `pg_relation_size` must be string literal.".into(), - ) - .into()) - } - })), - ] - )), - ("pg_table_size", guard_by_len(1, raw(|binder, inputs|{ - let input = &inputs[0]; - let bound_query = binder.bind_get_table_size_select("pg_table_size", input)?; - Ok(ExprImpl::Subquery(Box::new(Subquery::new( - BoundQuery { - body: BoundSetExpr::Select(Box::new(bound_query)), - order: vec![], - limit: None, - offset: None, - with_ties: false, - extra_order_exprs: vec![], - }, - SubqueryKind::Scalar, - )))) - } - ))), - ("pg_indexes_size", guard_by_len(1, raw(|binder, inputs|{ - let input = &inputs[0]; - let bound_query = binder.bind_get_indexes_size_select(input)?; - Ok(ExprImpl::Subquery(Box::new(Subquery::new( - BoundQuery { - body: BoundSetExpr::Select(Box::new(bound_query)), - order: vec![], - limit: None, - offset: None, - with_ties: false, - extra_order_exprs: vec![], - }, - SubqueryKind::Scalar, - )))) + ("pg_relation_size", raw(|_binder, mut inputs|{ + if inputs.is_empty() { + return Err(ErrorCode::ExprError( + "function pg_relation_size() does not exist".into(), + ) + .into()); } - ))), + inputs[0].cast_to_regclass_mut()?; + Ok(FunctionCall::new(ExprType::PgRelationSize, inputs)?.into()) + })), + ("pg_table_size", guard_by_len(1, raw(|_binder, mut inputs|{ + inputs[0].cast_to_regclass_mut()?; + Ok(FunctionCall::new(ExprType::PgRelationSize, inputs)?.into()) + }))), + ("pg_indexes_size", guard_by_len(1, raw(|_binder, mut inputs|{ + inputs[0].cast_to_regclass_mut()?; + Ok(FunctionCall::new(ExprType::PgIndexesSize, inputs)?.into()) + }))), ("pg_get_expr", raw(|_binder, inputs|{ if inputs.len() == 2 || inputs.len() == 3 { // TODO: implement pg_get_expr rather than just return empty as an workaround. diff --git a/src/frontend/src/binder/expr/mod.rs b/src/frontend/src/binder/expr/mod.rs index 2baf9cb1f84e..0bc12545984c 100644 --- a/src/frontend/src/binder/expr/mod.rs +++ b/src/frontend/src/binder/expr/mod.rs @@ -612,23 +612,9 @@ impl Binder { match &data_type { // Casting to Regclass type means getting the oid of expr. // See https://www.postgresql.org/docs/current/datatype-oid.html. - // Currently only string liter expr is supported since we cannot handle subquery in join - // on condition: https://github.com/risingwavelabs/risingwave/issues/6852 - // TODO: Add generic expr support when needed AstDataType::Regclass => { let input = self.bind_expr_inner(expr)?; - match input.return_type() { - DataType::Varchar => Ok(ExprImpl::FunctionCall(Box::new( - FunctionCall::new_unchecked( - ExprType::CastRegclass, - vec![input], - DataType::Int32, - ), - ))), - DataType::Int32 => Ok(input), - dt if dt.is_int() => Ok(input.cast_explicit(DataType::Int32)?), - _ => Err(ErrorCode::BindError("Unsupported input type".to_string()).into()), - } + Ok(input.cast_to_regclass()?) } AstDataType::Regproc => { let lhs = self.bind_expr_inner(expr)?; diff --git a/src/frontend/src/binder/select.rs b/src/frontend/src/binder/select.rs index 4afc350d3d60..d9848ed76973 100644 --- a/src/frontend/src/binder/select.rs +++ b/src/frontend/src/binder/select.rs @@ -16,13 +16,11 @@ use std::collections::{HashMap, HashSet}; use std::fmt::Debug; use itertools::Itertools; -use risingwave_common::catalog::{Field, Schema, PG_CATALOG_SCHEMA_NAME, RW_CATALOG_SCHEMA_NAME}; -use risingwave_common::types::{DataType, ScalarImpl}; +use risingwave_common::catalog::{Field, Schema}; +use risingwave_common::types::ScalarImpl; use risingwave_common::util::iter_util::ZipEqFast; -use risingwave_expr::aggregate::AggKind; use risingwave_sqlparser::ast::{ - BinaryOperator, DataType as AstDataType, Distinct, Expr, Ident, Join, JoinConstraint, - JoinOperator, ObjectName, Select, SelectItem, TableFactor, TableWithJoins, Value, + DataType as AstDataType, Distinct, Expr, Select, SelectItem, Value, }; use super::bind_context::{Clause, ColumnBinding}; @@ -31,10 +29,7 @@ use super::UNNAMED_COLUMN; use crate::binder::{Binder, Relation}; use crate::catalog::check_valid_column_name; use crate::error::{ErrorCode, Result, RwError}; -use crate::expr::{ - AggCall, CorrelatedId, CorrelatedInputRef, Depth, Expr as _, ExprImpl, ExprType, FunctionCall, - InputRef, -}; +use crate::expr::{CorrelatedId, Depth, Expr as _, ExprImpl, ExprType, FunctionCall, InputRef}; use crate::utils::group_by::GroupBy; #[derive(Debug, Clone)] @@ -523,207 +518,6 @@ impl Binder { Ok((returning_list, fields)) } - /// `bind_get_user_by_id_select` binds a select statement that returns a single user name by id, - /// this is used for function `pg_catalog.get_user_by_id()`. - pub fn bind_get_user_by_id_select(&mut self, input: &ExprImpl) -> Result { - let schema = Schema { - fields: vec![Field::with_name( - DataType::Varchar, - UNNAMED_COLUMN.to_string(), - )], - }; - let input = match input { - ExprImpl::InputRef(input_ref) => { - CorrelatedInputRef::new(input_ref.index(), input_ref.return_type(), 1).into() - } - ExprImpl::CorrelatedInputRef(col_input_ref) => CorrelatedInputRef::new( - col_input_ref.index(), - col_input_ref.return_type(), - col_input_ref.depth() + 1, - ) - .into(), - ExprImpl::Literal(_) => input.clone(), - _ => return Err(ErrorCode::BindError("Unsupported input type".to_string()).into()), - }; - let from = Some(self.bind_relation_by_name_inner( - Some(PG_CATALOG_SCHEMA_NAME), - "pg_user", - None, - false, - )?); - let select_items = vec![self.bind_column(&["pg_user".into(), "usename".into()])?]; - let where_clause = Some( - FunctionCall::new( - ExprType::Equal, - vec![ - input, - self.bind_column(&["pg_user".into(), "usesysid".into()])?, - ], - )? - .into(), - ); - - Ok(BoundSelect { - distinct: BoundDistinct::All, - select_items, - aliases: vec![None], - from, - where_clause, - group_by: GroupBy::GroupKey(vec![]), - having: None, - schema, - }) - } - - /// This returns the size of all the indexes that are on the specified table. - pub fn bind_get_indexes_size_select(&mut self, table: &ExprImpl) -> Result { - // this function is implemented with the following query: - // SELECT sum(total_key_size + total_value_size) - // FROM rw_catalog.rw_table_stats as stats - // JOIN pg_index on stats.id = pg_index.indexrelid - // WHERE pg_index.indrelid = 'table_name'::regclass - - // Filter to only the Indexes on this table - let table_id = self.table_id_query(table)?; - - let constraint = JoinConstraint::On(Expr::BinaryOp { - left: Box::new(Expr::Identifier(Ident::new_unchecked("id"))), - op: BinaryOperator::Eq, - right: Box::new(Expr::Identifier(Ident::new_unchecked("indexrelid"))), - }); - let indexes_with_stats = self.bind_table_with_joins(TableWithJoins { - relation: TableFactor::Table { - name: ObjectName(vec![RW_CATALOG_SCHEMA_NAME.into(), "rw_table_stats".into()]), - alias: None, - for_system_time_as_of_proctime: false, - }, - joins: vec![Join { - relation: TableFactor::Table { - name: ObjectName(vec![PG_CATALOG_SCHEMA_NAME.into(), "pg_index".into()]), - alias: None, - for_system_time_as_of_proctime: false, - }, - join_operator: JoinOperator::Inner(constraint), - }], - })?; - - // Get the size of an index by adding the size of the keys and the size of the values - let sum = FunctionCall::new( - ExprType::Add, - vec![ - self.bind_column(&["rw_table_stats".into(), "total_key_size".into()])?, - self.bind_column(&["rw_table_stats".into(), "total_value_size".into()])?, - ], - )? - .into(); - - // There could be multiple indexes on a table so aggregate the sizes of all indexes - let select_items: Vec = - vec![AggCall::new_unchecked(AggKind::Sum0, vec![sum], DataType::Int64)?.into()]; - - let indrelid_ref = self.bind_column(&["indrelid".into()])?; - let where_clause: Option = - Some(FunctionCall::new(ExprType::Equal, vec![indrelid_ref, table_id])?.into()); - - // define the output schema - let result_schema = Schema { - fields: vec![Field::with_name( - DataType::Int64, - "pg_indexes_size".to_string(), - )], - }; - - Ok(BoundSelect { - distinct: BoundDistinct::All, - select_items, - aliases: vec![None], - from: Some(indexes_with_stats), - where_clause, - group_by: GroupBy::GroupKey(vec![]), - having: None, - schema: result_schema, - }) - } - - pub fn bind_get_table_size_select( - &mut self, - output_name: &str, - table: &ExprImpl, - ) -> Result { - // define the output schema - let result_schema = Schema { - fields: vec![Field::with_name(DataType::Int64, output_name.to_string())], - }; - - // Get table stats data - let from = Some(self.bind_relation_by_name_inner( - Some(RW_CATALOG_SCHEMA_NAME), - "rw_table_stats", - None, - false, - )?); - - let table_id = self.table_id_query(table)?; - - // Filter to only the Indexes on this table - let where_clause: Option = Some( - FunctionCall::new( - ExprType::Equal, - vec![ - table_id, - self.bind_column(&["rw_table_stats".into(), "id".into()])?, - ], - )? - .into(), - ); - - // Add the space used by keys and the space used by values to get the total space used by - // the table - let key_value_size_sum = FunctionCall::new( - ExprType::Add, - vec![ - self.bind_column(&["rw_table_stats".into(), "total_key_size".into()])?, - self.bind_column(&["rw_table_stats".into(), "total_value_size".into()])?, - ], - )? - .into(); - let select_items = vec![key_value_size_sum]; - - Ok(BoundSelect { - distinct: BoundDistinct::All, - select_items, - aliases: vec![None], - from, - where_clause, - group_by: GroupBy::GroupKey(vec![]), - having: None, - schema: result_schema, - }) - } - - /// Given literal varchar this will return the Object ID of the table or index whose - /// name matches the varchar. Given a literal integer, this will return the integer regardless - /// of whether an object exists with an Object ID that matches the integer. - fn table_id_query(&mut self, table: &ExprImpl) -> Result { - match table.as_literal() { - Some(literal) if literal.return_type().is_int() => Ok(table.clone()), - Some(literal) if literal.return_type() == DataType::Varchar => { - let table_name = literal - .get_data() - .as_ref() - .expect("ExprImpl value is a Literal but cannot get ref to data") - .as_utf8(); - self.bind_cast( - Expr::Value(Value::SingleQuotedString(table_name.to_string())), - AstDataType::Regclass, - ) - } - _ => Err(RwError::from(ErrorCode::ExprError( - "Expected an integer or varchar literal".into(), - ))), - } - } - pub fn iter_bound_columns<'a>( column_binding: impl Iterator, ) -> (Vec, Vec>) { diff --git a/src/frontend/src/expr/function_call.rs b/src/frontend/src/expr/function_call.rs index 5868c74355cc..af1f84b321eb 100644 --- a/src/frontend/src/expr/function_call.rs +++ b/src/frontend/src/expr/function_call.rs @@ -425,7 +425,7 @@ pub fn is_row_function(expr: &ExprImpl) -> bool { #[derive(Debug, Error)] #[error("{0}")] -pub struct CastError(String); +pub struct CastError(pub(super) String); impl From for ErrorCode { fn from(value: CastError) -> Self { diff --git a/src/frontend/src/expr/function_impl/context.rs b/src/frontend/src/expr/function_impl/context.rs index 1a5ecf88ce44..74cc5001043a 100644 --- a/src/frontend/src/expr/function_impl/context.rs +++ b/src/frontend/src/expr/function_impl/context.rs @@ -22,6 +22,7 @@ use crate::session::AuthContext; // Only for local mode. define_context! { pub(super) CATALOG_READER: crate::catalog::CatalogReader, + pub(super) USER_INFO_READER: crate::user::user_service::UserInfoReader, pub(super) AUTH_CONTEXT: Arc, pub(super) DB_NAME: String, pub(super) SEARCH_PATH: SearchPath, diff --git a/src/frontend/src/expr/function_impl/mod.rs b/src/frontend/src/expr/function_impl/mod.rs index 0541c392d187..ad0b3b7a853d 100644 --- a/src/frontend/src/expr/function_impl/mod.rs +++ b/src/frontend/src/expr/function_impl/mod.rs @@ -16,4 +16,7 @@ mod cast_regclass; mod col_description; pub mod context; mod pg_get_indexdef; +mod pg_get_userbyid; mod pg_get_viewdef; +mod pg_indexes_size; +mod pg_relation_size; diff --git a/src/frontend/src/expr/function_impl/pg_get_userbyid.rs b/src/frontend/src/expr/function_impl/pg_get_userbyid.rs new file mode 100644 index 000000000000..175d07e6aef7 --- /dev/null +++ b/src/frontend/src/expr/function_impl/pg_get_userbyid.rs @@ -0,0 +1,31 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_expr::{capture_context, function, Result}; + +use super::context::USER_INFO_READER; +use crate::user::user_service::UserInfoReader; + +#[function("pg_get_userbyid(int4) -> varchar")] +fn pg_get_userbyid(oid: i32) -> Result>> { + pg_get_userbyid_impl_captured(oid) +} + +#[capture_context(USER_INFO_READER)] +fn pg_get_userbyid_impl(reader: &UserInfoReader, oid: i32) -> Result>> { + Ok(reader + .read_guard() + .get_user_name_by_id(oid as u32) + .map(|s| s.into_boxed_str())) +} diff --git a/src/frontend/src/expr/function_impl/pg_indexes_size.rs b/src/frontend/src/expr/function_impl/pg_indexes_size.rs new file mode 100644 index 000000000000..cbcf631d415e --- /dev/null +++ b/src/frontend/src/expr/function_impl/pg_indexes_size.rs @@ -0,0 +1,51 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_expr::{capture_context, function, ExprError, Result}; +use thiserror_ext::AsReport; + +use super::context::{CATALOG_READER, DB_NAME}; +use crate::catalog::CatalogReader; + +/// Computes the total disk space used by indexes attached to the specified table. +#[function("pg_indexes_size(int4) -> int8")] +fn pg_indexes_size(oid: i32) -> Result { + pg_indexes_size_impl_captured(oid) +} + +#[capture_context(CATALOG_READER, DB_NAME)] +fn pg_indexes_size_impl(catalog: &CatalogReader, db_name: &str, oid: i32) -> Result { + let catalog = catalog.read_guard(); + let database = catalog + .get_database_by_name(db_name) + .map_err(|e| ExprError::InvalidParam { + name: "oid", + reason: e.to_report_string().into(), + })?; + let mut sum = 0; + for schema in database.iter_schemas() { + for index in schema.iter_index() { + if index.primary_table.id().table_id == oid as u32 { + if let Some(table_stats) = catalog + .table_stats() + .table_stats + .get(&index.primary_table.id().table_id) + { + sum += table_stats.total_key_size + table_stats.total_value_size; + } + } + } + } + Ok(sum) +} diff --git a/src/frontend/src/expr/function_impl/pg_relation_size.rs b/src/frontend/src/expr/function_impl/pg_relation_size.rs new file mode 100644 index 000000000000..b851688ceede --- /dev/null +++ b/src/frontend/src/expr/function_impl/pg_relation_size.rs @@ -0,0 +1,50 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_expr::{capture_context, function, ExprError, Result}; + +use super::context::CATALOG_READER; +use crate::catalog::CatalogReader; + +/// Computes the disk space used by one “fork” of the specified relation. +#[function("pg_relation_size(int4) -> int8")] +fn pg_relation_size(oid: i32) -> Result { + pg_relation_size_impl_captured(oid, "main") +} + +#[function("pg_relation_size(int4, varchar) -> int8")] +fn pg_relation_size_fork(oid: i32, fork: &str) -> Result { + pg_relation_size_impl_captured(oid, fork) +} + +#[capture_context(CATALOG_READER)] +fn pg_relation_size_impl(catalog: &CatalogReader, oid: i32, fork: &str) -> Result { + match fork { + "main" => {} + // These options are invalid in RW so we return 0 value as the result + "fsm" | "vm" | "init" => return Ok(0), + _ => return Err(ExprError::InvalidParam { + name: "fork", + reason: + "invalid fork name. Valid fork names are \"main\", \"fsm\", \"vm\", and \"init\"" + .into(), + }), + } + let catalog = catalog.read_guard(); + if let Some(stats) = catalog.table_stats().table_stats.get(&(oid as u32)) { + Ok(stats.total_key_size + stats.total_value_size) + } else { + Ok(0) + } +} diff --git a/src/frontend/src/expr/mod.rs b/src/frontend/src/expr/mod.rs index 1241ef6e4e28..78ae2db726a3 100644 --- a/src/frontend/src/expr/mod.rs +++ b/src/frontend/src/expr/mod.rs @@ -265,6 +265,26 @@ impl ExprImpl { FunctionCall::cast_mut(self, target, CastContext::Explicit) } + /// Casting to Regclass type means getting the oid of expr. + /// See + pub fn cast_to_regclass(self) -> Result { + match self.return_type() { + DataType::Varchar => Ok(ExprImpl::FunctionCall(Box::new( + FunctionCall::new_unchecked(ExprType::CastRegclass, vec![self], DataType::Int32), + ))), + DataType::Int32 => Ok(self), + dt if dt.is_int() => Ok(self.cast_explicit(DataType::Int32)?), + _ => Err(CastError("Unsupported input type".to_string())), + } + } + + /// Shorthand to inplace cast expr to `regclass` type. + pub fn cast_to_regclass_mut(&mut self) -> Result<(), CastError> { + let owned = std::mem::replace(self, ExprImpl::literal_bool(false)); + *self = owned.cast_to_regclass()?; + Ok(()) + } + /// Ensure the return type of this expression is an array of some type. pub fn ensure_array_type(&self) -> Result<(), ErrorCode> { if self.is_untyped() { diff --git a/src/frontend/src/expr/pure.rs b/src/frontend/src/expr/pure.rs index c50f1cc2460b..5528b4614c35 100644 --- a/src/frontend/src/expr/pure.rs +++ b/src/frontend/src/expr/pure.rs @@ -250,6 +250,9 @@ impl ExprVisitor for ImpureAnalyzer { | expr_node::Type::PgGetIndexdef | expr_node::Type::ColDescription | expr_node::Type::PgGetViewdef + | expr_node::Type::PgGetUserbyid + | expr_node::Type::PgIndexesSize + | expr_node::Type::PgRelationSize | expr_node::Type::MakeTimestamptz => self.impure = true, } } diff --git a/src/frontend/src/optimizer/plan_node/batch_topn.rs b/src/frontend/src/optimizer/plan_node/batch_topn.rs index 3b0072821e38..d8f96668f88a 100644 --- a/src/frontend/src/optimizer/plan_node/batch_topn.rs +++ b/src/frontend/src/optimizer/plan_node/batch_topn.rs @@ -52,7 +52,9 @@ impl BatchTopN { self.core.limit_attr.with_ties(), ); let new_offset = 0; - let partial_input: PlanRef = if input.order().satisfies(&self.core.order) { + let partial_input: PlanRef = if input.order().satisfies(&self.core.order) + && !self.core.limit_attr.with_ties() + { let logical_partial_limit = generic::Limit::new(input, new_limit.limit(), new_offset); let batch_partial_limit = BatchLimit::new(logical_partial_limit); batch_partial_limit.into() @@ -63,17 +65,23 @@ impl BatchTopN { batch_partial_topn.into() }; - let single_dist = RequiredDist::single(); - let ensure_single_dist = if !partial_input.distribution().satisfies(&single_dist) { - single_dist.enforce_if_not_satisfies(partial_input, &Order::any())? - } else { - // The input's distribution is singleton, so use one phase topn is enough. - return Ok(partial_input); - }; + let ensure_single_dist = + RequiredDist::single().enforce_if_not_satisfies(partial_input, &Order::any())?; let batch_global_topn = self.clone_with_input(ensure_single_dist); Ok(batch_global_topn.into()) } + + fn one_phase_topn(&self, input: PlanRef) -> Result { + if input.order().satisfies(&self.core.order) && !self.core.limit_attr.with_ties() { + let logical_limit = + generic::Limit::new(input, self.core.limit_attr.limit(), self.core.offset); + let batch_limit = BatchLimit::new(logical_limit); + Ok(batch_limit.into()) + } else { + Ok(self.clone_with_input(input).into()) + } + } } impl_distill_by_unit!(BatchTopN, core, "BatchTopN"); @@ -94,7 +102,13 @@ impl_plan_tree_node_for_unary! {BatchTopN} impl ToDistributedBatch for BatchTopN { fn to_distributed(&self) -> Result { - self.two_phase_topn(self.input().to_distributed()?) + let input = self.input().to_distributed()?; + let single_dist = RequiredDist::single(); + if input.distribution().satisfies(&single_dist) { + self.one_phase_topn(input) + } else { + self.two_phase_topn(input) + } } } @@ -112,7 +126,13 @@ impl ToBatchPb for BatchTopN { impl ToLocalBatch for BatchTopN { fn to_local(&self) -> Result { - self.two_phase_topn(self.input().to_local()?) + let input = self.input().to_local()?; + let single_dist = RequiredDist::single(); + if input.distribution().satisfies(&single_dist) { + self.one_phase_topn(input) + } else { + self.two_phase_topn(input) + } } } diff --git a/src/frontend/src/scheduler/local.rs b/src/frontend/src/scheduler/local.rs index 2cb642dc9054..c155cfe9aa23 100644 --- a/src/frontend/src/scheduler/local.rs +++ b/src/frontend/src/scheduler/local.rs @@ -20,7 +20,7 @@ use std::time::Duration; use anyhow::anyhow; use futures::stream::BoxStream; -use futures::StreamExt; +use futures::{FutureExt, StreamExt}; use futures_async_stream::try_stream; use itertools::Itertools; use pgwire::pg_server::BoxedError; @@ -147,6 +147,7 @@ impl LocalQueryExecution { let shutdown_rx = self.shutdown_rx().clone(); let catalog_reader = self.front_env.catalog_reader().clone(); + let user_info_reader = self.front_env.user_info_reader().clone(); let auth_context = self.session.auth_context().clone(); let db_name = self.session.database().to_string(); let search_path = self.session.config().search_path(); @@ -173,14 +174,16 @@ impl LocalQueryExecution { use risingwave_expr::expr_context::TIME_ZONE; use crate::expr::function_impl::context::{ - AUTH_CONTEXT, CATALOG_READER, DB_NAME, SEARCH_PATH, + AUTH_CONTEXT, CATALOG_READER, DB_NAME, SEARCH_PATH, USER_INFO_READER, }; - let exec = async move { CATALOG_READER::scope(catalog_reader, exec).await }; - let exec = async move { DB_NAME::scope(db_name, exec).await }; - let exec = async move { SEARCH_PATH::scope(search_path, exec).await }; - let exec = async move { AUTH_CONTEXT::scope(auth_context, exec).await }; - let exec = async move { TIME_ZONE::scope(time_zone, exec).await }; + // box is necessary, otherwise the size of `exec` will double each time it is nested. + let exec = async move { CATALOG_READER::scope(catalog_reader, exec).await }.boxed(); + let exec = async move { USER_INFO_READER::scope(user_info_reader, exec).await }.boxed(); + let exec = async move { DB_NAME::scope(db_name, exec).await }.boxed(); + let exec = async move { SEARCH_PATH::scope(search_path, exec).await }.boxed(); + let exec = async move { AUTH_CONTEXT::scope(auth_context, exec).await }.boxed(); + let exec = async move { TIME_ZONE::scope(time_zone, exec).await }.boxed(); if let Some(timeout) = timeout { let exec = async move { diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml index 013ce2200f0d..20a82883e81c 100644 --- a/src/meta/Cargo.toml +++ b/src/meta/Cargo.toml @@ -61,15 +61,15 @@ risingwave_rpc_client = { workspace = true } risingwave_sqlparser = { workspace = true } rw_futures_util = { workspace = true } scopeguard = "1.2.0" -sea-orm = { version = "0.12.0", features = [ +sea-orm = { version = "0.12.14", features = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-native-tls", "macros", ] } -serde = { version = "1", features = ["derive"] } -serde_json = "1" +serde = { version = "1.0.196", features = ["derive"] } +serde_json = "1.0.113" strum = { version = "0.25", features = ["derive"] } sync-point = { path = "../utils/sync-point" } thiserror = "1" diff --git a/src/meta/model_v2/Cargo.toml b/src/meta/model_v2/Cargo.toml index 8cc1407983f3..9d75bf22b5c6 100644 --- a/src/meta/model_v2/Cargo.toml +++ b/src/meta/model_v2/Cargo.toml @@ -14,15 +14,16 @@ ignored = ["workspace-hack"] normal = ["workspace-hack"] [dependencies] +prost = { workspace = true } risingwave_common = { workspace = true } risingwave_hummock_sdk = { workspace = true } risingwave_pb = { workspace = true } -sea-orm = { version = "0.12.0", features = [ +sea-orm = { version = "0.12.14", features = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-native-tls", "macros", ] } -serde = { version = "1", features = ["derive"] } -serde_json = "1" +serde = { version = "1.0.196", features = ["derive"] } +serde_json = "1.0.113" diff --git a/src/meta/model_v2/migration/Cargo.toml b/src/meta/model_v2/migration/Cargo.toml index 4745125140a2..b99efe2b7b0a 100644 --- a/src/meta/model_v2/migration/Cargo.toml +++ b/src/meta/model_v2/migration/Cargo.toml @@ -18,5 +18,5 @@ async-std = { version = "1", features = ["attributes", "tokio1"] } uuid = { version = "1", features = ["v4"] } [dependencies.sea-orm-migration] -version = "0.12.0" +version = "0.12.14" features = ["sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-native-tls", "with-uuid"] diff --git a/src/meta/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/model_v2/migration/src/m20230908_072257_init.rs index cf4bd9d37559..2da479531abf 100644 --- a/src/meta/model_v2/migration/src/m20230908_072257_init.rs +++ b/src/meta/model_v2/migration/src/m20230908_072257_init.rs @@ -361,7 +361,7 @@ impl MigrationTrait for Migration { .string() .not_null(), ) - .col(ColumnDef::new(Fragment::StreamNode).json().not_null()) + .col(ColumnDef::new(Fragment::StreamNode).binary().not_null()) .col(ColumnDef::new(Fragment::VnodeMapping).json().not_null()) .col(ColumnDef::new(Fragment::StateTableIds).json()) .col(ColumnDef::new(Fragment::UpstreamFragmentId).json()) diff --git a/src/meta/model_v2/src/fragment.rs b/src/meta/model_v2/src/fragment.rs index af1d529a0598..27a589806459 100644 --- a/src/meta/model_v2/src/fragment.rs +++ b/src/meta/model_v2/src/fragment.rs @@ -12,10 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::fmt::Formatter; + use risingwave_pb::meta::table_fragments::fragment::PbFragmentDistributionType; +use risingwave_pb::stream_plan::PbStreamNode; use sea_orm::entity::prelude::*; -use crate::{FragmentId, FragmentVnodeMapping, I32Array, ObjectId, StreamNode}; +use crate::{FragmentId, FragmentVnodeMapping, I32Array, ObjectId}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "fragment")] @@ -31,6 +34,34 @@ pub struct Model { pub upstream_fragment_id: I32Array, } +/// This is a workaround to avoid stack overflow when deserializing the `StreamNode` field from sql +/// backend if we store it as Json. We'd better fix it before using it in production, because it's less +/// readable and maintainable. +#[derive(Clone, PartialEq, Eq, DeriveValueType)] +pub struct StreamNode(#[sea_orm] Vec); + +impl StreamNode { + pub fn to_protobuf(&self) -> PbStreamNode { + prost::Message::decode(self.0.as_slice()).unwrap() + } + + pub fn from_protobuf(val: &PbStreamNode) -> Self { + Self(prost::Message::encode_to_vec(val)) + } +} + +impl std::fmt::Debug for StreamNode { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.to_protobuf().fmt(f) + } +} + +impl Default for StreamNode { + fn default() -> Self { + Self::from_protobuf(&PbStreamNode::default()) + } +} + #[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] #[sea_orm(rs_type = "String", db_type = "String(None)")] pub enum DistributionType { diff --git a/src/meta/model_v2/src/lib.rs b/src/meta/model_v2/src/lib.rs index 8c85cad2a659..2abd66fae3fe 100644 --- a/src/meta/model_v2/src/lib.rs +++ b/src/meta/model_v2/src/lib.rs @@ -216,8 +216,6 @@ derive_from_json_struct!( ); derive_from_json_struct!(AuthInfo, risingwave_pb::user::PbAuthInfo); -derive_from_json_struct!(StreamNode, risingwave_pb::stream_plan::PbStreamNode); - derive_from_json_struct!(ConnectorSplits, risingwave_pb::source::ConnectorSplits); derive_from_json_struct!(VnodeBitmap, risingwave_pb::common::Buffer); derive_from_json_struct!(ActorMapping, risingwave_pb::stream_plan::PbActorMapping); diff --git a/src/meta/model_v2/src/prelude.rs b/src/meta/model_v2/src/prelude.rs index 6a5316b0e422..d2d4e1362f93 100644 --- a/src/meta/model_v2/src/prelude.rs +++ b/src/meta/model_v2/src/prelude.rs @@ -25,6 +25,7 @@ pub use super::fragment::Entity as Fragment; pub use super::function::Entity as Function; pub use super::hummock_pinned_snapshot::Entity as HummockPinnedSnapshot; pub use super::hummock_pinned_version::Entity as HummockPinnedVersion; +pub use super::hummock_sequence::Entity as HummockSequence; pub use super::hummock_version_delta::Entity as HummockVersionDelta; pub use super::hummock_version_stats::Entity as HummockVersionStats; pub use super::index::Entity as Index; diff --git a/src/meta/node/Cargo.toml b/src/meta/node/Cargo.toml index a799c99b98c8..0538fa19280e 100644 --- a/src/meta/node/Cargo.toml +++ b/src/meta/node/Cargo.toml @@ -32,7 +32,7 @@ risingwave_meta_model_migration = { workspace = true } risingwave_meta_service = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } -sea-orm = { version = "0.12.0", features = [ +sea-orm = { version = "0.12.14", features = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", diff --git a/src/meta/node/src/server.rs b/src/meta/node/src/server.rs index bd61debc3bf7..59e520a3316f 100644 --- a/src/meta/node/src/server.rs +++ b/src/meta/node/src/server.rs @@ -35,6 +35,7 @@ use risingwave_meta::controller::cluster::ClusterController; use risingwave_meta::manager::MetadataManager; use risingwave_meta::rpc::intercept::MetricsMiddlewareLayer; use risingwave_meta::rpc::ElectionClientRef; +use risingwave_meta::stream::ScaleController; use risingwave_meta::MetaStoreBackend; use risingwave_meta_model_migration::{Migrator, MigratorTrait}; use risingwave_meta_service::backup_service::BackupServiceImpl; @@ -537,6 +538,13 @@ pub async fn start_service_as_election_leader( let stream_rpc_manager = StreamRpcManager::new(env.clone()); + let scale_controller = Arc::new(ScaleController::new( + &metadata_manager, + source_manager.clone(), + stream_rpc_manager.clone(), + env.clone(), + )); + let barrier_manager = GlobalBarrierManager::new( scheduled_barriers, env.clone(), @@ -546,6 +554,7 @@ pub async fn start_service_as_election_leader( sink_manager.clone(), meta_metrics.clone(), stream_rpc_manager.clone(), + scale_controller.clone(), ); { @@ -563,6 +572,7 @@ pub async fn start_service_as_election_leader( source_manager.clone(), hummock_manager.clone(), stream_rpc_manager, + scale_controller.clone(), ) .unwrap(), ); @@ -627,6 +637,7 @@ pub async fn start_service_as_election_leader( source_manager, stream_manager.clone(), barrier_manager.context().clone(), + scale_controller.clone(), ); let cluster_srv = ClusterServiceImpl::new(metadata_manager.clone()); diff --git a/src/meta/service/Cargo.toml b/src/meta/service/Cargo.toml index 2ba993d7eab5..a91e570ccd6a 100644 --- a/src/meta/service/Cargo.toml +++ b/src/meta/service/Cargo.toml @@ -27,7 +27,7 @@ risingwave_hummock_sdk = { workspace = true } risingwave_meta = { workspace = true } risingwave_meta_model_v2 = { workspace = true } risingwave_pb = { workspace = true } -sea-orm = { version = "0.12.0", features = [ +sea-orm = { version = "0.12.14", features = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", diff --git a/src/meta/service/src/scale_service.rs b/src/meta/service/src/scale_service.rs index 33270fc2204f..33899856a57b 100644 --- a/src/meta/service/src/scale_service.rs +++ b/src/meta/service/src/scale_service.rs @@ -13,12 +13,11 @@ // limitations under the License. use std::collections::HashMap; -use std::sync::Arc; use risingwave_common::catalog; use risingwave_meta::manager::MetadataManager; use risingwave_meta::model::TableParallelism; -use risingwave_meta::stream::{ScaleController, ScaleControllerRef, TableRevision}; +use risingwave_meta::stream::{ScaleControllerRef, TableRevision}; use risingwave_meta_model_v2::FragmentId; use risingwave_pb::common::WorkerType; use risingwave_pb::meta::scale_service_server::ScaleService; @@ -49,14 +48,8 @@ impl ScaleServiceImpl { source_manager: SourceManagerRef, stream_manager: GlobalStreamManagerRef, barrier_manager: BarrierManagerRef, + scale_controller: ScaleControllerRef, ) -> Self { - let scale_controller = Arc::new(ScaleController::new( - &metadata_manager, - source_manager.clone(), - stream_manager.stream_rpc_manager.clone(), - stream_manager.env.clone(), - )); - Self { metadata_manager, source_manager, @@ -82,7 +75,7 @@ impl ScaleService for ScaleServiceImpl { &self, _: Request, ) -> Result, Status> { - let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; + let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await; let table_fragments = match &self.metadata_manager { MetadataManager::V1(mgr) => mgr @@ -149,7 +142,7 @@ impl ScaleService for ScaleServiceImpl { resolve_no_shuffle_upstream, } = request.into_inner(); - let _reschedule_job_lock = self.stream_manager.reschedule_lock.write().await; + let _reschedule_job_lock = self.stream_manager.reschedule_lock_write_guard().await; let current_revision = self.get_revision().await; @@ -239,7 +232,7 @@ impl ScaleService for ScaleServiceImpl { let req = request.into_inner(); - let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; + let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await; let current_revision = self.get_revision().await; diff --git a/src/meta/src/barrier/info.rs b/src/meta/src/barrier/info.rs index b52ab9095528..7e845be71d98 100644 --- a/src/meta/src/barrier/info.rs +++ b/src/meta/src/barrier/info.rs @@ -15,6 +15,7 @@ use std::collections::{HashMap, HashSet}; use risingwave_pb::common::PbWorkerNode; +use tracing::warn; use crate::manager::{ActorInfos, WorkerId}; use crate::model::ActorId; @@ -87,10 +88,16 @@ impl InflightActorInfo { /// Update worker nodes snapshot. We need to support incremental updates for it in the future. pub fn resolve_worker_nodes(&mut self, all_nodes: impl IntoIterator) { - self.node_map = all_nodes + let new_node_map = all_nodes .into_iter() .map(|node| (node.id, node)) .collect::>(); + for (actor_id, location) in &self.actor_location_map { + if !new_node_map.contains_key(location) { + warn!(actor_id, location, node = ?self.node_map.get(location), "node with running actors is deleted"); + } + } + self.node_map = new_node_map; } /// Apply some actor changes before issuing a barrier command, if the command contains any new added actors, we should update diff --git a/src/meta/src/barrier/mod.rs b/src/meta/src/barrier/mod.rs index 9de6df91fed3..3bad3cbd15a2 100644 --- a/src/meta/src/barrier/mod.rs +++ b/src/meta/src/barrier/mod.rs @@ -32,6 +32,7 @@ use risingwave_hummock_sdk::table_watermark::{ }; use risingwave_hummock_sdk::{ExtendedSstableInfo, HummockSstableObjectId}; use risingwave_pb::catalog::table::TableType; +use risingwave_pb::common::WorkerNode; use risingwave_pb::ddl_service::DdlProgress; use risingwave_pb::meta::subscribe_response::{Info, Operation}; use risingwave_pb::meta::PausedReason; @@ -41,7 +42,7 @@ use thiserror_ext::AsReport; use tokio::sync::oneshot::{Receiver, Sender}; use tokio::sync::Mutex; use tokio::task::JoinHandle; -use tracing::Instrument; +use tracing::{info, warn, Instrument}; use self::command::CommandContext; use self::notifier::Notifier; @@ -54,10 +55,12 @@ use crate::barrier::state::BarrierManagerState; use crate::barrier::BarrierEpochState::{Completed, InFlight}; use crate::hummock::{CommitEpochInfo, HummockManagerRef}; use crate::manager::sink_coordination::SinkCoordinatorManager; -use crate::manager::{LocalNotification, MetaSrvEnv, MetadataManager, WorkerId}; +use crate::manager::{ + ActiveStreamingWorkerNodes, LocalNotification, MetaSrvEnv, MetadataManager, WorkerId, +}; use crate::model::{ActorId, TableFragments}; use crate::rpc::metrics::MetaMetrics; -use crate::stream::{ScaleController, ScaleControllerRef, SourceManagerRef}; +use crate::stream::{ScaleControllerRef, SourceManagerRef}; use crate::{MetaError, MetaResult}; mod command; @@ -183,6 +186,8 @@ pub struct GlobalBarrierManager { checkpoint_control: CheckpointControl, rpc_manager: BarrierRpcManager, + + active_streaming_nodes: ActiveStreamingWorkerNodes, } /// Controls the concurrent execution of commands. @@ -385,6 +390,7 @@ impl GlobalBarrierManager { sink_manager: SinkCoordinatorManager, metrics: Arc, stream_rpc_manager: StreamRpcManager, + scale_controller: ScaleControllerRef, ) -> Self { let enable_recovery = env.opts.enable_recovery; let in_flight_barrier_nums = env.opts.in_flight_barrier_nums; @@ -396,14 +402,9 @@ impl GlobalBarrierManager { ); let checkpoint_control = CheckpointControl::new(metrics.clone()); - let tracker = CreateMviewProgressTracker::new(); + let active_streaming_nodes = ActiveStreamingWorkerNodes::uninitialized(); - let scale_controller = Arc::new(ScaleController::new( - &metadata_manager, - source_manager.clone(), - stream_rpc_manager.clone(), - env.clone(), - )); + let tracker = CreateMviewProgressTracker::new(); let context = GlobalBarrierManagerContext { status: Arc::new(Mutex::new(BarrierManagerStatus::Starting)), @@ -429,6 +430,7 @@ impl GlobalBarrierManager { state: initial_invalid_state, checkpoint_control, rpc_manager, + active_streaming_nodes, } } @@ -453,7 +455,7 @@ impl GlobalBarrierManager { .await .pause_on_next_bootstrap(); if paused { - tracing::warn!( + warn!( "The cluster will bootstrap with all data sources paused as specified by the system parameter `{}`. \ It will now be reset to `false`. \ To resume the data sources, either restart the cluster again or use `risectl meta resume`.", @@ -504,7 +506,7 @@ impl GlobalBarrierManager { } } - self.state = { + { let latest_snapshot = self.context.hummock_manager.latest_snapshot(); assert_eq!( latest_snapshot.committed_epoch, latest_snapshot.current_epoch, @@ -524,11 +526,10 @@ impl GlobalBarrierManager { let paused = self.take_pause_on_bootstrap().await.unwrap_or(false); let paused_reason = paused.then_some(PausedReason::Manual); - self.context - .recovery(prev_epoch, paused_reason, &self.scheduled_barriers) + self.recovery(prev_epoch, paused_reason) .instrument(span) - .await - }; + .await; + } self.context.set_status(BarrierManagerStatus::Running).await; @@ -551,6 +552,59 @@ impl GlobalBarrierManager { tracing::info!("Barrier manager is stopped"); break; } + + changed_worker = self.active_streaming_nodes.changed() => { + #[cfg(debug_assertions)] + { + match self + .context + .metadata_manager + .list_active_streaming_compute_nodes() + .await + { + Ok(worker_nodes) => { + let ignore_irrelevant_info = |node: &WorkerNode| { + ( + node.id, + WorkerNode { + id: node.id, + r#type: node.r#type, + host: node.host.clone(), + parallel_units: node.parallel_units.clone(), + property: node.property.clone(), + resource: node.resource.clone(), + ..Default::default() + }, + ) + }; + let worker_nodes: HashMap<_, _> = + worker_nodes.iter().map(ignore_irrelevant_info).collect(); + let curr_worker_nodes: HashMap<_, _> = self + .active_streaming_nodes + .current() + .values() + .map(ignore_irrelevant_info) + .collect(); + if worker_nodes != curr_worker_nodes { + warn!( + ?worker_nodes, + ?curr_worker_nodes, + "different to global snapshot" + ); + } + } + Err(e) => { + warn!(e = ?e.as_report(), "fail to list_active_streaming_compute_nodes to compare with local snapshot"); + } + } + } + + info!(?changed_worker, "worker changed"); + + self.state + .resolve_worker_nodes(self.active_streaming_nodes.current().values().cloned()); + } + // Checkpoint frequency changes. notification = local_notification_rx.recv() => { let notification = notification.unwrap(); @@ -601,13 +655,6 @@ impl GlobalBarrierManager { span, } = self.scheduled_barriers.pop_or_default().await; - let all_nodes = self - .context - .metadata_manager - .list_active_streaming_compute_nodes() - .await - .unwrap(); - self.state.resolve_worker_nodes(all_nodes); let info = self.state.apply_command(&command); let (prev_epoch, curr_epoch) = self.state.next_epoch_pair(); @@ -674,7 +721,7 @@ impl GlobalBarrierManager { // back to frontend fail_point!("inject_barrier_err_success"); let fail_node = self.checkpoint_control.barrier_failed(); - tracing::warn!(%prev_epoch, error = %err.as_report(), "Failed to complete epoch"); + warn!(%prev_epoch, error = %err.as_report(), "Failed to complete epoch"); self.failure_recovery(err, fail_node).await; return; } @@ -699,7 +746,7 @@ impl GlobalBarrierManager { .drain(index..) .chain(self.checkpoint_control.barrier_failed().into_iter()) .collect_vec(); - tracing::warn!(%prev_epoch, error = %err.as_report(), "Failed to commit epoch"); + warn!(%prev_epoch, error = %err.as_report(), "Failed to commit epoch"); self.failure_recovery(err, fail_nodes).await; } } @@ -740,11 +787,7 @@ impl GlobalBarrierManager { // No need to clean dirty tables for barrier recovery, // The foreground stream job should cleanup their own tables. - self.state = self - .context - .recovery(prev_epoch, None, &self.scheduled_barriers) - .instrument(span) - .await; + self.recovery(prev_epoch, None).instrument(span).await; self.context.set_status(BarrierManagerStatus::Running).await; } else { panic!("failed to execute barrier: {}", err.as_report()); @@ -920,23 +963,17 @@ impl GlobalBarrierManagerContext { /// Resolve actor information from cluster, fragment manager and `ChangedTableId`. /// We use `changed_table_id` to modify the actors to be sent or collected. Because these actor /// will create or drop before this barrier flow through them. - async fn resolve_actor_info(&self) -> MetaResult { + async fn resolve_actor_info( + &self, + all_nodes: Vec, + ) -> MetaResult { let info = match &self.metadata_manager { MetadataManager::V1(mgr) => { - let all_nodes = mgr - .cluster_manager - .list_active_streaming_compute_nodes() - .await; let all_actor_infos = mgr.fragment_manager.load_all_actors().await; InflightActorInfo::resolve(all_nodes, all_actor_infos) } MetadataManager::V2(mgr) => { - let all_nodes = mgr - .cluster_controller - .list_active_streaming_workers() - .await - .unwrap(); let all_actor_infos = mgr.catalog_controller.load_all_actors().await?; InflightActorInfo::resolve(all_nodes, all_actor_infos) diff --git a/src/meta/src/barrier/progress.rs b/src/meta/src/barrier/progress.rs index 0c753a3c3f02..f22c5a2bbb21 100644 --- a/src/meta/src/barrier/progress.rs +++ b/src/meta/src/barrier/progress.rs @@ -38,7 +38,7 @@ type ConsumedRows = u64; #[derive(Clone, Copy, Debug)] enum BackfillState { Init, - ConsumingUpstream(Epoch, ConsumedRows), + ConsumingUpstream(#[allow(dead_code)] Epoch, ConsumedRows), Done(ConsumedRows), } diff --git a/src/meta/src/barrier/recovery.rs b/src/meta/src/barrier/recovery.rs index e4b93a286a3f..858fa937044d 100644 --- a/src/meta/src/barrier/recovery.rs +++ b/src/meta/src/barrier/recovery.rs @@ -21,7 +21,7 @@ use itertools::Itertools; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::compaction_group::StateTableId; use risingwave_meta_model_v2::StreamingParallelism; -use risingwave_pb::common::ActorInfo; +use risingwave_pb::common::{ActorInfo, WorkerNode}; use risingwave_pb::meta::table_fragments::State; use risingwave_pb::meta::PausedReason; use risingwave_pb::stream_plan::barrier::BarrierKind; @@ -37,16 +37,15 @@ use crate::barrier::command::CommandContext; use crate::barrier::info::InflightActorInfo; use crate::barrier::notifier::Notifier; use crate::barrier::progress::CreateMviewProgressTracker; -use crate::barrier::schedule::ScheduledBarriers; use crate::barrier::state::BarrierManagerState; -use crate::barrier::{Command, GlobalBarrierManagerContext}; +use crate::barrier::{Command, GlobalBarrierManager, GlobalBarrierManagerContext}; use crate::controller::catalog::ReleaseContext; -use crate::manager::{MetadataManager, WorkerId}; +use crate::manager::{ActiveStreamingWorkerNodes, MetadataManager, WorkerId}; use crate::model::{MetadataModel, MigrationPlan, TableFragments, TableParallelism}; use crate::stream::{build_actor_connector_splits, RescheduleOptions, TableResizePolicy}; use crate::MetaResult; -impl GlobalBarrierManagerContext { +impl GlobalBarrierManager { // Retry base interval in milliseconds. const RECOVERY_RETRY_BASE_INTERVAL: u64 = 20; // Retry max interval. @@ -59,7 +58,9 @@ impl GlobalBarrierManagerContext { .max_delay(Self::RECOVERY_RETRY_MAX_INTERVAL) .map(jitter) } +} +impl GlobalBarrierManagerContext { /// Clean catalogs for creating streaming jobs that are in foreground mode or table fragments not persisted. async fn clean_dirty_streaming_jobs(&self) -> MetaResult<()> { match &self.metadata_manager { @@ -295,17 +296,18 @@ impl GlobalBarrierManagerContext { Ok(()) } +} +impl GlobalBarrierManager { /// Pre buffered drop and cancel command, return true if any. - async fn pre_apply_drop_cancel( - &self, - scheduled_barriers: &ScheduledBarriers, - ) -> MetaResult { - let (dropped_actors, cancelled) = - scheduled_barriers.pre_apply_drop_cancel_scheduled().await; + async fn pre_apply_drop_cancel(&self) -> MetaResult { + let (dropped_actors, cancelled) = self + .scheduled_barriers + .pre_apply_drop_cancel_scheduled() + .await; let applied = !dropped_actors.is_empty() || !cancelled.is_empty(); if !cancelled.is_empty() { - match &self.metadata_manager { + match &self.context.metadata_manager { MetadataManager::V1(mgr) => { mgr.fragment_manager .drop_table_fragments_vec(&cancelled) @@ -330,57 +332,74 @@ impl GlobalBarrierManagerContext { /// the cluster or `risectl` command. Used for debugging purpose. /// /// Returns the new state of the barrier manager after recovery. - pub async fn recovery( - &self, - prev_epoch: TracedEpoch, - paused_reason: Option, - scheduled_barriers: &ScheduledBarriers, - ) -> BarrierManagerState { + pub async fn recovery(&mut self, prev_epoch: TracedEpoch, paused_reason: Option) { // Mark blocked and abort buffered schedules, they might be dirty already. - scheduled_barriers + self.scheduled_barriers .abort_and_mark_blocked("cluster is under recovering") .await; tracing::info!("recovery start!"); - self.clean_dirty_streaming_jobs() + self.context + .clean_dirty_streaming_jobs() .await .expect("clean dirty streaming jobs"); - self.sink_manager.reset().await; + self.context.sink_manager.reset().await; let retry_strategy = Self::get_retry_strategy(); // Mview progress needs to be recovered. tracing::info!("recovering mview progress"); - self.recover_background_mv_progress() + self.context + .recover_background_mv_progress() .await .expect("recover mview progress should not fail"); tracing::info!("recovered mview progress"); // We take retry into consideration because this is the latency user sees for a cluster to // get recovered. - let recovery_timer = self.metrics.recovery_latency.start_timer(); + let recovery_timer = self.context.metrics.recovery_latency.start_timer(); - let state = tokio_retry::Retry::spawn(retry_strategy, || { + let (state, active_streaming_nodes) = tokio_retry::Retry::spawn(retry_strategy, || { async { let recovery_result: MetaResult<_> = try { // This is a quick path to accelerate the process of dropping and canceling streaming jobs. - let _ = self.pre_apply_drop_cancel(scheduled_barriers).await?; + let _ = self.pre_apply_drop_cancel().await?; + + let active_streaming_nodes = ActiveStreamingWorkerNodes::new_snapshot( + self.context.metadata_manager.clone(), + ) + .await?; + + let all_nodes = active_streaming_nodes + .current() + .values() + .cloned() + .collect_vec(); // Resolve actor info for recovery. If there's no actor to recover, most of the // following steps will be no-op, while the compute nodes will still be reset. let mut info = if !self.env.opts.disable_automatic_parallelism_control { - self.scale_actors().await.inspect_err(|err| { - warn!(error = %err.as_report(), "scale actors failed"); - })?; - - self.resolve_actor_info().await.inspect_err(|err| { - warn!(error = %err.as_report(), "resolve actor info failed"); - })? + self.context + .scale_actors(all_nodes.clone()) + .await + .inspect_err(|err| { + warn!(error = %err.as_report(), "scale actors failed"); + })?; + + self.context + .resolve_actor_info(all_nodes.clone()) + .await + .inspect_err(|err| { + warn!(error = %err.as_report(), "resolve actor info failed"); + })? } else { // Migrate actors in expired CN to newly joined one. - self.migrate_actors().await.inspect_err(|err| { - warn!(error = %err.as_report(), "migrate actors failed"); - })? + self.context + .migrate_actors(all_nodes.clone()) + .await + .inspect_err(|err| { + warn!(error = %err.as_report(), "migrate actors failed"); + })? }; // Reset all compute nodes, stop and drop existing actors. @@ -388,10 +407,14 @@ impl GlobalBarrierManagerContext { warn!(error = %err.as_report(), "reset compute nodes failed"); })?; - if self.pre_apply_drop_cancel(scheduled_barriers).await? { - info = self.resolve_actor_info().await.inspect_err(|err| { - warn!(error = %err.as_report(), "resolve actor info failed"); - })?; + if self.pre_apply_drop_cancel().await? { + info = self + .context + .resolve_actor_info(all_nodes.clone()) + .await + .inspect_err(|err| { + warn!(error = %err.as_report(), "resolve actor info failed"); + })? } // update and build all actors. @@ -403,7 +426,8 @@ impl GlobalBarrierManagerContext { })?; // get split assignments for all actors - let source_split_assignments = self.source_manager.list_assignments().await; + let source_split_assignments = + self.context.source_manager.list_assignments().await; let command = Command::Plain(Some(Mutation::Add(AddMutation { // Actors built during recovery is not treated as newly added actors. actor_dispatchers: Default::default(), @@ -423,7 +447,7 @@ impl GlobalBarrierManagerContext { paused_reason, command, BarrierKind::Initial, - self.clone(), + self.context.clone(), tracing::Span::current(), // recovery span )); @@ -431,13 +455,18 @@ impl GlobalBarrierManagerContext { { use risingwave_common::util::epoch::INVALID_EPOCH; - let mce = self.hummock_manager.get_current_max_committed_epoch().await; + let mce = self + .context + .hummock_manager + .get_current_max_committed_epoch() + .await; if mce != INVALID_EPOCH { command_ctx.wait_epoch_commit(mce).await?; } }; - let await_barrier_complete = self.inject_barrier(command_ctx.clone()).await; + let await_barrier_complete = + self.context.inject_barrier(command_ctx.clone()).await; let res = match await_barrier_complete.await.result { Ok(response) => { if let Err(err) = command_ctx.post_collect().await { @@ -454,10 +483,13 @@ impl GlobalBarrierManagerContext { }; let (new_epoch, _) = res?; - BarrierManagerState::new(new_epoch, info, command_ctx.next_paused_reason()) + ( + BarrierManagerState::new(new_epoch, info, command_ctx.next_paused_reason()), + active_streaming_nodes, + ) }; if recovery_result.is_err() { - self.metrics.recovery_failure_cnt.inc(); + self.context.metrics.recovery_failure_cnt.inc(); } recovery_result } @@ -467,7 +499,7 @@ impl GlobalBarrierManagerContext { .expect("Retry until recovery success."); recovery_timer.observe_duration(); - scheduled_barriers.mark_ready().await; + self.scheduled_barriers.mark_ready().await; tracing::info!( epoch = state.in_flight_prev_epoch().value().0, @@ -475,18 +507,21 @@ impl GlobalBarrierManagerContext { "recovery success" ); - state + self.state = state; + self.active_streaming_nodes = active_streaming_nodes; } +} +impl GlobalBarrierManagerContext { /// Migrate actors in expired CNs to newly joined ones, return true if any actor is migrated. - async fn migrate_actors(&self) -> MetaResult { + async fn migrate_actors(&self, all_nodes: Vec) -> MetaResult { match &self.metadata_manager { - MetadataManager::V1(_) => self.migrate_actors_v1().await, - MetadataManager::V2(_) => self.migrate_actors_v2().await, + MetadataManager::V1(_) => self.migrate_actors_v1(all_nodes).await, + MetadataManager::V2(_) => self.migrate_actors_v2(all_nodes).await, } } - async fn migrate_actors_v2(&self) -> MetaResult { + async fn migrate_actors_v2(&self, all_nodes: Vec) -> MetaResult { let mgr = self.metadata_manager.as_v2_ref(); let all_inuse_parallel_units: HashSet<_> = mgr @@ -495,12 +530,10 @@ impl GlobalBarrierManagerContext { .await? .into_iter() .collect(); - let active_parallel_units: HashSet<_> = mgr - .cluster_controller - .list_active_parallel_units() - .await? - .into_iter() - .map(|pu| pu.id as i32) + + let active_parallel_units: HashSet<_> = all_nodes + .iter() + .flat_map(|node| node.parallel_units.iter().map(|pu| pu.id as i32)) .collect(); let expired_parallel_units: BTreeSet<_> = all_inuse_parallel_units @@ -509,7 +542,7 @@ impl GlobalBarrierManagerContext { .collect(); if expired_parallel_units.is_empty() { debug!("no expired parallel units, skipping."); - return self.resolve_actor_info().await; + return self.resolve_actor_info(all_nodes.clone()).await; } debug!("start migrate actors."); @@ -526,12 +559,14 @@ impl GlobalBarrierManagerContext { let start = Instant::now(); let mut plan = HashMap::new(); 'discovery: while !to_migrate_parallel_units.is_empty() { - let new_parallel_units = mgr - .cluster_controller - .list_active_parallel_units() - .await? - .into_iter() - .filter(|pu| !inuse_parallel_units.contains(&(pu.id as i32))) + let new_parallel_units = all_nodes + .iter() + .flat_map(|node| { + node.parallel_units + .iter() + .filter(|pu| !inuse_parallel_units.contains(&(pu.id as _))) + }) + .cloned() .collect_vec(); if !new_parallel_units.is_empty() { debug!("new parallel units found: {:#?}", new_parallel_units); @@ -560,14 +595,14 @@ impl GlobalBarrierManagerContext { debug!("migrate actors succeed."); - self.resolve_actor_info().await + self.resolve_actor_info(all_nodes).await } /// Migrate actors in expired CNs to newly joined ones, return true if any actor is migrated. - async fn migrate_actors_v1(&self) -> MetaResult { + async fn migrate_actors_v1(&self, all_nodes: Vec) -> MetaResult { let mgr = self.metadata_manager.as_v1_ref(); - let info = self.resolve_actor_info().await?; + let info = self.resolve_actor_info(all_nodes.clone()).await?; // 1. get expired workers. let expired_workers: HashSet = info @@ -582,7 +617,9 @@ impl GlobalBarrierManagerContext { } debug!("start migrate actors."); - let migration_plan = self.generate_migration_plan(expired_workers).await?; + let migration_plan = self + .generate_migration_plan(expired_workers, &all_nodes) + .await?; // 2. start to migrate fragment one-by-one. mgr.fragment_manager .migrate_fragment_actors(&migration_plan) @@ -591,17 +628,18 @@ impl GlobalBarrierManagerContext { migration_plan.delete(self.env.meta_store_checked()).await?; debug!("migrate actors succeed."); - self.resolve_actor_info().await + self.resolve_actor_info(all_nodes).await } - async fn scale_actors(&self) -> MetaResult<()> { + async fn scale_actors(&self, all_nodes: Vec) -> MetaResult<()> { + let _guard = self.scale_controller.reschedule_lock.write().await; match &self.metadata_manager { - MetadataManager::V1(_) => self.scale_actors_v1().await, - MetadataManager::V2(_) => self.scale_actors_v2().await, + MetadataManager::V1(_) => self.scale_actors_v1(all_nodes).await, + MetadataManager::V2(_) => self.scale_actors_v2(all_nodes).await, } } - async fn scale_actors_v2(&self) -> MetaResult<()> { + async fn scale_actors_v2(&self, workers: Vec) -> MetaResult<()> { let mgr = self.metadata_manager.as_v2_ref(); debug!("start resetting actors distribution"); @@ -625,11 +663,6 @@ impl GlobalBarrierManagerContext { .collect() }; - let workers = mgr - .cluster_controller - .list_active_streaming_workers() - .await?; - let schedulable_worker_ids = workers .iter() .filter(|worker| { @@ -698,8 +731,8 @@ impl GlobalBarrierManagerContext { Ok(()) } - async fn scale_actors_v1(&self) -> MetaResult<()> { - let info = self.resolve_actor_info().await?; + async fn scale_actors_v1(&self, workers: Vec) -> MetaResult<()> { + let info = self.resolve_actor_info(workers.clone()).await?; let mgr = self.metadata_manager.as_v1_ref(); debug!("start resetting actors distribution"); @@ -762,11 +795,6 @@ impl GlobalBarrierManagerContext { .collect() }; - let workers = mgr - .cluster_manager - .list_active_streaming_compute_nodes() - .await; - let schedulable_worker_ids = workers .iter() .filter(|worker| { @@ -841,6 +869,7 @@ impl GlobalBarrierManagerContext { async fn generate_migration_plan( &self, expired_workers: HashSet, + all_nodes: &Vec, ) -> MetaResult { let mgr = self.metadata_manager.as_v1_ref(); @@ -890,10 +919,10 @@ impl GlobalBarrierManagerContext { let start = Instant::now(); // if in-used expire parallel units are not empty, should wait for newly joined worker. 'discovery: while !to_migrate_parallel_units.is_empty() { - let mut new_parallel_units = mgr - .cluster_manager - .list_active_streaming_parallel_units() - .await; + let mut new_parallel_units = all_nodes + .iter() + .flat_map(|worker| worker.parallel_units.iter().cloned()) + .collect_vec(); new_parallel_units.retain(|pu| !inuse_parallel_units.contains(&pu.id)); if !new_parallel_units.is_empty() { @@ -944,7 +973,9 @@ impl GlobalBarrierManagerContext { new_plan.insert(self.env.meta_store_checked()).await?; Ok(new_plan) } +} +impl GlobalBarrierManager { /// Update all actors in compute nodes. async fn update_actors(&self, info: &InflightActorInfo) -> MetaResult<()> { if info.actor_map.is_empty() { @@ -970,7 +1001,7 @@ impl GlobalBarrierManagerContext { .flatten_ok() .try_collect()?; - let mut all_node_actors = self.metadata_manager.all_node_actors(false).await?; + let mut all_node_actors = self.context.metadata_manager.all_node_actors(false).await?; // Check if any actors were dropped after info resolved. if all_node_actors.iter().any(|(node_id, node_actors)| { @@ -984,7 +1015,8 @@ impl GlobalBarrierManagerContext { return Err(anyhow!("actors dropped during update").into()); } - self.stream_rpc_manager + self.context + .stream_rpc_manager .broadcast_update_actor_info( &info.node_map, info.actor_map.keys().cloned(), @@ -1008,7 +1040,8 @@ impl GlobalBarrierManagerContext { return Ok(()); } - self.stream_rpc_manager + self.context + .stream_rpc_manager .build_actors( &info.node_map, info.actor_map.iter().map(|(node_id, actors)| { @@ -1024,7 +1057,8 @@ impl GlobalBarrierManagerContext { /// Reset all compute nodes by calling `force_stop_actors`. async fn reset_compute_nodes(&self, info: &InflightActorInfo) -> MetaResult<()> { debug!(worker = ?info.node_map.keys().collect_vec(), "force stop actors"); - self.stream_rpc_manager + self.context + .stream_rpc_manager .force_stop_actors(info.node_map.values()) .await?; diff --git a/src/meta/src/barrier/state.rs b/src/meta/src/barrier/state.rs index 560f17118c58..f24bbb563722 100644 --- a/src/meta/src/barrier/state.rs +++ b/src/meta/src/barrier/state.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_pb::common::PbWorkerNode; +use risingwave_pb::common::WorkerNode; use risingwave_pb::meta::PausedReason; use crate::barrier::info::InflightActorInfo; @@ -69,8 +69,7 @@ impl BarrierManagerState { (prev_epoch, next_epoch) } - // TODO: optimize it as incremental updates. - pub fn resolve_worker_nodes(&mut self, nodes: Vec) { + pub fn resolve_worker_nodes(&mut self, nodes: impl IntoIterator) { self.inflight_actor_infos.resolve_worker_nodes(nodes); } diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index 2ca8e9b98be8..76eab54f631a 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -656,7 +656,7 @@ impl CatalogController { .await?; pb_source.id = source_obj.oid as _; let source: source::ActiveModel = pb_source.clone().into(); - source.insert(&txn).await?; + Source::insert(source).exec(&txn).await?; if let Some(src_manager) = source_manager_ref { let ret = src_manager.register_source(&pb_source).await; @@ -698,7 +698,7 @@ impl CatalogController { .await?; pb_function.id = function_obj.oid as _; let function: function::ActiveModel = pb_function.clone().into(); - function.insert(&txn).await?; + Function::insert(function).exec(&txn).await?; txn.commit().await?; let version = self @@ -774,7 +774,7 @@ impl CatalogController { .await?; pb_connection.id = conn_obj.oid as _; let connection: connection::ActiveModel = pb_connection.clone().into(); - connection.insert(&txn).await?; + Connection::insert(connection).exec(&txn).await?; txn.commit().await?; @@ -869,17 +869,17 @@ impl CatalogController { .await?; pb_view.id = view_obj.oid as _; let view: view::ActiveModel = pb_view.clone().into(); - view.insert(&txn).await?; + View::insert(view).exec(&txn).await?; // todo: change `dependent_relations` to `dependent_objects`, which should includes connection and function as well. // todo: shall we need to check existence of them Or let database handle it by FOREIGN KEY constraint. for obj_id in &pb_view.dependent_relations { - object_dependency::ActiveModel { + ObjectDependency::insert(object_dependency::ActiveModel { oid: Set(*obj_id as _), used_by: Set(view_obj.oid), ..Default::default() - } - .insert(&txn) + }) + .exec(&txn) .await?; } diff --git a/src/meta/src/controller/cluster.rs b/src/meta/src/controller/cluster.rs index 622a3efa6b56..2a03f063befe 100644 --- a/src/meta/src/controller/cluster.rs +++ b/src/meta/src/controller/cluster.rs @@ -31,6 +31,7 @@ use risingwave_meta_model_v2::{worker, worker_property, I32Array, TransactionId, use risingwave_pb::common::worker_node::{PbProperty, PbResource, PbState}; use risingwave_pb::common::{ HostAddress, ParallelUnit, PbHostAddress, PbParallelUnit, PbWorkerNode, PbWorkerType, + WorkerNode, }; use risingwave_pb::meta::add_worker_node_request::Property as AddNodeProperty; use risingwave_pb::meta::heartbeat_request; @@ -43,6 +44,7 @@ use sea_orm::{ TransactionTrait, }; use thiserror_ext::AsReport; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; use tokio::sync::oneshot::Sender; use tokio::sync::{RwLock, RwLockReadGuard}; use tokio::task::JoinHandle; @@ -338,6 +340,22 @@ impl ClusterController { Ok(workers) } + pub(crate) async fn subscribe_active_streaming_compute_nodes( + &self, + ) -> MetaResult<(Vec, UnboundedReceiver)> { + let inner = self.inner.read().await; + let worker_nodes = inner.list_active_streaming_workers().await?; + let (tx, rx) = unbounded_channel(); + + // insert before release the read lock to ensure that we don't lose any update in between + self.env + .notification_manager() + .insert_local_sender(tx) + .await; + drop(inner); + Ok((worker_nodes, rx)) + } + /// A convenient method to get all running compute nodes that may have running actors on them /// i.e. CNs which are running pub async fn list_active_streaming_workers(&self) -> MetaResult> { diff --git a/src/meta/src/controller/fragment.rs b/src/meta/src/controller/fragment.rs index 4f7fb469aec9..833e642a83e7 100644 --- a/src/meta/src/controller/fragment.rs +++ b/src/meta/src/controller/fragment.rs @@ -20,11 +20,12 @@ use itertools::Itertools; use risingwave_common::bail; use risingwave_common::util::stream_graph_visitor::visit_stream_node; use risingwave_meta_model_v2::actor::ActorStatus; +use risingwave_meta_model_v2::fragment::StreamNode; use risingwave_meta_model_v2::prelude::{Actor, ActorDispatcher, Fragment, Sink, StreamingJob}; use risingwave_meta_model_v2::{ actor, actor_dispatcher, fragment, object, sink, streaming_job, ActorId, ConnectorSplits, ExprContext, FragmentId, FragmentVnodeMapping, I32Array, JobStatus, ObjectId, SinkId, SourceId, - StreamNode, StreamingParallelism, TableId, VnodeBitmap, WorkerId, + StreamingParallelism, TableId, VnodeBitmap, WorkerId, }; use risingwave_pb::common::PbParallelUnit; use risingwave_pb::meta::subscribe_response::{ @@ -281,7 +282,7 @@ impl CatalogController { let vnode_mapping = pb_vnode_mapping.map(FragmentVnodeMapping).unwrap(); - let stream_node = StreamNode(stream_node); + let stream_node = StreamNode::from_protobuf(&stream_node); let distribution_type = PbFragmentDistributionType::try_from(pb_distribution_type) .unwrap() @@ -367,7 +368,7 @@ impl CatalogController { upstream_fragment_id, } = fragment; - let stream_node_template = stream_node.into_inner(); + let stream_node_template = stream_node.to_protobuf(); let mut pb_actors = vec![]; @@ -1186,7 +1187,7 @@ impl CatalogController { let mut source_fragment_ids = HashMap::new(); for (fragment_id, _, stream_node) in fragments { - if let Some(source) = find_stream_source(stream_node.inner_ref()) { + if let Some(source) = find_stream_source(&stream_node.to_protobuf()) { source_fragment_ids .entry(source.source_id as SourceId) .or_insert_with(BTreeSet::new) @@ -1216,7 +1217,7 @@ impl CatalogController { let mut source_fragment_ids = HashMap::new(); for (fragment_id, _, stream_node) in fragments { - if let Some(source) = find_stream_source(stream_node.inner_ref()) { + if let Some(source) = find_stream_source(&stream_node.to_protobuf()) { source_fragment_ids .entry(source.source_id as SourceId) .or_insert_with(BTreeSet::new) @@ -1278,11 +1279,10 @@ mod tests { use risingwave_common::util::iter_util::ZipEqDebug; use risingwave_common::util::stream_graph_visitor::visit_stream_node; use risingwave_meta_model_v2::actor::ActorStatus; - use risingwave_meta_model_v2::fragment::DistributionType; + use risingwave_meta_model_v2::fragment::{DistributionType, StreamNode}; use risingwave_meta_model_v2::{ actor, actor_dispatcher, fragment, ActorId, ActorUpstreamActors, ConnectorSplits, - ExprContext, FragmentId, FragmentVnodeMapping, I32Array, ObjectId, StreamNode, TableId, - VnodeBitmap, + ExprContext, FragmentId, FragmentVnodeMapping, I32Array, ObjectId, TableId, VnodeBitmap, }; use risingwave_pb::common::ParallelUnit; use risingwave_pb::meta::table_fragments::actor_status::PbActorState; @@ -1448,13 +1448,13 @@ mod tests { actors: Vec, upstream_actor_ids: &HashMap>>, ) { - let stream_node_template = fragment.stream_node.clone(); + let stream_node_template = fragment.stream_node.to_protobuf(); for PbStreamActor { nodes, actor_id, .. } in actors { - let mut template_node = stream_node_template.clone().into_inner(); + let mut template_node = stream_node_template.clone(); let nodes = nodes.unwrap(); let actor_upstream_actor_ids = upstream_actor_ids.get(&(actor_id as _)).cloned().unwrap(); @@ -1553,7 +1553,7 @@ mod tests { job_id: TEST_JOB_ID, fragment_type_mask: 0, distribution_type: DistributionType::Hash, - stream_node: StreamNode(stream_node), + stream_node: StreamNode::from_protobuf(&stream_node), vnode_mapping: FragmentVnodeMapping(parallel_unit_mapping.to_protobuf()), state_table_ids: I32Array(vec![TEST_STATE_TABLE_ID]), upstream_fragment_id: I32Array::default(), diff --git a/src/meta/src/controller/streaming_job.rs b/src/meta/src/controller/streaming_job.rs index 8afc5ab6c836..f4004e25b500 100644 --- a/src/meta/src/controller/streaming_job.rs +++ b/src/meta/src/controller/streaming_job.rs @@ -22,16 +22,17 @@ use risingwave_common::util::column_index_mapping::ColIndexMapping; use risingwave_common::util::stream_graph_visitor::visit_stream_node; use risingwave_meta_model_v2::actor::ActorStatus; use risingwave_meta_model_v2::actor_dispatcher::DispatcherType; +use risingwave_meta_model_v2::fragment::StreamNode; use risingwave_meta_model_v2::object::ObjectType; use risingwave_meta_model_v2::prelude::{ - Actor, ActorDispatcher, Fragment, Index, Object, ObjectDependency, Source, + Actor, ActorDispatcher, Fragment, Index, Object, ObjectDependency, Sink, Source, StreamingJob as StreamingJobModel, Table, }; use risingwave_meta_model_v2::{ actor, actor_dispatcher, fragment, index, object, object_dependency, sink, source, streaming_job, table, ActorId, ActorUpstreamActors, CreateType, DatabaseId, ExprNodeArray, - FragmentId, I32Array, IndexId, JobStatus, ObjectId, SchemaId, SourceId, StreamNode, - StreamingParallelism, TableId, TableVersion, UserId, + FragmentId, I32Array, IndexId, JobStatus, ObjectId, SchemaId, SourceId, StreamingParallelism, + TableId, TableVersion, UserId, }; use risingwave_pb::catalog::source::PbOptionalAssociatedTableId; use risingwave_pb::catalog::table::{PbOptionalAssociatedSourceId, PbTableVersion}; @@ -137,7 +138,7 @@ impl CatalogController { .await?; table.id = job_id as _; let table: table::ActiveModel = table.clone().into(); - table.insert(&txn).await?; + Table::insert(table).exec(&txn).await?; } StreamingJob::Sink(sink, _) => { let job_id = Self::create_streaming_job_obj( @@ -153,7 +154,7 @@ impl CatalogController { .await?; sink.id = job_id as _; let sink: sink::ActiveModel = sink.clone().into(); - sink.insert(&txn).await?; + Sink::insert(sink).exec(&txn).await?; } StreamingJob::Table(src, table, _) => { let job_id = Self::create_streaming_job_obj( @@ -184,10 +185,10 @@ impl CatalogController { PbOptionalAssociatedSourceId::AssociatedSourceId(src_obj.oid as _), ); let source: source::ActiveModel = src.clone().into(); - source.insert(&txn).await?; + Source::insert(source).exec(&txn).await?; } let table: table::ActiveModel = table.clone().into(); - table.insert(&txn).await?; + Table::insert(table).exec(&txn).await?; } StreamingJob::Index(index, table) => { ensure_object_id(ObjectType::Table, index.primary_table_id as _, &txn).await?; @@ -207,18 +208,18 @@ impl CatalogController { index.index_table_id = job_id as _; table.id = job_id as _; - object_dependency::ActiveModel { + ObjectDependency::insert(object_dependency::ActiveModel { oid: Set(index.primary_table_id as _), used_by: Set(table.id as _), ..Default::default() - } - .insert(&txn) + }) + .exec(&txn) .await?; let table: table::ActiveModel = table.clone().into(); - table.insert(&txn).await?; + Table::insert(table).exec(&txn).await?; let index: index::ActiveModel = index.clone().into(); - index.insert(&txn).await?; + Index::insert(index).exec(&txn).await?; } StreamingJob::Source(src) => { let job_id = Self::create_streaming_job_obj( @@ -234,7 +235,7 @@ impl CatalogController { .await?; src.id = job_id as _; let source: source::ActiveModel = src.clone().into(); - source.insert(&txn).await?; + Source::insert(source).exec(&txn).await?; } } @@ -280,7 +281,7 @@ impl CatalogController { table.table_id = Set(table_id as _); table.belongs_to_job_id = Set(Some(job_id as _)); table.fragment_id = NotSet; - table.insert(&txn).await?; + Table::insert(table).exec(&txn).await?; } txn.commit().await?; @@ -304,15 +305,17 @@ impl CatalogController { .map(|(fragment, actors, actor_dispatchers)| (fragment, (actors, actor_dispatchers))) .unzip(); for fragment in fragments { + let fragment_id = fragment.fragment_id; + let state_table_ids = fragment.state_table_ids.inner_ref().clone(); let fragment = fragment.into_active_model(); - let fragment = fragment.insert(&txn).await?; + Fragment::insert(fragment).exec(&txn).await?; // Update fragment id for all state tables. if !for_replace { - for state_table_id in fragment.state_table_ids.into_inner() { + for state_table_id in state_table_ids { table::ActiveModel { table_id: Set(state_table_id as _), - fragment_id: Set(Some(fragment.fragment_id as _)), + fragment_id: Set(Some(fragment_id)), ..Default::default() } .update(&txn) @@ -325,13 +328,13 @@ impl CatalogController { for (actors, actor_dispatchers) in actor_with_dispatchers { for actor in actors { let actor = actor.into_active_model(); - actor.insert(&txn).await?; + Actor::insert(actor).exec(&txn).await?; } for (_, actor_dispatchers) in actor_dispatchers { for actor_dispatcher in actor_dispatchers { let mut actor_dispatcher = actor_dispatcher.into_active_model(); actor_dispatcher.id = NotSet; - actor_dispatcher.insert(&txn).await?; + ActorDispatcher::insert(actor_dispatcher).exec(&txn).await?; } } } @@ -632,8 +635,9 @@ impl CatalogController { .into_tuple::<(FragmentId, StreamNode, I32Array)>() .one(&txn) .await? + .map(|(id, node, upstream)| (id, node.to_protobuf(), upstream)) .ok_or_else(|| MetaError::catalog_id_not_found("fragment", fragment_id))?; - visit_stream_node(&mut stream_node.0, |body| { + visit_stream_node(&mut stream_node, |body| { if let PbNodeBody::Merge(m) = body && let Some((new_fragment_id, new_actor_ids)) = fragment_replace_map.get(&m.upstream_fragment_id) @@ -649,7 +653,7 @@ impl CatalogController { } fragment::ActiveModel { fragment_id: Set(fragment_id), - stream_node: Set(stream_node), + stream_node: Set(StreamNode::from_protobuf(&stream_node)), upstream_fragment_id: Set(upstream_fragment_id), ..Default::default() } @@ -773,7 +777,7 @@ impl CatalogController { ))); } - let mut fragments: Vec<(FragmentId, i32, StreamNode)> = Fragment::find() + let fragments: Vec<(FragmentId, i32, StreamNode)> = Fragment::find() .select_only() .columns([ fragment::Column::FragmentId, @@ -784,11 +788,15 @@ impl CatalogController { .into_tuple() .all(&txn) .await?; + let mut fragments = fragments + .into_iter() + .map(|(id, mask, stream_node)| (id, mask, stream_node.to_protobuf())) + .collect_vec(); fragments.retain_mut(|(_, fragment_type_mask, stream_node)| { let mut found = false; if *fragment_type_mask & PbFragmentTypeFlag::Source as i32 != 0 { - visit_stream_node(&mut stream_node.0, |node| { + visit_stream_node(stream_node, |node| { if let PbNodeBody::Source(node) = node { if let Some(node_inner) = &mut node.source_inner && node_inner.source_id == source_id as u32 @@ -810,7 +818,7 @@ impl CatalogController { for (id, _, stream_node) in fragments { fragment::ActiveModel { fragment_id: Set(id), - stream_node: Set(stream_node), + stream_node: Set(StreamNode::from_protobuf(&stream_node)), ..Default::default() } .update(&txn) @@ -833,7 +841,7 @@ impl CatalogController { let inner = self.inner.read().await; let txn = inner.db.begin().await?; - let mut fragments: Vec<(FragmentId, i32, StreamNode)> = Fragment::find() + let fragments: Vec<(FragmentId, i32, StreamNode)> = Fragment::find() .select_only() .columns([ fragment::Column::FragmentId, @@ -844,11 +852,15 @@ impl CatalogController { .into_tuple() .all(&txn) .await?; + let mut fragments = fragments + .into_iter() + .map(|(id, mask, stream_node)| (id, mask, stream_node.to_protobuf())) + .collect_vec(); fragments.retain_mut(|(_, fragment_type_mask, stream_node)| { let mut found = false; if *fragment_type_mask & PbFragmentTypeFlag::StreamScan as i32 != 0 { - visit_stream_node(&mut stream_node.0, |node| { + visit_stream_node(stream_node, |node| { if let PbNodeBody::StreamScan(node) = node { node.rate_limit = rate_limit; found = true; @@ -867,7 +879,7 @@ impl CatalogController { for (id, _, stream_node) in fragments { fragment::ActiveModel { fragment_id: Set(id), - stream_node: Set(stream_node), + stream_node: Set(StreamNode::from_protobuf(&stream_node)), ..Default::default() } .update(&txn) diff --git a/src/meta/src/controller/system_param.rs b/src/meta/src/controller/system_param.rs index 6802ad17e176..4b2e598a2c22 100644 --- a/src/meta/src/controller/system_param.rs +++ b/src/meta/src/controller/system_param.rs @@ -173,10 +173,7 @@ impl SystemParamsController { // delete all params first and then insert all params. It follows the same logic // as the old code, we'd better change it to another way later to keep consistency. SystemParameter::delete_many().exec(&txn).await?; - - for model in models { - model.insert(&txn).await?; - } + SystemParameter::insert_many(models).exec(&txn).await?; txn.commit().await?; Ok(()) } diff --git a/src/meta/src/controller/utils.rs b/src/meta/src/controller/utils.rs index 53ac3c9616e2..ff19892d516b 100644 --- a/src/meta/src/controller/utils.rs +++ b/src/meta/src/controller/utils.rs @@ -18,14 +18,14 @@ use anyhow::anyhow; use itertools::Itertools; use risingwave_meta_model_migration::WithQuery; use risingwave_meta_model_v2::actor::ActorStatus; -use risingwave_meta_model_v2::fragment::DistributionType; +use risingwave_meta_model_v2::fragment::{DistributionType, StreamNode}; use risingwave_meta_model_v2::object::ObjectType; use risingwave_meta_model_v2::prelude::*; use risingwave_meta_model_v2::{ actor, actor_dispatcher, connection, database, fragment, function, index, object, object_dependency, schema, sink, source, table, user, user_privilege, view, ActorId, DataTypeArray, DatabaseId, FragmentId, FragmentVnodeMapping, I32Array, ObjectId, PrivilegeId, - SchemaId, SourceId, StreamNode, UserId, + SchemaId, SourceId, UserId, }; use risingwave_pb::catalog::{PbConnection, PbFunction}; use risingwave_pb::meta::PbFragmentParallelUnitMapping; @@ -761,7 +761,7 @@ where let mut source_fragment_ids = HashMap::new(); for (fragment_id, _, stream_node) in fragments { - if let Some(source) = find_stream_source(stream_node.inner_ref()) { + if let Some(source) = find_stream_source(&stream_node.to_protobuf()) { source_fragment_ids .entry(source.source_id as SourceId) .or_insert_with(BTreeSet::new) diff --git a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs index 27a00b05ffcc..f46a99bc80c0 100644 --- a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs @@ -688,7 +688,7 @@ pub mod tests { // pick_l0_to_base_level let mut picker = ManualCompactionPicker::new(Arc::new(RangeOverlapStrategy::default()), option, 1); - let mut expected = vec![vec![5, 6], vec![7, 8], vec![9, 10]]; + let mut expected = [vec![5, 6], vec![7, 8], vec![9, 10]]; expected.reverse(); let result = picker .pick_compaction(&levels, &levels_handler, &mut local_stats) @@ -724,7 +724,7 @@ pub mod tests { }; let mut picker = ManualCompactionPicker::new(Arc::new(RangeOverlapStrategy::default()), option, 1); - let mut expected = vec![vec![5, 6], vec![7, 8]]; + let mut expected = [vec![5, 6], vec![7, 8]]; expected.reverse(); let result = picker .pick_compaction(&levels, &levels_handler, &mut local_stats) @@ -1012,7 +1012,7 @@ pub mod tests { } { - let expected_input_level_sst_ids = vec![vec![4], vec![2]]; + let expected_input_level_sst_ids = [vec![4], vec![2]]; let option = ManualCompactionOption { sst_ids: vec![], level: input_level, diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index 8a76cc56f6b3..e34963ab48f9 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -874,7 +874,7 @@ pub mod tests { #[test] fn test_trivial_move_bug() { - let levels = vec![ + let levels = [ Level { level_idx: 1, level_type: LevelType::Nonoverlapping as i32, diff --git a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs index 506815dbed40..5d05fedbe533 100644 --- a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs @@ -409,7 +409,7 @@ mod test { // cut range [3,4] [6] [8,9,10] levels.member_table_ids = vec![0, 1, 2, 5, 7]; let expect_task_file_count = [2, 1, 4]; - let expect_task_sst_id_range = vec![vec![3, 4], vec![6], vec![8, 9, 10, 11]]; + let expect_task_sst_id_range = [vec![3, 4], vec![6], vec![8, 9, 10, 11]]; for (index, x) in expect_task_file_count.iter().enumerate() { // // pick space reclaim let task = selector @@ -460,7 +460,7 @@ mod test { // cut range [3,4] [6] [8,9,10] levels.member_table_ids = vec![0, 1, 2, 5, 7]; let expect_task_file_count = [2, 1, 5]; - let expect_task_sst_id_range = vec![vec![3, 4], vec![6], vec![7, 8, 9, 10, 11]]; + let expect_task_sst_id_range = [vec![3, 4], vec![6], vec![7, 8, 9, 10, 11]]; for (index, x) in expect_task_file_count.iter().enumerate() { if index == expect_task_file_count.len() - 1 { levels.member_table_ids = vec![2, 5]; diff --git a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs index 307d8f2b0174..f690b0f80112 100644 --- a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs @@ -611,7 +611,7 @@ mod test { ); let expect_task_file_count = [1, 1, 1]; - let expect_task_sst_id_range = vec![vec![2], vec![3], vec![4]]; + let expect_task_sst_id_range = [vec![2], vec![3], vec![4]]; for (index, x) in expect_task_file_count.iter().enumerate() { // // pick ttl reclaim let task = selector @@ -694,7 +694,7 @@ mod test { ); let expect_task_file_count = [1, 1]; - let expect_task_sst_id_range = vec![vec![2], vec![3]]; + let expect_task_sst_id_range = [vec![2], vec![3]]; for (index, x) in expect_task_file_count.iter().enumerate() { if index == expect_task_file_count.len() - 1 { table_id_to_options.insert( diff --git a/src/meta/src/hummock/compaction/selector/level_selector.rs b/src/meta/src/hummock/compaction/selector/level_selector.rs index a24f88cbf0fc..5c118269cfee 100644 --- a/src/meta/src/hummock/compaction/selector/level_selector.rs +++ b/src/meta/src/hummock/compaction/selector/level_selector.rs @@ -50,14 +50,14 @@ pub enum PickerType { BottomLevel, } -impl ToString for PickerType { - fn to_string(&self) -> String { - match self { - PickerType::Tier => String::from("Tier"), - PickerType::Intra => String::from("Intra"), - PickerType::ToBase => String::from("ToBase"), - PickerType::BottomLevel => String::from("BottomLevel"), - } +impl std::fmt::Display for PickerType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + PickerType::Tier => "Tier", + PickerType::Intra => "Intra", + PickerType::ToBase => "ToBase", + PickerType::BottomLevel => "BottomLevel", + }) } } diff --git a/src/meta/src/hummock/manager/sequence.rs b/src/meta/src/hummock/manager/sequence.rs index c1cdda952c6e..ab154376404d 100644 --- a/src/meta/src/hummock/manager/sequence.rs +++ b/src/meta/src/hummock/manager/sequence.rs @@ -18,6 +18,7 @@ use std::sync::LazyLock; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_meta_model_v2::hummock_sequence; +use risingwave_meta_model_v2::prelude::HummockSequence; use sea_orm::{ActiveModelTrait, ActiveValue, DatabaseConnection, EntityTrait, TransactionTrait}; use tokio::sync::Mutex; @@ -71,7 +72,7 @@ impl SequenceGenerator { name: ActiveValue::set(ident.into()), seq: ActiveValue::set(init.checked_add(num as _).unwrap().try_into().unwrap()), }; - active_model.insert(&txn).await?; + HummockSequence::insert(active_model).exec(&txn).await?; init } Some(model) => { diff --git a/src/meta/src/manager/cluster.rs b/src/meta/src/manager/cluster.rs index 38e1ff3ca8f3..b6c31ddb5168 100644 --- a/src/meta/src/manager/cluster.rs +++ b/src/meta/src/manager/cluster.rs @@ -32,6 +32,7 @@ use risingwave_pb::meta::heartbeat_request; use risingwave_pb::meta::subscribe_response::{Info, Operation}; use risingwave_pb::meta::update_worker_node_schedulability_request::Schedulability; use thiserror_ext::AsReport; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; use tokio::sync::oneshot::Sender; use tokio::sync::{RwLock, RwLockReadGuard}; use tokio::task::JoinHandle; @@ -458,6 +459,22 @@ impl ClusterManager { core.list_worker_node(worker_type, worker_state) } + pub async fn subscribe_active_streaming_compute_nodes( + &self, + ) -> (Vec, UnboundedReceiver) { + let core = self.core.read().await; + let worker_nodes = core.list_streaming_worker_node(Some(State::Running)); + let (tx, rx) = unbounded_channel(); + + // insert before release the read lock to ensure that we don't lose any update in between + self.env + .notification_manager() + .insert_local_sender(tx) + .await; + drop(core); + (worker_nodes, rx) + } + /// A convenient method to get all running compute nodes that may have running actors on them /// i.e. CNs which are running pub async fn list_active_streaming_compute_nodes(&self) -> Vec { @@ -465,11 +482,6 @@ impl ClusterManager { core.list_streaming_worker_node(Some(State::Running)) } - pub async fn list_active_streaming_parallel_units(&self) -> Vec { - let core = self.core.read().await; - core.list_active_streaming_parallel_units() - } - /// Get the cluster info used for scheduling a streaming job, containing all nodes that are /// running and schedulable pub async fn list_active_serving_compute_nodes(&self) -> Vec { @@ -703,13 +715,6 @@ impl ClusterManagerCore { .collect() } - fn list_active_streaming_parallel_units(&self) -> Vec { - self.list_streaming_worker_node(Some(State::Running)) - .into_iter() - .flat_map(|w| w.parallel_units) - .collect() - } - // Lists active worker nodes fn get_streaming_cluster_info(&self) -> StreamingClusterInfo { let mut streaming_worker_node = self.list_streaming_worker_node(Some(State::Running)); @@ -969,7 +974,12 @@ mod tests { } async fn assert_cluster_manager(cluster_manager: &ClusterManager, parallel_count: usize) { - let parallel_units = cluster_manager.list_active_streaming_parallel_units().await; + let parallel_units = cluster_manager + .list_active_serving_compute_nodes() + .await + .into_iter() + .flat_map(|w| w.parallel_units) + .collect_vec(); assert_eq!(parallel_units.len(), parallel_count); } diff --git a/src/meta/src/manager/metadata.rs b/src/meta/src/manager/metadata.rs index be16ec060194..eef1c0c10125 100644 --- a/src/meta/src/manager/metadata.rs +++ b/src/meta/src/manager/metadata.rs @@ -18,16 +18,19 @@ use risingwave_common::catalog::{TableId, TableOption}; use risingwave_meta_model_v2::SourceId; use risingwave_pb::catalog::{PbSource, PbTable}; use risingwave_pb::common::worker_node::{PbResource, State}; -use risingwave_pb::common::{HostAddress, PbWorkerNode, PbWorkerType, WorkerType}; +use risingwave_pb::common::{HostAddress, PbWorkerNode, PbWorkerType, WorkerNode, WorkerType}; use risingwave_pb::meta::add_worker_node_request::Property as AddNodeProperty; use risingwave_pb::meta::table_fragments::{ActorStatus, Fragment, PbFragment}; use risingwave_pb::stream_plan::{PbDispatchStrategy, PbStreamActor, StreamActor}; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; +use tracing::warn; use crate::barrier::Reschedule; use crate::controller::catalog::CatalogControllerRef; use crate::controller::cluster::{ClusterControllerRef, WorkerExtraInfo}; use crate::manager::{ - CatalogManagerRef, ClusterManagerRef, FragmentManagerRef, StreamingClusterInfo, WorkerId, + CatalogManagerRef, ClusterManagerRef, FragmentManagerRef, LocalNotification, + StreamingClusterInfo, WorkerId, }; use crate::model::{ActorId, FragmentId, MetadataModel, TableFragments, TableParallelism}; use crate::stream::SplitAssignment; @@ -52,6 +55,127 @@ pub struct MetadataManagerV2 { pub catalog_controller: CatalogControllerRef, } +#[derive(Debug)] +pub(crate) enum ActiveStreamingWorkerChange { + Add(WorkerNode), + Remove(WorkerNode), + Update(WorkerNode), +} + +pub struct ActiveStreamingWorkerNodes { + worker_nodes: HashMap, + rx: UnboundedReceiver, +} + +impl ActiveStreamingWorkerNodes { + pub(crate) fn uninitialized() -> Self { + Self { + worker_nodes: Default::default(), + rx: unbounded_channel().1, + } + } + + /// Return an uninitialized one as a place holder for future initialized + pub(crate) async fn new_snapshot(meta_manager: MetadataManager) -> MetaResult { + let (nodes, rx) = meta_manager + .subscribe_active_streaming_compute_nodes() + .await?; + Ok(Self { + worker_nodes: nodes.into_iter().map(|node| (node.id, node)).collect(), + rx, + }) + } + + pub(crate) fn current(&self) -> &HashMap { + &self.worker_nodes + } + + pub(crate) async fn changed(&mut self) -> ActiveStreamingWorkerChange { + let ret = loop { + let notification = self + .rx + .recv() + .await + .expect("notification stopped or uninitialized"); + match notification { + LocalNotification::WorkerNodeDeleted(worker) => { + let is_streaming_compute_node = worker.r#type == WorkerType::ComputeNode as i32 + && worker.property.as_ref().unwrap().is_streaming; + let Some(prev_worker) = self.worker_nodes.remove(&worker.id) else { + if is_streaming_compute_node { + warn!( + ?worker, + "notify to delete an non-existing streaming compute worker" + ); + } + continue; + }; + if !is_streaming_compute_node { + warn!( + ?worker, + ?prev_worker, + "deleted worker has a different recent type" + ); + } + if worker.state == State::Starting as i32 { + warn!( + id = worker.id, + host = ?worker.host, + state = worker.state, + "a starting streaming worker is deleted" + ); + } + break ActiveStreamingWorkerChange::Remove(prev_worker); + } + LocalNotification::WorkerNodeActivated(worker) => { + if worker.r#type != WorkerType::ComputeNode as i32 + || !worker.property.as_ref().unwrap().is_streaming + { + if let Some(prev_worker) = self.worker_nodes.remove(&worker.id) { + warn!( + ?worker, + ?prev_worker, + "the type of a streaming worker is changed" + ); + break ActiveStreamingWorkerChange::Remove(prev_worker); + } else { + continue; + } + } + assert_eq!( + worker.state, + State::Running as i32, + "not started worker added: {:?}", + worker + ); + if let Some(prev_worker) = self.worker_nodes.insert(worker.id, worker.clone()) { + assert_eq!(prev_worker.host, worker.host); + assert_eq!(prev_worker.r#type, worker.r#type); + warn!( + ?prev_worker, + ?worker, + eq = prev_worker == worker, + "notify to update an existing active worker" + ); + if prev_worker == worker { + continue; + } else { + break ActiveStreamingWorkerChange::Update(worker); + } + } else { + break ActiveStreamingWorkerChange::Add(worker); + } + } + _ => { + continue; + } + } + }; + + ret + } +} + impl MetadataManager { pub fn new_v1( cluster_manager: ClusterManagerRef, @@ -171,6 +295,22 @@ impl MetadataManager { } } + pub async fn subscribe_active_streaming_compute_nodes( + &self, + ) -> MetaResult<(Vec, UnboundedReceiver)> { + match self { + MetadataManager::V1(mgr) => Ok(mgr + .cluster_manager + .subscribe_active_streaming_compute_nodes() + .await), + MetadataManager::V2(mgr) => { + mgr.cluster_controller + .subscribe_active_streaming_compute_nodes() + .await + } + } + } + pub async fn list_active_streaming_compute_nodes(&self) -> MetaResult> { match self { MetadataManager::V1(mgr) => Ok(mgr diff --git a/src/meta/src/manager/notification_version.rs b/src/meta/src/manager/notification_version.rs index 87ccd3ff7f54..18c8c7cc9d20 100644 --- a/src/meta/src/manager/notification_version.rs +++ b/src/meta/src/manager/notification_version.rs @@ -41,11 +41,11 @@ impl NotificationVersionGenerator { .await?; let current_version = model.as_ref().map(|m| m.version).unwrap_or(1) as u64; if model.is_none() { - catalog_version::ActiveModel { + CatalogVersion::insert(catalog_version::ActiveModel { name: Set(VersionCategory::Notification), version: Set(1), - } - .insert(&txn) + }) + .exec(&txn) .await?; txn.commit().await?; } diff --git a/src/meta/src/rpc/ddl_controller.rs b/src/meta/src/rpc/ddl_controller.rs index 84ff4ea1de68..7de106523599 100644 --- a/src/meta/src/rpc/ddl_controller.rs +++ b/src/meta/src/rpc/ddl_controller.rs @@ -654,7 +654,7 @@ impl DdlController { .acquire() .await .unwrap(); - let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; + let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await; let stream_ctx = StreamContext::from_protobuf(fragment_graph.get_ctx().unwrap()); @@ -1154,7 +1154,7 @@ impl DdlController { target_replace_info: Option, ) -> MetaResult { let mgr = self.metadata_manager.as_v1_ref(); - let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; + let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await; let (mut version, streaming_job_ids) = match job_id { StreamingJobId::MaterializedView(table_id) => { mgr.catalog_manager @@ -1647,7 +1647,7 @@ impl DdlController { .replace_table_v2(stream_job, fragment_graph, table_col_index_mapping) .await; }; - let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; + let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await; let stream_ctx = StreamContext::from_protobuf(fragment_graph.get_ctx().unwrap()); let fragment_graph = self diff --git a/src/meta/src/rpc/ddl_controller_v2.rs b/src/meta/src/rpc/ddl_controller_v2.rs index dd3defd141ae..748857d2c3da 100644 --- a/src/meta/src/rpc/ddl_controller_v2.rs +++ b/src/meta/src/rpc/ddl_controller_v2.rs @@ -76,7 +76,7 @@ impl DdlController { .acquire() .await .unwrap(); - let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; + let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await; // create streaming job. match self @@ -284,7 +284,7 @@ impl DdlController { let mgr = self.metadata_manager.as_v2_ref(); let job_id = streaming_job.id(); - let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; + let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await; let ctx = StreamContext::from_protobuf(fragment_graph.get_ctx().unwrap()); // 1. build fragment graph. diff --git a/src/meta/src/stream/scale.rs b/src/meta/src/stream/scale.rs index e598aefb291d..7f40f8e3da03 100644 --- a/src/meta/src/stream/scale.rs +++ b/src/meta/src/stream/scale.rs @@ -41,8 +41,8 @@ use risingwave_pb::meta::FragmentParallelUnitMappings; use risingwave_pb::stream_plan::stream_node::NodeBody; use risingwave_pb::stream_plan::{DispatcherType, FragmentTypeFlag, StreamActor, StreamNode}; use thiserror_ext::AsReport; -use tokio::sync::oneshot; use tokio::sync::oneshot::Receiver; +use tokio::sync::{oneshot, RwLock, RwLockReadGuard, RwLockWriteGuard}; use tokio::task::JoinHandle; use tokio::time::MissedTickBehavior; @@ -375,6 +375,8 @@ pub struct ScaleController { pub stream_rpc_manager: StreamRpcManager, pub env: MetaSrvEnv, + + pub reschedule_lock: RwLock<()>, } impl ScaleController { @@ -389,6 +391,7 @@ impl ScaleController { metadata_manager: metadata_manager.clone(), source_manager, env, + reschedule_lock: RwLock::new(()), } } @@ -2449,6 +2452,14 @@ pub struct TableResizePolicy { } impl GlobalStreamManager { + pub async fn reschedule_lock_read_guard(&self) -> RwLockReadGuard<'_, ()> { + self.scale_controller.reschedule_lock.read().await + } + + pub async fn reschedule_lock_write_guard(&self) -> RwLockWriteGuard<'_, ()> { + self.scale_controller.reschedule_lock.write().await + } + pub async fn reschedule_actors( &self, reschedules: HashMap, @@ -2518,7 +2529,7 @@ impl GlobalStreamManager { } async fn trigger_parallelism_control(&self) -> MetaResult<()> { - let _reschedule_job_lock = self.reschedule_lock.write().await; + let _reschedule_job_lock = self.reschedule_lock_write_guard().await; match &self.metadata_manager { MetadataManager::V1(mgr) => { diff --git a/src/meta/src/stream/stream_manager.rs b/src/meta/src/stream/stream_manager.rs index 5889292756ca..d7388f50da09 100644 --- a/src/meta/src/stream/stream_manager.rs +++ b/src/meta/src/stream/stream_manager.rs @@ -24,10 +24,10 @@ use risingwave_pb::stream_plan::update_mutation::MergeUpdate; use risingwave_pb::stream_plan::Dispatcher; use thiserror_ext::AsReport; use tokio::sync::mpsc::Sender; -use tokio::sync::{oneshot, Mutex, RwLock}; +use tokio::sync::{oneshot, Mutex}; use tracing::Instrument; -use super::{Locations, RescheduleOptions, ScaleController, ScaleControllerRef, TableResizePolicy}; +use super::{Locations, RescheduleOptions, ScaleControllerRef, TableResizePolicy}; use crate::barrier::{BarrierScheduler, Command, ReplaceTablePlan, StreamRpcManager}; use crate::hummock::HummockManagerRef; use crate::manager::{DdlType, MetaSrvEnv, MetadataManager, StreamingJob}; @@ -192,9 +192,7 @@ pub struct GlobalStreamManager { hummock_manager: HummockManagerRef, - pub reschedule_lock: RwLock<()>, - - pub(crate) scale_controller: ScaleControllerRef, + pub scale_controller: ScaleControllerRef, pub stream_rpc_manager: StreamRpcManager, } @@ -207,14 +205,8 @@ impl GlobalStreamManager { source_manager: SourceManagerRef, hummock_manager: HummockManagerRef, stream_rpc_manager: StreamRpcManager, + scale_controller: ScaleControllerRef, ) -> MetaResult { - let scale_controller = Arc::new(ScaleController::new( - &metadata_manager, - source_manager.clone(), - stream_rpc_manager.clone(), - env.clone(), - )); - Ok(Self { env, metadata_manager, @@ -222,7 +214,6 @@ impl GlobalStreamManager { source_manager, hummock_manager, creating_job_info: Arc::new(CreatingStreamingJobInfo::default()), - reschedule_lock: RwLock::new(()), scale_controller, stream_rpc_manager, }) @@ -629,7 +620,7 @@ impl GlobalStreamManager { return vec![]; } - let _reschedule_job_lock = self.reschedule_lock.read().await; + let _reschedule_job_lock = self.reschedule_lock_read_guard().await; let (receivers, recovered_job_ids) = self.creating_job_info.cancel_jobs(table_ids).await; let futures = receivers.into_iter().map(|(id, receiver)| async move { @@ -688,7 +679,7 @@ impl GlobalStreamManager { parallelism: TableParallelism, deferred: bool, ) -> MetaResult<()> { - let _reschedule_job_lock = self.reschedule_lock.write().await; + let _reschedule_job_lock = self.reschedule_lock_write_guard().await; let worker_nodes = self .metadata_manager @@ -786,7 +777,7 @@ mod tests { use crate::model::{ActorId, FragmentId}; use crate::rpc::ddl_controller::DropMode; use crate::rpc::metrics::MetaMetrics; - use crate::stream::SourceManager; + use crate::stream::{ScaleController, SourceManager}; use crate::MetaOpts; struct FakeFragmentState { @@ -985,6 +976,12 @@ mod tests { let (sink_manager, _) = SinkCoordinatorManager::start_worker(); let stream_rpc_manager = StreamRpcManager::new(env.clone()); + let scale_controller = Arc::new(ScaleController::new( + &metadata_manager, + source_manager.clone(), + stream_rpc_manager.clone(), + env.clone(), + )); let barrier_manager = GlobalBarrierManager::new( scheduled_barriers, @@ -995,6 +992,7 @@ mod tests { sink_manager, meta_metrics.clone(), stream_rpc_manager.clone(), + scale_controller.clone(), ); let stream_manager = GlobalStreamManager::new( @@ -1004,6 +1002,7 @@ mod tests { source_manager.clone(), hummock_manager, stream_rpc_manager, + scale_controller.clone(), )?; let (join_handle_2, shutdown_tx_2) = GlobalBarrierManager::start(barrier_manager); diff --git a/src/meta/src/telemetry.rs b/src/meta/src/telemetry.rs index 4bf7a2c0ba34..43533f96ae10 100644 --- a/src/meta/src/telemetry.rs +++ b/src/meta/src/telemetry.rs @@ -18,6 +18,7 @@ use risingwave_common::telemetry::{ current_timestamp, SystemData, TelemetryNodeType, TelemetryReport, TelemetryReportBase, TelemetryResult, }; +use risingwave_common::{GIT_SHA, RW_VERSION}; use risingwave_pb::common::WorkerType; use serde::{Deserialize, Serialize}; use thiserror_ext::AsReport; @@ -33,6 +34,12 @@ struct NodeCount { compactor_count: u64, } +#[derive(Debug, Serialize, Deserialize)] +struct RwVersion { + version: String, + git_sha: String, +} + #[derive(Debug, Serialize, Deserialize)] pub struct MetaTelemetryReport { #[serde(flatten)] @@ -41,6 +48,7 @@ pub struct MetaTelemetryReport { streaming_job_count: u64, // At this point, it will always be etcd, but we will enable telemetry when using memory. meta_backend: MetaBackend, + rw_version: RwVersion, } impl TelemetryReport for MetaTelemetryReport {} @@ -99,6 +107,10 @@ impl TelemetryReportCreator for MetaReportCreator { .map_err(|err| err.as_report().to_string())? as u64; Ok(MetaTelemetryReport { + rw_version: RwVersion { + version: RW_VERSION.to_string(), + git_sha: GIT_SHA.to_string(), + }, base: TelemetryReportBase { tracking_id, session_id, diff --git a/src/object_store/Cargo.toml b/src/object_store/Cargo.toml index 61f526511c3a..a7ae9a8bfb70 100644 --- a/src/object_store/Cargo.toml +++ b/src/object_store/Cargo.toml @@ -18,7 +18,7 @@ aws-smithy-runtime = { workspace = true } aws-smithy-runtime-api = { workspace = true } aws-smithy-types = { workspace = true } bytes = { version = "1", features = ["serde"] } -crc32fast = "1.3.2" +crc32fast = "1" either = "1" fail = "0.5" futures = { version = "0.3", default-features = false, features = ["alloc"] } diff --git a/src/rpc_client/Cargo.toml b/src/rpc_client/Cargo.toml index 7a43b6359cd6..1e438603fd49 100644 --- a/src/rpc_client/Cargo.toml +++ b/src/rpc_client/Cargo.toml @@ -17,7 +17,7 @@ normal = ["workspace-hack"] anyhow = "1" async-trait = "0.1" easy-ext = "1" -either = "1.9.0" +either = "1.10.0" futures = { version = "0.3", default-features = false, features = ["alloc"] } http = "0.2" hyper = "0.14" diff --git a/src/storage/benches/bench_fs_operation.rs b/src/storage/benches/bench_fs_operation.rs index 4f01af01e60c..0983883fdaa4 100644 --- a/src/storage/benches/bench_fs_operation.rs +++ b/src/storage/benches/bench_fs_operation.rs @@ -94,6 +94,7 @@ fn gen_tokio_files(path: &Path) -> impl IntoIterator impl IntoIterator + EmptySliceRef>( pub struct FullKeyTracker + Ord + Eq> { pub latest_full_key: FullKey, last_observed_epoch_with_gap: EpochWithGap, + /// TODO: Temporary bypass full key check. Remove this field after #15099 is resolved. + allow_same_full_key: bool, } impl + Ord + Eq> FullKeyTracker { pub fn new(init_full_key: FullKey) -> Self { + Self::with_config(init_full_key, false) + } + + pub fn with_config(init_full_key: FullKey, allow_same_full_key: bool) -> Self { let epoch_with_gap = init_full_key.epoch_with_gap; Self { latest_full_key: init_full_key, last_observed_epoch_with_gap: epoch_with_gap, + allow_same_full_key, } } @@ -1032,7 +1039,10 @@ impl + Ord + Eq> FullKeyTracker { )) } Ordering::Equal => { - if max_epoch_with_gap >= self.last_observed_epoch_with_gap { + if max_epoch_with_gap > self.last_observed_epoch_with_gap + || (!self.allow_same_full_key + && max_epoch_with_gap == self.last_observed_epoch_with_gap) + { // Epoch from the same user key should be monotonically decreasing panic!( "key {:?} epoch {:?} >= prev epoch {:?}", diff --git a/src/storage/hummock_test/src/test_utils.rs b/src/storage/hummock_test/src/test_utils.rs index 28cd9273bc25..554f7bd8b8be 100644 --- a/src/storage/hummock_test/src/test_utils.rs +++ b/src/storage/hummock_test/src/test_utils.rs @@ -111,6 +111,7 @@ impl TestIngestBatch for S { } } +#[cfg(test)] #[async_trait::async_trait] pub(crate) trait HummockStateStoreTestTrait: StateStore { fn get_pinned_version(&self) -> PinnedVersion; @@ -120,6 +121,7 @@ pub(crate) trait HummockStateStoreTestTrait: StateStore { } } +#[cfg(test)] impl HummockStateStoreTestTrait for HummockStorage { fn get_pinned_version(&self) -> PinnedVersion { self.get_pinned_version() diff --git a/src/storage/hummock_trace/src/write.rs b/src/storage/hummock_trace/src/write.rs index 3038ffb9a7ee..528cdac5f659 100644 --- a/src/storage/hummock_trace/src/write.rs +++ b/src/storage/hummock_trace/src/write.rs @@ -30,13 +30,6 @@ pub(crate) static MAGIC_BYTES: MagicBytes = 0x484D5452; // HMTR pub(crate) trait TraceWriter { fn write(&mut self, record: Record) -> Result; fn flush(&mut self) -> Result<()>; - fn write_all(&mut self, records: Vec) -> Result { - let mut total_size = 0; - for r in records { - total_size += self.write(r)? - } - Ok(total_size) - } } /// Serializer serializes a record to std write. diff --git a/src/storage/src/hummock/block_cache.rs b/src/storage/src/hummock/block_cache.rs index e71714e8f8bb..9023f045c055 100644 --- a/src/storage/src/hummock/block_cache.rs +++ b/src/storage/src/hummock/block_cache.rs @@ -34,9 +34,9 @@ const MIN_BUFFER_SIZE_PER_SHARD: usize = 256 * 1024 * 1024; type CachedBlockEntry = CacheableEntry<(HummockSstableObjectId, u64), Box>; enum BlockEntry { - Cache(CachedBlockEntry), - Owned(Box), - RefEntry(Arc), + Cache(#[allow(dead_code)] CachedBlockEntry), + Owned(#[allow(dead_code)] Box), + RefEntry(#[allow(dead_code)] Arc), } pub struct BlockHolder { diff --git a/src/storage/src/hummock/iterator/forward_user.rs b/src/storage/src/hummock/iterator/forward_user.rs index f84e0038312b..f006d041d554 100644 --- a/src/storage/src/hummock/iterator/forward_user.rs +++ b/src/storage/src/hummock/iterator/forward_user.rs @@ -78,7 +78,7 @@ impl> UserIterator { delete_range_iter, _version: version, is_current_pos_valid: false, - full_key_tracker: FullKeyTracker::new(FullKey::default()), + full_key_tracker: FullKeyTracker::with_config(FullKey::default(), true), } } @@ -136,7 +136,7 @@ impl> UserIterator { pub async fn rewind(&mut self) -> HummockResult<()> { // Reset self.is_current_pos_valid = false; - self.full_key_tracker = FullKeyTracker::new(FullKey::default()); + self.full_key_tracker = FullKeyTracker::with_config(FullKey::default(), true); // Handle range scan match &self.key_range.0 { @@ -180,7 +180,7 @@ impl> UserIterator { pub async fn seek(&mut self, user_key: UserKey<&[u8]>) -> HummockResult<()> { // Reset self.is_current_pos_valid = false; - self.full_key_tracker = FullKeyTracker::new(FullKey::default()); + self.full_key_tracker = FullKeyTracker::with_config(FullKey::default(), true); // Handle range scan when key < begin_key let user_key = match &self.key_range.0 { diff --git a/src/stream/src/common/table/state_table_cache.rs b/src/stream/src/common/table/state_table_cache.rs index 06acd7e710dc..8c8014548a1e 100644 --- a/src/stream/src/common/table/state_table_cache.rs +++ b/src/stream/src/common/table/state_table_cache.rs @@ -453,23 +453,23 @@ mod tests { #[test] fn test_watermark_cache_syncing() { - let v1 = vec![ + let v1 = [ Some(Timestamptz::from_secs(1000).unwrap().to_scalar_value()), Some(1000i64.into()), ]; - let v2 = vec![ + let v2 = [ Some(Timestamptz::from_secs(3000).unwrap().to_scalar_value()), Some(1000i64.into()), ]; - let v3 = vec![ + let v3 = [ Some(Timestamptz::from_secs(2000).unwrap().to_scalar_value()), Some(1000i64.into()), ]; let mut cache = StateTableWatermarkCache::new(3); let mut filler = cache.begin_syncing(); - filler.insert_unchecked(DefaultOrdered(v1.into_owned_row()), ()); - filler.insert_unchecked(DefaultOrdered(v2.into_owned_row()), ()); - filler.insert_unchecked(DefaultOrdered(v3.into_owned_row()), ()); + filler.insert_unchecked(DefaultOrdered(v1.to_owned_row()), ()); + filler.insert_unchecked(DefaultOrdered(v2.to_owned_row()), ()); + filler.insert_unchecked(DefaultOrdered(v3.to_owned_row()), ()); filler.finish(); assert_eq!(cache.len(), 3); assert_eq!( diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs index f47919299957..07903a2c7b34 100644 --- a/src/stream/src/executor/source/source_executor.rs +++ b/src/stream/src/executor/source/source_executor.rs @@ -391,8 +391,8 @@ impl SourceExecutor { }; let mut boot_state = Vec::default(); - if let Some(mutation) = barrier.mutation.as_ref() { - match mutation.as_ref() { + if let Some(mutation) = barrier.mutation.as_deref() { + match mutation { Mutation::Add(AddMutation { splits, .. }) | Mutation::Update(UpdateMutation { actor_splits: splits, @@ -460,179 +460,154 @@ impl SourceExecutor { let mut metric_row_per_barrier: u64 = 0; while let Some(msg) = stream.next().await { + let Ok(msg) = msg else { + tokio::time::sleep(Duration::from_millis(1000)).await; + self.rebuild_stream_reader_from_error( + &source_desc, + &mut stream, + &mut latest_split_info, + msg.unwrap_err(), + ) + .await?; + continue; + }; + match msg { - Err(e) => { - tokio::time::sleep(Duration::from_millis(1000)).await; - self.rebuild_stream_reader_from_error( - &source_desc, - &mut stream, - &mut latest_split_info, - e, - ) - .await?; - } - Ok(msg) => { - match msg { - // This branch will be preferred. - Either::Left(msg) => match &msg { - Message::Barrier(barrier) => { - last_barrier_time = Instant::now(); - - if self_paused { - stream.resume_stream(); - self_paused = false; - } - - let epoch = barrier.epoch; - - let mut target_state = None; - let mut should_trim_state = false; - - if let Some(ref mutation) = barrier.mutation.as_deref() { - match mutation { - Mutation::Pause => stream.pause_stream(), - Mutation::Resume => stream.resume_stream(), - Mutation::SourceChangeSplit(actor_splits) => { - tracing::info!( - actor_id = self.actor_ctx.id, - actor_splits = ?actor_splits, - "source change split received" - ); - - target_state = self - .apply_split_change( - &source_desc, - &mut stream, - actor_splits, - ) - .await?; - should_trim_state = true; - } - - Mutation::Update(UpdateMutation { - actor_splits, .. - }) => { - target_state = self - .apply_split_change( - &source_desc, - &mut stream, - actor_splits, - ) - .await?; - } - _ => {} - } - } - - if let Some(target_state) = &target_state { - latest_split_info = target_state.clone(); - } - - self.persist_state_and_clear_cache( - epoch, - target_state, - should_trim_state, - ) - .await?; - - self.metrics - .source_row_per_barrier - .with_label_values(&[ - self.actor_ctx.id.to_string().as_str(), - self.stream_source_core - .as_ref() - .unwrap() - .source_id - .to_string() - .as_ref(), - self.actor_ctx.fragment_id.to_string().as_str(), - ]) - .inc_by(metric_row_per_barrier); - metric_row_per_barrier = 0; - - yield msg; - } - _ => { - // For the source executor, the message we receive from this arm - // should always be barrier message. - unreachable!(); - } - }, - - Either::Right(chunk) => { - // TODO: confirm when split_offset_mapping is None - let split_offset_mapping = - get_split_offset_mapping_from_chunk(&chunk, split_idx, offset_idx); - if last_barrier_time.elapsed().as_millis() > max_wait_barrier_time_ms { - // Exceeds the max wait barrier time, the source will be paused. - // Currently we can guarantee the - // source is not paused since it received stream - // chunks. - self_paused = true; - tracing::warn!( - "source {} paused, wait barrier for {:?}", - self.info.identity, - last_barrier_time.elapsed() + // This branch will be preferred. + Either::Left(Message::Barrier(barrier)) => { + last_barrier_time = Instant::now(); + + if self_paused { + stream.resume_stream(); + self_paused = false; + } + + let epoch = barrier.epoch; + + let mut target_state = None; + let mut should_trim_state = false; + + if let Some(mutation) = barrier.mutation.as_deref() { + match mutation { + Mutation::Pause => stream.pause_stream(), + Mutation::Resume => stream.resume_stream(), + Mutation::SourceChangeSplit(actor_splits) => { + tracing::info!( + actor_id = self.actor_ctx.id, + actor_splits = ?actor_splits, + "source change split received" ); - stream.pause_stream(); - - // Only update `max_wait_barrier_time_ms` to capture - // `barrier_interval_ms` - // changes here to avoid frequently accessing the shared - // `system_params`. - max_wait_barrier_time_ms = - self.system_params.load().barrier_interval_ms() as u128 - * WAIT_BARRIER_MULTIPLE_TIMES; + + target_state = self + .apply_split_change(&source_desc, &mut stream, actor_splits) + .await?; + should_trim_state = true; } - if let Some(mapping) = split_offset_mapping { - let state: HashMap<_, _> = mapping - .iter() - .flat_map(|(split_id, offset)| { - let origin_split_impl = self - .stream_source_core - .as_mut() - .unwrap() - .stream_source_splits - .get_mut(split_id); - - origin_split_impl.map(|split_impl| { - split_impl.update_in_place(offset.clone())?; - Ok::<_, anyhow::Error>(( - split_id.clone(), - split_impl.clone(), - )) - }) - }) - .try_collect()?; - - self.stream_source_core - .as_mut() - .unwrap() - .state_cache - .extend(state); + + Mutation::Update(UpdateMutation { actor_splits, .. }) => { + target_state = self + .apply_split_change(&source_desc, &mut stream, actor_splits) + .await?; } - metric_row_per_barrier += chunk.cardinality() as u64; - - self.metrics - .source_output_row_count - .with_label_values( - &self - .get_metric_labels() - .iter() - .map(AsRef::as_ref) - .collect::>(), - ) - .inc_by(chunk.cardinality() as u64); - let chunk = prune_additional_cols( - &chunk, - split_idx, - offset_idx, - &source_desc.columns, - ); - yield Message::Chunk(chunk); - self.try_flush_data().await?; + _ => {} } } + + if let Some(target_state) = &target_state { + latest_split_info = target_state.clone(); + } + + self.persist_state_and_clear_cache(epoch, target_state, should_trim_state) + .await?; + + self.metrics + .source_row_per_barrier + .with_label_values(&[ + self.actor_ctx.id.to_string().as_str(), + self.stream_source_core + .as_ref() + .unwrap() + .source_id + .to_string() + .as_ref(), + self.actor_ctx.fragment_id.to_string().as_str(), + ]) + .inc_by(metric_row_per_barrier); + metric_row_per_barrier = 0; + + yield Message::Barrier(barrier); + } + Either::Left(_) => { + // For the source executor, the message we receive from this arm + // should always be barrier message. + unreachable!(); + } + + Either::Right(chunk) => { + // TODO: confirm when split_offset_mapping is None + let split_offset_mapping = + get_split_offset_mapping_from_chunk(&chunk, split_idx, offset_idx); + if last_barrier_time.elapsed().as_millis() > max_wait_barrier_time_ms { + // Exceeds the max wait barrier time, the source will be paused. + // Currently we can guarantee the + // source is not paused since it received stream + // chunks. + self_paused = true; + tracing::warn!( + "source {} paused, wait barrier for {:?}", + self.info.identity, + last_barrier_time.elapsed() + ); + stream.pause_stream(); + + // Only update `max_wait_barrier_time_ms` to capture + // `barrier_interval_ms` + // changes here to avoid frequently accessing the shared + // `system_params`. + max_wait_barrier_time_ms = self.system_params.load().barrier_interval_ms() + as u128 + * WAIT_BARRIER_MULTIPLE_TIMES; + } + if let Some(mapping) = split_offset_mapping { + let state: HashMap<_, _> = mapping + .iter() + .flat_map(|(split_id, offset)| { + let origin_split_impl = self + .stream_source_core + .as_mut() + .unwrap() + .stream_source_splits + .get_mut(split_id); + + origin_split_impl.map(|split_impl| { + split_impl.update_in_place(offset.clone())?; + Ok::<_, anyhow::Error>((split_id.clone(), split_impl.clone())) + }) + }) + .try_collect()?; + + self.stream_source_core + .as_mut() + .unwrap() + .state_cache + .extend(state); + } + metric_row_per_barrier += chunk.cardinality() as u64; + + self.metrics + .source_output_row_count + .with_label_values( + &self + .get_metric_labels() + .iter() + .map(AsRef::as_ref) + .collect::>(), + ) + .inc_by(chunk.cardinality() as u64); + let chunk = + prune_additional_cols(&chunk, split_idx, offset_idx, &source_desc.columns); + yield Message::Chunk(chunk); + self.try_flush_data().await?; } } } diff --git a/src/stream/src/executor/top_n/top_n_plain.rs b/src/stream/src/executor/top_n/top_n_plain.rs index 8536d9d3273c..5df41cfca3a0 100644 --- a/src/stream/src/executor/top_n/top_n_plain.rs +++ b/src/stream/src/executor/top_n/top_n_plain.rs @@ -761,7 +761,7 @@ mod tests { } fn create_source_new_before_recovery() -> Box { - let mut chunks = vec![ + let mut chunks = [ StreamChunk::from_pretty( " I I I I + 1 1 4 1001", @@ -792,7 +792,7 @@ mod tests { } fn create_source_new_after_recovery() -> Box { - let mut chunks = vec![ + let mut chunks = [ StreamChunk::from_pretty( " I I I I + 1 9 1 1003 @@ -1209,7 +1209,7 @@ mod tests { } fn create_source_before_recovery() -> Box { - let mut chunks = vec![ + let mut chunks = [ StreamChunk::from_pretty( " I I + 1 0 @@ -1248,7 +1248,7 @@ mod tests { } fn create_source_after_recovery() -> Box { - let mut chunks = vec![ + let mut chunks = [ StreamChunk::from_pretty( " I I - 1 0", diff --git a/src/stream/src/executor/top_n/top_n_state.rs b/src/stream/src/executor/top_n/top_n_state.rs index a979aeb62995..f8dbbcabe00d 100644 --- a/src/stream/src/executor/top_n/top_n_state.rs +++ b/src/stream/src/executor/top_n/top_n_state.rs @@ -364,8 +364,8 @@ mod tests { let row2_bytes = serialize_pk_to_cache_key(row2.clone(), &cache_key_serde); let row3_bytes = serialize_pk_to_cache_key(row3.clone(), &cache_key_serde); let row4_bytes = serialize_pk_to_cache_key(row4.clone(), &cache_key_serde); - let rows = vec![row1, row2, row3, row4]; - let ordered_rows = vec![row1_bytes, row2_bytes, row3_bytes, row4_bytes]; + let rows = [row1, row2, row3, row4]; + let ordered_rows = [row1_bytes, row2_bytes, row3_bytes, row4_bytes]; managed_state.insert(rows[3].clone()); // now ("ab", 4) @@ -446,7 +446,7 @@ mod tests { let row3_bytes = serialize_pk_to_cache_key(row3.clone(), &cache_key_serde); let row4_bytes = serialize_pk_to_cache_key(row4.clone(), &cache_key_serde); let row5_bytes = serialize_pk_to_cache_key(row5.clone(), &cache_key_serde); - let rows = vec![row1, row2, row3, row4, row5]; + let rows = [row1, row2, row3, row4, row5]; let ordered_rows = vec![row1_bytes, row2_bytes, row3_bytes, row4_bytes, row5_bytes]; let mut cache = TopNCache::::new(1, 1, data_types); diff --git a/src/utils/pgwire/src/pg_server.rs b/src/utils/pgwire/src/pg_server.rs index 9425781f5400..e545c8a2d724 100644 --- a/src/utils/pgwire/src/pg_server.rs +++ b/src/utils/pgwire/src/pg_server.rs @@ -130,7 +130,7 @@ pub struct ExecContext { /// `ExecContextGuard` holds a `Arc` pointer. Once `ExecContextGuard` is dropped, /// the inner `Arc` should not be referred anymore, so that its `Weak` reference (used in `SessionImpl`) will be the same lifecycle of the running sql execution context. -pub struct ExecContextGuard(Arc); +pub struct ExecContextGuard(#[allow(dead_code)] Arc); impl ExecContextGuard { pub fn new(exec_context: Arc) -> Self { diff --git a/src/utils/runtime/src/logger.rs b/src/utils/runtime/src/logger.rs index 6eab94af6c48..6bacbe28d264 100644 --- a/src/utils/runtime/src/logger.rs +++ b/src/utils/runtime/src/logger.rs @@ -216,6 +216,7 @@ pub fn init_risingwave_logger(settings: LoggerSettings) { .with_target("sled", Level::INFO) .with_target("cranelift", Level::INFO) .with_target("wasmtime", Level::INFO) + .with_target("sqlx", Level::WARN) // Expose hyper connection socket addr log. .with_target("hyper::client::connect::http", Level::DEBUG);