diff --git a/.github/workflows/cherry-pick-to-release-branch.yml b/.github/workflows/cherry-pick-to-release-branch.yml index ed6946e8c2f11..06ed9dbd2672b 100644 --- a/.github/workflows/cherry-pick-to-release-branch.yml +++ b/.github/workflows/cherry-pick-to-release-branch.yml @@ -22,3 +22,19 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + release_pull_request_1_2: + if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-v1.2') && github.event.pull_request.merged == true" + runs-on: ubuntu-latest + name: release_pull_request + steps: + - name: checkout + uses: actions/checkout@v1 + - name: Create PR to branch + uses: risingwavelabs/github-action-cherry-pick@master + with: + pr_branch: 'v1.2-rc' + pr_labels: 'cherry-pick' + pr_body: ${{ format('Cherry picking \#{0} onto branch v1.2-rc', github.event.number) }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + diff --git a/Cargo.lock b/Cargo.lock index af36a2c299d37..eebf31d098f16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -326,7 +326,7 @@ dependencies = [ "arrow-cast", "arrow-ipc", "arrow-schema", - "base64 0.21.2", + "base64 0.21.3", "bytes", "futures", "paste", @@ -448,7 +448,7 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8257238e2a3629ee5618502a75d1b91f8017c24638c75349fc8d2d80cf1f7c4c" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "bytes", "futures", "http", @@ -1082,9 +1082,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.2" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "base64-simd" @@ -1482,6 +1482,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg-or-panic" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf85d5384815558275789d91d1895d1d9919a6e2534d6144650f036ac65691a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + [[package]] name = "chrono" version = "0.4.26" @@ -1577,33 +1588,31 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.1" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27" +checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.4.1" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d" +checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" dependencies = [ "anstream", "anstyle", "clap_lex", - "once_cell", "strsim 0.10.0", ] [[package]] name = "clap_derive" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ "heck 0.4.1", "proc-macro2", @@ -1914,7 +1923,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.1", + "clap 4.4.2", "criterion-plot", "futures", "is-terminal", @@ -3224,7 +3233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "931bedb2264cb00f914b0a6a5c304e34865c34306632d3932e0951a073e4a67d" dependencies = [ "async-trait", - "base64 0.21.2", + "base64 0.21.3", "google-cloud-metadata", "google-cloud-token", "home", @@ -3905,7 +3914,7 @@ version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "pem", "ring", "serde", @@ -4058,7 +4067,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d8de370f98a6cb8a4606618e53e802f93b094ddec0f96988eaec2c27e6e9ce7" dependencies = [ - "clap 4.4.1", + "clap 4.4.2", "termcolor", "threadpool", ] @@ -4404,9 +4413,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.6.1" +version = "2.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f478948fd84d9f8e86967bf432640e46adfb5a4bd4f14ef7e864ab38220534ae" +checksum = "5486aed0026218e61b8a01d5fbd5a0a134649abb71a0e53b7bc088529dced86e" [[package]] name = "memcomparable" @@ -4966,7 +4975,7 @@ dependencies = [ "async-compat", "async-trait", "backon", - "base64 0.21.2", + "base64 0.21.3", "bytes", "chrono", "flagset", @@ -5018,11 +5027,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.56" +version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "cfg-if", "foreign-types", "libc", @@ -5292,7 +5301,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.21.2", + "base64 0.21.3", "brotli", "bytes", "chrono", @@ -5665,7 +5674,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "byteorder", "bytes", "fallible-iterator", @@ -5966,9 +5975,9 @@ dependencies = [ [[package]] name = "prost-reflect" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "000e1e05ebf7b26e1eba298e66fe4eee6eb19c567d0ffb35e0dd34231cdac4c8" +checksum = "6b823de344848e011658ac981009100818b322421676740546f8b52ed5249428" dependencies = [ "once_cell", "prost", @@ -6234,9 +6243,9 @@ dependencies = [ [[package]] name = "redis" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd6543a7bc6428396845f6854ccf3d1ae8823816592e2cbe74f20f50f209d02" +checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba" dependencies = [ "combine", "itoa", @@ -6326,7 +6335,7 @@ checksum = "3228e570df74d69d3d3236a71371f1edd748a3e4eb728ea1f29d403bc10fc727" dependencies = [ "anyhow", "async-trait", - "base64 0.21.2", + "base64 0.21.3", "chrono", "form_urlencoded", "hex", @@ -6351,11 +6360,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "bytes", "encoding_rs", "futures-core", @@ -6390,7 +6399,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.22.6", + "webpki-roots 0.25.2", "winreg", ] @@ -6424,7 +6433,7 @@ version = "1.1.0-alpha" dependencies = [ "anyhow", "chrono", - "clap 4.4.1", + "clap 4.4.2", "console", "fs-err", "glob", @@ -6452,7 +6461,7 @@ name = "risedev-config" version = "1.1.0-alpha" dependencies = [ "anyhow", - "clap 4.4.1", + "clap 4.4.2", "console", "dialoguer", "enum-iterator", @@ -6484,7 +6493,7 @@ dependencies = [ name = "risingwave_backup_cmd" version = "1.1.0-alpha" dependencies = [ - "clap 4.4.1", + "clap 4.4.2", "madsim-tokio", "prometheus", "risingwave_backup", @@ -6547,7 +6556,7 @@ dependencies = [ "bcc", "bytes", "bytesize", - "clap 4.4.1", + "clap 4.4.2", "futures", "hdrhistogram", "itertools 0.11.0", @@ -6573,7 +6582,7 @@ dependencies = [ name = "risingwave_cmd" version = "1.1.0-alpha" dependencies = [ - "clap 4.4.1", + "clap 4.4.2", "madsim-tokio", "prometheus", "risingwave_common", @@ -6594,7 +6603,7 @@ name = "risingwave_cmd_all" version = "1.1.0-alpha" dependencies = [ "anyhow", - "clap 4.4.1", + "clap 4.4.2", "console", "const-str", "expect-test", @@ -6636,7 +6645,7 @@ dependencies = [ "bytes", "chrono", "chrono-tz", - "clap 4.4.1", + "clap 4.4.2", "comfy-table", "crc32fast", "criterion", @@ -6749,7 +6758,7 @@ dependencies = [ "anyhow", "async-trait", "bytes", - "clap 4.4.1", + "clap 4.4.2", "futures", "madsim-tokio", "prometheus", @@ -6775,7 +6784,7 @@ dependencies = [ "anyhow", "async-trait", "await-tree", - "clap 4.4.1", + "clap 4.4.2", "madsim-tokio", "madsim-tonic", "parking_lot 0.12.1", @@ -6800,7 +6809,7 @@ dependencies = [ "async-trait", "await-tree", "chrono", - "clap 4.4.1", + "clap 4.4.2", "either", "futures", "futures-async-stream", @@ -6850,7 +6859,7 @@ dependencies = [ "aws-sdk-s3", "aws-smithy-http", "aws-types", - "base64 0.21.2", + "base64 0.21.3", "bincode 1.3.3", "byteorder", "bytes", @@ -6913,7 +6922,7 @@ dependencies = [ "anyhow", "bytes", "chrono", - "clap 4.4.1", + "clap 4.4.2", "comfy-table", "futures", "inquire", @@ -6946,7 +6955,7 @@ version = "1.1.0-alpha" dependencies = [ "anyhow", "chrono", - "clap 4.4.1", + "clap 4.4.2", "madsim-tokio", "pg_interval", "rust_decimal", @@ -7025,7 +7034,7 @@ dependencies = [ "auto_enums", "bk-tree", "bytes", - "clap 4.4.1", + "clap 4.4.2", "downcast-rs", "dyn-clone", "easy-ext", @@ -7098,7 +7107,7 @@ version = "1.1.0-alpha" dependencies = [ "async-trait", "bytes", - "clap 4.4.1", + "clap 4.4.2", "criterion", "expect-test", "fail", @@ -7180,7 +7189,7 @@ dependencies = [ "aws-sdk-ec2", "axum", "bytes", - "clap 4.4.1", + "clap 4.4.2", "crepe", "easy-ext", "either", @@ -7303,7 +7312,7 @@ name = "risingwave_regress_test" version = "1.1.0-alpha" dependencies = [ "anyhow", - "clap 4.4.1", + "clap 4.4.2", "madsim-tokio", "path-absolutize", "similar", @@ -7374,7 +7383,8 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "clap 4.4.1", + "cfg-or-panic", + "clap 4.4.2", "console", "futures", "glob", @@ -7469,7 +7479,7 @@ version = "1.1.0-alpha" dependencies = [ "anyhow", "chrono", - "clap 4.4.1", + "clap 4.4.2", "expect-test", "itertools 0.11.0", "libtest-mimic", @@ -7494,7 +7504,7 @@ name = "risingwave_state_cleaning_test" version = "1.1.0-alpha" dependencies = [ "anyhow", - "clap 4.4.1", + "clap 4.4.2", "futures", "madsim-tokio", "prometheus", @@ -7855,7 +7865,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", ] [[package]] @@ -8203,7 +8213,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ca3b16a3d82c4088f343b7480a93550b3eabe1a358569c2dfe38bbcead07237" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "chrono", "hex", "indexmap 1.9.3", @@ -8802,9 +8812,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sysinfo" -version = "0.29.8" +version = "0.29.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d10ed79c22663a35a255d289a7fdcb43559fc77ff15df5ce6c341809e7867528" +checksum = "a8d0e9cc2273cc8d31377bdd638d72e3ac3e5607b18621062b169d02787f1bab" dependencies = [ "cfg-if", "core-foundation-sys", @@ -9240,7 +9250,7 @@ dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.2", + "base64 0.21.3", "bytes", "flate2", "futures-core", @@ -9572,9 +9582,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -9778,9 +9788,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "wasm-streams" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" dependencies = [ "futures-util", "js-sys", @@ -9811,21 +9821,18 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.6" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" dependencies = [ - "webpki", + "rustls-webpki 0.100.2", ] [[package]] name = "webpki-roots" -version = "0.23.1" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" -dependencies = [ - "rustls-webpki 0.100.2", -] +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" [[package]] name = "which" @@ -10021,11 +10028,12 @@ dependencies = [ [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi", + "cfg-if", + "windows-sys 0.48.0", ] [[package]] @@ -10053,7 +10061,7 @@ dependencies = [ "bytes", "cc", "chrono", - "clap 4.4.1", + "clap 4.4.2", "clap_builder", "combine", "crossbeam-epoch", diff --git a/Makefile.toml b/Makefile.toml index 1faf715e984a7..1765307fe691a 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -671,7 +671,7 @@ script = """ set -e export CARGO_TARGET_DIR=target/coverage -cargo llvm-cov nextest --html +cargo llvm-cov nextest --html --workspace --exclude risingwave_simulation """ description = "Run unit tests and report coverage" @@ -682,7 +682,7 @@ script = """ #!/usr/bin/env bash set -e -cargo nextest run "$@" +cargo nextest run --workspace --exclude risingwave_simulation "$@" """ description = "🌟 Run unit tests" diff --git a/README.md b/README.md index 9daa030431951..d63369b0200a6 100644 --- a/README.md +++ b/README.md @@ -18,14 +18,14 @@ RisingWave ingests data from sources like Apache Kafka, Apache Pulsar, Amazon Ki RisingWave 1.0 is a battle-tested version that has undergone rigorous stress tests and performance evaluations. It has proven its reliability and efficiency through successful deployments in numerous production environments across dozens of companies. -Learn more at [Introduction to RisingWave](https://www.risingwave.dev/docs/current/intro/). +Learn more at [Introduction to RisingWave](https://docs.risingwave.com/docs/current/intro/). ![RisingWave](https://github.com/risingwavelabs/risingwave-docs/blob/0f7e1302b22493ba3c1c48e78810750ce9a5ff42/docs/images/archi_simple.png) ## RisingWave Cloud -RisingWave Cloud is a fully-managed and scalable stream processing platform powered by the open-source RisingWave project. Try it out for free at: [risingwave.com/cloud](https://risingwave.com/cloud). +RisingWave Cloud is a fully-managed and scalable stream processing platform powered by the open-source RisingWave project. Try it out for free at: [cloud.risingwave.com](https://cloud.risingwave.com). ## Notes on telemetry @@ -33,15 +33,15 @@ RisingWave collects anonymous usage statistics to better understand how the comm ## Get started -- To learn about how to install and run RisingWave, see [Get started](https://www.risingwave.dev/docs/current/get-started/). -- To learn about how to ingest data and the supported data sources, see [Sources](https://www.risingwave.dev/docs/current/data-ingestion/). -- To learn about how to transform data using the PostgreSQL-compatible SQL of RisingWave, see [SQL reference](https://www.risingwave.dev/docs/current/sql-references/). -- To learn about how to deliver data and the supported data sinks, see [Sinks](https://www.risingwave.dev/docs/current/data-delivery/). -- To learn about new features and changes in the current and previous versions, see [Release notes](https://www.risingwave.dev/docs/current/release-notes/). +- To learn about how to install and run RisingWave, see [Get started](https://docs.risingwave.com/docs/current/get-started/). +- To learn about how to ingest data and the supported data sources, see [Sources](https://docs.risingwave.com/docs/current/data-ingestion/). +- To learn about how to transform data using the PostgreSQL-compatible SQL of RisingWave, see [SQL reference](https://docs.risingwave.com/docs/current/sql-references/). +- To learn about how to deliver data and the supported data sinks, see [Sinks](https://docs.risingwave.com/docs/current/data-delivery/). +- To learn about new features and changes in the current and previous versions, see [Release notes](https://docs.risingwave.com/release-notes/). ## Documentation -To learn about how to use RisingWave, refer to [RisingWave User Documentation](https://www.risingwave.dev/). To learn about the development process, see the [developer guide](docs/developer-guide.md). To understand the design and implementation of RisingWave, refer to the design docs listed in [readme.md](docs/README.md). +To learn about how to use RisingWave, refer to [RisingWave User Documentation](https://docs.risingwave.com/). To learn about the development process, see the [developer guide](docs/developer-guide.md). To understand the design and implementation of RisingWave, refer to the design docs listed in [readme.md](docs/README.md). ## Community diff --git a/ci/scripts/run-unit-test.sh b/ci/scripts/run-unit-test.sh index f5afdc017e52c..4ba534559ee9f 100755 --- a/ci/scripts/run-unit-test.sh +++ b/ci/scripts/run-unit-test.sh @@ -12,8 +12,8 @@ cd ${REPO_ROOT} echo "+++ Run unit tests with coverage" # use tee to disable progress bar -NEXTEST_PROFILE=ci cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints,sync_point 2> >(tee); +NEXTEST_PROFILE=ci cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints,sync_point --workspace --exclude risingwave_simulation 2> >(tee); echo "--- Codecov upload coverage reports" curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov -./codecov -t "$CODECOV_TOKEN" -s . -F rust \ No newline at end of file +./codecov -t "$CODECOV_TOKEN" -s . -F rust diff --git a/e2e_test/batch/basic/func.slt.part b/e2e_test/batch/basic/func.slt.part index 2936991a8d360..31ae11c1226d1 100644 --- a/e2e_test/batch/basic/func.slt.part +++ b/e2e_test/batch/basic/func.slt.part @@ -544,4 +544,61 @@ XXX query T select regexp_replace('💩💩💩💩💩foo🤔️bar亲爱的😭baz这不是爱情❤️‍🔥', 'baz(...)', '这是🥵', 'ic'); ---- -💩💩💩💩💩foo🤔️bar亲爱的😭这是🥵爱情❤️‍🔥 \ No newline at end of file +💩💩💩💩💩foo🤔️bar亲爱的😭这是🥵爱情❤️‍🔥 + +query T +select regexp_count('ABCABCAXYaxy', 'A.'); +---- +3 + +query T +select regexp_count('ABCABCAX🤔️axy', 'A.'); +---- +3 + +query T +select regexp_count('ABCABCAXYaxy', 'A.', 1, 'i'); +---- +4 + +query T +select regexp_count('ABCABCAbXYaxy', 'AB'); +---- +2 + +query T +select regexp_count('ABCABCAbXYaxy', 'AB', 1, 'i'); +---- +3 + +query T +select regexp_count('ABCABCAXYaxy', '[a-c]', 1, 'i'); +---- +8 + +query T +select regexp_count('123ABC#456', '\d{3}'); +---- +2 + +query T +select regexp_count('foobarbaz', 'b..', 3); +---- +2 + +query error Expected start to be a Unsigned Int32 +select regexp_count('foobarbaz', 'b..', 'g'); + +query error Expected start to be a Unsigned Int32 +select regexp_count('foobarbaz', 'b..', 'i'); + +query error invalid regular expression option: "a" +select regexp_count('foobarbaz', 'b..', 3, 'a'); + +query error `regexp_count` does not support global flag option +select regexp_count('foobar', 'b..', 1, 'g'); + +query T +select regexp_count('💩💩💩💩💩foo🤔️bar亲爱的😭baz这不是爱情❤️‍🔥', '亲爱的😭', 1, 'i'); +---- +1 \ No newline at end of file diff --git a/e2e_test/sink/kafka/debezium1.result b/e2e_test/sink/kafka/debezium1.result index d65c39be72a38..6c728e55346d9 100644 --- a/e2e_test/sink/kafka/debezium1.result +++ b/e2e_test/sink/kafka/debezium1.result @@ -1,10 +1,10 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} diff --git a/e2e_test/sink/kafka/debezium2.result b/e2e_test/sink/kafka/debezium2.result index 47dd9fb96fdad..3012648769070 100644 --- a/e2e_test/sink/kafka/debezium2.result +++ b/e2e_test/sink/kafka/debezium2.result @@ -1,11 +1,11 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"before":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"op":"u","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683857916842},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820372},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"before":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"op":"u","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549594077},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} diff --git a/e2e_test/sink/kafka/debezium3.result b/e2e_test/sink/kafka/debezium3.result index a6157e518f0b4..de658fe7157f2 100644 --- a/e2e_test/sink/kafka/debezium3.result +++ b/e2e_test/sink/kafka/debezium3.result @@ -1,13 +1,13 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} null -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":null,"before":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"op":"d","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381221481},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"before":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"op":"u","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381191421},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176934},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} null +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":null,"before":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"op":"d","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549624995},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"before":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"op":"u","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549594077},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}} {"payload":{"after":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}} diff --git a/e2e_test/source/cdc/cdc.load.slt b/e2e_test/source/cdc/cdc.load.slt index bd2472f9947a4..39e99b39df4fb 100644 --- a/e2e_test/source/cdc/cdc.load.slt +++ b/e2e_test/source/cdc/cdc.load.slt @@ -1,5 +1,9 @@ # CDC source basic test +# enable cdc backfill in ci +statement ok +set cdc_backfill='true'; + statement ok create table products ( id INT, name STRING, diff --git a/proto/catalog.proto b/proto/catalog.proto index 25e311cd9410f..aa09274a0d818 100644 --- a/proto/catalog.proto +++ b/proto/catalog.proto @@ -114,6 +114,8 @@ message Sink { optional uint32 connection_id = 14; optional uint64 initialized_at_epoch = 15; optional uint64 created_at_epoch = 16; + string db_name = 17; + string sink_from_name = 18; } message Connection { diff --git a/proto/connector_service.proto b/proto/connector_service.proto index 3704897ecd595..54728bf4ecdec 100644 --- a/proto/connector_service.proto +++ b/proto/connector_service.proto @@ -24,6 +24,8 @@ message SinkParam { map properties = 2; TableSchema table_schema = 3; catalog.SinkType sink_type = 4; + string db_name = 5; + string sink_from_name = 6; } enum SinkPayloadFormat { diff --git a/proto/expr.proto b/proto/expr.proto index 01830afe6c6ca..1b3aeff6480ff 100644 --- a/proto/expr.proto +++ b/proto/expr.proto @@ -103,6 +103,7 @@ message ExprNode { OVERLAY = 231; REGEXP_MATCH = 232; REGEXP_REPLACE = 280; + REGEXP_COUNT = 281; POW = 233; EXP = 234; CHR = 235; diff --git a/proto/meta.proto b/proto/meta.proto index b5f3b7ce7ec5f..5c128630f3336 100644 --- a/proto/meta.proto +++ b/proto/meta.proto @@ -495,6 +495,7 @@ message SystemParams { optional bool telemetry_enabled = 10; optional uint32 parallel_compact_size_mb = 11; optional uint32 max_concurrent_creating_streaming_jobs = 12; + optional bool pause_on_next_bootstrap = 13; } message GetSystemParamsRequest {} diff --git a/proto/stream_plan.proto b/proto/stream_plan.proto index b5fa5d7074c55..691b338312c98 100644 --- a/proto/stream_plan.proto +++ b/proto/stream_plan.proto @@ -24,6 +24,9 @@ message AddMutation { // We may embed a source change split mutation here. // TODO: we may allow multiple mutations in a single barrier. map actor_splits = 2; + // We may embed a pause mutation here. + // TODO: we may allow multiple mutations in a single barrier. + bool pause = 4; } message StopMutation { @@ -171,6 +174,10 @@ message SinkDesc { map properties = 8; catalog.SinkType sink_type = 9; repeated plan_common.ColumnCatalog column_catalogs = 10; + string db_name = 11; + // If the sink is from table or mv, this is name of the table/mv. Otherwise + // it is the name of the sink itself. + string sink_from_name = 12; } enum SinkLogStoreType { diff --git a/scripts/check/check-trailing-spaces.sh b/scripts/check/check-trailing-spaces.sh index 4876a2265e7d9..b2aba451365c6 100755 --- a/scripts/check/check-trailing-spaces.sh +++ b/scripts/check/check-trailing-spaces.sh @@ -3,6 +3,8 @@ # Exits as soon as any line fails. set -euo pipefail +self=$0 + # Shell colors RED='\033[0;31m' BLUE='\033[0;34m' @@ -11,8 +13,12 @@ ORANGE='\033[0;33m' BOLD='\033[1m' NONE='\033[0m' -_echo_err() { - echo -e "${RED}$@${NONE}" +print_help() { + echo "Usage: $self [-f|--fix]" + echo + echo "Options:" + echo " -f, --fix Fix trailing spaces." + echo " -h, --help Show this help message and exit." } fix=false @@ -21,42 +27,42 @@ while [ $# -gt 0 ]; do -f | --fix) fix=true ;; + -h | --help) + print_help + exit 0 + ;; *) - _echo_err "$self: invalid option \`$1\`\n" + echo -e "${RED}${BOLD}$self: invalid option \`$1\`\n${NONE}" + print_help exit 1 ;; esac shift done -# The following is modified from https://github.com/raisedevs/find-trailing-whitespace/blob/restrict-to-plaintext-only/entrypoint.sh. +temp_file=$(mktemp) -has_trailing_spaces=false +echo -ne "${BLUE}" +git grep -nIP --untracked '[[:space:]]+$' -- ':!src/tests/regress/data' | tee $temp_file || true +echo -ne "${NONE}" -for file in $(git grep --cached -Il '' -- ':!src/tests/regress/data'); do - lines=$(grep -E -rnIH "[[:space:]]+$" "$file" | cut -f-2 -d ":" || echo "") - if [ ! -z "$lines" ]; then - if [[ $has_trailing_spaces == false ]]; then - echo -e "\nLines containing trailing whitespace:\n" - has_trailing_spaces=true - fi - if [[ $fix == true ]]; then +bad_files=$(cat $temp_file | cut -f1 -d ':' | sort -u) +rm $temp_file + +if [ ! -z "$bad_files" ]; then + if [[ $fix == true ]]; then + for file in $bad_files; do sed -i '' -e's/[[:space:]]*$//' "$file" - fi - echo -e "${BLUE}$lines${NONE}" - fi -done + done -if [[ $has_trailing_spaces == true ]]; then - if [[ $fix == false ]]; then echo - echo -e "${RED}${BOLD}Please clean all the trailing spaces.${NONE}" - echo -e "${BOLD}You can run 'scripts/check-trailing-spaces.sh --fix' for convenience.${NONE}" - exit 1 + echo -e "${GREEN}${BOLD}All trailing spaces listed above have been cleaned.${NONE}" + exit 0 else echo - echo -e "${GREEN}${BOLD}All trailing spaces have been cleaned.${NONE}" - exit 0 + echo -e "${RED}${BOLD}Please clean all the trailing spaces listed above.${NONE}" + echo -e "${BOLD}You can run '$self --fix' for convenience.${NONE}" + exit 1 fi else echo -e "${GREEN}${BOLD}No trailing spaces found.${NONE}" diff --git a/src/batch/src/task/task_manager.rs b/src/batch/src/task/task_manager.rs index dd2d0994e30a5..f84cb842eba9e 100644 --- a/src/batch/src/task/task_manager.rs +++ b/src/batch/src/task/task_manager.rs @@ -436,34 +436,6 @@ mod tests { assert!(!manager.tasks.lock().contains_key(&task_id)); } - #[tokio::test] - async fn test_task_cancel_for_block() { - let manager = Arc::new(BatchManager::new( - BatchConfig::default(), - BatchManagerMetrics::for_test(), - )); - let plan = PlanFragment { - root: Some(PlanNode { - children: vec![], - identity: "".to_string(), - node_body: Some(NodeBody::BlockExecutor(true)), - }), - exchange_info: Some(ExchangeInfo { - mode: DistributionMode::Single as i32, - distribution: None, - }), - }; - let task_id = PbTaskId { - query_id: "".to_string(), - stage_id: 0, - task_id: 0, - }; - manager.fire_task_for_test(&task_id, plan).await.unwrap(); - manager.cancel_task(&task_id); - let task_id = TaskId::from(&task_id); - assert!(!manager.tasks.lock().contains_key(&task_id)); - } - #[tokio::test] async fn test_task_abort_for_busy_loop() { let manager = Arc::new(BatchManager::new( @@ -496,39 +468,4 @@ mod tests { .abort("Abort Test".to_owned()); assert!(manager.wait_until_task_aborted(&task_id).await.is_ok()); } - - /// https://github.com/risingwavelabs/risingwave/issues/11979 - #[ignore] - #[tokio::test] - async fn test_task_abort_for_block() { - let manager = Arc::new(BatchManager::new( - BatchConfig::default(), - BatchManagerMetrics::for_test(), - )); - let plan = PlanFragment { - root: Some(PlanNode { - children: vec![], - identity: "".to_string(), - node_body: Some(NodeBody::BlockExecutor(true)), - }), - exchange_info: Some(ExchangeInfo { - mode: DistributionMode::Single as i32, - distribution: None, - }), - }; - let task_id = PbTaskId { - query_id: "".to_string(), - stage_id: 0, - task_id: 0, - }; - manager.fire_task_for_test(&task_id, plan).await.unwrap(); - let task_id = TaskId::from(&task_id); - manager - .tasks - .lock() - .get(&task_id) - .unwrap() - .abort("Abort Test".to_owned()); - assert!(manager.wait_until_task_aborted(&task_id).await.is_ok()); - } } diff --git a/src/common/src/config.rs b/src/common/src/config.rs index 5025d43e5e7fe..85760db453b67 100644 --- a/src/common/src/config.rs +++ b/src/common/src/config.rs @@ -775,6 +775,10 @@ pub struct SystemConfig { /// Max number of concurrent creating streaming jobs. #[serde(default = "default::system::max_concurrent_creating_streaming_jobs")] pub max_concurrent_creating_streaming_jobs: Option, + + /// Whether to pause all data sources on next bootstrap. + #[serde(default = "default::system::pause_on_next_bootstrap")] + pub pause_on_next_bootstrap: Option, } impl SystemConfig { @@ -792,6 +796,7 @@ impl SystemConfig { backup_storage_directory: self.backup_storage_directory, telemetry_enabled: self.telemetry_enabled, max_concurrent_creating_streaming_jobs: self.max_concurrent_creating_streaming_jobs, + pause_on_next_bootstrap: self.pause_on_next_bootstrap, } } } @@ -1165,55 +1170,7 @@ pub mod default { } pub mod system { - use crate::system_param; - - pub fn barrier_interval_ms() -> Option { - system_param::default::barrier_interval_ms() - } - - pub fn checkpoint_frequency() -> Option { - system_param::default::checkpoint_frequency() - } - - pub fn parallel_compact_size_mb() -> Option { - system_param::default::parallel_compact_size_mb() - } - - pub fn sstable_size_mb() -> Option { - system_param::default::sstable_size_mb() - } - - pub fn block_size_kb() -> Option { - system_param::default::block_size_kb() - } - - pub fn bloom_false_positive() -> Option { - system_param::default::bloom_false_positive() - } - - pub fn state_store() -> Option { - system_param::default::state_store() - } - - pub fn data_directory() -> Option { - system_param::default::data_directory() - } - - pub fn backup_storage_url() -> Option { - system_param::default::backup_storage_url() - } - - pub fn backup_storage_directory() -> Option { - system_param::default::backup_storage_directory() - } - - pub fn telemetry_enabled() -> Option { - system_param::default::telemetry_enabled() - } - - pub fn max_concurrent_creating_streaming_jobs() -> Option { - system_param::default::max_concurrent_creating_streaming_jobs() - } + pub use crate::system_param::default::*; } pub mod batch { diff --git a/src/common/src/row/project.rs b/src/common/src/row/project.rs index 06a2bc6fe5f1a..ab4be6d229243 100644 --- a/src/common/src/row/project.rs +++ b/src/common/src/row/project.rs @@ -60,9 +60,10 @@ impl<'i, R: Row> Project<'i, R> { pub(crate) fn new(row: R, indices: &'i [usize]) -> Self { if let Some(index) = indices.iter().find(|&&i| i >= row.len()) { panic!( - "index {} out of bounds for row of length {}", + "index {} out of bounds for row of length {}, row {:?}", index, - row.len() + row.len(), + row ); } Self { row, indices } diff --git a/src/common/src/session_config/mod.rs b/src/common/src/session_config/mod.rs index 40f7aa7451bd9..0b757bf40eeba 100644 --- a/src/common/src/session_config/mod.rs +++ b/src/common/src/session_config/mod.rs @@ -334,7 +334,7 @@ type LockTimeout = ConfigI32; type RowSecurity = ConfigBool; type StandardConformingStrings = ConfigString; type StreamingRateLimit = ConfigU64; -type CdcBackfill = ConfigBool; +type CdcBackfill = ConfigBool; /// Report status or notice to caller. pub trait ConfigReporter { diff --git a/src/common/src/system_param/mod.rs b/src/common/src/system_param/mod.rs index b7c1f7a65c886..ccb666367d729 100644 --- a/src/common/src/system_param/mod.rs +++ b/src/common/src/system_param/mod.rs @@ -56,7 +56,8 @@ macro_rules! for_all_undeprecated_params { { backup_storage_directory, String, Some("backup".to_string()), false }, { telemetry_enabled, bool, Some(true), true }, { max_concurrent_creating_streaming_jobs, u32, Some(1_u32), true }, - $({ $field, $type, $default },)* + { pause_on_next_bootstrap, bool, Some(false), true }, + $({ $field, $type, $default, $is_mutable },)* } }; } @@ -67,7 +68,7 @@ macro_rules! for_all_params { ($macro:ident) => { for_all_undeprecated_params!( $macro /* Define future deprecated params here, such as - * ,{ backup_storage_directory, String, "backup".to_string() } */ + * ,{ backup_storage_directory, String, "backup".to_string(), true } */ ); }; } @@ -370,6 +371,7 @@ mod tests { (BACKUP_STORAGE_DIRECTORY_KEY, "a"), (TELEMETRY_ENABLED_KEY, "false"), (MAX_CONCURRENT_CREATING_STREAMING_JOBS_KEY, "1"), + (PAUSE_ON_NEXT_BOOTSTRAP_KEY, "false"), ]; // To kv - missing field. diff --git a/src/common/src/system_param/reader.rs b/src/common/src/system_param/reader.rs index 63954d53f9d52..91cc893823b66 100644 --- a/src/common/src/system_param/reader.rs +++ b/src/common/src/system_param/reader.rs @@ -80,6 +80,10 @@ impl SystemParamsReader { self.prost.telemetry_enabled.unwrap() } + pub fn pause_on_next_bootstrap(&self) -> bool { + self.prost.pause_on_next_bootstrap.unwrap_or(false) + } + pub fn to_kv(&self) -> Vec<(String, String)> { system_params_to_kv(&self.prost).unwrap() } diff --git a/src/common/src/util/chunk_coalesce.rs b/src/common/src/util/chunk_coalesce.rs index 0a8d256a92de9..3c5e4b109da90 100644 --- a/src/common/src/util/chunk_coalesce.rs +++ b/src/common/src/util/chunk_coalesce.rs @@ -165,14 +165,7 @@ impl DataChunkBuilder { } } - /// Append one row from the given [`Row`]. - /// The caller is responsible to build the data chunk, - /// i.e. call the `build_data_chunk` function. - /// Used when caller wants to decide when to `build_data_chunk`. - /// For instance, in backfill, we want to build data chunk when: - /// 1. Buffer is full (typical case) - /// 2. On barrier, to flush the remaining data in `data_chunk_builder`. - pub fn append_one_row_no_finish(&mut self, row: impl Row) { + fn append_one_row_no_finish(&mut self, row: impl Row) { assert!(self.buffered_count < self.batch_size); self.ensure_builders(); self.do_append_one_row_from_datums(row.iter()); @@ -212,7 +205,7 @@ impl DataChunkBuilder { } } - pub fn build_data_chunk(&mut self) -> DataChunk { + fn build_data_chunk(&mut self) -> DataChunk { let mut finished_array_builders = vec![]; swap(&mut finished_array_builders, &mut self.array_builders); let cardinality = self.buffered_count; @@ -236,6 +229,17 @@ impl DataChunkBuilder { pub fn data_types(&self) -> Vec { self.data_types.clone() } + + pub fn is_empty(&self) -> bool { + self.buffered_count == 0 + } + + pub fn clear(&mut self) { + if !self.is_empty() { + self.array_builders.clear() + } + self.buffered_count = 0; + } } impl Drop for DataChunkBuilder { diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs index 56245f52c3a9a..ad3817df74507 100644 --- a/src/compute/src/server.rs +++ b/src/compute/src/server.rs @@ -225,13 +225,16 @@ pub async fn compute_node_serve( compaction_executor: Arc::new(CompactionExecutor::new(Some(1))), filter_key_extractor_manager: storage.filter_key_extractor_manager().clone(), memory_limiter, - sstable_object_id_manager: storage.sstable_object_id_manager().clone(), + task_progress_manager: Default::default(), await_tree_reg: None, running_task_count: Arc::new(AtomicU32::new(0)), }); - let (handle, shutdown_sender) = start_compactor(compactor_context); + let (handle, shutdown_sender) = start_compactor( + compactor_context, + storage.sstable_object_id_manager().clone(), + ); sub_tasks.push((handle, shutdown_sender)); } let flush_limiter = storage.get_memory_limiter(); diff --git a/src/config/example.toml b/src/config/example.toml index 2a0ac38e7c564..2ac774104af4c 100644 --- a/src/config/example.toml +++ b/src/config/example.toml @@ -151,3 +151,4 @@ backup_storage_url = "memory" backup_storage_directory = "backup" telemetry_enabled = true max_concurrent_creating_streaming_jobs = 1 +pause_on_next_bootstrap = false diff --git a/src/connector/Cargo.toml b/src/connector/Cargo.toml index 9fb93bad28aed..f19f027cf3033 100644 --- a/src/connector/Cargo.toml +++ b/src/connector/Cargo.toml @@ -62,7 +62,7 @@ opendal = "0.39" parking_lot = "0.12" prometheus = { version = "0.13", features = ["process"] } prost = { version = "0.11.9", features = ["no-recursion-limit"] } -prost-reflect = "0.11.4" +prost-reflect = "0.11.5" protobuf-native = "0.2.1" pulsar = { version = "6.0", default-features = false, features = [ "tokio-runtime", diff --git a/src/connector/src/sink/catalog/desc.rs b/src/connector/src/sink/catalog/desc.rs index 1e8aa6a068739..f00709188d23c 100644 --- a/src/connector/src/sink/catalog/desc.rs +++ b/src/connector/src/sink/catalog/desc.rs @@ -54,6 +54,13 @@ pub struct SinkDesc { // based on both its own derivation on the append-only attribute and other user-specified // options in `properties`. pub sink_type: SinkType, + + /// Name of the database + pub db_name: String, + + /// Name of the "table" field for Debezium. If the sink is from table or mv, + /// it is the name of table/mv. Otherwise, it is the name of the sink. + pub sink_from_name: String, } impl SinkDesc { @@ -82,6 +89,8 @@ impl SinkDesc { connection_id, created_at_epoch: None, initialized_at_epoch: None, + db_name: self.db_name, + sink_from_name: self.sink_from_name, } } @@ -100,6 +109,8 @@ impl SinkDesc { distribution_key: self.distribution_key.iter().map(|k| *k as _).collect_vec(), properties: self.properties.clone().into_iter().collect(), sink_type: self.sink_type.to_proto() as i32, + db_name: self.db_name.clone(), + sink_from_name: self.sink_from_name.clone(), } } } diff --git a/src/connector/src/sink/catalog/mod.rs b/src/connector/src/sink/catalog/mod.rs index 5cd291dc0ae75..16b53af17499a 100644 --- a/src/connector/src/sink/catalog/mod.rs +++ b/src/connector/src/sink/catalog/mod.rs @@ -150,6 +150,12 @@ pub struct SinkCatalog { pub created_at_epoch: Option, pub initialized_at_epoch: Option, + + /// Name of the database + pub db_name: String, + + /// Name for the table info for Debezium sink + pub sink_from_name: String, } impl SinkCatalog { @@ -183,6 +189,8 @@ impl SinkCatalog { connection_id: self.connection_id.map(|id| id.into()), initialized_at_epoch: self.initialized_at_epoch.map(|e| e.0), created_at_epoch: self.created_at_epoch.map(|e| e.0), + db_name: self.db_name.clone(), + sink_from_name: self.sink_from_name.clone(), } } @@ -257,6 +265,8 @@ impl From for SinkCatalog { connection_id: pb.connection_id.map(ConnectionId), created_at_epoch: pb.created_at_epoch.map(Epoch::from), initialized_at_epoch: pb.initialized_at_epoch.map(Epoch::from), + db_name: pb.db_name, + sink_from_name: pb.sink_from_name, } } } diff --git a/src/connector/src/sink/kafka.rs b/src/connector/src/sink/kafka.rs index f52eb0bfa36f8..4c7fc317edd3e 100644 --- a/src/connector/src/sink/kafka.rs +++ b/src/connector/src/sink/kafka.rs @@ -32,7 +32,8 @@ use serde_json::Value; use serde_with::{serde_as, DisplayFromStr}; use super::{ - Sink, SinkError, SINK_TYPE_APPEND_ONLY, SINK_TYPE_DEBEZIUM, SINK_TYPE_OPTION, SINK_TYPE_UPSERT, + Sink, SinkError, SinkParam, SINK_TYPE_APPEND_ONLY, SINK_TYPE_DEBEZIUM, SINK_TYPE_OPTION, + SINK_TYPE_UPSERT, }; use crate::common::KafkaCommon; use crate::sink::utils::{ @@ -260,20 +261,19 @@ pub struct KafkaSink { schema: Schema, pk_indices: Vec, is_append_only: bool, + db_name: String, + sink_from_name: String, } impl KafkaSink { - pub fn new( - config: KafkaConfig, - schema: Schema, - pk_indices: Vec, - is_append_only: bool, - ) -> Self { + pub fn new(config: KafkaConfig, param: SinkParam) -> Self { Self { config, - schema, - pk_indices, - is_append_only, + schema: param.schema(), + pk_indices: param.pk_indices, + is_append_only: param.sink_type.is_append_only(), + db_name: param.db_name, + sink_from_name: param.sink_from_name, } } } @@ -290,6 +290,8 @@ impl Sink for KafkaSink { self.schema.clone(), self.pk_indices.clone(), self.is_append_only, + self.db_name.clone(), + self.sink_from_name.clone(), format!("sink-{:?}", writer_param.executor_id), ) .await?, @@ -333,6 +335,8 @@ pub struct KafkaSinkWriter { schema: Schema, pk_indices: Vec, is_append_only: bool, + db_name: String, + sink_from_name: String, } impl KafkaSinkWriter { @@ -341,6 +345,8 @@ impl KafkaSinkWriter { schema: Schema, pk_indices: Vec, is_append_only: bool, + db_name: String, + sink_from_name: String, identifier: String, ) -> Result { let inner: FutureProducer = { @@ -376,6 +382,8 @@ impl KafkaSinkWriter { schema, pk_indices, is_append_only, + db_name, + sink_from_name, }) } @@ -430,10 +438,13 @@ impl KafkaSinkWriter { Err((e, rec)) => { err = e; record = rec; - // FIXME: Will there possibly exist other errors? - assert!(err == KafkaError::MessageProduction(RDKafkaErrorCode::QueueFull)); - tokio::time::sleep(self.config.retry_interval).await; - continue; + match err { + KafkaError::MessageProduction(RDKafkaErrorCode::QueueFull) => { + tokio::time::sleep(self.config.retry_interval).await; + continue; + } + _ => break, + } } } } @@ -467,6 +478,8 @@ impl KafkaSinkWriter { chunk, ts_ms, DebeziumAdapterOpts::default(), + &self.db_name, + &self.sink_from_name, ); #[for_await] @@ -727,6 +740,8 @@ mod test { pk_indices, true, "test_sink_1".to_string(), + "test_db".into(), + "test_table".into(), ) .await .unwrap(); @@ -815,8 +830,8 @@ mod test { ]); let json_chunk = chunk_to_json(chunk, &schema).unwrap(); - let schema_json = schema_to_json(&schema); - assert_eq!(schema_json, serde_json::from_str::("{\"fields\":[{\"field\":\"before\",\"fields\":[{\"field\":\"v1\",\"optional\":true,\"type\":\"int32\"},{\"field\":\"v2\",\"optional\":true,\"type\":\"float\"},{\"field\":\"v3\",\"optional\":true,\"type\":\"string\"}],\"name\":\"RisingWave.RisingWave.RisingWave.Key\",\"optional\":true,\"type\":\"struct\"},{\"field\":\"after\",\"fields\":[{\"field\":\"v1\",\"optional\":true,\"type\":\"int32\"},{\"field\":\"v2\",\"optional\":true,\"type\":\"float\"},{\"field\":\"v3\",\"optional\":true,\"type\":\"string\"}],\"name\":\"RisingWave.RisingWave.RisingWave.Key\",\"optional\":true,\"type\":\"struct\"},{\"field\":\"source\",\"fields\":[{\"field\":\"db\",\"optional\":false,\"type\":\"string\"},{\"field\":\"table\",\"optional\":true,\"type\":\"string\"}],\"name\":\"RisingWave.RisingWave.RisingWave.Source\",\"optional\":false,\"type\":\"struct\"},{\"field\":\"op\",\"optional\":false,\"type\":\"string\"},{\"field\":\"ts_ms\",\"optional\":false,\"type\":\"int64\"}],\"name\":\"RisingWave.RisingWave.RisingWave.Envelope\",\"optional\":false,\"type\":\"struct\"}").unwrap()); + let schema_json = schema_to_json(&schema, "test_db", "test_table"); + assert_eq!(schema_json, serde_json::from_str::("{\"fields\":[{\"field\":\"before\",\"fields\":[{\"field\":\"v1\",\"optional\":true,\"type\":\"int32\"},{\"field\":\"v2\",\"optional\":true,\"type\":\"float\"},{\"field\":\"v3\",\"optional\":true,\"type\":\"string\"}],\"name\":\"RisingWave.test_db.test_table.Key\",\"optional\":true,\"type\":\"struct\"},{\"field\":\"after\",\"fields\":[{\"field\":\"v1\",\"optional\":true,\"type\":\"int32\"},{\"field\":\"v2\",\"optional\":true,\"type\":\"float\"},{\"field\":\"v3\",\"optional\":true,\"type\":\"string\"}],\"name\":\"RisingWave.test_db.test_table.Key\",\"optional\":true,\"type\":\"struct\"},{\"field\":\"source\",\"fields\":[{\"field\":\"db\",\"optional\":false,\"type\":\"string\"},{\"field\":\"table\",\"optional\":true,\"type\":\"string\"}],\"name\":\"RisingWave.test_db.test_table.Source\",\"optional\":false,\"type\":\"struct\"},{\"field\":\"op\",\"optional\":false,\"type\":\"string\"},{\"field\":\"ts_ms\",\"optional\":false,\"type\":\"int64\"}],\"name\":\"RisingWave.test_db.test_table.Envelope\",\"optional\":false,\"type\":\"struct\"}").unwrap()); assert_eq!( serde_json::from_str::(&json_chunk[0]).unwrap(), serde_json::from_str::("{\"v1\":0,\"v2\":0.0,\"v3\":{\"v4\":0,\"v5\":0.0}}") diff --git a/src/connector/src/sink/kinesis.rs b/src/connector/src/sink/kinesis.rs index e303731e2afca..4a6f3a789bf14 100644 --- a/src/connector/src/sink/kinesis.rs +++ b/src/connector/src/sink/kinesis.rs @@ -30,6 +30,7 @@ use serde_with::serde_as; use tokio_retry::strategy::{jitter, ExponentialBackoff}; use tokio_retry::Retry; +use super::SinkParam; use crate::common::KinesisCommon; use crate::sink::utils::{ gen_append_only_message_stream, gen_debezium_message_stream, gen_upsert_message_stream, @@ -48,20 +49,19 @@ pub struct KinesisSink { schema: Schema, pk_indices: Vec, is_append_only: bool, + db_name: String, + sink_from_name: String, } impl KinesisSink { - pub fn new( - config: KinesisSinkConfig, - schema: Schema, - pk_indices: Vec, - is_append_only: bool, - ) -> Self { + pub fn new(config: KinesisSinkConfig, param: SinkParam) -> Self { Self { config, - schema, - pk_indices, - is_append_only, + schema: param.schema(), + pk_indices: param.pk_indices, + is_append_only: param.sink_type.is_append_only(), + db_name: param.db_name, + sink_from_name: param.sink_from_name, } } } @@ -100,6 +100,8 @@ impl Sink for KinesisSink { self.schema.clone(), self.pk_indices.clone(), self.is_append_only, + self.db_name.clone(), + self.sink_from_name.clone(), ) .await } @@ -142,6 +144,8 @@ pub struct KinesisSinkWriter { pk_indices: Vec, client: KinesisClient, is_append_only: bool, + db_name: String, + sink_from_name: String, } impl KinesisSinkWriter { @@ -150,6 +154,8 @@ impl KinesisSinkWriter { schema: Schema, pk_indices: Vec, is_append_only: bool, + db_name: String, + sink_from_name: String, ) -> Result { let client = config .common @@ -162,6 +168,8 @@ impl KinesisSinkWriter { pk_indices, client, is_append_only, + db_name, + sink_from_name, }) } @@ -225,6 +233,8 @@ impl KinesisSinkWriter { chunk, ts_ms, DebeziumAdapterOpts::default(), + &self.db_name, + &self.sink_from_name, ); crate::impl_load_stream_write_record!(dbz_stream, self.put_record); diff --git a/src/connector/src/sink/mod.rs b/src/connector/src/sink/mod.rs index 4ebe99009bc90..9e480376ecb41 100644 --- a/src/connector/src/sink/mod.rs +++ b/src/connector/src/sink/mod.rs @@ -74,6 +74,8 @@ pub struct SinkParam { pub columns: Vec, pub pk_indices: Vec, pub sink_type: SinkType, + pub db_name: String, + pub sink_from_name: String, } impl SinkParam { @@ -91,6 +93,8 @@ impl SinkParam { sink_type: SinkType::from_proto( PbSinkType::from_i32(pb_param.sink_type).expect("should be able to convert"), ), + db_name: pb_param.db_name, + sink_from_name: pb_param.sink_from_name, } } @@ -103,6 +107,8 @@ impl SinkParam { pk_indices: self.pk_indices.iter().map(|i| *i as u32).collect(), }), sink_type: self.sink_type.to_proto().into(), + db_name: self.db_name.clone(), + sink_from_name: self.sink_from_name.clone(), } } @@ -125,6 +131,8 @@ impl From for SinkParam { columns, pk_indices: sink_catalog.downstream_pk, sink_type: sink_catalog.sink_type, + db_name: sink_catalog.db_name, + sink_from_name: sink_catalog.sink_from_name, } } } @@ -419,24 +427,9 @@ macro_rules! dispatch_sink { impl SinkImpl { pub fn new(cfg: SinkConfig, param: SinkParam) -> Result { Ok(match cfg { - SinkConfig::Redis(cfg) => SinkImpl::Redis(RedisSink::new( - cfg, - param.schema(), - param.pk_indices, - param.sink_type.is_append_only(), - )?), - SinkConfig::Kafka(cfg) => SinkImpl::Kafka(KafkaSink::new( - *cfg, - param.schema(), - param.pk_indices, - param.sink_type.is_append_only(), - )), - SinkConfig::Kinesis(cfg) => SinkImpl::Kinesis(KinesisSink::new( - *cfg, - param.schema(), - param.pk_indices, - param.sink_type.is_append_only(), - )), + SinkConfig::Redis(cfg) => SinkImpl::Redis(RedisSink::new(cfg, param)?), + SinkConfig::Kafka(cfg) => SinkImpl::Kafka(KafkaSink::new(*cfg, param)), + SinkConfig::Kinesis(cfg) => SinkImpl::Kinesis(KinesisSink::new(*cfg, param)), SinkConfig::Remote(cfg) => SinkImpl::Remote(RemoteSink::new(cfg, param)), SinkConfig::BlackHole => SinkImpl::BlackHole(BlackHoleSink), SinkConfig::ClickHouse(cfg) => SinkImpl::ClickHouse(ClickHouseSink::new( diff --git a/src/connector/src/sink/redis.rs b/src/connector/src/sink/redis.rs index 268a1fdfe9e01..872713c0d902c 100644 --- a/src/connector/src/sink/redis.rs +++ b/src/connector/src/sink/redis.rs @@ -28,7 +28,7 @@ use risingwave_rpc_client::ConnectorClient; use serde_derive::Deserialize; use serde_with::serde_as; -use super::{SinkError, SINK_TYPE_APPEND_ONLY, SINK_TYPE_OPTION, SINK_TYPE_UPSERT}; +use super::{SinkError, SinkParam, SINK_TYPE_APPEND_ONLY, SINK_TYPE_OPTION, SINK_TYPE_UPSERT}; use crate::common::RedisCommon; use crate::sink::{DummySinkCommitCoordinator, Result, Sink, SinkWriter, SinkWriterParam}; @@ -69,22 +69,17 @@ pub struct RedisSink { } impl RedisSink { - pub fn new( - config: RedisConfig, - schema: Schema, - pk_indices: Vec, - is_append_only: bool, - ) -> Result { - if pk_indices.is_empty() { + pub fn new(config: RedisConfig, param: SinkParam) -> Result { + if param.pk_indices.is_empty() { return Err(SinkError::Config(anyhow!( "Redis Sink Primary Key must be specified." ))); } Ok(Self { config, - schema, - is_append_only, - pk_indices, + schema: param.schema(), + is_append_only: param.sink_type.is_append_only(), + pk_indices: param.pk_indices, }) } } diff --git a/src/connector/src/sink/utils.rs b/src/connector/src/sink/utils.rs index 9b9adb3796629..1eb2f91e9af22 100644 --- a/src/connector/src/sink/utils.rs +++ b/src/connector/src/sink/utils.rs @@ -27,6 +27,8 @@ use tracing::warn; use crate::sink::{Result, SinkError}; +const DEBEZIUM_NAME_FIELD_PREFIX: &str = "RisingWave"; + pub struct DebeziumAdapterOpts { gen_tombstone: bool, } @@ -39,6 +41,10 @@ impl Default for DebeziumAdapterOpts { } } +fn concat_debezium_name_field(db_name: &str, sink_from_name: &str, value: &str) -> String { + DEBEZIUM_NAME_FIELD_PREFIX.to_owned() + "." + db_name + "." + sink_from_name + "." + value +} + #[try_stream(ok = (Option, Option), error = SinkError)] pub async fn gen_debezium_message_stream<'a>( schema: &'a Schema, @@ -46,10 +52,12 @@ pub async fn gen_debezium_message_stream<'a>( chunk: StreamChunk, ts_ms: u64, opts: DebeziumAdapterOpts, + db_name: &'a str, + sink_from_name: &'a str, ) { let source_field = json!({ - "db": "RisingWave", - "table": "RisingWave", + "db": db_name, + "table": sink_from_name, }); let mut update_cache: Option> = None; @@ -60,13 +68,13 @@ pub async fn gen_debezium_message_stream<'a>( "type": "struct", "fields": fields_pk_to_json(&schema.fields, pk_indices), "optional": false, - "name": "RisingWave.RisingWave.RisingWave.Key", + "name": concat_debezium_name_field(db_name, sink_from_name, "Key"), }), "payload": pk_to_json(row, &schema.fields, pk_indices)?, })); let event_object: Option = match op { Op::Insert => Some(json!({ - "schema": schema_to_json(schema), + "schema": schema_to_json(schema, db_name, sink_from_name), "payload": { "before": null, "after": record_to_json(row, &schema.fields, TimestampHandlingMode::Milli)?, @@ -77,7 +85,7 @@ pub async fn gen_debezium_message_stream<'a>( })), Op::Delete => { let value_obj = Some(json!({ - "schema": schema_to_json(schema), + "schema": schema_to_json(schema, db_name, sink_from_name), "payload": { "before": record_to_json(row, &schema.fields, TimestampHandlingMode::Milli)?, "after": null, @@ -107,7 +115,7 @@ pub async fn gen_debezium_message_stream<'a>( Op::UpdateInsert => { if let Some(before) = update_cache.take() { Some(json!({ - "schema": schema_to_json(schema), + "schema": schema_to_json(schema, db_name, sink_from_name), "payload": { "before": before, "after": record_to_json(row, &schema.fields, TimestampHandlingMode::Milli)?, @@ -129,27 +137,27 @@ pub async fn gen_debezium_message_stream<'a>( } } -pub(crate) fn schema_to_json(schema: &Schema) -> Value { +pub(crate) fn schema_to_json(schema: &Schema, db_name: &str, sink_from_name: &str) -> Value { let mut schema_fields = Vec::new(); schema_fields.push(json!({ "type": "struct", "fields": fields_to_json(&schema.fields), "optional": true, "field": "before", - "name": "RisingWave.RisingWave.RisingWave.Key", + "name": concat_debezium_name_field(db_name, sink_from_name, "Key"), })); schema_fields.push(json!({ "type": "struct", "fields": fields_to_json(&schema.fields), "optional": true, "field": "after", - "name": "RisingWave.RisingWave.RisingWave.Key", + "name": concat_debezium_name_field(db_name, sink_from_name, "Key"), })); schema_fields.push(json!({ "type": "struct", "optional": false, - "name": "RisingWave.RisingWave.RisingWave.Source", + "name": concat_debezium_name_field(db_name, sink_from_name, "Source"), "fields": vec![ json!({ "type": "string", @@ -178,7 +186,7 @@ pub(crate) fn schema_to_json(schema: &Schema) -> Value { "type": "struct", "fields": schema_fields, "optional": false, - "name": "RisingWave.RisingWave.RisingWave.Envelope", + "name": concat_debezium_name_field(db_name, sink_from_name, "Envelope"), }) } diff --git a/src/connector/src/source/external.rs b/src/connector/src/source/external.rs index cf78a39c4e07c..3b66a06132b26 100644 --- a/src/connector/src/source/external.rs +++ b/src/connector/src/source/external.rs @@ -599,6 +599,8 @@ mod tests { ], pk_indices: vec![0], sink_type: SinkType::AppendOnly, + db_name: "db".into(), + sink_from_name: "table".into(), }; let rw_schema = param.schema(); diff --git a/src/expr/src/expr/build.rs b/src/expr/src/expr/build.rs index 472e57e044b99..55ec16b4fe0fe 100644 --- a/src/expr/src/expr/build.rs +++ b/src/expr/src/expr/build.rs @@ -32,6 +32,7 @@ use super::expr_some_all::SomeAllExpression; use super::expr_udf::UdfExpression; use super::expr_vnode::VnodeExpression; use crate::expr::expr_proctime::ProcTimeExpression; +use crate::expr::expr_regexp_count::RegexpCountExpression; use crate::expr::{ BoxedExpression, Expression, InputRefExpression, LiteralExpression, TryFromExprNodeBoxed, }; @@ -65,6 +66,7 @@ pub fn build_from_prost(prost: &ExprNode) -> Result { E::Row => NestedConstructExpression::try_from_boxed(prost), E::RegexpMatch => RegexpMatchExpression::try_from_boxed(prost), E::RegexpReplace => RegexpReplaceExpression::try_from_boxed(prost), + E::RegexpCount => RegexpCountExpression::try_from_boxed(prost), E::ArrayCat | E::ArrayAppend | E::ArrayPrepend => { // Now we implement these three functions as a single expression for the // sake of simplicity. If performance matters at some time, we can split diff --git a/src/expr/src/expr/expr_regexp.rs b/src/expr/src/expr/expr_regexp.rs index 2da46464633dd..7907bb45ab915 100644 --- a/src/expr/src/expr/expr_regexp.rs +++ b/src/expr/src/expr/expr_regexp.rs @@ -741,7 +741,6 @@ impl Expression for RegexpReplaceExpression { } async fn eval_row(&self, input: &OwnedRow) -> Result { - println!("Current Expr: {:?}", self); // Get the source text to match and replace let source = self.source.eval_row(input).await?; let source = match source { diff --git a/src/expr/src/expr/expr_regexp_count.rs b/src/expr/src/expr/expr_regexp_count.rs new file mode 100644 index 0000000000000..28e5b75ff74e9 --- /dev/null +++ b/src/expr/src/expr/expr_regexp_count.rs @@ -0,0 +1,229 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use risingwave_common::array::{ + Array, ArrayBuilder, ArrayImpl, ArrayRef, DataChunk, I32ArrayBuilder, +}; +use risingwave_common::row::OwnedRow; +use risingwave_common::types::{DataType, Datum, ScalarImpl}; +use risingwave_common::util::value_encoding::deserialize_datum; +use risingwave_common::{bail, ensure}; +use risingwave_pb::expr::expr_node::{RexNode, Type}; +use risingwave_pb::expr::ExprNode; + +use super::expr_regexp::RegexpContext; +use super::{build_from_prost as expr_build_from_prost, Expression}; +use crate::{ExprError, Result}; + +#[derive(Debug)] +pub struct RegexpCountExpression { + /// The source text + pub source: Box, + /// Relevant regex context, contains `flags` option + pub ctx: RegexpContext, + /// The start position to begin the counting process + pub start: Option, +} + +pub const NULL_PATTERN: &str = "a^"; + +/// This trait provides the transformation from `ExprNode` to `RegexpCountExpression` +impl<'a> TryFrom<&'a ExprNode> for RegexpCountExpression { + type Error = ExprError; + + fn try_from(prost: &'a ExprNode) -> Result { + // Sanity check first + ensure!(prost.get_function_type().unwrap() == Type::RegexpCount); + + let RexNode::FuncCall(func_call_node) = prost.get_rex_node().unwrap() else { + bail!("Expected RexNode::FuncCall"); + }; + + let mut children = func_call_node.children.iter(); + + let Some(source_node) = children.next() else { + bail!("Expected source text"); + }; + let source = expr_build_from_prost(source_node)?; + + let Some(pattern_node) = children.next() else { + bail!("Expected pattern text"); + }; + let pattern = match &pattern_node.get_rex_node()? { + RexNode::Constant(pattern_value) => { + let pattern_datum = deserialize_datum( + pattern_value.get_body().as_slice(), + &DataType::from(pattern_node.get_return_type().unwrap()), + ) + .map_err(|e| ExprError::Internal(e.into()))?; + + match pattern_datum { + Some(ScalarImpl::Utf8(pattern)) => pattern.to_string(), + // NULL pattern + None => NULL_PATTERN.to_string(), + _ => bail!("Expected pattern to be a String"), + } + } + _ => { + return Err(ExprError::UnsupportedFunction( + "non-constant pattern in `regexp_count`".to_string(), + )) + } + }; + + // Parsing for [ , start [, flags ]] + let mut flags: Option = None; + let mut start: Option = None; + + // See if `start` is specified + if let Some(start_node) = children.next() { + start = match &start_node.get_rex_node()? { + RexNode::Constant(start_value) => { + let start_datum = deserialize_datum( + start_value.get_body().as_slice(), + &DataType::from(start_node.get_return_type().unwrap()), + ) + .map_err(|e| ExprError::Internal(e.into()))?; + + match start_datum { + Some(ScalarImpl::Int32(start)) => { + if start <= 0 { + bail!("start must greater than zero"); + } + Some(start as u32) + } + _ => bail!("Expected start to be a Unsigned Int32"), + } + } + _ => { + return Err(ExprError::UnsupportedFunction( + "non-constant start in `regexp_count`".to_string(), + )) + } + }; + + // See if `flags` is specified + if let Some(flags_node) = children.next() { + flags = match &flags_node.get_rex_node()? { + RexNode::Constant(flags_value) => { + let flags_datum = deserialize_datum( + flags_value.get_body().as_slice(), + &DataType::from(flags_node.get_return_type().unwrap()), + ) + .map_err(|e| ExprError::Internal(e.into()))?; + + match flags_datum { + Some(ScalarImpl::Utf8(flags)) => Some(flags.to_string()), + _ => bail!("Expected flags to be a String"), + } + } + _ => { + return Err(ExprError::UnsupportedFunction( + "non-constant flags in `regexp_count`".to_string(), + )) + } + } + } + }; + + // Sanity check + if children.next().is_some() { + bail!("syntax error in `regexp_count`"); + } + + let flags = flags.unwrap_or_default(); + + if flags.contains('g') { + bail!("`regexp_count` does not support global flag option"); + } + + let ctx = RegexpContext::new(&pattern, &flags)?; + + Ok(Self { source, ctx, start }) + } +} + +impl RegexpCountExpression { + fn match_row(&self, text: Option<&str>) -> Option { + if let Some(text) = text { + // First get the start position to count for + let start = if let Some(s) = self.start { s - 1 } else { 0 }; + + // For unicode purpose + let mut start = match text.char_indices().nth(start as usize) { + Some((idx, _)) => idx, + // The `start` is out of bound + None => return Some(0), + }; + + let mut count = 0; + + while let Some(captures) = self.ctx.regex.captures(&text[start..]) { + count += 1; + start += captures.get(0).unwrap().end(); + } + + Some(count) + } else { + // Input string is None, the return value should be NULL + None + } + } +} + +#[async_trait::async_trait] +impl Expression for RegexpCountExpression { + fn return_type(&self) -> DataType { + DataType::Int32 + } + + async fn eval(&self, input: &DataChunk) -> Result { + let source_column = self.source.eval_checked(input).await?; + let source_column = source_column.as_utf8(); + + let row_len = input.capacity(); + let vis = input.vis(); + let mut builder: I32ArrayBuilder = ArrayBuilder::new(row_len); + + for row_idx in 0..row_len { + if !vis.is_set(row_idx) { + builder.append(None); + continue; + } + + let source = source_column.value_at(row_idx); + builder.append(self.match_row(source)); + } + + Ok(Arc::new(ArrayImpl::from(builder.finish()))) + } + + async fn eval_row(&self, input: &OwnedRow) -> Result { + let source = self.source.eval_row(input).await?; + // Will panic if the input text is not a String + let source = match source { + Some(ScalarImpl::Utf8(s)) => s, + None => return Ok(None), + // Other than the above cases + // The input is invalid and we should panic here + _ => bail!("source should be a String"), + }; + + Ok(self + .match_row(Some(&source)) + .map(|replaced| replaced.into())) + } +} diff --git a/src/expr/src/expr/mod.rs b/src/expr/src/expr/mod.rs index 5571fd0888b3b..60f2484bf010d 100644 --- a/src/expr/src/expr/mod.rs +++ b/src/expr/src/expr/mod.rs @@ -49,6 +49,7 @@ mod expr_literal; mod expr_nested_construct; mod expr_proctime; pub mod expr_regexp; +pub mod expr_regexp_count; mod expr_some_all; mod expr_timestamp_to_char_const_tmpl; mod expr_timestamptz_to_char_const_tmpl; diff --git a/src/frontend/planner_test/src/lib.rs b/src/frontend/planner_test/src/lib.rs index f31aa1443f74d..048851ebd17da 100644 --- a/src/frontend/planner_test/src/lib.rs +++ b/src/frontend/planner_test/src/lib.rs @@ -779,6 +779,8 @@ impl TestCase { format!("CREATE SINK {sink_name} AS {}", stmt), options, false, + "test_db".into(), + "test_table".into(), ) { Ok(sink_plan) => { ret.sink_plan = Some(explain_plan(&sink_plan.into())); diff --git a/src/frontend/src/binder/expr/function.rs b/src/frontend/src/binder/expr/function.rs index a1fac93305c2a..c505aaa18d2b2 100644 --- a/src/frontend/src/binder/expr/function.rs +++ b/src/frontend/src/binder/expr/function.rs @@ -760,6 +760,7 @@ impl Binder { ("bit_length", raw_call(ExprType::BitLength)), ("regexp_match", raw_call(ExprType::RegexpMatch)), ("regexp_replace", raw_call(ExprType::RegexpReplace)), + ("regexp_count", raw_call(ExprType::RegexpCount)), ("chr", raw_call(ExprType::Chr)), ("starts_with", raw_call(ExprType::StartsWith)), ("initcap", raw_call(ExprType::Initcap)), diff --git a/src/frontend/src/expr/pure.rs b/src/frontend/src/expr/pure.rs index a8cf6139c35d9..4316223ec07ae 100644 --- a/src/frontend/src/expr/pure.rs +++ b/src/frontend/src/expr/pure.rs @@ -104,6 +104,7 @@ impl ExprVisitor for ImpureAnalyzer { | expr_node::Type::Overlay | expr_node::Type::RegexpMatch | expr_node::Type::RegexpReplace + | expr_node::Type::RegexpCount | expr_node::Type::Pow | expr_node::Type::Exp | expr_node::Type::Ln diff --git a/src/frontend/src/expr/type_inference/func.rs b/src/frontend/src/expr/type_inference/func.rs index 186f5ac2ff95b..8cccde3b251de 100644 --- a/src/frontend/src/expr/type_inference/func.rs +++ b/src/frontend/src/expr/type_inference/func.rs @@ -467,6 +467,11 @@ fn infer_type_for_special( ensure_arity!("regexp_replace", 3 <= | inputs | <= 6); Ok(Some(DataType::Varchar)) } + ExprType::RegexpCount => { + // TODO: Preprocessing? + ensure_arity!("regexp_count", 2 <= | inputs | <= 4); + Ok(Some(DataType::Int32)) + } ExprType::ArrayCat => { ensure_arity!("array_cat", | inputs | == 2); let left_type = (!inputs[0].is_untyped()).then(|| inputs[0].return_type()); diff --git a/src/frontend/src/handler/create_sink.rs b/src/frontend/src/handler/create_sink.rs index 13940438dfe5d..3a8c701d77432 100644 --- a/src/frontend/src/handler/create_sink.rs +++ b/src/frontend/src/handler/create_sink.rs @@ -73,9 +73,17 @@ pub fn gen_sink_plan( let (sink_schema_name, sink_table_name) = Binder::resolve_schema_qualified_name(db_name, stmt.sink_name.clone())?; + // Used for debezium's table name + let sink_from_table_name; let query = match stmt.sink_from { - CreateSink::From(from_name) => Box::new(gen_sink_query_from_name(from_name)?), - CreateSink::AsQuery(query) => query, + CreateSink::From(from_name) => { + sink_from_table_name = from_name.0.last().unwrap().real_value(); + Box::new(gen_sink_query_from_name(from_name)?) + } + CreateSink::AsQuery(query) => { + sink_from_table_name = sink_table_name.clone(); + query + } }; let (sink_database_id, sink_schema_id) = @@ -117,6 +125,8 @@ pub fn gen_sink_plan( definition, with_options, emit_on_window_close, + db_name.to_owned(), + sink_from_table_name, )?; let sink_desc = sink_plan.sink_desc().clone(); let sink_plan: PlanRef = sink_plan.into(); diff --git a/src/frontend/src/optimizer/mod.rs b/src/frontend/src/optimizer/mod.rs index 4ba97f633bd3b..64c2d3c4ee9ea 100644 --- a/src/frontend/src/optimizer/mod.rs +++ b/src/frontend/src/optimizer/mod.rs @@ -553,12 +553,16 @@ impl PlanRoot { definition: String, properties: WithOptions, emit_on_window_close: bool, + db_name: String, + sink_from_table_name: String, ) -> Result { let stream_plan = self.gen_optimized_stream_plan(emit_on_window_close)?; StreamSink::create( stream_plan, sink_name, + db_name, + sink_from_table_name, self.required_dist.clone(), self.required_order.clone(), self.out_fields.clone(), diff --git a/src/frontend/src/optimizer/plan_node/stream_sink.rs b/src/frontend/src/optimizer/plan_node/stream_sink.rs index 57d50a8987ecd..60e89abf3f5c9 100644 --- a/src/frontend/src/optimizer/plan_node/stream_sink.rs +++ b/src/frontend/src/optimizer/plan_node/stream_sink.rs @@ -70,6 +70,8 @@ impl StreamSink { pub fn create( input: PlanRef, name: String, + db_name: String, + sink_from_table_name: String, user_distributed_by: RequiredDist, user_order_by: Order, user_cols: FixedBitSet, @@ -82,6 +84,8 @@ impl StreamSink { input, user_distributed_by, name, + db_name, + sink_from_table_name, user_order_by, columns, definition, @@ -95,6 +99,8 @@ impl StreamSink { input: PlanRef, user_distributed_by: RequiredDist, name: String, + db_name: String, + sink_from_name: String, user_order_by: Order, columns: Vec, definition: String, @@ -142,6 +148,8 @@ impl StreamSink { let sink_desc = SinkDesc { id: SinkId::placeholder(), name, + db_name, + sink_from_name, definition, columns, plan_pk: pk, diff --git a/src/meta/src/backup_restore/restore.rs b/src/meta/src/backup_restore/restore.rs index d99ee11f482c9..bac77639ba1c5 100644 --- a/src/meta/src/backup_restore/restore.rs +++ b/src/meta/src/backup_restore/restore.rs @@ -291,6 +291,7 @@ mod tests { use clap::Parser; use itertools::Itertools; use risingwave_backup::meta_snapshot::{ClusterMetadata, MetaSnapshot}; + use risingwave_common::config::SystemConfig; use risingwave_pb::hummock::HummockVersion; use risingwave_pb::meta::SystemParams; @@ -318,18 +319,9 @@ mod tests { fn get_system_params() -> SystemParams { SystemParams { - barrier_interval_ms: Some(101), - checkpoint_frequency: Some(102), - sstable_size_mb: Some(103), - block_size_kb: Some(104), - bloom_false_positive: Some(0.1), state_store: Some("state_store".to_string()), data_directory: Some("data_directory".to_string()), - backup_storage_url: Some("backup_storage_url".to_string()), - backup_storage_directory: Some("backup_storage_directory".to_string()), - telemetry_enabled: Some(false), - parallel_compact_size_mb: Some(255), - max_concurrent_creating_streaming_jobs: Some(1), + ..SystemConfig::default().into_init_system_params() } } diff --git a/src/meta/src/barrier/command.rs b/src/meta/src/barrier/command.rs index eb40c11812e1a..6c9fe0e5d4f77 100644 --- a/src/meta/src/barrier/command.rs +++ b/src/meta/src/barrier/command.rs @@ -36,7 +36,7 @@ use super::info::BarrierActorInfo; use super::trace::TracedEpoch; use crate::barrier::CommandChanges; use crate::manager::{FragmentManagerRef, WorkerId}; -use crate::model::{ActorId, DispatcherId, FragmentId, TableFragments}; +use crate::model::{ActorId, DispatcherId, FragmentId, PausedReason, TableFragments}; use crate::storage::MetaStore; use crate::stream::{build_actor_connector_splits, SourceManagerRef, SplitAssignment}; use crate::MetaResult; @@ -79,6 +79,15 @@ pub enum Command { /// After the barrier is collected, it does nothing. Plain(Option), + /// `Pause` command generates a `Pause` barrier with the provided [`PausedReason`] **only if** + /// the cluster is not already paused. Otherwise, a barrier with no mutation will be generated. + Pause(PausedReason), + + /// `Resume` command generates a `Resume` barrier with the provided [`PausedReason`] **only + /// if** the cluster is paused with the same reason. Otherwise, a barrier with no mutation + /// will be generated. + Resume(PausedReason), + /// `DropStreamingJobs` command generates a `Stop` barrier by the given /// [`HashSet`]. The catalog has ensured that these streaming jobs are safe to be /// dropped by reference counts before. @@ -142,18 +151,20 @@ impl Command { Self::Plain(None) } - pub fn pause() -> Self { - Self::Plain(Some(Mutation::Pause(PauseMutation {}))) + pub fn pause(reason: PausedReason) -> Self { + Self::Pause(reason) } - pub fn resume() -> Self { - Self::Plain(Some(Mutation::Resume(ResumeMutation {}))) + pub fn resume(reason: PausedReason) -> Self { + Self::Resume(reason) } /// Changes to the actors to be sent or collected after this command is committed. pub fn changes(&self) -> CommandChanges { match self { Command::Plain(_) => CommandChanges::None, + Command::Pause(_) => CommandChanges::None, + Command::Resume(_) => CommandChanges::None, Command::CreateStreamingJob { table_fragments, .. } => CommandChanges::CreateTable(table_fragments.table_id()), @@ -189,15 +200,15 @@ impl Command { /// injection. return true. pub fn should_pause_inject_barrier(&self) -> bool { // Note: the meaning for `Pause` is not pausing the periodic barrier injection, but for - // pausing the sources on compute nodes. However, `Pause` is used for configuration change - // like scaling and migration, which must pause the concurrent checkpoint to ensure the + // pausing the sources on compute nodes. However, when `Pause` is used for configuration + // change like scaling and migration, it must pause the concurrent checkpoint to ensure the // previous checkpoint has been done. - matches!(self, Self::Plain(Some(Mutation::Pause(_)))) + matches!(self, Self::Pause(PausedReason::ConfigChange)) } pub fn need_checkpoint(&self) -> bool { // todo! Reviewing the flow of different command to reduce the amount of checkpoint - !matches!(self, Command::Plain(None | Some(Mutation::Resume(_)))) + !matches!(self, Command::Plain(None) | Command::Resume(_)) } } @@ -215,6 +226,8 @@ pub struct CommandContext { pub prev_epoch: TracedEpoch, pub curr_epoch: TracedEpoch, + pub current_paused_reason: Option, + pub command: Command, pub kind: BarrierKind, @@ -237,6 +250,7 @@ impl CommandContext { info: BarrierActorInfo, prev_epoch: TracedEpoch, curr_epoch: TracedEpoch, + current_paused_reason: Option, command: Command, kind: BarrierKind, source_manager: SourceManagerRef, @@ -248,6 +262,7 @@ impl CommandContext { info: Arc::new(info), prev_epoch, curr_epoch, + current_paused_reason, command, kind, source_manager, @@ -265,6 +280,24 @@ where let mutation = match &self.command { Command::Plain(mutation) => mutation.clone(), + Command::Pause(_) => { + // Only pause when the cluster is not already paused. + if self.current_paused_reason.is_none() { + Some(Mutation::Pause(PauseMutation {})) + } else { + None + } + } + + Command::Resume(reason) => { + // Only resume when the cluster is paused with the same reason. + if self.current_paused_reason == Some(*reason) { + Some(Mutation::Resume(ResumeMutation {})) + } else { + None + } + } + Command::SourceSplitAssignment(change) => { let mut diff = HashMap::new(); @@ -308,6 +341,8 @@ where actor_dispatchers, added_actors, actor_splits, + // If the cluster is already paused, the new actors should be paused too. + pause: self.current_paused_reason.is_some(), })) } @@ -478,6 +513,31 @@ where Ok(mutation) } + /// Returns the paused reason after executing the current command. + pub fn next_paused_reason(&self) -> Option { + match &self.command { + Command::Pause(reason) => { + // Only pause when the cluster is not already paused. + if self.current_paused_reason.is_none() { + Some(*reason) + } else { + self.current_paused_reason + } + } + + Command::Resume(reason) => { + // Only resume when the cluster is paused with the same reason. + if self.current_paused_reason == Some(*reason) { + None + } else { + self.current_paused_reason + } + } + + _ => self.current_paused_reason, + } + } + /// For `CreateStreamingJob`, returns the actors of the `Chain` nodes. For other commands, /// returns an empty set. pub fn actors_to_track(&self) -> HashSet { @@ -562,18 +622,19 @@ where /// the given command. pub async fn post_collect(&self) -> MetaResult<()> { match &self.command { - #[allow(clippy::single_match)] - Command::Plain(mutation) => match mutation { - // After the `Pause` barrier is collected and committed, we must ensure that the - // storage version with this epoch is synced to all compute nodes before the - // execution of the next command of `Update`, as some newly created operators may - // immediately initialize their states on that barrier. - Some(Mutation::Pause(..)) => { + Command::Plain(_) => {} + + Command::Pause(reason) => { + if let PausedReason::ConfigChange = reason { + // After the `Pause` barrier is collected and committed, we must ensure that the + // storage version with this epoch is synced to all compute nodes before the + // execution of the next command of `Update`, as some newly created operators + // may immediately initialize their states on that barrier. self.wait_epoch_commit(self.prev_epoch.value().0).await?; } + } - _ => {} - }, + Command::Resume(_) => {} Command::SourceSplitAssignment(split_assignment) => { self.fragment_manager diff --git a/src/meta/src/barrier/mod.rs b/src/meta/src/barrier/mod.rs index f8ee3b3433c72..694bc81ca861c 100644 --- a/src/meta/src/barrier/mod.rs +++ b/src/meta/src/barrier/mod.rs @@ -25,6 +25,7 @@ use itertools::Itertools; use prometheus::HistogramTimer; use risingwave_common::bail; use risingwave_common::catalog::TableId; +use risingwave_common::system_param::PAUSE_ON_NEXT_BOOTSTRAP_KEY; use risingwave_common::util::tracing::TracingContext; use risingwave_hummock_sdk::{ExtendedSstableInfo, HummockSstableObjectId}; use risingwave_pb::ddl_service::DdlProgress; @@ -55,7 +56,7 @@ use crate::manager::{ CatalogManagerRef, ClusterManagerRef, FragmentManagerRef, LocalNotification, MetaSrvEnv, WorkerId, }; -use crate::model::{ActorId, BarrierManagerState}; +use crate::model::{ActorId, BarrierManagerState, PausedReason}; use crate::rpc::metrics::MetaMetrics; use crate::storage::meta_store::MetaStore; use crate::stream::SourceManagerRef; @@ -541,6 +542,23 @@ where *status = new_status; } + /// Check whether we should pause on bootstrap from the system parameter and reset it. + async fn take_pause_on_bootstrap(&self) -> MetaResult { + let pm = self.env.system_params_manager(); + let paused = pm.get_params().await.pause_on_next_bootstrap(); + if paused { + tracing::warn!( + "The cluster will bootstrap with all data sources paused as specified by the system parameter `{}`. \ + It will now be reset to `false`. \ + To resume the data sources, either restart the cluster again or use `risectl meta resume`.", + PAUSE_ON_NEXT_BOOTSTRAP_KEY + ); + pm.set_param(PAUSE_ON_NEXT_BOOTSTRAP_KEY, Some("false".to_owned())) + .await?; + } + Ok(paused) + } + /// Start an infinite loop to take scheduled barriers and send them. async fn run(&self, mut shutdown_rx: Receiver<()>) { // Initialize the barrier manager. @@ -579,9 +597,13 @@ where // inject the first `Initial` barrier. self.set_status(BarrierManagerStatus::Recovering).await; let span = tracing::info_span!("bootstrap_recovery", prev_epoch = prev_epoch.value().0); - let new_epoch = self.recovery(prev_epoch).instrument(span).await; - BarrierManagerState::new(new_epoch) + let paused = self.take_pause_on_bootstrap().await.unwrap_or(false); + let paused_reason = paused.then_some(PausedReason::Manual); + + self.recovery(prev_epoch, paused_reason) + .instrument(span) + .await }; self.set_status(BarrierManagerStatus::Running).await; @@ -682,6 +704,7 @@ where info, prev_epoch, curr_epoch, + state.paused_reason(), command, kind, self.source_manager.clone(), @@ -692,9 +715,12 @@ where send_latency_timer.observe_duration(); checkpoint_control.enqueue_command(command_ctx.clone(), notifiers); - self.inject_barrier(command_ctx, barrier_complete_tx) + self.inject_barrier(command_ctx.clone(), barrier_complete_tx) .instrument(span) .await; + + // Update the paused state after the barrier is injected. + state.set_paused_reason(command_ctx.next_paused_reason()); } /// Inject a barrier to all CNs and spawn a task to collect it @@ -918,9 +944,8 @@ where %err, prev_epoch = prev_epoch.value().0 ); - let new_epoch = self.recovery(prev_epoch).instrument(span).await; - *state = BarrierManagerState::new(new_epoch); + *state = self.recovery(prev_epoch, None).instrument(span).await; self.set_status(BarrierManagerStatus::Running).await; } else { panic!("failed to execute barrier: {:?}", err); diff --git a/src/meta/src/barrier/recovery.rs b/src/meta/src/barrier/recovery.rs index a049fe04f8877..8cfd5dacbf6e6 100644 --- a/src/meta/src/barrier/recovery.rs +++ b/src/meta/src/barrier/recovery.rs @@ -22,8 +22,7 @@ use risingwave_pb::common::ActorInfo; use risingwave_pb::stream_plan::barrier::{BarrierKind, Mutation}; use risingwave_pb::stream_plan::AddMutation; use risingwave_pb::stream_service::{ - BarrierCompleteResponse, BroadcastActorInfoTableRequest, BuildActorsRequest, - ForceStopActorsRequest, UpdateActorsRequest, + BroadcastActorInfoTableRequest, BuildActorsRequest, ForceStopActorsRequest, UpdateActorsRequest, }; use tokio_retry::strategy::{jitter, ExponentialBackoff}; use tracing::{debug, warn, Instrument}; @@ -34,7 +33,7 @@ use crate::barrier::command::CommandContext; use crate::barrier::info::BarrierActorInfo; use crate::barrier::{CheckpointControl, Command, GlobalBarrierManager}; use crate::manager::WorkerId; -use crate::model::MigrationPlan; +use crate::model::{BarrierManagerState, MigrationPlan, PausedReason}; use crate::storage::MetaStore; use crate::stream::build_actor_connector_splits; use crate::MetaResult; @@ -106,8 +105,16 @@ where /// Recovery the whole cluster from the latest epoch. /// - /// Returns the new epoch after recovery. - pub(crate) async fn recovery(&self, prev_epoch: TracedEpoch) -> TracedEpoch { + /// If `paused_reason` is `Some`, all data sources (including connectors and DMLs) will be + /// immediately paused after recovery, until the user manually resume them either by restarting + /// the cluster or `risectl` command. Used for debugging purpose. + /// + /// Returns the new state of the barrier manager after recovery. + pub(crate) async fn recovery( + &self, + prev_epoch: TracedEpoch, + paused_reason: Option, + ) -> BarrierManagerState { // Mark blocked and abort buffered schedules, they might be dirty already. self.scheduled_barriers .abort_and_mark_blocked("cluster is under recovering") @@ -123,9 +130,10 @@ where // We take retry into consideration because this is the latency user sees for a cluster to // get recovered. let recovery_timer = self.metrics.recovery_latency.start_timer(); - let (new_epoch, _responses) = tokio_retry::Retry::spawn(retry_strategy, || { + + let state = tokio_retry::Retry::spawn(retry_strategy, || { async { - let recovery_result: MetaResult<(TracedEpoch, Vec)> = try { + let recovery_result: MetaResult<_> = try { // Resolve actor info for recovery. If there's no actor to recover, most of the // following steps will be no-op, while the compute nodes will still be reset. let mut info = self.resolve_actor_info_for_recovery().await; @@ -158,6 +166,7 @@ where actor_dispatchers: Default::default(), added_actors: Default::default(), actor_splits: build_actor_connector_splits(&source_split_assignments), + pause: paused_reason.is_some(), }))); // Use a different `curr_epoch` for each recovery attempt. @@ -170,6 +179,7 @@ where info, prev_epoch.clone(), new_epoch.clone(), + paused_reason, command, BarrierKind::Initial, self.source_manager.clone(), @@ -205,7 +215,9 @@ where Err(err) } }; - res? + let (new_epoch, _) = res?; + + BarrierManagerState::new(new_epoch, command_ctx.next_paused_reason()) }; if recovery_result.is_err() { self.metrics.recovery_failure_cnt.inc(); @@ -216,11 +228,17 @@ where }) .await .expect("Retry until recovery success."); + recovery_timer.observe_duration(); self.scheduled_barriers.mark_ready().await; - tracing::info!("recovery success"); - new_epoch + tracing::info!( + epoch = state.in_flight_prev_epoch().value().0, + paused = ?state.paused_reason(), + "recovery success" + ); + + state } /// Migrate actors in expired CNs to newly joined ones, return true if any actor is migrated. diff --git a/src/meta/src/barrier/schedule.rs b/src/meta/src/barrier/schedule.rs index 1c93665002ac3..8734d2e230d7f 100644 --- a/src/meta/src/barrier/schedule.rs +++ b/src/meta/src/barrier/schedule.rs @@ -26,6 +26,7 @@ use tokio::sync::{oneshot, watch, RwLock}; use super::notifier::Notifier; use super::{Command, Scheduled}; use crate::hummock::HummockManagerRef; +use crate::model::PausedReason; use crate::rpc::metrics::MetaMetrics; use crate::storage::MetaStore; use crate::{MetaError, MetaResult}; @@ -289,9 +290,13 @@ impl BarrierScheduler { /// Run a command with a `Pause` command before and `Resume` command after it. Used for /// configuration change. - pub async fn run_command_with_paused(&self, command: Command) -> MetaResult<()> { - self.run_multiple_commands(vec![Command::pause(), command, Command::resume()]) - .await + pub async fn run_config_change_command_with_pause(&self, command: Command) -> MetaResult<()> { + self.run_multiple_commands(vec![ + Command::pause(PausedReason::ConfigChange), + command, + Command::resume(PausedReason::ConfigChange), + ]) + .await } /// Run a command and return when it's completely finished. diff --git a/src/meta/src/manager/sink_coordination/manager.rs b/src/meta/src/manager/sink_coordination/manager.rs index dfb511f08849a..522b40cd58001 100644 --- a/src/meta/src/manager/sink_coordination/manager.rs +++ b/src/meta/src/manager/sink_coordination/manager.rs @@ -422,6 +422,8 @@ mod tests { columns: vec![], pk_indices: vec![], sink_type: SinkType::AppendOnly, + db_name: "test".into(), + sink_from_name: "test".into(), }; let epoch1 = 233; @@ -593,6 +595,8 @@ mod tests { columns: vec![], pk_indices: vec![], sink_type: SinkType::AppendOnly, + db_name: "test".into(), + sink_from_name: "test".into(), }; let (manager, (_join_handle, _stop_tx)) = SinkCoordinatorManager::start_worker(None); @@ -629,6 +633,8 @@ mod tests { columns: vec![], pk_indices: vec![], sink_type: SinkType::AppendOnly, + db_name: "test".into(), + sink_from_name: "test".into(), }; let epoch = 233; @@ -709,6 +715,8 @@ mod tests { columns: vec![], pk_indices: vec![], sink_type: SinkType::AppendOnly, + db_name: "test".into(), + sink_from_name: "test".into(), }; let epoch = 233; diff --git a/src/meta/src/model/barrier.rs b/src/meta/src/model/barrier.rs index 0a603d7c3b100..a2315c9aa7ed8 100644 --- a/src/meta/src/model/barrier.rs +++ b/src/meta/src/model/barrier.rs @@ -14,6 +14,16 @@ use crate::barrier::TracedEpoch; +/// The reason why the data sources in the cluster are paused. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PausedReason { + /// The cluster is paused due to configuration change, e.g. altering table schema and scaling. + ConfigChange, + /// The cluster is paused due to manual operation, e.g. `risectl` command or the + /// `pause_on_next_bootstrap` system variable. + Manual, +} + /// `BarrierManagerState` defines the necessary state of `GlobalBarrierManager`. pub struct BarrierManagerState { /// The last sent `prev_epoch` @@ -21,15 +31,34 @@ pub struct BarrierManagerState { /// There's no need to persist this field. On recovery, we will restore this from the latest /// committed snapshot in `HummockManager`. in_flight_prev_epoch: TracedEpoch, + + /// Whether the cluster is paused and the reason. + paused_reason: Option, } impl BarrierManagerState { - pub fn new(in_flight_prev_epoch: TracedEpoch) -> Self { + pub fn new(in_flight_prev_epoch: TracedEpoch, paused_reason: Option) -> Self { Self { in_flight_prev_epoch, + paused_reason, + } + } + + pub fn paused_reason(&self) -> Option { + self.paused_reason + } + + pub fn set_paused_reason(&mut self, paused_reason: Option) { + if self.paused_reason != paused_reason { + tracing::info!(current = ?self.paused_reason, new = ?paused_reason, "update paused state"); + self.paused_reason = paused_reason; } } + pub fn in_flight_prev_epoch(&self) -> &TracedEpoch { + &self.in_flight_prev_epoch + } + /// Returns the epoch pair for the next barrier, and updates the state. pub fn next_epoch_pair(&mut self) -> (TracedEpoch, TracedEpoch) { let prev_epoch = self.in_flight_prev_epoch.clone(); diff --git a/src/meta/src/rpc/server.rs b/src/meta/src/rpc/server.rs index d9dd59650f5e2..fe6878b8e80f8 100644 --- a/src/meta/src/rpc/server.rs +++ b/src/meta/src/rpc/server.rs @@ -543,6 +543,7 @@ pub async fn start_service_as_election_leader( source_manager, catalog_manager.clone(), stream_manager.clone(), + barrier_manager.clone(), ); let cluster_srv = ClusterServiceImpl::::new(cluster_manager.clone()); diff --git a/src/meta/src/rpc/service/scale_service.rs b/src/meta/src/rpc/service/scale_service.rs index a770a1d726043..1bff2939d78b0 100644 --- a/src/meta/src/rpc/service/scale_service.rs +++ b/src/meta/src/rpc/service/scale_service.rs @@ -22,9 +22,9 @@ use risingwave_pb::meta::{ use risingwave_pb::source::{ConnectorSplit, ConnectorSplits}; use tonic::{Request, Response, Status}; -use crate::barrier::{BarrierScheduler, Command}; +use crate::barrier::{BarrierManagerRef, BarrierScheduler, Command}; use crate::manager::{CatalogManagerRef, ClusterManagerRef, FragmentManagerRef}; -use crate::model::MetadataModel; +use crate::model::{MetadataModel, PausedReason}; use crate::storage::MetaStore; use crate::stream::{ GlobalStreamManagerRef, ParallelUnitReschedule, RescheduleOptions, SourceManagerRef, @@ -37,6 +37,7 @@ pub struct ScaleServiceImpl { source_manager: SourceManagerRef, catalog_manager: CatalogManagerRef, stream_manager: GlobalStreamManagerRef, + barrier_manager: BarrierManagerRef, } impl ScaleServiceImpl @@ -50,6 +51,7 @@ where source_manager: SourceManagerRef, catalog_manager: CatalogManagerRef, stream_manager: GlobalStreamManagerRef, + barrier_manager: BarrierManagerRef, ) -> Self { Self { barrier_scheduler, @@ -58,6 +60,7 @@ where source_manager, catalog_manager, stream_manager, + barrier_manager, } } } @@ -69,14 +72,20 @@ where { #[cfg_attr(coverage, no_coverage)] async fn pause(&self, _: Request) -> Result, Status> { - self.barrier_scheduler.run_command(Command::pause()).await?; + // TODO: move this out of the scale service, as scaling actually executes `pause` and + // `resume` with `PausedReason::ConfigChange`. + self.barrier_scheduler + .run_command(Command::pause(PausedReason::Manual)) + .await?; Ok(Response::new(PauseResponse {})) } #[cfg_attr(coverage, no_coverage)] async fn resume(&self, _: Request) -> Result, Status> { + // TODO: move this out of the scale service, as scaling actually executes `pause` and + // `resume` with `PausedReason::ConfigChange`. self.barrier_scheduler - .run_command(Command::resume()) + .run_command(Command::resume(PausedReason::Manual)) .await?; Ok(Response::new(ResumeResponse {})) } @@ -137,6 +146,12 @@ where &self, request: Request, ) -> Result, Status> { + if !self.barrier_manager.is_running().await { + return Err(Status::unavailable( + "Rescheduling is unavailable for now. Likely the cluster is starting or recovering.", + )); + } + let RescheduleRequest { reschedules, revision, @@ -197,6 +212,12 @@ where ) -> Result, Status> { let req = request.into_inner(); + if !self.barrier_manager.is_running().await { + return Err(Status::unavailable( + "Rescheduling is unavailable for now. Likely the cluster is starting or recovering.", + )); + } + let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; let current_revision = self.fragment_manager.get_revision().await; diff --git a/src/meta/src/stream/scale.rs b/src/meta/src/stream/scale.rs index 83eb22f3f3cee..5afc29a699f2d 100644 --- a/src/meta/src/stream/scale.rs +++ b/src/meta/src/stream/scale.rs @@ -16,7 +16,7 @@ use std::cmp::{min, Ordering}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; use std::iter::repeat; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use futures::future::BoxFuture; use itertools::Itertools; use num_integer::Integer; @@ -1016,7 +1016,10 @@ where let upstream_worker_id = ctx .actor_id_to_parallel_unit(upstream_actor_id)? .worker_node_id; - let upstream_worker = ctx.worker_nodes.get(&upstream_worker_id).unwrap(); + let upstream_worker = + ctx.worker_nodes.get(&upstream_worker_id).with_context(|| { + format!("upstream worker {} not found", upstream_worker_id) + })?; // Force broadcast upstream actor info, because the actor information of the new // node may not have been synchronized yet @@ -1040,8 +1043,12 @@ where let downstream_worker_id = ctx .actor_id_to_parallel_unit(downstream_actor_id)? .worker_node_id; - let downstream_worker = - ctx.worker_nodes.get(&downstream_worker_id).unwrap(); + let downstream_worker = ctx + .worker_nodes + .get(&downstream_worker_id) + .with_context(|| { + format!("downstream worker {} not found", downstream_worker_id) + })?; actor_infos_to_broadcast.insert( *downstream_actor_id, @@ -1271,7 +1278,8 @@ where let worker_id = ctx .parallel_unit_id_to_worker_id .get(parallel_unit_id) - .unwrap(); + .with_context(|| format!("parallel unit {} not found", parallel_unit_id))?; + created_actors.insert( *actor_id, ( @@ -1308,7 +1316,7 @@ where tracing::debug!("reschedule plan: {:#?}", reschedule_fragment); self.barrier_scheduler - .run_command_with_paused(Command::RescheduleFragment { + .run_config_change_command_with_pause(Command::RescheduleFragment { reschedules: reschedule_fragment, }) .await?; diff --git a/src/meta/src/stream/stream_manager.rs b/src/meta/src/stream/stream_manager.rs index 802c04ed8935f..9d9006a0e29b7 100644 --- a/src/meta/src/stream/stream_manager.rs +++ b/src/meta/src/stream/stream_manager.rs @@ -489,7 +489,7 @@ where if let Err(err) = self .barrier_scheduler - .run_command_with_paused(Command::ReplaceTable { + .run_config_change_command_with_pause(Command::ReplaceTable { old_table_fragments, new_table_fragments: table_fragments, merge_updates, diff --git a/src/object_store/src/object/opendal_engine/opendal_object_store.rs b/src/object_store/src/object/opendal_engine/opendal_object_store.rs index bebfb660497f4..19787c53ec08d 100644 --- a/src/object_store/src/object/opendal_engine/opendal_object_store.rs +++ b/src/object_store/src/object/opendal_engine/opendal_object_store.rs @@ -193,7 +193,10 @@ impl StreamingUploader for OpenDalStreamingUploader { async fn finish(mut self: Box) -> ObjectResult<()> { match self.writer.close().await { Ok(_) => (), - Err(_) => self.writer.abort().await?, + Err(err) => { + self.writer.abort().await?; + return Err(err.into()); + } }; Ok(()) diff --git a/src/rpc_client/Cargo.toml b/src/rpc_client/Cargo.toml index f830e25641164..97225211d8ce9 100644 --- a/src/rpc_client/Cargo.toml +++ b/src/rpc_client/Cargo.toml @@ -41,7 +41,7 @@ tokio-stream = "0.1" tonic = { workspace = true } tower = "0.4" tracing = "0.1" -url = "2.4.0" +url = "2.4.1" [target.'cfg(not(madsim))'.dependencies] moka = { version = "0.11", features = ["future"] } diff --git a/src/storage/compactor/src/server.rs b/src/storage/compactor/src/server.rs index 679f9b5f05add..566ad8fd5e8ee 100644 --- a/src/storage/compactor/src/server.rs +++ b/src/storage/compactor/src/server.rs @@ -36,7 +36,9 @@ use risingwave_pb::common::WorkerType; use risingwave_pb::compactor::compactor_service_server::CompactorServiceServer; use risingwave_pb::monitor_service::monitor_service_server::MonitorServiceServer; use risingwave_rpc_client::MetaClient; -use risingwave_storage::filter_key_extractor::{FilterKeyExtractorManager, RemoteTableAccessor}; +use risingwave_storage::filter_key_extractor::{ + FilterKeyExtractorManager, RemoteTableAccessor, RpcFilterKeyExtractorManager, +}; use risingwave_storage::hummock::compactor::{CompactionExecutor, CompactorContext}; use risingwave_storage::hummock::hummock_meta_client::MonitoredHummockMetaClient; use risingwave_storage::hummock::{ @@ -156,7 +158,7 @@ pub async fn compactor_serve( let telemetry_enabled = system_params_reader.telemetry_enabled(); - let filter_key_extractor_manager = Arc::new(FilterKeyExtractorManager::new(Box::new( + let filter_key_extractor_manager = Arc::new(RpcFilterKeyExtractorManager::new(Box::new( RemoteTableAccessor::new(meta_client.clone()), ))); let system_params_manager = Arc::new(LocalSystemParamsManager::new(system_params_reader)); @@ -201,9 +203,11 @@ pub async fn compactor_serve( compaction_executor: Arc::new(CompactionExecutor::new( opts.compaction_worker_threads_number, )), - filter_key_extractor_manager: filter_key_extractor_manager.clone(), + filter_key_extractor_manager: FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + filter_key_extractor_manager.clone(), + ), memory_limiter, - sstable_object_id_manager: sstable_object_id_manager.clone(), + task_progress_manager: Default::default(), await_tree_reg: await_tree_reg.clone(), running_task_count: Arc::new(AtomicU32::new(0)), @@ -212,9 +216,12 @@ pub async fn compactor_serve( MetaClient::start_heartbeat_loop( meta_client.clone(), Duration::from_millis(config.server.heartbeat_interval_ms as u64), - vec![sstable_object_id_manager], + vec![sstable_object_id_manager.clone()], + ), + risingwave_storage::hummock::compactor::start_compactor( + compactor_context.clone(), + sstable_object_id_manager.clone(), ), - risingwave_storage::hummock::compactor::start_compactor(compactor_context.clone()), ]; let telemetry_manager = TelemetryManager::new( diff --git a/src/storage/hummock_test/src/bin/replay/main.rs b/src/storage/hummock_test/src/bin/replay/main.rs index 968f6437a2f43..8ca23d2ba0327 100644 --- a/src/storage/hummock_test/src/bin/replay/main.rs +++ b/src/storage/hummock_test/src/bin/replay/main.rs @@ -38,7 +38,7 @@ use risingwave_meta::hummock::test_utils::setup_compute_env; use risingwave_meta::hummock::MockHummockMetaClient; use risingwave_object_store::object::parse_remote_object_store; use risingwave_storage::filter_key_extractor::{ - FakeRemoteTableAccessor, FilterKeyExtractorManager, + FakeRemoteTableAccessor, RpcFilterKeyExtractorManager, }; use risingwave_storage::hummock::{FileCache, HummockStorage, SstableStore}; use risingwave_storage::monitor::{CompactorMetrics, HummockStateStoreMetrics, ObjectStoreMetrics}; @@ -137,7 +137,7 @@ async fn create_replay_hummock(r: Record, args: &Args) -> Result CompactorContext { CompactorContext { - storage_opts: options.clone(), + storage_opts: options, sstable_store, hummock_meta_client: hummock_meta_client.clone(), compactor_metrics: Arc::new(CompactorMetrics::unused()), is_share_buffer_compact: false, compaction_executor: Arc::new(CompactionExecutor::new(Some(1))), memory_limiter: MemoryLimiter::unlimit(), - filter_key_extractor_manager, - sstable_object_id_manager: Arc::new(SstableObjectIdManager::new( - hummock_meta_client.clone(), - options.sstable_id_remote_fetch_number, - )), + filter_key_extractor_manager: FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + filter_key_extractor_manager, + ), task_progress_manager: Default::default(), await_tree_reg: None, running_task_count: Arc::new(AtomicU32::new(0)), @@ -227,11 +225,26 @@ pub(crate) mod tests { Default::default(), ) .await; + + let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone() + { + FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + rpc_filter_key_extractor_manager, + ) => rpc_filter_key_extractor_manager, + FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(), + }; let compact_ctx = get_compactor_context_with_filter_key_extractor_manager( &storage, &hummock_meta_client, - storage.filter_key_extractor_manager().clone(), + rpc_filter_key_extractor_manager, ); + let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new( + hummock_meta_client.clone(), + storage + .storage_opts() + .clone() + .sstable_id_remote_fetch_number, + )); let worker_node2 = hummock_manager_ref .cluster_manager .add_worker_node( @@ -277,8 +290,13 @@ pub(crate) mod tests { compact_task.current_epoch_time = 0; let (_tx, rx) = tokio::sync::oneshot::channel(); - let (mut result_task, task_stats) = - compact(Arc::new(compact_ctx.clone()), compact_task.clone(), rx).await; + let (mut result_task, task_stats) = compact( + Arc::new(compact_ctx.clone()), + compact_task.clone(), + rx, + Box::new(sstable_object_id_manager.clone()), + ) + .await; hummock_manager_ref .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats))) @@ -371,12 +389,26 @@ pub(crate) mod tests { Default::default(), ) .await; + + let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone() + { + FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + rpc_filter_key_extractor_manager, + ) => rpc_filter_key_extractor_manager, + FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(), + }; let compact_ctx = get_compactor_context_with_filter_key_extractor_manager( &storage, &hummock_meta_client, - storage.filter_key_extractor_manager().clone(), + rpc_filter_key_extractor_manager, ); - + let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new( + hummock_meta_client.clone(), + storage + .storage_opts() + .clone() + .sstable_id_remote_fetch_number, + )); // 1. add sstables with 1MB value let mut key = BytesMut::default(); key.put_u16(0); @@ -421,8 +453,13 @@ pub(crate) mod tests { // 3. compact let (_tx, rx) = tokio::sync::oneshot::channel(); - let (mut result_task, task_stats) = - compact(Arc::new(compact_ctx), compact_task.clone(), rx).await; + let (mut result_task, task_stats) = compact( + Arc::new(compact_ctx), + compact_task.clone(), + rx, + Box::new(sstable_object_id_manager.clone()), + ) + .await; hummock_manager_ref .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats))) @@ -535,8 +572,14 @@ pub(crate) mod tests { hummock_meta_client: &Arc, existing_table_id: u32, ) -> CompactorContext { - let filter_key_extractor_manager = storage.filter_key_extractor_manager().clone(); - filter_key_extractor_manager.update( + let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone() + { + FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + rpc_filter_key_extractor_manager, + ) => rpc_filter_key_extractor_manager, + FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(), + }; + rpc_filter_key_extractor_manager.update( existing_table_id, Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)), ); @@ -544,7 +587,7 @@ pub(crate) mod tests { get_compactor_context_with_filter_key_extractor_manager( storage, hummock_meta_client, - filter_key_extractor_manager, + rpc_filter_key_extractor_manager, ) } @@ -636,13 +679,20 @@ pub(crate) mod tests { .new_local(NewLocalOptions::for_test(TableId::from(2))) .await; - let filter_key_extractor_manager = global_storage.filter_key_extractor_manager().clone(); - filter_key_extractor_manager.update( + let rpc_filter_key_extractor_manager = + match global_storage.filter_key_extractor_manager().clone() { + FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + rpc_filter_key_extractor_manager, + ) => rpc_filter_key_extractor_manager, + FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(), + }; + + rpc_filter_key_extractor_manager.update( 1, Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)), ); - filter_key_extractor_manager.update( + rpc_filter_key_extractor_manager.update( 2, Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)), ); @@ -651,9 +701,15 @@ pub(crate) mod tests { global_storage.storage_opts().clone(), global_storage.sstable_store(), &hummock_meta_client, - filter_key_extractor_manager.clone(), + rpc_filter_key_extractor_manager, ); - + let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new( + hummock_meta_client.clone(), + global_storage + .storage_opts() + .clone() + .sstable_id_remote_fetch_number, + )); // 1. add sstables let val = Bytes::from(b"0"[..].repeat(1 << 10)); // 1024 Byte value @@ -731,8 +787,13 @@ pub(crate) mod tests { // 4. compact let (_tx, rx) = tokio::sync::oneshot::channel(); - let (mut result_task, task_stats) = - compact(Arc::new(compact_ctx), compact_task.clone(), rx).await; + let (mut result_task, task_stats) = compact( + Arc::new(compact_ctx), + compact_task.clone(), + rx, + Box::new(sstable_object_id_manager.clone()), + ) + .await; hummock_manager_ref .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats))) @@ -815,13 +876,28 @@ pub(crate) mod tests { TableId::from(existing_table_id), ) .await; - let filter_key_extractor_manager = storage.filter_key_extractor_manager().clone(); + + let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone() + { + FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + rpc_filter_key_extractor_manager, + ) => rpc_filter_key_extractor_manager, + FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(), + }; + let compact_ctx = get_compactor_context_with_filter_key_extractor_manager( &storage, &hummock_meta_client, - filter_key_extractor_manager.clone(), + rpc_filter_key_extractor_manager.clone(), ); - filter_key_extractor_manager.update( + let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new( + hummock_meta_client.clone(), + storage + .storage_opts() + .clone() + .sstable_id_remote_fetch_number, + )); + rpc_filter_key_extractor_manager.update( 2, Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)), ); @@ -900,8 +976,13 @@ pub(crate) mod tests { // 3. compact let (_tx, rx) = tokio::sync::oneshot::channel(); - let (mut result_task, task_stats) = - compact(Arc::new(compact_ctx), compact_task.clone(), rx).await; + let (mut result_task, task_stats) = compact( + Arc::new(compact_ctx), + compact_task.clone(), + rx, + Box::new(sstable_object_id_manager.clone()), + ) + .await; hummock_manager_ref .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats))) @@ -990,20 +1071,32 @@ pub(crate) mod tests { ) .await; - let filter_key_extractor_manager = storage.filter_key_extractor_manager().clone(); - filter_key_extractor_manager.update( + let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone() + { + FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + rpc_filter_key_extractor_manager, + ) => rpc_filter_key_extractor_manager, + FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(), + }; + + rpc_filter_key_extractor_manager.update( existing_table_id, Arc::new(FilterKeyExtractorImpl::FixedLength( FixedLengthFilterKeyExtractor::new(TABLE_PREFIX_LEN + key_prefix.len()), )), ); - let compact_ctx = get_compactor_context_with_filter_key_extractor_manager( &storage, &hummock_meta_client, - filter_key_extractor_manager.clone(), + rpc_filter_key_extractor_manager, ); - + let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new( + hummock_meta_client.clone(), + storage + .storage_opts() + .clone() + .sstable_id_remote_fetch_number, + )); // 1. add sstables let val = Bytes::from(b"0"[..].to_vec()); // 1 Byte value let kv_count = 11; @@ -1069,8 +1162,13 @@ pub(crate) mod tests { // 3. compact let (_tx, rx) = tokio::sync::oneshot::channel(); - let (mut result_task, task_stats) = - compact(Arc::new(compact_ctx), compact_task.clone(), rx).await; + let (mut result_task, task_stats) = compact( + Arc::new(compact_ctx), + compact_task.clone(), + rx, + Box::new(sstable_object_id_manager.clone()), + ) + .await; hummock_manager_ref .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats))) @@ -1168,7 +1266,13 @@ pub(crate) mod tests { .await; let compact_ctx = prepare_compactor_and_filter(&storage, &hummock_meta_client, existing_table_id); - + let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new( + hummock_meta_client.clone(), + storage + .storage_opts() + .clone() + .sstable_id_remote_fetch_number, + )); prepare_data(hummock_meta_client.clone(), &storage, existing_table_id, 2).await; let mut local = storage .new_local(NewLocalOptions::for_test(existing_table_id.into())) @@ -1217,8 +1321,13 @@ pub(crate) mod tests { // 3. compact let (_tx, rx) = tokio::sync::oneshot::channel(); - let (mut result_task, task_stats) = - compact(Arc::new(compact_ctx), compact_task.clone(), rx).await; + let (mut result_task, task_stats) = compact( + Arc::new(compact_ctx), + compact_task.clone(), + rx, + Box::new(sstable_object_id_manager.clone()), + ) + .await; hummock_manager_ref .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats))) diff --git a/src/storage/hummock_test/src/sync_point_tests.rs b/src/storage/hummock_test/src/sync_point_tests.rs index 9533119ec3733..b0895ec64c1e9 100644 --- a/src/storage/hummock_test/src/sync_point_tests.rs +++ b/src/storage/hummock_test/src/sync_point_tests.rs @@ -39,7 +39,7 @@ use risingwave_pb::hummock::compact_task::TaskStatus; use risingwave_rpc_client::HummockMetaClient; use risingwave_storage::hummock::compactor::compactor_runner::compact; use risingwave_storage::hummock::compactor::CompactorContext; -use risingwave_storage::hummock::{CachePolicy, SstableObjectIdManager}; +use risingwave_storage::hummock::{CachePolicy, GetObjectId, SstableObjectIdManager}; use risingwave_storage::store::{LocalStateStore, NewLocalOptions, ReadOptions}; use risingwave_storage::StateStore; use serial_test::serial; @@ -73,7 +73,7 @@ async fn test_syncpoints_sstable_object_id_manager() { }); // Start the task that fetches new ids. - let sstable_object_id_manager_clone = sstable_object_id_manager.clone(); + let mut sstable_object_id_manager_clone = sstable_object_id_manager.clone(); let leader_task = tokio::spawn(async move { sstable_object_id_manager_clone .get_new_sst_object_id() @@ -90,7 +90,7 @@ async fn test_syncpoints_sstable_object_id_manager() { // Start tasks that waits to be notified. let mut follower_tasks = vec![]; for _ in 0..3 { - let sstable_object_id_manager_clone = sstable_object_id_manager.clone(); + let mut sstable_object_id_manager_clone = sstable_object_id_manager.clone(); let follower_task = tokio::spawn(async move { sstable_object_id_manager_clone .get_new_sst_object_id() @@ -137,7 +137,7 @@ async fn test_syncpoints_test_failpoints_fetch_ids() { }); // Start the task that fetches new ids. - let sstable_object_id_manager_clone = sstable_object_id_manager.clone(); + let mut sstable_object_id_manager_clone = sstable_object_id_manager.clone(); let leader_task = tokio::spawn(async move { fail::cfg("get_new_sst_ids_err", "return").unwrap(); sstable_object_id_manager_clone @@ -153,7 +153,7 @@ async fn test_syncpoints_test_failpoints_fetch_ids() { // Start tasks that waits to be notified. let mut follower_tasks = vec![]; for _ in 0..3 { - let sstable_object_id_manager_clone = sstable_object_id_manager.clone(); + let mut sstable_object_id_manager_clone = sstable_object_id_manager.clone(); let follower_task = tokio::spawn(async move { sstable_object_id_manager_clone .get_new_sst_object_id() @@ -230,6 +230,7 @@ async fn test_syncpoints_test_local_notification_receiver() { pub async fn compact_once( hummock_manager_ref: HummockManagerRef, compact_ctx: Arc, + sstable_object_id_manager: Arc, ) { // 2. get compact task let manual_compcation_option = ManualCompactionOption { @@ -251,7 +252,13 @@ pub async fn compact_once( compact_task.compaction_filter_mask = compaction_filter_flag.bits(); // 3. compact let (_tx, rx) = tokio::sync::oneshot::channel(); - let (mut result_task, task_stats) = compact(compact_ctx, compact_task.clone(), rx).await; + let (mut result_task, task_stats) = compact( + compact_ctx, + compact_task.clone(), + rx, + Box::new(sstable_object_id_manager), + ) + .await; hummock_manager_ref .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats))) @@ -288,6 +295,14 @@ async fn test_syncpoints_get_in_delete_range_boundary() { existing_table_id, )); + let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new( + hummock_meta_client.clone(), + storage + .storage_opts() + .clone() + .sstable_id_remote_fetch_number, + )); + let mut local = storage .new_local(NewLocalOptions::for_test(existing_table_id.into())) .await; @@ -320,7 +335,12 @@ async fn test_syncpoints_get_in_delete_range_boundary() { local.flush(Vec::new()).await.unwrap(); local.seal_current_epoch(101); flush_and_commit(&hummock_meta_client, &storage, 100).await; - compact_once(hummock_manager_ref.clone(), compact_ctx.clone()).await; + compact_once( + hummock_manager_ref.clone(), + compact_ctx.clone(), + sstable_object_id_manager.clone(), + ) + .await; local .insert(Bytes::from(b"\0\0aaa".as_slice()), val1.clone(), None) @@ -337,7 +357,12 @@ async fn test_syncpoints_get_in_delete_range_boundary() { .unwrap(); local.seal_current_epoch(102); flush_and_commit(&hummock_meta_client, &storage, 101).await; - compact_once(hummock_manager_ref.clone(), compact_ctx.clone()).await; + compact_once( + hummock_manager_ref.clone(), + compact_ctx.clone(), + sstable_object_id_manager.clone(), + ) + .await; local .insert(Bytes::from(b"\0\0hhh".as_slice()), val1.clone(), None) @@ -355,7 +380,12 @@ async fn test_syncpoints_get_in_delete_range_boundary() { local.seal_current_epoch(103); flush_and_commit(&hummock_meta_client, &storage, 102).await; // move this two file to the same level. - compact_once(hummock_manager_ref.clone(), compact_ctx.clone()).await; + compact_once( + hummock_manager_ref.clone(), + compact_ctx.clone(), + sstable_object_id_manager.clone(), + ) + .await; local .insert(Bytes::from(b"\0\0lll".as_slice()), val1.clone(), None) @@ -367,7 +397,12 @@ async fn test_syncpoints_get_in_delete_range_boundary() { local.seal_current_epoch(u64::MAX); flush_and_commit(&hummock_meta_client, &storage, 103).await; // move this two file to the same level. - compact_once(hummock_manager_ref.clone(), compact_ctx.clone()).await; + compact_once( + hummock_manager_ref.clone(), + compact_ctx.clone(), + sstable_object_id_manager.clone(), + ) + .await; // 4. get the latest version and check let version = hummock_manager_ref.get_current_version().await; diff --git a/src/storage/hummock_test/src/test_utils.rs b/src/storage/hummock_test/src/test_utils.rs index 523637f138a72..3ada8e61e4c56 100644 --- a/src/storage/hummock_test/src/test_utils.rs +++ b/src/storage/hummock_test/src/test_utils.rs @@ -32,8 +32,8 @@ use risingwave_pb::hummock::version_update_payload; use risingwave_rpc_client::HummockMetaClient; use risingwave_storage::error::StorageResult; use risingwave_storage::filter_key_extractor::{ - FilterKeyExtractorImpl, FilterKeyExtractorManager, FilterKeyExtractorManagerRef, - FullKeyFilterKeyExtractor, + FilterKeyExtractorImpl, FilterKeyExtractorManager, FullKeyFilterKeyExtractor, + RpcFilterKeyExtractorManager, }; use risingwave_storage::hummock::backup_reader::BackupReader; use risingwave_storage::hummock::event_handler::HummockEvent; @@ -66,7 +66,7 @@ pub async fn prepare_first_valid_version( let observer_manager = ObserverManager::new( notification_client, HummockObserverNode::new( - Arc::new(FilterKeyExtractorManager::default()), + Arc::new(RpcFilterKeyExtractorManager::default()), backup_manager, tx.clone(), write_limiter, @@ -163,11 +163,18 @@ pub async fn with_hummock_storage_v2( (hummock_storage, meta_client) } pub fn update_filter_key_extractor_for_table_ids( - filter_key_extractor_manager_ref: &FilterKeyExtractorManagerRef, + filter_key_extractor_manager_ref: &FilterKeyExtractorManager, table_ids: &[u32], ) { + let rpc_filter_key_extractor_manager = match filter_key_extractor_manager_ref { + FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + rpc_filter_key_extractor_manager, + ) => rpc_filter_key_extractor_manager, + FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(), + }; + for table_id in table_ids { - filter_key_extractor_manager_ref.update( + rpc_filter_key_extractor_manager.update( *table_id, Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)), ) @@ -175,7 +182,7 @@ pub fn update_filter_key_extractor_for_table_ids( } pub async fn register_tables_with_id_for_test( - filter_key_extractor_manager: &FilterKeyExtractorManagerRef, + filter_key_extractor_manager: &FilterKeyExtractorManager, hummock_manager_ref: &HummockManagerRef, table_ids: &[u32], ) { @@ -189,18 +196,24 @@ pub async fn register_tables_with_id_for_test( } pub fn update_filter_key_extractor_for_tables( - filter_key_extractor_manager_ref: &FilterKeyExtractorManagerRef, + filter_key_extractor_manager: &FilterKeyExtractorManager, tables: &[PbTable], ) { + let rpc_filter_key_extractor_manager = match filter_key_extractor_manager { + FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + rpc_filter_key_extractor_manager, + ) => rpc_filter_key_extractor_manager, + FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(), + }; for table in tables { - filter_key_extractor_manager_ref.update( + rpc_filter_key_extractor_manager.update( table.id, Arc::new(FilterKeyExtractorImpl::from_table(table)), ) } } pub async fn register_tables_with_catalog_for_test( - filter_key_extractor_manager: &FilterKeyExtractorManagerRef, + filter_key_extractor_manager: &FilterKeyExtractorManager, hummock_manager_ref: &HummockManagerRef, tables: &[Table], ) { diff --git a/src/storage/src/filter_key_extractor.rs b/src/storage/src/filter_key_extractor.rs index 77dd68a198854..47ce552300e22 100644 --- a/src/storage/src/filter_key_extractor.rs +++ b/src/storage/src/filter_key_extractor.rs @@ -258,7 +258,6 @@ impl StateTableAccessor for FakeRemoteTableAccessor { ))) } } - struct FilterKeyExtractorManagerInner { table_id_to_filter_key_extractor: RwLock>>, table_accessor: Box, @@ -334,19 +333,19 @@ impl FilterKeyExtractorManagerInner { } } -/// `FilterKeyExtractorManager` is a wrapper for inner, and provide a protected read and write +/// `RpcFilterKeyExtractorManager` is a wrapper for inner, and provide a protected read and write /// interface, its thread safe -pub struct FilterKeyExtractorManager { +pub struct RpcFilterKeyExtractorManager { inner: FilterKeyExtractorManagerInner, } -impl Default for FilterKeyExtractorManager { +impl Default for RpcFilterKeyExtractorManager { fn default() -> Self { Self::new(Box::::default()) } } -impl FilterKeyExtractorManager { +impl RpcFilterKeyExtractorManager { pub fn new(table_accessor: Box) -> Self { Self { inner: FilterKeyExtractorManagerInner { @@ -376,15 +375,60 @@ impl FilterKeyExtractorManager { /// Acquire a `MultiFilterKeyExtractor` by `table_id_set` /// Internally, try to get all `filter_key_extractor` from `hashmap`. Will block the caller if /// `table_id` does not util version update (notify), and retry to get + async fn acquire(&self, table_id_set: HashSet) -> HummockResult { + self.inner.acquire(table_id_set).await + } +} + +#[derive(Clone)] +pub enum FilterKeyExtractorManager { + RpcFilterKeyExtractorManager(Arc), + StaticFilterKeyExtractorManager(Arc), +} + +impl FilterKeyExtractorManager { pub async fn acquire( &self, table_id_set: HashSet, ) -> HummockResult { - self.inner.acquire(table_id_set).await + match self { + FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + rpc_filter_key_exactor_manager, + ) => rpc_filter_key_exactor_manager.acquire(table_id_set).await, + FilterKeyExtractorManager::StaticFilterKeyExtractorManager( + static_filter_key_extractor_manager, + ) => static_filter_key_extractor_manager.acquire(table_id_set), + } } } -pub type FilterKeyExtractorManagerRef = Arc; +#[derive(Clone)] +pub struct StaticFilterKeyExtractorManager { + id_to_table: HashMap, +} + +impl StaticFilterKeyExtractorManager { + pub fn new(id_to_table: HashMap) -> Self { + Self { id_to_table } + } + + fn acquire(&self, table_id_set: HashSet) -> HummockResult { + let mut multi_filter_key_extractor = MultiFilterKeyExtractor::default(); + for table_id in table_id_set { + if let Some(table) = self.id_to_table.get(&table_id) { + let key_extractor = Arc::new(FilterKeyExtractorImpl::from_table(table)); + multi_filter_key_extractor.register(table_id, key_extractor); + } else { + return Err(HummockError::other(format!( + "table {} is absent in id_to_table, need to request rpc list_tables to get the schema", table_id, + ))); + } + } + Ok(FilterKeyExtractorImpl::Multi(multi_filter_key_extractor)) + } +} + +pub type FilterKeyExtractorManagerRef = Arc; #[cfg(test)] mod tests { @@ -410,10 +454,9 @@ mod tests { use super::{DummyFilterKeyExtractor, FilterKeyExtractor, SchemaFilterKeyExtractor}; use crate::filter_key_extractor::{ - FilterKeyExtractorImpl, FilterKeyExtractorManager, FullKeyFilterKeyExtractor, - MultiFilterKeyExtractor, + FilterKeyExtractorImpl, FullKeyFilterKeyExtractor, MultiFilterKeyExtractor, + RpcFilterKeyExtractorManager, }; - const fn dummy_vnode() -> [u8; VirtualNode::SIZE] { VirtualNode::from_index(233).to_be_bytes() } @@ -618,7 +661,7 @@ mod tests { #[tokio::test] async fn test_filter_key_extractor_manager() { - let filter_key_extractor_manager = Arc::new(FilterKeyExtractorManager::default()); + let filter_key_extractor_manager = Arc::new(RpcFilterKeyExtractorManager::default()); filter_key_extractor_manager.update( 1, diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs index 9a5a6c11e3eee..f04121ff76179 100644 --- a/src/storage/src/hummock/compactor/compaction_utils.rs +++ b/src/storage/src/hummock/compactor/compaction_utils.rs @@ -35,13 +35,13 @@ use crate::hummock::compactor::{ use crate::hummock::multi_builder::TableBuilderFactory; use crate::hummock::sstable::DEFAULT_ENTRY_SIZE; use crate::hummock::{ - CachePolicy, FilterBuilder, HummockResult, MemoryLimiter, SstableBuilder, - SstableBuilderOptions, SstableObjectIdManagerRef, SstableWriterFactory, SstableWriterOptions, + CachePolicy, FilterBuilder, GetObjectId, HummockResult, MemoryLimiter, SstableBuilder, + SstableBuilderOptions, SstableWriterFactory, SstableWriterOptions, }; use crate::monitor::StoreLocalStatistic; pub struct RemoteBuilderFactory { - pub sstable_object_id_manager: SstableObjectIdManagerRef, + pub object_id_getter: Box, pub limiter: Arc, pub options: SstableBuilderOptions, pub policy: CachePolicy, @@ -58,10 +58,7 @@ impl TableBuilderFactory for RemoteBu async fn open_builder(&mut self) -> HummockResult> { let timer = Instant::now(); - let table_id = self - .sstable_object_id_manager - .get_new_sst_object_id() - .await?; + let table_id = self.object_id_getter.get_new_sst_object_id().await?; let cost = (timer.elapsed().as_secs_f64() * 1000000.0).round() as u64; self.remote_rpc_cost.fetch_add(cost, Ordering::Relaxed); let writer_options = SstableWriterOptions { diff --git a/src/storage/src/hummock/compactor/compactor_runner.rs b/src/storage/src/hummock/compactor/compactor_runner.rs index 4a40d288f2c08..cc22048489c20 100644 --- a/src/storage/src/hummock/compactor/compactor_runner.rs +++ b/src/storage/src/hummock/compactor/compactor_runner.rs @@ -45,7 +45,7 @@ use crate::hummock::sstable::CompactionDeleteRangesBuilder; use crate::hummock::value::HummockValue; use crate::hummock::{ BlockedXor16FilterBuilder, CachePolicy, CompactionDeleteRanges, CompressionAlgorithm, - HummockResult, MonotonicDeleteEvent, SstableBuilderOptions, SstableStoreRef, + GetObjectId, HummockResult, MonotonicDeleteEvent, SstableBuilderOptions, SstableStoreRef, }; use crate::monitor::{CompactorMetrics, StoreLocalStatistic}; @@ -58,7 +58,12 @@ pub struct CompactorRunner { } impl CompactorRunner { - pub fn new(split_index: usize, context: Arc, task: CompactTask) -> Self { + pub fn new( + split_index: usize, + context: Arc, + task: CompactTask, + object_id_getter: Box, + ) -> Self { let mut options: SstableBuilderOptions = context.storage_opts.as_ref().into(); options.compression_algorithm = match task.compression_algorithm { 0 => CompressionAlgorithm::None, @@ -98,6 +103,7 @@ impl CompactorRunner { split_weight_by_vnode: task.split_weight_by_vnode, use_block_based_filter, }, + object_id_getter, ); Self { @@ -232,32 +238,9 @@ pub async fn compact( compactor_context: Arc, mut compact_task: CompactTask, mut shutdown_rx: Receiver<()>, + object_id_getter: Box, ) -> (CompactTask, HashMap) { let context = compactor_context.clone(); - // Set a watermark SST id to prevent full GC from accidentally deleting SSTs for in-progress - // write op. The watermark is invalidated when this method exits. - let tracker_id = match context - .sstable_object_id_manager - .add_watermark_object_id(None) - .await - { - Ok(tracker_id) => tracker_id, - Err(err) => { - tracing::warn!("Failed to track pending SST object id. {:#?}", err); - - // return TaskStatus::TrackSstObjectIdFailed; - compact_task.set_task_status(TaskStatus::TrackSstObjectIdFailed); - return (compact_task, HashMap::default()); - } - }; - let sstable_object_id_manager_clone = context.sstable_object_id_manager.clone(); - let _guard = scopeguard::guard( - (tracker_id, sstable_object_id_manager_clone), - |(tracker_id, sstable_object_id_manager)| { - sstable_object_id_manager.remove_watermark_object_id(tracker_id); - }, - ); - let group_label = compact_task.compaction_group_id.to_string(); let cur_level_label = compact_task.input_ssts[0].level_idx.to_string(); let select_table_infos = compact_task @@ -327,7 +310,7 @@ pub async fn compact( .into_iter() .filter(|table_id| existing_table_ids.contains(table_id)), ); - let multi_filter_key_extractor = match context + let multi_filter_key_extractor = match compactor_context .filter_key_extractor_manager .acquire(compact_table_ids.clone()) .await @@ -457,8 +440,12 @@ pub async fn compact( for (split_index, _) in compact_task.splits.iter().enumerate() { let filter = multi_filter.clone(); let multi_filter_key_extractor = multi_filter_key_extractor.clone(); - let compactor_runner = - CompactorRunner::new(split_index, compactor_context.clone(), compact_task.clone()); + let compactor_runner = CompactorRunner::new( + split_index, + compactor_context.clone(), + compact_task.clone(), + object_id_getter.clone(), + ); let del_agg = delete_range_agg.clone(); let task_progress = task_progress_guard.progress.clone(); let runner = async move { diff --git a/src/storage/src/hummock/compactor/context.rs b/src/storage/src/hummock/compactor/context.rs index 32a23c1fdbe98..b5563ba391ccd 100644 --- a/src/storage/src/hummock/compactor/context.rs +++ b/src/storage/src/hummock/compactor/context.rs @@ -19,10 +19,10 @@ use parking_lot::RwLock; use risingwave_rpc_client::HummockMetaClient; use super::task_progress::TaskProgressManagerRef; -use crate::filter_key_extractor::FilterKeyExtractorManagerRef; +use crate::filter_key_extractor::FilterKeyExtractorManager; use crate::hummock::compactor::CompactionExecutor; use crate::hummock::sstable_store::SstableStoreRef; -use crate::hummock::{MemoryLimiter, SstableObjectIdManagerRef}; +use crate::hummock::MemoryLimiter; use crate::monitor::CompactorMetrics; use crate::opts::StorageOpts; @@ -46,12 +46,10 @@ pub struct CompactorContext { pub compaction_executor: Arc, - pub filter_key_extractor_manager: FilterKeyExtractorManagerRef, + pub filter_key_extractor_manager: FilterKeyExtractorManager, pub memory_limiter: Arc, - pub sstable_object_id_manager: SstableObjectIdManagerRef, - pub task_progress_manager: TaskProgressManagerRef, pub await_tree_reg: Option>>>, @@ -65,8 +63,7 @@ impl CompactorContext { sstable_store: SstableStoreRef, hummock_meta_client: Arc, compactor_metrics: Arc, - sstable_object_id_manager: SstableObjectIdManagerRef, - filter_key_extractor_manager: FilterKeyExtractorManagerRef, + filter_key_extractor_manager: FilterKeyExtractorManager, ) -> Self { let compaction_executor = if storage_opts.share_buffer_compaction_worker_threads_number == 0 { @@ -87,7 +84,6 @@ impl CompactorContext { compaction_executor, filter_key_extractor_manager, memory_limiter: MemoryLimiter::unlimit(), - sstable_object_id_manager, task_progress_manager: Default::default(), await_tree_reg: None, running_task_count: Arc::new(AtomicU32::new(0)), diff --git a/src/storage/src/hummock/compactor/mod.rs b/src/storage/src/hummock/compactor/mod.rs index c3a21050a08ef..c5d8cb05b96a4 100644 --- a/src/storage/src/hummock/compactor/mod.rs +++ b/src/storage/src/hummock/compactor/mod.rs @@ -41,6 +41,7 @@ pub use iterator::{ConcatSstableIterator, SstableStreamIterator}; use more_asserts::assert_ge; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; use risingwave_hummock_sdk::{HummockCompactionTaskId, LocalSstableInfo}; +use risingwave_pb::hummock::compact_task::TaskStatus; use risingwave_pb::hummock::subscribe_compaction_event_request::{ Event as RequestEvent, HeartBeat, PullTask, ReportTask, }; @@ -58,7 +59,10 @@ use tokio::time::Instant; pub use self::compaction_utils::{CompactionStatistics, RemoteBuilderFactory, TaskConfig}; pub use self::task_progress::TaskProgress; use super::multi_builder::CapacitySplitTableBuilder; -use super::{CompactionDeleteRanges, HummockResult, SstableBuilderOptions, Xor16FilterBuilder}; +use super::{ + CompactionDeleteRanges, GetObjectId, HummockResult, SstableBuilderOptions, + SstableObjectIdManager, Xor16FilterBuilder, +}; use crate::filter_key_extractor::FilterKeyExtractorImpl; use crate::hummock::compactor::compactor_runner::compact_and_build_sst; use crate::hummock::iterator::{Forward, HummockIterator}; @@ -73,6 +77,7 @@ use crate::hummock::{ pub struct Compactor { /// The context of the compactor. context: Arc, + object_id_getter: Box, task_config: TaskConfig, options: SstableBuilderOptions, get_id_time: Arc, @@ -86,12 +91,14 @@ impl Compactor { context: Arc, options: SstableBuilderOptions, task_config: TaskConfig, + object_id_getter: Box, ) -> Self { Self { context, options, task_config, get_id_time: Arc::new(AtomicU64::new(0)), + object_id_getter, } } @@ -137,6 +144,7 @@ impl Compactor { del_agg, filter_key_extractor, task_progress.clone(), + self.object_id_getter.clone(), ) .verbose_instrument_await("compact") .await? @@ -148,6 +156,7 @@ impl Compactor { del_agg, filter_key_extractor, task_progress.clone(), + self.object_id_getter.clone(), ) .verbose_instrument_await("compact") .await? @@ -162,6 +171,7 @@ impl Compactor { del_agg, filter_key_extractor, task_progress.clone(), + self.object_id_getter.clone(), ) .verbose_instrument_await("compact") .await? @@ -173,6 +183,7 @@ impl Compactor { del_agg, filter_key_extractor, task_progress.clone(), + self.object_id_getter.clone(), ) .verbose_instrument_await("compact") .await? @@ -253,9 +264,10 @@ impl Compactor { del_agg: Arc, filter_key_extractor: Arc, task_progress: Option>, + object_id_getter: Box, ) -> HummockResult<(Vec, CompactionStatistics)> { let builder_factory = RemoteBuilderFactory:: { - sstable_object_id_manager: self.context.sstable_object_id_manager.clone(), + object_id_getter, limiter: self.context.memory_limiter.clone(), options: self.options.clone(), policy: self.task_config.cache_policy, @@ -297,7 +309,10 @@ impl Compactor { /// The background compaction thread that receives compaction tasks from hummock compaction /// manager and runs compaction tasks. #[cfg_attr(coverage, no_coverage)] -pub fn start_compactor(compactor_context: Arc) -> (JoinHandle<()>, Sender<()>) { +pub fn start_compactor( + compactor_context: Arc, + sstable_object_id_manager: Arc, +) -> (JoinHandle<()>, Sender<()>) { let hummock_meta_client = compactor_context.hummock_meta_client.clone(); type CompactionShutdownMap = Arc>>>; let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); @@ -358,6 +373,7 @@ pub fn start_compactor(compactor_context: Arc) -> (JoinHandle< pin_mut!(response_event_stream); let executor = compactor_context.compaction_executor.clone(); + let sstable_object_id_manager = sstable_object_id_manager.clone(); let mut last_workload = CompactorWorkload::default(); // This inner loop is to consume stream or report task progress. @@ -495,6 +511,7 @@ pub fn start_compactor(compactor_context: Arc) -> (JoinHandle< .observe(consumed_latency_ms as _); let meta_client = hummock_meta_client.clone(); + let sstable_object_id_manager = sstable_object_id_manager.clone(); executor.spawn(async move { let running_task_count = running_task_count.clone(); match event { @@ -503,7 +520,26 @@ pub fn start_compactor(compactor_context: Arc) -> (JoinHandle< let (tx, rx) = tokio::sync::oneshot::channel(); let task_id = compact_task.task_id; shutdown.lock().unwrap().insert(task_id, tx); - let (compact_task, table_stats) = compactor_runner::compact(context, compact_task, rx).await; + let (compact_task, table_stats) = match sstable_object_id_manager.add_watermark_object_id(None).await + { + Ok(tracker_id) => { + let sstable_object_id_manager_clone = sstable_object_id_manager.clone(); + let _guard = scopeguard::guard( + (tracker_id, sstable_object_id_manager_clone), + |(tracker_id, sstable_object_id_manager)| { + sstable_object_id_manager.remove_watermark_object_id(tracker_id); + }, + ); + compactor_runner::compact(context, compact_task, rx, Box::new(sstable_object_id_manager.clone())).await + }, + Err(err) => { + tracing::warn!("Failed to track pending SST object id. {:#?}", err); + let mut compact_task = compact_task; + // return TaskStatus::TrackSstObjectIdFailed; + compact_task.set_task_status(TaskStatus::TrackSstObjectIdFailed); + (compact_task, HashMap::default()) + } + }; shutdown.lock().unwrap().remove(&task_id); running_task_count.fetch_sub(1, Ordering::SeqCst); diff --git a/src/storage/src/hummock/compactor/shared_buffer_compact.rs b/src/storage/src/hummock/compactor/shared_buffer_compact.rs index f46ac8006b00a..994dc3bab7799 100644 --- a/src/storage/src/hummock/compactor/shared_buffer_compact.rs +++ b/src/storage/src/hummock/compactor/shared_buffer_compact.rs @@ -46,7 +46,8 @@ use crate::hummock::utils::MemoryTracker; use crate::hummock::value::HummockValue; use crate::hummock::{ create_monotonic_events_from_compaction_delete_events, BlockedXor16FilterBuilder, CachePolicy, - CompactionDeleteRanges, HummockError, HummockResult, SstableBuilderOptions, + CompactionDeleteRanges, GetObjectId, HummockError, HummockResult, SstableBuilderOptions, + SstableObjectIdManagerRef, }; const GC_DELETE_KEYS_FOR_FLUSH: bool = false; @@ -55,6 +56,7 @@ const GC_WATERMARK_FOR_FLUSH: u64 = 0; /// Flush shared buffer to level0. Resulted SSTs are grouped by compaction group. pub async fn compact( context: Arc, + sstable_object_id_manager: SstableObjectIdManagerRef, payload: UploadTaskPayload, compaction_group_index: Arc>, ) -> HummockResult> { @@ -81,7 +83,12 @@ pub async fn compact( for (id, group_payload) in grouped_payload { let id_copy = id; futures.push( - compact_shared_buffer(context.clone(), group_payload).map_ok(move |results| { + compact_shared_buffer( + context.clone(), + sstable_object_id_manager.clone(), + group_payload, + ) + .map_ok(move |results| { results .into_iter() .map(move |mut result| { @@ -104,6 +111,7 @@ pub async fn compact( /// For compaction from shared buffer to level 0, this is the only function gets called. async fn compact_shared_buffer( context: Arc, + sstable_object_id_manager: SstableObjectIdManagerRef, mut payload: UploadTaskPayload, ) -> HummockResult> { // Local memory compaction looks at all key ranges. @@ -230,6 +238,7 @@ async fn compact_shared_buffer( sub_compaction_sstable_size as usize, split_weight_by_vnode as u32, use_block_based_filter, + Box::new(sstable_object_id_manager.clone()), ); let iter = OrderedMergeIteratorInner::new( payload.iter().map(|imm| imm.clone().into_forward_iter()), @@ -448,6 +457,7 @@ impl SharedBufferCompactRunner { sub_compaction_sstable_size: usize, split_weight_by_vnode: u32, use_block_based_filter: bool, + object_id_getter: Box, ) -> Self { let mut options: SstableBuilderOptions = context.storage_opts.as_ref().into(); options.capacity = sub_compaction_sstable_size; @@ -466,6 +476,7 @@ impl SharedBufferCompactRunner { split_weight_by_vnode, use_block_based_filter, }, + object_id_getter, ); Self { compactor, diff --git a/src/storage/src/hummock/event_handler/hummock_event_handler.rs b/src/storage/src/hummock/event_handler/hummock_event_handler.rs index 3a16bae1a15f7..6bf8f463c8189 100644 --- a/src/storage/src/hummock/event_handler/hummock_event_handler.rs +++ b/src/storage/src/hummock/event_handler/hummock_event_handler.rs @@ -44,7 +44,7 @@ use crate::hummock::store::version::{ }; use crate::hummock::utils::validate_table_key_range; use crate::hummock::{ - HummockError, HummockResult, MemoryLimiter, SstableObjectIdManagerRef, TrackerId, + HummockError, HummockResult, MemoryLimiter, SstableObjectIdManager, TrackerId, }; use crate::monitor::HummockStateStoreMetrics; use crate::opts::StorageOpts; @@ -127,26 +127,31 @@ pub struct HummockEventHandler { last_instance_id: LocalInstanceId, cache_refill_policy: Arc, - sstable_object_id_manager: SstableObjectIdManagerRef, + sstable_object_id_manager: Arc, } async fn flush_imms( payload: UploadTaskPayload, task_info: UploadTaskInfo, compactor_context: Arc, + sstable_object_id_manager: Arc, ) -> HummockResult> { for epoch in &task_info.epochs { - let _ = compactor_context - .sstable_object_id_manager + let _ = sstable_object_id_manager .add_watermark_object_id(Some(*epoch)) .await .inspect_err(|e| { error!("unable to set watermark sst id. epoch: {}, {:?}", epoch, e); }); } - compact(compactor_context, payload, task_info.compaction_group_index) - .verbose_instrument_await("shared_buffer_compact") - .await + compact( + compactor_context, + sstable_object_id_manager, + payload, + task_info.compaction_group_index, + ) + .verbose_instrument_await("shared_buffer_compact") + .await } impl HummockEventHandler { @@ -155,6 +160,7 @@ impl HummockEventHandler { hummock_event_rx: mpsc::UnboundedReceiver, pinned_version: PinnedVersion, compactor_context: Arc, + sstable_object_id_manager: Arc, state_store_metrics: Arc, refill_data_file_cache_levels: HashSet, ) -> Self { @@ -171,8 +177,8 @@ impl HummockEventHandler { ConflictDetector::new_from_config(&compactor_context.storage_opts); let sstable_store = compactor_context.sstable_store.clone(); let metrics = compactor_context.compactor_metrics.clone(); - let sstable_object_id_manager = compactor_context.sstable_object_id_manager.clone(); let upload_compactor_context = compactor_context.clone(); + let cloned_sstable_object_id_manager = sstable_object_id_manager.clone(); let uploader = HummockUploader::new( state_store_metrics, pinned_version.clone(), @@ -181,6 +187,7 @@ impl HummockEventHandler { payload, task_info, upload_compactor_context.clone(), + cloned_sstable_object_id_manager.clone(), )) }), buffer_tracker, diff --git a/src/storage/src/hummock/mod.rs b/src/storage/src/hummock/mod.rs index 76c5951e90ac0..fea6600f63b0f 100644 --- a/src/storage/src/hummock/mod.rs +++ b/src/storage/src/hummock/mod.rs @@ -32,6 +32,7 @@ use tracing::log::error; mod block_cache; pub use block_cache::*; +use crate::filter_key_extractor::RpcFilterKeyExtractorManager; use crate::hummock::store::state_store::LocalHummockStorage; use crate::opts::StorageOpts; @@ -73,7 +74,7 @@ use self::event_handler::ReadVersionMappingType; use self::iterator::HummockIterator; pub use self::sstable_store::*; use super::monitor::HummockStateStoreMetrics; -use crate::filter_key_extractor::{FilterKeyExtractorManager, FilterKeyExtractorManagerRef}; +use crate::filter_key_extractor::FilterKeyExtractorManager; use crate::hummock::backup_reader::{BackupReader, BackupReaderRef}; use crate::hummock::compactor::CompactorContext; use crate::hummock::event_handler::hummock_event_handler::BufferTracker; @@ -106,6 +107,8 @@ pub struct HummockStorage { context: Arc, + sstable_object_id_manager: SstableObjectIdManagerRef, + buffer_tracker: BufferTracker, version_update_notifier_tx: Arc>, @@ -136,7 +139,7 @@ impl HummockStorage { sstable_store: SstableStoreRef, hummock_meta_client: Arc, notification_client: impl NotificationClient, - filter_key_extractor_manager: Arc, + filter_key_extractor_manager: Arc, state_store_metrics: Arc, compactor_metrics: Arc, ) -> HummockResult { @@ -182,8 +185,9 @@ impl HummockStorage { sstable_store.clone(), hummock_meta_client.clone(), compactor_metrics.clone(), - sstable_object_id_manager.clone(), - filter_key_extractor_manager.clone(), + FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + filter_key_extractor_manager.clone(), + ), )); let seal_epoch = Arc::new(AtomicU64::new(pinned_version.max_committed_epoch())); @@ -193,6 +197,7 @@ impl HummockStorage { event_rx, pinned_version, compactor_context.clone(), + sstable_object_id_manager.clone(), state_store_metrics.clone(), options .data_file_cache_refill_levels @@ -203,6 +208,7 @@ impl HummockStorage { let instance = Self { context: compactor_context, + sstable_object_id_manager, buffer_tracker: hummock_event_handler.buffer_tracker().clone(), version_update_notifier_tx: hummock_event_handler.version_update_notifier_tx(), seal_epoch, @@ -253,10 +259,10 @@ impl HummockStorage { } pub fn sstable_object_id_manager(&self) -> &SstableObjectIdManagerRef { - &self.context.sstable_object_id_manager + &self.sstable_object_id_manager } - pub fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManagerRef { + pub fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManager { &self.context.filter_key_extractor_manager } @@ -327,7 +333,7 @@ impl HummockStorage { sstable_store, hummock_meta_client, notification_client, - Arc::new(FilterKeyExtractorManager::default()), + Arc::new(RpcFilterKeyExtractorManager::default()), Arc::new(HummockStateStoreMetrics::unused()), Arc::new(CompactorMetrics::unused()), ) diff --git a/src/storage/src/hummock/sstable/sstable_object_id_manager.rs b/src/storage/src/hummock/sstable/sstable_object_id_manager.rs index d37cceb3f8bc9..6ae7ddad4a7ea 100644 --- a/src/storage/src/hummock/sstable/sstable_object_id_manager.rs +++ b/src/storage/src/hummock/sstable/sstable_object_id_manager.rs @@ -14,7 +14,7 @@ use std::cmp; use std::collections::hash_map::Entry; -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::ops::DerefMut; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -28,9 +28,13 @@ use sync_point::sync_point; use tokio::sync::oneshot; use crate::hummock::{HummockError, HummockResult}; - pub type SstableObjectIdManagerRef = Arc; - +use dyn_clone::DynClone; +#[async_trait::async_trait] +pub trait GetObjectId: DynClone + Send + Sync { + async fn get_new_sst_object_id(&mut self) -> HummockResult; +} +dyn_clone::clone_trait_object!(GetObjectId); /// 1. Caches SST object ids fetched from meta. /// 2. Maintains GC watermark SST object id. /// @@ -59,15 +63,6 @@ impl SstableObjectIdManager { } } - /// Returns a new SST id. - /// The id is guaranteed to be monotonic increasing. - pub async fn get_new_sst_object_id(self: &Arc) -> HummockResult { - self.map_next_sst_object_id(|available_sst_object_ids| { - available_sst_object_ids.get_next_sst_object_id() - }) - .await - } - /// Executes `f` with next SST id. /// May fetch new SST ids via RPC. async fn map_next_sst_object_id( @@ -191,6 +186,41 @@ impl SstableObjectIdManager { } } +#[async_trait::async_trait] +impl GetObjectId for Arc { + /// Returns a new SST id. + /// The id is guaranteed to be monotonic increasing. + async fn get_new_sst_object_id(&mut self) -> HummockResult { + self.map_next_sst_object_id(|available_sst_object_ids| { + available_sst_object_ids.get_next_sst_object_id() + }) + .await + } +} + +/// `SharedComapctorObjectIdManager` is used to get output sst id for serverless compaction. +#[derive(Clone)] +pub struct SharedComapctorObjectIdManager { + output_object_ids: VecDeque, +} + +impl SharedComapctorObjectIdManager { + pub fn new(output_object_ids: VecDeque) -> Self { + Self { output_object_ids } + } +} + +#[async_trait::async_trait] +impl GetObjectId for SharedComapctorObjectIdManager { + async fn get_new_sst_object_id(&mut self) -> HummockResult { + if let Some(first_element) = self.output_object_ids.pop_front() { + Ok(first_element) + } else { + return Err(HummockError::other("Output object id runs out")); + } + } +} + #[async_trait::async_trait] impl ExtraInfoSource for SstableObjectIdManager { async fn get_extra_info(&self) -> Option { @@ -283,10 +313,14 @@ impl SstObjectIdTrackerInner { #[cfg(test)] mod test { + use std::collections::VecDeque; + use risingwave_common::try_match_expand; use crate::hummock::sstable::sstable_object_id_manager::AutoTrackerId; - use crate::hummock::{SstObjectIdTracker, TrackerId}; + use crate::hummock::{ + GetObjectId, SharedComapctorObjectIdManager, SstObjectIdTracker, TrackerId, + }; #[tokio::test] async fn test_object_id_tracker_basic() { @@ -356,4 +390,18 @@ mod test { object_id_tacker.remove_tracker(auto_id_3); assert!(object_id_tacker.tracking_object_ids().is_empty()); } + + #[tokio::test] + async fn test_shared_comapctor_object_id_manager() { + let mut pre_allocated_object_ids: VecDeque<_> = VecDeque::new(); + pre_allocated_object_ids.extend(vec![1, 3, 5]); + let mut object_id_manager = SharedComapctorObjectIdManager::new(pre_allocated_object_ids); + assert_eq!(object_id_manager.get_new_sst_object_id().await.unwrap(), 1); + + assert_eq!(object_id_manager.get_new_sst_object_id().await.unwrap(), 3); + + assert_eq!(object_id_manager.get_new_sst_object_id().await.unwrap(), 5); + + assert!(object_id_manager.get_new_sst_object_id().await.is_err()); + } } diff --git a/src/storage/src/store_impl.rs b/src/storage/src/store_impl.rs index 1655d774e2ee8..16b89868e8485 100644 --- a/src/storage/src/store_impl.rs +++ b/src/storage/src/store_impl.rs @@ -23,7 +23,7 @@ use risingwave_object_store::object::parse_remote_object_store; use crate::error::StorageResult; use crate::filter_key_extractor::{ - FilterKeyExtractorManager, FilterKeyExtractorManagerRef, RemoteTableAccessor, + FilterKeyExtractorManager, RemoteTableAccessor, RpcFilterKeyExtractorManager, }; use crate::hummock::backup_reader::BackupReaderRef; use crate::hummock::hummock_meta_client::MonitoredHummockMetaClient; @@ -635,7 +635,7 @@ impl StateStoreImpl { )); let notification_client = RpcNotificationClient::new(hummock_meta_client.get_inner().clone()); - let key_filter_manager = Arc::new(FilterKeyExtractorManager::new(Box::new( + let key_filter_manager = Arc::new(RpcFilterKeyExtractorManager::new(Box::new( RemoteTableAccessor::new(hummock_meta_client.get_inner().clone()), ))); let inner = HummockStorage::new( @@ -674,7 +674,7 @@ impl StateStoreImpl { pub trait HummockTrait { fn sstable_object_id_manager(&self) -> &SstableObjectIdManagerRef; fn sstable_store(&self) -> SstableStoreRef; - fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManagerRef; + fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManager; fn get_memory_limiter(&self) -> Arc; fn backup_reader(&self) -> BackupReaderRef; fn as_hummock(&self) -> Option<&HummockStorage>; @@ -689,7 +689,7 @@ impl HummockTrait for HummockStorage { self.sstable_store() } - fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManagerRef { + fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManager { self.filter_key_extractor_manager() } diff --git a/src/storage/src/table/mod.rs b/src/storage/src/table/mod.rs index 4eb5ca31077bb..aa876e2c6b88c 100644 --- a/src/storage/src/table/mod.rs +++ b/src/storage/src/table/mod.rs @@ -130,28 +130,22 @@ where /// Collects data chunks from stream of rows. pub async fn collect_data_chunk_with_builder( stream: &mut S, - chunk_size: Option, builder: &mut DataChunkBuilder, ) -> Result, E> where S: Stream> + Unpin, { - for _ in 0..chunk_size.unwrap_or(usize::MAX) { - match stream.next().await.transpose()? { - Some(row) => { - builder.append_one_row_no_finish(row); - } - None => break, + // TODO(kwannoel): If necessary, we can optimize it in the future. + // This can be done by moving the check if builder is full from `append_one_row` to here, + while let Some(row) = stream.next().await.transpose()? { + let result = builder.append_one_row(row); + if let Some(chunk) = result { + return Ok(Some(chunk)); } } - let chunk = builder.build_data_chunk(); - - if chunk.cardinality() == 0 { - Ok(None) - } else { - Ok(Some(chunk)) - } + let chunk = builder.consume_all(); + Ok(chunk) } pub fn get_second(arg: Result<(T, U), E>) -> Result { diff --git a/src/stream/src/executor/backfill/arrangement_backfill.rs b/src/stream/src/executor/backfill/arrangement_backfill.rs index 14da5930ddff6..77774528520d1 100644 --- a/src/stream/src/executor/backfill/arrangement_backfill.rs +++ b/src/stream/src/executor/backfill/arrangement_backfill.rs @@ -162,7 +162,6 @@ where let snapshot = Self::snapshot_read_per_vnode( &upstream_table, backfill_state.clone(), // FIXME: temporary workaround... How to avoid it? - self.chunk_size, &mut builders, ); pin_mut!(snapshot); @@ -244,7 +243,6 @@ where let right_snapshot = pin!(Self::snapshot_read_per_vnode( &upstream_table, backfill_state.clone(), // FIXME: temporary workaround, how to avoid it? - self.chunk_size, &mut builders, ) .map(Either::Right),); @@ -348,19 +346,27 @@ where // NOTE(kwannoel): `zip_eq_debug` does not work here, // we encounter "higher-ranked lifetime error". for (vnode, chunk) in vnodes.iter_vnodes().zip_eq(builders.iter_mut().map(|b| { - let chunk = b.build_data_chunk(); - let ops = vec![Op::Insert; chunk.capacity()]; - StreamChunk::from_parts(ops, chunk) + b.consume_all().map(|chunk| { + let ops = vec![Op::Insert; chunk.capacity()]; + StreamChunk::from_parts(ops, chunk) + }) })) { - // Raise the current position. - // As snapshot read streams are ordered by pk, so we can - // just use the last row to update `current_pos`. - update_pos_by_vnode(vnode, &chunk, &pk_in_output_indices, &mut backfill_state); - - let chunk_cardinality = chunk.cardinality() as u64; - cur_barrier_snapshot_processed_rows += chunk_cardinality; - total_snapshot_processed_rows += chunk_cardinality; - yield Message::Chunk(mapping_chunk(chunk, &self.output_indices)); + if let Some(chunk) = chunk { + // Raise the current position. + // As snapshot read streams are ordered by pk, so we can + // just use the last row to update `current_pos`. + update_pos_by_vnode( + vnode, + &chunk, + &pk_in_output_indices, + &mut backfill_state, + ); + + let chunk_cardinality = chunk.cardinality() as u64; + cur_barrier_snapshot_processed_rows += chunk_cardinality; + total_snapshot_processed_rows += chunk_cardinality; + yield Message::Chunk(mapping_chunk(chunk, &self.output_indices)); + } } // consume upstream buffer chunk @@ -496,10 +502,6 @@ where /// 3. Change it into a chunk iterator with `iter_chunks`. /// This means it should fetch a row from each iterator to form a chunk. /// - /// NOTE(kwannoel): We interleave at chunk per vnode level rather than rows. - /// This is so that we can compute `current_pos` once per chunk, since they correspond to 1 - /// vnode. - /// /// We will return chunks based on the `BackfillProgressPerVnode`. /// 1. Completed(vnode): Current iterator is complete, in that case we need to handle it /// in arrangement backfill. We should not buffer updates for this vnode, @@ -507,11 +509,21 @@ where /// 2. InProgress(CHUNK): Current iterator is not complete, in that case we /// need to buffer updates for this vnode. /// 3. Finished: All iterators finished. + /// + /// NOTE(kwannoel): We interleave at chunk per vnode level rather than rows. + /// This is so that we can compute `current_pos` once per chunk, since they correspond to 1 + /// vnode. + /// + /// NOTE(kwannoel): + /// The rows from upstream snapshot read will be buffered inside the `builder`. + /// If snapshot is dropped before its rows are consumed, + /// remaining data in `builder` must be flushed manually. + /// Otherwise when we scan a new snapshot, it is possible the rows in the `builder` would be + /// present, Then when we flush we contain duplicate rows. #[try_stream(ok = Option<(VirtualNode, StreamChunk)>, error = StreamExecutorError)] async fn snapshot_read_per_vnode<'a>( upstream_table: &'a ReplicatedStateTable, backfill_state: BackfillState, - chunk_size: usize, builders: &'a mut [DataChunkBuilder], ) { let mut streams = Vec::with_capacity(upstream_table.vnodes().len()); @@ -542,7 +554,7 @@ where // TODO: Is there some way to avoid double-pin here? let vnode_row_iter = Box::pin(owned_row_iter(vnode_row_iter)); - let vnode_chunk_iter = iter_chunks(vnode_row_iter, chunk_size, builder) + let vnode_chunk_iter = iter_chunks(vnode_row_iter, builder) .map_ok(move |chunk_opt| chunk_opt.map(|chunk| (vnode, chunk))); // TODO: Is there some way to avoid double-pin streams.push(Box::pin(vnode_chunk_iter)); diff --git a/src/stream/src/executor/backfill/cdc_backfill.rs b/src/stream/src/executor/backfill/cdc_backfill.rs index 22ef9be8b7900..55742348bbf03 100644 --- a/src/stream/src/executor/backfill/cdc_backfill.rs +++ b/src/stream/src/executor/backfill/cdc_backfill.rs @@ -27,8 +27,9 @@ use risingwave_common::array::StreamChunk; use risingwave_common::catalog::Schema; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::{JsonbVal, ScalarRefImpl}; +use risingwave_common::util::epoch::EpochPair; use risingwave_connector::source::external::{CdcOffset, DebeziumOffset, DebeziumSourceOffset}; -use risingwave_connector::source::{SplitImpl, SplitMetaData}; +use risingwave_connector::source::{SplitId, SplitImpl, SplitMetaData}; use risingwave_storage::StateStore; use serde_json::Value; @@ -43,7 +44,7 @@ use crate::executor::monitor::StreamingMetrics; use crate::executor::{ expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, Message, Mutation, PkIndices, PkIndicesRef, SourceStateTableHandler, - StreamExecutorError, + StreamExecutorError, StreamExecutorResult, }; use crate::task::{ActorId, CreateMviewProgress}; @@ -109,7 +110,6 @@ impl CdcBackfillExecutor { async fn execute_inner(mut self) { // The primary key columns, in the output columns of the upstream_table scan. let pk_in_output_indices = self.upstream_table.pk_in_output_indices().unwrap(); - let pk_order = self.upstream_table.pk_order_types().to_vec(); let upstream_table_id = self.upstream_table.table_id().table_id; @@ -121,6 +121,8 @@ impl CdcBackfillExecutor { // `None` means it starts from the beginning. let mut current_pk_pos: Option; + tracing::info!(upstream_table_id, ?pk_in_output_indices); + // Poll the upstream to get the first barrier. let first_barrier = expect_first_barrier(&mut upstream).await?; let init_epoch = first_barrier.epoch.prev; @@ -128,7 +130,7 @@ impl CdcBackfillExecutor { // Check whether this parallelism has been assigned splits, // if not, we should bypass the backfill directly. let mut invalid_backfill = false; - let mut split_id: Option> = None; + let mut split_id: Option = None; let mut cdc_split: Option = None; if let Some(mutation) = first_barrier.mutation.as_ref() { match mutation.as_ref() { @@ -254,10 +256,10 @@ impl CdcBackfillExecutor { // otherwise the upstream changelog may be blocked by the snapshot read stream let _ = Pin::new(&mut upstream).peek().await; - tracing::debug!( - "start the bacfill loop: [initial] binlog offset {:?}", - last_binlog_offset, - ); + tracing::info!( + upstream_table_id, + initial_binlog_offset = ?last_binlog_offset, + "start the bacfill loop"); 'backfill_loop: loop { let mut upstream_chunk_buffer: Vec = vec![]; @@ -335,10 +337,11 @@ impl CdcBackfillExecutor { } // seal current epoch even though there is no data - self.source_state_handler - .state_store - .commit(barrier.epoch) - .await?; + Self::persist_state( + &mut self.source_state_handler, + barrier.epoch, + ) + .await?; yield Message::Barrier(barrier); // Break the for loop and start a new snapshot read stream. @@ -383,9 +386,11 @@ impl CdcBackfillExecutor { Either::Right(msg) => { match msg? { None => { - tracing::debug!( - "snapshot read stream ends: last_binlog_offset {:?}, current_pk_pos {:?}", - last_binlog_offset, current_pk_pos + tracing::info!( + upstream_table_id, + ?last_binlog_offset, + ?current_pk_pos, + "snapshot read stream ends" ); // End of the snapshot read stream. // We should not mark the chunk anymore, @@ -402,59 +407,14 @@ impl CdcBackfillExecutor { )); } - // When snapshot read stream ends, we should persist two states: - // 1) a backfill finish flag to denote the backfill has done - // 2) a consumed binlog offset to denote the last binlog offset - if let Some(split_id) = split_id.as_ref() { - let mut key = split_id.to_string(); - key.push_str(BACKFILL_STATE_KEY_SUFFIX); - self.source_state_handler - .set(key.into(), JsonbVal::from(Value::Bool(true))) - .await?; - - if let Some(SplitImpl::MySqlCdc(split)) = cdc_split.as_mut() - && let Some(s) = split.mysql_split.as_mut() { - let start_offset = - last_binlog_offset.as_ref().map(|cdc_offset| { - let source_offset = - if let CdcOffset::MySql(o) = cdc_offset - { - DebeziumSourceOffset { - file: Some(o.filename.clone()), - pos: Some(o.position), - ..Default::default() - } - } else { - DebeziumSourceOffset::default() - }; - - let mut server = "RW_CDC_".to_string(); - server.push_str( - upstream_table_id.to_string().as_str(), - ); - DebeziumOffset { - source_partition: hashmap! { - "server".to_string() => server - }, - source_offset, - } - }); - - // persist the last binlog offset into split state - s.inner.start_offset = start_offset.map(|o| { - let value = serde_json::to_value(o).unwrap(); - value.to_string() - }); - s.inner.snapshot_done = true; - } - - if let Some(split_impl) = cdc_split { - self.source_state_handler - .set(split_impl.id(), split_impl.encode_to_json()) - .await? - } - } - + Self::write_backfill_state( + &mut self.source_state_handler, + upstream_table_id, + &split_id, + &mut cdc_split, + last_binlog_offset.clone(), + ) + .await?; break 'backfill_loop; } Some(chunk) => { @@ -481,6 +441,15 @@ impl CdcBackfillExecutor { } } } + } else { + Self::write_backfill_state( + &mut self.source_state_handler, + upstream_table_id, + &split_id, + &mut cdc_split, + None, + ) + .await?; } tracing::debug!( @@ -498,15 +467,81 @@ impl CdcBackfillExecutor { if let Some(msg) = mapping_message(msg?, &self.output_indices) { // persist the backfill state if any if let Message::Barrier(barrier) = &msg { - self.source_state_handler - .state_store - .commit(barrier.epoch) - .await?; + Self::persist_state(&mut self.source_state_handler, barrier.epoch).await?; } yield msg; } } } + + /// When snapshot read stream ends, we should persist two states: + /// 1) a backfill finish flag to denote the backfill has done + /// 2) a consumed binlog offset to denote the last binlog offset + /// which will be committed to the state store upon next barrier. + async fn write_backfill_state( + source_state_handler: &mut SourceStateTableHandler, + upstream_table_id: u32, + split_id: &Option, + cdc_split: &mut Option, + last_binlog_offset: Option, + ) -> StreamExecutorResult<()> { + if let Some(split_id) = split_id.as_ref() { + let mut key = split_id.to_string(); + key.push_str(BACKFILL_STATE_KEY_SUFFIX); + source_state_handler + .set(key.into(), JsonbVal::from(Value::Bool(true))) + .await?; + + if let Some(SplitImpl::MySqlCdc(split)) = cdc_split.as_mut() + && let Some(s) = split.mysql_split.as_mut() { + let start_offset = + last_binlog_offset.as_ref().map(|cdc_offset| { + let source_offset = + if let CdcOffset::MySql(o) = cdc_offset + { + DebeziumSourceOffset { + file: Some(o.filename.clone()), + pos: Some(o.position), + ..Default::default() + } + } else { + DebeziumSourceOffset::default() + }; + + let mut server = "RW_CDC_".to_string(); + server.push_str( + upstream_table_id.to_string().as_str(), + ); + DebeziumOffset { + source_partition: hashmap! { + "server".to_string() => server + }, + source_offset, + } + }); + + // persist the last binlog offset into split state + s.inner.start_offset = start_offset.map(|o| { + let value = serde_json::to_value(o).unwrap(); + value.to_string() + }); + s.inner.snapshot_done = true; + } + if let Some(split_impl) = cdc_split { + source_state_handler + .set(split_impl.id(), split_impl.encode_to_json()) + .await? + } + } + Ok(()) + } + + async fn persist_state( + source_state_handler: &mut SourceStateTableHandler, + new_epoch: EpochPair, + ) -> StreamExecutorResult<()> { + source_state_handler.state_store.commit(new_epoch).await + } } impl Executor for CdcBackfillExecutor { diff --git a/src/stream/src/executor/backfill/no_shuffle_backfill.rs b/src/stream/src/executor/backfill/no_shuffle_backfill.rs index 309bc1f7dec69..6040a6745f9e3 100644 --- a/src/stream/src/executor/backfill/no_shuffle_backfill.rs +++ b/src/stream/src/executor/backfill/no_shuffle_backfill.rs @@ -163,16 +163,20 @@ where // It is finished, so just assign a value to avoid accessing storage table again. false } else { - let snapshot = Self::snapshot_read( - &self.upstream_table, - init_epoch, - None, - false, - self.chunk_size, - &mut builder, - ); - pin_mut!(snapshot); - snapshot.try_next().await?.unwrap().is_none() + let snapshot_is_empty = { + let snapshot = Self::snapshot_read( + &self.upstream_table, + init_epoch, + None, + false, + &mut builder, + ); + pin_mut!(snapshot); + snapshot.try_next().await?.unwrap().is_none() + }; + let snapshot_buffer_is_empty = builder.is_empty(); + builder.clear(); + snapshot_is_empty && snapshot_buffer_is_empty } }; @@ -254,7 +258,6 @@ where snapshot_read_epoch, current_pos.clone(), true, - self.chunk_size, &mut builder ) .map(Either::Right),); @@ -348,9 +351,9 @@ where // - switch snapshot // Consume snapshot rows left in builder - let chunk = builder.build_data_chunk(); - let chunk_cardinality = chunk.cardinality() as u64; - if chunk_cardinality > 0 { + let chunk = builder.consume_all(); + if let Some(chunk) = chunk { + let chunk_cardinality = chunk.cardinality() as u64; let ops = vec![Op::Insert; chunk.capacity()]; let chunk = StreamChunk::from_parts(ops, chunk); current_pos = Some(get_new_pos(&chunk, &pk_in_output_indices)); @@ -476,13 +479,18 @@ where } } + /// Snapshot read the upstream mv. + /// The rows from upstream snapshot read will be buffered inside the `builder`. + /// If snapshot is dropped before its rows are consumed, + /// remaining data in `builder` must be flushed manually. + /// Otherwise when we scan a new snapshot, it is possible the rows in the `builder` would be + /// present, Then when we flush we contain duplicate rows. #[try_stream(ok = Option, error = StreamExecutorError)] async fn snapshot_read<'a>( upstream_table: &'a StorageTable, epoch: u64, current_pos: Option, ordered: bool, - chunk_size: usize, builder: &'a mut DataChunkBuilder, ) { let range_bounds = compute_bounds(upstream_table.pk_indices(), current_pos); @@ -510,7 +518,7 @@ where pin_mut!(row_iter); #[for_await] - for chunk in iter_chunks(row_iter, chunk_size, builder) { + for chunk in iter_chunks(row_iter, builder) { yield chunk?; } } diff --git a/src/stream/src/executor/backfill/upstream_table/snapshot.rs b/src/stream/src/executor/backfill/upstream_table/snapshot.rs index ee485ece4796d..806d78700154e 100644 --- a/src/stream/src/executor/backfill/upstream_table/snapshot.rs +++ b/src/stream/src/executor/backfill/upstream_table/snapshot.rs @@ -125,7 +125,7 @@ impl UpstreamTableRead for UpstreamTableReader { let mut builder = DataChunkBuilder::new(self.inner.schema().data_types(), args.chunk_size); - let chunk_stream = iter_chunks(row_stream, args.chunk_size, &mut builder); + let chunk_stream = iter_chunks(row_stream, &mut builder); #[for_await] for chunk in chunk_stream { yield chunk?; diff --git a/src/stream/src/executor/backfill/utils.rs b/src/stream/src/executor/backfill/utils.rs index 629626b19bda2..80f65e28aa8d5 100644 --- a/src/stream/src/executor/backfill/utils.rs +++ b/src/stream/src/executor/backfill/utils.rs @@ -217,30 +217,27 @@ fn mark_cdc_chunk_inner( // `_rw_offset` must be placed at the last column right now let offset_col_idx = data.dimension() - 1; - for v in data.rows_with_holes().map(|row| { + for v in data.rows().map(|row| { let offset_datum = row.datum_at(offset_col_idx).unwrap(); let event_offset = table_reader.parse_binlog_offset(offset_datum.into_utf8())?; - let visible = match row { - None => false, - Some(row) => { - // filter changelog events with binlog range - let in_binlog_range = if let Some(binlog_low) = &last_cdc_offset { - binlog_low <= &event_offset - } else { - true - }; - - if in_binlog_range { - let lhs = row.project(pk_in_output_indices); - let rhs = current_pos.project(pk_in_output_indices); - let order = cmp_datum_iter(lhs.iter(), rhs.iter(), pk_order.iter().copied()); - match order { - Ordering::Less | Ordering::Equal => true, - Ordering::Greater => false, - } - } else { - false + let visible = { + // filter changelog events with binlog range + let in_binlog_range = if let Some(binlog_low) = &last_cdc_offset { + binlog_low <= &event_offset + } else { + true + }; + + if in_binlog_range { + let lhs = row.project(pk_in_output_indices); + let rhs = current_pos; + let order = cmp_datum_iter(lhs.iter(), rhs.iter(), pk_order.iter().copied()); + match order { + Ordering::Less | Ordering::Equal => true, + Ordering::Greater => false, } + } else { + false } }; Ok::<_, ConnectorError>(visible) @@ -477,18 +474,14 @@ where } #[try_stream(ok = Option, error = StreamExecutorError)] -pub(crate) async fn iter_chunks<'a, S, E>( - mut iter: S, - chunk_size: usize, - builder: &'a mut DataChunkBuilder, -) where +pub(crate) async fn iter_chunks<'a, S, E>(mut iter: S, builder: &'a mut DataChunkBuilder) +where StreamExecutorError: From, S: Stream> + Unpin + 'a, { - while let Some(data_chunk) = - collect_data_chunk_with_builder(&mut iter, Some(chunk_size), builder) - .instrument_await("backfill_snapshot_read") - .await? + while let Some(data_chunk) = collect_data_chunk_with_builder(&mut iter, builder) + .instrument_await("backfill_snapshot_read") + .await? { debug_assert!(data_chunk.cardinality() > 0); let ops = vec![Op::Insert; data_chunk.capacity()]; diff --git a/src/stream/src/executor/chain.rs b/src/stream/src/executor/chain.rs index 56a10da734d37..ab3ef9ae44973 100644 --- a/src/stream/src/executor/chain.rs +++ b/src/stream/src/executor/chain.rs @@ -178,6 +178,7 @@ mod test { }, added_actors: maplit::hashset! { actor_id }, splits: Default::default(), + pause: false, })), Message::Chunk(StreamChunk::from_pretty("I\n + 3")), Message::Chunk(StreamChunk::from_pretty("I\n + 4")), diff --git a/src/stream/src/executor/dml.rs b/src/stream/src/executor/dml.rs index 16ae74841b7f0..b324f4666cc2e 100644 --- a/src/stream/src/executor/dml.rs +++ b/src/stream/src/executor/dml.rs @@ -119,9 +119,8 @@ impl DmlExecutor { // a round robin way. let mut stream = StreamReaderWithPause::::new(upstream, batch_reader); - // If the first barrier is configuration change, then the DML executor must be newly - // created, and we should start with the paused state. - if barrier.is_update() { + // If the first barrier requires us to pause on startup, pause the stream. + if barrier.is_pause_on_startup() { stream.pause_stream(); } diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs index e55f786aabb57..653c2c471a30d 100644 --- a/src/stream/src/executor/mod.rs +++ b/src/stream/src/executor/mod.rs @@ -235,6 +235,7 @@ pub enum Mutation { added_actors: HashSet, // TODO: remove this and use `SourceChangesSplit` after we support multiple mutations. splits: HashMap>, + pause: bool, }, SourceChangeSplit(HashMap>), Pause, @@ -321,9 +322,15 @@ impl Barrier { } } - /// Whether this barrier is for pause. - pub fn is_pause(&self) -> bool { - matches!(self.mutation.as_deref(), Some(Mutation::Pause)) + /// Whether this barrier requires the executor to pause its data stream on startup. + pub fn is_pause_on_startup(&self) -> bool { + match self.mutation.as_deref() { + Some( + Mutation::Update { .. } // new actors for scaling + | Mutation::Add { pause: true, .. } // new streaming job, or recovery + ) => true, + _ => false, + } } /// Whether this barrier is for configuration change. Used for source executor initialization. @@ -442,6 +449,7 @@ impl Mutation { adds, added_actors, splits, + pause, } => PbMutation::Add(AddMutation { actor_dispatchers: adds .iter() @@ -456,6 +464,7 @@ impl Mutation { .collect(), added_actors: added_actors.iter().copied().collect(), actor_splits: actor_splits_to_protobuf(splits), + pause: *pause, }), Mutation::SourceChangeSplit(changes) => PbMutation::Splits(SourceChangeSplitMutation { actor_splits: changes @@ -540,6 +549,7 @@ impl Mutation { ) }) .collect(), + pause: add.pause, }, PbMutation::Splits(s) => { diff --git a/src/stream/src/executor/sink.rs b/src/stream/src/executor/sink.rs index 810976a3f38dd..42656543549a5 100644 --- a/src/stream/src/executor/sink.rs +++ b/src/stream/src/executor/sink.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; @@ -25,7 +24,7 @@ use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::catalog::{ColumnCatalog, Field, Schema}; use risingwave_common::types::DataType; use risingwave_connector::dispatch_sink; -use risingwave_connector::sink::catalog::{SinkId, SinkType}; +use risingwave_connector::sink::catalog::SinkType; use risingwave_connector::sink::{ build_sink, Sink, SinkImpl, SinkParam, SinkWriter, SinkWriterParam, }; @@ -73,27 +72,13 @@ impl SinkExecutor { input: BoxedExecutor, metrics: Arc, sink_writer_param: SinkWriterParam, + sink_param: SinkParam, columns: Vec, - properties: HashMap, - pk_indices: Vec, - sink_type: SinkType, - sink_id: SinkId, actor_context: ActorContextRef, log_store_factory: F, ) -> StreamExecutorResult { let (log_reader, log_writer) = log_store_factory.build().await; - let sink_param = SinkParam { - sink_id, - properties, - columns: columns - .iter() - .filter(|col| !col.is_hidden) - .map(|col| col.column_desc.clone()) - .collect(), - pk_indices, - sink_type, - }; let sink = build_sink(sink_param.clone())?; let input_schema = columns .iter() @@ -381,15 +366,26 @@ mod test { ], ); + let sink_param = SinkParam { + sink_id: 0.into(), + properties, + columns: columns + .iter() + .filter(|col| !col.is_hidden) + .map(|col| col.column_desc.clone()) + .collect(), + pk_indices: pk.clone(), + sink_type: SinkType::ForceAppendOnly, + db_name: "test".into(), + sink_from_name: "test".into(), + }; + let sink_executor = SinkExecutor::new( Box::new(mock), Arc::new(StreamingMetrics::unused()), SinkWriterParam::default(), + sink_param, columns.clone(), - properties, - pk.clone(), - SinkType::ForceAppendOnly, - 0.into(), ActorContext::create(0), BoundedInMemLogStoreFactory::new(1), ) @@ -467,15 +463,26 @@ mod test { ], ); + let sink_param = SinkParam { + sink_id: 0.into(), + properties, + columns: columns + .iter() + .filter(|col| !col.is_hidden) + .map(|col| col.column_desc.clone()) + .collect(), + pk_indices: pk.clone(), + sink_type: SinkType::ForceAppendOnly, + db_name: "test".into(), + sink_from_name: "test".into(), + }; + let sink_executor = SinkExecutor::new( Box::new(mock), Arc::new(StreamingMetrics::unused()), SinkWriterParam::default(), + sink_param, columns, - properties, - pk.clone(), - SinkType::ForceAppendOnly, - 0.into(), ActorContext::create(0), BoundedInMemLogStoreFactory::new(1), ) diff --git a/src/stream/src/executor/source/fs_source_executor.rs b/src/stream/src/executor/source/fs_source_executor.rs index dd70e908eff03..aba89f817b09a 100644 --- a/src/stream/src/executor/source/fs_source_executor.rs +++ b/src/stream/src/executor/source/fs_source_executor.rs @@ -279,9 +279,8 @@ impl FsSourceExecutor { .build_fs_source_desc() .map_err(StreamExecutorError::connector_error)?; - // If the first barrier is configuration change, then the source executor must be newly - // created, and we should start with the paused state. - let start_with_paused = barrier.is_update(); + // If the first barrier requires us to pause on startup, pause the stream. + let start_with_paused = barrier.is_pause_on_startup(); let mut boot_state = Vec::default(); if let Some(mutation) = barrier.mutation.as_deref() { diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs index 2fdffeb475911..9e685bd67b34c 100644 --- a/src/stream/src/executor/source/source_executor.rs +++ b/src/stream/src/executor/source/source_executor.rs @@ -415,9 +415,8 @@ impl SourceExecutor { source_chunk_reader, ); - // If the first barrier is configuration change, then the source executor must be newly - // created, and we should start with the paused state. - if barrier.is_update() { + // If the first barrier requires us to pause on startup, pause the stream. + if barrier.is_pause_on_startup() { stream.pause_stream(); } @@ -727,6 +726,7 @@ mod tests { }), ], }, + pause: false, }); barrier_tx.send(init_barrier).unwrap(); @@ -818,6 +818,7 @@ mod tests { }), ], }, + pause: false, }); barrier_tx.send(init_barrier).unwrap(); diff --git a/src/stream/src/executor/values.rs b/src/stream/src/executor/values.rs index 29a9f8a77bee3..512e9f6c28da3 100644 --- a/src/stream/src/executor/values.rs +++ b/src/stream/src/executor/values.rs @@ -218,6 +218,7 @@ mod tests { adds: Default::default(), added_actors: maplit::hashset! {actor_id}, splits: Default::default(), + pause: false, }); tx.send(first_message).unwrap(); diff --git a/src/stream/src/from_proto/sink.rs b/src/stream/src/from_proto/sink.rs index d3f133822769f..0dd05d722b131 100644 --- a/src/stream/src/from_proto/sink.rs +++ b/src/stream/src/from_proto/sink.rs @@ -16,7 +16,7 @@ use std::sync::Arc; use risingwave_common::catalog::ColumnCatalog; use risingwave_connector::sink::catalog::SinkType; -use risingwave_connector::sink::SinkWriterParam; +use risingwave_connector::sink::{SinkParam, SinkWriterParam}; use risingwave_pb::stream_plan::{SinkLogStoreType, SinkNode}; use risingwave_storage::dispatch_state_store; @@ -42,6 +42,8 @@ impl ExecutorBuilder for SinkExecutorBuilder { let sink_desc = node.sink_desc.as_ref().unwrap(); let sink_type = SinkType::from_proto(sink_desc.get_sink_type().unwrap()); let sink_id = sink_desc.get_id().into(); + let db_name = sink_desc.get_db_name().into(); + let sink_from_name = sink_desc.get_sink_from_name().into(); let properties = sink_desc.get_properties().clone(); let pk_indices = sink_desc .downstream_pk @@ -54,6 +56,19 @@ impl ExecutorBuilder for SinkExecutorBuilder { .into_iter() .map(ColumnCatalog::from) .collect_vec(); + let sink_param = SinkParam { + sink_id, + properties, + columns: columns + .iter() + .filter(|col| !col.is_hidden) + .map(|col| col.column_desc.clone()) + .collect(), + pk_indices, + sink_type, + db_name, + sink_from_name, + }; match node.log_store_type() { // Default value is the normal in memory log store to be backward compatible with the @@ -70,11 +85,8 @@ impl ExecutorBuilder for SinkExecutorBuilder { vnode_bitmap: params.vnode_bitmap, meta_client: params.env.meta_client(), }, + sink_param, columns, - properties, - pk_indices, - sink_type, - sink_id, params.actor_context, factory, ) @@ -100,11 +112,8 @@ impl ExecutorBuilder for SinkExecutorBuilder { vnode_bitmap: params.vnode_bitmap, meta_client: params.env.meta_client(), }, + sink_param, columns, - properties, - pk_indices, - sink_type, - sink_id, params.actor_context, factory, ) diff --git a/src/tests/compaction_test/src/delete_range_runner.rs b/src/tests/compaction_test/src/delete_range_runner.rs index b55c6c90448b5..0b42262b09c83 100644 --- a/src/tests/compaction_test/src/delete_range_runner.rs +++ b/src/tests/compaction_test/src/delete_range_runner.rs @@ -41,6 +41,7 @@ use risingwave_pb::meta::SystemParams; use risingwave_rpc_client::HummockMetaClient; use risingwave_storage::filter_key_extractor::{ FilterKeyExtractorImpl, FilterKeyExtractorManager, FullKeyFilterKeyExtractor, + RpcFilterKeyExtractorManager, }; use risingwave_storage::hummock::compactor::{ start_compactor, CompactionExecutor, CompactorContext, @@ -212,13 +213,19 @@ async fn compaction_test( sstable_store.clone(), meta_client.clone(), get_notification_client_for_test(env, hummock_manager_ref.clone(), worker_node), - Arc::new(FilterKeyExtractorManager::default()), + Arc::new(RpcFilterKeyExtractorManager::default()), state_store_metrics.clone(), compactor_metrics.clone(), ) .await?; let sstable_object_id_manager = store.sstable_object_id_manager().clone(); - let filter_key_extractor_manager = store.filter_key_extractor_manager().clone(); + let filter_key_extractor_manager = match store.filter_key_extractor_manager().clone() { + FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + rpc_filter_key_extractor_manager, + ) => rpc_filter_key_extractor_manager, + FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(), + }; + filter_key_extractor_manager.update( 1, Arc::new(FilterKeyExtractorImpl::FullKey( @@ -560,7 +567,7 @@ fn run_compactor_thread( storage_opts: Arc, sstable_store: SstableStoreRef, meta_client: Arc, - filter_key_extractor_manager: Arc, + filter_key_extractor_manager: Arc, sstable_object_id_manager: Arc, compactor_metrics: Arc, ) -> ( @@ -574,14 +581,15 @@ fn run_compactor_thread( compactor_metrics, is_share_buffer_compact: false, compaction_executor: Arc::new(CompactionExecutor::new(None)), - filter_key_extractor_manager, + filter_key_extractor_manager: FilterKeyExtractorManager::RpcFilterKeyExtractorManager( + filter_key_extractor_manager, + ), memory_limiter: MemoryLimiter::unlimit(), - sstable_object_id_manager, task_progress_manager: Default::default(), await_tree_reg: None, running_task_count: Arc::new(AtomicU32::new(0)), }); - start_compactor(compactor_context) + start_compactor(compactor_context, sstable_object_id_manager) } #[cfg(test)] diff --git a/src/tests/simulation/Cargo.toml b/src/tests/simulation/Cargo.toml index 2e819327962a1..dd21ac0ac6949 100644 --- a/src/tests/simulation/Cargo.toml +++ b/src/tests/simulation/Cargo.toml @@ -14,6 +14,7 @@ normal = ["serde"] anyhow = "1.0" async-trait = "0.1" aws-sdk-s3 = { version = "0.2", package = "madsim-aws-sdk-s3" } +cfg-or-panic = "0.1" clap = { version = "4", features = ["derive"] } console = "0.15" etcd-client = { workspace = true } diff --git a/src/tests/simulation/src/client.rs b/src/tests/simulation/src/client.rs index 498aaa2500edf..089d67bceeeab 100644 --- a/src/tests/simulation/src/client.rs +++ b/src/tests/simulation/src/client.rs @@ -64,7 +64,7 @@ impl<'a, 'b> SetStmtsIterator<'a, 'b> { impl SetStmts { fn push(&mut self, sql: &str) { - let ast = Parser::parse_sql(&sql).expect("a set statement should be parsed successfully"); + let ast = Parser::parse_sql(sql).expect("a set statement should be parsed successfully"); match ast .into_iter() .exactly_one() @@ -134,7 +134,7 @@ impl RisingWave { .simple_query("SET RW_IMPLICIT_FLUSH TO true;") .await?; // replay all SET statements - for stmt in SetStmtsIterator::new(&set_stmts) { + for stmt in SetStmtsIterator::new(set_stmts) { client.simple_query(&stmt).await?; } Ok((client, task)) diff --git a/src/tests/simulation/src/cluster.rs b/src/tests/simulation/src/cluster.rs index 9c9b426a04001..95d3c90a81656 100644 --- a/src/tests/simulation/src/cluster.rs +++ b/src/tests/simulation/src/cluster.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![cfg_attr(not(madsim), allow(unused_imports))] + use std::collections::HashMap; use std::future::Future; use std::io::Write; @@ -20,17 +22,20 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{anyhow, bail, Result}; +use cfg_or_panic::cfg_or_panic; use clap::Parser; use futures::channel::{mpsc, oneshot}; use futures::future::join_all; use futures::{SinkExt, StreamExt}; use itertools::Itertools; -use madsim::net::ipvs::*; +#[cfg(madsim)] use madsim::runtime::{Handle, NodeHandle}; use rand::seq::IteratorRandom; use rand::Rng; use risingwave_pb::common::WorkerNode; use sqllogictest::AsyncDB; +#[cfg(not(madsim))] +use tokio::runtime::Handle; use crate::client::RisingWave; @@ -153,7 +158,9 @@ impl Configuration { pub struct Cluster { config: Configuration, handle: Handle, + #[cfg(madsim)] pub(crate) client: NodeHandle, + #[cfg(madsim)] pub(crate) ctl: NodeHandle, } @@ -161,7 +168,10 @@ impl Cluster { /// Start a RisingWave cluster for testing. /// /// This function should be called exactly once in a test. + #[cfg_or_panic(madsim)] pub async fn start(conf: Configuration) -> Result { + use madsim::net::ipvs::*; + let handle = madsim::runtime::Handle::current(); println!("seed = {}", handle.seed()); println!("{:#?}", conf); @@ -361,6 +371,7 @@ impl Cluster { } /// Start a SQL session on the client node. + #[cfg_or_panic(madsim)] pub fn start_session(&mut self) -> Session { let (query_tx, mut query_rx) = mpsc::channel::(0); @@ -404,6 +415,7 @@ impl Cluster { } /// Run a future on the client node. + #[cfg_or_panic(madsim)] pub async fn run_on_client(&self, future: F) -> F::Output where F: Future + Send + 'static, @@ -433,7 +445,7 @@ impl Cluster { timeout: Duration, ) -> Result { let fut = async move { - let mut interval = madsim::time::interval(interval); + let mut interval = tokio::time::interval(interval); loop { interval.tick().await; let result = self.run(sql.clone()).await?; @@ -443,7 +455,7 @@ impl Cluster { } }; - match madsim::time::timeout(timeout, fut).await { + match tokio::time::timeout(timeout, fut).await { Ok(r) => Ok(r?), Err(_) => bail!("wait_until timeout"), } @@ -460,7 +472,8 @@ impl Cluster { .await } - /// Kill some nodes and restart them in 2s + restart_delay_secs with a probability of 0.1. + /// Generate a list of random worker nodes to kill by `opts`, then call `kill_nodes` to kill and + /// restart them. pub async fn kill_node(&self, opts: &KillOpts) { let mut nodes = vec![]; if opts.kill_meta { @@ -515,11 +528,24 @@ impl Cluster { nodes.push(format!("compactor-{}", i)); } } - join_all(nodes.iter().map(|name| async move { + + self.kill_nodes(nodes, opts.restart_delay_secs).await + } + + /// Kill the given nodes by their names and restart them in 2s + restart_delay_secs with a + /// probability of 0.1. + #[cfg_or_panic(madsim)] + pub async fn kill_nodes( + &self, + nodes: impl IntoIterator>, + restart_delay_secs: u32, + ) { + join_all(nodes.into_iter().map(|name| async move { + let name = name.as_ref(); let t = rand::thread_rng().gen_range(Duration::from_secs(0)..Duration::from_secs(1)); tokio::time::sleep(t).await; tracing::info!("kill {name}"); - madsim::runtime::Handle::current().kill(name); + Handle::current().kill(name); let mut t = rand::thread_rng().gen_range(Duration::from_secs(0)..Duration::from_secs(1)); @@ -527,16 +553,17 @@ impl Cluster { // so that the node is expired and removed from the cluster if rand::thread_rng().gen_bool(0.1) { // max_heartbeat_interval_secs = 15 - t += Duration::from_secs(opts.restart_delay_secs as u64); + t += Duration::from_secs(restart_delay_secs as u64); } tokio::time::sleep(t).await; tracing::info!("restart {name}"); - madsim::runtime::Handle::current().restart(name); + Handle::current().restart(name); })) .await; } /// Create a node for kafka producer and prepare data. + #[cfg_or_panic(madsim)] pub async fn create_kafka_producer(&self, datadir: &str) { self.handle .create_node() @@ -552,6 +579,7 @@ impl Cluster { } /// Create a kafka topic. + #[cfg_or_panic(madsim)] pub fn create_kafka_topics(&self, topics: HashMap) { self.handle .create_node() @@ -570,6 +598,7 @@ impl Cluster { } /// Graceful shutdown all RisingWave nodes. + #[cfg_or_panic(madsim)] pub async fn graceful_shutdown(&self) { let mut nodes = vec![]; let mut metas = vec![]; @@ -592,12 +621,12 @@ impl Cluster { for node in &nodes { self.handle.send_ctrl_c(node); } - madsim::time::sleep(waiting_time).await; + tokio::time::sleep(waiting_time).await; // shutdown metas for meta in &metas { self.handle.send_ctrl_c(meta); } - madsim::time::sleep(waiting_time).await; + tokio::time::sleep(waiting_time).await; // check all nodes are exited for node in nodes.iter().chain(metas.iter()) { diff --git a/src/tests/simulation/src/ctl_ext.rs b/src/tests/simulation/src/ctl_ext.rs index feb2b40be7986..850f881031e73 100644 --- a/src/tests/simulation/src/ctl_ext.rs +++ b/src/tests/simulation/src/ctl_ext.rs @@ -12,18 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![cfg_attr(not(madsim), expect(unused_imports))] + use std::collections::{HashMap, HashSet}; use std::fmt::Write; use std::sync::Arc; use anyhow::{anyhow, Result}; +use cfg_or_panic::cfg_or_panic; use clap::Parser; use itertools::Itertools; -use madsim::rand::thread_rng; use rand::seq::{IteratorRandom, SliceRandom}; -use rand::Rng; +use rand::{thread_rng, Rng}; use risingwave_common::hash::ParallelUnitId; -use risingwave_pb::common::{HostAddress, WorkerNode}; use risingwave_pb::meta::get_reschedule_plan_request::PbPolicy; use risingwave_pb::meta::table_fragments::fragment::FragmentDistributionType; use risingwave_pb::meta::table_fragments::PbFragment; @@ -219,6 +220,7 @@ impl Fragment { impl Cluster { /// Locate fragments that satisfy all the predicates. + #[cfg_or_panic(madsim)] pub async fn locate_fragments( &mut self, predicates: impl IntoIterator, @@ -291,6 +293,7 @@ impl Cluster { self.locate_one_fragment([predicate::id(id)]).await } + #[cfg_or_panic(madsim)] pub async fn get_cluster_info(&self) -> Result { let response = self .ctl @@ -305,6 +308,7 @@ impl Cluster { } // update node schedulability + #[cfg_or_panic(madsim)] async fn update_worker_node_schedulability( &self, worker_ids: Vec, @@ -345,11 +349,12 @@ impl Cluster { self.reschedule_helper(plan, false).await } - /// Same as reschedule, but resolve the no_shuffle upstream + /// Same as reschedule, but resolve the no-shuffle upstream pub async fn reschedule_resolve_no_shuffle(&mut self, plan: impl Into) -> Result<()> { self.reschedule_helper(plan, true).await } + #[cfg_or_panic(madsim)] async fn reschedule_helper( &mut self, plan: impl Into, @@ -394,6 +399,33 @@ impl Cluster { Ok(()) } + /// Pause all data sources in the cluster. + #[cfg_or_panic(madsim)] + pub async fn pause(&mut self) -> Result<()> { + self.ctl + .spawn(async move { + let opts = risingwave_ctl::CliOpts::parse_from(["ctl", "meta", "pause"]); + risingwave_ctl::start(opts).await + }) + .await??; + + Ok(()) + } + + /// Resume all data sources in the cluster. + #[cfg_or_panic(madsim)] + pub async fn resume(&mut self) -> Result<()> { + self.ctl + .spawn(async move { + let opts = risingwave_ctl::CliOpts::parse_from(["ctl", "meta", "resume"]); + risingwave_ctl::start(opts).await + }) + .await??; + + Ok(()) + } + + #[cfg_or_panic(madsim)] pub async fn get_reschedule_plan(&self, policy: PbPolicy) -> Result { let revision = self .ctl diff --git a/src/tests/simulation/src/kafka.rs b/src/tests/simulation/src/kafka.rs index b62072961e629..0ab4829d9e1b8 100644 --- a/src/tests/simulation/src/kafka.rs +++ b/src/tests/simulation/src/kafka.rs @@ -15,9 +15,10 @@ use std::collections::HashMap; use std::time::SystemTime; +use itertools::Either; use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication}; use rdkafka::error::{KafkaError, RDKafkaErrorCode}; -use rdkafka::producer::{BaseProducer, BaseRecord}; +use rdkafka::producer::{BaseProducer, BaseRecord, Producer}; use rdkafka::ClientConfig; /// Create a kafka topic @@ -82,22 +83,21 @@ pub async fn producer(broker_addr: &str, datadir: String) { .expect("failed to create topic"); let content = std::fs::read(file.path()).unwrap(); - let msgs: Box, &[u8])> + Send> = - if topic.ends_with("bin") { - // binary message data, a file is a message - Box::new(std::iter::once((None, content.as_slice()))) - } else { - // text message data, a line is a message - Box::new( - content - .split(|&b| b == b'\n') - .filter(|line| !line.is_empty()) - .map(|line| match line.iter().position(|&b| b == KEY_DELIMITER) { - Some(pos) => (Some(&line[..pos]), &line[pos + 1..]), - None => (None, line), - }), - ) - }; + let msgs = if topic.ends_with("bin") { + // binary message data, a file is a message + Either::Left(std::iter::once((None, content.as_slice()))) + } else { + // text message data, a line is a message + Either::Right( + content + .split(|&b| b == b'\n') + .filter(|line| !line.is_empty()) + .map(|line| match line.iter().position(|&b| b == KEY_DELIMITER) { + Some(pos) => (Some(&line[..pos]), &line[pos + 1..]), + None => (None, line), + }), + ) + }; for (key, payload) in msgs { loop { let ts = SystemTime::now() diff --git a/src/tests/simulation/src/lib.rs b/src/tests/simulation/src/lib.rs index 8e0bd81ad6eec..68c1d0446944d 100644 --- a/src/tests/simulation/src/lib.rs +++ b/src/tests/simulation/src/lib.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![cfg(madsim)] #![feature(trait_alias)] #![feature(lint_reasons)] #![feature(lazy_cell)] diff --git a/src/tests/simulation/src/main.rs b/src/tests/simulation/src/main.rs index c37af587553f3..a3aa4ca056415 100644 --- a/src/tests/simulation/src/main.rs +++ b/src/tests/simulation/src/main.rs @@ -17,13 +17,9 @@ use std::path::PathBuf; +use cfg_or_panic::cfg_or_panic; use clap::Parser; -#[cfg(not(madsim))] -fn main() { - println!("This binary is only available in simulation."); -} - /// Deterministic simulation end-to-end test runner. /// /// ENVS: @@ -145,8 +141,8 @@ pub struct Args { e2e_extended_test: bool, } -#[cfg(madsim)] -#[madsim::main] +#[tokio::main] +#[cfg_or_panic(madsim)] async fn main() { use std::sync::Arc; diff --git a/src/tests/simulation/src/slt.rs b/src/tests/simulation/src/slt.rs index b78b11afb93a9..9df28c1ee24ff 100644 --- a/src/tests/simulation/src/slt.rs +++ b/src/tests/simulation/src/slt.rs @@ -24,11 +24,7 @@ use crate::cluster::{Cluster, KillOpts}; use crate::utils::TimedExt; fn is_create_table_as(sql: &str) -> bool { - let parts: Vec = sql - .trim_start() - .split_whitespace() - .map(|s| s.to_lowercase()) - .collect(); + let parts: Vec = sql.split_whitespace().map(|s| s.to_lowercase()).collect(); parts.len() >= 4 && parts[0] == "create" && parts[1] == "table" && parts[3] == "as" } diff --git a/src/tests/simulation/tests/integration_tests/backfill_tests.rs b/src/tests/simulation/tests/integration_tests/backfill_tests.rs index d42d4b3d500af..813bc3c0b7257 100644 --- a/src/tests/simulation/tests/integration_tests/backfill_tests.rs +++ b/src/tests/simulation/tests/integration_tests/backfill_tests.rs @@ -13,16 +13,14 @@ // limitations under the License. use anyhow::Result; -use itertools::{any, Itertools}; +use itertools::Itertools; use risingwave_simulation::cluster::{Cluster, Configuration}; const SET_PARALLELISM: &str = "SET STREAMING_PARALLELISM=1;"; const ROOT_TABLE_CREATE: &str = "create table t1 (_id int, data jsonb);"; const INSERT_SEED_SQL: &str = r#"insert into t1 values (1, '{"orders": {"id": 1, "price": "2.30", "customer_id": 2}}');"#; -const INSERT_AND_FLUSH_SQL: &str = r#"insert into t1 values (1, '{"orders": {"id": 1, "price": "2.30", "customer_id": 2}}'); FLUSH;"#; const INSERT_RECURSE_SQL: &str = "insert into t1 select _id + 1, data from t1;"; -const INSERT_RECURSE_AND_FLUSH_SQL: &str = "insert into t1 select _id + 1, data from t1;"; const MV1: &str = r#" create materialized view mv1 as with p1 as ( @@ -47,7 +45,7 @@ select from p2; "#; -#[madsim::test] +#[tokio::test] async fn test_backfill_with_upstream_and_snapshot_read() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_backfill()).await?; let mut session = cluster.start_session(); @@ -70,7 +68,7 @@ async fn test_backfill_with_upstream_and_snapshot_read() -> Result<()> { let sessions = (0..3).map(|_| cluster.start_session()).collect_vec(); // Create lots of base table update - for mut session in sessions.into_iter() { + for mut session in sessions { let task = tokio::spawn(async move { session.run(INSERT_RECURSE_SQL).await?; anyhow::Ok(()) @@ -82,7 +80,7 @@ async fn test_backfill_with_upstream_and_snapshot_read() -> Result<()> { let sessions = (0..10).map(|_| cluster.start_session()).collect_vec(); // Create lots of base table update - for mut session in sessions.into_iter() { + for mut session in sessions { let task = tokio::spawn(async move { for _ in 0..10 { session.run("FLUSH;").await?; diff --git a/src/tests/simulation/tests/integration_tests/batch/mod.rs b/src/tests/simulation/tests/integration_tests/batch/mod.rs index 52bce7212c41b..dfe5de1405bf9 100644 --- a/src/tests/simulation/tests/integration_tests/batch/mod.rs +++ b/src/tests/simulation/tests/integration_tests/batch/mod.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![cfg(madsim)] + use std::io::Write; -use anyhow::Result; use clap::Parser; use itertools::Itertools; -use madsim::runtime::Handle; use risingwave_simulation::cluster::{Cluster, ConfigPath, Configuration, Session}; use tokio::time::Duration; @@ -81,7 +81,7 @@ telemetry_enabled = false } } -#[madsim::test] +#[tokio::test] async fn test_serving_cluster_availability() { let config = cluster_config_no_compute_nodes(); let mut cluster = Cluster::start(config).await.unwrap(); diff --git a/src/tests/simulation/tests/integration_tests/main.rs b/src/tests/simulation/tests/integration_tests/main.rs index 0f4c10c8569ca..82ed948b39a51 100644 --- a/src/tests/simulation/tests/integration_tests/main.rs +++ b/src/tests/simulation/tests/integration_tests/main.rs @@ -18,7 +18,6 @@ //! for the rationale behind this approach. #![feature(stmt_expr_attributes)] -#![cfg(madsim)] #![feature(lazy_cell)] #![feature(drain_filter)] diff --git a/src/tests/simulation/tests/integration_tests/recovery/backfill.rs b/src/tests/simulation/tests/integration_tests/recovery/backfill.rs index a83c217cf151d..e907528826864 100644 --- a/src/tests/simulation/tests/integration_tests/recovery/backfill.rs +++ b/src/tests/simulation/tests/integration_tests/recovery/backfill.rs @@ -17,10 +17,10 @@ use std::time::Duration; use anyhow::Result; use itertools::Itertools; -use madsim::time::sleep; use risingwave_simulation::cluster::{Cluster, Configuration, Session}; use risingwave_simulation::ctl_ext::predicate::{identity_contains, no_identity_contains}; use risingwave_simulation::utils::AssertResult; +use tokio::time::sleep; const ROOT_TABLE_CREATE: &str = "create table t1 (v1 int);"; const ROOT_TABLE_DROP: &str = "drop table t1;"; @@ -46,7 +46,7 @@ async fn test_no_backfill_state(session: &mut Session) -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_snapshot_mv() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); @@ -98,7 +98,7 @@ async fn test_snapshot_mv() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_backfill_mv() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); @@ -151,7 +151,7 @@ async fn test_backfill_mv() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_index_backfill() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); diff --git a/src/tests/simulation/tests/integration_tests/recovery/mod.rs b/src/tests/simulation/tests/integration_tests/recovery/mod.rs index 3b45a0d63853e..565487e8d7dbd 100644 --- a/src/tests/simulation/tests/integration_tests/recovery/mod.rs +++ b/src/tests/simulation/tests/integration_tests/recovery/mod.rs @@ -14,3 +14,4 @@ mod backfill; mod nexmark_recovery; +mod pause_on_bootstrap; diff --git a/src/tests/simulation/tests/integration_tests/recovery/nexmark_recovery.rs b/src/tests/simulation/tests/integration_tests/recovery/nexmark_recovery.rs index 1720c2edc21e7..edd419adabf66 100644 --- a/src/tests/simulation/tests/integration_tests/recovery/nexmark_recovery.rs +++ b/src/tests/simulation/tests/integration_tests/recovery/nexmark_recovery.rs @@ -15,10 +15,10 @@ use std::time::Duration; use anyhow::Result; -use madsim::time::{sleep, Instant}; use risingwave_simulation::cluster::{Configuration, KillOpts}; -use risingwave_simulation::nexmark::{self, NexmarkCluster, THROUGHPUT}; +use risingwave_simulation::nexmark::{NexmarkCluster, THROUGHPUT}; use risingwave_simulation::utils::AssertResult; +use tokio::time::sleep; /// Setup a nexmark stream, inject failures, and verify results. async fn nexmark_recovery_common(create: &str, select: &str, drop: &str) -> Result<()> { @@ -54,7 +54,7 @@ async fn nexmark_recovery_common(create: &str, select: &str, drop: &str) -> Resu macro_rules! test { ($query:ident) => { paste::paste! { - #[madsim::test] + #[tokio::test] async fn [< nexmark_recovery_ $query >]() -> Result<()> { use risingwave_simulation::nexmark::queries::$query::*; nexmark_recovery_common(CREATE, SELECT, DROP) diff --git a/src/tests/simulation/tests/integration_tests/recovery/pause_on_bootstrap.rs b/src/tests/simulation/tests/integration_tests/recovery/pause_on_bootstrap.rs new file mode 100644 index 0000000000000..d0288e6931e88 --- /dev/null +++ b/src/tests/simulation/tests/integration_tests/recovery/pause_on_bootstrap.rs @@ -0,0 +1,125 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; + +use anyhow::Result; +use risingwave_simulation::cluster::Configuration; +use risingwave_simulation::nexmark::NexmarkCluster; +use risingwave_simulation::utils::AssertResult; +use tokio::time::{sleep, timeout}; + +const CREATE_TABLE: &str = "CREATE TABLE t (v int)"; +const INSERT_INTO_TABLE: &str = "INSERT INTO t VALUES (1)"; +const SELECT_COUNT_TABLE: &str = "SELECT COUNT(*) FROM t"; + +const CREATE: &str = "CREATE MATERIALIZED VIEW count_bid as SELECT COUNT(*) FROM bid"; +const SELECT: &str = "SELECT * FROM count_bid"; + +const CREATE_2: &str = "CREATE MATERIALIZED VIEW count_auction as SELECT COUNT(*) FROM auction"; +const SELECT_2: &str = "SELECT * FROM count_auction"; + +const SET_PARAMETER: &str = "ALTER SYSTEM SET pause_on_next_bootstrap TO true"; + +enum ResumeBy { + Risectl, + Restart, +} + +async fn test_impl(resume_by: ResumeBy) -> Result<()> { + let mut cluster = NexmarkCluster::new( + Configuration { + meta_nodes: 1, + ..Configuration::for_scale() + }, + 6, + None, + false, + ) + .await?; + + cluster.run(SET_PARAMETER).await?; + cluster.run(CREATE).await?; + cluster.run(CREATE_TABLE).await?; + + // Run for a while. + sleep(Duration::from_secs(10)).await; + + // Kill the meta node and wait for the service to recover. + cluster.kill_nodes(["meta-1"], 0).await; + sleep(Duration::from_secs(10)).await; + + // The source should be paused. + let count = cluster.run(SELECT).await?; + sleep(Duration::from_secs(10)).await; + cluster.run(SELECT).await?.assert_result_eq(&count); + + // Scaling will trigger a pair of `Pause` and `Resume`. However, this should not affect the + // "manual" pause. + let random_fragment_id = cluster.locate_random_fragment().await?; + cluster + .reschedule(random_fragment_id.random_reschedule()) + .await?; + sleep(Duration::from_secs(10)).await; + cluster.run(SELECT).await?.assert_result_eq(&count); + + // New streaming jobs should also start from paused. + cluster.run(CREATE_2).await?; + sleep(Duration::from_secs(10)).await; + cluster.run(SELECT_2).await?.assert_result_eq("0"); // even there's no data from source, the + // result will be 0 instead of empty or NULL + + // DML on tables should be blocked. + let result = timeout(Duration::from_secs(10), cluster.run(INSERT_INTO_TABLE)).await; + assert!(result.is_err()); + cluster.run(SELECT_COUNT_TABLE).await?.assert_result_eq("0"); + + match resume_by { + ResumeBy::Risectl => cluster.resume().await?, + ResumeBy::Restart => cluster.kill_nodes(["meta-1"], 0).await, + } + sleep(Duration::from_secs(10)).await; + + // The source should be resumed. + let new_count = cluster.run(SELECT).await?; + assert_ne!(count, new_count); + + // DML on tables should be allowed. However, we're uncertain whether the previous blocked DML is + // executed or not. So we just check the count difference. + { + let mut session = cluster.start_session(); + + session.run("FLUSH").await?; + let count: i64 = session.run(SELECT_COUNT_TABLE).await?.parse().unwrap(); + + session.run(INSERT_INTO_TABLE).await?; + session.run("FLUSH").await?; + session + .run(SELECT_COUNT_TABLE) + .await? + .assert_result_eq(format!("{}", count + 1)); + } + + Ok(()) +} + +#[tokio::test] +async fn test_pause_on_bootstrap_resume_by_risectl() -> Result<()> { + test_impl(ResumeBy::Risectl).await +} + +#[tokio::test] +async fn test_pause_on_bootstrap_resume_by_restart() -> Result<()> { + test_impl(ResumeBy::Restart).await +} diff --git a/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs b/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs index 968762570ec79..c05e52c927424 100644 --- a/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs +++ b/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs @@ -16,10 +16,10 @@ use std::time::Duration; use anyhow::Result; use itertools::Itertools; -use madsim::time::sleep; use risingwave_simulation::cluster::{Cluster, Configuration}; use risingwave_simulation::ctl_ext::predicate::{identity_contains, no_identity_contains}; use risingwave_simulation::utils::AssertResult; +use tokio::time::sleep; const ROOT_TABLE_CREATE: &str = "create table t1 (v1 int);"; const MV1: &str = "create materialized view m1 as select * from t1 where v1 > 5;"; @@ -28,7 +28,7 @@ const MV3: &str = "create materialized view m3 as select * from m2 where v1 < 15 const MV4: &str = "create materialized view m4 as select m1.v1 as m1v, m3.v1 as m3v from m1 join m3 on m1.v1 = m3.v1;"; const MV5: &str = "create materialized view m5 as select * from m4;"; -#[madsim::test] +#[tokio::test] async fn test_simple_cascade_materialized_view() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); @@ -113,7 +113,7 @@ async fn test_simple_cascade_materialized_view() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_diamond_cascade_materialized_view() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); diff --git a/src/tests/simulation/tests/integration_tests/scale/dynamic_filter.rs b/src/tests/simulation/tests/integration_tests/scale/dynamic_filter.rs index a9028836a8c54..e8e1af703c1c2 100644 --- a/src/tests/simulation/tests/integration_tests/scale/dynamic_filter.rs +++ b/src/tests/simulation/tests/integration_tests/scale/dynamic_filter.rs @@ -16,14 +16,14 @@ use std::collections::HashSet; use std::time::Duration; use anyhow::Result; -use madsim::time::sleep; use risingwave_simulation::cluster::{Cluster, Configuration}; use risingwave_simulation::ctl_ext::predicate::identity_contains; use risingwave_simulation::utils::AssertResult; +use tokio::time::sleep; const SELECT: &str = "select * from mv1 order by v1;"; -#[madsim::test] +#[tokio::test] async fn test_dynamic_filter() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); diff --git a/src/tests/simulation/tests/integration_tests/scale/mod.rs b/src/tests/simulation/tests/integration_tests/scale/mod.rs index 2fb163ffea443..5270247bf3997 100644 --- a/src/tests/simulation/tests/integration_tests/scale/mod.rs +++ b/src/tests/simulation/tests/integration_tests/scale/mod.rs @@ -21,5 +21,6 @@ mod no_shuffle; mod plan; mod schedulability; mod singleton_migration; +mod sink; mod streaming_parallelism; mod table; diff --git a/src/tests/simulation/tests/integration_tests/scale/nexmark_chaos.rs b/src/tests/simulation/tests/integration_tests/scale/nexmark_chaos.rs index a06fb25fe7a18..6ed30a7452dcc 100644 --- a/src/tests/simulation/tests/integration_tests/scale/nexmark_chaos.rs +++ b/src/tests/simulation/tests/integration_tests/scale/nexmark_chaos.rs @@ -17,11 +17,11 @@ use std::time::Duration; use anyhow::Result; use futures::future::BoxFuture; use itertools::Itertools; -use madsim::time::sleep; use risingwave_simulation::cluster::Configuration; use risingwave_simulation::ctl_ext::Fragment; use risingwave_simulation::nexmark::{NexmarkCluster, THROUGHPUT}; use risingwave_simulation::utils::AssertResult; +use tokio::time::sleep; /// Common code for Nexmark chaos tests. /// @@ -117,7 +117,7 @@ macro_rules! test { }; ($query:ident, $after_scale_duration:expr) => { paste::paste! { - #[madsim::test] + #[tokio::test] async fn [< nexmark_chaos_ $query _single >]() -> Result<()> { use risingwave_simulation::nexmark::queries::$query::*; nexmark_chaos_common( @@ -132,7 +132,7 @@ macro_rules! test { .await } - #[madsim::test] + #[tokio::test] async fn [< nexmark_chaos_ $query _multiple >]() -> Result<()> { use risingwave_simulation::nexmark::queries::$query::*; nexmark_chaos_common( diff --git a/src/tests/simulation/tests/integration_tests/scale/nexmark_q4.rs b/src/tests/simulation/tests/integration_tests/scale/nexmark_q4.rs index ed3f3d4ad580f..41a90fcf5d0d5 100644 --- a/src/tests/simulation/tests/integration_tests/scale/nexmark_q4.rs +++ b/src/tests/simulation/tests/integration_tests/scale/nexmark_q4.rs @@ -15,7 +15,6 @@ use std::time::Duration; use anyhow::Result; -use madsim::time::sleep; use risingwave_simulation::cluster::Configuration; use risingwave_simulation::ctl_ext::predicate::{ identity_contains, upstream_fragment_count, BoxedPredicate, @@ -23,6 +22,7 @@ use risingwave_simulation::ctl_ext::predicate::{ use risingwave_simulation::nexmark::queries::q4::*; use risingwave_simulation::nexmark::{NexmarkCluster, THROUGHPUT}; use risingwave_simulation::utils::AssertResult; +use tokio::time::sleep; #[cfg(target_os = "linux")] const RESULT: &str = r#" @@ -55,7 +55,7 @@ async fn wait_initial_data(cluster: &mut NexmarkCluster) -> Result { .await } -#[madsim::test] +#[tokio::test] async fn nexmark_q4_ref() -> Result<()> { let mut cluster = init().await?; @@ -93,7 +93,7 @@ async fn nexmark_q4_common(predicates: impl IntoIterator) Ok(()) } -#[madsim::test] +#[tokio::test] async fn nexmark_q4_materialize_agg() -> Result<()> { nexmark_q4_common([ identity_contains("materialize"), @@ -102,12 +102,12 @@ async fn nexmark_q4_materialize_agg() -> Result<()> { .await } -#[madsim::test] +#[tokio::test] async fn nexmark_q4_source() -> Result<()> { nexmark_q4_common([identity_contains("source: bid")]).await } -#[madsim::test] +#[tokio::test] async fn nexmark_q4_agg_join() -> Result<()> { nexmark_q4_common([ identity_contains("hashagg"), @@ -117,7 +117,7 @@ async fn nexmark_q4_agg_join() -> Result<()> { .await } -#[madsim::test] +#[tokio::test] async fn nexmark_q4_cascade() -> Result<()> { let mut cluster = init().await?; @@ -165,7 +165,7 @@ async fn nexmark_q4_cascade() -> Result<()> { } // https://github.com/risingwavelabs/risingwave/issues/5567 -#[madsim::test] +#[tokio::test] async fn nexmark_q4_materialize_agg_cache_invalidation() -> Result<()> { let mut cluster = init().await?; diff --git a/src/tests/simulation/tests/integration_tests/scale/nexmark_source.rs b/src/tests/simulation/tests/integration_tests/scale/nexmark_source.rs index d1d6d1814f38b..864195d0efd7b 100644 --- a/src/tests/simulation/tests/integration_tests/scale/nexmark_source.rs +++ b/src/tests/simulation/tests/integration_tests/scale/nexmark_source.rs @@ -15,17 +15,17 @@ use std::time::Duration; use anyhow::Result; -use madsim::time::sleep; use risingwave_simulation::cluster::Configuration; use risingwave_simulation::ctl_ext::predicate::identity_contains; use risingwave_simulation::nexmark::{NexmarkCluster, THROUGHPUT}; +use tokio::time::sleep; -#[madsim::test] +#[tokio::test] async fn nexmark_source() -> Result<()> { nexmark_source_inner(false).await } -#[madsim::test] +#[tokio::test] async fn nexmark_source_with_watermark() -> Result<()> { nexmark_source_inner(true).await } diff --git a/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs b/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs index 3a8402d5a83ae..0fde294d08e7f 100644 --- a/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs +++ b/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs @@ -18,7 +18,7 @@ use risingwave_simulation::cluster::{Cluster, Configuration}; use risingwave_simulation::ctl_ext::predicate::identity_contains; use risingwave_simulation::utils::AssertResult; -#[madsim::test] +#[tokio::test] async fn test_delta_join() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); @@ -120,7 +120,7 @@ async fn test_delta_join() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_share_multiple_no_shuffle_upstream() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); @@ -140,7 +140,7 @@ async fn test_share_multiple_no_shuffle_upstream() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_resolve_no_shuffle_upstream() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); diff --git a/src/tests/simulation/tests/integration_tests/scale/plan.rs b/src/tests/simulation/tests/integration_tests/scale/plan.rs index 5899ab7ba43b3..fdb72d35830a1 100644 --- a/src/tests/simulation/tests/integration_tests/scale/plan.rs +++ b/src/tests/simulation/tests/integration_tests/scale/plan.rs @@ -26,7 +26,7 @@ use risingwave_pb::meta::PbReschedule; use risingwave_simulation::cluster::{Cluster, Configuration}; use risingwave_simulation::ctl_ext::predicate::{identity_contains, no_identity_contains}; -#[madsim::test] +#[tokio::test] async fn test_resize_normal() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); @@ -92,7 +92,7 @@ async fn test_resize_normal() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_resize_single() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); @@ -135,7 +135,7 @@ async fn test_resize_single() -> Result<()> { .parallel_units .iter() .map(|parallel_unit| parallel_unit.id) - .contains(&used_parallel_unit_id) + .contains(used_parallel_unit_id) }) .collect_vec(); @@ -175,7 +175,7 @@ async fn test_resize_single() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_resize_single_failed() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); @@ -242,7 +242,7 @@ async fn test_resize_single_failed() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_resize_no_shuffle() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); diff --git a/src/tests/simulation/tests/integration_tests/scale/schedulability.rs b/src/tests/simulation/tests/integration_tests/scale/schedulability.rs index ed5c5ec468771..5ad82b5326732 100644 --- a/src/tests/simulation/tests/integration_tests/scale/schedulability.rs +++ b/src/tests/simulation/tests/integration_tests/scale/schedulability.rs @@ -19,7 +19,7 @@ use risingwave_common::hash::ParallelUnitId; use risingwave_pb::common::{WorkerNode, WorkerType}; use risingwave_simulation::cluster::{Cluster, Configuration}; -#[madsim::test] +#[tokio::test] async fn test_cordon_normal() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); @@ -78,7 +78,7 @@ async fn test_cordon_normal() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_cordon_no_shuffle_failed() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); diff --git a/src/tests/simulation/tests/integration_tests/scale/singleton_migration.rs b/src/tests/simulation/tests/integration_tests/scale/singleton_migration.rs index 51c4cb17a6387..6abd980bea3bd 100644 --- a/src/tests/simulation/tests/integration_tests/scale/singleton_migration.rs +++ b/src/tests/simulation/tests/integration_tests/scale/singleton_migration.rs @@ -16,18 +16,18 @@ use std::time::Duration; use anyhow::Result; use itertools::Itertools; -use madsim::time::sleep; use rand::prelude::SliceRandom; use rand::thread_rng; use risingwave_simulation::cluster::{Cluster, Configuration}; use risingwave_simulation::ctl_ext::predicate::identity_contains; use risingwave_simulation::utils::AssertResult; +use tokio::time::sleep; const ROOT_TABLE_CREATE: &str = "create table t (v1 int);"; const ROOT_MV: &str = "create materialized view m1 as select count(*) as c1 from t;"; const CASCADE_MV: &str = "create materialized view m2 as select * from m1;"; -#[madsim::test] +#[tokio::test] async fn test_singleton_migration() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); diff --git a/src/tests/simulation/tests/integration_tests/scale/sink.rs b/src/tests/simulation/tests/integration_tests/scale/sink.rs index 414acd0ad05e6..99f150e85b8a1 100644 --- a/src/tests/simulation/tests/integration_tests/scale/sink.rs +++ b/src/tests/simulation/tests/integration_tests/scale/sink.rs @@ -16,15 +16,16 @@ use std::collections::HashMap; use std::time::Duration; use anyhow::Result; -use madsim::export::futures::StreamExt; -use madsim::rand::prelude::SliceRandom; -use madsim::rand::thread_rng; -use madsim::time; -use rdkafka::consumer::{MessageStream, StreamConsumer}; +use futures::StreamExt; +use rand::prelude::SliceRandom; +use rand::thread_rng; +use rdkafka::consumer::{Consumer, StreamConsumer}; +use rdkafka::error::KafkaResult; use rdkafka::message::BorrowedMessage; use rdkafka::{ClientConfig, Message, TopicPartitionList}; use risingwave_simulation::cluster::{Cluster, Configuration}; use risingwave_simulation::ctl_ext::predicate::{identity_contains, no_identity_contains}; +use tokio::time; const ROOT_TABLE_CREATE: &str = "create table t (v1 int) append only;"; const APPEND_ONLY_SINK_CREATE: &str = "create sink s1 from t with (connector='kafka', properties.bootstrap.server='192.168.11.1:29092', topic='t_sink_append_only', type='append-only');"; @@ -70,7 +71,8 @@ pub struct Before { pub count: i64, } -#[madsim::test] +#[tokio::test] +#[ignore] // https://github.com/risingwavelabs/risingwave/issues/12003 async fn test_sink_append_only() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; @@ -122,7 +124,8 @@ async fn test_sink_append_only() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] +#[ignore] // https://github.com/risingwavelabs/risingwave/issues/12003 async fn test_sink_debezium() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; @@ -201,7 +204,7 @@ async fn test_sink_debezium() -> Result<()> { Ok(()) } -fn check_payload(msg: &BorrowedMessage, payload: &[u8], i: i64) { +fn check_payload(msg: &BorrowedMessage<'_>, payload: &[u8], i: i64) { match msg.topic() { APPEND_ONLY_TOPIC => { let data: AppendOnlyPayload = serde_json::from_slice(payload).unwrap(); @@ -228,7 +231,7 @@ fn check_payload(msg: &BorrowedMessage, payload: &[u8], i: i64) { async fn check_kafka_after_insert( cluster: &mut Cluster, - stream: &mut MessageStream<'_>, + stream: &mut (impl futures::Stream>> + Unpin), input: &[i64], ) -> Result<()> { for i in input { diff --git a/src/tests/simulation/tests/integration_tests/scale/streaming_parallelism.rs b/src/tests/simulation/tests/integration_tests/scale/streaming_parallelism.rs index 8828b7380616d..1e1260e7b0bd2 100644 --- a/src/tests/simulation/tests/integration_tests/scale/streaming_parallelism.rs +++ b/src/tests/simulation/tests/integration_tests/scale/streaming_parallelism.rs @@ -16,7 +16,7 @@ use anyhow::Result; use risingwave_simulation::cluster::{Cluster, Configuration}; use risingwave_simulation::ctl_ext::predicate::identity_contains; -#[madsim::test] +#[tokio::test] async fn test_streaming_parallelism_default() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let default_parallelism = cluster.config().compute_nodes * cluster.config().compute_node_cores; @@ -28,7 +28,7 @@ async fn test_streaming_parallelism_default() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_streaming_parallelism_set_some() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let default_parallelism = cluster.config().compute_nodes * cluster.config().compute_node_cores; @@ -48,7 +48,7 @@ async fn test_streaming_parallelism_set_some() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_streaming_parallelism_set_zero() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let default_parallelism = cluster.config().compute_nodes * cluster.config().compute_node_cores; @@ -64,7 +64,7 @@ async fn test_streaming_parallelism_set_zero() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_streaming_parallelism_mv_on_mv() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let default_parallelism = cluster.config().compute_nodes * cluster.config().compute_node_cores; @@ -101,7 +101,7 @@ async fn test_streaming_parallelism_mv_on_mv() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_streaming_parallelism_index() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let default_parallelism = cluster.config().compute_nodes * cluster.config().compute_node_cores; diff --git a/src/tests/simulation/tests/integration_tests/scale/table.rs b/src/tests/simulation/tests/integration_tests/scale/table.rs index 4ab4c2ce9ec25..8044818b43d0a 100644 --- a/src/tests/simulation/tests/integration_tests/scale/table.rs +++ b/src/tests/simulation/tests/integration_tests/scale/table.rs @@ -12,17 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::iter::{repeat, repeat_with}; -use std::time::Duration; +use std::iter::repeat_with; use anyhow::Result; use itertools::Itertools; -use madsim::time::sleep; -use rand::prelude::SliceRandom; -use rand::thread_rng; use risingwave_simulation::cluster::{Cluster, Configuration}; use risingwave_simulation::ctl_ext::predicate::identity_contains; -use risingwave_simulation::utils::AssertResult; const ROOT_TABLE_CREATE: &str = "create table t (v1 int);"; const MV1: &str = "create materialized view m1 as select * from t;"; @@ -41,7 +36,7 @@ macro_rules! insert_and_flush { }}; } -#[madsim::test] +#[tokio::test] async fn test_table() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; cluster.run(ROOT_TABLE_CREATE).await?; @@ -65,7 +60,7 @@ async fn test_table() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_mv_on_scaled_table() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; cluster.run(ROOT_TABLE_CREATE).await?; diff --git a/src/tests/simulation/tests/integration_tests/sink/basic.rs b/src/tests/simulation/tests/integration_tests/sink/basic.rs index 6747b8bbac4dd..d15e3a68ea2b6 100644 --- a/src/tests/simulation/tests/integration_tests/sink/basic.rs +++ b/src/tests/simulation/tests/integration_tests/sink/basic.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::future::pending; use std::io::Write; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; @@ -22,7 +21,6 @@ use std::time::Duration; use anyhow::Result; use async_trait::async_trait; use itertools::Itertools; -use madsim::time::{sleep, timeout}; use rand::prelude::SliceRandom; use risingwave_common::array::StreamChunk; use risingwave_common::buffer::Bitmap; @@ -30,6 +28,7 @@ use risingwave_connector::sink::boxed::{BoxCoordinator, BoxWriter}; use risingwave_connector::sink::test_sink::registry_build_sink; use risingwave_connector::sink::{Sink, SinkWriter, SinkWriterParam}; use risingwave_simulation::cluster::{Cluster, ConfigPath, Configuration}; +use tokio::time::sleep; struct TestWriter { row_counter: Arc, @@ -106,7 +105,7 @@ impl Sink for TestSink { } } -#[madsim::test] +#[tokio::test] async fn test_sink_basic() -> Result<()> { let config_path = { let mut file = tempfile::NamedTempFile::new().expect("failed to create temp config file"); @@ -133,7 +132,7 @@ async fn test_sink_basic() -> Result<()> { let _sink_guard = registry_build_sink({ let row_counter = row_counter.clone(); let parallelism_counter = parallelism_counter.clone(); - move |param| { + move |_param| { Ok(Box::new(TestSink { row_counter: row_counter.clone(), parallelism_counter: parallelism_counter.clone(), @@ -178,7 +177,7 @@ async fn test_sink_basic() -> Result<()> { Ok(()) } -#[madsim::test] +#[tokio::test] async fn test_sink_decouple_basic() -> Result<()> { let config_path = { let mut file = tempfile::NamedTempFile::new().expect("failed to create temp config file"); @@ -205,7 +204,7 @@ async fn test_sink_decouple_basic() -> Result<()> { let _sink_guard = registry_build_sink({ let row_counter = row_counter.clone(); let parallelism_counter = parallelism_counter.clone(); - move |param| { + move |_param| { Ok(Box::new(TestSink { row_counter: row_counter.clone(), parallelism_counter: parallelism_counter.clone(), diff --git a/src/tests/simulation/tests/integration_tests/sink/mod.rs b/src/tests/simulation/tests/integration_tests/sink/mod.rs index 6db319067a068..71a65bf062d9c 100644 --- a/src/tests/simulation/tests/integration_tests/sink/mod.rs +++ b/src/tests/simulation/tests/integration_tests/sink/mod.rs @@ -12,4 +12,5 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(madsim)] mod basic; diff --git a/src/tests/simulation/tests/integration_tests/storage.rs b/src/tests/simulation/tests/integration_tests/storage.rs index 193d00395d16d..e300a8194e453 100644 --- a/src/tests/simulation/tests/integration_tests/storage.rs +++ b/src/tests/simulation/tests/integration_tests/storage.rs @@ -16,10 +16,9 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use anyhow::Result; -use madsim::rand::thread_rng; use rand::distributions::{Alphanumeric, DistString}; use rand::rngs::SmallRng; -use rand::{RngCore, SeedableRng}; +use rand::{thread_rng, RngCore, SeedableRng}; use risingwave_simulation::cluster::{Cluster, Configuration}; use risingwave_simulation::utils::AssertResult; @@ -27,7 +26,7 @@ const NUM_ROWS: usize = 500; const NUM_OVERWRITES: usize = 5000; const MAX_STRING_LEN: usize = 150; -#[madsim::test] +#[tokio::test] async fn test_storage_with_random_writes() -> Result<()> { // TODO: Use backfill configuration for now let mut cluster = Cluster::start(Configuration::for_backfill()).await?; diff --git a/src/utils/pgwire/Cargo.toml b/src/utils/pgwire/Cargo.toml index b11bdd241eaf0..4ca6b556bf281 100644 --- a/src/utils/pgwire/Cargo.toml +++ b/src/utils/pgwire/Cargo.toml @@ -20,7 +20,7 @@ byteorder = "1.4" bytes = "1" futures = { version = "0.3", default-features = false, features = ["alloc"] } itertools = "0.11" -openssl = "0.10.56" +openssl = "0.10.57" panic-message = "0.3" risingwave_common = { workspace = true } risingwave_sqlparser = { workspace = true }