diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 235fb3bbacfbb..acd75f253b699 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -3,5 +3,5 @@ contact_links: url: https://github.com/risingwavelabs/risingwave/discussions about: Have questions? Welcome to open a discussion. - name: Community Chat - url: https://join.slack.com/t/risingwave-community/shared_invite/zt-120rft0mr-d8uGk3d~NZiZAQWPnElOfw + url: https://risingwave.com/slack about: Join the RisingWave Slack community and chat with us. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b2d58279b5290..51242d0425e28 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,6 +16,7 @@ updates: arrow: patterns: - "arrow*" + - "parquet" aws: patterns: - "aws*" diff --git a/.github/workflows/auto-create-doc-issue-by-issue.yml b/.github/workflows/auto-create-doc-issue-by-issue.yml new file mode 100644 index 0000000000000..0c8d78062977a --- /dev/null +++ b/.github/workflows/auto-create-doc-issue-by-issue.yml @@ -0,0 +1,31 @@ +name: Issue Documentation Checker + +on: + issues: + types: + - closed + - labeled + +jobs: + create-issue: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Log the event payload + run: echo "${{ toJSON(github.event) }}" + - name: Check if issue is done and labeled 'user-facing-changes' + uses: dacbd/create-issue-action@main + if: ${{ github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user-facing-changes') }} + with: + token: ${{ secrets.ACCESS_TOKEN }} + owner: risingwavelabs + repo: risingwave-docs + title: | + Document: ${{ github.event.issue.title }} + body: | + ## Context + Source Issue URL: ${{ github.event.issue.html_url }} + Created At: ${{ github.event.issue.created_at }} + Created By: ${{ github.event.issue.user.login }} + Closed At: ${{ github.event.issue.closed_at }} diff --git a/.github/workflows/auto-create-docs-pr.yml b/.github/workflows/auto-create-doc-issue-by-pr.yml similarity index 100% rename from .github/workflows/auto-create-docs-pr.yml rename to .github/workflows/auto-create-doc-issue-by-pr.yml diff --git a/.github/workflows/nightly-rust.yml b/.github/workflows/nightly-rust.yml index a89d86ac9e560..5219b4805c74d 100644 --- a/.github/workflows/nightly-rust.yml +++ b/.github/workflows/nightly-rust.yml @@ -21,6 +21,13 @@ jobs: remove-docker-images: 'true' root-reserve-mb: 2048 - uses: actions/checkout@v3 + if: ${{ github.event_name == 'schedule' }} + with: + # For daily scheduled run, we use a fixed branch, so that we can apply patches to fix compile errors earlier. + # We can also ensure the regression is due to new rust instead of new RisingWave code. + ref: xxchan/latest-nightly-rust + - uses: actions/checkout@v3 + if: ${{ !(github.event_name == 'schedule') }} - name: Setup Rust toolchain run: | rustup override set nightly diff --git a/.gitignore b/.gitignore index 19fb6643dd8a6..375738f67093e 100644 --- a/.gitignore +++ b/.gitignore @@ -74,4 +74,7 @@ simulation-it-test.tar.zst # hummock-trace .trace +# spark binary +e2e_test/iceberg/spark-*-bin* + **/poetry.lock \ No newline at end of file diff --git a/.licenserc.yaml b/.licenserc.yaml index c1745a4d1ad74..7b49108b6b2f3 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -17,6 +17,6 @@ header: - "**/*.d.ts" - "src/sqlparser/**/*.rs" - "java/connector-node/risingwave-source-cdc/src/main/java/com/risingwave/connector/cdc/debezium/internal/*.java" - - "src/meta/src/model_v2/migration/**/*.rs" + - "src/meta/model_v2/migration/**/*.rs" comment: on-failure diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b519c16010ba..c0b3991fc1f61 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Thanks for your interest in contributing to RisingWave! We welcome and appreciat This document describes how to submit your code changes. To learn about the development process, see the [developer guide](docs/developer-guide.md). To understand the design and implementation of RisingWave, refer to the design docs listed in [docs/README.md](docs/README.md). -If you have questions, you can search for existing discussions or start a new discussion in the [Discussions forum of RisingWave](https://github.com/risingwavelabs/risingwave/discussions), or ask in the RisingWave Community channel on Slack. Please use the [invitation link](https://join.slack.com/t/risingwave-community/shared_invite/zt-120rft0mr-d8uGk3d~NZiZAQWPnElOfw) to join the channel. +If you have questions, you can search for existing discussions or start a new discussion in the [Discussions forum of RisingWave](https://github.com/risingwavelabs/risingwave/discussions), or ask in the RisingWave Community channel on Slack. Please use the [invitation link](https://risingwave.com/slack) to join the channel. To report bugs, create a [GitHub issue](https://github.com/risingwavelabs/risingwave/issues/new/choose). diff --git a/Cargo.lock b/Cargo.lock index 94c5ca804ea42..498002620379d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -243,9 +243,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow-arith" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1d4e368e87ad9ee64f28b9577a3834ce10fe2703a26b28417d485bbbdff956" +checksum = "c5c3d17fc5b006e7beeaebfb1d2edfc92398b981f82d9744130437909b72a468" dependencies = [ "arrow-array", "arrow-buffer", @@ -258,9 +258,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d02efa7253ede102d45a4e802a129e83bcc3f49884cab795b1ac223918e4318d" +checksum = "55705ada5cdde4cb0f202ffa6aa756637e33fea30e13d8d0d0fd6a24ffcee1e3" dependencies = [ "ahash 0.8.3", "arrow-buffer", @@ -274,9 +274,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda119225204141138cb0541c692fbfef0e875ba01bfdeaed09e9d354f9d6195" +checksum = "a722f90a09b94f295ab7102542e97199d3500128843446ef63e410ad546c5333" dependencies = [ "bytes", "half 2.3.1", @@ -285,9 +285,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d825d51b9968868d50bc5af92388754056796dbc62a4e25307d588a1fc84dee" +checksum = "af01fc1a06f6f2baf31a04776156d47f9f31ca5939fe6d00cd7a059f95a46ff1" dependencies = [ "arrow-array", "arrow-buffer", @@ -302,9 +302,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "475a4c3699c8b4095ca61cecf15da6f67841847a5f5aac983ccb9a377d02f73a" +checksum = "d0a547195e607e625e7fafa1a7269b8df1a4a612c919efd9b26bd86e74538f3a" dependencies = [ "arrow-buffer", "arrow-schema", @@ -314,9 +314,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd938ea4a0e8d0db2b9f47ebba792f73f6188f4289707caeaf93a3be705e5ed5" +checksum = "c58645809ced5acd6243e89a63ae8535a2ab50d780affcd7efe8c7473a0da661" dependencies = [ "arrow-array", "arrow-buffer", @@ -334,9 +334,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1248005c8ac549f869b7a840859d942bf62471479c1a2d82659d453eebcd166a" +checksum = "e36bf091502ab7e37775ff448413ef1ffff28ff93789acb669fffdd51b394d51" dependencies = [ "arrow-array", "arrow-buffer", @@ -348,9 +348,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b87aa408ea6a6300e49eb2eba0c032c88ed9dc19e0a9948489c55efdca71f4" +checksum = "4502123d2397319f3a13688432bc678c61cb1582f2daa01253186da650bf5841" dependencies = [ "arrow-array", "arrow-buffer", @@ -363,9 +363,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "114a348ab581e7c9b6908fcab23cb39ff9f060eb19e72b13f8fb8eaa37f65d22" +checksum = "249fc5a07906ab3f3536a6e9f118ec2883fbcde398a97a5ba70053f0276abda4" dependencies = [ "ahash 0.8.3", "arrow-array", @@ -378,15 +378,15 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d179c117b158853e0101bfbed5615e86fe97ee356b4af901f1c5001e1ce4b" +checksum = "9d7a8c3f97f5ef6abd862155a6f39aaba36b029322462d72bbcfa69782a50614" [[package]] name = "arrow-select" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5c71e003202e67e9db139e5278c79f5520bb79922261dfe140e4637ee8b6108" +checksum = "f868f4a5001429e20f7c1994b5cd1aa68b82e3db8cf96c559cdb56dc8be21410" dependencies = [ "ahash 0.8.3", "arrow-array", @@ -480,7 +480,7 @@ dependencies = [ "log", "parking", "polling", - "rustix 0.37.23", + "rustix 0.37.26", "slab", "socket2 0.4.9", "waker-fn", @@ -1816,18 +1816,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ "bytes", + "futures-core", "memchr", + "pin-project-lite", + "tokio", + "tokio-util", ] [[package]] name = "comfy-table" -version = "7.0.1" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b" +checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" dependencies = [ - "crossterm 0.26.1", - "strum 0.24.1", - "strum_macros 0.24.3", + "crossterm 0.27.0", + "strum 0.25.0", + "strum_macros 0.25.2", "unicode-width", ] @@ -2136,17 +2140,14 @@ dependencies = [ [[package]] name = "crossterm" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" +checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "crossterm_winapi", "libc", - "mio", "parking_lot 0.12.1", - "signal-hook", - "signal-hook-mio", "winapi", ] @@ -2444,10 +2445,11 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" dependencies = [ + "powerfmt", "serde", ] @@ -2550,8 +2552,7 @@ checksum = "86e3bdc80eee6e16b2b6b0f87fbc98c04bee3455e35174c0de1a125d0688c632" [[package]] name = "dlv-list" version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aead04dc46b5f263c25721cf25c9e595951d15055f8063f92392fa0d7f64cf4" +source = "git+https://github.com/sgodwincs/dlv-list-rs.git?rev=5bbc5d0#5bbc5d0cc84f257e173d851f8dc1674fb6e46f95" dependencies = [ "const-random", ] @@ -2734,6 +2735,26 @@ dependencies = [ "syn 2.0.37", ] +[[package]] +name = "enumflags2" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5998b4f30320c9d93aed72f63af821bfdac50465b75428fce77b48ec482c3939" +dependencies = [ + "enumflags2_derive", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + [[package]] name = "env_logger" version = "0.10.0" @@ -2811,9 +2832,9 @@ dependencies = [ [[package]] name = "ethnum" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8ff382b2fa527fb7fb06eeebfc5bbb3f17e3cc6b9d70b006c41daa8824adac" +checksum = "b90ca2580b73ab6a1f724b76ca11ab632df820fd6040c336200d2c1df7b3c82c" dependencies = [ "serde", ] @@ -3014,7 +3035,7 @@ dependencies = [ [[package]] name = "foyer" version = "0.1.0" -source = "git+https://github.com/mrcroxx/foyer?rev=438eec8#438eec87e90c7a80cb53a06b711c6ea1ad7a0f41" +source = "git+https://github.com/MrCroxx/foyer?rev=2261151#2261151107ad362851f5fff9ce4fa56e61911b10" dependencies = [ "foyer-common", "foyer-intrusive", @@ -3025,10 +3046,11 @@ dependencies = [ [[package]] name = "foyer-common" version = "0.1.0" -source = "git+https://github.com/mrcroxx/foyer?rev=438eec8#438eec87e90c7a80cb53a06b711c6ea1ad7a0f41" +source = "git+https://github.com/MrCroxx/foyer?rev=2261151#2261151107ad362851f5fff9ce4fa56e61911b10" dependencies = [ "bytes", "foyer-workspace-hack", + "itertools 0.11.0", "madsim-tokio", "parking_lot 0.12.1", "paste", @@ -3039,13 +3061,13 @@ dependencies = [ [[package]] name = "foyer-intrusive" version = "0.1.0" -source = "git+https://github.com/mrcroxx/foyer?rev=438eec8#438eec87e90c7a80cb53a06b711c6ea1ad7a0f41" +source = "git+https://github.com/MrCroxx/foyer?rev=2261151#2261151107ad362851f5fff9ce4fa56e61911b10" dependencies = [ "bytes", "cmsketch", "foyer-common", "foyer-workspace-hack", - "itertools 0.10.5", + "itertools 0.11.0", "memoffset", "parking_lot 0.12.1", "paste", @@ -3056,7 +3078,7 @@ dependencies = [ [[package]] name = "foyer-storage" version = "0.1.0" -source = "git+https://github.com/mrcroxx/foyer?rev=438eec8#438eec87e90c7a80cb53a06b711c6ea1ad7a0f41" +source = "git+https://github.com/MrCroxx/foyer?rev=2261151#2261151107ad362851f5fff9ce4fa56e61911b10" dependencies = [ "anyhow", "async-channel", @@ -3085,7 +3107,7 @@ dependencies = [ [[package]] name = "foyer-workspace-hack" version = "0.1.0" -source = "git+https://github.com/mrcroxx/foyer?rev=438eec8#438eec87e90c7a80cb53a06b711c6ea1ad7a0f41" +source = "git+https://github.com/MrCroxx/foyer?rev=2261151#2261151107ad362851f5fff9ce4fa56e61911b10" dependencies = [ "crossbeam-utils", "either", @@ -3094,7 +3116,7 @@ dependencies = [ "futures-sink", "futures-util", "hyper", - "itertools 0.10.5", + "itertools 0.11.0", "libc", "memchr", "parking_lot 0.12.1", @@ -3223,9 +3245,9 @@ dependencies = [ [[package]] name = "futures-async-stream" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f529ccdeacfa2446a9577041686cf1abb839b1b3e15fee4c1b1232ab3b7d799f" +checksum = "379790776b0d953337df4ab7ecc51936c66ea112484cad7912907b1d34253ebf" dependencies = [ "futures-async-stream-macro", "futures-core", @@ -3234,13 +3256,13 @@ dependencies = [ [[package]] name = "futures-async-stream-macro" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b48ee06dc8d2808ba5ebad075d06c3406085bb19deaac33be64c39113bf80" +checksum = "5df2c13d48c8cb8a3ec093ede6f0f4482f327d7bb781120c5fb483ef0f17e758" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.37", ] [[package]] @@ -3850,7 +3872,7 @@ dependencies = [ [[package]] name = "icelake" version = "0.0.10" -source = "git+https://github.com/icelake-io/icelake?rev=16dab0e36ab337e58ee8002d828def2d212fa116#16dab0e36ab337e58ee8002d828def2d212fa116" +source = "git+https://github.com/icelake-io/icelake?rev=186fde7663545d1d6a5856ce9fbbc541224eadfb#186fde7663545d1d6a5856ce9fbbc541224eadfb" dependencies = [ "anyhow", "apache-avro 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3867,6 +3889,7 @@ dependencies = [ "bytes", "chrono", "csv", + "derive_builder", "enum-display", "faster-hex", "futures", @@ -3883,7 +3906,7 @@ dependencies = [ "serde", "serde_bytes", "serde_json", - "serde_with 3.3.0", + "serde_with 3.4.0", "tokio", "toml 0.7.8", "url", @@ -4036,12 +4059,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "iter-chunks" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7abddfc4e19bc38f3922e41b341fedb4e1470e922f024c4e5ae5922f56c7593" - [[package]] name = "itertools" version = "0.10.5" @@ -4118,6 +4135,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonbb" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44376417b2ff0cd879b5c84976fa9e0855c316321b4e0502e33e52963bf84f74" +dependencies = [ + "bytes", + "serde", + "serde_json", + "smallvec", +] + [[package]] name = "jsonschema-transpiler" version = "1.10.0" @@ -4441,6 +4470,15 @@ dependencies = [ "libc", ] +[[package]] +name = "lz4_flex" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" +dependencies = [ + "twox-hash", +] + [[package]] name = "lzma-sys" version = "0.1.20" @@ -4519,7 +4557,7 @@ dependencies = [ "http", "madsim", "serde", - "serde_with 3.3.0", + "serde_with 3.4.0", "spin 0.9.8", "thiserror", "tokio", @@ -4566,9 +4604,9 @@ dependencies = [ [[package]] name = "madsim-tokio" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3b6df81cfc5c561311fa2e85ed52e8c6f3990b700751be914797addf984e72d" +checksum = "e5611fd0eb96867dd03a9fd2494d4c1bb126f413519673195065b6ea011e8c68" dependencies = [ "madsim", "spin 0.9.8", @@ -4593,9 +4631,9 @@ dependencies = [ [[package]] name = "madsim-tonic-build" -version = "0.4.1+0.10.0" +version = "0.4.2+0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3434b3d59001abcce56b9170fbd7982377858d8b931e8472056bf0c894ab257" +checksum = "4a2ad2776ba20221ccbe4e136e2fa0f7ab90eebd608373177f3e74a198a288ec" dependencies = [ "prettyplease 0.2.15", "proc-macro2", @@ -4759,15 +4797,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "model_migration" -version = "0.1.0" -dependencies = [ - "async-std", - "sea-orm-migration", - "uuid", -] - [[package]] name = "moka" version = "0.12.0" @@ -5240,9 +5269,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "opendal" -version = "0.39.0" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad95e460e5976ab1b74f398ab856c59f8417b3dd32202329e3491dcbe3a6b84" +checksum = "ddba7299bab261d3ae2f37617fb7f45b19ed872752bb4e22cf93a69d979366c5" dependencies = [ "anyhow", "async-compat", @@ -5261,6 +5290,7 @@ dependencies = [ "parking_lot 0.12.1", "percent-encoding", "pin-project", + "prometheus", "quick-xml 0.29.0", "reqsign", "reqwest", @@ -5273,9 +5303,9 @@ dependencies = [ [[package]] name = "opendal" -version = "0.40.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddba7299bab261d3ae2f37617fb7f45b19ed872752bb4e22cf93a69d979366c5" +checksum = "e31b48f0af6de5b3b344c1acc1e06c4581dca3e13cd5ba05269927fc2abf953a" dependencies = [ "anyhow", "async-compat", @@ -5294,8 +5324,7 @@ dependencies = [ "parking_lot 0.12.1", "percent-encoding", "pin-project", - "prometheus", - "quick-xml 0.29.0", + "quick-xml 0.30.0", "reqsign", "reqwest", "serde", @@ -5496,8 +5525,7 @@ dependencies = [ [[package]] name = "ordered-multimap" version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e" +source = "git+https://github.com/risingwavelabs/ordered-multimap-rs.git?rev=19c743f#19c743f3e3d106c99ba37628f06a2ca6faa2284f" dependencies = [ "dlv-list", "hashbrown 0.13.2", @@ -5620,9 +5648,9 @@ dependencies = [ [[package]] name = "parquet" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0463cc3b256d5f50408c49a4be3a16674f4c8ceef60941709620a062b1f6bf4d" +checksum = "239229e6a668ab50c61de3dce61cf0fa1069345f7aa0f4c934491f92205a4945" dependencies = [ "ahash 0.8.3", "arrow-array", @@ -5639,7 +5667,7 @@ dependencies = [ "flate2", "futures", "hashbrown 0.14.0", - "lz4", + "lz4_flex", "num", "num-bigint", "paste", @@ -5648,7 +5676,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd 0.12.4", + "zstd 0.13.0", ] [[package]] @@ -6027,6 +6055,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "pprof" version = "0.13.0" @@ -6214,7 +6248,7 @@ dependencies = [ "byteorder", "hex", "lazy_static", - "rustix 0.36.15", + "rustix 0.36.16", ] [[package]] @@ -6227,7 +6261,7 @@ dependencies = [ "byteorder", "hex", "lazy_static", - "rustix 0.36.15", + "rustix 0.36.16", ] [[package]] @@ -6357,7 +6391,6 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.37", - "workspace-hack", ] [[package]] @@ -6531,6 +6564,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.33" @@ -6632,18 +6675,31 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "redact" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b97c0a6319ae55341eb213c8ef97002630a3a5bd6f287f0124d077121d3f2a5" + [[package]] name = "redis" version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba" dependencies = [ + "async-std", + "async-trait", + "bytes", "combine", + "futures-util", "itoa", "percent-encoding", + "pin-project-lite", "ryu", "sha1_smol", "socket2 0.4.9", + "tokio", + "tokio-util", "url", ] @@ -6844,7 +6900,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "serde_with 3.3.0", + "serde_with 3.4.0", "serde_yaml", "tempfile", "tracing", @@ -6893,7 +6949,6 @@ dependencies = [ "anyhow", "assert_matches", "async-recursion", - "async-stream", "async-trait", "criterion", "either", @@ -6927,7 +6982,6 @@ dependencies = [ "tokio-metrics", "tokio-stream", "tracing", - "uuid", "workspace-hack", ] @@ -6977,7 +7031,7 @@ dependencies = [ "risingwave_ctl", "risingwave_expr_impl", "risingwave_frontend", - "risingwave_meta", + "risingwave_meta_node", "risingwave_rt", "task_stats_alloc", "tikv-jemallocator", @@ -7003,7 +7057,7 @@ dependencies = [ "risingwave_ctl", "risingwave_expr_impl", "risingwave_frontend", - "risingwave_meta", + "risingwave_meta_node", "risingwave_rt", "shell-words", "strum 0.25.0", @@ -7044,6 +7098,7 @@ dependencies = [ "educe", "either", "enum-as-inner", + "enumflags2", "ethnum", "fixedbitset", "fs-err", @@ -7056,6 +7111,7 @@ dependencies = [ "hytra", "itertools 0.11.0", "itoa", + "jsonbb", "libc", "lru 0.7.6", "mach2", @@ -7089,7 +7145,7 @@ dependencies = [ "serde_bytes", "serde_default", "serde_json", - "serde_with 3.3.0", + "serde_with 3.4.0", "smallbitset", "speedate", "static_assertions", @@ -7098,7 +7154,6 @@ dependencies = [ "sysinfo", "tempfile", "thiserror", - "tikv-jemalloc-ctl", "tinyvec", "toml 0.8.2", "tower-layer", @@ -7112,6 +7167,19 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "risingwave_common_heap_profiling" +version = "1.3.0-alpha" +dependencies = [ + "anyhow", + "chrono", + "madsim-tokio", + "parking_lot 0.12.1", + "risingwave_common", + "tikv-jemalloc-ctl", + "tracing", +] + [[package]] name = "risingwave_common_proc_macro" version = "1.3.0-alpha" @@ -7121,7 +7189,6 @@ dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "workspace-hack", ] [[package]] @@ -7160,6 +7227,7 @@ dependencies = [ "risingwave_hummock_sdk", "risingwave_hummock_test", "risingwave_meta", + "risingwave_meta_node", "risingwave_object_store", "risingwave_pb", "risingwave_rpc_client", @@ -7173,22 +7241,20 @@ dependencies = [ name = "risingwave_compactor" version = "1.3.0-alpha" dependencies = [ - "anyhow", "async-trait", "await-tree", "clap", "madsim-tokio", "madsim-tonic", "parking_lot 0.12.1", - "prometheus", "risingwave_common", + "risingwave_common_heap_profiling", "risingwave_common_service", "risingwave_object_store", "risingwave_pb", "risingwave_rpc_client", "risingwave_storage", "serde", - "serde_json", "tracing", "workspace-hack", ] @@ -7215,9 +7281,11 @@ dependencies = [ "rand", "risingwave_batch", "risingwave_common", + "risingwave_common_heap_profiling", "risingwave_common_service", "risingwave_connector", "risingwave_hummock_sdk", + "risingwave_jni_core", "risingwave_pb", "risingwave_rpc_client", "risingwave_source", @@ -7252,7 +7320,6 @@ dependencies = [ "aws-smithy-http", "aws-types", "base64 0.21.4", - "bincode 1.3.3", "byteorder", "bytes", "chrono", @@ -7276,15 +7343,12 @@ dependencies = [ "jsonschema-transpiler", "madsim-rdkafka", "madsim-tokio", - "madsim-tonic", "maplit", "moka", "mysql_async", "mysql_common", "nexmark", - "nkeys", "num-bigint", - "opendal 0.39.0", "parking_lot 0.12.1", "paste", "prometheus", @@ -7295,6 +7359,8 @@ dependencies = [ "protobuf-native", "pulsar", "rand", + "redis", + "regex", "reqwest", "risingwave_common", "risingwave_jni_core", @@ -7304,7 +7370,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "serde_with 3.3.0", + "serde_with 3.4.0", "simd-json", "strum 0.25.0", "strum_macros 0.25.2", @@ -7317,6 +7383,7 @@ dependencies = [ "tonic 0.9.2", "tracing", "tracing-futures", + "tracing-subscriber", "tracing-test", "url", "urlencoding", @@ -7423,6 +7490,7 @@ dependencies = [ "futures-util", "hex", "itertools 0.11.0", + "jsonbb", "madsim-tokio", "md5", "num-traits", @@ -7462,6 +7530,7 @@ dependencies = [ "async-recursion", "async-trait", "auto_enums", + "auto_impl", "bk-tree", "bytes", "clap", @@ -7479,7 +7548,6 @@ dependencies = [ "madsim-tonic", "maplit", "md5", - "more-asserts", "num-integer", "parking_lot 0.12.1", "parse-display", @@ -7604,6 +7672,7 @@ dependencies = [ name = "risingwave_jni_core" version = "0.1.0" dependencies = [ + "anyhow", "bytes", "cfg-or-panic", "futures", @@ -7654,22 +7723,21 @@ dependencies = [ "maplit", "memcomparable", "mime_guess", - "model_migration", "num-integer", "num-traits", "parking_lot 0.12.1", - "parse-display", "prometheus", "prometheus-http-query", "prost 0.12.1", "rand", - "regex", "reqwest", "risingwave_backup", "risingwave_common", - "risingwave_common_service", + "risingwave_common_heap_profiling", "risingwave_connector", "risingwave_hummock_sdk", + "risingwave_meta_model_migration", + "risingwave_meta_model_v2", "risingwave_object_store", "risingwave_pb", "risingwave_rpc_client", @@ -7679,10 +7747,7 @@ dependencies = [ "sea-orm", "serde", "serde_json", - "sqlx", - "static_assertions", "sync-point", - "tempfile", "thiserror", "tokio-retry", "tokio-stream", @@ -7694,6 +7759,78 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "risingwave_meta_model_migration" +version = "1.3.0-alpha" +dependencies = [ + "async-std", + "sea-orm-migration", + "uuid", +] + +[[package]] +name = "risingwave_meta_model_v2" +version = "1.3.0-alpha" +dependencies = [ + "risingwave_pb", + "sea-orm", + "serde", + "serde_json", +] + +[[package]] +name = "risingwave_meta_node" +version = "1.3.0-alpha" +dependencies = [ + "anyhow", + "clap", + "either", + "futures", + "itertools 0.11.0", + "madsim-etcd-client", + "madsim-tokio", + "madsim-tonic", + "prometheus-http-query", + "redact", + "regex", + "risingwave_common", + "risingwave_common_heap_profiling", + "risingwave_common_service", + "risingwave_meta", + "risingwave_meta_model_migration", + "risingwave_meta_service", + "risingwave_pb", + "risingwave_rpc_client", + "sea-orm", + "tracing", + "workspace-hack", +] + +[[package]] +name = "risingwave_meta_service" +version = "1.3.0-alpha" +dependencies = [ + "anyhow", + "async-trait", + "either", + "futures", + "itertools 0.11.0", + "madsim-tokio", + "madsim-tonic", + "rand", + "regex", + "risingwave_common", + "risingwave_connector", + "risingwave_meta", + "risingwave_meta_model_v2", + "risingwave_pb", + "sea-orm", + "sync-point", + "tokio-stream", + "tracing", + "workspace-hack", +] + [[package]] name = "risingwave_object_store" version = "1.3.0-alpha" @@ -7706,6 +7843,7 @@ dependencies = [ "aws-smithy-types", "bytes", "crc32fast", + "either", "fail", "futures", "hyper", @@ -7713,7 +7851,7 @@ dependencies = [ "itertools 0.11.0", "madsim-aws-sdk-s3", "madsim-tokio", - "opendal 0.39.0", + "opendal 0.41.0", "prometheus", "risingwave_common", "spin 0.9.8", @@ -7754,7 +7892,7 @@ dependencies = [ "risingwave_frontend", "risingwave_sqlparser", "serde", - "serde_with 3.3.0", + "serde_with 3.4.0", "serde_yaml", "tempfile", "walkdir", @@ -7809,7 +7947,6 @@ name = "risingwave_rt" version = "1.3.0-alpha" dependencies = [ "await-tree", - "chrono", "console", "console-subscriber", "either", @@ -7821,7 +7958,6 @@ dependencies = [ "opentelemetry-semantic-conventions", "parking_lot 0.12.1", "pprof", - "prometheus", "risingwave_common", "risingwave_variables", "rlimit", @@ -7863,7 +7999,7 @@ dependencies = [ "risingwave_e2e_extended_mode_test", "risingwave_expr_impl", "risingwave_frontend", - "risingwave_meta", + "risingwave_meta_node", "risingwave_pb", "risingwave_rpc_client", "risingwave_sqlparser", @@ -7887,7 +8023,6 @@ dependencies = [ "anyhow", "assert_matches", "criterion", - "easy-ext", "futures", "futures-async-stream", "itertools 0.11.0", @@ -7924,7 +8059,7 @@ dependencies = [ "madsim-tokio", "risingwave_sqlparser", "serde", - "serde_with 3.3.0", + "serde_with 3.4.0", "serde_yaml", "walkdir", "workspace-hack", @@ -7969,7 +8104,7 @@ dependencies = [ "regex", "risingwave_rt", "serde", - "serde_with 3.3.0", + "serde_with 3.4.0", "tokio-postgres", "tokio-stream", "toml 0.8.2", @@ -7981,7 +8116,6 @@ dependencies = [ name = "risingwave_storage" version = "1.3.0-alpha" dependencies = [ - "anyhow", "arc-swap", "async-trait", "auto_enums", @@ -8054,7 +8188,6 @@ dependencies = [ "await-tree", "bytes", "criterion", - "dyn-clone", "educe", "either", "enum-as-inner", @@ -8063,7 +8196,6 @@ dependencies = [ "futures-async-stream", "governor", "hytra", - "iter-chunks", "itertools 0.11.0", "local_stats_alloc", "lru 0.7.6", @@ -8072,9 +8204,7 @@ dependencies = [ "maplit", "memcomparable", "multimap 0.9.0", - "num-traits", "parking_lot 0.12.1", - "parse-display", "pin-project", "prometheus", "prost 0.12.1", @@ -8093,7 +8223,6 @@ dependencies = [ "serde_json", "serde_yaml", "smallvec", - "spin 0.9.8", "static_assertions", "task_stats_alloc", "thiserror", @@ -8252,9 +8381,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.15" +version = "0.36.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c37f1bd5ef1b5422177b7646cba67430579cfe2ace80f284fee876bca52ad941" +checksum = "6da3636faa25820d8648e0e31c5d519bbb01f72fdf57131f0f5f7da5fed36eab" dependencies = [ "bitflags 1.3.2", "errno", @@ -8266,9 +8395,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.23" +version = "0.37.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +checksum = "84f3f8f960ed3b5a59055428714943298bf3fa2d4a1d53135084e0544829d995" dependencies = [ "bitflags 1.3.2", "errno", @@ -8811,9 +8940,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca3b16a3d82c4088f343b7480a93550b3eabe1a358569c2dfe38bbcead07237" +checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" dependencies = [ "base64 0.21.4", "chrono", @@ -8822,7 +8951,7 @@ dependencies = [ "indexmap 2.0.0", "serde", "serde_json", - "serde_with_macros 3.3.0", + "serde_with_macros 3.4.0", "time", ] @@ -8840,9 +8969,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e6be15c453eb305019bfa438b1593c731f36a289a7853f7707ee29e870b3b3c" +checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" dependencies = [ "darling 0.20.3", "proc-macro2", @@ -9758,14 +9887,15 @@ dependencies = [ [[package]] name = "time" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ "deranged", "itoa", "libc", "num_threads", + "powerfmt", "serde", "time-core", "time-macros", @@ -9773,15 +9903,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -10443,9 +10573,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" dependencies = [ "getrandom", "rand", @@ -10896,6 +11026,7 @@ dependencies = [ "ahash 0.8.3", "allocator-api2", "anyhow", + "async-std", "auto_enums", "aws-credential-types", "aws-sdk-s3", @@ -10929,7 +11060,6 @@ dependencies = [ "futures-util", "hashbrown 0.12.3", "hashbrown 0.14.0", - "heck 0.4.1", "hyper", "indexmap 1.9.3", "itertools 0.10.5", @@ -10971,6 +11101,7 @@ dependencies = [ "rand", "rand_chacha", "rand_core", + "redis", "regex", "regex-automata 0.4.1", "regex-syntax 0.8.0", @@ -10984,7 +11115,7 @@ dependencies = [ "sea-query-binder", "serde", "serde_json", - "serde_with 3.3.0", + "serde_with 3.4.0", "sha1", "sha2", "signature", diff --git a/Cargo.toml b/Cargo.toml index 81d914b416d17..f0fbb8485d318 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "src/cmd_all", "src/common", "src/common/common_service", + "src/common/heap_profiling", "src/compute", "src/connector", "src/ctl", @@ -18,7 +19,10 @@ members = [ "src/java_binding", "src/jni_core", "src/meta", - "src/meta/src/model_v2/migration", + "src/meta/model_v2", + "src/meta/model_v2/migration", + "src/meta/node", + "src/meta/service", "src/object_store", "src/prost", "src/prost/helpers", @@ -94,7 +98,7 @@ aws-smithy-types = "0.55" aws-endpoint = "0.55" aws-types = "0.55" etcd-client = { package = "madsim-etcd-client", version = "0.4" } -futures-async-stream = "0.2" +futures-async-stream = "0.2.9" hytra = "0.1" rdkafka = { package = "madsim-rdkafka", version = "0.3.0", features = [ "cmake-build", @@ -106,16 +110,16 @@ hashbrown = { version = "0.14.0", features = [ ] } criterion = { version = "0.5", features = ["async_futures"] } tonic = { package = "madsim-tonic", version = "0.4.0" } -tonic-build = { package = "madsim-tonic-build", version = "0.4.0" } +tonic-build = { package = "madsim-tonic-build", version = "0.4.2" } prost = { version = "0.12" } -icelake = { git = "https://github.com/icelake-io/icelake", rev = "16dab0e36ab337e58ee8002d828def2d212fa116" } -arrow-array = "47" -arrow-cast = "47" -arrow-schema = "47" -arrow-buffer = "47" -arrow-flight = "47" -arrow-select = "47" -arrow-ord = "47" +icelake = { git = "https://github.com/icelake-io/icelake", rev = "186fde7663545d1d6a5856ce9fbbc541224eadfb" } +arrow-array = "48" +arrow-cast = "48" +arrow-schema = "48" +arrow-buffer = "48" +arrow-flight = "48" +arrow-select = "48" +arrow-ord = "48" tikv-jemalloc-ctl = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9" } tikv-jemallocator = { git = "https://github.com/risingwavelabs/jemallocator.git", features = [ "profiling", @@ -127,6 +131,7 @@ risingwave_batch = { path = "./src/batch" } risingwave_cmd = { path = "./src/cmd" } risingwave_common = { path = "./src/common" } risingwave_common_service = { path = "./src/common/common_service" } +risingwave_common_heap_profiling = { path = "./src/common/heap_profiling" } risingwave_compactor = { path = "./src/storage/compactor" } risingwave_compute = { path = "./src/compute" } risingwave_ctl = { path = "./src/ctl" } @@ -138,6 +143,10 @@ risingwave_hummock_sdk = { path = "./src/storage/hummock_sdk" } risingwave_hummock_test = { path = "./src/storage/hummock_test" } risingwave_hummock_trace = { path = "./src/storage/hummock_trace" } risingwave_meta = { path = "./src/meta" } +risingwave_meta_service = { path = "./src/meta/service" } +risingwave_meta_model_migration = { path = "src/meta/model_v2/migration" } +risingwave_meta_model_v2 = { path = "./src/meta/model_v2" } +risingwave_meta_node = { path = "./src/meta/node" } risingwave_object_store = { path = "./src/object_store" } risingwave_pb = { path = "./src/prost" } risingwave_rpc_client = { path = "./src/rpc_client" } @@ -159,6 +168,8 @@ unused_must_use = "forbid" future_incompatible = "warn" nonstandard_style = "warn" rust_2018_idioms = "warn" +# Backward compatibility is not important for an application. +async_fn_in_trait = "allow" [workspace.lints.clippy] uninlined_format_args = "allow" @@ -223,8 +234,8 @@ opt-level = 2 incremental = false debug = 1 -# Patch third-party crates for deterministic simulation. [patch.crates-io] +# Patch third-party crates for deterministic simulation. quanta = { git = "https://github.com/madsim-rs/quanta.git", rev = "948bdc3" } getrandom = { git = "https://github.com/madsim-rs/getrandom.git", rev = "8daf97e" } tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "fe39bb8e" } @@ -232,3 +243,8 @@ tokio-retry = { git = "https://github.com/madsim-rs/rust-tokio-retry.git", rev = tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "ac00d88" } # patch: unlimit 4MB message size for grpc client etcd-client = { git = "https://github.com/risingwavelabs/etcd-client.git", rev = "4e84d40" } + +# Patch for coverage_attribute. +# https://github.com/sgodwincs/dlv-list-rs/pull/19#issuecomment-1774786289 +dlv-list = { git = "https://github.com/sgodwincs/dlv-list-rs.git", rev = "5bbc5d0" } +ordered-multimap = { git = "https://github.com/risingwavelabs/ordered-multimap-rs.git", rev = "19c743f" } diff --git a/Makefile.toml b/Makefile.toml index a0c6213a63c82..86e9cea136bb5 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -406,6 +406,7 @@ condition = { env_set = [ "ENABLE_BUILD_DASHBOARD", ], files_modified = { input = [ "./dashboard/**/*.js", + "./dashboard/**/*.ts*", "./dashboard/package.json", "./dashboard/next.config.js", ], output = [ @@ -1316,7 +1317,6 @@ dependencies = ["k", "l", "check-logs", "wait-processes-exit"] description = "Kill cluster, dump logs and check logs" [tasks.wait-processes-exit] -private = true category = "Misc" description = "Wait for RisingWave processes to exit" script = """ diff --git a/README.md b/README.md index c1878a2717159..091bccc0de9b4 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +

@@ -5,47 +6,131 @@

-[![Slack](https://badgen.net/badge/Slack/Join%20RisingWave/0abd59?icon=slack)](https://risingwave.com/slack) -[![Build status](https://badge.buildkite.com/9394d2bca0f87e2e97aa78b25f765c92d4207c0b65e7f6648f.svg)](https://buildkite.com/risingwavelabs/main) -[![codecov](https://codecov.io/gh/risingwavelabs/risingwave/branch/main/graph/badge.svg?token=EB44K9K38B)](https://codecov.io/gh/risingwavelabs/risingwave) - -RisingWave is a distributed SQL streaming database. It is designed to reduce the complexity and cost of building stream processing applications. RisingWave consumes streaming data, performs incremental computations when new data comes in, and updates results dynamically. As a database system, RisingWave maintains results inside its own storage so that users can access data efficiently. -RisingWave offers wire compatibility with PostgreSQL and demonstrates exceptional performance surpassing the previous generation of stream processing systems, including Apache Flink, by several orders of magnitude. -It particularly excels in handling complex stateful operations like multi-stream joins. +
-RisingWave ingests data from sources like Apache Kafka, Apache Pulsar, Amazon Kinesis, Redpanda, and materialized CDC sources. Data in RisingWave can be delivered to external targets such as message brokers, data warehouses, and data lakes for storage or additional processing. +### 🌊Stream Processing Redefined. -RisingWave 1.0 is a battle-tested version that has undergone rigorous stress tests and performance evaluations. It has proven its reliability and efficiency through successful deployments in numerous production environments across dozens of companies. +
-Learn more at [Introduction to RisingWave](https://docs.risingwave.com/docs/current/intro/). +

+ Documentation   ðŸ“‘    + Hands-on Tutorials   ðŸŽ¯    + RisingWave Cloud   ðŸš€    + + Get Instant Help + +

+
+ + Slack + + + Build status + + + codecov + +
+ +RisingWave is a distributed SQL streaming database that enables cost-efficient and reliable processing of streaming data. ![RisingWave](https://github.com/risingwavelabs/risingwave-docs/blob/0f7e1302b22493ba3c1c48e78810750ce9a5ff42/docs/images/archi_simple.png) +## Try it out in 5 minutes +**Mac** +``` +brew tap risingwavelabs/risingwave +brew install risingwave +risingwave playground +``` +**Ubuntu** +``` +wget https://github.com/risingwavelabs/risingwave/releases/download/v1.3.0/risingwave-v1.3.0-x86_64-unknown-linux.tar.gz +tar xvf risingwave-v1.3.0-x86_64-unknown-linux.tar.gz +./risingwave playground +``` +Now connect to RisingWave using `psql`: +``` +psql -h localhost -p 4566 -d dev -U root +``` +Learn more at [Quick Start](https://docs.risingwave.com/docs/current/get-started/). + +## Production deployments +For **single-node Docker deployments**, please refer to [Docker Compose](https://docs.risingwave.com/docs/current/risingwave-trial/?method=docker-compose). + +For **Kubernetes deployments**, please refer to [Kubernetes with Helm](https://docs.risingwave.com/docs/current/risingwave-k8s-helm/) or [Kubernetes with Operator](https://docs.risingwave.com/docs/current/risingwave-kubernetes/). + +**RisingWave Cloud** the easiest way to run a fully-fledged RisingWave cluster. Try it out for free at: [cloud.risingwave.com](https://cloud.risingwave.com). + + +## Why RisingWave for stream processing? +RisingWave adaptly addresses some of the most challenging problems in stream processing. Compared to existing stream processing systems like [Apache Flink](https://flink.apache.org/), [Apache Spark Streaming](https://spark.apache.org/docs/latest/streaming-programming-guide.html), and [KsqlDB](https://ksqldb.io/), RisingWave stands out in two primary dimensions: **Ease-of-use** and **efficiency**, thanks to its **[PostgreSQL](https://www.postgresql.org/)-style interaction experience** and **[Snowflake](https://snowflake.com/)-like architectural design** (i.e., compute-storage decoupling). +### Ease-of-use +* **Simple to learn** + * RisingWave speaks PostgreSQL-style SQL, enabling users to dive into stream processing in much the same way as operating a PostgreSQL database. +* **Simple to verify correctness** + * RisingWave persists results in materialized views and allow users to break down complex stream computation programs into stacked materialized views, simplifying program development and result verification. +* **Simple to maintain and operate** + * RisingWave abstracts away unnecessary low-level details, allowing users to concentrate solely on SQL code-level issues. +* **Simple to integrate** + * With integrations to a diverse range of cloud systems and the PostgreSQL ecosystem, RisingWave boasts a rich and expansive ecosystem, making it straightforward to incorporate into existing infrastructures. + +### Efficiency +* **High resource utilization** + * Queries in RisingWave leverage shared computational resources, eliminating the need for users to manually allocate resources for each query. +* **No compromise on large state management** + * The decoupled compute-storage architecture of RisingWave ensures remote persistence of internal states, and users never need to worry about the size of internal states when handling complex queries. +* **Highly efficient in multi-stream joins** + * RisingWave has made significant optimizations for multiple stream join scenarios. Users can easily join 10-20 streams (or more) efficiently in a production environment. +* **Transparent dynamic scaling** + * RisingWave supports near-instantaneous dynamic scaling without any service interruptions. +* **Instant failure recovery** + * RisingWave's state management mechanism allows it to recover from failure in seconds, not minutes or hours. +* **Simplified data stack** + * RisingWave's ability to store data and serve queries eliminates the need for separate maintenance of stream processors and databases. Users can effortlessly connect RisingWave to their preferred BI tools or through client libraries. + +## RisingWave's limitations +RisingWave isn’t a panacea for all data engineering hurdles. It has its own set of limitations: +* **No programmable interfaces** + * RisingWave does not provide low-level APIs in languages like Java and Scala, and does not allow users to manage internal states manually (unless you want to hack!). For coding in Java, Scala, and other languages, please consider using RisingWave's User-Defined Functions (UDF). +* **No support for transaction processing** + * RisingWave isn’t cut out for transactional workloads, thus it’s not a viable substitute for operational databases dedicated to transaction processing. However, it supports read-only transactions, ensuring data freshness and consistency. It also comprehends the transactional semantics of upstream database Change Data Capture (CDC). +* **Not tailored for ad-hoc analytical queries** + * RisingWave's row store design is tailored for optimal stream processing performance rather than interactive analytical workloads. Hence, it's not a suitable replacement for OLAP databases. Yet, a reliable integration with many OLAP databases exists, and a collaborative use of RisingWave and OLAP databases is a common practice among many users. + + +## In-production use cases +Like other stream processing systems, the primary use cases of RisingWave include monitoring, alerting, real-time dashboard reporting, streaming ETL (Extract, Transform, Load), machine learning feature engineering, and more. It has already been adopted in fields such as financial trading, manufacturing, new media, logistics, gaming, and more. Check out [customer stories](https://www.risingwave.com/use-cases/). -## RisingWave Cloud +## Community -RisingWave Cloud is a fully-managed and scalable stream processing platform powered by the open-source RisingWave project. Try it out for free at: [cloud.risingwave.com](https://cloud.risingwave.com). +Looking for help, discussions, collaboration opportunities, or a casual afternoon chat with our fellow engineers and community members? Join our [Slack workspace](https://risingwave.com/slack)! ## Notes on telemetry -RisingWave collects anonymous usage statistics to better understand how the community is using RisingWave. The sole intention of this exercise is to help improve the product. These statistics are related to system resource usage, OS versions and system uptime. RisingWave doesn't have access to any user data or metadata running on RisingWave clusters including source and sink connection parameters, sources, sinks, materialized views, and tables. Users have the option to opt out of this collection using a system parameter. Please refer to the RisingWave user documentation for more details. - -## Get started - -- To learn about how to install and run RisingWave, see [Get started](https://docs.risingwave.com/docs/current/get-started/). -- To learn about how to ingest data and the supported data sources, see [Sources](https://docs.risingwave.com/docs/current/data-ingestion/). -- To learn about how to transform data using the PostgreSQL-compatible SQL of RisingWave, see [SQL reference](https://docs.risingwave.com/docs/current/sql-references/). -- To learn about how to deliver data and the supported data sinks, see [Sinks](https://docs.risingwave.com/docs/current/data-delivery/). -- To learn about new features and changes in the current and previous versions, see [Release notes](https://docs.risingwave.com/release-notes/). - -## Documentation - -To learn about how to use RisingWave, refer to [RisingWave User Documentation](https://docs.risingwave.com/). To learn about the development process, see the [developer guide](docs/developer-guide.md). To understand the design and implementation of RisingWave, refer to the design docs listed in [readme.md](docs/README.md). - -## Community - -Looking for help, discussions, collaboration opportunities, or a casual afternoon chat with our fellow engineers and community members? Join our [Slack workspace](https://risingwave.com/slack)! +RisingWave collects anonymous usage statistics to better understand how the community is using RisingWave. The sole intention of this exercise is to help improve the product. Users may opt out easily at any time. Please refer to the [user documentation](https://docs.risingwave.com/docs/current/telemetry/) for more details. ## License diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh index 43ff81ade2b85..59c88e5e9a9ae 100755 --- a/ci/build-ci-image.sh +++ b/ci/build-ci-image.sh @@ -13,7 +13,7 @@ cat ../rust-toolchain # !!! CHANGE THIS WHEN YOU WANT TO BUMP CI IMAGE !!! # # AND ALSO docker-compose.yml # ###################################################### -export BUILD_ENV_VERSION=v20230919 +export BUILD_ENV_VERSION=v20231022 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}" diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index 6fe7cfbfdeca2..66dd2d175e675 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -71,7 +71,7 @@ services: retries: 5 source-test-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230919 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231022 depends_on: - mysql - db @@ -81,10 +81,11 @@ services: - ..:/risingwave sink-test-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230919 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231022 depends_on: - mysql - db + - message_queue - elasticsearch - clickhouse-server - pulsar @@ -92,12 +93,12 @@ services: - ..:/risingwave rw-build-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230919 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231022 volumes: - ..:/risingwave ci-flamegraph-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230919 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231022 # NOTE(kwannoel): This is used in order to permit # syscalls for `nperf` (perf_event_open), # so it can do CPU profiling. @@ -108,7 +109,7 @@ services: - ..:/risingwave regress-test-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230919 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231022 depends_on: db: condition: service_healthy diff --git a/ci/rust-toolchain b/ci/rust-toolchain index ebc0b6c285a4e..fe2a026f6e40f 100644 --- a/ci/rust-toolchain +++ b/ci/rust-toolchain @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2023-09-09" +channel = "nightly-2023-10-21" diff --git a/ci/scripts/deterministic-it-test.sh b/ci/scripts/deterministic-it-test.sh index f281eaa467bfd..5d76a6677d580 100755 --- a/ci/scripts/deterministic-it-test.sh +++ b/ci/scripts/deterministic-it-test.sh @@ -16,6 +16,7 @@ mv target/ci-sim target/sim echo "--- Run integration tests in deterministic simulation mode" seq $TEST_NUM | parallel MADSIM_TEST_SEED={} NEXTEST_PROFILE=ci-sim \ cargo nextest run \ + --no-capture \ --no-fail-fast \ --cargo-metadata target/nextest/cargo-metadata.json \ --binaries-metadata target/nextest/binaries-metadata.json \ diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh index 0d3a7b3fabed4..c5f89a2bbc7e0 100755 --- a/ci/scripts/deterministic-recovery-test.sh +++ b/ci/scripts/deterministic-recovery-test.sh @@ -9,11 +9,20 @@ echo "--- Download artifacts" download-and-decompress-artifact risingwave_simulation . chmod +x ./risingwave_simulation -export RUST_LOG="info,risingwave_meta::barrier::recovery=debug" +export RUST_LOG="info,\ +risingwave_meta::barrier::recovery=debug,\ +risingwave_meta::manager::catalog=debug,\ +risingwave_meta::rpc::ddl_controller=debug,\ +risingwave_meta::barrier::mod=debug,\ +risingwave_simulation=debug" export LOGDIR=.risingwave/log mkdir -p $LOGDIR +# FIXME(kwannoel): Why is this failing? +# echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, background_ddl" +# seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} ./e2e_test/background_ddl/sim/basic.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log' + echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, ddl" seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} ./e2e_test/ddl/\*\*/\*.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log' diff --git a/ci/scripts/e2e-iceberg-cdc.sh b/ci/scripts/e2e-iceberg-cdc.sh new file mode 100755 index 0000000000000..1cdc4abf26487 --- /dev/null +++ b/ci/scripts/e2e-iceberg-cdc.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# Exits as soon as any line fails. +set -euo pipefail + +source ci/scripts/common.sh + +# prepare environment +export CONNECTOR_LIBS_PATH="./connector-node/libs" + +while getopts 'p:' opt; do + case ${opt} in + p ) + profile=$OPTARG + ;; + \? ) + echo "Invalid Option: -$OPTARG" 1>&2 + exit 1 + ;; + : ) + echo "Invalid option: $OPTARG requires an argument" 1>&2 + ;; + esac +done +shift $((OPTIND -1)) + +download_and_prepare_rw "$profile" source + +echo "--- Download connector node package" +buildkite-agent artifact download risingwave-connector.tar.gz ./ +mkdir ./connector-node +tar xf ./risingwave-connector.tar.gz -C ./connector-node + +echo "--- e2e, ci-1cn-1fe, iceberg cdc" + +echo "--- starting risingwave cluster" + +RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ +cargo make ci-start ci-1cn-1fe-with-recovery + +# prepare minio iceberg sink +echo "--- preparing iceberg" +.risingwave/bin/mcli -C .risingwave/config/mcli mb hummock-minio/icebergdata + +cd e2e_test/iceberg +bash ./start_spark_connect_server.sh + +# Don't remove the `--quiet` option since poetry has a bug when printing output, see +# https://github.com/python-poetry/poetry/issues/3412 +"$HOME"/.local/bin/poetry update --quiet + +# 1. import data to mysql +mysql --host=mysql --port=3306 -u root -p123456 < ./test_case/cdc/mysql_cdc.sql + +# 2. create table and sink +"$HOME"/.local/bin/poetry run python main.py -t ./test_case/cdc/no_partition_cdc_init.toml + +# 3. insert new data to mysql +mysql --host=mysql --port=3306 -u root -p123456 < ./test_case/cdc/mysql_cdc_insert.sql + +sleep 20 + +# 4. check change +"$HOME"/.local/bin/poetry run python main.py -t ./test_case/cdc/no_partition_cdc.toml \ No newline at end of file diff --git a/ci/scripts/e2e-iceberg-sink-test.sh b/ci/scripts/e2e-iceberg-sink-test.sh index 1a12225ab5435..998ca52e51262 100755 --- a/ci/scripts/e2e-iceberg-sink-test.sh +++ b/ci/scripts/e2e-iceberg-sink-test.sh @@ -31,9 +31,8 @@ buildkite-agent artifact download risingwave-connector.tar.gz ./ mkdir ./connector-node tar xf ./risingwave-connector.tar.gz -C ./connector-node -echo "--- starting risingwave cluster with connector node" +echo "--- starting risingwave cluster" mkdir -p .risingwave/log -./connector-node/start-service.sh -p 50051 > .risingwave/log/connector-sink.log 2>&1 & cargo make ci-start ci-iceberg-test sleep 1 @@ -84,4 +83,3 @@ fi echo "--- Kill cluster" cargo make ci-kill -pkill -f connector-node diff --git a/ci/scripts/e2e-kafka-sink-test.sh b/ci/scripts/e2e-kafka-sink-test.sh index 85aad20749d61..d51482a912235 100755 --- a/ci/scripts/e2e-kafka-sink-test.sh +++ b/ci/scripts/e2e-kafka-sink-test.sh @@ -3,9 +3,10 @@ # Exits as soon as any line fails. set -euo pipefail -./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only --create > /dev/null 2>&1 -./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert --create > /dev/null 2>&1 -./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --create > /dev/null 2>&1 +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only --create > /dev/null 2>&1 +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert --create > /dev/null 2>&1 +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert-schema --create > /dev/null 2>&1 +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-debezium --create > /dev/null 2>&1 sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/create_sink.slt' sleep 2 @@ -13,7 +14,7 @@ sleep 2 # test append-only kafka sink echo "testing append-only kafka sink" diff ./e2e_test/sink/kafka/append_only1.result \ -<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only --from-beginning --max-messages 10 | sort) 2> /dev/null) +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only --from-beginning --max-messages 10 | sort) 2> /dev/null) if [ $? -ne 0 ]; then echo "The output for append-only sink is not as expected." exit 1 @@ -22,15 +23,24 @@ fi # test upsert kafka sink echo "testing upsert kafka sink" diff ./e2e_test/sink/kafka/upsert1.result \ -<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert --from-beginning --property print.key=true --max-messages 10 | sort) 2> /dev/null) +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert --from-beginning --property print.key=true --max-messages 10 | sort) 2> /dev/null) if [ $? -ne 0 ]; then echo "The output for upsert sink is not as expected." exit 1 fi +# test upsert kafka sink with schema +echo "testing upsert kafka sink with schema" +diff ./e2e_test/sink/kafka/upsert_schema1.result \ +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 10 | sort) 2> /dev/null) +if [ $? -ne 0 ]; then + echo "The output for upsert sink with schema is not as expected." + exit 1 +fi + # test debezium kafka sink echo "testing debezium kafka sink" -(./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 10 | sort) > ./e2e_test/sink/kafka/debezium1.tmp.result 2> /dev/null +(./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 10 | sort) > ./e2e_test/sink/kafka/debezium1.tmp.result 2> /dev/null python3 e2e_test/sink/kafka/debezium.py e2e_test/sink/kafka/debezium1.result e2e_test/sink/kafka/debezium1.tmp.result if [ $? -ne 0 ]; then echo "The output for debezium sink is not as expected." @@ -47,7 +57,7 @@ psql -h localhost -p 4566 -d dev -U root -c "update t_kafka set v_varchar = '', # test append-only kafka sink after update echo "testing append-only kafka sink after updating data" diff ./e2e_test/sink/kafka/append_only2.result \ -<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only --from-beginning --max-messages 11 | sort) 2> /dev/null) +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only --from-beginning --max-messages 11 | sort) 2> /dev/null) if [ $? -ne 0 ]; then echo "The output for append-only sink after update is not as expected." exit 1 @@ -56,15 +66,24 @@ fi # test upsert kafka sink after update echo "testing upsert kafka sink after updating data" diff ./e2e_test/sink/kafka/upsert2.result \ -<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert --from-beginning --property print.key=true --max-messages 11 | sort) 2> /dev/null) +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert --from-beginning --property print.key=true --max-messages 11 | sort) 2> /dev/null) if [ $? -ne 0 ]; then echo "The output for upsert sink after update is not as expected." exit 1 fi +# test upsert kafka sink with schema after update +echo "testing upsert kafka sink with schema after updating data" +diff ./e2e_test/sink/kafka/upsert_schema2.result \ +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 11 | sort) 2> /dev/null) +if [ $? -ne 0 ]; then + echo "The output for upsert sink with schema is not as expected." + exit 1 +fi + # test debezium kafka sink after update echo "testing debezium kafka sink after updating data" -(./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 11 | sort) > ./e2e_test/sink/kafka/debezium2.tmp.result 2> /dev/null +(./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 11 | sort) > ./e2e_test/sink/kafka/debezium2.tmp.result 2> /dev/null python3 e2e_test/sink/kafka/debezium.py e2e_test/sink/kafka/debezium2.result e2e_test/sink/kafka/debezium2.tmp.result if [ $? -ne 0 ]; then echo "The output for debezium sink after update is not as expected." @@ -81,15 +100,24 @@ psql -h localhost -p 4566 -d dev -U root -c "delete from t_kafka where id = 1;" # test upsert kafka sink after delete echo "testing upsert kafka sink after deleting data" diff ./e2e_test/sink/kafka/upsert3.result \ -<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert --from-beginning --property print.key=true --max-messages 12 | sort) 2> /dev/null) +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert --from-beginning --property print.key=true --max-messages 12 | sort) 2> /dev/null) if [ $? -ne 0 ]; then echo "The output for upsert sink after update is not as expected." exit 1 fi +# test upsert kafka sink with schema after delete +echo "testing upsert kafka sink with schema after deleting data" +diff ./e2e_test/sink/kafka/upsert_schema3.result \ +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 12 | sort) 2> /dev/null) +if [ $? -ne 0 ]; then + echo "The output for upsert sink with schema is not as expected." + exit 1 +fi + # test debezium kafka sink after delete echo "testing debezium kafka sink after deleting data" -(./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 13 | sort) > ./e2e_test/sink/kafka/debezium3.tmp.result 2> /dev/null +(./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 13 | sort) > ./e2e_test/sink/kafka/debezium3.tmp.result 2> /dev/null python3 e2e_test/sink/kafka/debezium.py e2e_test/sink/kafka/debezium3.result e2e_test/sink/kafka/debezium3.tmp.result if [ $? -ne 0 ]; then echo "The output for debezium sink after delete is not as expected." @@ -100,6 +128,21 @@ else fi sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/drop_sink.slt' -./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only --delete > /dev/null 2>&1 -./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert --delete > /dev/null 2>&1 -./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --delete > /dev/null 2>&1 +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only --delete > /dev/null 2>&1 +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert --delete > /dev/null 2>&1 +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-debezium --delete > /dev/null 2>&1 + +# test different encoding +echo "testing protobuf" +cp src/connector/src/test_data/proto_recursive/recursive.pb ./proto-recursive +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only-protobuf --create > /dev/null 2>&1 +sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/protobuf.slt' +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only-protobuf --delete > /dev/null 2>&1 + +echo "testing avro" +python3 -m pip install requests confluent-kafka +python3 e2e_test/sink/kafka/register_schema.py 'http://message_queue:8081' 'test-rw-sink-upsert-avro-value' src/connector/src/test_data/all-types.avsc +python3 e2e_test/sink/kafka/register_schema.py 'http://message_queue:8081' 'test-rw-sink-upsert-avro-key' src/connector/src/test_data/all-types.avsc 'string_field,int32_field' +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert-avro --create > /dev/null 2>&1 +sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/avro.slt' +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert-avro --delete > /dev/null 2>&1 diff --git a/ci/scripts/e2e-sink-test.sh b/ci/scripts/e2e-sink-test.sh index 2dc02f0eada7a..0dbdce47abf55 100755 --- a/ci/scripts/e2e-sink-test.sh +++ b/ci/scripts/e2e-sink-test.sh @@ -53,30 +53,8 @@ psql -h db -U postgres -d test -c "CREATE TABLE t4 (v1 int PRIMARY KEY, v2 int); psql -h db -U postgres -d test -c "create table t5 (v1 smallint primary key, v2 int, v3 bigint, v4 float4, v5 float8, v6 decimal, v7 varchar, v8 timestamp, v9 boolean);" psql -h db -U postgres -d test < ./e2e_test/sink/remote/pg_create_table.sql -node_port=50051 -node_timeout=10 - -echo "--- starting risingwave cluster with connector node" -cargo make ci-start ci-kafka -./connector-node/start-service.sh -p $node_port > .risingwave/log/connector-node.log 2>&1 & - -echo "waiting for connector node to start" -start_time=$(date +%s) -while : -do - if nc -z localhost $node_port; then - echo "Port $node_port is listened! Connector Node is up!" - break - fi - - current_time=$(date +%s) - elapsed_time=$((current_time - start_time)) - if [ $elapsed_time -ge $node_timeout ]; then - echo "Timeout waiting for port $node_port to be listened!" - exit 1 - fi - sleep 0.1 -done +echo "--- starting risingwave cluster" +cargo make ci-start ci-1cn-1fe echo "--- testing common sinks" sqllogictest -p 4566 -d dev './e2e_test/sink/append_only_sink.slt' @@ -141,7 +119,6 @@ fi echo "--- Kill cluster" cargo make ci-kill -pkill -f connector-node echo "--- e2e, ci-1cn-1fe, nexmark endless" RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh index 59618d24641aa..12ea66f9f1ac4 100755 --- a/ci/scripts/e2e-source-test.sh +++ b/ci/scripts/e2e-source-test.sh @@ -6,7 +6,6 @@ set -euo pipefail source ci/scripts/common.sh # prepare environment -export CONNECTOR_RPC_ENDPOINT="localhost:50051" export CONNECTOR_LIBS_PATH="./connector-node/libs" while getopts 'p:' opt; do @@ -49,38 +48,12 @@ export PGHOST=db PGUSER=postgres PGPASSWORD=postgres PGDATABASE=cdc_test createdb psql < ./e2e_test/source/cdc/postgres_cdc.sql -node_port=50051 -node_timeout=10 - -wait_for_connector_node_start() { - start_time=$(date +%s) - while : - do - if nc -z localhost $node_port; then - echo "Port $node_port is listened! Connector Node is up!" - break - fi - - current_time=$(date +%s) - elapsed_time=$((current_time - start_time)) - if [ $elapsed_time -ge $node_timeout ]; then - echo "Timeout waiting for port $node_port to be listened!" - exit 1 - fi - sleep 0.1 - done - sleep 2 -} - -echo "--- starting risingwave cluster with connector node" +echo "--- starting risingwave cluster" RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ cargo make ci-start ci-1cn-1fe-with-recovery -./connector-node/start-service.sh -p $node_port > .risingwave/log/connector-node.log 2>&1 & - -echo "waiting for connector node to start" -wait_for_connector_node_start echo "--- inline cdc test" +export MYSQL_HOST=mysql MYSQL_TCP_PORT=3306 MYSQL_PWD=123456 sqllogictest -p 4566 -d dev './e2e_test/source/cdc_inline/**/*.slt' echo "--- mysql & postgres cdc validate test" @@ -95,7 +68,6 @@ sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check.slt' # kill cluster and the connector node cargo make kill -pkill -f connector-node echo "cluster killed " # insert new rows @@ -105,10 +77,6 @@ echo "inserted new rows into mysql and postgres" # start cluster w/o clean-data RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ -touch .risingwave/log/connector-node.log -./connector-node/start-service.sh -p $node_port >> .risingwave/log/connector-node.log 2>&1 & -echo "(recovery) waiting for connector node to start" -wait_for_connector_node_start cargo make dev ci-1cn-1fe-with-recovery echo "wait for cluster recovery finish" @@ -119,7 +87,6 @@ sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check_new_rows.slt' echo "--- Kill cluster" cargo make ci-kill -pkill -f connector-node echo "--- e2e, ci-1cn-1fe, protobuf schema registry" RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index fee45e6afc4f8..08e5794f173cd 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -11,7 +11,7 @@ if [ "${BUILDKITE_SOURCE}" != "schedule" ] && [ "${BUILDKITE_SOURCE}" != "webhoo fi echo "--- Install java and maven" -yum install -y java-11-openjdk wget python3 cyrus-sasl-devel +yum install -y java-11-openjdk java-11-openjdk-devel wget python3 cyrus-sasl-devel pip3 install toml-cli wget https://ci-deps-dist.s3.amazonaws.com/apache-maven-3.9.3-bin.tar.gz && tar -zxvf apache-maven-3.9.3-bin.tar.gz export PATH="${REPO_ROOT}/apache-maven-3.9.3/bin:$PATH" @@ -64,6 +64,10 @@ elif [[ -n "${BINARY_NAME+x}" ]]; then aws s3 cp risingwave-${BINARY_NAME}-x86_64-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary fi +echo "--- Build connector node" +cd ${REPO_ROOT}/java && mvn -B package -Dmaven.test.skip=true -Dno-build-rust +cd ${REPO_ROOT} && mv ${REPO_ROOT}/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz risingwave-connector-"${BUILDKITE_TAG}".tar.gz + if [[ -n "${BUILDKITE_TAG}" ]]; then echo "--- Install gh cli" yum install -y dnf @@ -87,8 +91,6 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz echo "--- Release build and upload risingwave connector node jar asset" - cd ${REPO_ROOT}/java && mvn -B package -Dmaven.test.skip=true -Dno-build-rust - cd connector-node/assembly/target && mv risingwave-connector-1.0.0.tar.gz risingwave-connector-"${BUILDKITE_TAG}".tar.gz gh release upload "${BUILDKITE_TAG}" risingwave-connector-"${BUILDKITE_TAG}".tar.gz fi diff --git a/ci/scripts/run-backfill-tests.sh b/ci/scripts/run-backfill-tests.sh index 86dbcb376c255..d0d5eafb3c917 100755 --- a/ci/scripts/run-backfill-tests.sh +++ b/ci/scripts/run-backfill-tests.sh @@ -1,22 +1,31 @@ #!/usr/bin/env bash # Runs backfill tests. -# NOTE(kwannoel): -# The following scenario is adapted in madsim's integration tests as well. -# But this script reproduces it more reliably (I'm not sure why.) -# Hence keeping it in case we ever need to debug backfill again. # USAGE: # ```sh # cargo make ci-start ci-backfill # ./ci/scripts/run-backfill-tests.sh # ``` +# Example progress: +# dev=> select * from rw_catalog.rw_ddl_progress; +# ddl_id | ddl_statement | progress | initialized_at +#--------+------------------------------------------------+----------+------------------------------- +# 1002 | CREATE MATERIALIZED VIEW m1 AS SELECT * FROM t | 56.12% | 2023-09-27 06:37:06.636+00:00 +#(1 row) set -euo pipefail PARENT_PATH=$(dirname "${BASH_SOURCE[0]}") +TEST_DIR=$PWD/e2e_test +BACKGROUND_DDL_DIR=$TEST_DIR/background_ddl +COMMON_DIR=$BACKGROUND_DDL_DIR/common + +CLUSTER_PROFILE='ci-1cn-1fe-with-recovery' +export RUST_LOG="risingwave_meta=debug" + run_sql_file() { psql -h localhost -p 4566 -d dev -U root -f "$@" } @@ -29,40 +38,309 @@ flush() { run_sql "FLUSH;" } -basic() { - echo "--- e2e, test_backfill_basic" +cancel_stream_jobs() { + ID=$(run_sql "select ddl_id from rw_catalog.rw_ddl_progress;" | tail -3 | head -1 | grep -E -o "[0-9]*") + echo "CANCELLING STREAM_JOB: $ID" + run_sql "CANCEL JOBS $ID;" .risingwave/log/compute-node.log 2>&1 & +} + +# Test snapshot and upstream read. +test_snapshot_and_upstream_read() { + echo "--- e2e, ci-backfill, test_snapshot_and_upstream_read" cargo make ci-start ci-backfill + run_sql_file "$PARENT_PATH"/sql/backfill/create_base_table.sql # Provide snapshot run_sql_file "$PARENT_PATH"/sql/backfill/insert.sql + + # Provide updates ... run_sql_file "$PARENT_PATH"/sql/backfill/insert.sql & + + # ... and concurrently create mv. run_sql_file "$PARENT_PATH"/sql/backfill/create_mv.sql & wait run_sql_file "$PARENT_PATH"/sql/backfill/select.sql microbench_instance_type.txt + buildkite-agent artifact upload ./microbench_instance_type.txt + if [[ $instance_type != "m6i.4xlarge" ]]; then + echo "Only m6i.4xlarge is supported, skipping microbenchmark" + exit 0 + fi + # We need cargo criterion to generate machine-readable benchmark results from # microbench. echo "--- Installing cargo criterion" diff --git a/ci/scripts/sql/backfill/insert.sql b/ci/scripts/sql/backfill/insert.sql index 6c12a3e0b897e..f25f8b09cb2b8 100644 --- a/ci/scripts/sql/backfill/insert.sql +++ b/ci/scripts/sql/backfill/insert.sql @@ -2,5 +2,5 @@ insert into t1 SELECT generate_series, '{"orders": {"id": 1, "price": "2.30", "customer_id": 2}}'::jsonb -FROM generate_series(1, 100000); +FROM generate_series(1, 50000); FLUSH; \ No newline at end of file diff --git a/ci/scripts/upload-micro-bench-results.sh b/ci/scripts/upload-micro-bench-results.sh index 2644ca936c5da..e72b69950bb7b 100755 --- a/ci/scripts/upload-micro-bench-results.sh +++ b/ci/scripts/upload-micro-bench-results.sh @@ -36,6 +36,19 @@ get_commit() { | sed 's/\"//g' } +get_machine() { + buildkite-agent artifact download microbench_instance_type.txt ./ + cat ./microbench_instance_type.txt +} + +echo "--- Checking microbench_instance_type" +INSTANCE_TYPE=$(get_machine) +echo "instance type: $INSTANCE_TYPE" +if [[ $INSTANCE_TYPE != "m6i.4xlarge" ]]; then + echo "Only m6i.4xlarge is supported, microbenchmark was skipped" + exit 0 +fi + setup BUILDKITE_BUILD_URL="https://buildkite.com/risingwavelabs/main-cron/builds/$BUILDKITE_BUILD_NUMBER" diff --git a/ci/workflows/integration-tests.yml b/ci/workflows/integration-tests.yml index 6c4851c0c669c..455f29b210ec1 100644 --- a/ci/workflows/integration-tests.yml +++ b/ci/workflows/integration-tests.yml @@ -29,7 +29,8 @@ steps: - "postgres-cdc" - "mysql-sink" - "postgres-sink" - - "iceberg-sink" + - "iceberg-cdc" + # - "iceberg-sink" - "debezium-mysql" format: - "json" @@ -75,8 +76,12 @@ steps: testcase: "postgres-sink" format: "protobuf" skip: true + # - with: + # testcase: "iceberg-sink" + # format: "protobuf" + # skip: true - with: - testcase: "iceberg-sink" + testcase: "iceberg-cdc" format: "protobuf" skip: true - with: diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml index 349f6d8359f2b..d8e78952c141f 100644 --- a/ci/workflows/main-cron.yml +++ b/ci/workflows/main-cron.yml @@ -478,7 +478,7 @@ steps: config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 10 + timeout_in_minutes: 20 retry: *auto-retry - label: "e2e standalone binary test" diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml index 367aaee3730e0..3aaa09f0d7716 100644 --- a/ci/workflows/pull-request.yml +++ b/ci/workflows/pull-request.yml @@ -209,6 +209,21 @@ steps: timeout_in_minutes: 10 retry: *auto-retry + - label: "end-to-end iceberg cdc test" + if: build.pull_request.labels includes "ci/run-e2e-iceberg-sink-tests" + command: "ci/scripts/e2e-iceberg-cdc.sh -p ci-dev" + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v4.9.0: + run: sink-test-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 10 + retry: *auto-retry + - label: "end-to-end pulsar sink test" if: build.pull_request.labels includes "ci/run-e2e-pulsar-sink-tests" command: "ci/scripts/e2e-pulsar-sink-test.sh -p ci-dev" @@ -486,11 +501,11 @@ steps: - "build" plugins: - docker-compose#v4.9.0: - run: ci-flamegraph-env + run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 10 + timeout_in_minutes: 30 - label: "e2e standalone binary test" command: "ci/scripts/e2e-test.sh -p ci-dev -m standalone" @@ -529,4 +544,35 @@ steps: if: build.pull_request.labels includes "ci/skip-ci" && !build.pull_request.draft commands: - echo "ci/skip-ci is only usable for draft Pull Requests" - - exit 1 \ No newline at end of file + - exit 1 + + - label: "micro benchmark" + command: "ci/scripts/run-micro-benchmarks.sh" + key: "run-micro-benchmarks" + if: build.pull_request.labels includes "ci/run-micro-benchmarks" + plugins: + - docker-compose#v4.9.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + timeout_in_minutes: 60 + retry: *auto-retry + + - label: "upload micro-benchmark" + if: build.pull_request.labels includes "ci/run-upload-micro-benchmark" + command: + - "BUILDKITE_BUILD_NUMBER=$BUILDKITE_BUILD_NUMBER ci/scripts/upload-micro-bench-results.sh" + depends_on: "run-micro-benchmarks" + plugins: + - seek-oss/aws-sm#v2.3.1: + env: + BUILDKITE_TOKEN: buildkite_token + GITHUB_TOKEN: github-token + - docker-compose#v4.9.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + environment: + - BUILDKITE_TOKEN + - GITHUB_TOKEN + timeout_in_minutes: 5 diff --git a/dashboard/pages/await_tree.tsx b/dashboard/pages/await_tree.tsx index 8908e121deabd..3db6b6677274a 100644 --- a/dashboard/pages/await_tree.tsx +++ b/dashboard/pages/await_tree.tsx @@ -36,22 +36,32 @@ import { getClusterInfoComputeNode } from "./api/cluster" import useFetch from "./api/fetch" const SIDEBAR_WIDTH = 200 +const ALL_COMPUTE_NODES = "" export default function AwaitTreeDump() { const { response: computeNodes } = useFetch(getClusterInfoComputeNode) - const [computeNodeId, setComputeNodeId] = useState() - const [dump, setDump] = useState("") + const [computeNodeId, setComputeNodeId] = useState() + const [dump, setDump] = useState("") useEffect(() => { - if (computeNodes && !computeNodeId && computeNodes.length > 0) { - setComputeNodeId(computeNodes[0].id) + if (computeNodes && !computeNodeId) { + setComputeNodeId(ALL_COMPUTE_NODES) } }, [computeNodes, computeNodeId]) const dumpTree = async () => { - const title = `Await-Tree Dump of Compute Node ${computeNodeId}:` - setDump(undefined) + if (computeNodeId === undefined) { + return + } + + let title + if (computeNodeId === ALL_COMPUTE_NODES) { + title = "Await-Tree Dump of All Compute Nodes:" + } else { + title = `Await-Tree Dump of Compute Node ${computeNodeId}:` + } + setDump("Loading...") let result @@ -92,10 +102,13 @@ export default function AwaitTreeDump() { Compute Nodes