diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index eb5173e599e7..cdd8b591d65c 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -3,17 +3,23 @@ name: Mark stale issues and pull requests on: schedule: - cron: '30 1 * * *' + workflow_dispatch: + inputs: + # https://github.com/marketplace/actions/close-stale-issues#operations-per-run + operationsPerRun: + description: 'Max number of operations per run' + required: true + default: 30 jobs: stale: - runs-on: ubuntu-latest permissions: issues: write pull-requests: write steps: - - uses: actions/stale@v5 + - uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: > @@ -25,3 +31,6 @@ jobs: stale-issue-label: 'no-issue-activity' stale-pr-label: 'no-pr-activity' days-before-close: -1 + days-before-pr-close: 7 + operations-per-run: ${{ github.event.inputs.operationsPerRun }} + enable-statistics: true diff --git a/Cargo.lock b/Cargo.lock index eb2ae2ed948c..462b542829ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1097,6 +1097,19 @@ dependencies = [ "syn 2.0.57", ] +[[package]] +name = "asynchronous-codec" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4057f2c32adbb2fc158e22fb38433c8e9bbf76b75a4732c7c0cbaf695fb65568" +dependencies = [ + "bytes", + "futures-sink", + "futures-util", + "memchr", + "pin-project-lite", +] + [[package]] name = "atoi" version = "2.0.0" @@ -2654,6 +2667,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "connection-string" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "510ca239cf13b7f8d16a2b48f263de7b4f8c566f0af58d901031473c76afb1e3" + [[package]] name = "console" version = "0.15.7" @@ -4540,6 +4559,70 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encoding" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" +dependencies = [ + "encoding-index-japanese", + "encoding-index-korean", + "encoding-index-simpchinese", + "encoding-index-singlebyte", + "encoding-index-tradchinese", +] + +[[package]] +name = "encoding-index-japanese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-korean" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-simpchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-singlebyte" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-tradchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding_index_tests" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" + [[package]] name = "encoding_rs" version = "0.8.33" @@ -8955,6 +9038,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04bfa62906ce8d9badf8d1764501640ae7f0bcea3437a209315830e0f73564d1" +[[package]] +name = "pretty-hex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5" + [[package]] name = "pretty-xmlish" version = "0.1.13" @@ -10653,6 +10742,7 @@ dependencies = [ "tempfile", "thiserror", "thiserror-ext", + "tiberius", "time", "tokio-postgres", "tokio-retry", @@ -11066,14 +11156,25 @@ dependencies = [ name = "risingwave_java_binding" version = "0.1.0" dependencies = [ + "anyhow", + "bytes", + "cfg-or-panic", + "foyer", + "futures", "jni", + "madsim-tokio", "prost 0.12.1", "risingwave_common", "risingwave_expr", + "risingwave_hummock_sdk", "risingwave_jni_core", + "risingwave_object_store", "risingwave_pb", + "risingwave_storage", + "rw_futures_util", "serde", "serde_json", + "tracing", ] [[package]] @@ -11096,9 +11197,7 @@ dependencies = [ "risingwave_common", "risingwave_expr", "risingwave_hummock_sdk", - "risingwave_object_store", "risingwave_pb", - "risingwave_storage", "rw_futures_util", "serde", "serde_json", @@ -11991,6 +12090,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" +dependencies = [ + "log", + "ring 0.16.20", + "sct", + "webpki", +] + [[package]] name = "rustls" version = "0.21.11" @@ -14294,6 +14405,37 @@ dependencies = [ "ordered-float 2.10.0", ] +[[package]] +name = "tiberius" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6e2bf3e4b5be181a2a2ceff4b9b12e2684010d436a6958bd564fbc8094d44d" +dependencies = [ + "async-trait", + "asynchronous-codec", + "bigdecimal 0.3.1", + "byteorder", + "bytes", + "chrono", + "connection-string", + "encoding", + "enumflags2", + "futures-util", + "num-traits", + "once_cell", + "pin-project-lite", + "pretty-hex", + "rust_decimal", + "rustls-native-certs 0.6.3", + "rustls-pemfile 1.0.4", + "thiserror", + "time", + "tokio-rustls 0.23.4", + "tokio-util", + "tracing", + "uuid", +] + [[package]] name = "tikv-jemalloc-ctl" version = "0.5.4" @@ -14497,6 +14639,17 @@ dependencies = [ "rand", ] +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls 0.20.9", + "tokio", + "webpki", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -14559,6 +14712,7 @@ checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -15909,6 +16063,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" +dependencies = [ + "ring 0.17.5", + "untrusted 0.9.0", +] + [[package]] name = "webpki-roots" version = "0.25.2" diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index 4786f85327d1..b2a885a4ba2e 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.9" services: db: image: postgres:15-alpine @@ -100,6 +99,7 @@ services: - doris-server - starrocks-fe-server - starrocks-be-server + - sqlserver-server volumes: - ..:/risingwave @@ -204,6 +204,16 @@ services: timeout: 5s retries: 30 + sqlserver-server: + container_name: sqlserver-server + image: mcr.microsoft.com/mssql/server:2022-latest + hostname: sqlserver-server + ports: + - 1433:1433 + environment: + ACCEPT_EULA: 'Y' + SA_PASSWORD: 'SomeTestOnly@SA' + starrocks-fe-server: container_name: starrocks-fe-server image: starrocks/fe-ubuntu:3.1.7 diff --git a/ci/scripts/e2e-sqlserver-sink-test.sh b/ci/scripts/e2e-sqlserver-sink-test.sh new file mode 100755 index 000000000000..f1f62941375c --- /dev/null +++ b/ci/scripts/e2e-sqlserver-sink-test.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# Exits as soon as any line fails. +set -euo pipefail + +source ci/scripts/common.sh + +while getopts 'p:' opt; do + case ${opt} in + p ) + profile=$OPTARG + ;; + \? ) + echo "Invalid Option: -$OPTARG" 1>&2 + exit 1 + ;; + : ) + echo "Invalid option: $OPTARG requires an argument" 1>&2 + ;; + esac +done +shift $((OPTIND -1)) + +download_and_prepare_rw "$profile" source + +echo "--- starting risingwave cluster" +risedev ci-start ci-sink-test +sleep 1 + +echo "--- create SQL Server table" +curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - +curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list +apt-get update -y +ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y mssql-tools unixodbc-dev +export PATH="/opt/mssql-tools/bin/:$PATH" +sleep 2 + +sqlcmd -S sqlserver-server -U SA -P SomeTestOnly@SA -Q " +CREATE DATABASE SinkTest; +GO +USE SinkTest; +CREATE TABLE t_many_data_type ( + k1 int, k2 int, + c_boolean bit, + c_int16 smallint, + c_int32 int, + c_int64 bigint, + c_float32 float, + c_float64 float, + c_decimal decimal, + c_date date, + c_time time, + c_timestamp datetime2, + c_timestampz datetime2, + c_nvarchar nvarchar(1024), + c_varbinary varbinary(1024), +PRIMARY KEY (k1,k2)); +GO" +sleep 2 + +echo "--- testing sinks" +sqllogictest -p 4566 -d dev './e2e_test/sink/sqlserver_sink.slt' +sleep 1 +sqlcmd -S sqlserver-server -U SA -P SomeTestOnly@SA -h -1 -Q " +SELECT * FROM SinkTest.dbo.t_many_data_type; +GO" > ./query_result.txt + +mapfile -t actual < <(tr -s '[:space:]' '\n' < query_result.txt) +actual=("${actual[@]:1}") +expected=(0 0 0 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 1 1 0 55 55 1 1.0 1.0 1 2022-04-08 18:20:49.0000000 2022-03-13 01:00:00.0000000 2022-03-13 01:00:00.0000000 Hello World! 0xDE00BEEF 1 2 0 66 66 1 1.0 1.0 1 2022-04-08 18:20:49.0000000 2022-03-13 01:00:00.0000000 2022-03-13 01:00:00.0000000 Hello World! 0xDE00BEEF 1 4 0 2 2 1 1.0 1.0 1 2022-04-08 18:20:49.0000000 2022-03-13 01:00:00.0000000 2022-03-13 01:00:00.0000000 Hello World! 0xDE00BEEF "(4" rows "affected)") + +if [[ ${#actual[@]} -eq ${#expected[@]} && ${actual[@]} == ${expected[@]} ]]; then + echo "SQL Server sink check passed" +else + cat ./query_result.txt + echo "The output is not as expected." +fi + +echo "--- Kill cluster" +risedev ci-kill diff --git a/ci/scripts/run-e2e-test.sh b/ci/scripts/run-e2e-test.sh index 94398a77a830..4736f4aa53a8 100755 --- a/ci/scripts/run-e2e-test.sh +++ b/ci/scripts/run-e2e-test.sh @@ -92,7 +92,11 @@ sqllogictest -p 4566 -d dev './e2e_test/ddl/**/*.slt' --junit "batch-ddl-${profi if [[ "$mode" != "single-node" ]]; then sqllogictest -p 4566 -d dev './e2e_test/background_ddl/basic.slt' --junit "batch-ddl-${profile}" fi -sqllogictest -p 4566 -d dev './e2e_test/visibility_mode/*.slt' --junit "batch-${profile}" + +if [[ $mode != "single-node" ]]; then + sqllogictest -p 4566 -d dev './e2e_test/visibility_mode/*.slt' --junit "batch-${profile}" +fi + sqllogictest -p 4566 -d dev './e2e_test/ttl/ttl.slt' sqllogictest -p 4566 -d dev './e2e_test/database/prepare.slt' sqllogictest -p 4566 -d test './e2e_test/database/test.slt' diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml index 14f3a23161c8..c9eaf5cf0c38 100644 --- a/ci/workflows/main-cron.yml +++ b/ci/workflows/main-cron.yml @@ -7,10 +7,6 @@ auto-retry: &auto-retry steps: - label: "build" command: "ci/scripts/build.sh -p ci-release" - if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null - || build.pull_request.labels includes "ci/run-build" - || build.env("CI_STEPS") =~ /(^|,)build(,|$$)/ key: "build" plugins: - docker-compose#v5.1.0: @@ -22,10 +18,6 @@ steps: - label: "build other components" command: "ci/scripts/build-other.sh" - if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null - || build.pull_request.labels includes "ci/run-build-other" - || build.env("CI_STEPS") =~ /(^|,)build-other(,|$$)/ key: "build-other" plugins: - seek-oss/aws-sm#v2.3.1: @@ -42,10 +34,6 @@ steps: - label: "build simulation test" command: "ci/scripts/build-simulation.sh" - if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null - || build.pull_request.labels includes "ci/run-build-simulation" - || build.env("CI_STEPS") =~ /(^|,)build-simulation(,|$$)/ key: "build-simulation" plugins: - docker-compose#v5.1.0: @@ -57,10 +45,6 @@ steps: - label: "docslt" command: "ci/scripts/docslt.sh" - if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null - || build.pull_request.labels includes "ci/run-docslt" - || build.env("CI_STEPS") =~ /(^|,)docslt(,|$$)/ key: "docslt" plugins: - docker-compose#v5.1.0: @@ -74,7 +58,7 @@ steps: key: "e2e-test-release" command: "ci/scripts/cron-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-test" || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/ depends_on: @@ -94,7 +78,7 @@ steps: key: "slow-e2e-test-release" command: "ci/scripts/slow-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-slow-e2e-tests" || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/ depends_on: @@ -113,7 +97,7 @@ steps: key: "e2e-meta-backup-test-release" command: "ci/scripts/run-meta-backup-test.sh -p ci-release -m ci-3streaming-2serving-3fe" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-meta-backup-test" || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/ depends_on: @@ -133,7 +117,7 @@ steps: key: "e2e-test-release-parallel" command: "ci/scripts/e2e-test-parallel.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-parallel-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-tests?(,|$$)/ depends_on: @@ -158,7 +142,7 @@ steps: key: "e2e-test-release-parallel-memory" command: "ci/scripts/e2e-test-parallel-in-memory.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-parallel-in-memory-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-in-memory-tests?(,|$$)/ depends_on: @@ -177,7 +161,7 @@ steps: key: "e2e-test-release-source" command: "ci/scripts/e2e-source-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-source-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-source-tests?(,|$$)/ depends_on: @@ -196,7 +180,7 @@ steps: key: "e2e-test-release-sink" command: "ci/scripts/e2e-sink-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-sink-tests?(,|$$)/ depends_on: @@ -215,7 +199,7 @@ steps: key: "fuzz-test" command: "ci/scripts/cron-fuzz-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-sqlsmith-fuzzing-tests" || build.env("CI_STEPS") =~ /(^|,)sqlsmith-fuzzing-tests?(,|$$)/ depends_on: @@ -237,7 +221,7 @@ steps: key: "unit-test" command: "ci/scripts/unit-test.sh" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-unit-test" || build.env("CI_STEPS") =~ /(^|,)unit-tests?(,|$$)/ plugins: @@ -257,7 +241,7 @@ steps: key: "unit-test-deterministic" command: "MADSIM_TEST_NUM=100 timeout 30m ci/scripts/deterministic-unit-test.sh" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-unit-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)unit-tests?-deterministic-simulation(,|$$)/ plugins: @@ -272,7 +256,7 @@ steps: key: "integration-test-deterministic-scale" command: "TEST_NUM=60 ci/scripts/deterministic-it-test.sh scale::" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" @@ -289,7 +273,7 @@ steps: key: "integration-test-deterministic-recovery" command: "TEST_NUM=60 ci/scripts/deterministic-it-test.sh recovery::" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" @@ -306,7 +290,7 @@ steps: key: "integration-test-deterministic-backfill" command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh backfill_tests::" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" @@ -323,7 +307,7 @@ steps: key: "integration-test-deterministic-storage" command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh storage::" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" @@ -340,7 +324,7 @@ steps: key: "integration-test-deterministic-sink" command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh sink::" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" @@ -357,7 +341,7 @@ steps: key: "e2e-test-deterministic" command: "TEST_NUM=64 timeout 75m ci/scripts/deterministic-e2e-test.sh" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)e2e-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" @@ -379,7 +363,7 @@ steps: key: "recovery-test-deterministic" command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.0 timeout 65m ci/scripts/deterministic-recovery-test.sh" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)recovery-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" @@ -398,7 +382,7 @@ steps: key: "background-ddl-arrangement-backfill-recovery-test-deterministic" command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.8 USE_ARRANGEMENT_BACKFILL=--use-arrangement-backfill timeout 65m ci/scripts/deterministic-recovery-test.sh" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)recovery-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" @@ -416,7 +400,7 @@ steps: key: "e2e-iceberg-sink-test" command: "ci/scripts/e2e-iceberg-sink-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-iceberg-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-iceberg-sink-tests?(,|$$)/ depends_on: @@ -436,7 +420,7 @@ steps: key: "e2e-iceberg-sink-v2-test" command: "ci/scripts/e2e-iceberg-sink-v2-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-iceberg-sink-v2-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-iceberg-sink-v2-tests?(,|$$)/ depends_on: @@ -455,7 +439,7 @@ steps: key: "e2e-java-binding-tests" command: "ci/scripts/java-binding-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-java-binding-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-java-binding-tests?(,|$$)/ depends_on: @@ -476,7 +460,7 @@ steps: key: "s3-v2-source-check-aws-json-parser" command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_source_v2.py -t json" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-s3-source-tests" || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/ depends_on: build @@ -498,7 +482,7 @@ steps: key: "s3-v2-source-batch-read-check-aws-json-parser" command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_source_batch.py -t json" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-s3-source-tests" || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/ depends_on: build @@ -520,7 +504,7 @@ steps: key: "s3-v2-source-check-aws-csv-parser" command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_source_v2.py -t csv_without_header" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-s3-source-tests" || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/ depends_on: build @@ -542,7 +526,7 @@ steps: key: "s3-source-test-for-opendal-fs-engine-csv-parser" command: "ci/scripts/s3-source-test.sh -p ci-release -s posix_fs_source.py -t csv_without_header" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-s3-source-tests" || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/ depends_on: build @@ -560,7 +544,7 @@ steps: # key: "s3-source-test-for-opendal-fs-engine" # command: "ci/scripts/s3-source-test-for-opendal-fs-engine.sh -p ci-release -s gcs_source.py" # if: | - # !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + # !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null # || build.pull_request.labels includes "ci/run-s3-source-tests" # || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/ # depends_on: build @@ -582,7 +566,7 @@ steps: key: "pulsar-source-tests" command: "ci/scripts/pulsar-source-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-pulsar-source-tests" || build.env("CI_STEPS") =~ /(^|,)pulsar-source-tests?(,|$$)/ depends_on: @@ -607,7 +591,7 @@ steps: key: "run-micro-benchmarks" command: "ci/scripts/run-micro-benchmarks.sh" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-micro-benchmarks" || build.env("CI_STEPS") =~ /(^|,)micro-benchmarks?(,|$$)/ plugins: @@ -622,7 +606,7 @@ steps: key: "upload-micro-benchmarks" if: | build.branch == "main" - || !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + || !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-micro-benchmarks" || build.env("CI_STEPS") =~ /(^|,)micro-benchmarks?(,|$$)/ command: @@ -647,7 +631,7 @@ steps: key: "backwards-compat-tests" command: "VERSION_OFFSET={{matrix.version_offset}} RW_COMMIT=$BUILDKITE_COMMIT ci/scripts/backwards-compat-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-backwards-compat-tests" || build.env("CI_STEPS") =~ /(^|,)backwards?-compat-tests?(,|$$)/ depends_on: @@ -681,7 +665,7 @@ steps: key: "sqlsmith-differential-tests" command: "ci/scripts/sqlsmith-differential-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-sqlsmith-differential-tests" || build.env("CI_STEPS") =~ /(^|,)sqlsmith-differential-tests?(,|$$)/ depends_on: @@ -697,7 +681,7 @@ steps: key: "backfill-tests" command: "BUILDKITE=${BUILDKITE:-} ci/scripts/backfill-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-backfill-tests" || build.env("CI_STEPS") =~ /(^|,)backfill-tests?(,|$$)/ depends_on: @@ -715,7 +699,7 @@ steps: key: "e2e-standalone-binary-tests" command: "ci/scripts/e2e-test.sh -p ci-release -m standalone" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-standalone-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-standalone-tests?(,|$$)/ depends_on: @@ -735,7 +719,7 @@ steps: key: "e2e-single-node-binary-tests" command: "ci/scripts/e2e-test.sh -p ci-release -m single-node" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-single-node-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-single-node-tests?(,|$$)/ depends_on: @@ -755,7 +739,7 @@ steps: key: "e2e-test-opendal-parallel" command: "ci/scripts/e2e-test-parallel-for-opendal.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-parallel-tests-for-opendal" || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-tests?-for-opendal(,|$$)/ depends_on: @@ -774,7 +758,7 @@ steps: key: "e2e-deltalake-sink-rust-tests" command: "ci/scripts/e2e-deltalake-sink-rust-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-deltalake-sink-rust-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-deltalake-sink-rust-tests?(,|$$)/ depends_on: @@ -793,7 +777,7 @@ steps: key: "e2e-redis-sink-tests" command: "ci/scripts/e2e-redis-sink-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-redis-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-redis-sink-tests?(,|$$)/ depends_on: @@ -812,7 +796,7 @@ steps: key: "e2e-doris-sink-tests" command: "ci/scripts/e2e-doris-sink-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/ depends_on: @@ -831,7 +815,7 @@ steps: key: "e2e-starrocks-sink-tests" command: "ci/scripts/e2e-starrocks-sink-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-starrocks-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-starrocks-sink-tests?(,|$$)/ depends_on: @@ -850,7 +834,7 @@ steps: key: "e2e-cassandra-sink-tests" command: "ci/scripts/e2e-cassandra-sink-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-cassandra-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-cassandra-sink-tests?(,|$$)/ depends_on: @@ -869,7 +853,7 @@ steps: key: "e2e-clickhouse-sink-tests" command: "ci/scripts/e2e-clickhouse-sink-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-clickhouse-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-clickhouse-sink-tests?(,|$$)/ depends_on: @@ -884,11 +868,30 @@ steps: timeout_in_minutes: 10 retry: *auto-retry + - label: "end-to-end sqlserver sink test" + key: "e2e-sqlserver-sink-tests" + command: "ci/scripts/e2e-sqlserver-sink-test.sh -p ci-release" + if: | + !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + || build.pull_request.labels includes "ci/run-e2e-sqlserver-sink-tests" + || build.env("CI_STEPS") =~ /(^|,)e2e-sqlserver-sink-tests?(,|$$)/ + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v5.1.0: + run: sink-test-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 10 + retry: *auto-retry + - label: "end-to-end pulsar sink test" key: "e2e-pulsar-sink-tests" command: "ci/scripts/e2e-pulsar-sink-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-pulsar-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-pulsar-sink-tests?(,|$$)/ depends_on: @@ -907,7 +910,7 @@ steps: key: "e2e-mqtt-sink-tests" command: "ci/scripts/e2e-mqtt-sink-test.sh -p ci-release" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-mqtt-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-mqtt-sink-tests?(,|$$)/ depends_on: @@ -926,7 +929,7 @@ steps: key: "connector-node-integration-test" command: "ci/scripts/connector-node-integration-test.sh -p ci-release -v {{matrix.java_version}}" if: | - !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-connector-node-integration-tests" || build.env("CI_STEPS") =~ /(^|,)connector-node-integration-tests?(,|$$)/ depends_on: diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml index 3f0c307bdab3..cc5e670fe078 100644 --- a/ci/workflows/pull-request.yml +++ b/ci/workflows/pull-request.yml @@ -34,10 +34,6 @@ steps: - label: "build" command: "ci/scripts/build.sh -p ci-dev" key: "build" - if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null - || build.pull_request.labels includes "ci/run-build" - || build.env("CI_STEPS") =~ /(^|,)build(,|$$)/ plugins: - gencer/cache#v2.4.10: *cargo-cache - docker-compose#v5.1.0: @@ -50,10 +46,6 @@ steps: - label: "build other components" command: "ci/scripts/build-other.sh" key: "build-other" - if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null - || build.pull_request.labels includes "ci/run-build-other" - || build.env("CI_STEPS") =~ /(^|,)build-other(,|$$)/ plugins: - gencer/cache#v2.4.10: *cargo-cache - seek-oss/aws-sm#v2.3.1: @@ -71,10 +63,6 @@ steps: - label: "build (deterministic simulation)" command: "ci/scripts/build-simulation.sh" key: "build-simulation" - if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null - || build.pull_request.labels includes "ci/run-build-simulation" - || build.env("CI_STEPS") =~ /(^|,)build-simulation(,|$$)/ plugins: - gencer/cache#v2.4.10: *cargo-cache - docker-compose#v5.1.0: @@ -86,10 +74,6 @@ steps: - label: "docslt" command: "ci/scripts/docslt.sh" key: "docslt" - if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null - || build.pull_request.labels includes "ci/run-docslt" - || build.env("CI_STEPS") =~ /(^|,)docslt(,|$$)/ plugins: - gencer/cache#v2.4.10: *cargo-cache - docker-compose#v5.1.0: @@ -102,7 +86,7 @@ steps: - label: "end-to-end test" command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-test" || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/ depends_on: @@ -122,7 +106,7 @@ steps: key: "slow-e2e-test" command: "ci/scripts/slow-e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-slow-e2e-tests" || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/ depends_on: @@ -137,10 +121,28 @@ steps: timeout_in_minutes: 8 retry: *auto-retry + - label: "meta backup test" + key: "e2e-meta-backup-test" + command: "ci/scripts/run-meta-backup-test.sh -p ci-dev -m ci-3streaming-2serving-3fe" + if: | + build.pull_request.labels includes "ci/run-e2e-meta-backup-test" + depends_on: + - "build" + - "build-other" + - "docslt" + plugins: + - docker-compose#v5.1.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 45 + retry: *auto-retry + - label: "end-to-end test (parallel)" command: "ci/scripts/e2e-test-parallel.sh -p ci-dev" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-parallel-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-tests?(,|$$)/ depends_on: @@ -186,7 +188,7 @@ steps: - label: "end-to-end source test" command: "ci/scripts/e2e-source-test.sh -p ci-dev" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-source-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-source-tests?(,|$$)/ depends_on: @@ -205,7 +207,7 @@ steps: - label: "end-to-end sink test" command: "ci/scripts/e2e-sink-test.sh -p ci-dev" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-sink-tests?(,|$$)/ depends_on: @@ -331,6 +333,21 @@ steps: timeout_in_minutes: 10 retry: *auto-retry + - label: "end-to-end sqlserver sink test" + if: build.pull_request.labels includes "ci/run-e2e-sqlserver-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-sqlserver-sink-tests?(,|$$)/ + command: "ci/scripts/e2e-sqlserver-sink-test.sh -p ci-dev" + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v5.1.0: + run: sink-test-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 10 + retry: *auto-retry + - label: "end-to-end deltalake sink test" if: build.pull_request.labels includes "ci/run- e2e-deltalake-sink-rust-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-deltalake-sink-rust-tests?(,|$$)/ command: "ci/scripts/e2e-deltalake-sink-rust-test.sh -p ci-dev" @@ -424,7 +441,7 @@ steps: - label: "regress test" command: "ci/scripts/regress-test.sh -p ci-dev" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-regress-test" || build.env("CI_STEPS") =~ /(^|,)regress-tests?(,|$$)/ depends_on: "build" @@ -443,7 +460,7 @@ steps: - label: "unit test" command: "ci/scripts/pr-unit-test.sh" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-unit-test" || build.env("CI_STEPS") =~ /(^|,)unit-tests?(,|$$)/ plugins: @@ -463,7 +480,7 @@ steps: - label: "check" command: "ci/scripts/check.sh" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-check" || build.env("CI_STEPS") =~ /(^|,)check(,|$$)/ plugins: @@ -477,7 +494,7 @@ steps: - label: "check dylint" command: "ci/scripts/check-dylint.sh" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-check" || build.env("CI_STEPS") =~ /(^|,)check(,|$$)/ plugins: @@ -491,7 +508,7 @@ steps: - label: "unit test (deterministic simulation)" command: "ci/scripts/deterministic-unit-test.sh" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-unit-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)unit-tests?-deterministic-simulation(,|$$)/ plugins: @@ -506,7 +523,7 @@ steps: - label: "integration test (deterministic simulation)" command: "TEST_NUM=5 ci/scripts/deterministic-it-test.sh" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" @@ -522,7 +539,7 @@ steps: - label: "end-to-end test (deterministic simulation)" command: "TEST_NUM=16 ci/scripts/deterministic-e2e-test.sh" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-e2e-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)e2e-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" @@ -544,7 +561,7 @@ steps: - label: "recovery test (deterministic simulation)" command: "TEST_NUM=8 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.0 ci/scripts/deterministic-recovery-test.sh" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)recovery-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" @@ -568,7 +585,7 @@ steps: - label: "misc check" command: "ci/scripts/misc-check.sh" if: | - !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-misc-check" || build.env("CI_STEPS") =~ /(^|,)misc-check(,|$$)/ plugins: @@ -786,10 +803,10 @@ steps: timeout_in_minutes: 15 retry: *auto-retry - - label: "enable ci/skip-ci only in draft PRs" - if: build.pull_request.labels includes "ci/skip-ci" && !build.pull_request.draft + - label: "enable ci/pr/run-selected only in draft PRs" + if: build.pull_request.labels includes "ci/pr/run-selected" && !build.pull_request.draft commands: - - echo "ci/skip-ci is only usable for draft Pull Requests" + - echo "ci/pr/run-selected is only usable for draft Pull Requests" - exit 1 - label: "micro benchmark" diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 5bf9ae127252..857888e4d4cf 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -3870,12 +3870,12 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -6325,9 +6325,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "dependencies": { "to-regex-range": "^5.0.1" @@ -14516,12 +14516,12 @@ } }, "braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "requires": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" } }, "browser-process-hrtime": { @@ -16350,9 +16350,9 @@ } }, "fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "requires": { "to-regex-range": "^5.0.1" diff --git a/docker/docker-compose-distributed.yml b/docker/docker-compose-distributed.yml index f94437f08919..80ce64cb9053 100644 --- a/docker/docker-compose-distributed.yml +++ b/docker/docker-compose-distributed.yml @@ -1,5 +1,4 @@ --- -version: "3" x-image: &image image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1-rc.2} services: diff --git a/docker/docker-compose-with-azblob.yml b/docker/docker-compose-with-azblob.yml index 9fc76c405181..e1bf11bb28ff 100644 --- a/docker/docker-compose-with-azblob.yml +++ b/docker/docker-compose-with-azblob.yml @@ -1,5 +1,4 @@ --- -version: "3" x-image: &image image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1-rc.2} services: diff --git a/docker/docker-compose-with-gcs.yml b/docker/docker-compose-with-gcs.yml index 36433d549c1c..fcad2692f474 100644 --- a/docker/docker-compose-with-gcs.yml +++ b/docker/docker-compose-with-gcs.yml @@ -1,5 +1,4 @@ --- -version: "3" x-image: &image image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1-rc.2} services: diff --git a/docker/docker-compose-with-hdfs.yml b/docker/docker-compose-with-hdfs.yml index cf2b45078bac..974cf922e77b 100644 --- a/docker/docker-compose-with-hdfs.yml +++ b/docker/docker-compose-with-hdfs.yml @@ -1,5 +1,4 @@ --- -version: "3" services: compactor-0: image: ghcr.io/risingwavelabs/risingwave:RisingWave_1.6.1_HDFS_2.7-x86_64 diff --git a/docker/docker-compose-with-local-fs.yml b/docker/docker-compose-with-local-fs.yml index ab4545d64982..d7b5e22c3624 100644 --- a/docker/docker-compose-with-local-fs.yml +++ b/docker/docker-compose-with-local-fs.yml @@ -1,5 +1,4 @@ --- -version: "3" x-image: &image image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1-rc.2} services: diff --git a/docker/docker-compose-with-obs.yml b/docker/docker-compose-with-obs.yml index b92d0cb077a1..45c84e32f652 100644 --- a/docker/docker-compose-with-obs.yml +++ b/docker/docker-compose-with-obs.yml @@ -1,5 +1,4 @@ --- -version: "3" x-image: &image image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1-rc.2} services: diff --git a/docker/docker-compose-with-oss.yml b/docker/docker-compose-with-oss.yml index adbfee86cc83..25aeae746e3b 100644 --- a/docker/docker-compose-with-oss.yml +++ b/docker/docker-compose-with-oss.yml @@ -1,5 +1,4 @@ --- -version: "3" x-image: &image image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1-rc.2} services: diff --git a/docker/docker-compose-with-s3.yml b/docker/docker-compose-with-s3.yml index eabe1f023ced..75ffded8edcb 100644 --- a/docker/docker-compose-with-s3.yml +++ b/docker/docker-compose-with-s3.yml @@ -1,5 +1,4 @@ --- -version: "3" x-image: &image image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1-rc.2} services: diff --git a/docker/docker-compose-with-sqlite.yml b/docker/docker-compose-with-sqlite.yml index f65a4af7bc29..065a60610f33 100644 --- a/docker/docker-compose-with-sqlite.yml +++ b/docker/docker-compose-with-sqlite.yml @@ -1,5 +1,4 @@ --- -version: "3" x-image: &image image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1-rc.2} services: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5ba67f78c3f5..6042fbba261d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" x-image: &image image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1-rc.2} services: diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 054d33d4d727..9b03c60f3d9a 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -548,17 +548,18 @@ Instructions about submitting PRs are included in the [contribution guidelines]( ## CI Labels Guide -- `[ci/run-xxx ...]`: Run additional steps indicated by `ci/run-xxx` in your PR. -- `ci/skip-ci` + `[ci/run-xxx ...]` : Skip steps except for those indicated by `ci/run-xxx` in your **DRAFT PR.** -- `ci/run-main-cron`: Run full `main-cron`. -- `ci/run-main-cron` + `ci/main-cron/skip-ci` + `[ci/run-xxx …]` : Run specific steps indicated by `ci/run-xxx` +- `[ci/run-xxx ...]`: Run additional steps in the PR workflow indicated by `ci/run-xxx` in your PR. +- `ci/pr/run-selected` + `[ci/run-xxx ...]` : Only run selected steps indicated by `ci/run-xxx` in your **DRAFT PR.** +- `ci/main-cron/run-all`: Run full `main-cron` workflow for your PR. +- `ci/main-cron/run-selected` + `[ci/run-xxx …]` : Run specific steps indicated by `ci/run-xxx` from the `main-cron` workflow, in your PR. Can use to verify some `main-cron` fix works as expected. - To reference `[ci/run-xxx ...]` labels, you may look at steps from `pull-request.yml` and `main-cron.yml`. -- **Be sure to add all the dependencies.** - - For example to run `e2e-test` for `main-cron` in your pull request: - 1. Add `ci/run-build`, `ci/run-build-other`, `ci/run-docslt` . - These correspond to its `depends` field in `pull-request.yml` and `main-cron.yml` . - 2. Add `ci/run-e2e-test` to run the step as well. - 3. Add `ci/run-main-cron` to run `main-cron` workflow in your pull request, - 4. Add `ci/main-cron/skip-ci` to skip all other steps which were not selected with `ci/run-xxx`. + +### Example + +https://github.com/risingwavelabs/risingwave/pull/17197 + +To run `e2e-test` and `e2e-source-test` for `main-cron` in your pull request: +1. Add `ci/run-e2e-test`. +2. Add `ci/run-e2e-source-tests`. +3. Add `ci/main-cron/run-selected` to skip all other steps which were not selected with `ci/run-xxx`. \ No newline at end of file diff --git a/e2e_test/sink/sqlserver_sink.slt b/e2e_test/sink/sqlserver_sink.slt new file mode 100644 index 000000000000..156b8b865ffc --- /dev/null +++ b/e2e_test/sink/sqlserver_sink.slt @@ -0,0 +1,59 @@ +statement ok +create table t_many_data_type_rw ( + k1 int, k2 int, + c_boolean bool, + c_int16 smallint, + c_int32 int, + c_int64 bigint, + c_float32 float, + c_float64 double, + c_decimal decimal, + c_date date, + c_time time, + c_timestamp timestamp, + c_timestampz timestamp, + c_nvarchar string, + c_varbinary bytea); + +statement ok +create sink s_many_data_type from t_many_data_type_rw with ( + connector = 'sqlserver', + type = 'upsert', + sqlserver.host = 'sqlserver-server', + sqlserver.port = 1433, + sqlserver.user = 'SA', + sqlserver.password = 'SomeTestOnly@SA', + sqlserver.database = 'SinkTest', + sqlserver.table = 't_many_data_type', + primary_key = 'k1,k2', +); + +statement ok +insert into t_many_data_type_rw values +(0,0,false,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL), +(1,1,false,1,1,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf'), +(1,2,false,2,2,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf'), +(1,3,false,2,2,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf'), +(1,4,false,2,2,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf'), +(1,1,false,2,2,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf'); +flush; + +statement ok +delete from t_many_data_type_rw where k1=1 and k2=2; +delete from t_many_data_type_rw where k1=1 and k2=3; +flush; + +statement ok +insert into t_many_data_type_rw values +(1,1,false,55,55,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf'), +(1,2,false,66,66,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf'); +flush; + +statement ok +FLUSH; + +statement ok +DROP SINK s_many_data_type; + +statement ok +DROP TABLE t_many_data_type_rw; diff --git a/integration_tests/ad-click/docker-compose.yml b/integration_tests/ad-click/docker-compose.yml index 62d5c3fb7651..2b84bb00d995 100644 --- a/integration_tests/ad-click/docker-compose.yml +++ b/integration_tests/ad-click/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/ad-ctr/docker-compose.yml b/integration_tests/ad-ctr/docker-compose.yml index 0298f014db11..2bec6c35295b 100644 --- a/integration_tests/ad-ctr/docker-compose.yml +++ b/integration_tests/ad-ctr/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/big-query-sink/docker-compose.yml b/integration_tests/big-query-sink/docker-compose.yml index 6c93903df8bb..279f43a43b21 100644 --- a/integration_tests/big-query-sink/docker-compose.yml +++ b/integration_tests/big-query-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/cassandra-and-scylladb-sink/docker-compose.yml b/integration_tests/cassandra-and-scylladb-sink/docker-compose.yml index 0fa224ddab9d..9f09b203ef70 100644 --- a/integration_tests/cassandra-and-scylladb-sink/docker-compose.yml +++ b/integration_tests/cassandra-and-scylladb-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: cassandra: image: cassandra:4.0 diff --git a/integration_tests/cdn-metrics/docker-compose.yml b/integration_tests/cdn-metrics/docker-compose.yml index 87adef35f8cf..05d3d786e627 100644 --- a/integration_tests/cdn-metrics/docker-compose.yml +++ b/integration_tests/cdn-metrics/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/citus-cdc/docker-compose.yml b/integration_tests/citus-cdc/docker-compose.yml index 6ce8341047ee..8afb665e02cd 100644 --- a/integration_tests/citus-cdc/docker-compose.yml +++ b/integration_tests/citus-cdc/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/clickhouse-sink/docker-compose.yml b/integration_tests/clickhouse-sink/docker-compose.yml index 1cf61ff8dfa3..beb2ee125473 100644 --- a/integration_tests/clickhouse-sink/docker-compose.yml +++ b/integration_tests/clickhouse-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: clickhouse-server: image: clickhouse/clickhouse-server:23.3.8.21-alpine diff --git a/integration_tests/clickstream/docker-compose.yml b/integration_tests/clickstream/docker-compose.yml index 857c93f0d757..4015a3a976ce 100644 --- a/integration_tests/clickstream/docker-compose.yml +++ b/integration_tests/clickstream/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/client-library/docker-compose.yml b/integration_tests/client-library/docker-compose.yml index c6868eaa4214..c8a03d353b18 100644 --- a/integration_tests/client-library/docker-compose.yml +++ b/integration_tests/client-library/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/cockroach-sink/docker-compose.yml b/integration_tests/cockroach-sink/docker-compose.yml index b6b0c8d9e6c5..d325c57865ba 100644 --- a/integration_tests/cockroach-sink/docker-compose.yml +++ b/integration_tests/cockroach-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/debezium-mysql/docker-compose.yml b/integration_tests/debezium-mysql/docker-compose.yml index 3462e5e3d09d..6cb577ac2388 100644 --- a/integration_tests/debezium-mysql/docker-compose.yml +++ b/integration_tests/debezium-mysql/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/debezium-postgres/docker-compose.yml b/integration_tests/debezium-postgres/docker-compose.yml index c81c33fb3e45..327cb44d6db7 100644 --- a/integration_tests/debezium-postgres/docker-compose.yml +++ b/integration_tests/debezium-postgres/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/debezium-sqlserver/docker-compose.yml b/integration_tests/debezium-sqlserver/docker-compose.yml index e88cb36e548b..9d4bbbf0a5bb 100644 --- a/integration_tests/debezium-sqlserver/docker-compose.yml +++ b/integration_tests/debezium-sqlserver/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/deltalake-sink/docker-compose.yml b/integration_tests/deltalake-sink/docker-compose.yml index 70b1e3c22e32..2a799f9fcf45 100644 --- a/integration_tests/deltalake-sink/docker-compose.yml +++ b/integration_tests/deltalake-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: spark: image: apache/spark:3.3.1 diff --git a/integration_tests/doris-sink/docker-compose.yml b/integration_tests/doris-sink/docker-compose.yml index e1a7f1ef5e90..4b43632f5131 100644 --- a/integration_tests/doris-sink/docker-compose.yml +++ b/integration_tests/doris-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: fe: platform: linux/amd64 diff --git a/integration_tests/elasticsearch-sink/docker-compose.yml b/integration_tests/elasticsearch-sink/docker-compose.yml index c885b7136a60..097de4beb549 100644 --- a/integration_tests/elasticsearch-sink/docker-compose.yml +++ b/integration_tests/elasticsearch-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: elasticsearch7: image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 diff --git a/integration_tests/feature-store/docker-compose.yml b/integration_tests/feature-store/docker-compose.yml index 71633cce20a1..77de22a8c352 100644 --- a/integration_tests/feature-store/docker-compose.yml +++ b/integration_tests/feature-store/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: kafka: image: confluentinc/cp-kafka:7.1.0 diff --git a/integration_tests/http-sink/docker-compose.yml b/integration_tests/http-sink/docker-compose.yml index 12546c4f5dd2..9a7c42b1443e 100644 --- a/integration_tests/http-sink/docker-compose.yml +++ b/integration_tests/http-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/iceberg-sink/docker-compose.yml b/integration_tests/iceberg-sink/docker-compose.yml index 91cec5dd2443..84bda01b21ce 100644 --- a/integration_tests/iceberg-sink/docker-compose.yml +++ b/integration_tests/iceberg-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" x-airflow-common: &airflow-common image: apache/airflow:2.6.2-python3.10 diff --git a/integration_tests/kafka-cdc-sink/docker-compose.yml b/integration_tests/kafka-cdc-sink/docker-compose.yml index 81f892354b8a..1cebe9b73f28 100644 --- a/integration_tests/kafka-cdc-sink/docker-compose.yml +++ b/integration_tests/kafka-cdc-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/kafka-cdc/docker-compose.yml b/integration_tests/kafka-cdc/docker-compose.yml index f42c4399178d..6eaa5b5ead7a 100644 --- a/integration_tests/kafka-cdc/docker-compose.yml +++ b/integration_tests/kafka-cdc/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/kinesis-s3-source/docker-compose.yml b/integration_tests/kinesis-s3-source/docker-compose.yml index dc91e2095cbd..74dabde96f7b 100644 --- a/integration_tests/kinesis-s3-source/docker-compose.yml +++ b/integration_tests/kinesis-s3-source/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/livestream/docker-compose.yml b/integration_tests/livestream/docker-compose.yml index 8dffce371562..e263b704bc90 100644 --- a/integration_tests/livestream/docker-compose.yml +++ b/integration_tests/livestream/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/mindsdb/docker-compose.yml b/integration_tests/mindsdb/docker-compose.yml index 40fe4e6192fa..0cd82b10a652 100644 --- a/integration_tests/mindsdb/docker-compose.yml +++ b/integration_tests/mindsdb/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/mongodb-cdc/docker-compose.yaml b/integration_tests/mongodb-cdc/docker-compose.yaml index eaf519b44056..de09a204d991 100644 --- a/integration_tests/mongodb-cdc/docker-compose.yaml +++ b/integration_tests/mongodb-cdc/docker-compose.yaml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/mongodb/docker-compose.yaml b/integration_tests/mongodb/docker-compose.yaml index 59ac89215ec1..a2855c200e6b 100644 --- a/integration_tests/mongodb/docker-compose.yaml +++ b/integration_tests/mongodb/docker-compose.yaml @@ -1,4 +1,3 @@ -version: "3" services: mongodb: image: mongo:4.4 diff --git a/integration_tests/mqtt/docker-compose.yml b/integration_tests/mqtt/docker-compose.yml index 04f73404be6a..b91ddd482509 100644 --- a/integration_tests/mqtt/docker-compose.yml +++ b/integration_tests/mqtt/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/mysql-cdc/docker-compose.yml b/integration_tests/mysql-cdc/docker-compose.yml index c0bba2ccc008..b2779c42c05b 100644 --- a/integration_tests/mysql-cdc/docker-compose.yml +++ b/integration_tests/mysql-cdc/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/mysql-sink/docker-compose.yml b/integration_tests/mysql-sink/docker-compose.yml index 3e1fc5544276..8f8c4f9aa433 100644 --- a/integration_tests/mysql-sink/docker-compose.yml +++ b/integration_tests/mysql-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/nats/docker-compose.yml b/integration_tests/nats/docker-compose.yml index 891c86574474..930f1a719fd7 100644 --- a/integration_tests/nats/docker-compose.yml +++ b/integration_tests/nats/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/pinot-sink/docker-compose.yml b/integration_tests/pinot-sink/docker-compose.yml index fc4ad250880c..c7d08dcc005e 100644 --- a/integration_tests/pinot-sink/docker-compose.yml +++ b/integration_tests/pinot-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/postgres-cdc/docker-compose.yml b/integration_tests/postgres-cdc/docker-compose.yml index 7650da077917..333ee2f4080c 100644 --- a/integration_tests/postgres-cdc/docker-compose.yml +++ b/integration_tests/postgres-cdc/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/postgres-sink/docker-compose.yml b/integration_tests/postgres-sink/docker-compose.yml index 4d8638fdc3c0..6f5a16db64c2 100644 --- a/integration_tests/postgres-sink/docker-compose.yml +++ b/integration_tests/postgres-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/presto-trino/docker-compose.yml b/integration_tests/presto-trino/docker-compose.yml index a56135a4ae59..5de9dc34eb78 100644 --- a/integration_tests/presto-trino/docker-compose.yml +++ b/integration_tests/presto-trino/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/prometheus/docker-compose.yml b/integration_tests/prometheus/docker-compose.yml index de3249df9253..cd840807ea1a 100644 --- a/integration_tests/prometheus/docker-compose.yml +++ b/integration_tests/prometheus/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/redis-sink/docker-compose.yml b/integration_tests/redis-sink/docker-compose.yml index dce27ae99895..8f3c3eb9cdd8 100644 --- a/integration_tests/redis-sink/docker-compose.yml +++ b/integration_tests/redis-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: redis: image: 'redis:latest' diff --git a/integration_tests/schema-registry/docker-compose.yml b/integration_tests/schema-registry/docker-compose.yml index 80d4b90e4f7d..2209fb7e20ae 100644 --- a/integration_tests/schema-registry/docker-compose.yml +++ b/integration_tests/schema-registry/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/starrocks-sink/docker-compose.yml b/integration_tests/starrocks-sink/docker-compose.yml index 70918713643d..e3a06cc33587 100644 --- a/integration_tests/starrocks-sink/docker-compose.yml +++ b/integration_tests/starrocks-sink/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: starrocks-fe: image: starrocks/fe-ubuntu:3.1.7 diff --git a/integration_tests/superset/docker-compose.yml b/integration_tests/superset/docker-compose.yml index 746a80fb9a06..3d7b9ed2494c 100644 --- a/integration_tests/superset/docker-compose.yml +++ b/integration_tests/superset/docker-compose.yml @@ -9,7 +9,6 @@ x-superset-volumes: - ./docker:/app/docker - superset_home:/app/superset_home -version: "3.7" services: risingwave-standalone: extends: diff --git a/integration_tests/twitter-pulsar/docker-compose.yml b/integration_tests/twitter-pulsar/docker-compose.yml index d684be6b876a..a3e91c9f8751 100644 --- a/integration_tests/twitter-pulsar/docker-compose.yml +++ b/integration_tests/twitter-pulsar/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/twitter/docker-compose.yml b/integration_tests/twitter/docker-compose.yml index 37b2723cb8e5..e59e71b3839c 100644 --- a/integration_tests/twitter/docker-compose.yml +++ b/integration_tests/twitter/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/integration_tests/upsert-avro/docker-compose.yml b/integration_tests/upsert-avro/docker-compose.yml index 291528f6fb31..9176ca053ba4 100644 --- a/integration_tests/upsert-avro/docker-compose.yml +++ b/integration_tests/upsert-avro/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: @@ -53,4 +52,4 @@ volumes: external: false message_queue: external: false -name: risingwave-compose \ No newline at end of file +name: risingwave-compose diff --git a/integration_tests/vector/docker-compose.yml b/integration_tests/vector/docker-compose.yml index 4c2e6100b714..13101925ac4a 100644 --- a/integration_tests/vector/docker-compose.yml +++ b/integration_tests/vector/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3" services: risingwave-standalone: extends: diff --git a/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java b/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java index a16acda73e7f..db832566fdfa 100644 --- a/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java +++ b/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java @@ -26,6 +26,8 @@ public class Binding { } } + static void ensureInitialized() {} + public static native void tracingSlf4jEvent( String threadName, String name, int level, String message); @@ -33,10 +35,6 @@ public static native void tracingSlf4jEvent( public static native int vnodeCount(); - // hummock iterator method - // Return a pointer to the iterator - static native long iteratorNewHummock(byte[] readPlan); - static native long iteratorNewStreamChunk(long pointer); static native boolean iteratorNext(long pointer); diff --git a/java/java-binding/src/main/java/com/risingwave/java/binding/HummockIterator.java b/java/java-binding/src/main/java/com/risingwave/java/binding/HummockIterator.java index 03282a2dce52..a30391edbd38 100644 --- a/java/java-binding/src/main/java/com/risingwave/java/binding/HummockIterator.java +++ b/java/java-binding/src/main/java/com/risingwave/java/binding/HummockIterator.java @@ -20,8 +20,16 @@ public class HummockIterator implements AutoCloseable { private final long pointer; private boolean isClosed; + static { + Binding.ensureInitialized(); + } + + // hummock iterator method + // Return a pointer to the iterator + private static native long iteratorNewHummock(byte[] readPlan); + public HummockIterator(ReadPlan readPlan) { - this.pointer = Binding.iteratorNewHummock(readPlan.toByteArray()); + this.pointer = iteratorNewHummock(readPlan.toByteArray()); this.isClosed = false; } diff --git a/proto/backup_service.proto b/proto/backup_service.proto index 75420149042d..48fe46ed7eac 100644 --- a/proto/backup_service.proto +++ b/proto/backup_service.proto @@ -49,6 +49,7 @@ message MetaSnapshotMetadata { uint64 safe_epoch = 4; optional uint32 format_version = 5; optional string remarks = 6; + optional string rw_version = 7; } service BackupService { diff --git a/proto/hummock.proto b/proto/hummock.proto index 2fc4c1a6d4b4..c4ac75f2fed6 100644 --- a/proto/hummock.proto +++ b/proto/hummock.proto @@ -706,6 +706,7 @@ message RiseCtlUpdateCompactionConfigRequest { bool enable_emergency_picker = 15; uint32 tombstone_reclaim_ratio = 16; CompressionAlgorithm compression_algorithm = 17; + uint32 max_l0_compact_level_count = 18; } } repeated uint64 compaction_group_ids = 1; @@ -885,6 +886,9 @@ message CompactionConfig { uint32 level0_overlapping_sub_level_compact_level_count = 18; uint32 tombstone_reclaim_ratio = 19; bool enable_emergency_picker = 20; + + // The limitation of the level count of l0 compaction + uint32 max_l0_compact_level_count = 21; } message TableStats { diff --git a/src/batch/src/executor/merge_sort_exchange.rs b/src/batch/src/executor/merge_sort_exchange.rs index 3c0f13198a3a..e2779967dbcb 100644 --- a/src/batch/src/executor/merge_sort_exchange.rs +++ b/src/batch/src/executor/merge_sort_exchange.rs @@ -123,7 +123,7 @@ impl MergeSortExchangeEx // Check whether there is indeed a chunk and there is a visible row sitting at `row_idx` // in the chunk before calling this function. - fn push_row_into_heap(&mut self, source_idx: usize, row_idx: usize) -> Result<()> { + fn push_row_into_heap(&mut self, source_idx: usize, row_idx: usize) { assert!(source_idx < self.source_inputs.len()); let chunk_ref = self.source_inputs[source_idx].as_ref().unwrap(); self.min_heap.push(HeapElem::new( @@ -133,14 +133,6 @@ impl MergeSortExchangeEx row_idx, None, )); - - if self.min_heap.mem_context().check_memory_usage() { - Ok(()) - } else { - Err(BatchError::OutOfMemory( - self.min_heap.mem_context().mem_limit(), - )) - } } } @@ -176,7 +168,7 @@ impl MergeSortExchangeEx // exchange, therefore we are sure that there is at least // one visible row. let next_row_idx = chunk.next_visible_row_idx(0); - self.push_row_into_heap(source_idx, next_row_idx.unwrap())?; + self.push_row_into_heap(source_idx, next_row_idx.unwrap()); } } @@ -211,13 +203,13 @@ impl MergeSortExchangeEx let possible_next_row_idx = cur_chunk.next_visible_row_idx(row_idx + 1); match possible_next_row_idx { Some(next_row_idx) => { - self.push_row_into_heap(child_idx, next_row_idx)?; + self.push_row_into_heap(child_idx, next_row_idx); } None => { self.get_source_chunk(child_idx).await?; if let Some(chunk) = &self.source_inputs[child_idx] { let next_row_idx = chunk.next_visible_row_idx(0); - self.push_row_into_heap(child_idx, next_row_idx.unwrap())?; + self.push_row_into_heap(child_idx, next_row_idx.unwrap()); } } } diff --git a/src/batch/src/executor/order_by.rs b/src/batch/src/executor/order_by.rs index fd07b4fab845..05cd0f8c94fa 100644 --- a/src/batch/src/executor/order_by.rs +++ b/src/batch/src/executor/order_by.rs @@ -19,6 +19,7 @@ use risingwave_common::memory::MemoryContext; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; use risingwave_common::util::memcmp_encoding::encode_chunk; use risingwave_common::util::sort_util::ColumnOrder; +use risingwave_common_estimate_size::EstimateSize; use risingwave_pb::batch_plan::plan_node::NodeBody; use super::{BoxedDataChunkStream, BoxedExecutor, BoxedExecutorBuilder, Executor, ExecutorBuilder}; @@ -91,7 +92,12 @@ impl SortExecutor { #[for_await] for chunk in self.child.execute() { - chunks.push(chunk?.compact()); + let chunk = chunk?.compact(); + let chunk_estimated_heap_size = chunk.estimated_heap_size(); + chunks.push(chunk); + if !self.mem_context.add(chunk_estimated_heap_size as i64) { + Err(BatchError::OutOfMemory(self.mem_context.mem_limit()))?; + } } let mut encoded_rows = @@ -99,12 +105,19 @@ impl SortExecutor { for chunk in &chunks { let encoded_chunk = encode_chunk(chunk, &self.column_orders)?; + let chunk_estimated_heap_size = encoded_chunk + .iter() + .map(|x| x.estimated_heap_size()) + .sum::(); encoded_rows.extend( encoded_chunk .into_iter() .enumerate() .map(|(row_id, row)| (chunk.row_at_unchecked_vis(row_id), row)), ); + if !self.mem_context.add(chunk_estimated_heap_size as i64) { + Err(BatchError::OutOfMemory(self.mem_context.mem_limit()))?; + } } encoded_rows.sort_unstable_by(|(_, a), (_, b)| a.cmp(b)); diff --git a/src/common/src/config.rs b/src/common/src/config.rs index 6395cbf37ed1..6cad79849153 100644 --- a/src/common/src/config.rs +++ b/src/common/src/config.rs @@ -1786,8 +1786,8 @@ pub mod default { const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 12; const DEFAULT_TOMBSTONE_RATIO_PERCENT: u32 = 40; const DEFAULT_EMERGENCY_PICKER: bool = true; - const DEFAULT_MAX_LEVEL: u32 = 6; + const DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT: u32 = 42; use crate::catalog::hummock::CompactionFilterFlag; @@ -1854,6 +1854,10 @@ pub mod default { pub fn max_level() -> u32 { DEFAULT_MAX_LEVEL } + + pub fn max_l0_compact_level_count() -> u32 { + DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT + } } pub mod object_store_config { diff --git a/src/connector/Cargo.toml b/src/connector/Cargo.toml index 1464fcc215b3..7cb6f23e5ec7 100644 --- a/src/connector/Cargo.toml +++ b/src/connector/Cargo.toml @@ -136,6 +136,7 @@ strum_macros = "0.26" tempfile = "3" thiserror = "1" thiserror-ext = { workspace = true } +tiberius = { version = "0.12", default-features = false, features = ["chrono", "time", "tds73", "rust_decimal", "bigdecimal", "rustls"] } time = "0.3.30" tokio = { version = "0.2", package = "madsim-tokio", features = [ "rt", diff --git a/src/connector/src/sink/mod.rs b/src/connector/src/sink/mod.rs index 6bfe20fa18c1..e5a5f6143e41 100644 --- a/src/connector/src/sink/mod.rs +++ b/src/connector/src/sink/mod.rs @@ -38,6 +38,7 @@ pub mod redis; pub mod remote; pub mod snowflake; pub mod snowflake_connector; +pub mod sqlserver; pub mod starrocks; pub mod test_sink; pub mod trivial; @@ -100,6 +101,7 @@ macro_rules! for_all_sinks { { DeltaLake, $crate::sink::deltalake::DeltaLakeSink }, { BigQuery, $crate::sink::big_query::BigQuerySink }, { DynamoDb, $crate::sink::dynamodb::DynamoDbSink }, + { SqlServer, $crate::sink::sqlserver::SqlServerSink }, { Test, $crate::sink::test_sink::TestSink }, { Table, $crate::sink::trivial::TableSink } } @@ -577,6 +579,12 @@ pub enum SinkError { #[backtrace] anyhow::Error, ), + #[error("SQL Server error: {0}")] + SqlServer( + #[source] + #[backtrace] + anyhow::Error, + ), #[error(transparent)] Connector( #[from] @@ -614,3 +622,9 @@ impl From for SinkError { SinkError::Redis(value.to_report_string()) } } + +impl From for SinkError { + fn from(err: tiberius::error::Error) -> Self { + SinkError::SqlServer(anyhow!(err)) + } +} diff --git a/src/connector/src/sink/sqlserver.rs b/src/connector/src/sink/sqlserver.rs new file mode 100644 index 000000000000..acdad3c47627 --- /dev/null +++ b/src/connector/src/sink/sqlserver.rs @@ -0,0 +1,649 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +use anyhow::anyhow; +use async_trait::async_trait; +use risingwave_common::array::{Op, RowRef, StreamChunk}; +use risingwave_common::buffer::Bitmap; +use risingwave_common::catalog::Schema; +use risingwave_common::row::{OwnedRow, Row}; +use risingwave_common::types::{DataType, Decimal}; +use serde_derive::Deserialize; +use serde_with::{serde_as, DisplayFromStr}; +use simd_json::prelude::ArrayTrait; +use tiberius::numeric::Numeric; +use tiberius::{AuthMethod, Client, ColumnData, Config, Query}; +use tokio::net::TcpStream; +use tokio_util::compat::TokioAsyncWriteCompatExt; +use with_options::WithOptions; + +use super::{SinkError, SINK_TYPE_APPEND_ONLY, SINK_TYPE_OPTION, SINK_TYPE_UPSERT}; +use crate::sink::writer::{LogSinkerOf, SinkWriter, SinkWriterExt}; +use crate::sink::{DummySinkCommitCoordinator, Result, Sink, SinkParam, SinkWriterParam}; + +pub const SQLSERVER_SINK: &str = "sqlserver"; + +fn default_max_batch_rows() -> usize { + 1024 +} + +#[serde_as] +#[derive(Clone, Debug, Deserialize, WithOptions)] +pub struct SqlServerConfig { + #[serde(rename = "sqlserver.host")] + pub host: String, + #[serde(rename = "sqlserver.port")] + #[serde_as(as = "DisplayFromStr")] + pub port: u16, + #[serde(rename = "sqlserver.user")] + pub user: String, + #[serde(rename = "sqlserver.password")] + pub password: String, + #[serde(rename = "sqlserver.database")] + pub database: String, + #[serde(rename = "sqlserver.table")] + pub table: String, + #[serde( + rename = "sqlserver.max_batch_rows", + default = "default_max_batch_rows" + )] + #[serde_as(as = "DisplayFromStr")] + pub max_batch_rows: usize, + pub r#type: String, // accept "append-only" or "upsert" +} + +impl SqlServerConfig { + pub fn from_btreemap(properties: BTreeMap) -> Result { + let config = + serde_json::from_value::(serde_json::to_value(properties).unwrap()) + .map_err(|e| SinkError::Config(anyhow!(e)))?; + if config.r#type != SINK_TYPE_APPEND_ONLY && config.r#type != SINK_TYPE_UPSERT { + return Err(SinkError::Config(anyhow!( + "`{}` must be {}, or {}", + SINK_TYPE_OPTION, + SINK_TYPE_APPEND_ONLY, + SINK_TYPE_UPSERT + ))); + } + Ok(config) + } +} + +#[derive(Debug)] +pub struct SqlServerSink { + pub config: SqlServerConfig, + schema: Schema, + pk_indices: Vec, + is_append_only: bool, +} + +impl SqlServerSink { + pub fn new( + mut config: SqlServerConfig, + schema: Schema, + pk_indices: Vec, + is_append_only: bool, + ) -> Result { + // Rewrite config because tiberius allows a maximum of 2100 params in one query request. + const TIBERIUS_PARAM_MAX: usize = 2000; + let params_per_op = schema.fields().len(); + let tiberius_max_batch_rows = if params_per_op == 0 { + config.max_batch_rows + } else { + ((TIBERIUS_PARAM_MAX as f64 / params_per_op as f64).floor()) as usize + }; + if tiberius_max_batch_rows == 0 { + return Err(SinkError::SqlServer(anyhow!(format!( + "too many column {}", + params_per_op + )))); + } + config.max_batch_rows = std::cmp::min(config.max_batch_rows, tiberius_max_batch_rows); + Ok(Self { + config, + schema, + pk_indices, + is_append_only, + }) + } +} + +impl TryFrom for SqlServerSink { + type Error = SinkError; + + fn try_from(param: SinkParam) -> std::result::Result { + let schema = param.schema(); + let config = SqlServerConfig::from_btreemap(param.properties)?; + SqlServerSink::new( + config, + schema, + param.downstream_pk, + param.sink_type.is_append_only(), + ) + } +} + +impl Sink for SqlServerSink { + type Coordinator = DummySinkCommitCoordinator; + type LogSinker = LogSinkerOf; + + const SINK_NAME: &'static str = SQLSERVER_SINK; + + async fn validate(&self) -> Result<()> { + if !self.is_append_only && self.pk_indices.is_empty() { + return Err(SinkError::Config(anyhow!( + "Primary key not defined for upsert SQL Server sink (please define in `primary_key` field)"))); + } + + for f in self.schema.fields() { + check_data_type_compatibility(&f.data_type)?; + } + + // Query table metadata from SQL Server. + let mut sql_server_table_metadata = HashMap::new(); + let mut sql_client = SqlClient::new(&self.config).await?; + let query_table_metadata_error = || { + SinkError::SqlServer(anyhow!(format!( + "SQL Server table {} metadata error", + self.config.table + ))) + }; + static QUERY_TABLE_METADATA: &str = r#" +SELECT + col.name AS ColumnName, + pk.index_id AS PkIndex +FROM + sys.columns col +LEFT JOIN + sys.index_columns ic ON ic.object_id = col.object_id AND ic.column_id = col.column_id +LEFT JOIN + sys.indexes pk ON pk.object_id = col.object_id AND pk.index_id = ic.index_id AND pk.is_primary_key = 1 +WHERE + col.object_id = OBJECT_ID(@P1) +ORDER BY + col.column_id;"#; + let rows = sql_client + .client + .query(QUERY_TABLE_METADATA, &[&self.config.table]) + .await? + .into_results() + .await?; + for row in rows.into_iter().flatten() { + let mut iter = row.into_iter(); + let ColumnData::String(Some(col_name)) = + iter.next().ok_or_else(query_table_metadata_error)? + else { + return Err(query_table_metadata_error()); + }; + let ColumnData::I32(col_pk_index) = + iter.next().ok_or_else(query_table_metadata_error)? + else { + return Err(query_table_metadata_error()); + }; + sql_server_table_metadata.insert(col_name.into_owned(), col_pk_index.is_some()); + } + + // Validate Column name and Primary Key + for (idx, col) in self.schema.fields().iter().enumerate() { + let rw_is_pk = self.pk_indices.contains(&idx); + match sql_server_table_metadata.get(&col.name) { + None => { + return Err(SinkError::SqlServer(anyhow!(format!( + "column {} not found in the downstream SQL Server table {}", + col.name, self.config.table + )))); + } + Some(sql_server_is_pk) => { + if self.is_append_only { + continue; + } + if rw_is_pk && !*sql_server_is_pk { + return Err(SinkError::SqlServer(anyhow!(format!( + "column {} specified in primary_key mismatches with the downstream SQL Server table {} PK", + col.name, self.config.table, + )))); + } + if !rw_is_pk && *sql_server_is_pk { + return Err(SinkError::SqlServer(anyhow!(format!( + "column {} unspecified in primary_key mismatches with the downstream SQL Server table {} PK", + col.name, self.config.table, + )))); + } + } + } + } + + if !self.is_append_only { + let sql_server_pk_count = sql_server_table_metadata + .values() + .filter(|is_pk| **is_pk) + .count(); + if sql_server_pk_count != self.pk_indices.len() { + return Err(SinkError::SqlServer(anyhow!(format!( + "primary key does not match between RisingWave sink ({}) and SQL Server table {} ({})", + self.pk_indices.len(), + self.config.table, + sql_server_pk_count, + )))); + } + } + + Ok(()) + } + + async fn new_log_sinker(&self, writer_param: SinkWriterParam) -> Result { + Ok(SqlServerSinkWriter::new( + self.config.clone(), + self.schema.clone(), + self.pk_indices.clone(), + self.is_append_only, + ) + .await? + .into_log_sinker(writer_param.sink_metrics)) + } +} + +enum SqlOp { + Insert(OwnedRow), + Merge(OwnedRow), + Delete(OwnedRow), +} + +pub struct SqlServerSinkWriter { + config: SqlServerConfig, + schema: Schema, + pk_indices: Vec, + is_append_only: bool, + sql_client: SqlClient, + ops: Vec, +} + +impl SqlServerSinkWriter { + async fn new( + config: SqlServerConfig, + schema: Schema, + pk_indices: Vec, + is_append_only: bool, + ) -> Result { + let sql_client = SqlClient::new(&config).await?; + let writer = Self { + config, + schema, + pk_indices, + is_append_only, + sql_client, + ops: vec![], + }; + Ok(writer) + } + + async fn delete_one(&mut self, row: RowRef<'_>) -> Result<()> { + if self.ops.len() + 1 >= self.config.max_batch_rows { + self.flush().await?; + } + self.ops.push(SqlOp::Delete(row.into_owned_row())); + Ok(()) + } + + async fn upsert_one(&mut self, row: RowRef<'_>) -> Result<()> { + if self.ops.len() + 1 >= self.config.max_batch_rows { + self.flush().await?; + } + self.ops.push(SqlOp::Merge(row.into_owned_row())); + Ok(()) + } + + async fn insert_one(&mut self, row: RowRef<'_>) -> Result<()> { + if self.ops.len() + 1 >= self.config.max_batch_rows { + self.flush().await?; + } + self.ops.push(SqlOp::Insert(row.into_owned_row())); + Ok(()) + } + + async fn flush(&mut self) -> Result<()> { + use std::fmt::Write; + if self.ops.is_empty() { + return Ok(()); + } + let mut query_str = String::new(); + let col_num = self.schema.fields.len(); + let mut next_param_id = 1; + let non_pk_col_indices = (0..col_num) + .filter(|idx| !self.pk_indices.contains(idx)) + .collect::>(); + let all_col_names = self + .schema + .fields + .iter() + .map(|f| format!("[{}]", f.name)) + .collect::>() + .join(","); + let all_source_col_names = self + .schema + .fields + .iter() + .map(|f| format!("[SOURCE].[{}]", f.name)) + .collect::>() + .join(","); + let pk_match = self + .pk_indices + .iter() + .map(|idx| { + format!( + "[SOURCE].[{}]=[TARGET].[{}]", + &self.schema[*idx].name, &self.schema[*idx].name + ) + }) + .collect::>() + .join(" AND "); + let param_placeholders = |param_id: &mut usize| { + let params = (*param_id..(*param_id + col_num)) + .map(|i| format!("@P{}", i)) + .collect::>() + .join(","); + *param_id += col_num; + params + }; + let set_all_source_col = non_pk_col_indices + .iter() + .map(|idx| { + format!( + "[{}]=[SOURCE].[{}]", + &self.schema[*idx].name, &self.schema[*idx].name + ) + }) + .collect::>() + .join(","); + // TODO: avoid repeating the SQL + for op in &self.ops { + match op { + SqlOp::Insert(_) => { + write!( + &mut query_str, + "INSERT INTO [{}] ({}) VALUES ({});", + self.config.table, + all_col_names, + param_placeholders(&mut next_param_id), + ) + .unwrap(); + } + SqlOp::Merge(_) => { + write!( + &mut query_str, + r#"MERGE [{}] AS [TARGET] + USING (VALUES ({})) AS [SOURCE] ({}) + ON {} + WHEN MATCHED THEN UPDATE SET {} + WHEN NOT MATCHED THEN INSERT ({}) VALUES ({});"#, + self.config.table, + param_placeholders(&mut next_param_id), + all_col_names, + pk_match, + set_all_source_col, + all_col_names, + all_source_col_names, + ) + .unwrap(); + } + SqlOp::Delete(_) => { + write!( + &mut query_str, + r#"DELETE FROM [{}] WHERE {};"#, + self.config.table, + self.pk_indices + .iter() + .map(|idx| { + let condition = + format!("[{}]=@P{}", self.schema[*idx].name, next_param_id); + next_param_id += 1; + condition + }) + .collect::>() + .join(" AND "), + ) + .unwrap(); + } + } + } + + let mut query = Query::new(query_str); + for op in self.ops.drain(..) { + match op { + SqlOp::Insert(row) => { + bind_params(&mut query, row, &self.schema, 0..col_num)?; + } + SqlOp::Merge(row) => { + bind_params(&mut query, row, &self.schema, 0..col_num)?; + } + SqlOp::Delete(row) => { + bind_params( + &mut query, + row, + &self.schema, + self.pk_indices.iter().copied(), + )?; + } + } + } + query.execute(&mut self.sql_client.client).await?; + Ok(()) + } +} + +#[async_trait] +impl SinkWriter for SqlServerSinkWriter { + async fn begin_epoch(&mut self, _epoch: u64) -> Result<()> { + Ok(()) + } + + async fn write_batch(&mut self, chunk: StreamChunk) -> Result<()> { + for (op, row) in chunk.rows() { + match op { + Op::Insert => { + if self.is_append_only { + self.insert_one(row).await?; + } else { + self.upsert_one(row).await?; + } + } + Op::UpdateInsert => { + debug_assert!(!self.is_append_only); + self.upsert_one(row).await?; + } + Op::Delete => { + debug_assert!(!self.is_append_only); + self.delete_one(row).await?; + } + Op::UpdateDelete => {} + } + } + Ok(()) + } + + async fn barrier(&mut self, is_checkpoint: bool) -> Result { + if is_checkpoint { + self.flush().await?; + } + Ok(()) + } + + async fn abort(&mut self) -> Result<()> { + Ok(()) + } + + async fn update_vnode_bitmap(&mut self, _vnode_bitmap: Arc) -> Result<()> { + Ok(()) + } +} + +struct SqlClient { + client: Client>, +} + +impl SqlClient { + async fn new(msconfig: &SqlServerConfig) -> Result { + let mut config = Config::new(); + config.host(&msconfig.host); + config.port(msconfig.port); + config.authentication(AuthMethod::sql_server(&msconfig.user, &msconfig.password)); + config.database(&msconfig.database); + config.trust_cert(); + + let tcp = TcpStream::connect(config.get_addr()).await.unwrap(); + tcp.set_nodelay(true).unwrap(); + let client = Client::connect(config, tcp.compat_write()).await?; + Ok(Self { client }) + } +} + +fn bind_params( + query: &mut Query<'_>, + row: impl Row, + schema: &Schema, + col_indices: impl Iterator, +) -> Result<()> { + use risingwave_common::types::ScalarRefImpl; + for col_idx in col_indices { + match row.datum_at(col_idx) { + Some(data_ref) => match data_ref { + ScalarRefImpl::Int16(v) => query.bind(v), + ScalarRefImpl::Int32(v) => query.bind(v), + ScalarRefImpl::Int64(v) => query.bind(v), + ScalarRefImpl::Float32(v) => query.bind(v.into_inner()), + ScalarRefImpl::Float64(v) => query.bind(v.into_inner()), + ScalarRefImpl::Utf8(v) => query.bind(v.to_owned()), + ScalarRefImpl::Bool(v) => query.bind(v), + ScalarRefImpl::Decimal(v) => match v { + Decimal::Normalized(d) => { + query.bind(decimal_to_sql(&d)); + } + Decimal::NaN | Decimal::PositiveInf | Decimal::NegativeInf => { + tracing::warn!( + "Inf, -Inf, Nan in RisingWave decimal is converted into SQL Server null!" + ); + query.bind(None as Option); + } + }, + ScalarRefImpl::Date(v) => query.bind(v.0), + ScalarRefImpl::Timestamp(v) => query.bind(v.0), + ScalarRefImpl::Timestamptz(v) => query.bind(v.timestamp_micros()), + ScalarRefImpl::Time(v) => query.bind(v.0), + ScalarRefImpl::Bytea(v) => query.bind(v.to_vec()), + ScalarRefImpl::Interval(_) => return Err(data_type_not_supported("Interval")), + ScalarRefImpl::Jsonb(_) => return Err(data_type_not_supported("Jsonb")), + ScalarRefImpl::Struct(_) => return Err(data_type_not_supported("Struct")), + ScalarRefImpl::List(_) => return Err(data_type_not_supported("List")), + ScalarRefImpl::Int256(_) => return Err(data_type_not_supported("Int256")), + ScalarRefImpl::Serial(_) => return Err(data_type_not_supported("Serial")), + }, + None => match schema[col_idx].data_type { + DataType::Boolean => { + query.bind(None as Option); + } + DataType::Int16 => { + query.bind(None as Option); + } + DataType::Int32 => { + query.bind(None as Option); + } + DataType::Int64 => { + query.bind(None as Option); + } + DataType::Float32 => { + query.bind(None as Option); + } + DataType::Float64 => { + query.bind(None as Option); + } + DataType::Decimal => { + query.bind(None as Option); + } + DataType::Date => { + query.bind(None as Option); + } + DataType::Time => { + query.bind(None as Option); + } + DataType::Timestamp => { + query.bind(None as Option); + } + DataType::Timestamptz => { + query.bind(None as Option); + } + DataType::Varchar => { + query.bind(None as Option); + } + DataType::Bytea => { + query.bind(None as Option>); + } + DataType::Interval => return Err(data_type_not_supported("Interval")), + DataType::Struct(_) => return Err(data_type_not_supported("Struct")), + DataType::List(_) => return Err(data_type_not_supported("List")), + DataType::Jsonb => return Err(data_type_not_supported("Jsonb")), + DataType::Serial => return Err(data_type_not_supported("Serial")), + DataType::Int256 => return Err(data_type_not_supported("Int256")), + }, + }; + } + Ok(()) +} + +fn data_type_not_supported(data_type_name: &str) -> SinkError { + SinkError::SqlServer(anyhow!(format!( + "{data_type_name} is not supported in SQL Server" + ))) +} + +fn check_data_type_compatibility(data_type: &DataType) -> Result<()> { + match data_type { + DataType::Boolean + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::Float32 + | DataType::Float64 + | DataType::Decimal + | DataType::Date + | DataType::Varchar + | DataType::Time + | DataType::Timestamp + | DataType::Timestamptz + | DataType::Bytea => Ok(()), + DataType::Interval => Err(data_type_not_supported("Interval")), + DataType::Struct(_) => Err(data_type_not_supported("Struct")), + DataType::List(_) => Err(data_type_not_supported("List")), + DataType::Jsonb => Err(data_type_not_supported("Jsonb")), + DataType::Serial => Err(data_type_not_supported("Serial")), + DataType::Int256 => Err(data_type_not_supported("Int256")), + } +} + +/// The implementation is copied from tiberius crate. +fn decimal_to_sql(decimal: &rust_decimal::Decimal) -> Numeric { + let unpacked = decimal.unpack(); + + let mut value = (((unpacked.hi as u128) << 64) + + ((unpacked.mid as u128) << 32) + + unpacked.lo as u128) as i128; + + if decimal.is_sign_negative() { + value = -value; + } + + Numeric::new_with_scale(value, decimal.scale() as u8) +} diff --git a/src/connector/src/with_options.rs b/src/connector/src/with_options.rs index 7e0eb4ce7160..3207a7bbbde2 100644 --- a/src/connector/src/with_options.rs +++ b/src/connector/src/with_options.rs @@ -53,6 +53,7 @@ impl WithOptions for BTreeMap {} impl WithOptions for String {} impl WithOptions for bool {} impl WithOptions for usize {} +impl WithOptions for u16 {} impl WithOptions for u32 {} impl WithOptions for u64 {} impl WithOptions for i32 {} diff --git a/src/connector/with_options_sink.yaml b/src/connector/with_options_sink.yaml index 54bde5a8325e..731bb900335e 100644 --- a/src/connector/with_options_sink.yaml +++ b/src/connector/with_options_sink.yaml @@ -751,6 +751,33 @@ SnowflakeConfig: field_type: String comments: The s3 region, e.g., us-east-2 required: true +SqlServerConfig: + fields: + - name: sqlserver.host + field_type: String + required: true + - name: sqlserver.port + field_type: u16 + required: true + - name: sqlserver.user + field_type: String + required: true + - name: sqlserver.password + field_type: String + required: true + - name: sqlserver.database + field_type: String + required: true + - name: sqlserver.table + field_type: String + required: true + - name: sqlserver.max_batch_rows + field_type: usize + required: false + default: '1024' + - name: r#type + field_type: String + required: true StarrocksConfig: fields: - name: starrocks.host diff --git a/src/ctl/src/cmd_impl/hummock/compaction_group.rs b/src/ctl/src/cmd_impl/hummock/compaction_group.rs index 3fb83e9d16cb..d58aeb7bffe7 100644 --- a/src/ctl/src/cmd_impl/hummock/compaction_group.rs +++ b/src/ctl/src/cmd_impl/hummock/compaction_group.rs @@ -65,6 +65,7 @@ pub fn build_compaction_config_vec( enable_emergency_picker: Option, tombstone_reclaim_ratio: Option, compress_algorithm: Option, + max_l0_compact_level: Option, ) -> Vec { let mut configs = vec![]; if let Some(c) = max_bytes_for_level_base { @@ -115,6 +116,9 @@ pub fn build_compaction_config_vec( if let Some(c) = compress_algorithm { configs.push(MutableConfig::CompressionAlgorithm(c)) } + if let Some(c) = max_l0_compact_level { + configs.push(MutableConfig::MaxL0CompactLevelCount(c)) + } configs } diff --git a/src/ctl/src/lib.rs b/src/ctl/src/lib.rs index 1276d4bfce43..1f50250276d6 100644 --- a/src/ctl/src/lib.rs +++ b/src/ctl/src/lib.rs @@ -259,6 +259,8 @@ enum HummockCommands { compression_level: Option, #[clap(long)] compression_algorithm: Option, + #[clap(long)] + max_l0_compact_level: Option, }, /// Split given compaction group into two. Moves the given tables to the new group. SplitCompactionGroup { @@ -690,6 +692,7 @@ async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> { tombstone_reclaim_ratio, compression_level, compression_algorithm, + max_l0_compact_level, }) => { cmd_impl::hummock::update_compaction_config( context, @@ -719,6 +722,7 @@ async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> { } else { None }, + max_l0_compact_level, ), ) .await? diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs index e3c0578ac686..f7d265485f70 100644 --- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs +++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs @@ -106,7 +106,7 @@ fn version_to_compaction_group_rows(version: &HummockVersion) -> Vec { Err(ErrorCode::SinkError(Box::new(Error::new( ErrorKind::InvalidInput, - format!("The sink cannot be append-only. Please add \"force_append_only='true'\" in {} options to force the sink to be append-only. Notice that this will cause the sink executor to drop any UPDATE or DELETE message.", if syntax_legacy { "WITH" } else { "FORMAT ENCODE" }), + format!( + "The sink cannot be append-only. Please add \"force_append_only='true'\" in {} options to force the sink to be append-only. \ + Notice that this will cause the sink executor to drop DELETE messages and convert UPDATE messages to INSERT.", + if syntax_legacy { "WITH" } else { "FORMAT ENCODE" } + ), ))) .into()) } diff --git a/src/java_binding/Cargo.toml b/src/java_binding/Cargo.toml index 477f19878cbd..0966b700a713 100644 --- a/src/java_binding/Cargo.toml +++ b/src/java_binding/Cargo.toml @@ -10,13 +10,32 @@ ignored = ["workspace-hack"] normal = ["workspace-hack"] [dependencies] +anyhow = "1" +bytes = "1" +cfg-or-panic = "0.2" +foyer ={ workspace = true } +futures = { version = "0.3", default-features = false, features = ["alloc"] } jni = "0.21.1" prost = { workspace = true } risingwave_common = { workspace = true } +risingwave_hummock_sdk = { workspace = true } risingwave_jni_core = { workspace = true } +risingwave_object_store = { workspace = true } risingwave_pb = { workspace = true } +risingwave_storage = { workspace = true } +rw_futures_util = { workspace = true } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +tokio = { version = "0.2", package = "madsim-tokio", features = [ + "fs", + "rt", + "rt-multi-thread", + "sync", + "macros", + "time", + "signal", +] } +tracing = "0.1" [dev-dependencies] risingwave_expr = { workspace = true } diff --git a/src/jni_core/src/hummock_iterator.rs b/src/java_binding/src/hummock_iterator.rs similarity index 87% rename from src/jni_core/src/hummock_iterator.rs rename to src/java_binding/src/hummock_iterator.rs index c4445dece131..4b6fc5b01742 100644 --- a/src/jni_core/src/hummock_iterator.rs +++ b/src/java_binding/src/hummock_iterator.rs @@ -14,9 +14,10 @@ use std::sync::Arc; +use anyhow::anyhow; use bytes::Bytes; use foyer::HybridCacheBuilder; -use futures::{Stream, TryFutureExt, TryStreamExt}; +use futures::{TryFutureExt, TryStreamExt}; use risingwave_common::catalog::ColumnDesc; use risingwave_common::config::{MetricLevel, ObjectStoreConfig}; use risingwave_common::hash::VirtualNode; @@ -25,6 +26,7 @@ use risingwave_common::util::value_encoding::column_aware_row_encoding::ColumnAw use risingwave_common::util::value_encoding::{BasicSerde, EitherSerde, ValueRowDeserializer}; use risingwave_hummock_sdk::key::{prefixed_range_with_vnode, TableKeyRange}; use risingwave_hummock_sdk::version::HummockVersion; +use risingwave_jni_core::HummockJavaBindingIterator; use risingwave_object_store::object::build_remote_object_store; use risingwave_object_store::object::object_metrics::ObjectStoreMetrics; use risingwave_pb::java_binding::key_range::Bound; @@ -39,35 +41,34 @@ use risingwave_storage::hummock::{ use risingwave_storage::monitor::{global_hummock_state_store_metrics, HummockStateStoreMetrics}; use risingwave_storage::row_serde::value_serde::ValueRowSerdeNew; use risingwave_storage::store::{ReadOptions, StateStoreIterExt}; -use risingwave_storage::table::KeyedRow; use rw_futures_util::select_all; use tokio::sync::mpsc::unbounded_channel; -type SelectAllIterStream = impl Stream>> + Unpin; -type SingleIterStream = impl Stream>>; +type SingleIterStream = HummockJavaBindingIterator; -fn select_all_vnode_stream(streams: Vec) -> SelectAllIterStream { - select_all(streams.into_iter().map(Box::pin)) +fn select_all_vnode_stream(streams: Vec) -> HummockJavaBindingIterator { + Box::pin(select_all(streams)) } fn to_deserialized_stream( iter: HummockStorageIterator, row_serde: EitherSerde, ) -> SingleIterStream { - iter.into_stream(move |(key, value)| { - Ok(KeyedRow::new( - key.user_key.table_key.copy_into(), - row_serde.deserialize(value).map(OwnedRow::new)?, - )) - }) + Box::pin( + iter.into_stream(move |(key, value)| { + Ok(( + Bytes::copy_from_slice(key.user_key.table_key.0), + row_serde.deserialize(value).map(OwnedRow::new)?, + )) + }) + .map_err(|e| anyhow!(e)), + ) } -pub struct HummockJavaBindingIterator { - stream: SelectAllIterStream, -} - -impl HummockJavaBindingIterator { - pub async fn new(read_plan: ReadPlan) -> StorageResult { +pub(crate) async fn new_hummock_java_binding_iter( + read_plan: ReadPlan, +) -> StorageResult { + { // Note(bugen): should we forward the implementation to the `StorageTable`? let object_store = Arc::new( build_remote_object_store( @@ -170,11 +171,7 @@ impl HummockJavaBindingIterator { let stream = select_all_vnode_stream(streams); - Ok(Self { stream }) - } - - pub async fn next(&mut self) -> StorageResult>> { - self.stream.try_next().await + Ok(stream) } } diff --git a/src/java_binding/src/lib.rs b/src/java_binding/src/lib.rs index ef5bb228b0ca..4fd089918bd5 100644 --- a/src/java_binding/src/lib.rs +++ b/src/java_binding/src/lib.rs @@ -12,16 +12,83 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![feature(type_alias_impl_trait)] +#![feature(try_blocks)] + +mod hummock_iterator; use std::ffi::c_void; +use std::ops::Deref; +use anyhow::anyhow; +use cfg_or_panic::cfg_or_panic; +use jni::objects::JByteArray; use jni::sys::{jint, JNI_VERSION_1_2}; -use jni::JavaVM; -use risingwave_jni_core::register_native_method_for_jvm; +use jni::{JNIEnv, JavaVM}; +use prost::Message; +use risingwave_common::error::AsReport; +use risingwave_jni_core::jvm_runtime::{jvm_env, register_java_binding_native_methods}; +use risingwave_jni_core::{ + execute_and_catch, gen_class_name, to_guarded_slice, EnvParam, JavaBindingIterator, Pointer, + JAVA_BINDING_ASYNC_RUNTIME, +}; + +use crate::hummock_iterator::new_hummock_java_binding_iter; + +fn register_hummock_java_binding_native_methods( + env: &mut JNIEnv<'_>, +) -> Result<(), jni::errors::Error> { + let binding_class = env + .find_class(gen_class_name!(com.risingwave.java.binding.HummockIterator)) + .inspect_err(|e| tracing::error!(error = ?e.as_report(), "jvm find class error"))?; + macro_rules! gen_native_method_array { + () => {{ + risingwave_jni_core::split_extract_plain_native_methods! {{long iteratorNewHummock(byte[] readPlan);}, gen_native_method_array} + }}; + ({$({ $func_name:ident, {$($ret:tt)+}, {$($args:tt)*} })*}) => { + [ + $( + risingwave_jni_core::gen_native_method_entry! { + Java_com_risingwave_java_binding_HummockIterator_, $func_name, {$($ret)+}, {$($args)*} + }, + )* + ] + } + } + env.register_native_methods(binding_class, &gen_native_method_array!()) + .inspect_err( + |e| tracing::error!(error = ?e.as_report(), "jvm register native methods error"), + )?; + + tracing::info!("register native methods for jvm successfully"); + Ok(()) +} #[no_mangle] #[allow(non_snake_case)] pub extern "system" fn JNI_OnLoad(jvm: JavaVM, _reserved: *mut c_void) -> jint { - let _ = register_native_method_for_jvm(&jvm) - .inspect_err(|_e| eprintln!("unable to register native method")); + let result: Result<(), jni::errors::Error> = try { + let mut env = jvm_env(&jvm)?; + register_java_binding_native_methods(&mut env)?; + register_hummock_java_binding_native_methods(&mut env)?; + }; + let _ = + result.inspect_err(|e| eprintln!("unable to register native method: {:?}", e.as_report())); + JNI_VERSION_1_2 } + +#[cfg_or_panic(not(madsim))] +#[no_mangle] +extern "system" fn Java_com_risingwave_java_binding_HummockIterator_iteratorNewHummock<'a>( + env: EnvParam<'a>, + read_plan: JByteArray<'a>, +) -> Pointer<'static, JavaBindingIterator<'static>> { + execute_and_catch(env, move |env| { + let read_plan = Message::decode(to_guarded_slice(&read_plan, env)?.deref())?; + let iter = JAVA_BINDING_ASYNC_RUNTIME + .block_on(new_hummock_java_binding_iter(read_plan)) + .map_err(|e| anyhow!(e))?; + let iter = JavaBindingIterator::new_hummock_iter(iter); + Ok(iter.into()) + }) +} diff --git a/src/jni_core/Cargo.toml b/src/jni_core/Cargo.toml index 4d9c6cab092a..a16776add6c6 100644 --- a/src/jni_core/Cargo.toml +++ b/src/jni_core/Cargo.toml @@ -22,10 +22,7 @@ jni = { version = "0.21.1", features = ["invocation"] } paste = "1" prost = { workspace = true } risingwave_common = { workspace = true } -risingwave_hummock_sdk = { workspace = true } -risingwave_object_store = { workspace = true } risingwave_pb = { workspace = true } -risingwave_storage = { workspace = true } rw_futures_util = { workspace = true } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/src/jni_core/src/jvm_runtime.rs b/src/jni_core/src/jvm_runtime.rs index f848b7d44240..e596d5664dac 100644 --- a/src/jni_core/src/jvm_runtime.rs +++ b/src/jni_core/src/jvm_runtime.rs @@ -20,8 +20,7 @@ use anyhow::{bail, Context}; use fs_err as fs; use fs_err::PathExt; use jni::objects::{JObject, JString}; -use jni::strings::JNIString; -use jni::{InitArgsBuilder, JNIEnv, JNIVersion, JavaVM, NativeMethod}; +use jni::{AttachGuard, InitArgsBuilder, JNIEnv, JNIVersion, JavaVM}; use risingwave_common::util::resource_util::memory::system_memory_available_bytes; use thiserror_ext::AsReport; use tracing::error; @@ -122,19 +121,27 @@ impl JavaVmWrapper { tracing::info!("initialize JVM successfully"); - register_native_method_for_jvm(&jvm).context("failed to register native method")?; + let result: std::result::Result<(), jni::errors::Error> = try { + let mut env = jvm_env(&jvm)?; + register_java_binding_native_methods(&mut env)?; + }; + + result.context("failed to register native method")?; Ok(jvm) } } -pub fn register_native_method_for_jvm(jvm: &JavaVM) -> Result<(), jni::errors::Error> { - let mut env = jvm - .attach_current_thread() - .inspect_err(|e| tracing::error!(error = ?e.as_report(), "jvm attach thread error"))?; +pub fn jvm_env(jvm: &JavaVM) -> Result, jni::errors::Error> { + jvm.attach_current_thread() + .inspect_err(|e| tracing::error!(error = ?e.as_report(), "jvm attach thread error")) +} +pub fn register_java_binding_native_methods( + env: &mut JNIEnv<'_>, +) -> Result<(), jni::errors::Error> { let binding_class = env - .find_class("com/risingwave/java/binding/Binding") + .find_class(gen_class_name!(com.risingwave.java.binding.Binding)) .inspect_err(|e| tracing::error!(error = ?e.as_report(), "jvm find class error"))?; use crate::*; macro_rules! gen_native_method_array { @@ -144,14 +151,8 @@ pub fn register_native_method_for_jvm(jvm: &JavaVM) -> Result<(), jni::errors::E ({$({ $func_name:ident, {$($ret:tt)+}, {$($args:tt)*} })*}) => { [ $( - { - let fn_ptr = paste::paste! {[ ]} as *mut c_void; - let sig = $crate::gen_jni_sig! { {$($ret)+}, {$($args)*}}; - NativeMethod { - name: JNIString::from(stringify! {$func_name}), - sig: JNIString::from(sig), - fn_ptr, - } + $crate::gen_native_method_entry! { + Java_com_risingwave_java_binding_Binding_, $func_name, {$($ret)+}, {$($args)*} }, )* ] diff --git a/src/jni_core/src/lib.rs b/src/jni_core/src/lib.rs index 7ff8e5aa930e..18d1807948d2 100644 --- a/src/jni_core/src/lib.rs +++ b/src/jni_core/src/lib.rs @@ -18,7 +18,6 @@ #![feature(type_alias_impl_trait)] #![feature(try_blocks)] -pub mod hummock_iterator; pub mod jvm_runtime; mod macros; mod tracing_slf4j; @@ -33,6 +32,8 @@ use anyhow::anyhow; use bytes::Bytes; use cfg_or_panic::cfg_or_panic; use chrono::{Datelike, NaiveDateTime, Timelike}; +use futures::stream::BoxStream; +use futures::TryStreamExt; use jni::objects::{ AutoElements, GlobalRef, JByteArray, JClass, JMethodID, JObject, JStaticMethodID, JString, JValueOwned, ReleaseMode, @@ -42,6 +43,7 @@ use jni::sys::{ jboolean, jbyte, jdouble, jfloat, jint, jlong, jshort, jsize, jvalue, JNI_FALSE, JNI_TRUE, }; use jni::JNIEnv; +pub use paste::paste; use prost::{DecodeError, Message}; use risingwave_common::array::{ArrayError, StreamChunk}; use risingwave_common::hash::VirtualNode; @@ -54,17 +56,14 @@ use risingwave_pb::connector_service::{ SinkWriterStreamRequest, SinkWriterStreamResponse, }; use risingwave_pb::data::Op; -use risingwave_storage::error::StorageError; use thiserror::Error; use thiserror_ext::AsReport; use tokio::runtime::Runtime; use tokio::sync::mpsc::{Receiver, Sender}; use tracing_slf4j::*; -use crate::hummock_iterator::HummockJavaBindingIterator; -pub use crate::jvm_runtime::register_native_method_for_jvm; - -static RUNTIME: LazyLock = LazyLock::new(|| tokio::runtime::Runtime::new().unwrap()); +pub static JAVA_BINDING_ASYNC_RUNTIME: LazyLock = + LazyLock::new(|| tokio::runtime::Runtime::new().unwrap()); #[derive(Error, Debug)] pub enum BindingError { @@ -78,7 +77,7 @@ pub enum BindingError { #[error("StorageError {error}")] Storage { #[from] - error: StorageError, + error: anyhow::Error, backtrace: Backtrace, }, @@ -201,7 +200,7 @@ impl<'a> EnvParam<'a> { } } -fn execute_and_catch<'env, F, Ret>(mut env: EnvParam<'env>, inner: F) -> Ret +pub fn execute_and_catch<'env, F, Ret>(mut env: EnvParam<'env>, inner: F) -> Ret where F: FnOnce(&mut EnvParam<'env>) -> Result, Ret: Default + 'env, @@ -245,9 +244,10 @@ struct JavaClassMethodCache { } // TODO: may only return a RowRef -type StreamChunkRowIterator<'a> = impl Iterator + 'a; +pub type StreamChunkRowIterator<'a> = impl Iterator + 'a; +pub type HummockJavaBindingIterator = BoxStream<'static, anyhow::Result<(Bytes, OwnedRow)>>; -enum JavaBindingIteratorInner<'a> { +pub enum JavaBindingIteratorInner<'a> { Hummock(HummockJavaBindingIterator), StreamChunk(StreamChunkRowIterator<'a>), } @@ -288,12 +288,22 @@ struct RowCursor { extra: RowExtra, } -struct JavaBindingIterator<'a> { +pub struct JavaBindingIterator<'a> { inner: JavaBindingIteratorInner<'a>, cursor: Option, class_cache: JavaClassMethodCache, } +impl JavaBindingIterator<'static> { + pub fn new_hummock_iter(iter: HummockJavaBindingIterator) -> Self { + Self { + inner: JavaBindingIteratorInner::Hummock(iter), + cursor: None, + class_cache: Default::default(), + } + } +} + impl<'a> Deref for JavaBindingIterator<'a> { type Target = OwnedRow; @@ -311,24 +321,6 @@ extern "system" fn Java_com_risingwave_java_binding_Binding_vnodeCount(_env: Env VirtualNode::COUNT as jint } -#[cfg_or_panic(not(madsim))] -#[no_mangle] -extern "system" fn Java_com_risingwave_java_binding_Binding_iteratorNewHummock<'a>( - env: EnvParam<'a>, - read_plan: JByteArray<'a>, -) -> Pointer<'static, JavaBindingIterator<'static>> { - execute_and_catch(env, move |env| { - let read_plan = Message::decode(to_guarded_slice(&read_plan, env)?.deref())?; - let iter = RUNTIME.block_on(HummockJavaBindingIterator::new(read_plan))?; - let iter = JavaBindingIterator { - inner: JavaBindingIteratorInner::Hummock(iter), - cursor: None, - class_cache: Default::default(), - }; - Ok(iter.into()) - }) -} - #[cfg_or_panic(not(madsim))] #[no_mangle] extern "system" fn Java_com_risingwave_java_binding_Binding_iteratorNewStreamChunk<'a>( @@ -355,16 +347,15 @@ extern "system" fn Java_com_risingwave_java_binding_Binding_iteratorNext<'a>( let iter = pointer.as_mut(); match &mut iter.inner { JavaBindingIteratorInner::Hummock(ref mut hummock_iter) => { - match RUNTIME.block_on(hummock_iter.next())? { + match JAVA_BINDING_ASYNC_RUNTIME.block_on(hummock_iter.try_next())? { None => { iter.cursor = None; Ok(JNI_FALSE) } - Some(keyed_row) => { - let (key, row) = keyed_row.into_parts(); + Some((key, row)) => { iter.cursor = Some(RowCursor { row, - extra: RowExtra::Key(key.0), + extra: RowExtra::Key(key), }); Ok(JNI_TRUE) } diff --git a/src/jni_core/src/macros.rs b/src/jni_core/src/macros.rs index 1b2f79f82956..982ccda06ecf 100644 --- a/src/jni_core/src/macros.rs +++ b/src/jni_core/src/macros.rs @@ -448,10 +448,6 @@ macro_rules! for_all_plain_native_methods { public static native int vnodeCount(); - // hummock iterator method - // Return a pointer to the iterator - static native long iteratorNewHummock(byte[] readPlan); - static native long iteratorNewStreamChunk(long pointer); static native boolean iteratorNext(long pointer); @@ -839,6 +835,23 @@ macro_rules! call_method { }}; } +#[macro_export] +macro_rules! gen_native_method_entry { + ( + $class_prefix:ident, $func_name:ident, {$($ret:tt)+}, {$($args:tt)*} + ) => {{ + { + let fn_ptr = $crate::paste! {[<$class_prefix $func_name> ]} as *mut c_void; + let sig = $crate::gen_jni_sig! { {$($ret)+}, {$($args)*}}; + jni::NativeMethod { + name: jni::strings::JNIString::from(stringify! {$func_name}), + sig: jni::strings::JNIString::from(sig), + fn_ptr, + } + } + }}; +} + #[cfg(test)] mod tests { use std::fmt::Formatter; @@ -891,7 +904,6 @@ mod tests { tracingSlf4jEvent (Ljava/lang/String;Ljava/lang/String;ILjava/lang/String;)V, tracingSlf4jEventEnabled (I)Z, vnodeCount ()I, - iteratorNewHummock ([B)J, iteratorNewStreamChunk (J)J, iteratorNext (J)Z, iteratorClose (J)V, diff --git a/src/meta/model_v2/src/secret.rs b/src/meta/model_v2/src/secret.rs index af3590dd0de5..0d122267bb4b 100644 --- a/src/meta/model_v2/src/secret.rs +++ b/src/meta/model_v2/src/secret.rs @@ -15,8 +15,9 @@ use risingwave_pb::catalog::PbSecret; use sea_orm::entity::prelude::*; use sea_orm::ActiveValue::Set; +use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] #[sea_orm(table_name = "secret")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] diff --git a/src/meta/src/backup_restore/meta_snapshot_builder.rs b/src/meta/src/backup_restore/meta_snapshot_builder.rs index 323f49b68663..bb8a1eb919fd 100644 --- a/src/meta/src/backup_restore/meta_snapshot_builder.rs +++ b/src/meta/src/backup_restore/meta_snapshot_builder.rs @@ -187,10 +187,8 @@ mod tests { let meta_store = MemStore::new(); let mut builder = MetaSnapshotBuilder::new(meta_store.clone()); - let hummock_version = HummockVersion { - id: 1, - ..Default::default() - }; + let mut hummock_version = HummockVersion::default(); + hummock_version.id = 1; let get_ckpt_builder = |v: &HummockVersion| { let v_ = v.clone(); async move { v_ } diff --git a/src/meta/src/backup_restore/meta_snapshot_builder_v2.rs b/src/meta/src/backup_restore/meta_snapshot_builder_v2.rs index c2bafcd071c6..4a81ae9d6ad0 100644 --- a/src/meta/src/backup_restore/meta_snapshot_builder_v2.rs +++ b/src/meta/src/backup_restore/meta_snapshot_builder_v2.rs @@ -31,6 +31,25 @@ fn map_db_err(e: DbErr) -> BackupError { BackupError::MetaStorage(e.into()) } +macro_rules! define_set_metadata { + ($( {$name:ident, $mod_path:ident::$mod_name:ident} ),*) => { + pub async fn set_metadata( + metadata: &mut MetadataV2, + txn: &sea_orm::DatabaseTransaction, + ) -> BackupResult<()> { + $( + metadata.$name = $mod_path::$mod_name::Entity::find() + .all(txn) + .await + .map_err(map_db_err)?; + )* + Ok(()) + } + }; +} + +risingwave_backup::for_all_metadata_models_v2!(define_set_metadata); + pub struct MetaSnapshotV2Builder { snapshot: MetaSnapshotV2, meta_store: SqlMetaStore, @@ -91,151 +110,14 @@ impl MetaSnapshotV2Builder { } redo_state }; - let version_stats = model_v2::prelude::HummockVersionStats::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let compaction_configs = model_v2::prelude::CompactionConfig::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let actors = model_v2::prelude::Actor::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let clusters = model_v2::prelude::Cluster::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let actor_dispatchers = model_v2::prelude::ActorDispatcher::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let catalog_versions = model_v2::prelude::CatalogVersion::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let connections = model_v2::prelude::Connection::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let databases = model_v2::prelude::Database::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let fragments = model_v2::prelude::Fragment::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let functions = model_v2::prelude::Function::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let indexes = model_v2::prelude::Index::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let objects = model_v2::prelude::Object::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let object_dependencies = model_v2::prelude::ObjectDependency::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let schemas = model_v2::prelude::Schema::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let sinks = model_v2::prelude::Sink::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let sources = model_v2::prelude::Source::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let streaming_jobs = model_v2::prelude::StreamingJob::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let subscriptions = model_v2::prelude::Subscription::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let system_parameters = model_v2::prelude::SystemParameter::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let tables = model_v2::prelude::Table::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let users = model_v2::prelude::User::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let user_privileges = model_v2::prelude::UserPrivilege::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let views = model_v2::prelude::View::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let workers = model_v2::prelude::Worker::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let worker_properties = model_v2::prelude::WorkerProperty::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let hummock_sequences = model_v2::prelude::HummockSequence::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let seaql_migrations = model_v2::serde_seaql_migration::Entity::find() - .all(&txn) - .await - .map_err(map_db_err)?; - let session_parameters = model_v2::prelude::SessionParameter::find() - .all(&txn) - .await - .map_err(map_db_err)?; - - txn.commit().await.map_err(map_db_err)?; - self.snapshot.metadata = MetadataV2 { - seaql_migrations, + let mut metadata = MetadataV2 { hummock_version, - version_stats, - compaction_configs, - actors, - clusters, - actor_dispatchers, - catalog_versions, - connections, - databases, - fragments, - functions, - indexes, - objects, - object_dependencies, - schemas, - sinks, - sources, - streaming_jobs, - subscriptions, - system_parameters, - tables, - users, - user_privileges, - views, - workers, - worker_properties, - hummock_sequences, - session_parameters, + ..Default::default() }; + set_metadata(&mut metadata, &txn).await?; + + txn.commit().await.map_err(map_db_err)?; + self.snapshot.metadata = metadata; Ok(()) } diff --git a/src/meta/src/backup_restore/restore.rs b/src/meta/src/backup_restore/restore.rs index f94737a1bd49..1ba13c8cbf8b 100644 --- a/src/meta/src/backup_restore/restore.rs +++ b/src/meta/src/backup_restore/restore.rs @@ -266,9 +266,10 @@ mod tests { let snapshot = MetaSnapshot { id: opts.meta_snapshot_id, metadata: ClusterMetadata { - hummock_version: HummockVersion { - id: 123, - ..Default::default() + hummock_version: { + let mut version = HummockVersion::default(); + version.id = 123; + version }, system_param: system_param.clone(), ..Default::default() @@ -448,9 +449,10 @@ mod tests { memcomparable::to_vec(&"some_value_2".to_string()).unwrap(), ), ]), - hummock_version: HummockVersion { - id: 123, - ..Default::default() + hummock_version: { + let mut version = HummockVersion::default(); + version.id = 123; + version }, system_param: system_param.clone(), ..Default::default() diff --git a/src/meta/src/backup_restore/restore_impl/v2.rs b/src/meta/src/backup_restore/restore_impl/v2.rs index cccb5c9b641d..13492c56316a 100644 --- a/src/meta/src/backup_restore/restore_impl/v2.rs +++ b/src/meta/src/backup_restore/restore_impl/v2.rs @@ -93,40 +93,28 @@ impl WriterModelV2ToMetaStoreV2 { } } +macro_rules! define_write_model_v2_to_meta_store_v2 { + ($( {$name:ident, $mod_path:ident::$mod_name:ident} ),*) => { + async fn write_model_v2_to_meta_store_v2( + metadata: &risingwave_backup::meta_snapshot_v2::MetadataV2, + db: &sea_orm::DatabaseConnection, + ) -> BackupResult<()> { + $( + insert_models(metadata.$name.clone(), db).await?; + )* + Ok(()) + } + }; +} + +risingwave_backup::for_all_metadata_models_v2!(define_write_model_v2_to_meta_store_v2); + #[async_trait::async_trait] impl Writer for WriterModelV2ToMetaStoreV2 { async fn write(&self, target_snapshot: MetaSnapshot) -> BackupResult<()> { let metadata = target_snapshot.metadata; let db = &self.meta_store.conn; - insert_models(metadata.seaql_migrations.clone(), db).await?; - insert_models(metadata.clusters.clone(), db).await?; - insert_models(metadata.version_stats.clone(), db).await?; - insert_models(metadata.compaction_configs.clone(), db).await?; - insert_models(metadata.hummock_sequences.clone(), db).await?; - insert_models(metadata.workers.clone(), db).await?; - insert_models(metadata.worker_properties.clone(), db).await?; - insert_models(metadata.users.clone(), db).await?; - insert_models(metadata.user_privileges.clone(), db).await?; - insert_models(metadata.objects.clone(), db).await?; - insert_models(metadata.object_dependencies.clone(), db).await?; - insert_models(metadata.databases.clone(), db).await?; - insert_models(metadata.schemas.clone(), db).await?; - insert_models(metadata.streaming_jobs.clone(), db).await?; - insert_models(metadata.fragments.clone(), db).await?; - insert_models(metadata.actors.clone(), db).await?; - insert_models(metadata.actor_dispatchers.clone(), db).await?; - insert_models(metadata.connections.clone(), db).await?; - insert_models(metadata.sources.clone(), db).await?; - insert_models(metadata.tables.clone(), db).await?; - insert_models(metadata.sinks.clone(), db).await?; - insert_models(metadata.views.clone(), db).await?; - insert_models(metadata.indexes.clone(), db).await?; - insert_models(metadata.functions.clone(), db).await?; - insert_models(metadata.system_parameters.clone(), db).await?; - insert_models(metadata.catalog_versions.clone(), db).await?; - insert_models(metadata.subscriptions.clone(), db).await?; - insert_models(metadata.session_parameters.clone(), db).await?; - + write_model_v2_to_meta_store_v2(&metadata, db).await?; // update_auto_inc must be called last. update_auto_inc(&metadata, db).await?; Ok(()) diff --git a/src/meta/src/hummock/compaction/compaction_config.rs b/src/meta/src/hummock/compaction/compaction_config.rs index e6750084432e..de91bf4f79de 100644 --- a/src/meta/src/hummock/compaction/compaction_config.rs +++ b/src/meta/src/hummock/compaction/compaction_config.rs @@ -64,6 +64,7 @@ impl CompactionConfigBuilder { compaction_config::level0_overlapping_sub_level_compact_level_count(), tombstone_reclaim_ratio: compaction_config::tombstone_reclaim_ratio(), enable_emergency_picker: compaction_config::enable_emergency_picker(), + max_l0_compact_level_count: compaction_config::max_l0_compact_level_count(), }, } } diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs index f98e14203d95..a2c3a7d52802 100644 --- a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs @@ -166,6 +166,7 @@ impl LevelCompactionPicker { self.config.level0_max_compact_file_number, overlap_strategy.clone(), self.developer_config.enable_check_task_level_overlap, + self.config.max_l0_compact_level_count as usize, ); let mut max_vnode_partition_idx = 0; diff --git a/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs b/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs index 29119ae283b0..c7dd27a6b190 100644 --- a/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs +++ b/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use risingwave_pb::hummock::CompactionConfig; -use super::{CompactionInput, LocalPickerStatistic, MAX_COMPACT_LEVEL_COUNT}; +use super::{CompactionInput, LocalPickerStatistic}; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum ValidationRuleType { @@ -89,14 +89,8 @@ struct TierCompactionTaskValidationRule { impl CompactionTaskValidationRule for TierCompactionTaskValidationRule { fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool { - // Limit sstable file count to avoid using too much memory. - let overlapping_max_compact_file_numer = std::cmp::min( - self.config.level0_max_compact_file_number, - MAX_COMPACT_LEVEL_COUNT as u64, - ); - - if input.total_file_count >= overlapping_max_compact_file_numer - || input.input_levels.len() >= MAX_COMPACT_LEVEL_COUNT + if input.total_file_count >= self.config.level0_max_compact_file_number + || input.input_levels.len() >= self.config.max_l0_compact_level_count as usize { return true; } @@ -130,7 +124,7 @@ impl CompactionTaskValidationRule for IntraCompactionTaskValidationRule { fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool { if (input.total_file_count >= self.config.level0_max_compact_file_number && input.input_levels.len() > 1) - || input.input_levels.len() >= MAX_COMPACT_LEVEL_COUNT + || input.input_levels.len() >= self.config.max_l0_compact_level_count as usize { return true; } @@ -178,7 +172,7 @@ struct BaseCompactionTaskValidationRule { impl CompactionTaskValidationRule for BaseCompactionTaskValidationRule { fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool { if input.total_file_count >= self.config.level0_max_compact_file_number - || input.input_levels.len() >= MAX_COMPACT_LEVEL_COUNT + || input.input_levels.len() >= self.config.max_l0_compact_level_count as usize { return true; } diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs index 993ad79d59b2..6b5dcae7d0c3 100644 --- a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs @@ -144,6 +144,7 @@ impl IntraCompactionPicker { self.config.level0_max_compact_file_number, overlap_strategy.clone(), self.developer_config.enable_check_task_level_overlap, + self.config.max_l0_compact_level_count as usize, ); let l0_select_tables_vec = non_overlap_sub_level_picker diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index a0d896daa439..57dd5469d42a 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -20,7 +20,7 @@ use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; use risingwave_pb::hummock::hummock_version::Levels; use risingwave_pb::hummock::{InputLevel, Level, LevelType, SstableInfo}; -use super::{CompactionInput, CompactionPicker, LocalPickerStatistic, MAX_COMPACT_LEVEL_COUNT}; +use super::{CompactionInput, CompactionPicker, LocalPickerStatistic}; use crate::hummock::compaction::overlap_strategy::OverlapStrategy; use crate::hummock::level_handler::LevelHandler; @@ -197,6 +197,7 @@ impl NonOverlapSubLevelPicker { max_file_count: u64, overlap_strategy: Arc, enable_check_task_level_overlap: bool, + max_expected_level_count: usize, ) -> Self { Self { min_compaction_bytes, @@ -205,7 +206,7 @@ impl NonOverlapSubLevelPicker { max_file_count, overlap_strategy, enable_check_task_level_overlap, - max_expected_level_count: MAX_COMPACT_LEVEL_COUNT, + max_expected_level_count, } } @@ -533,6 +534,8 @@ impl NonOverlapSubLevelPicker { pub mod tests { use std::collections::BTreeSet; + use risingwave_common::config::default::compaction_config; + use super::*; use crate::hummock::compaction::overlap_strategy::RangeOverlapStrategy; use crate::hummock::compaction::selector::tests::{ @@ -736,6 +739,7 @@ pub mod tests { 10000, Arc::new(RangeOverlapStrategy::default()), true, + compaction_config::max_l0_compact_level_count() as usize, ); let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]); assert_eq!(6, ret.len()); @@ -750,6 +754,7 @@ pub mod tests { 10000, Arc::new(RangeOverlapStrategy::default()), true, + compaction_config::max_l0_compact_level_count() as usize, ); let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]); assert_eq!(6, ret.len()); @@ -764,6 +769,7 @@ pub mod tests { 5, Arc::new(RangeOverlapStrategy::default()), true, + compaction_config::max_l0_compact_level_count() as usize, ); let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]); assert_eq!(6, ret.len()); @@ -839,6 +845,7 @@ pub mod tests { 10000, Arc::new(RangeOverlapStrategy::default()), true, + compaction_config::max_l0_compact_level_count() as usize, ); let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]); assert_eq!(6, ret.len()); @@ -854,6 +861,7 @@ pub mod tests { 10000, Arc::new(RangeOverlapStrategy::default()), true, + compaction_config::max_l0_compact_level_count() as usize, ); let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]); assert_eq!(6, ret.len()); @@ -869,6 +877,7 @@ pub mod tests { max_file_count, Arc::new(RangeOverlapStrategy::default()), true, + compaction_config::max_l0_compact_level_count() as usize, ); let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]); assert_eq!(6, ret.len()); @@ -892,6 +901,7 @@ pub mod tests { 10000, Arc::new(RangeOverlapStrategy::default()), true, + compaction_config::max_l0_compact_level_count() as usize, ); let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]); assert_eq!(3, ret.len()); @@ -1019,6 +1029,7 @@ pub mod tests { 10000, Arc::new(RangeOverlapStrategy::default()), true, + compaction_config::max_l0_compact_level_count() as usize, ); let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]); { @@ -1036,6 +1047,7 @@ pub mod tests { 10000, Arc::new(RangeOverlapStrategy::default()), true, + compaction_config::max_l0_compact_level_count() as usize, ); let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]); { @@ -1053,6 +1065,7 @@ pub mod tests { 3, Arc::new(RangeOverlapStrategy::default()), true, + compaction_config::max_l0_compact_level_count() as usize, ); let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]); { diff --git a/src/meta/src/hummock/compaction/picker/mod.rs b/src/meta/src/hummock/compaction/picker/mod.rs index f6dc46c99106..6d464b9a33bc 100644 --- a/src/meta/src/hummock/compaction/picker/mod.rs +++ b/src/meta/src/hummock/compaction/picker/mod.rs @@ -43,8 +43,6 @@ pub use ttl_reclaim_compaction_picker::{TtlPickerState, TtlReclaimCompactionPick use crate::hummock::level_handler::LevelHandler; -pub const MAX_COMPACT_LEVEL_COUNT: usize = 42; - #[derive(Default, Debug)] pub struct LocalPickerStatistic { pub skip_by_write_amp_limit: u64, diff --git a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs index 9ed22ba551fc..ce86b523f6e8 100644 --- a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs @@ -21,7 +21,6 @@ use super::{ CompactionInput, CompactionPicker, CompactionTaskValidator, LocalPickerStatistic, ValidationRuleType, }; -use crate::hummock::compaction::picker::MAX_COMPACT_LEVEL_COUNT; use crate::hummock::level_handler::LevelHandler; pub struct TierCompactionPicker { @@ -87,10 +86,7 @@ impl TierCompactionPicker { let mut compaction_bytes = level.total_file_size; let mut compact_file_count = level.table_infos.len() as u64; // Limit sstable file count to avoid using too much memory. - let overlapping_max_compact_file_numer = std::cmp::min( - self.config.level0_max_compact_file_number, - MAX_COMPACT_LEVEL_COUNT as u64, - ); + let overlapping_max_compact_file_numer = self.config.level0_max_compact_file_number; for other in &l0.sub_levels[idx + 1..] { if compaction_bytes > max_compaction_bytes { diff --git a/src/meta/src/hummock/manager/commit_epoch.rs b/src/meta/src/hummock/manager/commit_epoch.rs index 1471ddd32af5..d42c032d5a05 100644 --- a/src/meta/src/hummock/manager/commit_epoch.rs +++ b/src/meta/src/hummock/manager/commit_epoch.rs @@ -338,21 +338,23 @@ impl HummockManager { // update state table info new_version_delta.with_latest_version(|version, delta| { - for table_id in new_table_ids - .into_iter() - .flat_map(|ids| ids.into_iter().map(|table_id| table_id.table_id)) - .chain( - version - .levels - .values() - .flat_map(|group| group.member_table_ids.iter().cloned()), - ) - { + if let Some(new_table_ids) = new_table_ids { + for table_id in new_table_ids { + delta.state_table_info_delta.insert( + table_id, + StateTableInfoDelta { + committed_epoch: epoch, + safe_epoch: epoch, + }, + ); + } + } + for (table_id, info) in version.state_table_info.info() { delta.state_table_info_delta.insert( - TableId::new(table_id), + *table_id, StateTableInfoDelta { committed_epoch: epoch, - safe_epoch: version.safe_epoch, + safe_epoch: info.safe_epoch, }, ); } diff --git a/src/meta/src/hummock/manager/compaction.rs b/src/meta/src/hummock/manager/compaction.rs index 1603969e9f7b..cc5c878a8bae 100644 --- a/src/meta/src/hummock/manager/compaction.rs +++ b/src/meta/src/hummock/manager/compaction.rs @@ -26,6 +26,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::cmp::min; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::{Arc, LazyLock}; use std::time::{Instant, SystemTime}; @@ -182,25 +183,33 @@ impl<'a> HummockVersionTransaction<'a> { }; group_deltas.push(group_delta); version_delta.safe_epoch = std::cmp::max( - version_delta.latest_version().safe_epoch, + version_delta.latest_version().visible_table_safe_epoch(), compact_task.watermark, ); - if version_delta.latest_version().safe_epoch < version_delta.safe_epoch { - version_delta.state_table_info_delta = version_delta - .latest_version() - .state_table_info - .info() - .iter() - .map(|(table_id, info)| { - ( - *table_id, - StateTableInfoDelta { - committed_epoch: info.committed_epoch, - safe_epoch: version_delta.safe_epoch, - }, - ) - }) - .collect(); + if version_delta.latest_version().visible_table_safe_epoch() < version_delta.safe_epoch { + version_delta.with_latest_version(|version, version_delta| { + for (table_id, info) in version.state_table_info.info() { + let new_safe_epoch = min(version_delta.safe_epoch, info.committed_epoch); + if new_safe_epoch > info.safe_epoch { + if new_safe_epoch != version_delta.safe_epoch { + warn!( + new_safe_epoch, + committed_epoch = info.committed_epoch, + global_safe_epoch = version_delta.safe_epoch, + table_id = table_id.table_id, + "table has different safe epoch to global" + ); + } + version_delta.state_table_info_delta.insert( + *table_id, + StateTableInfoDelta { + committed_epoch: info.committed_epoch, + safe_epoch: new_safe_epoch, + }, + ); + } + } + }); } version_delta.pre_apply(); } diff --git a/src/meta/src/hummock/manager/compaction_group_manager.rs b/src/meta/src/hummock/manager/compaction_group_manager.rs index 7d56e2f9dc0d..063ce4e0a58b 100644 --- a/src/meta/src/hummock/manager/compaction_group_manager.rs +++ b/src/meta/src/hummock/manager/compaction_group_manager.rs @@ -188,10 +188,7 @@ impl HummockManager { &self.metrics, ); let mut new_version_delta = version.new_delta(); - let (committed_epoch, safe_epoch) = { - let version = new_version_delta.latest_version(); - (version.max_committed_epoch, version.safe_epoch) - }; + let epoch = new_version_delta.latest_version().max_committed_epoch; for (table_id, raw_group_id) in pairs { let mut group_id = *raw_group_id; @@ -247,8 +244,8 @@ impl HummockManager { .insert( TableId::new(*table_id), StateTableInfoDelta { - committed_epoch, - safe_epoch, + committed_epoch: epoch, + safe_epoch: epoch, } ) .is_none()); @@ -888,11 +885,13 @@ fn update_compaction_config(target: &mut CompactionConfig, items: &[MutableConfi MutableConfig::TombstoneReclaimRatio(c) => { target.tombstone_reclaim_ratio = *c; } - MutableConfig::CompressionAlgorithm(c) => { target.compression_algorithm[c.get_level() as usize] .clone_from(&c.compression_algorithm); } + MutableConfig::MaxL0CompactLevelCount(c) => { + target.max_l0_compact_level_count = *c; + } } } } diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs index 1c0bbde4eb33..47209ddf1fff 100644 --- a/src/meta/src/hummock/manager/mod.rs +++ b/src/meta/src/hummock/manager/mod.rs @@ -392,7 +392,7 @@ impl HummockManager { .read() .await .default_compaction_config(); - let checkpoint_version = create_init_version(default_compaction_config); + let checkpoint_version = HummockVersion::create_init_version(default_compaction_config); tracing::info!("init hummock version checkpoint"); versioning_guard.checkpoint = HummockVersionCheckpoint { version: checkpoint_version.clone(), diff --git a/src/meta/src/hummock/manager/transaction.rs b/src/meta/src/hummock/manager/transaction.rs index e5f2ba4b325b..c467e95adfdb 100644 --- a/src/meta/src/hummock/manager/transaction.rs +++ b/src/meta/src/hummock/manager/transaction.rs @@ -15,7 +15,6 @@ use std::collections::BTreeMap; use std::ops::{Deref, DerefMut}; -use risingwave_hummock_sdk::compaction_group::hummock_version_ext::build_version_delta_after_version; use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta}; use risingwave_hummock_sdk::HummockVersionId; use risingwave_pb::hummock::HummockVersionStats; @@ -38,7 +37,9 @@ fn trigger_version_stat(metrics: &MetaMetrics, current_version: &HummockVersion) metrics .version_size .set(current_version.estimated_encode_len() as i64); - metrics.safe_epoch.set(current_version.safe_epoch as i64); + metrics + .safe_epoch + .set(current_version.visible_table_safe_epoch() as i64); metrics.current_version_id.set(current_version.id as i64); } @@ -86,7 +87,7 @@ impl<'a> HummockVersionTransaction<'a> { } pub(super) fn new_delta<'b>(&'b mut self) -> SingleDeltaTransaction<'a, 'b> { - let delta = build_version_delta_after_version(self.latest_version()); + let delta = self.latest_version().version_delta_after(); SingleDeltaTransaction { version_txn: self, delta: Some(delta), diff --git a/src/meta/src/hummock/manager/versioning.rs b/src/meta/src/hummock/manager/versioning.rs index e04af41d30d4..790ac6b54fef 100644 --- a/src/meta/src/hummock/manager/versioning.rs +++ b/src/meta/src/hummock/manager/versioning.rs @@ -17,25 +17,20 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use itertools::Itertools; use risingwave_common::catalog::TableId; -use risingwave_common::util::epoch::INVALID_EPOCH; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ - build_initial_compaction_group_levels, get_compaction_group_ids, - get_table_compaction_group_id_mapping, BranchedSstInfo, + get_compaction_group_ids, get_table_compaction_group_id_mapping, BranchedSstInfo, }; -use risingwave_hummock_sdk::compaction_group::{StateTableId, StaticCompactionGroupId}; +use risingwave_hummock_sdk::compaction_group::StateTableId; use risingwave_hummock_sdk::table_stats::add_prost_table_stats_map; -use risingwave_hummock_sdk::version::{ - HummockVersion, HummockVersionDelta, HummockVersionStateTableInfo, -}; +use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta}; use risingwave_hummock_sdk::{ CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId, - FIRST_VERSION_ID, }; use risingwave_pb::common::WorkerNode; use risingwave_pb::hummock::write_limits::WriteLimit; use risingwave_pb::hummock::{ - CompactionConfig, HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot, - HummockVersionStats, SstableInfo, TableStats, + HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot, HummockVersionStats, SstableInfo, + TableStats, }; use risingwave_pb::meta::subscribe_response::{Info, Operation}; @@ -349,28 +344,6 @@ pub(super) fn calc_new_write_limits( new_write_limits } -pub(super) fn create_init_version(default_compaction_config: CompactionConfig) -> HummockVersion { - let mut init_version = HummockVersion { - id: FIRST_VERSION_ID, - levels: Default::default(), - max_committed_epoch: INVALID_EPOCH, - safe_epoch: INVALID_EPOCH, - table_watermarks: HashMap::new(), - table_change_log: HashMap::new(), - state_table_info: HummockVersionStateTableInfo::empty(), - }; - for group_id in [ - StaticCompactionGroupId::StateDefault as CompactionGroupId, - StaticCompactionGroupId::MaterializedView as CompactionGroupId, - ] { - init_version.levels.insert( - group_id, - build_initial_compaction_group_levels(group_id, &default_compaction_config), - ); - } - init_version -} - /// Rebuilds table stats from the given version. /// Note that the result is approximate value. See `estimate_table_stats`. fn rebuild_table_stats(version: &HummockVersion) -> HummockVersionStats { @@ -575,10 +548,9 @@ mod tests { ); } - let mut version = HummockVersion { - id: 123, - ..Default::default() - }; + let mut version = HummockVersion::default(); + version.id = 123; + for cg in 1..3 { version.levels.insert( cg, diff --git a/src/risedevtool/src/bin/risedev-compose.rs b/src/risedevtool/src/bin/risedev-compose.rs index 5ff56916deca..89bb0592d0d8 100644 --- a/src/risedevtool/src/bin/risedev-compose.rs +++ b/src/risedevtool/src/bin/risedev-compose.rs @@ -245,7 +245,6 @@ fn main() -> Result<()> { } }); let compose_file = ComposeFile { - version: "3".into(), services: services.clone(), volumes: node_volumes, name: format!("risingwave-{}", opts.profile), @@ -303,7 +302,6 @@ fn main() -> Result<()> { } } let compose_file = ComposeFile { - version: "3".into(), services, volumes, name: format!("risingwave-{}", opts.profile), diff --git a/src/risedevtool/src/compose.rs b/src/risedevtool/src/compose.rs index 779ca2355762..a49056015752 100644 --- a/src/risedevtool/src/compose.rs +++ b/src/risedevtool/src/compose.rs @@ -56,7 +56,6 @@ pub struct HealthCheck { #[derive(Debug, Clone, Serialize)] pub struct ComposeFile { - pub version: String, pub services: BTreeMap, pub volumes: BTreeMap, pub name: String, diff --git a/src/storage/backup/src/lib.rs b/src/storage/backup/src/lib.rs index a1acfde20e40..ed8dccf8d1e4 100644 --- a/src/storage/backup/src/lib.rs +++ b/src/storage/backup/src/lib.rs @@ -36,6 +36,7 @@ use std::collections::HashSet; use std::hash::Hasher; use itertools::Itertools; +use risingwave_common::RW_VERSION; use risingwave_hummock_sdk::version::HummockVersion; use risingwave_hummock_sdk::{HummockSstableObjectId, HummockVersionId}; use risingwave_pb::backup_service::{PbMetaSnapshotManifest, PbMetaSnapshotMetadata}; @@ -57,6 +58,7 @@ pub struct MetaSnapshotMetadata { #[serde(default)] pub format_version: u32, pub remarks: Option, + pub rw_version: Option, } impl MetaSnapshotMetadata { @@ -71,9 +73,10 @@ impl MetaSnapshotMetadata { hummock_version_id: v.id, ssts: v.get_object_ids(), max_committed_epoch: v.max_committed_epoch, - safe_epoch: v.safe_epoch, + safe_epoch: v.visible_table_safe_epoch(), format_version, remarks, + rw_version: Some(RW_VERSION.to_owned()), } } } @@ -112,6 +115,7 @@ impl From<&MetaSnapshotMetadata> for PbMetaSnapshotMetadata { safe_epoch: m.safe_epoch, format_version: Some(m.format_version), remarks: m.remarks.clone(), + rw_version: m.rw_version.clone(), } } } diff --git a/src/storage/backup/src/meta_snapshot_v2.rs b/src/storage/backup/src/meta_snapshot_v2.rs index feb5f12540d0..bec07a80cf19 100644 --- a/src/storage/backup/src/meta_snapshot_v2.rs +++ b/src/storage/backup/src/meta_snapshot_v2.rs @@ -16,7 +16,6 @@ use std::fmt::{Display, Formatter}; use bytes::{Buf, BufMut}; use risingwave_hummock_sdk::version::HummockVersion; -use risingwave_meta_model_v2 as model_v2; use serde::{Deserialize, Serialize}; use crate::meta_snapshot::{MetaSnapshot, Metadata}; @@ -29,39 +28,100 @@ impl From for BackupError { } } -#[derive(Default)] -pub struct MetadataV2 { - pub seaql_migrations: Vec, - pub hummock_version: HummockVersion, - pub version_stats: Vec, - pub compaction_configs: Vec, - pub actors: Vec, - pub clusters: Vec, - pub actor_dispatchers: Vec, - pub catalog_versions: Vec, - pub connections: Vec, - pub databases: Vec, - pub fragments: Vec, - pub functions: Vec, - pub indexes: Vec, - pub objects: Vec, - pub object_dependencies: Vec, - pub schemas: Vec, - pub sinks: Vec, - pub sources: Vec, - pub streaming_jobs: Vec, - pub subscriptions: Vec, - pub system_parameters: Vec, - pub tables: Vec, - pub users: Vec, - pub user_privileges: Vec, - pub views: Vec, - pub workers: Vec, - pub worker_properties: Vec, - pub hummock_sequences: Vec, - pub session_parameters: Vec, +/// Add new item in the end. Do not change the order. +#[macro_export] +macro_rules! for_all_metadata_models_v2 { + ($macro:ident) => { + $macro! { + {seaql_migrations, risingwave_meta_model_v2::serde_seaql_migration}, + {version_stats, risingwave_meta_model_v2::hummock_version_stats}, + {compaction_configs, risingwave_meta_model_v2::compaction_config}, + {actors, risingwave_meta_model_v2::actor}, + {clusters, risingwave_meta_model_v2::cluster}, + {actor_dispatchers, risingwave_meta_model_v2::actor_dispatcher}, + {catalog_versions, risingwave_meta_model_v2::catalog_version}, + {connections, risingwave_meta_model_v2::connection}, + {databases, risingwave_meta_model_v2::database}, + {fragments, risingwave_meta_model_v2::fragment}, + {functions, risingwave_meta_model_v2::function}, + {indexes, risingwave_meta_model_v2::index}, + {objects, risingwave_meta_model_v2::object}, + {object_dependencies, risingwave_meta_model_v2::object_dependency}, + {schemas, risingwave_meta_model_v2::schema}, + {sinks, risingwave_meta_model_v2::sink}, + {sources, risingwave_meta_model_v2::source}, + {streaming_jobs, risingwave_meta_model_v2::streaming_job}, + {subscriptions, risingwave_meta_model_v2::subscription}, + {system_parameters, risingwave_meta_model_v2::system_parameter}, + {tables, risingwave_meta_model_v2::table}, + {users, risingwave_meta_model_v2::user}, + {user_privileges, risingwave_meta_model_v2::user_privilege}, + {views, risingwave_meta_model_v2::view}, + {workers, risingwave_meta_model_v2::worker}, + {worker_properties, risingwave_meta_model_v2::worker_property}, + {hummock_sequences, risingwave_meta_model_v2::hummock_sequence}, + {session_parameters, risingwave_meta_model_v2::session_parameter}, + {secrets, risingwave_meta_model_v2::secret} + } + }; } +macro_rules! define_metadata_v2 { + ($({ $name:ident, $mod_path:ident::$mod_name:ident }),*) => { + #[derive(Default)] + pub struct MetadataV2 { + pub hummock_version: HummockVersion, + $( + pub $name: Vec<$mod_path::$mod_name::Model>, + )* + } + }; +} + +for_all_metadata_models_v2!(define_metadata_v2); + +macro_rules! define_encode_metadata { + ($( {$name:ident, $mod_path:ident::$mod_name:ident} ),*) => { + fn encode_metadata( + metadata: &MetadataV2, + buf: &mut Vec, + ) -> BackupResult<()> { + let mut _idx = 0; + $( + if _idx == 1 { + put_1(buf, &metadata.hummock_version.to_protobuf())?; + } + put_n(buf, &metadata.$name)?; + _idx += 1; + )* + Ok(()) + } + }; +} + +for_all_metadata_models_v2!(define_encode_metadata); + +macro_rules! define_decode_metadata { + ($( {$name:ident, $mod_path:ident::$mod_name:ident} ),*) => { + fn decode_metadata( + metadata: &mut MetadataV2, + mut buf: &[u8], + ) -> BackupResult<()> { + let mut _idx = 0; + $( + if _idx == 1 { + metadata.hummock_version = HummockVersion::from_persisted_protobuf(&get_1(&mut buf)?); + } + metadata.$name = get_n(&mut buf)?; + _idx += 1; + )* + Ok(()) + } + }; +} + +for_all_metadata_models_v2!(define_decode_metadata); + impl Display for MetadataV2 { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { writeln!(f, "clusters: {:#?}", self.clusters)?; @@ -77,102 +137,16 @@ impl Display for MetadataV2 { impl Metadata for MetadataV2 { fn encode_to(&self, buf: &mut Vec) -> BackupResult<()> { - put_n(buf, &self.seaql_migrations)?; - put_1(buf, &self.hummock_version.to_protobuf())?; - put_n(buf, &self.version_stats)?; - put_n(buf, &self.compaction_configs)?; - put_n(buf, &self.actors)?; - put_n(buf, &self.clusters)?; - put_n(buf, &self.actor_dispatchers)?; - put_n(buf, &self.catalog_versions)?; - put_n(buf, &self.connections)?; - put_n(buf, &self.databases)?; - put_n(buf, &self.fragments)?; - put_n(buf, &self.functions)?; - put_n(buf, &self.indexes)?; - put_n(buf, &self.objects)?; - put_n(buf, &self.object_dependencies)?; - put_n(buf, &self.schemas)?; - put_n(buf, &self.sinks)?; - put_n(buf, &self.sources)?; - put_n(buf, &self.streaming_jobs)?; - put_n(buf, &self.subscriptions)?; - put_n(buf, &self.system_parameters)?; - put_n(buf, &self.tables)?; - put_n(buf, &self.users)?; - put_n(buf, &self.user_privileges)?; - put_n(buf, &self.views)?; - put_n(buf, &self.workers)?; - put_n(buf, &self.worker_properties)?; - put_n(buf, &self.hummock_sequences)?; - put_n(buf, &self.session_parameters)?; - Ok(()) + encode_metadata(self, buf) } - fn decode(mut buf: &[u8]) -> BackupResult + fn decode(buf: &[u8]) -> BackupResult where Self: Sized, { - let seaql_migrations = get_n(&mut buf)?; - let pb_hummock_version = get_1(&mut buf)?; - let version_stats = get_n(&mut buf)?; - let compaction_configs = get_n(&mut buf)?; - let actors = get_n(&mut buf)?; - let clusters = get_n(&mut buf)?; - let actor_dispatchers = get_n(&mut buf)?; - let catalog_versions = get_n(&mut buf)?; - let connections = get_n(&mut buf)?; - let databases = get_n(&mut buf)?; - let fragments = get_n(&mut buf)?; - let functions = get_n(&mut buf)?; - let indexes = get_n(&mut buf)?; - let objects = get_n(&mut buf)?; - let object_dependencies = get_n(&mut buf)?; - let schemas = get_n(&mut buf)?; - let sinks = get_n(&mut buf)?; - let sources = get_n(&mut buf)?; - let streaming_jobs = get_n(&mut buf)?; - let subscriptions = get_n(&mut buf)?; - let system_parameters = get_n(&mut buf)?; - let tables = get_n(&mut buf)?; - let users = get_n(&mut buf)?; - let user_privileges = get_n(&mut buf)?; - let views = get_n(&mut buf)?; - let workers = get_n(&mut buf)?; - let worker_properties = get_n(&mut buf)?; - let hummock_sequences = get_n(&mut buf)?; - let session_parameters = get_n(&mut buf)?; - Ok(Self { - seaql_migrations, - hummock_version: HummockVersion::from_persisted_protobuf(&pb_hummock_version), - version_stats, - compaction_configs, - actors, - clusters, - actor_dispatchers, - catalog_versions, - connections, - databases, - fragments, - functions, - indexes, - objects, - object_dependencies, - schemas, - sinks, - sources, - streaming_jobs, - subscriptions, - system_parameters, - tables, - users, - user_privileges, - views, - workers, - worker_properties, - hummock_sequences, - session_parameters, - }) + let mut metadata = Self::default(); + decode_metadata(&mut metadata, buf)?; + Ok(metadata) } fn hummock_version_ref(&self) -> &HummockVersion { diff --git a/src/storage/benches/bench_table_watermarks.rs b/src/storage/benches/bench_table_watermarks.rs index 11ec3c4bdcb5..ce8980598f77 100644 --- a/src/storage/benches/bench_table_watermarks.rs +++ b/src/storage/benches/bench_table_watermarks.rs @@ -15,7 +15,7 @@ #![feature(lazy_cell)] use std::collections::hash_map::Entry; -use std::collections::{HashMap, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, LazyLock}; use bytes::Bytes; @@ -28,8 +28,9 @@ use risingwave_common::util::epoch::test_epoch; use risingwave_hummock_sdk::table_watermark::{ TableWatermarks, TableWatermarksIndex, VnodeWatermark, WatermarkDirection, }; -use risingwave_hummock_sdk::version::HummockVersion; +use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionStateTableInfo}; use risingwave_hummock_sdk::HummockEpoch; +use risingwave_pb::hummock::StateTableInfoDelta; use risingwave_storage::hummock::local_version::pinned_version::PinnedVersion; use spin::Mutex; use tokio::sync::mpsc::unbounded_channel; @@ -115,17 +116,30 @@ fn gen_version( new_epoch_idx, vnode_part_count, )); - // let table_watermarks = - // gen_committed_table_watermarks(old_epoch_idx, new_epoch_idx, vnode_part_count); - HummockVersion { - id: new_epoch_idx as _, - max_committed_epoch: test_epoch(new_epoch_idx as _), - safe_epoch: test_epoch(old_epoch_idx as _), - table_watermarks: (0..table_count) - .map(|table_id| (TableId::new(table_id as _), table_watermarks.clone())) + let mut version = HummockVersion::default(); + let committed_epoch = test_epoch(new_epoch_idx as _); + version.id = new_epoch_idx as _; + version.max_committed_epoch = committed_epoch; + version.table_watermarks = (0..table_count) + .map(|table_id| (TableId::new(table_id as _), table_watermarks.clone())) + .collect(); + let mut state_table_info = HummockVersionStateTableInfo::empty(); + state_table_info.apply_delta( + &(0..table_count) + .map(|table_id| { + ( + TableId::new(table_id as _), + StateTableInfoDelta { + committed_epoch, + safe_epoch: test_epoch(old_epoch_idx as _), + }, + ) + }) .collect(), - ..Default::default() - } + &HashSet::new(), + ); + version.state_table_info = state_table_info; + version } fn bench_table_watermarks(c: &mut Criterion) { diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 52f3c1cb15ca..2231878dc9ef 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -246,8 +246,15 @@ impl HummockVersion { if !existing_table_ids.contains(&u32_table_id) { None } else { - extract_single_table_watermark(table_watermarks, self.safe_epoch) - .map(|table_watermarks| (table_id.table_id, table_watermarks)) + extract_single_table_watermark( + table_watermarks, + self.state_table_info + .info() + .get(table_id) + .expect("table should exist") + .safe_epoch, + ) + .map(|table_watermarks| (table_id.table_id, table_watermarks)) } }) .collect() @@ -584,7 +591,7 @@ impl HummockVersion { } self.id = version_delta.id; self.max_committed_epoch = version_delta.max_committed_epoch; - self.safe_epoch = version_delta.safe_epoch; + self.set_safe_epoch(version_delta.safe_epoch); // apply to table watermark @@ -1116,21 +1123,6 @@ pub fn insert_new_sub_level( l0.sub_levels.insert(insert_pos, level); } -pub fn build_version_delta_after_version(version: &HummockVersion) -> HummockVersionDelta { - HummockVersionDelta { - id: version.next_version_id(), - prev_id: version.id, - safe_epoch: version.safe_epoch, - trivial_move: false, - max_committed_epoch: version.max_committed_epoch, - group_deltas: Default::default(), - new_table_watermarks: HashMap::new(), - removed_table_ids: HashSet::new(), - change_log_delta: HashMap::new(), - state_table_info_delta: Default::default(), - } -} - /// Delete sstables if the table id is in the id set. /// /// Return `true` if some sst is deleted, and `false` is the deletion is trivial @@ -1209,10 +1201,11 @@ pub fn validate_version(version: &HummockVersion) -> Vec { let mut res = Vec::new(); // Ensure safe_epoch <= max_committed_epoch - if version.safe_epoch > version.max_committed_epoch { + if version.visible_table_safe_epoch() > version.max_committed_epoch { res.push(format!( "VERSION: safe_epoch {} > max_committed_epoch {}", - version.safe_epoch, version.max_committed_epoch + version.visible_table_safe_epoch(), + version.max_committed_epoch )); } @@ -1353,22 +1346,20 @@ mod tests { #[test] fn test_get_sst_object_ids() { - let mut version = HummockVersion { - id: 0, - levels: HashMap::from_iter([( - 0, - Levels { - levels: vec![], - l0: Some(OverlappingLevel { - sub_levels: vec![], - total_file_size: 0, - uncompressed_file_size: 0, - }), - ..Default::default() - }, - )]), - ..Default::default() - }; + let mut version = HummockVersion::default(); + version.id = 0; + version.levels = HashMap::from_iter([( + 0, + Levels { + levels: vec![], + l0: Some(OverlappingLevel { + sub_levels: vec![], + total_file_size: 0, + uncompressed_file_size: 0, + }), + ..Default::default() + }, + )]); assert_eq!(version.get_object_ids().len(), 0); // Add to sub level @@ -1404,32 +1395,30 @@ mod tests { #[test] fn test_apply_version_delta() { - let mut version = HummockVersion { - id: 0, - levels: HashMap::from_iter([ - ( + let mut version = HummockVersion::default(); + version.id = 0; + version.levels = HashMap::from_iter([ + ( + 0, + build_initial_compaction_group_levels( 0, - build_initial_compaction_group_levels( - 0, - &CompactionConfig { - max_level: 6, - ..Default::default() - }, - ), + &CompactionConfig { + max_level: 6, + ..Default::default() + }, ), - ( + ), + ( + 1, + build_initial_compaction_group_levels( 1, - build_initial_compaction_group_levels( - 1, - &CompactionConfig { - max_level: 6, - ..Default::default() - }, - ), + &CompactionConfig { + max_level: 6, + ..Default::default() + }, ), - ]), - ..Default::default() - }; + ), + ]); let version_delta = HummockVersionDelta { id: 1, group_deltas: HashMap::from_iter([ @@ -1492,25 +1481,23 @@ mod tests { }], ..Default::default() }; - assert_eq!( - version, - HummockVersion { - id: 1, - levels: HashMap::from_iter([ - ( + assert_eq!(version, { + let mut version = HummockVersion::default(); + version.id = 1; + version.levels = HashMap::from_iter([ + ( + 2, + build_initial_compaction_group_levels( 2, - build_initial_compaction_group_levels( - 2, - &CompactionConfig { - max_level: 6, - ..Default::default() - } - ), + &CompactionConfig { + max_level: 6, + ..Default::default() + }, ), - (1, cg1,), - ]), - ..Default::default() - } - ); + ), + (1, cg1), + ]); + version + }); } } diff --git a/src/storage/hummock_sdk/src/table_watermark.rs b/src/storage/hummock_sdk/src/table_watermark.rs index 24d28dec9ba1..73f7bac358e1 100644 --- a/src/storage/hummock_sdk/src/table_watermark.rs +++ b/src/storage/hummock_sdk/src/table_watermark.rs @@ -1031,11 +1031,8 @@ mod tests { watermark3.clone(), ); - let mut version = HummockVersion { - max_committed_epoch: EPOCH1, - safe_epoch: EPOCH1, - ..Default::default() - }; + let mut version = HummockVersion::default(); + version.max_committed_epoch = EPOCH1; let test_table_id = TableId::from(233); version.table_watermarks.insert( test_table_id, diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs index 130c5f6f523d..51780b1bc733 100644 --- a/src/storage/hummock_sdk/src/version.rs +++ b/src/storage/hummock_sdk/src/version.rs @@ -19,18 +19,22 @@ use std::sync::Arc; use prost::Message; use risingwave_common::catalog::TableId; +use risingwave_common::util::epoch::INVALID_EPOCH; use risingwave_pb::hummock::group_delta::DeltaType; use risingwave_pb::hummock::hummock_version::Levels as PbLevels; use risingwave_pb::hummock::hummock_version_delta::{ChangeLogDelta, GroupDeltas as PbGroupDeltas}; use risingwave_pb::hummock::{ - HummockVersion as PbHummockVersion, HummockVersionDelta as PbHummockVersionDelta, SstableInfo, - StateTableInfo as PbStateTableInfo, StateTableInfo, StateTableInfoDelta, + CompactionConfig, HummockVersion as PbHummockVersion, + HummockVersionDelta as PbHummockVersionDelta, SstableInfo, StateTableInfo as PbStateTableInfo, + StateTableInfo, StateTableInfoDelta, }; use tracing::warn; use crate::change_log::TableChangeLog; +use crate::compaction_group::hummock_version_ext::build_initial_compaction_group_levels; +use crate::compaction_group::StaticCompactionGroupId; use crate::table_watermark::TableWatermarks; -use crate::{CompactionGroupId, HummockSstableObjectId, HummockVersionId}; +use crate::{CompactionGroupId, HummockSstableObjectId, HummockVersionId, FIRST_VERSION_ID}; #[derive(Debug, Clone, PartialEq)] pub struct HummockVersionStateTableInfo { @@ -85,7 +89,16 @@ impl HummockVersionStateTableInfo { }; match self.state_table_info.entry(*table_id) { Entry::Occupied(mut entry) => { - let prev_info = replace(entry.get_mut(), new_info); + let prev_info = entry.get_mut(); + assert!( + new_info.safe_epoch >= prev_info.safe_epoch + && new_info.committed_epoch >= prev_info.committed_epoch, + "state table info regress. table id: {}, prev_info: {:?}, new_info: {:?}", + table_id.table_id, + prev_info, + new_info + ); + let prev_info = replace(prev_info, new_info); changed_table.insert(*table_id, Some(prev_info)); } Entry::Vacant(entry) => { @@ -107,7 +120,7 @@ pub struct HummockVersion { pub id: u64, pub levels: HashMap, pub max_committed_epoch: u64, - pub safe_epoch: u64, + safe_epoch: u64, pub table_watermarks: HashMap>, pub table_change_log: HashMap, pub state_table_info: HummockVersionStateTableInfo, @@ -244,6 +257,51 @@ impl HummockVersion { } } } + + pub(crate) fn set_safe_epoch(&mut self, safe_epoch: u64) { + self.safe_epoch = safe_epoch; + } + + pub fn visible_table_safe_epoch(&self) -> u64 { + self.safe_epoch + } + + pub fn create_init_version(default_compaction_config: CompactionConfig) -> HummockVersion { + let mut init_version = HummockVersion { + id: FIRST_VERSION_ID, + levels: Default::default(), + max_committed_epoch: INVALID_EPOCH, + safe_epoch: INVALID_EPOCH, + table_watermarks: HashMap::new(), + table_change_log: HashMap::new(), + state_table_info: HummockVersionStateTableInfo::empty(), + }; + for group_id in [ + StaticCompactionGroupId::StateDefault as CompactionGroupId, + StaticCompactionGroupId::MaterializedView as CompactionGroupId, + ] { + init_version.levels.insert( + group_id, + build_initial_compaction_group_levels(group_id, &default_compaction_config), + ); + } + init_version + } + + pub fn version_delta_after(&self) -> HummockVersionDelta { + HummockVersionDelta { + id: self.next_version_id(), + prev_id: self.id, + safe_epoch: self.safe_epoch, + trivial_move: false, + max_committed_epoch: self.max_committed_epoch, + group_deltas: Default::default(), + new_table_watermarks: HashMap::new(), + removed_table_ids: HashSet::new(), + change_log_delta: HashMap::new(), + state_table_info_delta: Default::default(), + } + } } #[derive(Debug, PartialEq, Clone)] diff --git a/src/storage/hummock_test/src/hummock_storage_tests.rs b/src/storage/hummock_test/src/hummock_storage_tests.rs index 6962fc506ccf..5c6e1607de92 100644 --- a/src/storage/hummock_test/src/hummock_storage_tests.rs +++ b/src/storage/hummock_test/src/hummock_storage_tests.rs @@ -530,6 +530,9 @@ async fn test_state_store_sync() { .await .unwrap(); + let epoch3 = epoch2.next_epoch(); + hummock_storage.seal_current_epoch(epoch3, SealCurrentEpochOptions::for_test()); + let res = test_env.storage.seal_and_sync_epoch(epoch1).await.unwrap(); test_env .meta_client @@ -829,14 +832,15 @@ async fn test_delete_get() { .await .unwrap(); + let epoch2 = epoch1.next_epoch(); + hummock_storage.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test()); let res = test_env.storage.seal_and_sync_epoch(epoch1).await.unwrap(); test_env .meta_client .commit_epoch(epoch1, res) .await .unwrap(); - let epoch2 = epoch1.next_epoch(); - hummock_storage.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test()); + let batch2 = vec![( gen_key_from_str(VirtualNode::ZERO, "bb"), StorageValue::new_delete(), @@ -851,6 +855,7 @@ async fn test_delete_get() { ) .await .unwrap(); + hummock_storage.seal_current_epoch(u64::MAX, SealCurrentEpochOptions::for_test()); let res = test_env.storage.seal_and_sync_epoch(epoch2).await.unwrap(); test_env .meta_client @@ -1005,6 +1010,8 @@ async fn test_multiple_epoch_sync() { }; test_get().await; + let epoch4 = epoch3.next_epoch(); + hummock_storage.seal_current_epoch(epoch4, SealCurrentEpochOptions::for_test()); test_env.storage.seal_epoch(epoch1, false); let sync_result2 = test_env.storage.seal_and_sync_epoch(epoch2).await.unwrap(); let sync_result3 = test_env.storage.seal_and_sync_epoch(epoch3).await.unwrap(); @@ -1079,6 +1086,9 @@ async fn test_iter_with_min_epoch() { .await .unwrap(); + let epoch3 = (33 * 1000) << 16; + hummock_storage.seal_current_epoch(epoch3, SealCurrentEpochOptions::for_test()); + { // test before sync { @@ -1329,6 +1339,9 @@ async fn test_hummock_version_reader() { .await .unwrap(); + let epoch4 = (34 * 1000) << 16; + hummock_storage.seal_current_epoch(epoch4, SealCurrentEpochOptions::for_test()); + { // test before sync { @@ -1739,6 +1752,8 @@ async fn test_get_with_min_epoch() { .await .unwrap(); + hummock_storage.seal_current_epoch(u64::MAX, SealCurrentEpochOptions::for_test()); + { // test before sync let k = gen_key(0); diff --git a/src/storage/hummock_test/src/state_store_tests.rs b/src/storage/hummock_test/src/state_store_tests.rs index 4e14e006f009..fe129deb5224 100644 --- a/src/storage/hummock_test/src/state_store_tests.rs +++ b/src/storage/hummock_test/src/state_store_tests.rs @@ -1046,11 +1046,12 @@ async fn test_delete_get_v2() { ) .await .unwrap(); - let res = hummock_storage.seal_and_sync_epoch(epoch1).await.unwrap(); - meta_client.commit_epoch(epoch1, res).await.unwrap(); let epoch2 = epoch1.next_epoch(); local.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test()); + let res = hummock_storage.seal_and_sync_epoch(epoch1).await.unwrap(); + meta_client.commit_epoch(epoch1, res).await.unwrap(); + let batch2 = vec![( gen_key_from_str(VirtualNode::ZERO, "bb"), StorageValue::new_delete(), diff --git a/src/storage/src/hummock/error.rs b/src/storage/src/hummock/error.rs index 3019e65fc4e3..48f71b919933 100644 --- a/src/storage/src/hummock/error.rs +++ b/src/storage/src/hummock/error.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use risingwave_common::catalog::TableId; use risingwave_object_store::object::ObjectError; use thiserror::Error; use thiserror_ext::AsReport; @@ -48,7 +49,11 @@ pub enum HummockErrorInner { #[error("Barrier read is unavailable for now. Likely the cluster is recovering")] ReadCurrentEpoch, #[error("Expired Epoch: watermark {safe_epoch}, epoch {epoch}")] - ExpiredEpoch { safe_epoch: u64, epoch: u64 }, + ExpiredEpoch { + table_id: u32, + safe_epoch: u64, + epoch: u64, + }, #[error("CompactionExecutor error: {0}")] CompactionExecutor(String), #[error("FileCache error: {0}")] @@ -108,8 +113,13 @@ impl HummockError { HummockErrorInner::ReadCurrentEpoch.into() } - pub fn expired_epoch(safe_epoch: u64, epoch: u64) -> HummockError { - HummockErrorInner::ExpiredEpoch { safe_epoch, epoch }.into() + pub fn expired_epoch(table_id: TableId, safe_epoch: u64, epoch: u64) -> HummockError { + HummockErrorInner::ExpiredEpoch { + table_id: table_id.table_id, + safe_epoch, + epoch, + } + .into() } pub fn is_expired_epoch(&self) -> bool { diff --git a/src/storage/src/hummock/event_handler/hummock_event_handler.rs b/src/storage/src/hummock/event_handler/hummock_event_handler.rs index c01a97563237..277984d3545d 100644 --- a/src/storage/src/hummock/event_handler/hummock_event_handler.rs +++ b/src/storage/src/hummock/event_handler/hummock_event_handler.rs @@ -196,7 +196,7 @@ pub struct HummockEventHandler { version_update_rx: UnboundedReceiver, read_version_mapping: Arc>, /// A copy of `read_version_mapping` but owned by event handler - local_read_version_mapping: HashMap, + local_read_version_mapping: HashMap, version_update_notifier_tx: Arc>, pinned_version: Arc>, @@ -455,7 +455,7 @@ impl HummockEventHandler { let mut pending = VecDeque::new(); let mut total_count = 0; for instance_id in instances { - let Some(read_version) = self.local_read_version_mapping.get(&instance_id) else { + let Some((_, read_version)) = self.local_read_version_mapping.get(&instance_id) else { continue; }; total_count += 1; @@ -475,7 +475,7 @@ impl HummockEventHandler { const TRY_LOCK_TIMEOUT: Duration = Duration::from_millis(1); while let Some(instance_id) = pending.pop_front() { - let read_version = self + let (_, read_version) = self .local_read_version_mapping .get(&instance_id) .expect("have checked exist before"); @@ -520,7 +520,6 @@ impl HummockEventHandler { prev_epoch, max_committed_epoch = self.uploader.max_committed_epoch(), max_synced_epoch = self.uploader.max_synced_epoch(), - max_sealed_epoch = self.uploader.max_sealed_epoch(), "handle clear event" ); @@ -588,7 +587,7 @@ impl HummockEventHandler { "read version mapping not empty when clear. remaining tables: {:?}", self.local_read_version_mapping .values() - .map(|read_version| read_version.read().table_id()) + .map(|(_, read_version)| read_version.read().table_id()) .collect_vec() ); @@ -784,6 +783,18 @@ impl HummockEventHandler { HummockEvent::Shutdown => { unreachable!("shutdown is handled specially") } + HummockEvent::InitEpoch { + instance_id, + init_epoch, + } => { + let table_id = self + .local_read_version_mapping + .get(&instance_id) + .expect("should exist") + .0; + self.uploader + .init_instance(instance_id, table_id, init_epoch); + } HummockEvent::ImmToUploader { instance_id, imm } => { assert!( self.local_read_version_mapping.contains_key(&instance_id), @@ -795,29 +806,13 @@ impl HummockEventHandler { self.uploader.may_flush(); } - HummockEvent::SealEpoch { - epoch, - is_checkpoint: _, - } => { - self.uploader.seal_epoch(epoch); - } - HummockEvent::LocalSealEpoch { - epoch, + next_epoch, opts, - table_id, instance_id, } => { - assert!( - self.local_read_version_mapping - .contains_key(&instance_id), - "seal epoch from non-existing read version instance: instance_id: {}, table_id: {}, epoch: {}", - instance_id, table_id, epoch, - ); - if let Some((direction, watermarks)) = opts.table_watermarks { - self.uploader - .add_table_watermarks(epoch, table_id, watermarks, direction) - } + self.uploader + .local_seal_epoch(instance_id, next_epoch, opts); } #[cfg(any(test, feature = "test"))] @@ -852,7 +847,7 @@ impl HummockEventHandler { { self.local_read_version_mapping - .insert(instance_id, basic_read_version.clone()); + .insert(instance_id, (table_id, basic_read_version.clone())); let mut read_version_mapping_guard = self.read_version_mapping.write(); read_version_mapping_guard @@ -876,33 +871,29 @@ impl HummockEventHandler { table_id, instance_id ); guard.event_sender.take().expect("sender is just set"); - self.destroy_read_version(table_id, instance_id); + self.destroy_read_version(instance_id); } } } - HummockEvent::DestroyReadVersion { - table_id, - instance_id, - } => { - self.destroy_read_version(table_id, instance_id); + HummockEvent::DestroyReadVersion { instance_id } => { + self.uploader.may_destroy_instance(instance_id); + self.destroy_read_version(instance_id); } } } - fn destroy_read_version(&mut self, table_id: TableId, instance_id: LocalInstanceId) { + fn destroy_read_version(&mut self, instance_id: LocalInstanceId) { { { - debug!( - "read version deregister: table_id: {}, instance_id: {}", - table_id, instance_id - ); - self.local_read_version_mapping + debug!("read version deregister: instance_id: {}", instance_id); + let (table_id, _) = self + .local_read_version_mapping .remove(&instance_id) .unwrap_or_else(|| { panic!( - "DestroyHummockInstance inexist instance table_id {} instance_id {}", - table_id, instance_id + "DestroyHummockInstance inexist instance instance_id {}", + instance_id ) }); let mut read_version_mapping_guard = self.read_version_mapping.write(); @@ -994,6 +985,7 @@ mod tests { use crate::hummock::test_utils::default_opts_for_test; use crate::hummock::HummockError; use crate::monitor::HummockStateStoreMetrics; + use crate::store::SealCurrentEpochOptions; #[tokio::test] async fn test_clear_shared_buffer() { @@ -1197,6 +1189,11 @@ mod tests { rx.await.unwrap() }; + send_event(HummockEvent::InitEpoch { + instance_id: guard.instance_id, + init_epoch: epoch1, + }); + let imm1 = gen_imm(epoch1).await; read_version .write() @@ -1207,6 +1204,12 @@ mod tests { imm: imm1, }); + send_event(HummockEvent::LocalSealEpoch { + instance_id: guard.instance_id, + next_epoch: epoch2, + opts: SealCurrentEpochOptions::for_test(), + }); + let imm2 = gen_imm(epoch2).await; read_version .write() @@ -1217,20 +1220,19 @@ mod tests { imm: imm2, }); - send_event(HummockEvent::SealEpoch { - epoch: epoch1, - is_checkpoint: true, + let epoch3 = epoch2.next_epoch(); + send_event(HummockEvent::LocalSealEpoch { + instance_id: guard.instance_id, + next_epoch: epoch3, + opts: SealCurrentEpochOptions::for_test(), }); + let (tx1, mut rx1) = oneshot::channel(); send_event(HummockEvent::SyncEpoch { new_sync_epoch: epoch1, sync_result_sender: tx1, }); assert!(poll_fn(|cx| Poll::Ready(rx1.poll_unpin(cx).is_pending())).await); - send_event(HummockEvent::SealEpoch { - epoch: epoch2, - is_checkpoint: true, - }); let (tx2, mut rx2) = oneshot::channel(); send_event(HummockEvent::SyncEpoch { new_sync_epoch: epoch2, diff --git a/src/storage/src/hummock/event_handler/mod.rs b/src/storage/src/hummock/event_handler/mod.rs index c28dd6d25c3a..bbf69ae194f7 100644 --- a/src/storage/src/hummock/event_handler/mod.rs +++ b/src/storage/src/hummock/event_handler/mod.rs @@ -72,15 +72,14 @@ pub enum HummockEvent { imm: ImmutableMemtable, }, - SealEpoch { - epoch: HummockEpoch, - is_checkpoint: bool, + InitEpoch { + instance_id: LocalInstanceId, + init_epoch: HummockEpoch, }, LocalSealEpoch { instance_id: LocalInstanceId, - table_id: TableId, - epoch: HummockEpoch, + next_epoch: HummockEpoch, opts: SealCurrentEpochOptions, }, @@ -97,7 +96,6 @@ pub enum HummockEvent { }, DestroyReadVersion { - table_id: TableId, instance_id: LocalInstanceId, }, } @@ -116,27 +114,25 @@ impl HummockEvent { HummockEvent::Shutdown => "Shutdown".to_string(), + HummockEvent::InitEpoch { + instance_id, + init_epoch, + } => { + format!("InitEpoch {} {}", instance_id, init_epoch) + } + HummockEvent::ImmToUploader { instance_id, imm } => { format!("ImmToUploader {} {}", instance_id, imm.batch_id()) } - HummockEvent::SealEpoch { - epoch, - is_checkpoint, - } => format!( - "SealEpoch epoch {:?} is_checkpoint {:?}", - epoch, is_checkpoint - ), - HummockEvent::LocalSealEpoch { - epoch, instance_id, - table_id, + next_epoch, opts, } => { format!( - "LocalSealEpoch epoch: {}, table_id: {}, instance_id: {}, opts: {:?}", - epoch, table_id.table_id, instance_id, opts + "LocalSealEpoch next_epoch: {}, instance_id: {}, opts: {:?}", + next_epoch, instance_id, opts ) } @@ -150,13 +146,9 @@ impl HummockEvent { table_id, is_replicated ), - HummockEvent::DestroyReadVersion { - table_id, - instance_id, - } => format!( - "DestroyReadVersion table_id {:?} instance_id {:?}", - table_id, instance_id - ), + HummockEvent::DestroyReadVersion { instance_id } => { + format!("DestroyReadVersion instance_id {:?}", instance_id) + } #[cfg(any(test, feature = "test"))] HummockEvent::FlushEvent(_) => "FlushEvent".to_string(), @@ -210,7 +202,6 @@ impl Drop for LocalInstanceGuard { // need to handle failure sender .send(HummockEvent::DestroyReadVersion { - table_id: self.table_id, instance_id: self.instance_id, }) .unwrap_or_else(|err| { diff --git a/src/storage/src/hummock/event_handler/uploader.rs b/src/storage/src/hummock/event_handler/uploader.rs index 9f40e9ac2896..f768aa23dcd8 100644 --- a/src/storage/src/hummock/event_handler/uploader.rs +++ b/src/storage/src/hummock/event_handler/uploader.rs @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; +use std::cmp::Ordering; +use std::collections::btree_map::Entry; use std::collections::{BTreeMap, HashMap, VecDeque}; use std::fmt::{Debug, Display, Formatter}; use std::future::{poll_fn, Future}; -use std::mem::{replace, take}; +use std::mem::{replace, swap, take}; use std::pin::Pin; use std::sync::Arc; use std::task::{ready, Context, Poll}; @@ -38,7 +39,7 @@ use risingwave_hummock_sdk::{CompactionGroupId, HummockEpoch, LocalSstableInfo, use thiserror_ext::AsReport; use tokio::sync::oneshot; use tokio::task::JoinHandle; -use tracing::{debug, error, info}; +use tracing::{debug, error, info, warn}; use crate::hummock::event_handler::hummock_event_handler::{send_sync_result, BufferTracker}; use crate::hummock::event_handler::uploader::uploader_imm::UploaderImm; @@ -50,6 +51,17 @@ use crate::hummock::{HummockError, HummockResult, ImmutableMemtable}; use crate::mem_table::ImmId; use crate::monitor::HummockStateStoreMetrics; use crate::opts::StorageOpts; +use crate::store::SealCurrentEpochOptions; + +/// Take epoch data inclusively before `epoch` out from `data` +fn take_before_epoch( + data: &mut BTreeMap, + epoch: HummockEpoch, +) -> BTreeMap { + let mut before_epoch_data = data.split_off(&(epoch + 1)); + swap(&mut before_epoch_data, data); + before_epoch_data +} type UploadTaskInput = HashMap>; pub type UploadTaskPayload = HashMap>; @@ -329,11 +341,6 @@ struct SpilledData { } impl SpilledData { - #[cfg(test)] - fn is_empty(&self) -> bool { - self.uploading_tasks.is_empty() && self.uploaded_data.is_empty() - } - fn add_task(&mut self, task: UploadingTask) { self.uploading_tasks.push_front(task); } @@ -360,20 +367,16 @@ impl SpilledData { } #[derive(Default, Debug)] -struct UnsealedEpochData { - // newer data at the front - imms: HashMap>, +struct EpochData { spilled_data: SpilledData, - - table_watermarks: HashMap, BitmapBuilder)>, } -impl UnsealedEpochData { - fn flush(&mut self, context: &UploaderContext) -> usize { - let imms: HashMap<_, _> = take(&mut self.imms) - .into_iter() - .map(|(id, imms)| (id, imms.into_iter().collect_vec())) - .collect(); +impl EpochData { + fn flush( + &mut self, + context: &UploaderContext, + imms: HashMap>, + ) -> usize { if !imms.is_empty() { let task = UploadingTask::new(imms, context); context.stats.spill_task_counts_from_unsealed.inc(); @@ -389,10 +392,12 @@ impl UnsealedEpochData { 0 } } +} +impl TableUnsyncData { fn add_table_watermarks( &mut self, - table_id: TableId, + epoch: HummockEpoch, table_watermarks: Vec, direction: WatermarkDirection, ) { @@ -411,45 +416,50 @@ impl UnsealedEpochData { } } } - match self.table_watermarks.entry(table_id) { - Entry::Occupied(mut entry) => { - let (prev_direction, prev_watermarks, vnode_bitmap) = entry.get_mut(); + match &mut self.table_watermarks { + Some((prev_direction, prev_watermarks)) => { assert_eq!( *prev_direction, direction, "table id {} new watermark direction not match with previous", - table_id + self.table_id ); - apply_new_vnodes(vnode_bitmap, &table_watermarks); - prev_watermarks.extend(table_watermarks); + match prev_watermarks.entry(epoch) { + Entry::Occupied(mut entry) => { + let (prev_watermarks, vnode_bitmap) = entry.get_mut(); + apply_new_vnodes(vnode_bitmap, &table_watermarks); + prev_watermarks.extend(table_watermarks); + } + Entry::Vacant(entry) => { + let mut vnode_bitmap = BitmapBuilder::zeroed(VirtualNode::COUNT); + apply_new_vnodes(&mut vnode_bitmap, &table_watermarks); + entry.insert((table_watermarks, vnode_bitmap)); + } + } } - Entry::Vacant(entry) => { + None => { let mut vnode_bitmap = BitmapBuilder::zeroed(VirtualNode::COUNT); apply_new_vnodes(&mut vnode_bitmap, &table_watermarks); - entry.insert((direction, table_watermarks, vnode_bitmap)); + self.table_watermarks = Some(( + direction, + BTreeMap::from_iter([(epoch, (table_watermarks, vnode_bitmap))]), + )); } } } } #[derive(Default)] -/// Data at the sealed stage. We will ensure that data in `imms` are newer than the data in the -/// `spilled_data`, and that data in the `uploading_tasks` in `spilled_data` are newer than data in -/// the `uploaded_data` in `spilled_data`. -struct SealedData { +struct SyncDataBuilder { // newer epochs come first epochs: VecDeque, - // Sealed imms grouped by table shard. - // newer data (larger imm id) at the front - imms_by_table_shard: HashMap>, - spilled_data: SpilledData, table_watermarks: HashMap, } -impl SealedData { - /// Add the data of a newly sealed epoch. +impl SyncDataBuilder { + /// Add the data of a new epoch. /// /// Note: it may happen that, for example, currently we hold `imms` and `spilled_data` of epoch /// 3, and after we add the spilled data of epoch 4, both `imms` and `spilled_data` hold data @@ -459,9 +469,9 @@ impl SealedData { /// data of `imms` must not overlap with the epoch 4 data of `spilled_data`. The explanation is /// as followed: /// - /// First, unsealed data has 3 stages, from earlier to later, imms, uploading task, and - /// uploaded. When we try to spill unsealed data, we first pick the imms of older epoch until - /// the imms of older epoch are all picked. When we try to poll the uploading tasks of unsealed + /// First, unsync data has 3 stages, from earlier to later, imms, uploading task, and + /// uploaded. When we try to spill unsync data, we first pick the imms of older epoch until + /// the imms of older epoch are all picked. When we try to poll the uploading tasks of unsync /// data, we first poll the task of older epoch, until there is no uploading task in older /// epoch. Therefore, we can reach that, if two data are in the same stage, but /// different epochs, data in the older epoch will always enter the next stage earlier than data @@ -475,28 +485,19 @@ impl SealedData { /// Based on the two points above, we can reach that, if two data of a same key appear in /// different epochs, the data of older epoch will not appear at a later stage than the data /// of newer epoch. Therefore, we can safely merge the data of each stage when we seal an epoch. - fn seal_new_epoch(&mut self, epoch: HummockEpoch, mut unseal_epoch_data: UnsealedEpochData) { - if let Some(prev_max_sealed_epoch) = self.epochs.front() { + fn add_new_epoch(&mut self, epoch: HummockEpoch, mut unseal_epoch_data: EpochData) { + if let Some(prev_max_epoch) = self.epochs.front() { assert!( - epoch > *prev_max_sealed_epoch, - "epoch {} to seal not greater than prev max sealed epoch {}", + epoch > *prev_max_epoch, + "epoch {} to seal not greater than prev max epoch {}", epoch, - prev_max_sealed_epoch + prev_max_epoch ); } - // rearrange sealed imms by table shard and in epoch descending order - for (instance_id, imms) in unseal_epoch_data.imms { - let queue = self.imms_by_table_shard.entry(instance_id).or_default(); - for imm in imms.into_iter().rev() { - if let Some(front) = queue.front() { - assert_gt!(imm.batch_id(), front.batch_id()); - } - queue.push_front(imm); - } - } - self.epochs.push_front(epoch); + // for each local instance, earlier data must be spilled at earlier epoch. Therefore, since we add spill data from old epoch + // to new epoch, unseal_epoch_data .spilled_data .uploading_tasks @@ -507,64 +508,403 @@ impl SealedData { .append(&mut self.spilled_data.uploaded_data); self.spilled_data.uploading_tasks = unseal_epoch_data.spilled_data.uploading_tasks; self.spilled_data.uploaded_data = unseal_epoch_data.spilled_data.uploaded_data; - for (table_id, (direction, watermarks, _)) in unseal_epoch_data.table_watermarks { - match self.table_watermarks.entry(table_id) { - Entry::Occupied(mut entry) => { - entry.get_mut().add_new_epoch_watermarks( + } + + fn add_table_watermarks( + &mut self, + table_id: TableId, + direction: WatermarkDirection, + watermarks: impl Iterator)>, + ) { + let mut table_watermarks: Option = None; + for (epoch, watermarks) in watermarks { + match &mut table_watermarks { + Some(prev_watermarks) => { + prev_watermarks.add_new_epoch_watermarks( epoch, Arc::from(watermarks), direction, ); } - Entry::Vacant(entry) => { - entry.insert(TableWatermarks::single_epoch(epoch, watermarks, direction)); + None => { + table_watermarks = + Some(TableWatermarks::single_epoch(epoch, watermarks, direction)); } - }; + } + } + if let Some(table_watermarks) = table_watermarks { + assert!(self + .table_watermarks + .insert(table_id, table_watermarks) + .is_none()); } } - // Flush can be triggered by either a sync_epoch or a spill (`may_flush`) request. - fn flush(&mut self, context: &UploaderContext, is_spilled: bool) -> usize { - let payload: HashMap<_, Vec<_>> = take(&mut self.imms_by_table_shard) - .into_iter() - .map(|(id, imms)| (id, imms.into_iter().collect())) - .collect(); - + fn flush(&mut self, context: &UploaderContext, payload: UploadTaskInput) { if !payload.is_empty() { let task = UploadingTask::new(payload, context); - let size = task.task_info.task_size; - if is_spilled { - context.stats.spill_task_counts_from_sealed.inc(); - context - .stats - .spill_task_size_from_sealed - .inc_by(task.task_info.task_size as u64); - info!("Spill sealed data. Task: {}", task.get_task_info()); - } self.spilled_data.add_task(task); - size + } + } +} + +struct LocalInstanceEpochData { + epoch: HummockEpoch, + // newer data comes first. + imms: VecDeque, + has_spilled: bool, +} + +impl LocalInstanceEpochData { + fn new(epoch: HummockEpoch) -> Self { + Self { + epoch, + imms: VecDeque::new(), + has_spilled: false, + } + } + + fn epoch(&self) -> HummockEpoch { + self.epoch + } + + fn add_imm(&mut self, imm: UploaderImm) { + assert_eq!(imm.max_epoch(), imm.min_epoch()); + assert_eq!(self.epoch, imm.min_epoch()); + if let Some(prev_imm) = self.imms.front() { + assert_gt!(imm.batch_id(), prev_imm.batch_id()); + } + self.imms.push_front(imm); + } + + fn is_empty(&self) -> bool { + self.imms.is_empty() + } +} + +struct LocalInstanceUnsyncData { + table_id: TableId, + instance_id: LocalInstanceId, + // None means that the current instance should have stopped advancing + current_epoch_data: Option, + // newer data comes first. + sealed_data: VecDeque, + // newer data comes first + flushing_imms: VecDeque, +} + +impl LocalInstanceUnsyncData { + fn new(table_id: TableId, instance_id: LocalInstanceId, init_epoch: HummockEpoch) -> Self { + Self { + table_id, + instance_id, + current_epoch_data: Some(LocalInstanceEpochData::new(init_epoch)), + sealed_data: VecDeque::new(), + flushing_imms: Default::default(), + } + } + + fn add_imm(&mut self, imm: UploaderImm) { + assert_eq!(self.table_id, imm.table_id); + self.current_epoch_data + .as_mut() + .expect("should be Some when adding new imm") + .add_imm(imm); + } + + fn local_seal_epoch(&mut self, next_epoch: HummockEpoch) -> HummockEpoch { + let data = self + .current_epoch_data + .as_mut() + .expect("should be Some when seal new epoch"); + let current_epoch = data.epoch; + debug!( + instance_id = self.instance_id, + next_epoch, current_epoch, "local seal epoch" + ); + assert_gt!(next_epoch, current_epoch); + let epoch_data = replace(data, LocalInstanceEpochData::new(next_epoch)); + if !epoch_data.is_empty() { + self.sealed_data.push_front(epoch_data); + } + current_epoch + } + + // imm_ids from old to new, which means in ascending order + fn ack_flushed(&mut self, imm_ids: impl Iterator) { + for imm_id in imm_ids { + assert_eq!(self.flushing_imms.pop_back().expect("should exist"), imm_id); + } + } + + fn spill(&mut self, epoch: HummockEpoch) -> Vec { + let imms = if let Some(oldest_sealed_epoch) = self.sealed_data.back() { + match oldest_sealed_epoch.epoch.cmp(&epoch) { + Ordering::Less => { + unreachable!( + "should not spill at this epoch because there \ + is unspilled data in previous epoch: prev epoch {}, spill epoch {}", + oldest_sealed_epoch.epoch, epoch + ); + } + Ordering::Equal => { + let epoch_data = self.sealed_data.pop_back().unwrap(); + assert_eq!(epoch, epoch_data.epoch); + epoch_data.imms + } + Ordering::Greater => VecDeque::new(), + } } else { - 0 + let Some(current_epoch_data) = &mut self.current_epoch_data else { + return Vec::new(); + }; + match current_epoch_data.epoch.cmp(&epoch) { + Ordering::Less => { + assert!( + current_epoch_data.imms.is_empty(), + "should not spill at this epoch because there \ + is unspilled data in current epoch epoch {}, spill epoch {}", + current_epoch_data.epoch, + epoch + ); + VecDeque::new() + } + Ordering::Equal => { + if !current_epoch_data.imms.is_empty() { + current_epoch_data.has_spilled = true; + take(&mut current_epoch_data.imms) + } else { + VecDeque::new() + } + } + Ordering::Greater => VecDeque::new(), + } + }; + self.add_flushing_imm(imms.iter().rev().map(|imm| imm.batch_id())); + imms.into_iter().collect() + } + + fn add_flushing_imm(&mut self, imm_ids: impl Iterator) { + for imm_id in imm_ids { + if let Some(prev_imm_id) = self.flushing_imms.front() { + assert_gt!(imm_id, *prev_imm_id); + } + self.flushing_imms.push_front(imm_id); } } - /// Clear self and return the current sealed data - fn drain(&mut self) -> SealedData { - take(self) + // start syncing the imm inclusively before the `epoch` + // returning data with newer data coming first + fn sync(&mut self, epoch: HummockEpoch) -> Vec { + // firstly added from old to new + let mut ret = Vec::new(); + while let Some(epoch_data) = self.sealed_data.back() + && epoch_data.epoch() <= epoch + { + let imms = self.sealed_data.pop_back().expect("checked exist").imms; + self.add_flushing_imm(imms.iter().rev().map(|imm| imm.batch_id())); + ret.extend(imms.into_iter().rev()); + } + // reverse so that newer data comes first + ret.reverse(); + if let Some(latest_epoch_data) = &self.current_epoch_data { + if latest_epoch_data.epoch <= epoch { + assert!(self.sealed_data.is_empty()); + assert!(latest_epoch_data.is_empty()); + assert!(!latest_epoch_data.has_spilled); + if cfg!(debug_assertions) { + panic!("sync epoch exceeds latest epoch, and the current instance should have be archived"); + } + warn!( + instance_id = self.instance_id, + table_id = self.table_id.table_id, + "sync epoch exceeds latest epoch, and the current instance should have be archived" + ); + self.current_epoch_data = None; + } + } + ret } +} - #[cfg(test)] - fn imm_count(&self) -> usize { - self.imms_by_table_shard - .values() - .map(|imms| imms.len()) - .sum() +struct TableUnsyncData { + table_id: TableId, + instance_data: HashMap, + #[expect(clippy::type_complexity)] + table_watermarks: Option<( + WatermarkDirection, + BTreeMap, BitmapBuilder)>, + )>, +} + +impl TableUnsyncData { + fn new(table_id: TableId) -> Self { + Self { + table_id, + instance_data: Default::default(), + table_watermarks: None, + } + } + + fn sync( + &mut self, + epoch: HummockEpoch, + ) -> ( + impl Iterator)> + '_, + Option<( + WatermarkDirection, + impl Iterator)>, + )>, + ) { + ( + self.instance_data + .iter_mut() + .map(move |(instance_id, data)| (*instance_id, data.sync(epoch))), + self.table_watermarks + .as_mut() + .map(|(direction, watermarks)| { + let watermarks = take_before_epoch(watermarks, epoch); + ( + *direction, + watermarks + .into_iter() + .map(|(epoch, (watermarks, _))| (epoch, watermarks)), + ) + }), + ) + } +} + +#[derive(Default)] +/// Unsync data, can be either imm or spilled sst, and some aggregated epoch information. +/// +/// `instance_data` holds the imm of each individual local instance, and data are first added here. +/// The aggregated epoch information (table watermarks, etc.) and the spilled sst will be added to `epoch_data`. +struct UnsyncData { + table_data: HashMap, + // An index as a mapping from instance id to its table id + instance_table_id: HashMap, + epoch_data: BTreeMap, +} + +impl UnsyncData { + fn init_instance( + &mut self, + table_id: TableId, + instance_id: LocalInstanceId, + init_epoch: HummockEpoch, + ) { + debug!( + table_id = table_id.table_id, + instance_id, init_epoch, "init epoch" + ); + let table_data = self + .table_data + .entry(table_id) + .or_insert_with(|| TableUnsyncData::new(table_id)); + assert!(table_data + .instance_data + .insert( + instance_id, + LocalInstanceUnsyncData::new(table_id, instance_id, init_epoch) + ) + .is_none()); + assert!(self + .instance_table_id + .insert(instance_id, table_id) + .is_none()); + self.epoch_data.entry(init_epoch).or_default(); + } + + fn instance_data( + &mut self, + instance_id: LocalInstanceId, + ) -> Option<&mut LocalInstanceUnsyncData> { + self.instance_table_id + .get_mut(&instance_id) + .cloned() + .map(move |table_id| { + self.table_data + .get_mut(&table_id) + .expect("should exist") + .instance_data + .get_mut(&instance_id) + .expect("should exist") + }) + } + + fn add_imm(&mut self, instance_id: LocalInstanceId, imm: UploaderImm) { + self.instance_data(instance_id) + .expect("should exist") + .add_imm(imm); + } + + fn local_seal_epoch( + &mut self, + instance_id: LocalInstanceId, + next_epoch: HummockEpoch, + opts: SealCurrentEpochOptions, + ) { + let table_id = self.instance_table_id[&instance_id]; + let table_data = self.table_data.get_mut(&table_id).expect("should exist"); + let instance_data = table_data + .instance_data + .get_mut(&instance_id) + .expect("should exist"); + let epoch = instance_data.local_seal_epoch(next_epoch); + self.epoch_data.entry(next_epoch).or_default(); + if let Some((direction, table_watermarks)) = opts.table_watermarks { + table_data.add_table_watermarks(epoch, table_watermarks, direction); + } + } + + fn may_destroy_instance(&mut self, instance_id: LocalInstanceId) { + if let Some(table_id) = self.instance_table_id.remove(&instance_id) { + debug!(instance_id, "destroy instance"); + let table_data = self.table_data.get_mut(&table_id).expect("should exist"); + assert!(table_data.instance_data.remove(&instance_id).is_some()); + if table_data.instance_data.is_empty() { + self.table_data.remove(&table_id); + } + } + } + + fn sync(&mut self, epoch: HummockEpoch, context: &UploaderContext) -> SyncDataBuilder { + let sync_epoch_data = take_before_epoch(&mut self.epoch_data, epoch); + + let mut sync_data = SyncDataBuilder::default(); + for (epoch, epoch_data) in sync_epoch_data { + sync_data.add_new_epoch(epoch, epoch_data); + } + + let mut flush_payload = HashMap::new(); + for (table_id, table_data) in &mut self.table_data { + let (unflushed_payload, table_watermarks) = table_data.sync(epoch); + for (instance_id, payload) in unflushed_payload { + if !payload.is_empty() { + flush_payload.insert(instance_id, payload); + } + } + if let Some((direction, watermarks)) = table_watermarks { + sync_data.add_table_watermarks(*table_id, direction, watermarks); + } + } + sync_data.flush(context, flush_payload); + sync_data + } + + fn ack_flushed(&mut self, sstable_info: &StagingSstableInfo) { + for (instance_id, imm_ids) in sstable_info.imm_ids() { + if let Some(instance_data) = self.instance_data(*instance_id) { + // take `rev` to let old imm id goes first + instance_data.ack_flushed(imm_ids.iter().rev().cloned()); + } + } } } struct SyncingData { - // newer epochs come first - epochs: Vec, + sync_epoch: HummockEpoch, // TODO: may replace `TryJoinAll` with a future that will abort other join handles once // one join handle failed. // None means there is no pending uploading tasks @@ -575,13 +915,7 @@ struct SyncingData { sync_result_sender: oneshot::Sender>, } -impl SyncingData { - fn sync_epoch(&self) -> HummockEpoch { - *self.epochs.first().expect("non-empty") - } -} - -pub(super) struct SyncedData { +pub struct SyncedData { pub newly_upload_ssts: Vec, pub uploaded_ssts: VecDeque, pub table_watermarks: HashMap, @@ -615,12 +949,7 @@ impl UploaderContext { #[derive(Default)] struct UploaderData { - /// Data that are not sealed yet. `epoch` satisfies `epoch > max_sealed_epoch`. - unsealed_data: BTreeMap, - - /// Data that are sealed but not synced yet. `epoch` satisfies - /// `max_syncing_epoch < epoch <= max_sealed_epoch`. - sealed_data: SealedData, + unsync_data: UnsyncData, /// Data that has started syncing but not synced yet. `epoch` satisfies /// `max_synced_epoch < epoch <= max_syncing_epoch`. @@ -630,9 +959,8 @@ struct UploaderData { impl UploaderData { fn abort(self, err: impl Fn() -> HummockError) { - self.sealed_data.spilled_data.abort(); - for (_, unsealed_data) in self.unsealed_data { - unsealed_data.spilled_data.abort(); + for (_, epoch_data) in self.unsync_data.epoch_data { + epoch_data.spilled_data.abort(); } // TODO: call `abort` on the uploading task join handle of syncing_data for syncing_data in self.syncing_data { @@ -651,20 +979,17 @@ enum UploaderState { /// An uploader for hummock data. /// -/// Data have 4 sequential stages: unsealed, sealed, syncing, synced. +/// Data have 3 sequential stages: unsync (inside each local instance, data can be unsealed, sealed), syncing, synced. /// -/// The 4 stages are divided by 3 marginal epochs: `max_sealed_epoch`, `max_syncing_epoch`, +/// The 3 stages are divided by 2 marginal epochs: `max_syncing_epoch`, /// `max_synced_epoch`. Epochs satisfy the following inequality. /// /// (epochs of `synced_data`) <= `max_synced_epoch` < (epochs of `syncing_data`) <= -/// `max_syncing_epoch` < (epochs of `sealed_data`) <= `max_sealed_epoch` < (epochs of -/// `unsealed_data`) +/// `max_syncing_epoch` < (epochs of `unsync_data`) /// /// Data are mostly stored in `VecDeque`, and the order stored in the `VecDeque` indicates the data /// order. Data at the front represents ***newer*** data. pub struct HummockUploader { - /// The maximum epoch that is sealed - max_sealed_epoch: HummockEpoch, /// The maximum epoch that has started syncing max_syncing_epoch: HummockEpoch, /// The maximum epoch that has been synced @@ -685,12 +1010,10 @@ impl HummockUploader { ) -> Self { let initial_epoch = pinned_version.version().max_committed_epoch; Self { - max_sealed_epoch: initial_epoch, max_syncing_epoch: initial_epoch, max_synced_epoch: initial_epoch, state: UploaderState::Working(UploaderData { - unsealed_data: Default::default(), - sealed_data: Default::default(), + unsync_data: Default::default(), syncing_data: Default::default(), }), context: UploaderContext::new( @@ -707,10 +1030,6 @@ impl HummockUploader { &self.context.buffer_tracker } - pub(super) fn max_sealed_epoch(&self) -> HummockEpoch { - self.max_sealed_epoch - } - pub(super) fn max_synced_epoch(&self) -> HummockEpoch { self.max_synced_epoch } @@ -727,78 +1046,36 @@ impl HummockUploader { let UploaderState::Working(data) = &mut self.state else { return; }; - let epoch = imm.min_epoch(); - assert!( - epoch > self.max_sealed_epoch, - "imm epoch {} older than max sealed epoch {}", - epoch, - self.max_sealed_epoch - ); - let unsealed_data = data.unsealed_data.entry(epoch).or_default(); - unsealed_data - .imms - .entry(instance_id) - .or_default() - .push_front(UploaderImm::new(imm, &self.context)); + let imm = UploaderImm::new(imm, &self.context); + data.unsync_data.add_imm(instance_id, imm); } - pub(super) fn add_table_watermarks( + pub(super) fn init_instance( &mut self, - epoch: u64, + instance_id: LocalInstanceId, table_id: TableId, - table_watermarks: Vec, - direction: WatermarkDirection, + init_epoch: HummockEpoch, ) { let UploaderState::Working(data) = &mut self.state else { return; }; - assert!( - epoch > self.max_sealed_epoch, - "imm epoch {} older than max sealed epoch {}", - epoch, - self.max_sealed_epoch - ); - data.unsealed_data - .entry(epoch) - .or_default() - .add_table_watermarks(table_id, table_watermarks, direction); + assert_gt!(init_epoch, self.max_syncing_epoch); + data.unsync_data + .init_instance(table_id, instance_id, init_epoch); } - pub(super) fn seal_epoch(&mut self, epoch: HummockEpoch) { + pub(super) fn local_seal_epoch( + &mut self, + instance_id: LocalInstanceId, + next_epoch: HummockEpoch, + opts: SealCurrentEpochOptions, + ) { let UploaderState::Working(data) = &mut self.state else { return; }; - debug!("epoch {} is sealed", epoch); - assert!( - epoch > self.max_sealed_epoch, - "sealing a sealed epoch {}. {}", - epoch, - self.max_sealed_epoch - ); - self.max_sealed_epoch = epoch; - let unsealed_data = - if let Some((&smallest_unsealed_epoch, _)) = data.unsealed_data.first_key_value() { - assert!( - smallest_unsealed_epoch >= epoch, - "some epoch {} older than epoch to seal {}", - smallest_unsealed_epoch, - epoch - ); - if smallest_unsealed_epoch == epoch { - let (_, unsealed_data) = data - .unsealed_data - .pop_first() - .expect("we have checked non-empty"); - unsealed_data - } else { - debug!("epoch {} to seal has no data", epoch); - UnsealedEpochData::default() - } - } else { - debug!("epoch {} to seal has no data", epoch); - UnsealedEpochData::default() - }; - data.sealed_data.seal_new_epoch(epoch, unsealed_data); + assert_gt!(next_epoch, self.max_syncing_epoch); + data.unsync_data + .local_seal_epoch(instance_id, next_epoch, opts); } pub(super) fn start_sync_epoch( @@ -827,20 +1104,12 @@ impl HummockUploader { epoch, self.max_syncing_epoch ); - assert_eq!( - epoch, self.max_sealed_epoch, - "we must start syncing all the sealed data", - ); self.max_syncing_epoch = epoch; - // flush imms to SST file, the output SSTs will be uploaded to object store - // return unfinished merging task - data.sealed_data.flush(&self.context, false); + let sync_data = data.unsync_data.sync(epoch, &self.context); - let SealedData { - epochs, - imms_by_table_shard, + let SyncDataBuilder { spilled_data: SpilledData { uploading_tasks, @@ -848,14 +1117,7 @@ impl HummockUploader { }, table_watermarks, .. - } = data.sealed_data.drain(); - - assert!( - imms_by_table_shard.is_empty(), - "after flush, imms must be empty" - ); - - assert_eq!(epoch, *epochs.front().expect("non-empty epoch")); + } = sync_data; let try_join_all_upload_task = if uploading_tasks.is_empty() { None @@ -864,7 +1126,7 @@ impl HummockUploader { }; data.syncing_data.push_front(SyncingData { - epochs: epochs.into_iter().collect(), + sync_epoch: epoch, uploading_tasks: try_join_all_upload_task, uploaded: uploaded_data, table_watermarks, @@ -902,34 +1164,21 @@ impl HummockUploader { self.context.pinned_version = pinned_version; if self.max_synced_epoch < max_committed_epoch { self.max_synced_epoch = max_committed_epoch; - if let UploaderState::Working(data) = &mut self.state { - if let Some(syncing_data) = data.syncing_data.back() { - // there must not be any syncing data below MCE - assert_gt!( - *syncing_data - .epochs - .last() - .expect("epoch should not be empty"), - max_committed_epoch - ); - } - }; } if self.max_syncing_epoch < max_committed_epoch { self.max_syncing_epoch = max_committed_epoch; - if let UploaderState::Working(data) = &mut self.state { - // there must not be any sealed data below MCE - if let Some(&epoch) = data.sealed_data.epochs.back() { - assert_gt!(epoch, max_committed_epoch); - } - } - } - if self.max_sealed_epoch < max_committed_epoch { - self.max_sealed_epoch = max_committed_epoch; - if let UploaderState::Working(data) = &mut self.state { - // there must not be any unsealed data below MCE - if let Some((&epoch, _)) = data.unsealed_data.first_key_value() { - assert_gt!(epoch, max_committed_epoch); + if let UploaderState::Working(data) = &self.state { + for instance_data in data + .unsync_data + .table_data + .values() + .flat_map(|data| data.instance_data.values()) + { + if let Some(oldest_epoch) = instance_data.sealed_data.back() { + assert_gt!(oldest_epoch.epoch, max_committed_epoch); + } else if let Some(current_epoch) = &instance_data.current_epoch_data { + assert_gt!(current_epoch.epoch, max_committed_epoch); + } } } } @@ -941,26 +1190,28 @@ impl HummockUploader { }; if self.context.buffer_tracker.need_flush() { let mut curr_batch_flush_size = 0; - if self.context.buffer_tracker.need_flush() { - curr_batch_flush_size += data.sealed_data.flush(&self.context, true); - } - - if self - .context - .buffer_tracker - .need_more_flush(curr_batch_flush_size) - { - // iterate from older epoch to newer epoch - for unsealed_data in data.unsealed_data.values_mut() { - curr_batch_flush_size += unsealed_data.flush(&self.context); - if !self - .context - .buffer_tracker - .need_more_flush(curr_batch_flush_size) - { - break; + // iterate from older epoch to newer epoch + for (epoch, epoch_data) in &mut data.unsync_data.epoch_data { + if !self + .context + .buffer_tracker + .need_more_flush(curr_batch_flush_size) + { + break; + } + let mut payload = HashMap::new(); + for (instance_id, instance_data) in data + .unsync_data + .table_data + .values_mut() + .flat_map(|data| data.instance_data.iter_mut()) + { + let instance_payload = instance_data.spill(*epoch); + if !instance_payload.is_empty() { + payload.insert(*instance_id, instance_payload); } } + curr_batch_flush_size += epoch_data.flush(&self.context, payload); } curr_batch_flush_size > 0 } else { @@ -972,7 +1223,6 @@ impl HummockUploader { let max_committed_epoch = self.context.pinned_version.max_committed_epoch(); self.max_synced_epoch = max_committed_epoch; self.max_syncing_epoch = max_committed_epoch; - self.max_sealed_epoch = max_committed_epoch; if let UploaderState::Working(data) = replace( &mut self.state, UploaderState::Working(UploaderData::default()), @@ -984,6 +1234,13 @@ impl HummockUploader { self.context.stats.uploader_syncing_epoch_count.set(0); } + + pub(crate) fn may_destroy_instance(&mut self, instance_id: LocalInstanceId) { + let UploaderState::Working(data) = &mut self.state else { + return; + }; + data.unsync_data.may_destroy_instance(instance_id); + } } impl UploaderData { @@ -1015,12 +1272,18 @@ impl UploaderData { .stats .uploader_syncing_epoch_count .set(self.syncing_data.len() as _); - let epoch = syncing_data.sync_epoch(); + let epoch = syncing_data.sync_epoch; - let result = result.map(|newly_uploaded_sstable_infos| SyncedData { - newly_upload_ssts: newly_uploaded_sstable_infos, - uploaded_ssts: syncing_data.uploaded, - table_watermarks: syncing_data.table_watermarks, + let result = result.map(|newly_uploaded_sstable_infos| { + // take `rev` so that old data is acked first + for sstable_info in newly_uploaded_sstable_infos.iter().rev() { + self.unsync_data.ack_flushed(sstable_info); + } + SyncedData { + newly_upload_ssts: newly_uploaded_sstable_infos, + uploaded_ssts: syncing_data.uploaded, + table_watermarks: syncing_data.table_watermarks, + } }); Poll::Ready(Some((epoch, result, syncing_data.sync_result_sender))) @@ -1029,23 +1292,15 @@ impl UploaderData { } } - /// Poll the success of the oldest spilled task of sealed data. Return `Poll::Ready(None)` if - /// there is no spilling task. - fn poll_sealed_spill_task(&mut self, cx: &mut Context<'_>) -> Poll> { - self.sealed_data.spilled_data.poll_success_spill(cx) - } - - /// Poll the success of the oldest spilled task of unsealed data. Return `Poll::Ready(None)` if + /// Poll the success of the oldest spilled task of unsync spill data. Return `Poll::Ready(None)` if /// there is no spilling task. - fn poll_unsealed_spill_task( - &mut self, - cx: &mut Context<'_>, - ) -> Poll> { + fn poll_spill_task(&mut self, cx: &mut Context<'_>) -> Poll> { // iterator from older epoch to new epoch so that the spill task are finished in epoch order - for unsealed_data in self.unsealed_data.values_mut() { - // if None, there is no spilling task. Search for the unsealed data of the next epoch in + for epoch_data in self.unsync_data.epoch_data.values_mut() { + // if None, there is no spilling task. Search for the unsync data of the next epoch in // the next iteration. - if let Some(sstable_info) = ready!(unsealed_data.spilled_data.poll_success_spill(cx)) { + if let Some(sstable_info) = ready!(epoch_data.spilled_data.poll_success_spill(cx)) { + self.unsync_data.ack_flushed(&sstable_info); return Poll::Ready(Some(sstable_info)); } } @@ -1093,17 +1348,12 @@ impl HummockUploader { data.abort(|| { HummockError::other(format!("previous epoch {} failed to sync", epoch)) }); - return Poll::Pending; } - } - } - - if let Some(sstable_info) = ready!(data.poll_sealed_spill_task(cx)) { - return Poll::Ready(UploaderEvent::DataSpilled(sstable_info)); + }; } - if let Some(sstable_info) = ready!(data.poll_unsealed_spill_task(cx)) { + if let Some(sstable_info) = ready!(data.poll_spill_task(cx)) { return Poll::Ready(UploaderEvent::DataSpilled(sstable_info)); } @@ -1156,6 +1406,7 @@ pub(crate) mod tests { use crate::mem_table::{ImmId, ImmutableMemtable}; use crate::monitor::HummockStateStoreMetrics; use crate::opts::StorageOpts; + use crate::store::SealCurrentEpochOptions; const INITIAL_EPOCH: HummockEpoch = test_epoch(5); pub(crate) const TEST_TABLE_ID: TableId = TableId { table_id: 233 }; @@ -1176,11 +1427,10 @@ pub(crate) mod tests { } fn test_hummock_version(epoch: HummockEpoch) -> HummockVersion { - HummockVersion { - id: epoch, - max_committed_epoch: epoch, - ..Default::default() - } + let mut version = HummockVersion::default(); + version.id = epoch; + version.max_committed_epoch = epoch; + version } fn initial_pinned_version() -> PinnedVersion { @@ -1314,6 +1564,16 @@ pub(crate) mod tests { )]) } + impl HummockUploader { + fn local_seal_epoch_for_test(&mut self, instance_id: LocalInstanceId, epoch: HummockEpoch) { + self.local_seal_epoch( + instance_id, + epoch.next_epoch(), + SealCurrentEpochOptions::for_test(), + ); + } + } + #[tokio::test] pub async fn test_uploading_task_future() { let uploader_context = test_uploader_context(dummy_success_upload_future); @@ -1390,36 +1650,15 @@ pub(crate) mod tests { let mut uploader = test_uploader(dummy_success_upload_future); let epoch1 = INITIAL_EPOCH.next_epoch(); let imm = gen_imm(epoch1).await; - + uploader.init_instance(TEST_LOCAL_INSTANCE_ID, TEST_TABLE_ID, epoch1); uploader.add_imm(TEST_LOCAL_INSTANCE_ID, imm.clone()); - assert_eq!(1, uploader.data().unsealed_data.len()); - assert_eq!( - epoch1 as HummockEpoch, - *uploader.data().unsealed_data.first_key_value().unwrap().0 - ); - assert_eq!( - 1, - uploader - .data() - .unsealed_data - .first_key_value() - .unwrap() - .1 - .imms - .len() - ); - uploader.seal_epoch(epoch1); - assert_eq!(epoch1, uploader.max_sealed_epoch); - assert!(uploader.data().unsealed_data.is_empty()); - assert_eq!(1, uploader.data().sealed_data.imm_count()); + uploader.local_seal_epoch_for_test(TEST_LOCAL_INSTANCE_ID, epoch1); uploader.start_sync_epoch_for_test(epoch1); assert_eq!(epoch1 as HummockEpoch, uploader.max_syncing_epoch); - assert_eq!(0, uploader.data().sealed_data.imm_count()); - assert!(uploader.data().sealed_data.spilled_data.is_empty()); assert_eq!(1, uploader.data().syncing_data.len()); let syncing_data = uploader.data().syncing_data.front().unwrap(); - assert_eq!(epoch1 as HummockEpoch, syncing_data.sync_epoch()); + assert_eq!(epoch1 as HummockEpoch, syncing_data.sync_epoch); assert!(syncing_data.uploaded.is_empty()); assert!(syncing_data.uploading_tasks.is_some()); @@ -1457,6 +1696,32 @@ pub(crate) mod tests { assert_eq!(epoch1, uploader.max_committed_epoch()); } + #[tokio::test] + async fn test_empty_uploader_sync() { + let mut uploader = test_uploader(dummy_success_upload_future); + let epoch1 = INITIAL_EPOCH.next_epoch(); + + uploader.start_sync_epoch_for_test(epoch1); + assert_eq!(epoch1, uploader.max_syncing_epoch); + + match uploader.next_event().await { + UploaderEvent::SyncFinish(finished_epoch, data) => { + assert_eq!(epoch1, finished_epoch); + assert!(data.uploaded_ssts.is_empty()); + assert!(data.newly_upload_ssts.is_empty()); + } + _ => unreachable!(), + }; + assert_eq!(epoch1, uploader.max_synced_epoch()); + let new_pinned_version = uploader + .context + .pinned_version + .new_pin_version(test_hummock_version(epoch1)); + uploader.update_pinned_version(new_pinned_version); + assert!(uploader.data().syncing_data.is_empty()); + assert_eq!(epoch1, uploader.max_committed_epoch()); + } + #[tokio::test] async fn test_uploader_empty_epoch() { let mut uploader = test_uploader(dummy_success_upload_future); @@ -1464,9 +1729,9 @@ pub(crate) mod tests { let epoch2 = epoch1.next_epoch(); let imm = gen_imm(epoch2).await; // epoch1 is empty while epoch2 is not. Going to seal empty epoch1. + uploader.init_instance(TEST_LOCAL_INSTANCE_ID, TEST_TABLE_ID, epoch1); + uploader.local_seal_epoch_for_test(TEST_LOCAL_INSTANCE_ID, epoch1); uploader.add_imm(TEST_LOCAL_INSTANCE_ID, imm); - uploader.seal_epoch(epoch1); - assert_eq!(epoch1, uploader.max_sealed_epoch); uploader.start_sync_epoch_for_test(epoch1); assert_eq!(epoch1, uploader.max_syncing_epoch); @@ -1496,12 +1761,7 @@ pub(crate) mod tests { assert!(poll_fn(|cx| data.poll_syncing_task(cx, &uploader.context)) .await .is_none()); - assert!(poll_fn(|cx| data.poll_sealed_spill_task(cx)) - .await - .is_none()); - assert!(poll_fn(|cx| data.poll_unsealed_spill_task(cx)) - .await - .is_none()); + assert!(poll_fn(|cx| data.poll_spill_task(cx)).await.is_none()); } #[tokio::test] @@ -1522,27 +1782,23 @@ pub(crate) mod tests { uploader.update_pinned_version(version1); assert_eq!(epoch1, uploader.max_synced_epoch); assert_eq!(epoch1, uploader.max_syncing_epoch); - assert_eq!(epoch1, uploader.max_sealed_epoch); + uploader.init_instance(TEST_LOCAL_INSTANCE_ID, TEST_TABLE_ID, epoch6); uploader.add_imm(TEST_LOCAL_INSTANCE_ID, gen_imm(epoch6).await); uploader.update_pinned_version(version2); assert_eq!(epoch2, uploader.max_synced_epoch); assert_eq!(epoch2, uploader.max_syncing_epoch); - assert_eq!(epoch2, uploader.max_sealed_epoch); - uploader.seal_epoch(epoch6); - assert_eq!(epoch6, uploader.max_sealed_epoch); + uploader.local_seal_epoch_for_test(TEST_LOCAL_INSTANCE_ID, epoch6); uploader.update_pinned_version(version3); assert_eq!(epoch3, uploader.max_synced_epoch); assert_eq!(epoch3, uploader.max_syncing_epoch); - assert_eq!(epoch6, uploader.max_sealed_epoch); uploader.start_sync_epoch_for_test(epoch6); assert_eq!(epoch6, uploader.max_syncing_epoch); uploader.update_pinned_version(version4); assert_eq!(epoch4, uploader.max_synced_epoch); assert_eq!(epoch6, uploader.max_syncing_epoch); - assert_eq!(epoch6, uploader.max_sealed_epoch); match uploader.next_event().await { UploaderEvent::SyncFinish(epoch, _) => { @@ -1553,7 +1809,6 @@ pub(crate) mod tests { uploader.update_pinned_version(version5); assert_eq!(epoch6, uploader.max_synced_epoch); assert_eq!(epoch6, uploader.max_syncing_epoch); - assert_eq!(epoch6, uploader.max_sealed_epoch); } fn prepare_uploader_order_test( @@ -1649,6 +1904,9 @@ pub(crate) mod tests { let instance_id1 = 1; let instance_id2 = 2; + uploader.init_instance(instance_id1, TEST_TABLE_ID, epoch1); + uploader.init_instance(instance_id2, TEST_TABLE_ID, epoch2); + // imm2 contains data in newer epoch, but added first let imm2 = gen_imm_with_limiter(epoch2, memory_limiter).await; uploader.add_imm(instance_id2, imm2.clone()); @@ -1701,18 +1959,19 @@ pub(crate) mod tests { let epoch1_sync_payload = HashMap::from_iter([(instance_id1, vec![imm1_4.clone()])]); let (await_start1_4, finish_tx1_4) = new_task_notifier(get_payload_imm_ids(&epoch1_sync_payload)); - uploader.seal_epoch(epoch1); + uploader.local_seal_epoch_for_test(instance_id1, epoch1); uploader.start_sync_epoch_for_test(epoch1); await_start1_4.await; + let epoch3 = epoch2.next_epoch(); - uploader.seal_epoch(epoch2); + uploader.local_seal_epoch_for_test(instance_id1, epoch2); + uploader.local_seal_epoch_for_test(instance_id2, epoch2); // current uploader state: // unsealed: empty // sealed: epoch2: uploaded sst([imm2]) // syncing: epoch1: uploading: [imm1_4], [imm1_3], uploaded: sst([imm1_2, imm1_1]) - let epoch3 = epoch2.next_epoch(); let imm3_1 = gen_imm_with_limiter(epoch3, memory_limiter).await; let epoch3_spill_payload1 = HashMap::from_iter([(instance_id1, vec![imm3_1.clone()])]); uploader.add_imm(instance_id1, imm3_1.clone()); @@ -1736,6 +1995,7 @@ pub(crate) mod tests { // syncing: epoch1: uploading: [imm1_4], [imm1_3], uploaded: sst([imm1_2, imm1_1]) let epoch4 = epoch3.next_epoch(); + uploader.local_seal_epoch_for_test(instance_id1, epoch3); let imm4 = gen_imm_with_limiter(epoch4, memory_limiter).await; uploader.add_imm(instance_id1, imm4.clone()); assert_uploader_pending(&mut uploader).await; @@ -1801,7 +2061,7 @@ pub(crate) mod tests { // synced: epoch1: sst([imm1_4]), sst([imm1_3]), sst([imm1_2, imm1_1]) // epoch2: sst([imm2]) - uploader.seal_epoch(epoch3); + uploader.local_seal_epoch_for_test(instance_id2, epoch3); if let UploaderEvent::DataSpilled(sst) = uploader.next_event().await { assert_eq!(&get_payload_imm_ids(&epoch3_spill_payload1), sst.imm_ids()); } else { @@ -1815,7 +2075,8 @@ pub(crate) mod tests { // synced: epoch1: sst([imm1_4]), sst([imm1_3]), sst([imm1_2, imm1_1]) // epoch2: sst([imm2]) - uploader.seal_epoch(epoch4); + uploader.local_seal_epoch_for_test(instance_id1, epoch4); + uploader.local_seal_epoch_for_test(instance_id2, epoch4); let epoch4_sync_payload = HashMap::from_iter([(instance_id1, vec![imm4, imm3_3])]); let (await_start4_with_3_3, finish_tx4_with_3_3) = new_task_notifier(get_payload_imm_ids(&epoch4_sync_payload)); @@ -1878,9 +2139,14 @@ pub(crate) mod tests { let epoch1 = INITIAL_EPOCH.next_epoch(); let epoch2 = epoch1.next_epoch(); + let instance_id1 = 1; + let instance_id2 = 2; let flush_threshold = buffer_tracker.flush_threshold(); let memory_limiter = buffer_tracker.get_memory_limiter().clone(); + uploader.init_instance(instance_id1, TEST_TABLE_ID, epoch1); + uploader.init_instance(instance_id2, TEST_TABLE_ID, epoch2); + // imm2 contains data in newer epoch, but added first let mut total_memory = 0; while total_memory < flush_threshold { @@ -1889,15 +2155,15 @@ pub(crate) mod tests { if total_memory > flush_threshold { break; } - uploader.add_imm(TEST_LOCAL_INSTANCE_ID, imm); + uploader.add_imm(instance_id2, imm); } let imm = gen_imm_with_limiter(epoch1, Some(memory_limiter.as_ref())).await; - uploader.add_imm(TEST_LOCAL_INSTANCE_ID, imm); + uploader.add_imm(instance_id1, imm); assert!(uploader.may_flush()); for _ in 0..10 { let imm = gen_imm_with_limiter(epoch1, Some(memory_limiter.as_ref())).await; - uploader.add_imm(TEST_LOCAL_INSTANCE_ID, imm); + uploader.add_imm(instance_id1, imm); assert!(!uploader.may_flush()); } } diff --git a/src/storage/src/hummock/local_version/pinned_version.rs b/src/storage/src/hummock/local_version/pinned_version.rs index da9569e6bb83..6302f91739c2 100644 --- a/src/storage/src/hummock/local_version/pinned_version.rs +++ b/src/storage/src/hummock/local_version/pinned_version.rs @@ -155,10 +155,6 @@ impl PinnedVersion { self.version.max_committed_epoch } - pub fn safe_epoch(&self) -> u64 { - self.version.safe_epoch - } - /// ret value can't be used as `HummockVersion`. it must be modified with delta pub fn version(&self) -> &HummockVersion { &self.version diff --git a/src/storage/src/hummock/store/hummock_storage.rs b/src/storage/src/hummock/store/hummock_storage.rs index bfe24e555937..2d89fdd401fa 100644 --- a/src/storage/src/hummock/store/hummock_storage.rs +++ b/src/storage/src/hummock/store/hummock_storage.rs @@ -29,7 +29,7 @@ use risingwave_hummock_sdk::key::{ is_empty_key_range, vnode, vnode_range, TableKey, TableKeyRange, }; use risingwave_hummock_sdk::table_watermark::TableWatermarksIndex; -use risingwave_hummock_sdk::{HummockReadEpoch, SyncResult}; +use risingwave_hummock_sdk::HummockReadEpoch; use risingwave_pb::hummock::SstableInfo; use risingwave_rpc_client::HummockMetaClient; use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; @@ -313,7 +313,8 @@ impl HummockStorage { ) -> StorageResult<(TableKeyRange, ReadVersionTuple)> { match self.backup_reader.try_get_hummock_version(epoch).await { Ok(Some(backup_version)) => { - validate_safe_epoch(backup_version.safe_epoch(), epoch)?; + validate_safe_epoch(backup_version.version(), table_id, epoch)?; + Ok(get_committed_read_version_tuple( backup_version, table_id, @@ -337,7 +338,7 @@ impl HummockStorage { key_range: TableKeyRange, ) -> StorageResult<(TableKeyRange, ReadVersionTuple)> { let pinned_version = self.pinned_version.load(); - validate_safe_epoch(pinned_version.safe_epoch(), epoch)?; + validate_safe_epoch(pinned_version.version(), table_id, epoch)?; // check epoch if lower mce let ret = if epoch <= pinned_version.max_committed_epoch() { @@ -582,12 +583,6 @@ impl StateStore for HummockStorage { MemOrdering::SeqCst, ); } - self.hummock_event_sender - .send(HummockEvent::SealEpoch { - epoch, - is_checkpoint, - }) - .expect("should send success"); StoreLocalStatistic::flush_all(); } @@ -643,7 +638,10 @@ use risingwave_hummock_sdk::version::HummockVersion; #[cfg(any(test, feature = "test"))] impl HummockStorage { - pub async fn seal_and_sync_epoch(&self, epoch: u64) -> StorageResult { + pub async fn seal_and_sync_epoch( + &self, + epoch: u64, + ) -> StorageResult { self.seal_epoch(epoch, true); self.sync(epoch).await } diff --git a/src/storage/src/hummock/store/local_hummock_storage.rs b/src/storage/src/hummock/store/local_hummock_storage.rs index 2f0ad3437efc..a14f3b450adf 100644 --- a/src/storage/src/hummock/store/local_hummock_storage.rs +++ b/src/storage/src/hummock/store/local_hummock_storage.rs @@ -513,6 +513,14 @@ impl LocalStateStore for LocalHummockStorage { "local state store of table id {:?} is init for more than once", self.table_id ); + if !self.is_replicated { + self.event_sender + .send(HummockEvent::InitEpoch { + instance_id: self.instance_id(), + init_epoch: options.epoch.curr, + }) + .expect("should succeed"); + } Ok(()) } @@ -544,14 +552,15 @@ impl LocalStateStore for LocalHummockStorage { }); } } - self.event_sender - .send(HummockEvent::LocalSealEpoch { - instance_id: self.instance_id(), - table_id: self.table_id, - epoch: prev_epoch, - opts, - }) - .expect("should be able to send") + if !self.is_replicated { + self.event_sender + .send(HummockEvent::LocalSealEpoch { + instance_id: self.instance_id(), + next_epoch, + opts, + }) + .expect("should be able to send"); + } } fn update_vnode_bitmap(&mut self, vnodes: Arc) -> Arc { diff --git a/src/storage/src/hummock/utils.rs b/src/storage/src/hummock/utils.rs index 4d61e7cd3367..4c270ee736b9 100644 --- a/src/storage/src/hummock/utils.rs +++ b/src/storage/src/hummock/utils.rs @@ -70,9 +70,19 @@ where !too_left && !too_right } -pub fn validate_safe_epoch(safe_epoch: u64, epoch: u64) -> HummockResult<()> { - if epoch < safe_epoch { - return Err(HummockError::expired_epoch(safe_epoch, epoch)); +pub fn validate_safe_epoch( + version: &HummockVersion, + table_id: TableId, + epoch: u64, +) -> HummockResult<()> { + if let Some(info) = version.state_table_info.info().get(&table_id) + && epoch < info.safe_epoch + { + return Err(HummockError::expired_epoch( + table_id, + info.safe_epoch, + epoch, + )); } Ok(()) diff --git a/src/stream/src/executor/actor.rs b/src/stream/src/executor/actor.rs index 249546031e22..1c73a3aeddad 100644 --- a/src/stream/src/executor/actor.rs +++ b/src/stream/src/executor/actor.rs @@ -223,14 +223,15 @@ where ) .into())); - // Collect barriers to local barrier manager - self.barrier_manager.collect(id, &barrier); - // Then stop this actor if asked if barrier.is_stop(id) { - break Ok(()); + debug!(actor_id = id, epoch = ?barrier.epoch, "stop at barrier"); + break Ok(barrier); } + // Collect barriers to local barrier manager + self.barrier_manager.collect(id, &barrier); + // Tracing related work last_epoch = Some(barrier.epoch); span = barrier.tracing_context().attach(new_span(last_epoch)); @@ -238,7 +239,12 @@ where spawn_blocking_drop_stream(stream).await; - tracing::trace!(actor_id = id, "actor exit"); + let result = result.map(|stop_barrier| { + // Collect the stop barrier after the stream has been dropped to ensure that all resources + self.barrier_manager.collect(id, &stop_barrier); + }); + + tracing::debug!(actor_id = id, ok = result.is_ok(), "actor exit"); result } } diff --git a/src/stream/src/executor/backfill/arrangement_backfill.rs b/src/stream/src/executor/backfill/arrangement_backfill.rs index 557dda4f535e..7920e8dceee8 100644 --- a/src/stream/src/executor/backfill/arrangement_backfill.rs +++ b/src/stream/src/executor/backfill/arrangement_backfill.rs @@ -551,7 +551,8 @@ where // If not finished then we need to update state, otherwise no need. if let Message::Barrier(barrier) = &msg { if is_completely_finished { - // If already finished, no need to persist any state. + // If already finished, no need to persist any state. But we need to advance the epoch anyway + self.state_table.commit(barrier.epoch).await?; } else { // If snapshot was empty, we do not need to backfill, // but we still need to persist the finished state. @@ -595,6 +596,10 @@ where #[for_await] for msg in upstream { if let Some(msg) = mapping_message(msg?, &self.output_indices) { + if let Message::Barrier(barrier) = &msg { + // If already finished, no need persist any state, but we need to advance the epoch of the state table anyway. + self.state_table.commit(barrier.epoch).await?; + } yield msg; } } diff --git a/src/stream/src/executor/backfill/no_shuffle_backfill.rs b/src/stream/src/executor/backfill/no_shuffle_backfill.rs index bd130fd8f52a..e368086a9773 100644 --- a/src/stream/src/executor/backfill/no_shuffle_backfill.rs +++ b/src/stream/src/executor/backfill/no_shuffle_backfill.rs @@ -498,7 +498,10 @@ where // If not finished then we need to update state, otherwise no need. if let Message::Barrier(barrier) = &msg { if is_finished { - // If already finished, no need persist any state. + // If already finished, no need persist any state, but we need to advance the epoch of the state table anyway. + if let Some(table) = &mut self.state_table { + table.commit(barrier.epoch).await?; + } } else { // If snapshot was empty, we do not need to backfill, // but we still need to persist the finished state. @@ -564,6 +567,13 @@ where #[for_await] for msg in upstream { if let Some(msg) = mapping_message(msg?, &self.output_indices) { + if let Message::Barrier(barrier) = &msg { + // If already finished, no need persist any state, but we need to advance the epoch of the state table anyway. + if let Some(table) = &mut self.state_table { + table.commit(barrier.epoch).await?; + } + } + yield msg; } } diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs index 8b1daa9695c4..19124fe8c22d 100644 --- a/src/stream/src/executor/merge.rs +++ b/src/stream/src/executor/merge.rs @@ -18,6 +18,9 @@ use std::task::{Context, Poll}; use anyhow::Context as _; use futures::stream::{FusedStream, FuturesUnordered, StreamFuture}; +use prometheus::Histogram; +use risingwave_common::config::MetricLevel; +use risingwave_common::metrics::LabelGuardedMetric; use tokio::time::Instant; use super::exchange::input::BoxedInput; @@ -92,12 +95,24 @@ impl MergeExecutor { #[try_stream(ok = Message, error = StreamExecutorError)] async fn execute_inner(mut self: Box) { + let merge_barrier_align_duration = if self.metrics.level >= MetricLevel::Debug { + Some( + self.metrics + .merge_barrier_align_duration + .with_label_values(&[ + &self.actor_context.id.to_string(), + &self.actor_context.fragment_id.to_string(), + ]), + ) + } else { + None + }; + // Futures of all active upstreams. let select_all = SelectReceivers::new( self.actor_context.id, - self.actor_context.fragment_id, self.upstreams, - self.metrics.clone(), + merge_barrier_align_duration.clone(), ); let actor_id = self.actor_context.id; @@ -189,9 +204,8 @@ impl MergeExecutor { // the one we polled from original upstreams. let mut select_new = SelectReceivers::new( self.actor_context.id, - self.fragment_id, new_upstreams, - self.metrics.clone(), + merge_barrier_align_duration.clone(), ); let new_barrier = expect_first_barrier(&mut select_new).await?; assert_eq!(barrier, &new_barrier); @@ -256,12 +270,10 @@ pub struct SelectReceivers { /// The actor id of this fragment. actor_id: u32, - /// The fragment id - fragment_id: u32, /// watermark column index -> `BufferedWatermarks` buffered_watermarks: BTreeMap>, - /// Streaming Metrics - metrics: Arc, + /// If None, then we don't take `Instant::now()` and `observe` during `poll_next` + merge_barrier_align_duration: Option>, } impl Stream for SelectReceivers { @@ -274,10 +286,6 @@ impl Stream for SelectReceivers { return Poll::Ready(None); } - let merge_barrier_align_duration = self - .metrics - .merge_barrier_align_duration - .with_label_values(&[&self.actor_id.to_string(), &self.fragment_id.to_string()]); let mut start = None; loop { match futures::ready!(self.active.poll_next_unpin(cx)) { @@ -303,7 +311,9 @@ impl Stream for SelectReceivers { } Message::Barrier(barrier) => { // Block this upstream by pushing it to `blocked`. - if self.blocked.is_empty() { + if self.blocked.is_empty() + && self.merge_barrier_align_duration.is_some() + { start = Some(Instant::now()); } self.blocked.push(remaining); @@ -332,7 +342,11 @@ impl Stream for SelectReceivers { Some((None, _)) => unreachable!(), // There's no active upstreams. Process the barrier and resume the blocked ones. None => { - if let Some(start) = start { + if let Some(start) = start + && let Some(merge_barrier_align_duration) = + &self.merge_barrier_align_duration + { + // Observe did a few atomic operation inside, we want to avoid the overhead. merge_barrier_align_duration.observe(start.elapsed().as_secs_f64()) } break; @@ -360,9 +374,8 @@ impl Stream for SelectReceivers { impl SelectReceivers { fn new( actor_id: u32, - fragment_id: u32, upstreams: Vec, - metrics: Arc, + merge_barrier_align_duration: Option>, ) -> Self { assert!(!upstreams.is_empty()); let upstream_actor_ids = upstreams.iter().map(|input| input.actor_id()).collect(); @@ -370,11 +383,10 @@ impl SelectReceivers { blocked: Vec::with_capacity(upstreams.len()), active: Default::default(), actor_id, - fragment_id, barrier: None, upstream_actor_ids, buffered_watermarks: Default::default(), - metrics, + merge_barrier_align_duration, }; this.extend_active(upstreams); this