diff --git a/.github/workflows/cherry-pick-to-release-branch.yml b/.github/workflows/cherry-pick-to-release-branch.yml index 2d7bcdc31224a..b6f70202855ff 100644 --- a/.github/workflows/cherry-pick-to-release-branch.yml +++ b/.github/workflows/cherry-pick-to-release-branch.yml @@ -3,7 +3,7 @@ on: pull_request: branches: - main - types: ["closed"] + types: ["closed", "labeled"] jobs: release_pull_request_1_7: diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index 0a81a772b1eaf..6450e7baf4c64 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -116,7 +116,7 @@ services: volumes: - ..:/risingwave - release-env: + release-env-x86: # build binaries on a earlier Linux distribution (therefore with earlier version GLIBC) # See https://github.com/risingwavelabs/risingwave/issues/4556 for more details. # @@ -132,6 +132,12 @@ services: volumes: - ..:/mnt + release-env-arm: + image: quay.io/pypa/manylinux2014_aarch64 + working_dir: /mnt + volumes: + - ..:/mnt + elasticsearch: container_name: elasticsearch image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index 32d9b801a7e26..1596d44518cd3 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -4,6 +4,7 @@ set -euo pipefail REPO_ROOT=${PWD} +ARCH="$(uname -m)" echo "--- Check env" if [ "${BUILDKITE_SOURCE}" != "schedule" ] && [ "${BUILDKITE_SOURCE}" != "webhook" ] && [[ -z "${BINARY_NAME+x}" ]]; then @@ -11,22 +12,22 @@ if [ "${BUILDKITE_SOURCE}" != "schedule" ] && [ "${BUILDKITE_SOURCE}" != "webhoo fi echo "--- Install aws cli" -curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip" unzip -q awscliv2.zip && ./aws/install && mv /usr/local/bin/aws /bin/aws echo "--- Install lld" # The lld in the CentOS 7 repository is too old and contains a bug that causes a linker error. # So we install a newer version here. (17.0.6, latest version at the time of writing) # It is manually built in the same environent and uploaded to S3. -aws s3 cp s3://ci-deps-dist/llvm-lld-manylinux2014_x86_64.tar.gz . -tar -zxvf llvm-lld-manylinux2014_x86_64.tar.gz --directory=/usr/local +aws s3 cp s3://ci-deps-dist/llvm-lld-manylinux2014_${ARCH}.tar.gz . +tar -zxvf llvm-lld-manylinux2014_${ARCH}.tar.gz --directory=/usr/local ld.lld --version echo "--- Install dependencies for openssl" yum install -y perl-core echo "--- Install java and maven" -yum install -y java-11-openjdk java-11-openjdk-devel wget python3 cyrus-sasl-devel +yum install -y java-11-openjdk java-11-openjdk-devel wget python3 python3-devel cyrus-sasl-devel pip3 install toml-cli wget https://ci-deps-dist.s3.amazonaws.com/apache-maven-3.9.3-bin.tar.gz && tar -zxvf apache-maven-3.9.3-bin.tar.gz export PATH="${REPO_ROOT}/apache-maven-3.9.3/bin:$PATH" @@ -40,8 +41,13 @@ source ci/scripts/common.sh unset RUSTC_WRAPPER # disable sccache echo "--- Install protoc3" -curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip -unzip -o protoc-3.15.8-linux-x86_64.zip -d protoc +PROTOC_ARCH=${ARCH} +if [ ${ARCH} == "aarch64" ]; then + # shellcheck disable=SC1068 + PROTOC_ARCH="aarch_64" +fi +curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-${PROTOC_ARCH}.zip +unzip -o protoc-3.15.8-linux-${PROTOC_ARCH}.zip -d protoc mv ./protoc/bin/protoc /usr/local/bin/ mv ./protoc/include/* /usr/local/include/ @@ -63,11 +69,11 @@ cd target/release && chmod +x risingwave risectl echo "--- Upload nightly binary to s3" if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then - tar -czvf risingwave-"$(date '+%Y%m%d')"-x86_64-unknown-linux.tar.gz risingwave - aws s3 cp risingwave-"$(date '+%Y%m%d')"-x86_64-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary + tar -czvf risingwave-"$(date '+%Y%m%d')"-${ARCH}-unknown-linux.tar.gz risingwave + aws s3 cp risingwave-"$(date '+%Y%m%d')"-${ARCH}-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary elif [[ -n "${BINARY_NAME+x}" ]]; then - tar -czvf risingwave-${BINARY_NAME}-x86_64-unknown-linux.tar.gz risingwave - aws s3 cp risingwave-${BINARY_NAME}-x86_64-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary + tar -czvf risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz risingwave + aws s3 cp risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary fi echo "--- Build connector node" @@ -88,23 +94,29 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then dnf install -y gh echo "--- Release create" - gh release create "${BUILDKITE_TAG}" --notes "release ${BUILDKITE_TAG}" -d -p + response=$(gh api repos/risingwavelabs/risingwave/releases/tags/${BUILDKITE_TAG} 2>&1) + if [[ $response == *"Not Found"* ]]; then + echo "Tag ${BUILDKITE_TAG} does not exist. Creating release..." + gh release create "${BUILDKITE_TAG}" --notes "release ${BUILDKITE_TAG}" -d -p + else + echo "Tag ${BUILDKITE_TAG} already exists. Skipping release creation." + fi echo "--- Release upload risingwave asset" - tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz risingwave - gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz + tar -czvf risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.tar.gz risingwave + gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.tar.gz echo "--- Release upload risingwave debug info" - tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.dwp.tar.gz risingwave.dwp - gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.dwp.tar.gz + tar -czvf risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.dwp.tar.gz risingwave.dwp + gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.dwp.tar.gz echo "--- Release upload risectl asset" - tar -czvf risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz risectl - gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz + tar -czvf risectl-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.tar.gz risectl + gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.tar.gz echo "--- Release upload risingwave-all-in-one asset" - tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux-all-in-one.tar.gz risingwave libs - gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux-all-in-one.tar.gz + tar -czvf risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux-all-in-one.tar.gz risingwave libs + gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-{ARCH}-unknown-linux-all-in-one.tar.gz fi diff --git a/ci/workflows/docker.yml b/ci/workflows/docker.yml index d97d99af7691d..bcf1b605eb96b 100644 --- a/ci/workflows/docker.yml +++ b/ci/workflows/docker.yml @@ -46,14 +46,14 @@ steps: DOCKER_TOKEN: docker-token retry: *auto-retry - - label: "pre build binary" + - label: "pre build binary: amd64" command: "ci/scripts/release.sh" plugins: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - docker-compose#v4.9.0: - run: release-env + run: release-env-x86 config: ci/docker-compose.yml mount-buildkite-agent: true propagate-environment: true @@ -61,3 +61,21 @@ steps: - BINARY_NAME - GITHUB_TOKEN retry: *auto-retry + + - label: "pre build binary: aarch64 " + command: "ci/scripts/release.sh" + plugins: + - seek-oss/aws-sm#v2.3.1: + env: + GITHUB_TOKEN: github-token + - docker-compose#v4.9.0: + run: release-env-arm + config: ci/docker-compose.yml + mount-buildkite-agent: true + propagate-environment: true + environment: + - BINARY_NAME + - GITHUB_TOKEN + agents: + queue: "linux-arm64" + retry: *auto-retry diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml index fab1f958f717e..835c46fb01e60 100644 --- a/ci/workflows/main-cron.yml +++ b/ci/workflows/main-cron.yml @@ -877,7 +877,7 @@ steps: timeout_in_minutes: 10 retry: *auto-retry - - label: "release" + - label: "release amd64" command: "ci/scripts/release.sh" if: build.tag != null plugins: @@ -885,7 +885,7 @@ steps: env: GITHUB_TOKEN: github-token - docker-compose#v4.9.0: - run: release-env + run: release-env-x86 config: ci/docker-compose.yml mount-buildkite-agent: true environment: @@ -895,6 +895,26 @@ steps: timeout_in_minutes: 60 retry: *auto-retry + - label: "release aarch64" + command: "ci/scripts/release.sh" + if: build.tag != null + plugins: + - seek-oss/aws-sm#v2.3.1: + env: + GITHUB_TOKEN: github-token + - docker-compose#v4.9.0: + run: release-env-arm + config: ci/docker-compose.yml + mount-buildkite-agent: true + environment: + - GITHUB_TOKEN + - BUILDKITE_TAG + - BUILDKITE_SOURCE + agents: + queue: "linux-arm64" + timeout_in_minutes: 60 + retry: *auto-retry + - label: "release docker image: amd64" command: "ci/scripts/docker.sh" key: "build-amd64" diff --git a/e2e_test/batch/basic/logical_view.slt.part b/e2e_test/batch/basic/logical_view.slt.part index 4adc05b386067..9c6b4c0f7360d 100644 --- a/e2e_test/batch/basic/logical_view.slt.part +++ b/e2e_test/batch/basic/logical_view.slt.part @@ -40,7 +40,7 @@ SELECT * FROM v3; statement error DROP TABLE t; -statement error other relation\(s\) depend on it +statement error Permission denied DROP VIEW v2; statement ok diff --git a/e2e_test/ddl/alter_rename.slt b/e2e_test/ddl/alter_rename.slt index 11b7007f72bf0..5171f7f3cdad1 100644 --- a/e2e_test/ddl/alter_rename.slt +++ b/e2e_test/ddl/alter_rename.slt @@ -229,7 +229,7 @@ DROP SCHEMA schema1; statement ok DROP SINK sink1; -statement error other relation\(s\) depend on it +statement error Permission denied DROP VIEW v5; statement ok diff --git a/e2e_test/ddl/dependency_check.slt b/e2e_test/ddl/dependency_check.slt index 88a89975d07ef..413579bb13e7b 100644 --- a/e2e_test/ddl/dependency_check.slt +++ b/e2e_test/ddl/dependency_check.slt @@ -16,7 +16,7 @@ create index i_b1 on b(b1); statement ok create materialized view mv1 as select * from a join b on a.a1 = b.b1; -statement error other relation\(s\) depend on it +statement error Permission denied drop index i_a1; statement ok @@ -25,7 +25,7 @@ drop materialized view mv1; statement ok create materialized view mv2 as with ctx as (select a1 from a) select b1 from b; -statement error other relation\(s\) depend on it +statement error Permission denied drop table a; statement ok @@ -48,13 +48,13 @@ create view v2 as select * from v; statement ok create materialized view mv3 as select * from v2; -statement error other relation\(s\) depend on it +statement error Permission denied drop source src; -statement error other relation\(s\) depend on it +statement error Permission denied drop view v; -statement error other relation\(s\) depend on it +statement error Permission denied drop view v2; statement ok diff --git a/e2e_test/over_window/generated/batch/basic/cross_check.slt.part b/e2e_test/over_window/generated/batch/basic/cross_check.slt.part index 34ef01e43b837..764a60cfba86e 100644 --- a/e2e_test/over_window/generated/batch/basic/cross_check.slt.part +++ b/e2e_test/over_window/generated/batch/basic/cross_check.slt.part @@ -21,6 +21,16 @@ select from v_a natural join v_c; ---- +query i +select + id, out1, out10, out11 +from v_a_d +except +select + id, out1, out10, out11 +from v_a natural join v_d; +---- + query i select id, out3, out4, out7, out8, out9 @@ -31,6 +41,26 @@ select from v_b natural join v_c; ---- +query i +select + id, out3, out4, out10, out11 +from v_b_d +except +select + id, out3, out4, out10, out11 +from v_b natural join v_d; +---- + +query i +select + id, out7, out8, out9, out10, out11 +from v_c_d +except +select + id, out7, out8, out9, out10, out11 +from v_c natural join v_d; +---- + query i select id, out1, out2, out3, out4, out5, out6, out7, out8, out9 @@ -40,3 +70,13 @@ select id, out1, out2, out3, out4, out5, out6, out7, out8, out9 from v_a natural join v_b natural join v_c; ---- + +query i +select + id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11 +from v_a_b_c_d +except +select + id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11 +from v_a natural join v_b natural join v_c natural join v_d; +---- diff --git a/e2e_test/over_window/generated/batch/basic/mod.slt.part b/e2e_test/over_window/generated/batch/basic/mod.slt.part index a8c74b16bf790..1c5ff0f9b460c 100644 --- a/e2e_test/over_window/generated/batch/basic/mod.slt.part +++ b/e2e_test/over_window/generated/batch/basic/mod.slt.part @@ -35,6 +35,14 @@ select * from v_c order by id; 100003 100 208 2 723 807 723 NULL NULL NULL NULL 100004 103 200 2 702 808 702 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2124 +100002 100 200 2 700 806 1 2124 +100003 100 208 2 723 807 1 2124 +100004 103 200 2 702 808 2 702 + include ./cross_check.slt.part statement ok @@ -72,6 +80,16 @@ select * from v_c order by id; 100005 100 200 3 717 810 717 700 700 NULL NULL 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2124 +100002 100 200 2 700 806 1 2841 +100003 100 208 2 723 807 1 2841 +100004 103 200 2 702 808 2 702 +100005 100 200 3 717 810 1 2140 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part statement ok @@ -113,6 +131,16 @@ select * from v_c order by id; 100005 100 200 1 717 810 717 723 701 806 806 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2940 +100002 100 200 2 799 806 1 2940 +100003 100 200 2 723 807 1 2940 +100004 103 200 2 702 808 2 702 +100005 100 200 1 717 810 1 2940 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part statement ok @@ -139,6 +167,13 @@ select * from v_c order by id; 100005 100 200 1 717 810 717 701 701 NULL NULL 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 1418 +100005 100 200 1 717 810 1 1418 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part include ./teardown.slt.part diff --git a/e2e_test/over_window/generated/batch/basic/setup.slt.part b/e2e_test/over_window/generated/batch/basic/setup.slt.part index 2ffc1b055334d..578a43df88113 100644 --- a/e2e_test/over_window/generated/batch/basic/setup.slt.part +++ b/e2e_test/over_window/generated/batch/basic/setup.slt.part @@ -40,6 +40,15 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +# range frame +statement ok +create view v_d as +select + * + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create view v_a_b as select @@ -59,6 +68,15 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +statement ok +create view v_a_d as +select + * + , first_value(v1) over (partition by p1, p2 order by time, id rows 3 preceding) as out1 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create view v_b_c as select @@ -70,6 +88,27 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +statement ok +create view v_b_d as +select + * + , sum(v1) over (partition by p1, p2 order by time, id rows between unbounded preceding and current row) as out3 + , min(v1) over (partition by p1, p2 order by time, id rows between current row and unbounded following) as out4 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + +statement ok +create view v_c_d as +select + * + , lag(v1) over (partition by p1, p2 order by time, id) as out7 + , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 + , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create view v_a_b_c as select @@ -84,3 +123,20 @@ select , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; + +statement ok +create view v_a_b_c_d as +select + * + , first_value(v1) over (partition by p1, p2 order by time, id rows 3 preceding) as out1 + , avg(v1) over (partition by p1) as out2 + , sum(v1) over (partition by p1, p2 order by time, id rows between unbounded preceding and current row) as out3 + , min(v1) over (partition by p1, p2 order by time, id rows between current row and unbounded following) as out4 + , lag(v1, 0) over (partition by p1 order by id) as out5 + , lag(v1, 1) over (partition by p1, p2 order by id) as out6 + , lag(v1) over (partition by p1, p2 order by time, id) as out7 + , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 + , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; diff --git a/e2e_test/over_window/generated/batch/basic/teardown.slt.part b/e2e_test/over_window/generated/batch/basic/teardown.slt.part index 97f416dd49c9e..b0314f2ce346b 100644 --- a/e2e_test/over_window/generated/batch/basic/teardown.slt.part +++ b/e2e_test/over_window/generated/batch/basic/teardown.slt.part @@ -9,17 +9,32 @@ drop view v_b; statement ok drop view v_c; +statement ok +drop view v_d; + statement ok drop view v_a_b; +statement ok +drop view v_a_c; + +statement ok +drop view v_a_d; + statement ok drop view v_b_c; statement ok -drop view v_a_c; +drop view v_b_d; + +statement ok +drop view v_c_d; statement ok drop view v_a_b_c; +statement ok +drop view v_a_b_c_d; + statement ok drop table t; diff --git a/e2e_test/over_window/generated/streaming/basic/cross_check.slt.part b/e2e_test/over_window/generated/streaming/basic/cross_check.slt.part index 34ef01e43b837..764a60cfba86e 100644 --- a/e2e_test/over_window/generated/streaming/basic/cross_check.slt.part +++ b/e2e_test/over_window/generated/streaming/basic/cross_check.slt.part @@ -21,6 +21,16 @@ select from v_a natural join v_c; ---- +query i +select + id, out1, out10, out11 +from v_a_d +except +select + id, out1, out10, out11 +from v_a natural join v_d; +---- + query i select id, out3, out4, out7, out8, out9 @@ -31,6 +41,26 @@ select from v_b natural join v_c; ---- +query i +select + id, out3, out4, out10, out11 +from v_b_d +except +select + id, out3, out4, out10, out11 +from v_b natural join v_d; +---- + +query i +select + id, out7, out8, out9, out10, out11 +from v_c_d +except +select + id, out7, out8, out9, out10, out11 +from v_c natural join v_d; +---- + query i select id, out1, out2, out3, out4, out5, out6, out7, out8, out9 @@ -40,3 +70,13 @@ select id, out1, out2, out3, out4, out5, out6, out7, out8, out9 from v_a natural join v_b natural join v_c; ---- + +query i +select + id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11 +from v_a_b_c_d +except +select + id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11 +from v_a natural join v_b natural join v_c natural join v_d; +---- diff --git a/e2e_test/over_window/generated/streaming/basic/mod.slt.part b/e2e_test/over_window/generated/streaming/basic/mod.slt.part index a8c74b16bf790..1c5ff0f9b460c 100644 --- a/e2e_test/over_window/generated/streaming/basic/mod.slt.part +++ b/e2e_test/over_window/generated/streaming/basic/mod.slt.part @@ -35,6 +35,14 @@ select * from v_c order by id; 100003 100 208 2 723 807 723 NULL NULL NULL NULL 100004 103 200 2 702 808 702 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2124 +100002 100 200 2 700 806 1 2124 +100003 100 208 2 723 807 1 2124 +100004 103 200 2 702 808 2 702 + include ./cross_check.slt.part statement ok @@ -72,6 +80,16 @@ select * from v_c order by id; 100005 100 200 3 717 810 717 700 700 NULL NULL 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2124 +100002 100 200 2 700 806 1 2841 +100003 100 208 2 723 807 1 2841 +100004 103 200 2 702 808 2 702 +100005 100 200 3 717 810 1 2140 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part statement ok @@ -113,6 +131,16 @@ select * from v_c order by id; 100005 100 200 1 717 810 717 723 701 806 806 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2940 +100002 100 200 2 799 806 1 2940 +100003 100 200 2 723 807 1 2940 +100004 103 200 2 702 808 2 702 +100005 100 200 1 717 810 1 2940 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part statement ok @@ -139,6 +167,13 @@ select * from v_c order by id; 100005 100 200 1 717 810 717 701 701 NULL NULL 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 1418 +100005 100 200 1 717 810 1 1418 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part include ./teardown.slt.part diff --git a/e2e_test/over_window/generated/streaming/basic/setup.slt.part b/e2e_test/over_window/generated/streaming/basic/setup.slt.part index cc46c4066f0f4..b4457cde0419b 100644 --- a/e2e_test/over_window/generated/streaming/basic/setup.slt.part +++ b/e2e_test/over_window/generated/streaming/basic/setup.slt.part @@ -40,6 +40,15 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +# range frame +statement ok +create materialized view v_d as +select + * + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create materialized view v_a_b as select @@ -59,6 +68,15 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +statement ok +create materialized view v_a_d as +select + * + , first_value(v1) over (partition by p1, p2 order by time, id rows 3 preceding) as out1 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create materialized view v_b_c as select @@ -70,6 +88,27 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +statement ok +create materialized view v_b_d as +select + * + , sum(v1) over (partition by p1, p2 order by time, id rows between unbounded preceding and current row) as out3 + , min(v1) over (partition by p1, p2 order by time, id rows between current row and unbounded following) as out4 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + +statement ok +create materialized view v_c_d as +select + * + , lag(v1) over (partition by p1, p2 order by time, id) as out7 + , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 + , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create materialized view v_a_b_c as select @@ -84,3 +123,20 @@ select , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; + +statement ok +create materialized view v_a_b_c_d as +select + * + , first_value(v1) over (partition by p1, p2 order by time, id rows 3 preceding) as out1 + , avg(v1) over (partition by p1) as out2 + , sum(v1) over (partition by p1, p2 order by time, id rows between unbounded preceding and current row) as out3 + , min(v1) over (partition by p1, p2 order by time, id rows between current row and unbounded following) as out4 + , lag(v1, 0) over (partition by p1 order by id) as out5 + , lag(v1, 1) over (partition by p1, p2 order by id) as out6 + , lag(v1) over (partition by p1, p2 order by time, id) as out7 + , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 + , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; diff --git a/e2e_test/over_window/generated/streaming/basic/teardown.slt.part b/e2e_test/over_window/generated/streaming/basic/teardown.slt.part index 2089fefcac249..041677a88c268 100644 --- a/e2e_test/over_window/generated/streaming/basic/teardown.slt.part +++ b/e2e_test/over_window/generated/streaming/basic/teardown.slt.part @@ -9,17 +9,32 @@ drop materialized view v_b; statement ok drop materialized view v_c; +statement ok +drop materialized view v_d; + statement ok drop materialized view v_a_b; +statement ok +drop materialized view v_a_c; + +statement ok +drop materialized view v_a_d; + statement ok drop materialized view v_b_c; statement ok -drop materialized view v_a_c; +drop materialized view v_b_d; + +statement ok +drop materialized view v_c_d; statement ok drop materialized view v_a_b_c; +statement ok +drop materialized view v_a_b_c_d; + statement ok drop table t; diff --git a/e2e_test/over_window/templates/basic/cross_check.slt.part b/e2e_test/over_window/templates/basic/cross_check.slt.part index b2a8041257143..0731d52637f45 100644 --- a/e2e_test/over_window/templates/basic/cross_check.slt.part +++ b/e2e_test/over_window/templates/basic/cross_check.slt.part @@ -19,6 +19,16 @@ select from v_a natural join v_c; ---- +query i +select + id, out1, out10, out11 +from v_a_d +except +select + id, out1, out10, out11 +from v_a natural join v_d; +---- + query i select id, out3, out4, out7, out8, out9 @@ -29,6 +39,26 @@ select from v_b natural join v_c; ---- +query i +select + id, out3, out4, out10, out11 +from v_b_d +except +select + id, out3, out4, out10, out11 +from v_b natural join v_d; +---- + +query i +select + id, out7, out8, out9, out10, out11 +from v_c_d +except +select + id, out7, out8, out9, out10, out11 +from v_c natural join v_d; +---- + query i select id, out1, out2, out3, out4, out5, out6, out7, out8, out9 @@ -38,3 +68,13 @@ select id, out1, out2, out3, out4, out5, out6, out7, out8, out9 from v_a natural join v_b natural join v_c; ---- + +query i +select + id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11 +from v_a_b_c_d +except +select + id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11 +from v_a natural join v_b natural join v_c natural join v_d; +---- diff --git a/e2e_test/over_window/templates/basic/mod.slt.part b/e2e_test/over_window/templates/basic/mod.slt.part index 421f5a911f468..d2ed6fdc1e107 100644 --- a/e2e_test/over_window/templates/basic/mod.slt.part +++ b/e2e_test/over_window/templates/basic/mod.slt.part @@ -33,6 +33,14 @@ select * from v_c order by id; 100003 100 208 2 723 807 723 NULL NULL NULL NULL 100004 103 200 2 702 808 702 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2124 +100002 100 200 2 700 806 1 2124 +100003 100 208 2 723 807 1 2124 +100004 103 200 2 702 808 2 702 + include ./cross_check.slt.part statement ok @@ -70,6 +78,16 @@ select * from v_c order by id; 100005 100 200 3 717 810 717 700 700 NULL NULL 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2124 +100002 100 200 2 700 806 1 2841 +100003 100 208 2 723 807 1 2841 +100004 103 200 2 702 808 2 702 +100005 100 200 3 717 810 1 2140 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part statement ok @@ -111,6 +129,16 @@ select * from v_c order by id; 100005 100 200 1 717 810 717 723 701 806 806 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2940 +100002 100 200 2 799 806 1 2940 +100003 100 200 2 723 807 1 2940 +100004 103 200 2 702 808 2 702 +100005 100 200 1 717 810 1 2940 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part statement ok @@ -137,6 +165,13 @@ select * from v_c order by id; 100005 100 200 1 717 810 717 701 701 NULL NULL 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 1418 +100005 100 200 1 717 810 1 1418 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part include ./teardown.slt.part diff --git a/e2e_test/over_window/templates/basic/setup.slt.part b/e2e_test/over_window/templates/basic/setup.slt.part index d989d50029430..f08fc7e4ea20d 100644 --- a/e2e_test/over_window/templates/basic/setup.slt.part +++ b/e2e_test/over_window/templates/basic/setup.slt.part @@ -38,6 +38,15 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +# range frame +statement ok +create $view_type v_d as +select + * + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create $view_type v_a_b as select @@ -57,6 +66,15 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +statement ok +create $view_type v_a_d as +select + * + , first_value(v1) over (partition by p1, p2 order by time, id rows 3 preceding) as out1 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create $view_type v_b_c as select @@ -68,6 +86,27 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +statement ok +create $view_type v_b_d as +select + * + , sum(v1) over (partition by p1, p2 order by time, id rows between unbounded preceding and current row) as out3 + , min(v1) over (partition by p1, p2 order by time, id rows between current row and unbounded following) as out4 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + +statement ok +create $view_type v_c_d as +select + * + , lag(v1) over (partition by p1, p2 order by time, id) as out7 + , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 + , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create $view_type v_a_b_c as select @@ -82,3 +121,20 @@ select , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; + +statement ok +create $view_type v_a_b_c_d as +select + * + , first_value(v1) over (partition by p1, p2 order by time, id rows 3 preceding) as out1 + , avg(v1) over (partition by p1) as out2 + , sum(v1) over (partition by p1, p2 order by time, id rows between unbounded preceding and current row) as out3 + , min(v1) over (partition by p1, p2 order by time, id rows between current row and unbounded following) as out4 + , lag(v1, 0) over (partition by p1 order by id) as out5 + , lag(v1, 1) over (partition by p1, p2 order by id) as out6 + , lag(v1) over (partition by p1, p2 order by time, id) as out7 + , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 + , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; diff --git a/e2e_test/over_window/templates/basic/teardown.slt.part b/e2e_test/over_window/templates/basic/teardown.slt.part index 89915395703ad..a9bf9252c385a 100644 --- a/e2e_test/over_window/templates/basic/teardown.slt.part +++ b/e2e_test/over_window/templates/basic/teardown.slt.part @@ -7,17 +7,32 @@ drop $view_type v_b; statement ok drop $view_type v_c; +statement ok +drop $view_type v_d; + statement ok drop $view_type v_a_b; +statement ok +drop $view_type v_a_c; + +statement ok +drop $view_type v_a_d; + statement ok drop $view_type v_b_c; statement ok -drop $view_type v_a_c; +drop $view_type v_b_d; + +statement ok +drop $view_type v_c_d; statement ok drop $view_type v_a_b_c; +statement ok +drop $view_type v_a_b_c_d; + statement ok drop table t; diff --git a/e2e_test/source/basic/ddl.slt b/e2e_test/source/basic/ddl.slt index f959ce7fa49a7..6e640e047d4c2 100644 --- a/e2e_test/source/basic/ddl.slt +++ b/e2e_test/source/basic/ddl.slt @@ -134,7 +134,7 @@ create source s ( statement ok create materialized view mv_1 as select * from s -statement error other relation\(s\) depend on it +statement error Permission denied drop source s statement ok diff --git a/e2e_test/source/basic/old_row_format_syntax/ddl.slt b/e2e_test/source/basic/old_row_format_syntax/ddl.slt index 2ec7239e61d2c..d5c41d4ded878 100644 --- a/e2e_test/source/basic/old_row_format_syntax/ddl.slt +++ b/e2e_test/source/basic/old_row_format_syntax/ddl.slt @@ -90,7 +90,7 @@ create source s ( statement ok create materialized view mv_1 as select * from s -statement error other relation\(s\) depend on it +statement error Permission denied drop source s statement ok diff --git a/e2e_test/udf/sql_udf.slt b/e2e_test/udf/sql_udf.slt index 758ec43ca53fc..bfae0f7045888 100644 --- a/e2e_test/udf/sql_udf.slt +++ b/e2e_test/udf/sql_udf.slt @@ -1,325 +1,351 @@ statement ok SET RW_IMPLICIT_FLUSH TO true; -# Create an anonymous function with double dollar as clause +############################################################# +# Basic tests for sql udf with [unnamed / named] parameters # +############################################################# + +# Create a sql udf function with unnamed parameters with double dollar as clause statement ok create function add(INT, INT) returns int language sql as $$select $1 + $2$$; -# Create an anonymous function with single quote as clause +query I +select add(1, -1); +---- +0 + +# Create a sql udf function with unnamed parameters with single quote as clause statement ok create function sub(INT, INT) returns int language sql as 'select $1 - $2'; -# Create an anonymous function that calls other pre-defined sql udfs +query I +select sub(1, 1); +---- +0 + +# Create a sql udf function with unamed parameters that calls other pre-defined sql udfs statement ok create function add_sub_binding() returns int language sql as 'select add(1, 1) + sub(2, 2)'; -# Create a named sql udf +query I +select add_sub_binding(); +---- +2 + +# Use them all together +query III +select add(1, -1), sub(1, 1), add_sub_binding(); +---- +0 0 2 + +# Create a sql udf with named parameters with single quote as clause statement ok create function add_named(a INT, b INT) returns int language sql as 'select a + b'; -# Create another named sql udf +query I +select add_named(1, -1); +---- +0 + +# Create another sql udf with named parameters with double dollar as clause statement ok -create function sub_named(a INT, b INT) returns int language sql as 'select a - b'; +create function sub_named(a INT, b INT) returns int language sql as $$select a - b$$; + +query I +select sub_named(1, 1); +---- +0 -# Mixed parameter with named / anonymous parameters +# Mixed with named / unnamed parameters statement ok create function add_sub_mix(INT, a INT, INT) returns int language sql as 'select $1 - a + $3'; -# Mixed parameter with calling inner sql udfs -# statement ok -# create function add_sub_mix_wrapper(INT, a INT, INT) returns int language sql as 'select add($1, a) + a + sub(a, $3)'; +query I +select add_sub_mix(1, 2, 3); +---- +2 -# Named sql udf with corner case +# Call sql udf with unnamed parameters inside sql udf with named parameters statement ok -create function corner_case(INT, a INT, INT) returns varchar language sql as $$select '$1 + a + $3'$$; +create function add_named_wrapper(a INT, b INT) returns int language sql as 'select add(a, b)'; -# Named sql udf with invalid parameter in body definition -# Will be rejected at creation time -statement error failed to find named parameter aa -create function unknown_parameter(a INT) returns int language sql as 'select a + aa + a'; +query I +select add_named_wrapper(1, -1); +---- +0 -# Call anonymous sql udf inside named sql udf +# Create a sql udf with unnamed parameters with return expression statement ok -create function add_named_wrapper(a INT, b INT) returns int language sql as 'select add(a, b)'; +create function add_return(INT, INT) returns int language sql return $1 + $2; + +query I +select add_return(1, 1); +---- +2 -# Create an anonymous function that calls built-in functions -# Note that double dollar signs should be used otherwise the parsing will fail, as illutrates below statement ok -create function call_regexp_replace() returns varchar language sql as $$select regexp_replace('💩💩💩💩💩foo🤔️bar亲爱的😭baz这不是爱情❤️‍🔥', 'baz(...)', '这是🥵', 'ic')$$; +create function add_return_binding() returns int language sql return add_return(1, 1) + add_return(1, 1); -statement error Expected end of statement, found: 💩 -create function call_regexp_replace() returns varchar language sql as 'select regexp_replace('💩💩💩💩💩foo🤔️bar亲爱的😭baz这不是爱情❤️‍🔥', 'baz(...)', '这是🥵', 'ic')'; +query I +select add_return_binding(); +---- +4 -# Create an anonymous function with return expression statement ok -create function add_return(INT, INT) returns int language sql return $1 + $2; +create function print(INT) returns int language sql as 'select $1'; +query T +select print(114514); +---- +114514 + +# Multiple type interleaving sql udf statement ok -create function add_return_binding() returns int language sql return add_return(1, 1) + add_return(1, 1); +create function add_sub(INT, FLOAT, INT) returns float language sql as $$select -$1 + $2 - $3$$; -# Recursive definition can NOT be accepted at present due to semantic check -statement error failed to conduct semantic check, please see if you are calling non-existence functions -create function recursive(INT, INT) returns int language sql as 'select recursive($1, $2) + recursive($1, $2)'; +query I +select add_sub(1, 5.1415926, 1); +---- +3.1415926 -# Complex but error-prone definition, recursive & normal sql udfs interleaving -statement error failed to conduct semantic check, please see if you are calling non-existence functions -create function recursive_non_recursive(INT, INT) returns int language sql as 'select recursive($1, $2) + sub($1, $2)'; +query III +select add(1, -1), sub(1, 1), add_sub(1, 5.1415926, 1); +---- +0 0 3.1415926 -# Recursive corner case +# Complex types interleaving statement ok -create function foo(INT) returns varchar language sql as $$select 'foo(INT)'$$; +create function add_sub_types(INT, BIGINT, FLOAT, DECIMAL, REAL) returns double language sql as 'select $1 + $2 - $3 + $4 + $5'; + +query I +select add_sub_types(1, 1919810114514, 3.1415926, 1.123123, 101010.191919); +---- +1919810215523.1734494 + +statement ok +create function add_sub_return(INT, FLOAT, INT) returns float language sql return -$1 + $2 - $3; + +query I +select add_sub_return(1, 5.1415926, 1); +---- +3.1415926 # Create a wrapper function for `add` & `sub` statement ok create function add_sub_wrapper(INT, INT) returns int language sql as 'select add($1, $2) + sub($1, $2) + 114512'; -# Create a valid recursive function -# Please note we do NOT support actual running the recursive sql udf at present -statement error failed to conduct semantic check, please see if you are calling non-existence functions -create function fib(INT) returns int - language sql as 'select case - when $1 = 0 then 0 - when $1 = 1 then 1 - when $1 = 2 then 1 - when $1 = 3 then 2 - else fib($1 - 1) + fib($1 - 2) - end;'; - -# The execution will eventually exceed the pre-defined max stack depth -# statement error function fib calling stack depth limit exceeded -# select fib(100); +query I +select add_sub_wrapper(1, 1); +---- +114514 -# Currently create a materialized view with a recursive sql udf will be rejected -# statement error function fib calling stack depth limit exceeded -# create materialized view foo_mv as select fib(100); +########################################################## +# Basic sql udfs integrated with the use of mock tables # +# P.S. This is also a simulation of real world use cases # +########################################################## statement ok -create function regexp_replace_wrapper(varchar) returns varchar language sql as $$select regexp_replace($1, 'baz(...)', '这是🥵', 'ic')$$; +create table t1 (c1 INT, c2 INT); statement ok -create function print(INT) returns int language sql as 'select $1'; +create table t2 (c1 INT, c2 FLOAT, c3 INT); -# Adjust the input value of the calling function (i.e., `print` here) with the actual input parameter +# Special table for named sql udf statement ok -create function print_add_one(INT) returns int language sql as 'select print($1 + 1)'; +create table t3 (a INT, b INT); statement ok -create function print_add_two(INT) returns int language sql as 'select print($1 + $1)'; - -# Calling a non-existence function -statement error failed to conduct semantic check, please see if you are calling non-existence functions -create function non_exist(INT) returns int language sql as 'select yo(114514)'; - -# Try to create an anonymous sql udf whose return type mismatches with the sql body definition -statement error return type mismatch detected -create function type_mismatch(INT) returns varchar language sql as 'select $1 + 114514 + $1'; +insert into t1 values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5); -# A valid example statement ok -create function type_match(INT) returns varchar language sql as $$select '$1 + 114514 + $1'$$; +insert into t2 values (1, 3.14, 2), (2, 4.44, 5), (20, 10.30, 02); -query T -select type_match(114514); ----- -$1 + 114514 + $1 +statement ok +insert into t3 values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5); -# Call the defined anonymous sql udfs query I -select add(1, -1); +select c1, c2, add_return(c1, c2) from t1 order by c1 asc; ---- -0 +1 1 2 +2 2 4 +3 3 6 +4 4 8 +5 5 10 -query I -select sub(1, 1); +query III +select sub(c1, c2), c1, c2, add(c1, c2) from t1 order by c1 asc; ---- -0 +0 1 1 2 +0 2 2 4 +0 3 3 6 +0 4 4 8 +0 5 5 10 -# Call the defined named sql udfs -query I -select add_named(1, -1); +query IIIIII +select c1, c2, c3, add(c1, c3), sub(c1, c3), add_sub(c1, c2, c3) from t2 order by c1 asc; ---- -0 +1 3.14 2 3 -1 0.14000000000000012 +2 4.44 5 7 -3 -2.5599999999999996 +20 10.3 2 22 18 -11.7 -query I -select sub_named(1, 1); +query IIIIII +select c1, c2, c3, add(c1, c3), sub(c1, c3), add_sub_return(c1, c2, c3) from t2 order by c1 asc; ---- -0 +1 3.14 2 3 -1 0.14000000000000012 +2 4.44 5 7 -3 -2.5599999999999996 +20 10.3 2 22 18 -11.7 query I -select add_sub_mix(1, 2, 3); +select add_named(a, b) from t3 order by a asc; ---- 2 +4 +6 +8 +10 -query T -select corner_case(1, 2, 3); ----- -$1 + a + $3 +################################ +# Corner & Special cases tests # +################################ -query I -select add_named_wrapper(1, -1); ----- -0 +# Mixed parameter with calling inner sql udfs +statement ok +create function add_sub_mix_wrapper(INT, a INT, INT) returns int language sql as 'select add($1, a) + a + sub(a, $3)'; query I -select add_sub_binding(); +select add_sub_mix_wrapper(1, 2, 3); ---- -2 +4 -query III -select add(1, -1), sub(1, 1), add_sub_binding(); ----- -0 0 2 +# Named sql udf with corner case +statement ok +create function corner_case(INT, a INT, INT) returns varchar language sql as $$select '$1 + a + $3'$$; -query I -select add_return(1, 1); +query T +select corner_case(1, 2, 3); ---- -2 +$1 + a + $3 -query I -select add_return_binding(); ----- -4 +# Create a sql udf with unnamed parameters that calls built-in functions +# Note that double dollar signs should be used otherwise the parsing will fail, as illutrates below +statement ok +create function call_regexp_replace() returns varchar language sql as $$select regexp_replace('💩💩💩💩💩foo🤔️bar亲爱的😭baz这不是爱情❤️‍🔥', 'baz(...)', '这是🥵', 'ic')$$; query T select call_regexp_replace(); ---- 💩💩💩💩💩foo🤔️bar亲爱的😭这是🥵爱情❤️‍🔥 +statement ok +create function regexp_replace_wrapper(varchar) returns varchar language sql as $$select regexp_replace($1, 'baz(...)', '这是🥵', 'ic')$$; + query T select regexp_replace_wrapper('💩💩💩💩💩foo🤔️bar亲爱的😭baz这不是爱情❤️‍🔥'); ---- 💩💩💩💩💩foo🤔️bar亲爱的😭这是🥵爱情❤️‍🔥 +# Recursive corner case (i.e., valid definition should not be rejected) +statement ok +create function foo(INT) returns varchar language sql as $$select 'foo(INT)'$$; + query T select foo(114514); ---- foo(INT) -# Rejected deep calling stack -# statement error function recursive calling stack depth limit exceeded -# select recursive(1, 1); - -# Same as above -# statement error function recursive calling stack depth limit exceeded -# select recursive_non_recursive(1, 1); +# Adjust the input value of the calling function (i.e., `print` here) with the actual input parameter +statement ok +create function print_add_one(INT) returns int language sql as 'select print($1 + 1)'; -query I -select add_sub_wrapper(1, 1); ----- -114514 +statement ok +create function print_add_two(INT) returns int language sql as 'select print($1 + $1)'; query III select print_add_one(1), print_add_one(114513), print_add_two(2); ---- 2 114514 4 -# Create a mock table for anonymous sql udf +# Note: the valid example of `type_mismatch` in the below test section statement ok -create table t1 (c1 INT, c2 INT); +create function type_match(INT) returns varchar language sql as $$select '$1 + 114514 + $1'$$; -# Create a mock table for named sql udf -statement ok -create table t3 (a INT, b INT); +query T +select type_match(114514); +---- +$1 + 114514 + $1 -# Insert some data into the mock table -statement ok -insert into t1 values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5); +################################################################################# +# Invalid definition (and maybe not yet supported features 🤪) / use case tests # +################################################################################# -statement ok -insert into t3 values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5); +# Named sql udf with invalid parameter in body definition +# Will be rejected at creation time +statement error failed to find named parameter aa +create function unknown_parameter(a INT) returns int language sql as 'select a + aa + a'; -query I -select add_named(a, b) from t3 order by a asc; ----- -2 -4 -6 -8 -10 +statement error Expected end of statement, found: 💩 +create function call_regexp_replace() returns varchar language sql as 'select regexp_replace('💩💩💩💩💩foo🤔️bar亲爱的😭baz这不是爱情❤️‍🔥', 'baz(...)', '这是🥵', 'ic')'; -query III -select sub(c1, c2), c1, c2, add(c1, c2) from t1 order by c1 asc; ----- -0 1 1 2 -0 2 2 4 -0 3 3 6 -0 4 4 8 -0 5 5 10 +# Recursive definition can NOT be accepted at present due to semantic check +statement error failed to conduct semantic check, please see if you are calling non-existent functions +create function recursive(INT, INT) returns int language sql as 'select recursive($1, $2) + recursive($1, $2)'; -query I -select c1, c2, add_return(c1, c2) from t1 order by c1 asc; ----- -1 1 2 -2 2 4 -3 3 6 -4 4 8 -5 5 10 +# Complex but error-prone definition, recursive & normal sql udfs interleaving +statement error failed to conduct semantic check, please see if you are calling non-existent functions +create function recursive_non_recursive(INT, INT) returns int language sql as 'select recursive($1, $2) + sub($1, $2)'; -# Recursive sql udf with normal table +# Create a valid recursive function +# Please note we do NOT support actual running the recursive sql udf at present +statement error failed to conduct semantic check, please see if you are calling non-existent functions +create function fib(INT) returns int + language sql as 'select case + when $1 = 0 then 0 + when $1 = 1 then 1 + when $1 = 2 then 1 + when $1 = 3 then 2 + else fib($1 - 1) + fib($1 - 2) + end;'; + +# The execution will eventually exceed the pre-defined max stack depth # statement error function fib calling stack depth limit exceeded -# select fib(c1) from t1; +# select fib(100); -# Recursive sql udf with materialized view +# Currently create a materialized view with a recursive sql udf will be rejected # statement error function fib calling stack depth limit exceeded -# create materialized view bar_mv as select fib(c1) from t1; +# create materialized view foo_mv as select fib(100); + +# Calling a non-existent function +statement error failed to conduct semantic check, please see if you are calling non-existent functions +create function non_exist(INT) returns int language sql as 'select yo(114514)'; + +# Try to create an sql udf with unnamed parameters whose return type mismatches with the sql body definition +statement error return type mismatch detected +create function type_mismatch(INT) returns varchar language sql as 'select $1 + 114514 + $1'; # Invalid function body syntax statement error Expected an expression:, found: EOF at the end create function add_error(INT, INT) returns int language sql as $$select $1 + $2 +$$; -# Multiple type interleaving sql udf -statement ok -create function add_sub(INT, FLOAT, INT) returns float language sql as $$select -$1 + $2 - $3$$; - -# Complex types interleaving -statement ok -create function add_sub_types(INT, BIGINT, FLOAT, DECIMAL, REAL) returns double language sql as 'select $1 + $2 - $3 + $4 + $5'; - -statement ok -create function add_sub_return(INT, FLOAT, INT) returns float language sql return -$1 + $2 - $3; - -query I -select add_sub(1, 5.1415926, 1); ----- -3.1415926 - -query I -select add_sub_return(1, 5.1415926, 1); ----- -3.1415926 - -query III -select add(1, -1), sub(1, 1), add_sub(1, 5.1415926, 1); ----- -0 0 3.1415926 - -query I -select add_sub_types(1, 1919810114514, 3.1415926, 1.123123, 101010.191919); ----- -1919810215523.1734494 +# Rejected deep calling stack +# statement error function recursive calling stack depth limit exceeded +# select recursive(1, 1); -# Create another mock table -statement ok -create table t2 (c1 INT, c2 FLOAT, c3 INT); +# Same as above +# statement error function recursive calling stack depth limit exceeded +# select recursive_non_recursive(1, 1); -statement ok -insert into t2 values (1, 3.14, 2), (2, 4.44, 5), (20, 10.30, 02); +# Recursive sql udf with normal table +# statement error function fib calling stack depth limit exceeded +# select fib(c1) from t1; -query IIIIII -select c1, c2, c3, add(c1, c3), sub(c1, c3), add_sub(c1, c2, c3) from t2 order by c1 asc; ----- -1 3.14 2 3 -1 0.14000000000000012 -2 4.44 5 7 -3 -2.5599999999999996 -20 10.3 2 22 18 -11.7 +# Recursive sql udf with materialized view +# statement error function fib calling stack depth limit exceeded +# create materialized view bar_mv as select fib(c1) from t1; -query IIIIII -select c1, c2, c3, add(c1, c3), sub(c1, c3), add_sub_return(c1, c2, c3) from t2 order by c1 asc; ----- -1 3.14 2 3 -1 0.14000000000000012 -2 4.44 5 7 -3 -2.5599999999999996 -20 10.3 2 22 18 -11.7 +################################################## +# Clean up the funtions / mock tables at the end # +################################################## -# Drop the functions statement ok drop function add; @@ -347,21 +373,12 @@ drop function call_regexp_replace; statement ok drop function add_sub_wrapper; -# statement ok -# drop function recursive; - statement ok drop function foo; -# statement ok -# drop function recursive_non_recursive; - statement ok drop function add_sub_types; -# statement ok -# drop function fib; - statement ok drop function print; @@ -392,7 +409,9 @@ drop function add_named_wrapper; statement ok drop function type_match; -# Drop the mock table +statement ok +drop function add_sub_mix_wrapper; + statement ok drop table t1; diff --git a/proto/expr.proto b/proto/expr.proto index e26b725b49737..14f9eb8c102cd 100644 --- a/proto/expr.proto +++ b/proto/expr.proto @@ -418,9 +418,11 @@ message AggCall { message WindowFrame { enum Type { TYPE_UNSPECIFIED = 0; - // RANGE = 1; - TYPE_ROWS = 2; - // GROUPS = 3; + + TYPE_ROWS_LEGACY = 2 [deprecated = true]; // Deprecated since we introduced `RANGE` frame. + + TYPE_ROWS = 5; + TYPE_RANGE = 10; } enum BoundType { BOUND_TYPE_UNSPECIFIED = 0; @@ -430,7 +432,9 @@ message WindowFrame { BOUND_TYPE_FOLLOWING = 4; BOUND_TYPE_UNBOUNDED_FOLLOWING = 5; } + // Deprecated since we introduced `RANGE` frame. message Bound { + option deprecated = true; BoundType type = 1; oneof offset { uint64 integer = 2; @@ -444,11 +448,38 @@ message WindowFrame { // EXCLUSION_TIES = 3; EXCLUSION_NO_OTHERS = 4; } + message RowsFrameBounds { + RowsFrameBound start = 1; + RowsFrameBound end = 2; + } + message RowsFrameBound { + BoundType type = 1; + optional uint64 offset = 3; + } + message RangeFrameBounds { + RangeFrameBound start = 1; + RangeFrameBound end = 2; + + data.DataType order_data_type = 10; + common.OrderType order_type = 15; + data.DataType offset_data_type = 20; + } + message RangeFrameBound { + BoundType type = 1; + optional data.Datum offset = 3; + } Type type = 1; - Bound start = 2; - Bound end = 3; + + Bound start = 2 [deprecated = true]; // Deprecated since we introduced `RANGE` frame. + Bound end = 3 [deprecated = true]; // Deprecated since we introduced `RANGE` frame. + Exclusion exclusion = 4; + + oneof bounds { + RowsFrameBounds rows = 10; + RangeFrameBounds range = 15; + } } message WindowFunction { diff --git a/proto/plan_common.proto b/proto/plan_common.proto index c85ea9ef8a603..a914ab9d2da25 100644 --- a/proto/plan_common.proto +++ b/proto/plan_common.proto @@ -61,7 +61,7 @@ message ColumnDesc { ColumnDescVersion version = 10; - AdditionalColumn additional_columns = 11; + AdditionalColumn additional_column = 11; } message ColumnCatalog { diff --git a/proto/telemetry.proto b/proto/telemetry.proto new file mode 100644 index 0000000000000..2da24cc823085 --- /dev/null +++ b/proto/telemetry.proto @@ -0,0 +1,95 @@ +syntax = "proto3"; + +package telemetry; + +enum MetaBackend { + META_BACKEND_UNSPECIFIED = 0; + META_BACKEND_MEMORY = 1; + META_BACKEND_ETCD = 2; + META_BACKEND_RDB = 3; +} + +enum TelemetryNodeType { + TELEMETRY_NODE_TYPE_UNSPECIFIED = 0; + TELEMETRY_NODE_TYPE_META = 1; + TELEMETRY_NODE_TYPE_COMPUTE = 2; + TELEMETRY_NODE_TYPE_FRONTEND = 3; + TELEMETRY_NODE_TYPE_COMPACTOR = 4; +} + +message SystemMemory { + uint64 used = 1; + uint64 total = 2; +} + +message SystemOs { + string name = 1; + string version = 2; + string kernel_version = 3; +} + +message SystemCpu { + float available = 1; +} + +message SystemData { + SystemMemory memory = 1; + SystemOs os = 2; + SystemCpu cpu = 3; +} + +// NodeCount represents how many nodes in this cluster +message NodeCount { + uint32 meta = 1; + uint32 compute = 2; + uint32 frontend = 3; + uint32 compactor = 4; +} + +// RwVersion represents the version of RisingWave +message RwVersion { + // Version is the Cargo package version of RisingWave + string rw_version = 1; + // GitSHA is the Git commit SHA of RisingWave + string git_sha = 2; +} + +message ReportBase { + // tracking_id is persistent in meta data + string tracking_id = 1; + // session_id is reset every time node restarts + string session_id = 2; + // system_data is hardware and os info + SystemData system_data = 3; + // up_time is how long the node has been running + uint64 up_time = 4; + // report_time is when the report is created + uint64 report_time = 5; + // node_type is the node that creates the report + TelemetryNodeType node_type = 6; +} + +message MetaReport { + ReportBase base = 1; + // meta_backend is the backend of meta data + MetaBackend meta_backend = 2; + // node_count is the count of each node type + NodeCount node_count = 3; + // rw_version is the version of RisingWave + RwVersion rw_version = 4; + // This field represents the "number of running streaming jobs" + // and is used to indicate whether the cluster is active. + uint32 stream_job_count = 5; +} + +message ComputeReport { + ReportBase base = 1; +} + +message FrontendReport { + ReportBase base = 1; +} + +message CompactorReport { + ReportBase base = 1; +} diff --git a/src/common/src/catalog/column.rs b/src/common/src/catalog/column.rs index 09fd3db09561f..f82e96a80c0e2 100644 --- a/src/common/src/catalog/column.rs +++ b/src/common/src/catalog/column.rs @@ -103,7 +103,7 @@ pub struct ColumnDesc { pub type_name: String, pub generated_or_default_column: Option, pub description: Option, - pub additional_columns: AdditionalColumn, + pub additional_column: AdditionalColumn, pub version: ColumnDescVersion, } @@ -117,7 +117,7 @@ impl ColumnDesc { type_name: String::new(), generated_or_default_column: None, description: None, - additional_columns: AdditionalColumn { column_type: None }, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, } } @@ -131,7 +131,7 @@ impl ColumnDesc { type_name: String::new(), generated_or_default_column: None, description: None, - additional_columns: AdditionalColumn { column_type: None }, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, } } @@ -150,7 +150,7 @@ impl ColumnDesc { type_name: String::new(), generated_or_default_column: None, description: None, - additional_columns: additional_column_type, + additional_column: additional_column_type, version: ColumnDescVersion::Pr13707, } } @@ -170,7 +170,7 @@ impl ColumnDesc { type_name: self.type_name.clone(), generated_or_default_column: self.generated_or_default_column.clone(), description: self.description.clone(), - additional_columns: Some(self.additional_columns.clone()), + additional_column: Some(self.additional_column.clone()), version: self.version as i32, } } @@ -198,7 +198,7 @@ impl ColumnDesc { type_name: "".to_string(), generated_or_default_column: None, description: None, - additional_columns: AdditionalColumn { column_type: None }, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, } } @@ -221,7 +221,7 @@ impl ColumnDesc { type_name: type_name.to_string(), generated_or_default_column: None, description: None, - additional_columns: AdditionalColumn { column_type: None }, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, } } @@ -239,7 +239,7 @@ impl ColumnDesc { type_name: field.type_name.clone(), description: None, generated_or_default_column: None, - additional_columns: AdditionalColumn { column_type: None }, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, } } @@ -265,8 +265,8 @@ impl ColumnDesc { impl From for ColumnDesc { fn from(prost: PbColumnDesc) -> Self { - let additional_columns = prost - .get_additional_columns() + let additional_column = prost + .get_additional_column() .unwrap_or(&AdditionalColumn { column_type: None }) .clone(); let version = prost.version(); @@ -283,7 +283,7 @@ impl From for ColumnDesc { field_descs, generated_or_default_column: prost.generated_or_default_column, description: prost.description.clone(), - additional_columns, + additional_column, version, } } @@ -305,7 +305,7 @@ impl From<&ColumnDesc> for PbColumnDesc { type_name: c.type_name.clone(), generated_or_default_column: c.generated_or_default_column.clone(), description: c.description.clone(), - additional_columns: c.additional_columns.clone().into(), + additional_column: c.additional_column.clone().into(), version: c.version as i32, } } diff --git a/src/common/src/catalog/test_utils.rs b/src/common/src/catalog/test_utils.rs index ca154b9bf0b0b..9930a5717b849 100644 --- a/src/common/src/catalog/test_utils.rs +++ b/src/common/src/catalog/test_utils.rs @@ -35,7 +35,7 @@ impl ColumnDescTestExt for ColumnDesc { column_type: Some(data_type), column_id, name: name.to_string(), - additional_columns: Some(AdditionalColumn { column_type: None }), + additional_column: Some(AdditionalColumn { column_type: None }), version: ColumnDescVersion::Pr13707 as i32, ..Default::default() } @@ -60,7 +60,7 @@ impl ColumnDescTestExt for ColumnDesc { field_descs: fields, generated_or_default_column: None, description: None, - additional_columns: Some(AdditionalColumn { column_type: None }), + additional_column: Some(AdditionalColumn { column_type: None }), version: ColumnDescVersion::Pr13707 as i32, } } diff --git a/src/common/src/telemetry/mod.rs b/src/common/src/telemetry/mod.rs index e25e312013593..382e50d8eb927 100644 --- a/src/common/src/telemetry/mod.rs +++ b/src/common/src/telemetry/mod.rs @@ -17,6 +17,11 @@ pub mod report; use std::time::SystemTime; +use risingwave_pb::telemetry::{ + ReportBase as PbTelemetryReportBase, SystemCpu as PbSystemCpu, SystemData as PbSystemData, + SystemMemory as PbSystemMemory, SystemOs as PbSystemOs, + TelemetryNodeType as PbTelemetryNodeType, +}; use serde::{Deserialize, Serialize}; use sysinfo::System; use thiserror_ext::AsReport; @@ -65,6 +70,19 @@ pub struct TelemetryReportBase { pub node_type: TelemetryNodeType, } +impl From for PbTelemetryReportBase { + fn from(val: TelemetryReportBase) -> Self { + PbTelemetryReportBase { + tracking_id: val.tracking_id, + session_id: val.session_id, + system_data: Some(val.system_data.into()), + up_time: val.up_time, + report_time: val.time_stamp, + node_type: from_telemetry_node_type(val.node_type) as i32, + } + } +} + pub trait TelemetryReport: Serialize {} #[derive(Debug, Serialize, Deserialize)] @@ -155,6 +173,63 @@ pub fn current_timestamp() -> u64 { .as_secs() } +fn from_telemetry_node_type(t: TelemetryNodeType) -> PbTelemetryNodeType { + match t { + TelemetryNodeType::Meta => PbTelemetryNodeType::Meta, + TelemetryNodeType::Compute => PbTelemetryNodeType::Compute, + TelemetryNodeType::Frontend => PbTelemetryNodeType::Frontend, + TelemetryNodeType::Compactor => PbTelemetryNodeType::Compactor, + } +} + +impl From for PbTelemetryNodeType { + fn from(val: TelemetryNodeType) -> Self { + match val { + TelemetryNodeType::Meta => PbTelemetryNodeType::Meta, + TelemetryNodeType::Compute => PbTelemetryNodeType::Compute, + TelemetryNodeType::Frontend => PbTelemetryNodeType::Frontend, + TelemetryNodeType::Compactor => PbTelemetryNodeType::Compactor, + } + } +} + +impl From for PbSystemCpu { + fn from(val: Cpu) -> Self { + PbSystemCpu { + available: val.available, + } + } +} + +impl From for PbSystemMemory { + fn from(val: Memory) -> Self { + PbSystemMemory { + used: val.used as u64, + total: val.total as u64, + } + } +} + +impl From for PbSystemOs { + fn from(val: Os) -> Self { + PbSystemOs { + name: val.name, + kernel_version: val.kernel_version, + version: val.version, + } + } +} + +impl From for PbSystemData { + fn from(val: SystemData) -> Self { + PbSystemData { + memory: Some(val.memory.into()), + os: Some(val.os.into()), + cpu: Some(val.cpu.into()), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/common/src/types/interval.rs b/src/common/src/types/interval.rs index 44470e29acf77..0160ff7955591 100644 --- a/src/common/src/types/interval.rs +++ b/src/common/src/types/interval.rs @@ -384,6 +384,11 @@ impl Interval { self > &Self::from_month_day_usec(0, 0, 0) } + /// Checks if all fields of [`Interval`] are all non-negative. + pub fn is_never_negative(&self) -> bool { + self.months >= 0 && self.days >= 0 && self.usecs >= 0 + } + /// Truncate the interval to the precision of milliseconds. /// /// # Example diff --git a/src/common/src/types/mod.rs b/src/common/src/types/mod.rs index 0b80f57e132b2..e45601d2e0240 100644 --- a/src/common/src/types/mod.rs +++ b/src/common/src/types/mod.rs @@ -293,6 +293,36 @@ impl From for PbTypeName { } } +/// Convenient macros to generate match arms for [`DataType`](crate::types::DataType). +pub mod data_types { + /// Numeric [`DataType`](crate::types::DataType)s supported to be `offset` of `RANGE` frame. + #[macro_export] + macro_rules! range_frame_numeric { + () => { + DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::Float32 + | DataType::Float64 + | DataType::Decimal + }; + } + pub use range_frame_numeric; + + /// Date/time [`DataType`](crate::types::DataType)s supported to be `offset` of `RANGE` frame. + #[macro_export] + macro_rules! range_frame_datetime { + () => { + DataType::Date + | DataType::Time + | DataType::Timestamp + | DataType::Timestamptz + | DataType::Interval + }; + } + pub use range_frame_datetime; +} + impl DataType { pub fn create_array_builder(&self, capacity: usize) -> ArrayBuilderImpl { use crate::array::*; @@ -374,7 +404,7 @@ impl DataType { matches!(self, DataType::Int16 | DataType::Int32 | DataType::Int64) } - /// Returns the output type of window function on a given input type. + /// Returns the output type of time window function on a given input type. pub fn window_of(input: &DataType) -> Option { match input { DataType::Timestamptz => Some(DataType::Timestamptz), @@ -570,6 +600,27 @@ pub trait ToOwnedDatum { fn to_owned_datum(self) -> Datum; } +impl ToOwnedDatum for Datum { + #[inline(always)] + fn to_owned_datum(self) -> Datum { + self + } +} + +impl ToOwnedDatum for &Datum { + #[inline(always)] + fn to_owned_datum(self) -> Datum { + self.clone() + } +} + +impl ToOwnedDatum for Option<&ScalarImpl> { + #[inline(always)] + fn to_owned_datum(self) -> Datum { + self.cloned() + } +} + impl ToOwnedDatum for DatumRef<'_> { #[inline(always)] fn to_owned_datum(self) -> Datum { diff --git a/src/common/src/types/sentinel.rs b/src/common/src/types/sentinel.rs index e220f1d210cca..22eb91d4d476b 100644 --- a/src/common/src/types/sentinel.rs +++ b/src/common/src/types/sentinel.rs @@ -54,6 +54,21 @@ impl Sentinelled { (Normal(a), Normal(b)) => cmp_fn(a, b), } } + + pub fn map(self, map_fn: impl FnOnce(T) -> U) -> Sentinelled { + use Sentinelled::*; + match self { + Smallest => Smallest, + Normal(inner) => Normal(map_fn(inner)), + Largest => Largest, + } + } +} + +impl From for Sentinelled { + fn from(inner: T) -> Self { + Self::Normal(inner) + } } impl PartialOrd for Sentinelled diff --git a/src/connector/src/parser/avro/util.rs b/src/connector/src/parser/avro/util.rs index a6c5c6fbef5d1..8d2d4265883e6 100644 --- a/src/connector/src/parser/avro/util.rs +++ b/src/connector/src/parser/avro/util.rs @@ -61,7 +61,7 @@ fn avro_field_to_column_desc( type_name: schema_name.to_string(), generated_or_default_column: None, description: None, - additional_columns: Some(AdditionalColumn { column_type: None }), + additional_column: Some(AdditionalColumn { column_type: None }), version: ColumnDescVersion::Pr13707 as i32, }) } @@ -71,7 +71,7 @@ fn avro_field_to_column_desc( column_type: Some(data_type.to_protobuf()), column_id: *index, name: name.to_owned(), - additional_columns: Some(AdditionalColumn { column_type: None }), + additional_column: Some(AdditionalColumn { column_type: None }), version: ColumnDescVersion::Pr13707 as i32, ..Default::default() }) diff --git a/src/connector/src/parser/debezium/simd_json_parser.rs b/src/connector/src/parser/debezium/simd_json_parser.rs index 87aead865c6db..a1bf9c9977860 100644 --- a/src/connector/src/parser/debezium/simd_json_parser.rs +++ b/src/connector/src/parser/debezium/simd_json_parser.rs @@ -567,7 +567,7 @@ mod tests { column_type: SourceColumnType::Normal, is_pk: false, is_hidden_addition_col: false, - additional_column_type: AdditionalColumn { column_type: None }, + additional_column: AdditionalColumn { column_type: None }, }, SourceColumnDesc::simple("o_enum", DataType::Varchar, ColumnId::from(8)), SourceColumnDesc::simple("o_char", DataType::Varchar, ColumnId::from(9)), diff --git a/src/connector/src/parser/json_parser.rs b/src/connector/src/parser/json_parser.rs index 8480138e02c07..053b9eb1fc2bc 100644 --- a/src/connector/src/parser/json_parser.rs +++ b/src/connector/src/parser/json_parser.rs @@ -582,7 +582,7 @@ mod tests { column_type: SourceColumnType::Normal, is_pk: true, is_hidden_addition_col: false, - additional_column_type: AdditionalColumn { + additional_column: AdditionalColumn { column_type: Some(AdditionalColumnType::Key(AdditionalColumnKey {})), }, }; diff --git a/src/connector/src/parser/mod.rs b/src/connector/src/parser/mod.rs index 7884eb62bf139..bcc946c28cd1f 100644 --- a/src/connector/src/parser/mod.rs +++ b/src/connector/src/parser/mod.rs @@ -318,7 +318,7 @@ impl SourceStreamChunkRowWriter<'_> { mut f: impl FnMut(&SourceColumnDesc) -> AccessResult, ) -> AccessResult<()> { let mut wrapped_f = |desc: &SourceColumnDesc| { - match (&desc.column_type, &desc.additional_column_type.column_type) { + match (&desc.column_type, &desc.additional_column.column_type) { (&SourceColumnType::Offset | &SourceColumnType::RowId, _) => { // SourceColumnType is for CDC source only. Ok(A::output_for( diff --git a/src/connector/src/parser/protobuf/parser.rs b/src/connector/src/parser/protobuf/parser.rs index 13da5c5b86b2d..4793c3d08f5ce 100644 --- a/src/connector/src/parser/protobuf/parser.rs +++ b/src/connector/src/parser/protobuf/parser.rs @@ -180,7 +180,7 @@ impl ProtobufParserConfig { type_name: m.full_name().to_string(), generated_or_default_column: None, description: None, - additional_columns: Some(AdditionalColumn { column_type: None }), + additional_column: Some(AdditionalColumn { column_type: None }), version: ColumnDescVersion::Pr13707 as i32, }) } else { @@ -189,7 +189,7 @@ impl ProtobufParserConfig { column_id: *index, name: field_descriptor.name().to_string(), column_type: Some(field_type.to_protobuf()), - additional_columns: Some(AdditionalColumn { column_type: None }), + additional_column: Some(AdditionalColumn { column_type: None }), version: ColumnDescVersion::Pr13707 as i32, ..Default::default() }) diff --git a/src/connector/src/parser/unified/upsert.rs b/src/connector/src/parser/unified/upsert.rs index 8fbed9dc2ac52..9129f0d16d864 100644 --- a/src/connector/src/parser/unified/upsert.rs +++ b/src/connector/src/parser/unified/upsert.rs @@ -105,7 +105,7 @@ where } fn access_field(&self, desc: &SourceColumnDesc) -> super::AccessResult { - match desc.additional_column_type.column_type { + match desc.additional_column.column_type { Some(AdditionalColumnType::Key(_)) => { if let Some(key_as_column_name) = &self.key_column_name && &desc.name == key_as_column_name diff --git a/src/connector/src/parser/upsert_parser.rs b/src/connector/src/parser/upsert_parser.rs index fbd82f6f3c167..4f4c23f881ec0 100644 --- a/src/connector/src/parser/upsert_parser.rs +++ b/src/connector/src/parser/upsert_parser.rs @@ -54,7 +54,7 @@ async fn build_accessor_builder( pub fn get_key_column_name(columns: &[SourceColumnDesc]) -> Option { columns.iter().find_map(|column| { if matches!( - column.additional_column_type.column_type, + column.additional_column.column_type, Some(AdditionalColumnType::Key(_)) ) { Some(column.name.clone()) diff --git a/src/connector/src/source/kafka/source/reader.rs b/src/connector/src/source/kafka/source/reader.rs index 2a0ef8babe992..691590e361cde 100644 --- a/src/connector/src/source/kafka/source/reader.rs +++ b/src/connector/src/source/kafka/source/reader.rs @@ -198,7 +198,7 @@ impl CommonSplitReader for KafkaSplitReader { // ingest kafka message header can be expensive, do it only when required let require_message_header = self.parser_config.common.rw_columns.iter().any(|col_desc| { matches!( - col_desc.additional_column_type.column_type, + col_desc.additional_column.column_type, Some(AdditionalColumnType::Headers(_) | AdditionalColumnType::HeaderInner(_)) ) }); diff --git a/src/connector/src/source/manager.rs b/src/connector/src/source/manager.rs index 049515d6091a8..a5584f6af83d5 100644 --- a/src/connector/src/source/manager.rs +++ b/src/connector/src/source/manager.rs @@ -37,10 +37,10 @@ pub struct SourceColumnDesc { /// `is_hidden_addition_col` is used to indicate whether the column is a hidden addition column. pub is_hidden_addition_col: bool, - /// `additional_column_type` and `column_type` are orthogonal - /// `additional_column_type` is used to indicate the column is from which part of the message + /// `additional_column` and `column_type` are orthogonal + /// `additional_column` is used to indicate the column is from which part of the message /// `column_type` is used to indicate the type of the column, only used in cdc scenario - pub additional_column_type: AdditionalColumn, + pub additional_column: AdditionalColumn, } /// `SourceColumnType` is used to indicate the type of a column emitted by the Source. @@ -91,7 +91,7 @@ impl SourceColumnDesc { column_type: SourceColumnType::Normal, is_pk: false, is_hidden_addition_col: false, - additional_column_type: AdditionalColumn { column_type: None }, + additional_column: AdditionalColumn { column_type: None }, } } @@ -131,7 +131,7 @@ impl From<&ColumnDesc> for SourceColumnDesc { column_type, is_pk: false, is_hidden_addition_col: false, - additional_column_type: c.additional_columns.clone(), + additional_column: c.additional_column.clone(), } } } @@ -146,7 +146,7 @@ impl From<&SourceColumnDesc> for ColumnDesc { type_name: "".to_string(), generated_or_default_column: None, description: None, - additional_columns: s.additional_column_type.clone(), + additional_column: s.additional_column.clone(), version: ColumnDescVersion::Pr13707, } } diff --git a/src/connector/src/source/reader/desc.rs b/src/connector/src/source/reader/desc.rs index e049be8bbe940..79ba66a0f7e38 100644 --- a/src/connector/src/source/reader/desc.rs +++ b/src/connector/src/source/reader/desc.rs @@ -139,7 +139,7 @@ impl SourceDescBuilder { // Check if partition/file/offset columns are included explicitly. for col in &self.columns { - match col.column_desc.as_ref().unwrap().get_additional_columns() { + match col.column_desc.as_ref().unwrap().get_additional_column() { Ok(AdditionalColumn { column_type: Some(ColumnType::Partition(_) | ColumnType::Filename(_)), }) => { diff --git a/src/expr/core/src/window_function/call.rs b/src/expr/core/src/window_function/call.rs index 6c5839c07bc46..f5714c64345a1 100644 --- a/src/expr/core/src/window_function/call.rs +++ b/src/expr/core/src/window_function/call.rs @@ -13,17 +13,34 @@ // limitations under the License. use std::fmt::Display; +use std::ops::Deref; +use std::sync::Arc; +use anyhow::Context; +use educe::Educe; use enum_as_inner::EnumAsInner; +use futures_util::FutureExt; use parse_display::Display; -use risingwave_common::bail; -use risingwave_common::types::DataType; -use risingwave_pb::expr::window_frame::{PbBound, PbExclusion}; +use risingwave_common::row::OwnedRow; +use risingwave_common::types::{ + DataType, Datum, IsNegative, ScalarImpl, ScalarRefImpl, Sentinelled, ToOwnedDatum, ToText, +}; +use risingwave_common::util::sort_util::{Direction, OrderType}; +use risingwave_common::util::value_encoding::{DatumFromProtoExt, DatumToProtoExt}; +use risingwave_common::{bail, must_match}; +use risingwave_pb::expr::window_frame::{ + PbBound, PbBoundType, PbBounds, PbExclusion, PbRangeFrameBound, PbRangeFrameBounds, + PbRowsFrameBound, PbRowsFrameBounds, +}; use risingwave_pb::expr::{PbWindowFrame, PbWindowFunction}; use FrameBound::{CurrentRow, Following, Preceding, UnboundedFollowing, UnboundedPreceding}; use super::WindowFuncKind; use crate::aggregate::AggArgs; +use crate::expr::{ + build_func, BoxedExpression, Expression, ExpressionBoxExt, InputRefExpression, + LiteralExpression, +}; use crate::Result; #[derive(Debug, Clone)] @@ -63,7 +80,7 @@ impl Display for Frame { } impl Frame { - pub fn rows(start: FrameBound, end: FrameBound) -> Self { + pub fn rows(start: RowsFrameBound, end: RowsFrameBound) -> Self { Self { bounds: FrameBounds::Rows(RowsFrameBounds { start, end }), exclusion: FrameExclusion::default(), @@ -71,8 +88,8 @@ impl Frame { } pub fn rows_with_exclusion( - start: FrameBound, - end: FrameBound, + start: RowsFrameBound, + end: RowsFrameBound, exclusion: FrameExclusion, ) -> Self { Self { @@ -87,11 +104,19 @@ impl Frame { use risingwave_pb::expr::window_frame::PbType; let bounds = match frame.get_type()? { PbType::Unspecified => bail!("unspecified type of `WindowFrame`"), - PbType::Rows => { - let start = FrameBound::from_protobuf(frame.get_start()?)?; - let end = FrameBound::from_protobuf(frame.get_end()?)?; + PbType::RowsLegacy => { + let start = FrameBound::::from_protobuf_legacy(frame.get_start()?)?; + let end = FrameBound::::from_protobuf_legacy(frame.get_end()?)?; FrameBounds::Rows(RowsFrameBounds { start, end }) } + PbType::Rows => { + let bounds = must_match!(frame.get_bounds()?, PbBounds::Rows(bounds) => bounds); + FrameBounds::Rows(RowsFrameBounds::from_protobuf(bounds)?) + } + PbType::Range => { + let bounds = must_match!(frame.get_bounds()?, PbBounds::Range(bounds) => bounds); + FrameBounds::Range(RangeFrameBounds::from_protobuf(bounds)?) + } }; let exclusion = FrameExclusion::from_protobuf(frame.get_exclusion()?)?; Ok(Self { bounds, exclusion }) @@ -101,11 +126,21 @@ impl Frame { use risingwave_pb::expr::window_frame::PbType; let exclusion = self.exclusion.to_protobuf() as _; match &self.bounds { - FrameBounds::Rows(RowsFrameBounds { start, end }) => PbWindowFrame { + #[expect(deprecated)] + FrameBounds::Rows(bounds) => PbWindowFrame { r#type: PbType::Rows as _, - start: Some(start.to_protobuf()), - end: Some(end.to_protobuf()), + start: None, // deprecated + end: None, // deprecated exclusion, + bounds: Some(PbBounds::Rows(bounds.to_protobuf())), + }, + #[expect(deprecated)] + FrameBounds::Range(bounds) => PbWindowFrame { + r#type: PbType::Range as _, + start: None, // deprecated + end: None, // deprecated + exclusion, + bounds: Some(PbBounds::Range(bounds.to_protobuf())), }, } } @@ -116,25 +151,28 @@ impl Frame { pub enum FrameBounds { Rows(RowsFrameBounds), // Groups(GroupsFrameBounds), - // Range(RangeFrameBounds), + Range(RangeFrameBounds), } impl FrameBounds { pub fn validate(&self) -> Result<()> { match self { Self::Rows(bounds) => bounds.validate(), + Self::Range(bounds) => bounds.validate(), } } pub fn start_is_unbounded(&self) -> bool { match self { Self::Rows(RowsFrameBounds { start, .. }) => start.is_unbounded_preceding(), + Self::Range(RangeFrameBounds { start, .. }) => start.is_unbounded_preceding(), } } pub fn end_is_unbounded(&self) -> bool { match self { Self::Rows(RowsFrameBounds { end, .. }) => end.is_unbounded_following(), + Self::Range(RangeFrameBounds { end, .. }) => end.is_unbounded_following(), } } @@ -150,8 +188,23 @@ pub trait FrameBoundsImpl { #[derive(Display, Debug, Clone, Eq, PartialEq, Hash)] #[display("ROWS BETWEEN {start} AND {end}")] pub struct RowsFrameBounds { - pub start: FrameBound, - pub end: FrameBound, + pub start: RowsFrameBound, + pub end: RowsFrameBound, +} + +impl RowsFrameBounds { + fn from_protobuf(bounds: &PbRowsFrameBounds) -> Result { + let start = FrameBound::::from_protobuf(bounds.get_start()?)?; + let end = FrameBound::::from_protobuf(bounds.get_end()?)?; + Ok(Self { start, end }) + } + + fn to_protobuf(&self) -> PbRowsFrameBounds { + PbRowsFrameBounds { + start: Some(self.start.to_protobuf()), + end: Some(self.end.to_protobuf()), + } + } } impl RowsFrameBounds { @@ -192,7 +245,261 @@ impl RowsFrameBounds { impl FrameBoundsImpl for RowsFrameBounds { fn validate(&self) -> Result<()> { - FrameBound::validate_bounds(&self.start, &self.end) + FrameBound::validate_bounds(&self.start, &self.end, |_| Ok(())) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct RangeFrameBounds { + pub order_data_type: DataType, + pub order_type: OrderType, + pub offset_data_type: DataType, + pub start: RangeFrameBound, + pub end: RangeFrameBound, +} + +impl RangeFrameBounds { + fn from_protobuf(bounds: &PbRangeFrameBounds) -> Result { + let order_data_type = DataType::from(bounds.get_order_data_type()?); + let order_type = OrderType::from_protobuf(bounds.get_order_type()?); + let offset_data_type = DataType::from(bounds.get_offset_data_type()?); + let start = FrameBound::::from_protobuf( + bounds.get_start()?, + &order_data_type, + &offset_data_type, + )?; + let end = FrameBound::::from_protobuf( + bounds.get_end()?, + &order_data_type, + &offset_data_type, + )?; + Ok(Self { + order_data_type, + order_type, + offset_data_type, + start, + end, + }) + } + + fn to_protobuf(&self) -> PbRangeFrameBounds { + PbRangeFrameBounds { + start: Some(self.start.to_protobuf()), + end: Some(self.end.to_protobuf()), + order_data_type: Some(self.order_data_type.to_protobuf()), + order_type: Some(self.order_type.to_protobuf()), + offset_data_type: Some(self.offset_data_type.to_protobuf()), + } + } +} + +/// The wrapper type for [`ScalarImpl`] range frame offset, containing +/// two expressions to help adding and subtracting the offset. +#[derive(Debug, Clone, Educe)] +#[educe(PartialEq, Eq, Hash)] +pub struct RangeFrameOffset { + /// The original offset value. + offset: ScalarImpl, + /// Built expression for `$0 + offset`. + #[educe(PartialEq(ignore), Hash(ignore))] + add_expr: Option>, + /// Built expression for `$0 - offset`. + #[educe(PartialEq(ignore), Hash(ignore))] + sub_expr: Option>, +} + +impl RangeFrameOffset { + pub fn new(offset: ScalarImpl) -> Self { + Self { + offset, + add_expr: None, + sub_expr: None, + } + } + + fn build_exprs( + &mut self, + order_data_type: &DataType, + offset_data_type: &DataType, + ) -> Result<()> { + use risingwave_pb::expr::expr_node::PbType as PbExprType; + + let input_expr = InputRefExpression::new(order_data_type.clone(), 0); + let offset_expr = + LiteralExpression::new(offset_data_type.clone(), Some(self.offset.clone())); + self.add_expr = Some(Arc::new(build_func( + PbExprType::Add, + order_data_type.clone(), + vec![input_expr.clone().boxed(), offset_expr.clone().boxed()], + )?)); + self.sub_expr = Some(Arc::new(build_func( + PbExprType::Subtract, + order_data_type.clone(), + vec![input_expr.boxed(), offset_expr.boxed()], + )?)); + Ok(()) + } + + pub fn new_for_test( + offset: ScalarImpl, + order_data_type: &DataType, + offset_data_type: &DataType, + ) -> Self { + let mut offset = Self::new(offset); + offset + .build_exprs(order_data_type, offset_data_type) + .unwrap(); + offset + } +} + +impl Deref for RangeFrameOffset { + type Target = ScalarImpl; + + fn deref(&self) -> &Self::Target { + &self.offset + } +} + +impl Display for RangeFrameBounds { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "RANGE BETWEEN {} AND {}", + self.start.for_display(), + self.end.for_display() + )?; + Ok(()) + } +} + +impl FrameBoundsImpl for RangeFrameBounds { + fn validate(&self) -> Result<()> { + fn validate_non_negative(val: impl IsNegative + Display) -> Result<()> { + if val.is_negative() { + bail!( + "frame bound offset should be non-negative, but {} is given", + val + ); + } + Ok(()) + } + + FrameBound::validate_bounds(&self.start, &self.end, |offset| { + match offset.as_scalar_ref_impl() { + // TODO(rc): use decl macro? + ScalarRefImpl::Int16(val) => validate_non_negative(val)?, + ScalarRefImpl::Int32(val) => validate_non_negative(val)?, + ScalarRefImpl::Int64(val) => validate_non_negative(val)?, + ScalarRefImpl::Float32(val) => validate_non_negative(val)?, + ScalarRefImpl::Float64(val) => validate_non_negative(val)?, + ScalarRefImpl::Decimal(val) => validate_non_negative(val)?, + ScalarRefImpl::Interval(val) => { + if !val.is_never_negative() { + bail!( + "for frame bound offset of type `interval`, each field should be non-negative, but {} is given", + val + ); + } + if matches!(self.order_data_type, DataType::Timestamptz) { + // for `timestamptz`, we only support offset without `month` and `day` fields + if val.months() != 0 || val.days() != 0 { + bail!( + "for frame order column of type `timestamptz`, offset should not have non-zero `month` and `day`", + ); + } + } + }, + _ => unreachable!("other order column data types are not supported and should be banned in frontend"), + } + Ok(()) + }) + } +} + +impl RangeFrameBounds { + /// Get the frame start for a given order column value. + /// + /// ## Examples + /// + /// For the following frames: + /// + /// ```sql + /// ORDER BY x ASC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + /// ORDER BY x DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + /// ``` + /// + /// For any CURRENT ROW with any order value, the frame start is always the first-most row, which is + /// represented by [`Sentinelled::Smallest`]. + /// + /// For the following frame: + /// + /// ```sql + /// ORDER BY x ASC RANGE BETWEEN 10 PRECEDING AND CURRENT ROW + /// ``` + /// + /// For CURRENT ROW with order value `100`, the frame start is the **FIRST** row with order value `90`. + /// + /// For the following frame: + /// + /// ```sql + /// ORDER BY x DESC RANGE BETWEEN 10 PRECEDING AND CURRENT ROW + /// ``` + /// + /// For CURRENT ROW with order value `100`, the frame start is the **FIRST** row with order value `110`. + pub fn frame_start_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled { + self.start.for_calc().bound_of(order_value, self.order_type) + } + + /// Get the frame end for a given order column value. It's very similar to `frame_start_of`, just with + /// everything on the other direction. + pub fn frame_end_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled { + self.end.for_calc().bound_of(order_value, self.order_type) + } + + /// Get the order value of the CURRENT ROW of the first frame that includes the given order value. + /// + /// ## Examples + /// + /// For the following frames: + /// + /// ```sql + /// ORDER BY x ASC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + /// ORDER BY x DESC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + /// ``` + /// + /// For any given order value, the first CURRENT ROW is always the first-most row, which is + /// represented by [`Sentinelled::Smallest`]. + /// + /// For the following frame: + /// + /// ```sql + /// ORDER BY x ASC RANGE BETWEEN CURRENT ROW AND 10 FOLLOWING + /// ``` + /// + /// For a given order value `100`, the first CURRENT ROW should have order value `90`. + /// + /// For the following frame: + /// + /// ```sql + /// ORDER BY x DESC RANGE BETWEEN CURRENT ROW AND 10 FOLLOWING + /// ``` + /// + /// For a given order value `100`, the first CURRENT ROW should have order value `110`. + pub fn first_curr_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled { + self.end + .for_calc() + .reverse() + .bound_of(order_value, self.order_type) + } + + /// Get the order value of the CURRENT ROW of the last frame that includes the given order value. + /// It's very similar to `first_curr_of`, just with everything on the other direction. + pub fn last_curr_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled { + self.start + .for_calc() + .reverse() + .bound_of(order_value, self.order_type) } } @@ -208,11 +515,27 @@ pub enum FrameBound { UnboundedFollowing, } +pub type RowsFrameBound = FrameBound; +pub type RangeFrameBound = FrameBound; + impl FrameBound { - fn validate_bounds(start: &Self, end: &Self) -> Result<()> { + fn offset_value(&self) -> Option<&T> { + match self { + UnboundedPreceding | UnboundedFollowing | CurrentRow => None, + Preceding(offset) | Following(offset) => Some(offset), + } + } + + fn validate_bounds( + start: &Self, + end: &Self, + offset_checker: impl Fn(&T) -> Result<()>, + ) -> Result<()> { match (start, end) { (_, UnboundedPreceding) => bail!("frame end cannot be UNBOUNDED PRECEDING"), - (UnboundedFollowing, _) => bail!("frame start cannot be UNBOUNDED FOLLOWING"), + (UnboundedFollowing, _) => { + bail!("frame start cannot be UNBOUNDED FOLLOWING") + } (Following(_), CurrentRow) | (Following(_), Preceding(_)) => { bail!("frame starting from following row cannot have preceding rows") } @@ -221,49 +544,85 @@ impl FrameBound { } _ => {} } + + for bound in [start, end] { + if let Some(offset) = bound.offset_value() { + offset_checker(offset)?; + } + } + Ok(()) } + + pub fn map(self, f: impl Fn(T) -> U) -> FrameBound { + match self { + UnboundedPreceding => UnboundedPreceding, + Preceding(offset) => Preceding(f(offset)), + CurrentRow => CurrentRow, + Following(offset) => Following(f(offset)), + UnboundedFollowing => UnboundedFollowing, + } + } +} + +impl FrameBound +where + T: Copy, +{ + fn reverse(self) -> FrameBound { + match self { + UnboundedPreceding => UnboundedFollowing, + Preceding(offset) => Following(offset), + CurrentRow => CurrentRow, + Following(offset) => Preceding(offset), + UnboundedFollowing => UnboundedPreceding, + } + } } -impl FrameBound { - pub fn from_protobuf(bound: &PbBound) -> Result { +impl RowsFrameBound { + fn from_protobuf_legacy(bound: &PbBound) -> Result { use risingwave_pb::expr::window_frame::bound::PbOffset; - use risingwave_pb::expr::window_frame::PbBoundType; let offset = bound.get_offset()?; let bound = match offset { - PbOffset::Integer(offset) => match bound.get_type()? { - PbBoundType::Unspecified => bail!("unspecified type of `FrameBound`"), - PbBoundType::UnboundedPreceding => Self::UnboundedPreceding, - PbBoundType::Preceding => Self::Preceding(*offset as usize), - PbBoundType::CurrentRow => Self::CurrentRow, - PbBoundType::Following => Self::Following(*offset as usize), - PbBoundType::UnboundedFollowing => Self::UnboundedFollowing, - }, - PbOffset::Datum(_) => bail!("offset of `FrameBound` must be `Integer`"), + PbOffset::Integer(offset) => Self::from_protobuf(&PbRowsFrameBound { + r#type: bound.get_type()? as _, + offset: Some(*offset), + })?, + PbOffset::Datum(_) => bail!("offset of `RowsFrameBound` must be `Integer`"), }; Ok(bound) } - pub fn to_protobuf(&self) -> PbBound { - use risingwave_pb::expr::window_frame::bound::PbOffset; - use risingwave_pb::expr::window_frame::PbBoundType; + fn from_protobuf(bound: &PbRowsFrameBound) -> Result { + let bound = match bound.get_type()? { + PbBoundType::Unspecified => bail!("unspecified type of `RowsFrameBound`"), + PbBoundType::UnboundedPreceding => Self::UnboundedPreceding, + PbBoundType::Preceding => Self::Preceding(*bound.get_offset()? as usize), + PbBoundType::CurrentRow => Self::CurrentRow, + PbBoundType::Following => Self::Following(*bound.get_offset()? as usize), + PbBoundType::UnboundedFollowing => Self::UnboundedFollowing, + }; + Ok(bound) + } + fn to_protobuf(&self) -> PbRowsFrameBound { let (r#type, offset) = match self { - Self::UnboundedPreceding => (PbBoundType::UnboundedPreceding, PbOffset::Integer(0)), - Self::Preceding(offset) => (PbBoundType::Preceding, PbOffset::Integer(*offset as _)), - Self::CurrentRow => (PbBoundType::CurrentRow, PbOffset::Integer(0)), - Self::Following(offset) => (PbBoundType::Following, PbOffset::Integer(*offset as _)), - Self::UnboundedFollowing => (PbBoundType::UnboundedFollowing, PbOffset::Integer(0)), + Self::UnboundedPreceding => (PbBoundType::UnboundedPreceding, None), + Self::Preceding(offset) => (PbBoundType::Preceding, Some(*offset as _)), + Self::CurrentRow => (PbBoundType::CurrentRow, None), + Self::Following(offset) => (PbBoundType::Following, Some(*offset as _)), + Self::UnboundedFollowing => (PbBoundType::UnboundedFollowing, None), }; - PbBound { + PbRowsFrameBound { r#type: r#type as _, - offset: Some(offset), + offset, } } } -impl FrameBound { +impl RowsFrameBound { /// Convert the bound to sized offset from current row. `None` if the bound is unbounded. pub fn to_offset(&self) -> Option { match self { @@ -275,6 +634,118 @@ impl FrameBound { } } +impl RangeFrameBound { + fn from_protobuf( + bound: &PbRangeFrameBound, + order_data_type: &DataType, + offset_data_type: &DataType, + ) -> Result { + let bound = match bound.get_type()? { + PbBoundType::Unspecified => bail!("unspecified type of `RangeFrameBound`"), + PbBoundType::UnboundedPreceding => Self::UnboundedPreceding, + PbBoundType::CurrentRow => Self::CurrentRow, + PbBoundType::UnboundedFollowing => Self::UnboundedFollowing, + bound_type @ (PbBoundType::Preceding | PbBoundType::Following) => { + let offset_value = Datum::from_protobuf(bound.get_offset()?, offset_data_type) + .context("offset `Datum` is not decodable")? + .context("offset of `RangeFrameBound` must be non-NULL")?; + let mut offset = RangeFrameOffset::new(offset_value); + offset.build_exprs(order_data_type, offset_data_type)?; + if bound_type == PbBoundType::Preceding { + Self::Preceding(offset) + } else { + Self::Following(offset) + } + } + }; + Ok(bound) + } + + fn to_protobuf(&self) -> PbRangeFrameBound { + let (r#type, offset) = match self { + Self::UnboundedPreceding => (PbBoundType::UnboundedPreceding, None), + Self::Preceding(offset) => ( + PbBoundType::Preceding, + Some(Some(offset.as_scalar_ref_impl()).to_protobuf()), + ), + Self::CurrentRow => (PbBoundType::CurrentRow, None), + Self::Following(offset) => ( + PbBoundType::Following, + Some(Some(offset.as_scalar_ref_impl()).to_protobuf()), + ), + Self::UnboundedFollowing => (PbBoundType::UnboundedFollowing, None), + }; + PbRangeFrameBound { + r#type: r#type as _, + offset, + } + } +} + +impl RangeFrameBound { + fn for_display(&self) -> FrameBound { + match self { + UnboundedPreceding => UnboundedPreceding, + Preceding(offset) => Preceding(offset.as_scalar_ref_impl().to_text()), + CurrentRow => CurrentRow, + Following(offset) => Following(offset.as_scalar_ref_impl().to_text()), + UnboundedFollowing => UnboundedFollowing, + } + } + + fn for_calc(&self) -> FrameBound> { + match self { + UnboundedPreceding => UnboundedPreceding, + Preceding(offset) => Preceding(RangeFrameOffsetRef { + add_expr: offset.add_expr.as_ref().unwrap().as_ref(), + sub_expr: offset.sub_expr.as_ref().unwrap().as_ref(), + }), + CurrentRow => CurrentRow, + Following(offset) => Following(RangeFrameOffsetRef { + add_expr: offset.add_expr.as_ref().unwrap().as_ref(), + sub_expr: offset.sub_expr.as_ref().unwrap().as_ref(), + }), + UnboundedFollowing => UnboundedFollowing, + } + } +} + +#[derive(Debug, Educe)] +#[educe(Clone, Copy)] +pub struct RangeFrameOffsetRef<'a> { + /// Built expression for `$0 + offset`. + add_expr: &'a dyn Expression, + /// Built expression for `$0 - offset`. + sub_expr: &'a dyn Expression, +} + +impl FrameBound> { + fn bound_of(self, order_value: impl ToOwnedDatum, order_type: OrderType) -> Sentinelled { + let expr = match (self, order_type.direction()) { + (UnboundedPreceding, _) => return Sentinelled::Smallest, + (UnboundedFollowing, _) => return Sentinelled::Largest, + (CurrentRow, _) => return Sentinelled::Normal(order_value.to_owned_datum()), + (Preceding(offset), Direction::Ascending) + | (Following(offset), Direction::Descending) => { + // should SUBTRACT the offset + offset.sub_expr + } + (Following(offset), Direction::Ascending) + | (Preceding(offset), Direction::Descending) => { + // should ADD the offset + offset.add_expr + } + }; + let row = OwnedRow::new(vec![order_value.to_owned_datum()]); + Sentinelled::Normal( + expr.eval_row(&row) + .now_or_never() + .expect("frame bound calculation should finish immediately") + .expect("just simple calculation, should succeed"), // TODO(rc): handle overflow + ) + } +} + #[derive(Display, Debug, Copy, Clone, Eq, PartialEq, Hash, Default, EnumAsInner)] #[display("EXCLUDE {}", style = "TITLE CASE")] pub enum FrameExclusion { @@ -286,7 +757,7 @@ pub enum FrameExclusion { } impl FrameExclusion { - pub fn from_protobuf(exclusion: PbExclusion) -> Result { + fn from_protobuf(exclusion: PbExclusion) -> Result { let excl = match exclusion { PbExclusion::Unspecified => bail!("unspecified type of `FrameExclusion`"), PbExclusion::CurrentRow => Self::CurrentRow, @@ -295,7 +766,7 @@ impl FrameExclusion { Ok(excl) } - pub fn to_protobuf(self) -> PbExclusion { + fn to_protobuf(self) -> PbExclusion { match self { Self::CurrentRow => PbExclusion::CurrentRow, Self::NoOthers => PbExclusion::NoOthers, diff --git a/src/expr/core/src/window_function/state/aggregate.rs b/src/expr/core/src/window_function/state/aggregate.rs index 7d3f4fe20fe57..08689bff6da72 100644 --- a/src/expr/core/src/window_function/state/aggregate.rs +++ b/src/expr/core/src/window_function/state/aggregate.rs @@ -22,7 +22,7 @@ use risingwave_common::util::iter_util::ZipEqFast; use risingwave_common::{bail, must_match}; use smallvec::SmallVec; -use super::buffer::{RowsWindow, WindowBuffer, WindowImpl}; +use super::buffer::{RangeWindow, RowsWindow, WindowBuffer, WindowImpl}; use super::{BoxedWindowState, StateEvictHint, StateKey, StatePos, WindowState}; use crate::aggregate::{ AggArgs, AggCall, AggregateFunction, AggregateState as AggImplState, BoxedAggregateFunction, @@ -90,6 +90,17 @@ pub(super) fn new(call: &WindowFuncCall) -> Result { ), buffer_heap_size: KvSize::new(), }) as BoxedWindowState, + FrameBounds::Range(frame_bounds) => Box::new(AggregateState { + agg_func, + agg_impl, + arg_data_types, + buffer: WindowBuffer::>::new( + RangeWindow::new(frame_bounds.clone()), + call.frame.exclusion, + enable_delta, + ), + buffer_heap_size: KvSize::new(), + }) as BoxedWindowState, }; Ok(this) } diff --git a/src/expr/core/src/window_function/state/buffer.rs b/src/expr/core/src/window_function/state/buffer.rs index 34b727fbd475e..ba064042c0021 100644 --- a/src/expr/core/src/window_function/state/buffer.rs +++ b/src/expr/core/src/window_function/state/buffer.rs @@ -17,10 +17,13 @@ use std::ops::Range; use educe::Educe; use risingwave_common::array::Op; +use risingwave_common::types::Sentinelled; +use risingwave_common::util::memcmp_encoding; use super::range_utils::range_except; +use super::StateKey; use crate::window_function::state::range_utils::range_diff; -use crate::window_function::{FrameExclusion, RowsFrameBounds}; +use crate::window_function::{FrameExclusion, RangeFrameBounds, RowsFrameBounds}; /// A common sliding window buffer. pub(super) struct WindowBuffer { @@ -333,6 +336,106 @@ impl WindowImpl for RowsWindow { } } +/// The sliding window implementation for `RANGE` frames. +pub(super) struct RangeWindow { + frame_bounds: RangeFrameBounds, + _phantom: std::marker::PhantomData, +} + +impl RangeWindow { + pub fn new(frame_bounds: RangeFrameBounds) -> Self { + Self { + frame_bounds, + _phantom: std::marker::PhantomData, + } + } +} + +impl WindowImpl for RangeWindow { + type Key = StateKey; + type Value = V; + + fn preceding_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool { + buffer_ref.curr_idx < buffer_ref.buffer.len() && { + // XXX(rc): It seems that preceding saturation is not important, may remove later. + true + } + } + + fn following_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool { + buffer_ref.curr_idx < buffer_ref.buffer.len() + && { + // Left OK? (note that `left_idx` can be greater than `right_idx`) + // The following line checks whether the left value is the last one in the buffer. + // Here we adopt a conservative approach, which means we assume the next future value + // is likely to be the same as the last value in the current window, in which case + // we can't say the current window is saturated. + buffer_ref.left_idx < buffer_ref.buffer.len() /* non-zero */ - 1 + } + && { + // Right OK? Ditto. + buffer_ref.right_excl_idx < buffer_ref.buffer.len() + } + } + + fn recalculate_left_right(&self, buffer_ref: BufferRefMut<'_, Self::Key, Self::Value>) { + if buffer_ref.buffer.is_empty() { + *buffer_ref.left_idx = 0; + *buffer_ref.right_excl_idx = 0; + } + + let Some(entry) = buffer_ref.buffer.get(*buffer_ref.curr_idx) else { + // If the current index has been moved to a future position, we can't touch anything + // because the next coming key may equal to the previous one which means the left and + // right indices will be the same. + return; + }; + let curr_key = &entry.key; + + let curr_order_value = memcmp_encoding::decode_value( + &self.frame_bounds.order_data_type, + &curr_key.order_key, + self.frame_bounds.order_type, + ) + .expect("no reason to fail here because we just encoded it in memory"); + + match self.frame_bounds.frame_start_of(&curr_order_value) { + Sentinelled::Smallest => { + // unbounded frame start + assert_eq!( + *buffer_ref.left_idx, 0, + "for unbounded start, left index should always be 0" + ); + } + Sentinelled::Normal(value) => { + // bounded, find the start position + let value_enc = memcmp_encoding::encode_value(value, self.frame_bounds.order_type) + .expect("no reason to fail here"); + *buffer_ref.left_idx = buffer_ref + .buffer + .partition_point(|elem| elem.key.order_key < value_enc); + } + Sentinelled::Largest => unreachable!("frame start never be UNBOUNDED FOLLOWING"), + } + + match self.frame_bounds.frame_end_of(curr_order_value) { + Sentinelled::Largest => { + // unbounded frame end + *buffer_ref.right_excl_idx = buffer_ref.buffer.len(); + } + Sentinelled::Normal(value) => { + // bounded, find the end position + let value_enc = memcmp_encoding::encode_value(value, self.frame_bounds.order_type) + .expect("no reason to fail here"); + *buffer_ref.right_excl_idx = buffer_ref + .buffer + .partition_point(|elem| elem.key.order_key <= value_enc); + } + Sentinelled::Smallest => unreachable!("frame end never be UNBOUNDED PRECEDING"), + } + } +} + #[cfg(test)] mod tests { use itertools::Itertools; diff --git a/src/expr/core/src/window_function/state/mod.rs b/src/expr/core/src/window_function/state/mod.rs index fbaec55a84c38..69162f33e5f5b 100644 --- a/src/expr/core/src/window_function/state/mod.rs +++ b/src/expr/core/src/window_function/state/mod.rs @@ -17,7 +17,7 @@ use std::collections::BTreeSet; use itertools::Itertools; use risingwave_common::estimate_size::EstimateSize; use risingwave_common::row::OwnedRow; -use risingwave_common::types::{Datum, DefaultOrdered, Sentinelled}; +use risingwave_common::types::{Datum, DefaultOrdered}; use risingwave_common::util::memcmp_encoding::MemcmpEncoded; use smallvec::SmallVec; @@ -36,12 +36,6 @@ pub struct StateKey { pub pk: DefaultOrdered, } -impl From for Sentinelled { - fn from(key: StateKey) -> Self { - Self::Normal(key) - } -} - #[derive(Debug)] pub struct StatePos<'a> { /// Only 2 cases in which the `key` is `None`: diff --git a/src/frontend/planner_test/tests/testdata/input/over_window_function.yaml b/src/frontend/planner_test/tests/testdata/input/over_window_function.yaml index bfa3bd0f9d6c6..4406089b273b8 100644 --- a/src/frontend/planner_test/tests/testdata/input/over_window_function.yaml +++ b/src/frontend/planner_test/tests/testdata/input/over_window_function.yaml @@ -134,13 +134,17 @@ create table t(x int, y int); select x, y, max(x) over(PARTITION BY y ORDER BY x RANGE 100 PRECEDING) from t; expected_outputs: - - binder_error + - logical_plan + - stream_plan + - batch_plan - id: aggregate with over clause, range frame definition with between sql: | create table t(x int, y int); select x, y, max(x) over(PARTITION BY y ORDER BY x RANGE BETWEEN 100 PRECEDING and UNBOUNDED FOLLOWING) from t; expected_outputs: - - binder_error + - logical_plan + - stream_plan + - batch_plan - id: aggregate with over clause, unbounded range, with ORDER BY sql: | create table t(x int, y int); @@ -547,3 +551,43 @@ expected_outputs: - optimized_logical_plan_for_stream - optimized_logical_plan_for_batch + +# Range frames +- sql: | + create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval); + select + count(*) over (partition by 1::int order by i range 1 preceding) as col1, + count(*) over (partition by 1::int order by bi range 1 preceding) as col2, + count(*) over (partition by 1::int order by d range 1.5 preceding) as col3, + count(*) over (partition by 1::int order by f range 1.5 preceding) as col4, + -- count(*) over (partition by 1::int order by da range '1 day' preceding) as col5, -- `date` not supported yet + -- count(*) over (partition by 1::int order by t range '1 min' preceding) as col6, -- `time` not supported yet + count(*) over (partition by 1::int order by ts range '1 day 1 hour' preceding) as col7, + count(*) over (partition by 1::int order by tstz range '1 min' preceding) as col8 + from t; + expected_outputs: + - logical_plan + - optimized_logical_plan_for_stream + - stream_plan + - batch_plan +- sql: | + create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval); + select + count(*) over (partition by 1::int order by da range '1 day' preceding) -- `date` not supported yet + from t; + expected_outputs: + - binder_error +- sql: | + create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval); + select + count(*) over (partition by 1::int order by t range '1 min' preceding) -- `time` not supported yet + from t; + expected_outputs: + - binder_error +- sql: | + create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval); + select + count(*) over (partition by 1::int order by tstz range '1 day 1 hour' preceding) -- `timestamptz` +/- 'x month x day' not supported yet + from t; + expected_outputs: + - binder_error diff --git a/src/frontend/planner_test/tests/testdata/output/over_window_function.yaml b/src/frontend/planner_test/tests/testdata/output/over_window_function.yaml index 444297bf30a77..6999cdeededfc 100644 --- a/src/frontend/planner_test/tests/testdata/output/over_window_function.yaml +++ b/src/frontend/planner_test/tests/testdata/output/over_window_function.yaml @@ -259,22 +259,42 @@ sql: | create table t(x int, y int); select x, y, max(x) over(PARTITION BY y ORDER BY x RANGE 100 PRECEDING) from t; - binder_error: | - Failed to bind expression: max(x) OVER (PARTITION BY y ORDER BY x RANGE 100 PRECEDING) - - Caused by: - Feature is not yet implemented: window frame in `RANGE` mode is not supported yet - Tracking issue: https://github.com/risingwavelabs/risingwave/issues/9124 + logical_plan: |- + LogicalProject { exprs: [t.x, t.y, max] } + └─LogicalOverWindow { window_functions: [max(t.x) OVER(PARTITION BY t.y ORDER BY t.x ASC RANGE BETWEEN 100 PRECEDING AND CURRENT ROW)] } + └─LogicalProject { exprs: [t.x, t.y, t._row_id] } + └─LogicalScan { table: t, columns: [t.x, t.y, t._row_id] } + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchOverWindow { window_functions: [max(t.x) OVER(PARTITION BY t.y ORDER BY t.x ASC RANGE BETWEEN 100 PRECEDING AND CURRENT ROW)] } + └─BatchExchange { order: [t.y ASC, t.x ASC], dist: HashShard(t.y) } + └─BatchSort { order: [t.y ASC, t.x ASC] } + └─BatchScan { table: t, columns: [t.x, t.y], distribution: SomeShard } + stream_plan: |- + StreamMaterialize { columns: [x, y, t._row_id(hidden), max], stream_key: [t._row_id, y], pk_columns: [t._row_id, y], pk_conflict: NoCheck } + └─StreamOverWindow { window_functions: [max(t.x) OVER(PARTITION BY t.y ORDER BY t.x ASC RANGE BETWEEN 100 PRECEDING AND CURRENT ROW)] } + └─StreamExchange { dist: HashShard(t.y) } + └─StreamTableScan { table: t, columns: [t.x, t.y, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } - id: aggregate with over clause, range frame definition with between sql: | create table t(x int, y int); select x, y, max(x) over(PARTITION BY y ORDER BY x RANGE BETWEEN 100 PRECEDING and UNBOUNDED FOLLOWING) from t; - binder_error: | - Failed to bind expression: max(x) OVER (PARTITION BY y ORDER BY x RANGE BETWEEN 100 PRECEDING AND UNBOUNDED FOLLOWING) - - Caused by: - Feature is not yet implemented: window frame in `RANGE` mode is not supported yet - Tracking issue: https://github.com/risingwavelabs/risingwave/issues/9124 + logical_plan: |- + LogicalProject { exprs: [t.x, t.y, max] } + └─LogicalOverWindow { window_functions: [max(t.x) OVER(PARTITION BY t.y ORDER BY t.x ASC RANGE BETWEEN 100 PRECEDING AND UNBOUNDED FOLLOWING)] } + └─LogicalProject { exprs: [t.x, t.y, t._row_id] } + └─LogicalScan { table: t, columns: [t.x, t.y, t._row_id] } + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchOverWindow { window_functions: [max(t.x) OVER(PARTITION BY t.y ORDER BY t.x ASC RANGE BETWEEN 100 PRECEDING AND UNBOUNDED FOLLOWING)] } + └─BatchExchange { order: [t.y ASC, t.x ASC], dist: HashShard(t.y) } + └─BatchSort { order: [t.y ASC, t.x ASC] } + └─BatchScan { table: t, columns: [t.x, t.y], distribution: SomeShard } + stream_plan: |- + StreamMaterialize { columns: [x, y, t._row_id(hidden), max], stream_key: [t._row_id, y], pk_columns: [t._row_id, y], pk_conflict: NoCheck } + └─StreamOverWindow { window_functions: [max(t.x) OVER(PARTITION BY t.y ORDER BY t.x ASC RANGE BETWEEN 100 PRECEDING AND UNBOUNDED FOLLOWING)] } + └─StreamExchange { dist: HashShard(t.y) } + └─StreamTableScan { table: t, columns: [t.x, t.y, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } - id: aggregate with over clause, unbounded range, with ORDER BY sql: | create table t(x int, y int); @@ -1115,3 +1135,108 @@ └─LogicalOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.b ASC ROWS BETWEEN 10 FOLLOWING AND 1 FOLLOWING)] } └─LogicalProject { exprs: [t.b, 1:Int32] } └─LogicalScan { table: t, columns: [t.b] } +- sql: | + create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval); + select + count(*) over (partition by 1::int order by i range 1 preceding) as col1, + count(*) over (partition by 1::int order by bi range 1 preceding) as col2, + count(*) over (partition by 1::int order by d range 1.5 preceding) as col3, + count(*) over (partition by 1::int order by f range 1.5 preceding) as col4, + -- count(*) over (partition by 1::int order by da range '1 day' preceding) as col5, -- `date` not supported yet + -- count(*) over (partition by 1::int order by t range '1 min' preceding) as col6, -- `time` not supported yet + count(*) over (partition by 1::int order by ts range '1 day 1 hour' preceding) as col7, + count(*) over (partition by 1::int order by tstz range '1 min' preceding) as col8 + from t; + logical_plan: |- + LogicalProject { exprs: [count, count, count, count, count, count] } + └─LogicalOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.i ASC RANGE BETWEEN 1 PRECEDING AND CURRENT ROW), count() OVER(PARTITION BY 1:Int32 ORDER BY t.bi ASC RANGE BETWEEN 1 PRECEDING AND CURRENT ROW), count() OVER(PARTITION BY 1:Int32 ORDER BY t.d ASC RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW), count() OVER(PARTITION BY 1:Int32 ORDER BY t.f ASC RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW), count() OVER(PARTITION BY 1:Int32 ORDER BY t.ts ASC RANGE BETWEEN 1 day 01:00:00 PRECEDING AND CURRENT ROW), count() OVER(PARTITION BY 1:Int32 ORDER BY t.tstz ASC RANGE BETWEEN 00:01:00 PRECEDING AND CURRENT ROW)] } + └─LogicalProject { exprs: [t.i, t.bi, t.d, t.f, t.da, t.t, t.ts, t.tstz, t.itv, t._row_id, 1:Int32] } + └─LogicalScan { table: t, columns: [t.i, t.bi, t.d, t.f, t.da, t.t, t.ts, t.tstz, t.itv, t._row_id] } + optimized_logical_plan_for_stream: |- + LogicalProject { exprs: [count, count, count, count, count, count] } + └─LogicalOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.tstz ASC RANGE BETWEEN 00:01:00 PRECEDING AND CURRENT ROW)] } + └─LogicalProject { exprs: [t.tstz, 1:Int32, count, count, count, count, count] } + └─LogicalOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.ts ASC RANGE BETWEEN 1 day 01:00:00 PRECEDING AND CURRENT ROW)] } + └─LogicalProject { exprs: [t.ts, t.tstz, 1:Int32, count, count, count, count] } + └─LogicalOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.f ASC RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW)] } + └─LogicalProject { exprs: [t.f, t.ts, t.tstz, 1:Int32, count, count, count] } + └─LogicalOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.d ASC RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW)] } + └─LogicalProject { exprs: [t.d, t.f, t.ts, t.tstz, 1:Int32, count, count] } + └─LogicalOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.bi ASC RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)] } + └─LogicalProject { exprs: [t.bi, t.d, t.f, t.ts, t.tstz, 1:Int32, count] } + └─LogicalOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.i ASC RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)] } + └─LogicalProject { exprs: [t.i, t.bi, t.d, t.f, t.ts, t.tstz, 1:Int32] } + └─LogicalScan { table: t, columns: [t.i, t.bi, t.d, t.f, t.ts, t.tstz] } + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchProject { exprs: [count, count, count, count, count, count] } + └─BatchOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.tstz ASC RANGE BETWEEN 00:01:00 PRECEDING AND CURRENT ROW)] } + └─BatchSort { order: [1:Int32 ASC, t.tstz ASC] } + └─BatchProject { exprs: [t.tstz, 1:Int32, count, count, count, count, count] } + └─BatchOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.ts ASC RANGE BETWEEN 1 day 01:00:00 PRECEDING AND CURRENT ROW)] } + └─BatchSort { order: [1:Int32 ASC, t.ts ASC] } + └─BatchProject { exprs: [t.ts, t.tstz, 1:Int32, count, count, count, count] } + └─BatchOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.f ASC RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW)] } + └─BatchSort { order: [1:Int32 ASC, t.f ASC] } + └─BatchProject { exprs: [t.f, t.ts, t.tstz, 1:Int32, count, count, count] } + └─BatchOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.d ASC RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW)] } + └─BatchSort { order: [1:Int32 ASC, t.d ASC] } + └─BatchProject { exprs: [t.d, t.f, t.ts, t.tstz, 1:Int32, count, count] } + └─BatchOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.bi ASC RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)] } + └─BatchSort { order: [1:Int32 ASC, t.bi ASC] } + └─BatchProject { exprs: [t.bi, t.d, t.f, t.ts, t.tstz, 1:Int32, count] } + └─BatchOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.i ASC RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)] } + └─BatchExchange { order: [1:Int32 ASC, t.i ASC], dist: HashShard(1:Int32) } + └─BatchSort { order: [1:Int32 ASC, t.i ASC] } + └─BatchProject { exprs: [t.i, t.bi, t.d, t.f, t.ts, t.tstz, 1:Int32] } + └─BatchScan { table: t, columns: [t.i, t.bi, t.d, t.f, t.ts, t.tstz], distribution: SomeShard } + stream_plan: |- + StreamMaterialize { columns: [col1, col2, col3, col4, col7, col8, t._row_id(hidden), 1:Int32(hidden)], stream_key: [t._row_id, 1:Int32], pk_columns: [t._row_id, 1:Int32], pk_conflict: NoCheck } + └─StreamProject { exprs: [count, count, count, count, count, count, t._row_id, 1:Int32] } + └─StreamOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.tstz ASC RANGE BETWEEN 00:01:00 PRECEDING AND CURRENT ROW)] } + └─StreamProject { exprs: [t.tstz, 1:Int32, count, count, count, count, count, t._row_id] } + └─StreamOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.ts ASC RANGE BETWEEN 1 day 01:00:00 PRECEDING AND CURRENT ROW)] } + └─StreamProject { exprs: [t.ts, t.tstz, 1:Int32, count, count, count, count, t._row_id] } + └─StreamOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.f ASC RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW)] } + └─StreamProject { exprs: [t.f, t.ts, t.tstz, 1:Int32, count, count, count, t._row_id] } + └─StreamOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.d ASC RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW)] } + └─StreamProject { exprs: [t.d, t.f, t.ts, t.tstz, 1:Int32, count, count, t._row_id] } + └─StreamOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.bi ASC RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)] } + └─StreamProject { exprs: [t.bi, t.d, t.f, t.ts, t.tstz, 1:Int32, count, t._row_id] } + └─StreamOverWindow { window_functions: [count() OVER(PARTITION BY 1:Int32 ORDER BY t.i ASC RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)] } + └─StreamExchange { dist: HashShard(1:Int32) } + └─StreamProject { exprs: [t.i, t.bi, t.d, t.f, t.ts, t.tstz, 1:Int32, t._row_id] } + └─StreamTableScan { table: t, columns: [t.i, t.bi, t.d, t.f, t.ts, t.tstz, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } +- sql: | + create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval); + select + count(*) over (partition by 1::int order by da range '1 day' preceding) -- `date` not supported yet + from t; + binder_error: | + Failed to bind expression: count(*) OVER (PARTITION BY CAST(1 AS INT) ORDER BY da RANGE '1 day' PRECEDING) + + Caused by: + Feature is not yet implemented: `RANGE` frame with offset of type `date` is not implemented yet, please manually cast the `ORDER BY` column to `timestamp` + No tracking issue yet. Feel free to submit a feature request at https://github.com/risingwavelabs/risingwave/issues/new?labels=type%2Ffeature&template=feature_request.yml +- sql: | + create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval); + select + count(*) over (partition by 1::int order by t range '1 min' preceding) -- `time` not supported yet + from t; + binder_error: | + Failed to bind expression: count(*) OVER (PARTITION BY CAST(1 AS INT) ORDER BY t RANGE '1 min' PRECEDING) + + Caused by: + Feature is not yet implemented: `RANGE` frame with offset of type `time without time zone` is not implemented yet, please manually cast the `ORDER BY` column to `timestamp` + No tracking issue yet. Feel free to submit a feature request at https://github.com/risingwavelabs/risingwave/issues/new?labels=type%2Ffeature&template=feature_request.yml +- sql: | + create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval); + select + count(*) over (partition by 1::int order by tstz range '1 day 1 hour' preceding) -- `timestamptz` +/- 'x month x day' not supported yet + from t; + binder_error: | + Failed to bind expression: count(*) OVER (PARTITION BY CAST(1 AS INT) ORDER BY tstz RANGE '1 day 1 hour' PRECEDING) + + Caused by these errors (recent errors listed first): + 1: Expr error + 2: for frame order column of type `timestamptz`, offset should not have non-zero `month` and `day` diff --git a/src/frontend/src/binder/expr/column.rs b/src/frontend/src/binder/expr/column.rs index 9fb17c6e43520..c2fd536c1ccc4 100644 --- a/src/frontend/src/binder/expr/column.rs +++ b/src/frontend/src/binder/expr/column.rs @@ -45,7 +45,7 @@ impl Binder { // to the name of the defined sql udf parameters stored in `udf_context`. // If so, we will treat this bind as an special bind, the actual expression // stored in `udf_context` will then be bound instead of binding the non-existing column. - if self.udf_binding_flag { + if self.udf_context.global_count() != 0 { if let Some(expr) = self.udf_context.get_expr(&column_name) { return Ok(expr.clone()); } else { diff --git a/src/frontend/src/binder/expr/function.rs b/src/frontend/src/binder/expr/function.rs index 5372ff596cbbc..bb6e4ee14c335 100644 --- a/src/frontend/src/binder/expr/function.rs +++ b/src/frontend/src/binder/expr/function.rs @@ -23,11 +23,12 @@ use risingwave_common::array::ListValue; use risingwave_common::catalog::{INFORMATION_SCHEMA_SCHEMA_NAME, PG_CATALOG_SCHEMA_NAME}; use risingwave_common::error::{ErrorCode, Result, RwError}; use risingwave_common::session_config::USER_NAME_WILD_CARD; -use risingwave_common::types::{DataType, ScalarImpl, Timestamptz}; +use risingwave_common::types::{data_types, DataType, ScalarImpl, Timestamptz}; use risingwave_common::{bail_not_implemented, current_cluster_version, no_function}; use risingwave_expr::aggregate::{agg_kinds, AggKind}; use risingwave_expr::window_function::{ - Frame, FrameBound, FrameBounds, FrameExclusion, RowsFrameBounds, WindowFuncKind, + Frame, FrameBound, FrameBounds, FrameExclusion, RangeFrameBounds, RangeFrameOffset, + RowsFrameBounds, WindowFuncKind, }; use risingwave_sqlparser::ast::{ self, Function, FunctionArg, FunctionArgExpr, Ident, WindowFrameBound, WindowFrameExclusion, @@ -239,12 +240,16 @@ impl Binder { } if let Ok(expr) = UdfContext::extract_udf_expression(ast) { - self.set_udf_binding_flag(); let bind_result = self.bind_expr(expr); - self.unset_udf_binding_flag(); + + // We should properly decrement global count after a successful binding + // Since the subsequent probe operation in `bind_column` or + // `bind_parameter` relies on global counting + self.udf_context.decr_global_count(); // Restore context information for subsequent binding self.udf_context.update_context(stashed_udf_context); + return bind_result; } else { return Err(ErrorCode::InvalidInputSyntax( @@ -568,6 +573,8 @@ impl Binder { Ok((vec![], args, order_by)) } + /// Bind window function calls according to PostgreSQL syntax. + /// See for syntax detail. pub(super) fn bind_window_function( &mut self, kind: WindowFuncKind, @@ -607,34 +614,76 @@ impl Binder { }; let bounds = match frame.units { WindowFrameUnits::Rows => { - let convert_bound = |bound| match bound { - WindowFrameBound::CurrentRow => FrameBound::CurrentRow, - WindowFrameBound::Preceding(None) => FrameBound::UnboundedPreceding, - WindowFrameBound::Preceding(Some(offset)) => { - FrameBound::Preceding(offset as usize) + let (start, end) = + self.bind_window_frame_usize_bounds(frame.start_bound, frame.end_bound)?; + FrameBounds::Rows(RowsFrameBounds { start, end }) + } + WindowFrameUnits::Range => { + let order_by_expr = order_by + .sort_exprs + .iter() + // for `RANGE` frame, there should be exactly one `ORDER BY` column + .exactly_one() + .map_err(|_| { + ErrorCode::InvalidInputSyntax( + "there should be exactly one ordering column for `RANGE` frame" + .to_string(), + ) + })?; + let order_data_type = order_by_expr.expr.return_type(); + let order_type = order_by_expr.order_type; + + let offset_data_type = match &order_data_type { + // for numeric ordering columns, `offset` should be the same type + // NOTE: actually in PG it can be a larger type, but we don't support this here + t @ data_types::range_frame_numeric!() => t.clone(), + // for datetime ordering columns, `offset` should be interval + t @ data_types::range_frame_datetime!() => { + if matches!(t, DataType::Date | DataType::Time) { + bail_not_implemented!( + "`RANGE` frame with offset of type `{}` is not implemented yet, please manually cast the `ORDER BY` column to `timestamp`", + t + ); + } + DataType::Interval } - WindowFrameBound::Following(None) => FrameBound::UnboundedFollowing, - WindowFrameBound::Following(Some(offset)) => { - FrameBound::Following(offset as usize) + // other types are not supported + t => { + return Err(ErrorCode::NotSupported( + format!( + "`RANGE` frame with offset of type `{}` is not supported", + t + ), + "Please re-consider the `ORDER BY` column".to_string(), + ) + .into()) } }; - let start = convert_bound(frame.start_bound); - let end = if let Some(end_bound) = frame.end_bound { - convert_bound(end_bound) - } else { - FrameBound::CurrentRow - }; - FrameBounds::Rows(RowsFrameBounds { start, end }) + + let (start, end) = self.bind_window_frame_scalar_impl_bounds( + frame.start_bound, + frame.end_bound, + &offset_data_type, + )?; + FrameBounds::Range(RangeFrameBounds { + order_data_type, + order_type, + offset_data_type, + start: start.map(RangeFrameOffset::new), + end: end.map(RangeFrameOffset::new), + }) } - WindowFrameUnits::Range | WindowFrameUnits::Groups => { + WindowFrameUnits::Groups => { bail_not_implemented!( issue = 9124, - "window frame in `{}` mode is not supported yet", - frame.units + "window frame in `GROUPS` mode is not supported yet", ); } }; + + // Validate the frame bounds, may return `ExprError` to user if the bounds given are not valid. bounds.validate()?; + Some(Frame { bounds, exclusion }) } else { None @@ -642,6 +691,102 @@ impl Binder { Ok(WindowFunction::new(kind, partition_by, order_by, inputs, frame)?.into()) } + fn bind_window_frame_usize_bounds( + &mut self, + start: WindowFrameBound, + end: Option, + ) -> Result<(FrameBound, FrameBound)> { + let mut convert_offset = |offset: Box| -> Result { + let offset = self + .bind_window_frame_bound_offset(*offset, DataType::Int64)? + .into_int64(); + if offset < 0 { + return Err(ErrorCode::InvalidInputSyntax( + "offset in window frame bounds must be non-negative".to_string(), + ) + .into()); + } + Ok(offset as usize) + }; + let mut convert_bound = |bound| -> Result> { + Ok(match bound { + WindowFrameBound::CurrentRow => FrameBound::CurrentRow, + WindowFrameBound::Preceding(None) => FrameBound::UnboundedPreceding, + WindowFrameBound::Preceding(Some(offset)) => { + FrameBound::Preceding(convert_offset(offset)?) + } + WindowFrameBound::Following(None) => FrameBound::UnboundedFollowing, + WindowFrameBound::Following(Some(offset)) => { + FrameBound::Following(convert_offset(offset)?) + } + }) + }; + let start = convert_bound(start)?; + let end = if let Some(end_bound) = end { + convert_bound(end_bound)? + } else { + FrameBound::CurrentRow + }; + Ok((start, end)) + } + + fn bind_window_frame_scalar_impl_bounds( + &mut self, + start: WindowFrameBound, + end: Option, + offset_data_type: &DataType, + ) -> Result<(FrameBound, FrameBound)> { + let mut convert_bound = |bound| -> Result> { + Ok(match bound { + WindowFrameBound::CurrentRow => FrameBound::CurrentRow, + WindowFrameBound::Preceding(None) => FrameBound::UnboundedPreceding, + WindowFrameBound::Preceding(Some(offset)) => FrameBound::Preceding( + self.bind_window_frame_bound_offset(*offset, offset_data_type.clone())?, + ), + WindowFrameBound::Following(None) => FrameBound::UnboundedFollowing, + WindowFrameBound::Following(Some(offset)) => FrameBound::Following( + self.bind_window_frame_bound_offset(*offset, offset_data_type.clone())?, + ), + }) + }; + let start = convert_bound(start)?; + let end = if let Some(end_bound) = end { + convert_bound(end_bound)? + } else { + FrameBound::CurrentRow + }; + Ok((start, end)) + } + + fn bind_window_frame_bound_offset( + &mut self, + offset: ast::Expr, + cast_to: DataType, + ) -> Result { + let mut offset = self.bind_expr(offset)?; + if !offset.is_const() { + return Err(ErrorCode::InvalidInputSyntax( + "offset in window frame bounds must be constant".to_string(), + ) + .into()); + } + if offset.cast_implicit_mut(cast_to.clone()).is_err() { + return Err(ErrorCode::InvalidInputSyntax(format!( + "offset in window frame bounds must be castable to {}", + cast_to + )) + .into()); + } + let offset = offset.fold_const()?; + let Some(offset) = offset else { + return Err(ErrorCode::InvalidInputSyntax( + "offset in window frame bounds must not be NULL".to_string(), + ) + .into()); + }; + Ok(offset) + } + fn bind_builtin_scalar_function( &mut self, function_name: &str, diff --git a/src/frontend/src/binder/expr/mod.rs b/src/frontend/src/binder/expr/mod.rs index 7da1518ceb552..7cd9eab4c1bec 100644 --- a/src/frontend/src/binder/expr/mod.rs +++ b/src/frontend/src/binder/expr/mod.rs @@ -386,11 +386,13 @@ impl Binder { fn bind_parameter(&mut self, index: u64) -> Result { // Special check for sql udf - // Note: This is specific to anonymous sql udf, since the + // Note: This is specific to sql udf with unnamed parameters, since the // parameters will be parsed and treated as `Parameter`. // For detailed explanation, consider checking `bind_column`. - if let Some(expr) = self.udf_context.get_expr(&format!("${index}")) { - return Ok(expr.clone()); + if self.udf_context.global_count() != 0 { + if let Some(expr) = self.udf_context.get_expr(&format!("${index}")) { + return Ok(expr.clone()); + } } Ok(Parameter::new(index, self.param_types.clone()).into()) @@ -701,7 +703,7 @@ pub fn bind_struct_field(column_def: &StructField) -> Result { type_name: "".to_string(), generated_or_default_column: None, description: None, - additional_columns: AdditionalColumn { column_type: None }, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, }) } diff --git a/src/frontend/src/binder/mod.rs b/src/frontend/src/binder/mod.rs index e726d7dc01d4c..f1c7d97c57fa2 100644 --- a/src/frontend/src/binder/mod.rs +++ b/src/frontend/src/binder/mod.rs @@ -122,10 +122,6 @@ pub struct Binder { /// The sql udf context that will be used during binding phase udf_context: UdfContext, - - /// Udf binding flag, used to distinguish between - /// columns and named parameters during sql udf binding - udf_binding_flag: bool, } #[derive(Clone, Debug, Default)] @@ -155,6 +151,10 @@ impl UdfContext { self.udf_global_counter += 1; } + pub fn decr_global_count(&mut self) { + self.udf_global_counter -= 1; + } + pub fn _is_empty(&self) -> bool { self.udf_param_context.is_empty() } @@ -219,6 +219,8 @@ impl UdfContext { Ok(expr) } + /// Create the sql udf context + /// used per `bind_function` for sql udf & semantic check at definition time pub fn create_udf_context( args: &[FunctionArg], catalog: &Arc, @@ -228,9 +230,10 @@ impl UdfContext { match current_arg { FunctionArg::Unnamed(arg) => { let FunctionArgExpr::Expr(e) = arg else { - return Err( - ErrorCode::InvalidInputSyntax("invalid syntax".to_string()).into() - ); + return Err(ErrorCode::InvalidInputSyntax( + "expect `FunctionArgExpr` for unnamed argument".to_string(), + ) + .into()); }; if catalog.arg_names[i].is_empty() { ret.insert(format!("${}", i + 1), e.clone()); @@ -240,7 +243,12 @@ impl UdfContext { ret.insert(catalog.arg_names[i].clone(), e.clone()); } } - _ => return Err(ErrorCode::InvalidInputSyntax("invalid syntax".to_string()).into()), + _ => { + return Err(ErrorCode::InvalidInputSyntax( + "expect unnamed argument when creating sql udf context".to_string(), + ) + .into()) + } } } Ok(ret) @@ -347,7 +355,6 @@ impl Binder { included_relations: HashSet::new(), param_types: ParameterTypes::new(param_types), udf_context: UdfContext::new(), - udf_binding_flag: false, } } @@ -497,14 +504,6 @@ impl Binder { pub fn udf_context_mut(&mut self) -> &mut UdfContext { &mut self.udf_context } - - pub fn set_udf_binding_flag(&mut self) { - self.udf_binding_flag = true; - } - - pub fn unset_udf_binding_flag(&mut self) { - self.udf_binding_flag = false; - } } /// The column name stored in [`BindContext`] for a column without an alias. diff --git a/src/frontend/src/catalog/table_catalog.rs b/src/frontend/src/catalog/table_catalog.rs index 37e79664c666a..965105e1e710e 100644 --- a/src/frontend/src/catalog/table_catalog.rs +++ b/src/frontend/src/catalog/table_catalog.rs @@ -694,7 +694,7 @@ mod tests { type_name: ".test.Country".to_string(), description: None, generated_or_default_column: None, - additional_columns: AdditionalColumn { column_type: None }, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, }, is_hidden: false diff --git a/src/frontend/src/handler/create_source.rs b/src/frontend/src/handler/create_source.rs index 5f25d12650f0c..abe046f08fce1 100644 --- a/src/frontend/src/handler/create_source.rs +++ b/src/frontend/src/handler/create_source.rs @@ -685,7 +685,7 @@ pub(crate) async fn bind_source_pk( // return the key column names if exists columns.iter().find_map(|catalog| { if matches!( - catalog.column_desc.additional_columns.column_type, + catalog.column_desc.additional_column.column_type, Some(AdditionalColumnType::Key(_)) ) { Some(catalog.name().to_string()) @@ -697,7 +697,7 @@ pub(crate) async fn bind_source_pk( let additional_column_names = columns .iter() .filter_map(|col| { - if col.column_desc.additional_columns.column_type.is_some() { + if col.column_desc.additional_column.column_type.is_some() { Some(col.name().to_string()) } else { None @@ -848,7 +848,7 @@ fn check_and_add_timestamp_column( if is_kafka_connector(with_properties) { if columns.iter().any(|col| { matches!( - col.column_desc.additional_columns.column_type, + col.column_desc.additional_column.column_type, Some(AdditionalColumnType::Timestamp(_)) ) }) { diff --git a/src/frontend/src/handler/create_sql_function.rs b/src/frontend/src/handler/create_sql_function.rs index 4eaa78f82533e..311664735603f 100644 --- a/src/frontend/src/handler/create_sql_function.rs +++ b/src/frontend/src/handler/create_sql_function.rs @@ -188,7 +188,9 @@ pub async fn handle_create_sql_function( arg_names.clone(), )); - binder.set_udf_binding_flag(); + // Need to set the initial global count to 1 + // otherwise the context will not be probed during the semantic check + binder.udf_context_mut().incr_global_count(); if let Ok(expr) = UdfContext::extract_udf_expression(ast) { match binder.bind_expr(expr) { @@ -204,7 +206,7 @@ pub async fn handle_create_sql_function( } } Err(e) => return Err(ErrorCode::InvalidInputSyntax(format!( - "failed to conduct semantic check, please see if you are calling non-existence functions: {}", + "failed to conduct semantic check, please see if you are calling non-existence functions or parameters\ndetailed error message: {}", e.as_report() )) .into()), @@ -217,8 +219,6 @@ pub async fn handle_create_sql_function( ) .into()); } - - binder.unset_udf_binding_flag(); } // Create the actual function, will be stored in function catalog diff --git a/src/frontend/src/handler/create_table.rs b/src/frontend/src/handler/create_table.rs index ee871bc68702f..b69d2e5cebbd3 100644 --- a/src/frontend/src/handler/create_table.rs +++ b/src/frontend/src/handler/create_table.rs @@ -214,7 +214,7 @@ pub fn bind_sql_columns(column_defs: &[ColumnDef]) -> Result> type_name: "".to_string(), generated_or_default_column: None, description: None, - additional_columns: AdditionalColumn { column_type: None }, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, }, is_hidden: false, diff --git a/src/frontend/src/optimizer/plan_node/logical_over_window.rs b/src/frontend/src/optimizer/plan_node/logical_over_window.rs index b70b1a82b132d..a1d3909b54d08 100644 --- a/src/frontend/src/optimizer/plan_node/logical_over_window.rs +++ b/src/frontend/src/optimizer/plan_node/logical_over_window.rs @@ -713,12 +713,10 @@ impl PredicatePushdown for LogicalOverWindow { impl ToBatch for LogicalOverWindow { fn to_batch(&self) -> Result { - if !self.core.funcs_have_same_partition_and_order() { - return Err(ErrorCode::InvalidInputSyntax( - "All window functions must have the same PARTITION BY and ORDER BY".to_string(), - ) - .into()); - } + assert!( + self.core.funcs_have_same_partition_and_order(), + "must apply OverWindowSplitRule before generating physical plan" + ); // TODO(rc): Let's not introduce too many cases at once. Later we may decide to support // empty PARTITION BY by simply removing the following check. @@ -744,18 +742,16 @@ impl ToStream for LogicalOverWindow { fn to_stream(&self, ctx: &mut ToStreamContext) -> Result { use super::stream::prelude::*; + assert!( + self.core.funcs_have_same_partition_and_order(), + "must apply OverWindowSplitRule before generating physical plan" + ); + let stream_input = self.core.input.to_stream(ctx)?; if ctx.emit_on_window_close() { // Emit-On-Window-Close case - if !self.core.funcs_have_same_partition_and_order() { - return Err(ErrorCode::InvalidInputSyntax( - "All window functions must have the same PARTITION BY and ORDER BY".to_string(), - ) - .into()); - } - let order_by = &self.window_functions()[0].order_by; if order_by.len() != 1 || order_by[0].order_type != OrderType::ascending() { return Err(ErrorCode::InvalidInputSyntax( @@ -797,13 +793,6 @@ impl ToStream for LogicalOverWindow { } else { // General (Emit-On-Update) case - if !self.core.funcs_have_same_partition_and_order() { - return Err(ErrorCode::InvalidInputSyntax( - "All window functions must have the same PARTITION BY and ORDER BY".to_string(), - ) - .into()); - } - // TODO(rc): Let's not introduce too many cases at once. Later we may decide to support // empty PARTITION BY by simply removing the following check. let partition_key_indices = self.window_functions()[0] diff --git a/src/frontend/src/optimizer/rule/over_window_to_agg_and_join_rule.rs b/src/frontend/src/optimizer/rule/over_window_to_agg_and_join_rule.rs index ac236d0a2e6f0..279d495e4681c 100644 --- a/src/frontend/src/optimizer/rule/over_window_to_agg_and_join_rule.rs +++ b/src/frontend/src/optimizer/rule/over_window_to_agg_and_join_rule.rs @@ -37,9 +37,9 @@ impl Rule for OverWindowToAggAndJoinRule { let over_window = plan.as_logical_over_window()?; let window_functions = over_window.window_functions(); if window_functions.iter().any(|window| { - !window.order_by.is_empty() - || !window.frame.bounds.start_is_unbounded() - || !window.frame.bounds.end_is_unbounded() + !(window.order_by.is_empty() + && window.frame.bounds.start_is_unbounded() + && window.frame.bounds.end_is_unbounded()) }) { return None; } diff --git a/src/meta/model_v2/src/table.rs b/src/meta/model_v2/src/table.rs index 446c928718cf6..f0d83cb2d847d 100644 --- a/src/meta/model_v2/src/table.rs +++ b/src/meta/model_v2/src/table.rs @@ -24,7 +24,7 @@ use crate::{ SourceId, TableId, TableVersion, }; -#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] +#[derive(Clone, Debug, PartialEq, Copy, Eq, EnumIter, DeriveActiveEnum)] #[sea_orm(rs_type = "String", db_type = "String(None)")] pub enum TableType { #[sea_orm(string_value = "TABLE")] diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index a89fe23b43e7d..8d34e4076c1e9 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -1131,10 +1131,6 @@ impl CatalogController { if obj.schema_id == Some(new_schema) { return Ok(IGNORED_NOTIFICATION_VERSION); } - - let mut obj = obj.into_active_model(); - obj.schema_id = Set(Some(new_schema)); - let obj = obj.update(&txn).await?; let database_id = obj.database_id.unwrap(); let mut relations = vec![]; @@ -1145,9 +1141,16 @@ impl CatalogController { .await? .ok_or_else(|| MetaError::catalog_id_not_found("table", object_id))?; check_relation_name_duplicate(&table.name, database_id, new_schema, &txn).await?; + let (associated_src_id, table_type) = + (table.optional_associated_source_id, table.table_type); + + let mut obj = obj.into_active_model(); + obj.schema_id = Set(Some(new_schema)); + let obj = obj.update(&txn).await?; + relations.push(PbRelationInfo::Table(ObjectModel(table, obj).into())); // associated source. - if let Some(associated_source_id) = table.optional_associated_source_id { + if let Some(associated_source_id) = associated_src_id { let src_obj = object::ActiveModel { oid: Set(associated_source_id as _), schema_id: Set(Some(new_schema)), @@ -1168,7 +1171,7 @@ impl CatalogController { let (index_ids, (index_names, mut table_ids)): ( Vec, (Vec, Vec), - ) = if table.table_type == TableType::Table { + ) = if table_type == TableType::Table { Index::find() .select_only() .columns([ @@ -1186,7 +1189,6 @@ impl CatalogController { } else { (vec![], (vec![], vec![])) }; - relations.push(PbRelationInfo::Table(ObjectModel(table, obj).into())); // internal tables. let internal_tables: Vec = Table::find() @@ -1220,18 +1222,6 @@ impl CatalogController { .await?; } - if !index_ids.is_empty() { - let index_objs = Index::find() - .find_also_related(Object) - .filter(index::Column::IndexId.is_in(index_ids)) - .all(&txn) - .await?; - for (index, index_obj) in index_objs { - relations.push(PbRelationInfo::Index( - ObjectModel(index, index_obj.unwrap()).into(), - )); - } - } if !table_ids.is_empty() { let table_objs = Table::find() .find_also_related(Object) @@ -1244,6 +1234,18 @@ impl CatalogController { )); } } + if !index_ids.is_empty() { + let index_objs = Index::find() + .find_also_related(Object) + .filter(index::Column::IndexId.is_in(index_ids)) + .all(&txn) + .await?; + for (index, index_obj) in index_objs { + relations.push(PbRelationInfo::Index( + ObjectModel(index, index_obj.unwrap()).into(), + )); + } + } } ObjectType::Source => { let source = Source::find_by_id(object_id) @@ -1251,6 +1253,10 @@ impl CatalogController { .await? .ok_or_else(|| MetaError::catalog_id_not_found("source", object_id))?; check_relation_name_duplicate(&source.name, database_id, new_schema, &txn).await?; + + let mut obj = obj.into_active_model(); + obj.schema_id = Set(Some(new_schema)); + let obj = obj.update(&txn).await?; relations.push(PbRelationInfo::Source(ObjectModel(source, obj).into())); } ObjectType::Sink => { @@ -1259,6 +1265,10 @@ impl CatalogController { .await? .ok_or_else(|| MetaError::catalog_id_not_found("sink", object_id))?; check_relation_name_duplicate(&sink.name, database_id, new_schema, &txn).await?; + + let mut obj = obj.into_active_model(); + obj.schema_id = Set(Some(new_schema)); + let obj = obj.update(&txn).await?; relations.push(PbRelationInfo::Sink(ObjectModel(sink, obj).into())); // internal tables. @@ -1298,6 +1308,10 @@ impl CatalogController { .await? .ok_or_else(|| MetaError::catalog_id_not_found("view", object_id))?; check_relation_name_duplicate(&view.name, database_id, new_schema, &txn).await?; + + let mut obj = obj.into_active_model(); + obj.schema_id = Set(Some(new_schema)); + let obj = obj.update(&txn).await?; relations.push(PbRelationInfo::View(ObjectModel(view, obj).into())); } ObjectType::Function => { @@ -1305,9 +1319,19 @@ impl CatalogController { .one(&txn) .await? .ok_or_else(|| MetaError::catalog_id_not_found("function", object_id))?; - let pb_function: PbFunction = ObjectModel(function, obj).into(); + + let mut pb_function: PbFunction = ObjectModel(function, obj).into(); + pb_function.schema_id = new_schema as _; check_function_signature_duplicate(&pb_function, &txn).await?; + object::ActiveModel { + oid: Set(object_id), + schema_id: Set(Some(new_schema)), + ..Default::default() + } + .update(&txn) + .await?; + txn.commit().await?; let version = self .notify_frontend( @@ -1322,9 +1346,19 @@ impl CatalogController { .one(&txn) .await? .ok_or_else(|| MetaError::catalog_id_not_found("connection", object_id))?; - let pb_connection: PbConnection = ObjectModel(connection, obj).into(); + + let mut pb_connection: PbConnection = ObjectModel(connection, obj).into(); + pb_connection.schema_id = new_schema as _; check_connection_name_duplicate(&pb_connection, &txn).await?; + object::ActiveModel { + oid: Set(object_id), + schema_id: Set(Some(new_schema)), + ..Default::default() + } + .update(&txn) + .await?; + txn.commit().await?; let version = self .notify_frontend( @@ -1337,6 +1371,7 @@ impl CatalogController { _ => unreachable!("not supported object type: {:?}", object_type), } + txn.commit().await?; let version = self .notify_frontend( Operation::Update, @@ -1748,7 +1783,7 @@ impl CatalogController { let obj = obj.unwrap(); let old_name = relation.name.clone(); relation.name = object_name.into(); - if obj.obj_type != ObjectType::Index { + if obj.obj_type != ObjectType::View { relation.definition = alter_relation_rename(&relation.definition, object_name); } let active_model = $table::ActiveModel { @@ -1778,6 +1813,7 @@ impl CatalogController { .unwrap(); index.name = object_name.into(); let index_table_id = index.index_table_id; + let old_name = rename_relation!(Table, table, table_id, index_table_id); // the name of index and its associated table is the same. let active_model = index::ActiveModel { @@ -1791,7 +1827,7 @@ impl CatalogController { ObjectModel(index, obj.unwrap()).into(), )), }); - rename_relation!(Table, table, table_id, index_table_id) + old_name } _ => unreachable!("only relation name can be altered."), }; diff --git a/src/meta/src/controller/streaming_job.rs b/src/meta/src/controller/streaming_job.rs index 86a527c6a0eb0..7b9426ec78559 100644 --- a/src/meta/src/controller/streaming_job.rs +++ b/src/meta/src/controller/streaming_job.rs @@ -38,7 +38,7 @@ use risingwave_pb::catalog::table::{PbOptionalAssociatedSourceId, PbTableVersion use risingwave_pb::catalog::{PbCreateType, PbTable}; use risingwave_pb::meta::relation::PbRelationInfo; use risingwave_pb::meta::subscribe_response::{ - Info as NotificationInfo, Info, Operation as NotificationOperation, Operation, + Info as NotificationInfo, Operation as NotificationOperation, Operation, }; use risingwave_pb::meta::table_fragments::PbActorStatus; use risingwave_pb::meta::{ @@ -543,7 +543,7 @@ impl CatalogController { let table = table::ActiveModel::from(table).update(&txn).await?; - let old_fragment_mappings = get_fragment_mappings(&txn, job_id).await?; + // let old_fragment_mappings = get_fragment_mappings(&txn, job_id).await?; // 1. replace old fragments/actors with new ones. Fragment::delete_many() .filter(fragment::Column::JobId.eq(job_id)) @@ -701,8 +701,11 @@ impl CatalogController { txn.commit().await?; - self.notify_fragment_mapping(NotificationOperation::Delete, old_fragment_mappings) - .await; + // FIXME: Do not notify frontend currently, because frontend nodes might refer to old table + // catalog and need to access the old fragment. Let frontend nodes delete the old fragment + // when they receive table catalog change. + // self.notify_fragment_mapping(NotificationOperation::Delete, old_fragment_mappings) + // .await; self.notify_fragment_mapping(NotificationOperation::Add, fragment_mapping) .await; let version = self @@ -1200,13 +1203,8 @@ impl CatalogController { } txn.commit().await?; - - for mapping in fragment_mapping_to_notify { - self.env - .notification_manager() - .notify_frontend(Operation::Update, Info::ParallelUnitMapping(mapping)) - .await; - } + self.notify_fragment_mapping(Operation::Update, fragment_mapping_to_notify) + .await; Ok(()) } diff --git a/src/meta/src/stream/scale.rs b/src/meta/src/stream/scale.rs index eca6a78f35e94..6dd0d1e3b72cb 100644 --- a/src/meta/src/stream/scale.rs +++ b/src/meta/src/stream/scale.rs @@ -1732,16 +1732,21 @@ impl ScaleController { "no shuffle should have exactly one downstream actor id", ); - let downstream_fragment_id = actor_fragment_id_map_for_check - .get(downstream_actor_id) - .unwrap(); - - // dispatcher_id of dispatcher should be exactly same as downstream fragment id - // but we need to check it to make sure - debug_assert_eq!( - *downstream_fragment_id, - dispatcher.dispatcher_id as FragmentId - ); + if let Some(downstream_fragment_id) = + actor_fragment_id_map_for_check.get(downstream_actor_id) + { + // dispatcher_id of dispatcher should be exactly same as downstream fragment id + // but we need to check it to make sure + debug_assert_eq!( + *downstream_fragment_id, + dispatcher.dispatcher_id as FragmentId + ); + } else { + tracing::warn!( + "downstream actor id {} not found in fragment_actor_id_map", + downstream_actor_id + ); + } no_shuffle_target_fragment_ids .insert(dispatcher.dispatcher_id as FragmentId); diff --git a/src/prost/build.rs b/src/prost/build.rs index 3af4c9873863d..cd65b20d6f192 100644 --- a/src/prost/build.rs +++ b/src/prost/build.rs @@ -51,6 +51,7 @@ fn main() -> Result<(), Box> { "stream_plan", "stream_service", "task_service", + "telemetry", "user", ]; let protos: Vec = proto_files diff --git a/src/prost/src/lib.rs b/src/prost/src/lib.rs index dafa3a568780f..82f399d1a3de6 100644 --- a/src/prost/src/lib.rs +++ b/src/prost/src/lib.rs @@ -89,6 +89,9 @@ pub mod java_binding; #[cfg_attr(madsim, path = "sim/health.rs")] pub mod health; #[rustfmt::skip] +#[path = "sim/telemetry.rs"] +pub mod telemetry; +#[rustfmt::skip] #[path = "connector_service.serde.rs"] pub mod connector_service_serde; #[rustfmt::skip] @@ -151,6 +154,9 @@ pub mod backup_service_serde; #[rustfmt::skip] #[path = "java_binding.serde.rs"] pub mod java_binding_serde; +#[rustfmt::skip] +#[path = "telemetry.serde.rs"] +pub mod telemetry_serde; #[derive(Clone, PartialEq, Eq, Debug, Error)] #[error("field `{0}` not found")] diff --git a/src/sqlparser/src/ast/mod.rs b/src/sqlparser/src/ast/mod.rs index a5ebb66955f47..952ce05eb933f 100644 --- a/src/sqlparser/src/ast/mod.rs +++ b/src/sqlparser/src/ast/mod.rs @@ -814,10 +814,10 @@ impl fmt::Display for WindowFrameUnits { pub enum WindowFrameBound { /// `CURRENT ROW` CurrentRow, - /// ` PRECEDING` or `UNBOUNDED PRECEDING` - Preceding(Option), - /// ` FOLLOWING` or `UNBOUNDED FOLLOWING`. - Following(Option), + /// ` PRECEDING` or `UNBOUNDED PRECEDING` + Preceding(Option>), + /// ` FOLLOWING` or `UNBOUNDED FOLLOWING`. + Following(Option>), } impl fmt::Display for WindowFrameBound { diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs index 9257c28e94d39..87263dff16ee2 100644 --- a/src/sqlparser/src/parser.rs +++ b/src/sqlparser/src/parser.rs @@ -923,7 +923,7 @@ impl Parser { }) } - /// Parse `CURRENT ROW` or `{ | UNBOUNDED } { PRECEDING | FOLLOWING }` + /// Parse `CURRENT ROW` or `{ | UNBOUNDED } { PRECEDING | FOLLOWING }` pub fn parse_window_frame_bound(&mut self) -> Result { if self.parse_keywords(&[Keyword::CURRENT, Keyword::ROW]) { Ok(WindowFrameBound::CurrentRow) @@ -931,7 +931,7 @@ impl Parser { let rows = if self.parse_keyword(Keyword::UNBOUNDED) { None } else { - Some(self.parse_literal_uint()?) + Some(Box::new(self.parse_expr()?)) }; if self.parse_keyword(Keyword::PRECEDING) { Ok(WindowFrameBound::Preceding(rows)) diff --git a/src/storage/src/row_serde/mod.rs b/src/storage/src/row_serde/mod.rs index 2668d75bc1b93..4cb6696cce931 100644 --- a/src/storage/src/row_serde/mod.rs +++ b/src/storage/src/row_serde/mod.rs @@ -95,7 +95,7 @@ mod test { type_name: "", generated_or_default_column: None, description: None, - additional_columns: AdditionalColumn { + additional_column: AdditionalColumn { column_type: None, }, version: Pr13707, @@ -108,7 +108,7 @@ mod test { type_name: "", generated_or_default_column: None, description: None, - additional_columns: AdditionalColumn { + additional_column: AdditionalColumn { column_type: None, }, version: Pr13707, @@ -141,7 +141,7 @@ mod test { type_name: "", generated_or_default_column: None, description: None, - additional_columns: AdditionalColumn { + additional_column: AdditionalColumn { column_type: None, }, version: Pr13707, @@ -154,7 +154,7 @@ mod test { type_name: "", generated_or_default_column: None, description: None, - additional_columns: AdditionalColumn { + additional_column: AdditionalColumn { column_type: None, }, version: Pr13707, diff --git a/src/stream/src/executor/over_window/frame_finder.rs b/src/stream/src/executor/over_window/frame_finder.rs index 704e955508591..3154284653f11 100644 --- a/src/stream/src/executor/over_window/frame_finder.rs +++ b/src/stream/src/executor/over_window/frame_finder.rs @@ -18,8 +18,12 @@ use std::ops::Bound; use delta_btree_map::DeltaBTreeMap; +use itertools::Itertools; use risingwave_common::row::OwnedRow; -use risingwave_expr::window_function::{FrameBound, RowsFrameBounds}; +use risingwave_common::types::{Datum, Sentinelled, ToDatumRef}; +use risingwave_common::util::memcmp_encoding; +use risingwave_common::util::sort_util::cmp_datum; +use risingwave_expr::window_function::{FrameBound, RangeFrameBounds, RowsFrameBounds, StateKey}; use super::over_partition::CacheKey; @@ -143,6 +147,73 @@ pub(super) fn find_frame_end_for_rows_frame<'cache>( find_boundary_for_rows_frame::(frame_bounds, part_with_delta, curr_key) } +/// Given the first and last key in delta, calculate the order values of the first +/// and the last frames logically affected by some `RANGE` frames. +pub(super) fn calc_logical_curr_for_range_frames( + range_frames: &[&RangeFrameBounds], + delta_first_key: &StateKey, + delta_last_key: &StateKey, +) -> Option<(Sentinelled, Sentinelled)> { + calc_logical_ord_for_range_frames( + range_frames, + delta_first_key, + delta_last_key, + |bounds, v| bounds.first_curr_of(v), + |bounds, v| bounds.last_curr_of(v), + ) +} + +/// Given the curr keys of the first and the last affected frames, calculate the order +/// values of the logical start row of the first frame and the logical end row of the +/// last frame. +pub(super) fn calc_logical_boundary_for_range_frames( + range_frames: &[&RangeFrameBounds], + first_curr_key: &StateKey, + last_curr_key: &StateKey, +) -> Option<(Sentinelled, Sentinelled)> { + calc_logical_ord_for_range_frames( + range_frames, + first_curr_key, + last_curr_key, + |bounds, v| bounds.frame_start_of(v), + |bounds, v| bounds.frame_end_of(v), + ) +} + +/// Given a left logical order value (e.g. first curr order value, first delta order value), +/// find the most closed cache key in `part_with_delta`. Ideally this function returns +/// the smallest key that is larger than or equal to the given logical order (using `lower_bound`). +pub(super) fn find_left_for_range_frames<'cache>( + range_frames: &[&RangeFrameBounds], + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + logical_order_value: impl ToDatumRef, + cache_key_pk_len: usize, // this is dirty but we have no better choice +) -> &'cache CacheKey { + find_for_range_frames::( + range_frames, + part_with_delta, + logical_order_value, + cache_key_pk_len, + ) +} + +/// Given a right logical order value (e.g. last curr order value, last delta order value), +/// find the most closed cache key in `part_with_delta`. Ideally this function returns +/// the largest key that is smaller than or equal to the given logical order (using `lower_bound`). +pub(super) fn find_right_for_range_frames<'cache>( + range_frames: &[&RangeFrameBounds], + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + logical_order_value: impl ToDatumRef, + cache_key_pk_len: usize, // this is dirty but we have no better choice +) -> &'cache CacheKey { + find_for_range_frames::( + range_frames, + part_with_delta, + logical_order_value, + cache_key_pk_len, + ) +} + // -------------------------- ↑ PUBLIC INTERFACE ↑ -------------------------- fn find_curr_for_rows_frame<'cache, const LEFT: bool>( @@ -310,9 +381,151 @@ fn find_boundary_for_rows_frame<'cache, const LEFT: bool>( .unwrap() } +/// Given a pair of left and right state keys, calculate the leftmost (smallest) and rightmost +/// (largest) order values after the two given `offset_fn`s are applied, for all range frames. +/// +/// A more vivid description may be: Given a pair of left and right keys, this function pushes +/// the keys leftward and rightward respectively according to the given `offset_fn`s. +/// +/// This is not very understandable but we have to extract the code to a function to avoid +/// repeating. Check [`calc_logical_curr_for_range_frames`] and [`calc_logical_boundary_for_range_frames`] +/// if you cannot understand the purpose of this function. +fn calc_logical_ord_for_range_frames( + range_frames: &[&RangeFrameBounds], + left_key: &StateKey, + right_key: &StateKey, + left_offset_fn: impl Fn(&RangeFrameBounds, &Datum) -> Sentinelled, + right_offset_fn: impl Fn(&RangeFrameBounds, &Datum) -> Sentinelled, +) -> Option<(Sentinelled, Sentinelled)> { + if range_frames.is_empty() { + return None; + } + + let (data_type, order_type) = range_frames + .iter() + .map(|bounds| (&bounds.order_data_type, bounds.order_type)) + .all_equal_value() + .unwrap(); + + let datum_cmp = |a: &Datum, b: &Datum| cmp_datum(a, b, order_type); + + let left_given_ord = memcmp_encoding::decode_value(data_type, &left_key.order_key, order_type) + .expect("no reason to fail because we just encoded it in memory"); + let right_given_ord = + memcmp_encoding::decode_value(data_type, &right_key.order_key, order_type) + .expect("no reason to fail because we just encoded it in memory"); + + let logical_left_offset_ord = { + let mut order_value = None; + for bounds in range_frames { + let new_order_value = left_offset_fn(bounds, &left_given_ord); + order_value = match (order_value, new_order_value) { + (None, any_new) => Some(any_new), + (Some(old), new) => Some(std::cmp::min_by(old, new, |x, y| x.cmp_by(y, datum_cmp))), + }; + if !order_value.as_ref().unwrap().is_normal() { + // already unbounded + assert!( + order_value.as_ref().unwrap().is_smallest(), + "left order value should never be `Largest`" + ); + break; + } + } + order_value.expect("# of range frames > 0") + }; + + let logical_right_offset_ord = { + let mut order_value = None; + for bounds in range_frames { + let new_order_value = right_offset_fn(bounds, &right_given_ord); + order_value = match (order_value, new_order_value) { + (None, any_new) => Some(any_new), + (Some(old), new) => Some(std::cmp::max_by(old, new, |x, y| x.cmp_by(y, datum_cmp))), + }; + if !order_value.as_ref().unwrap().is_normal() { + // already unbounded + assert!( + order_value.as_ref().unwrap().is_largest(), + "right order value should never be `Smallest`" + ); + break; + } + } + order_value.expect("# of range frames > 0") + }; + + Some((logical_left_offset_ord, logical_right_offset_ord)) +} + +fn find_for_range_frames<'cache, const LEFT: bool>( + range_frames: &[&RangeFrameBounds], + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + logical_order_value: impl ToDatumRef, + cache_key_pk_len: usize, +) -> &'cache CacheKey { + debug_assert!( + part_with_delta.first_key().is_some(), + "must have something in the range cache after applying delta" + ); + + let order_type = range_frames + .iter() + .map(|bounds| bounds.order_type) + .all_equal_value() + .unwrap(); + + let search_key = Sentinelled::Normal(StateKey { + order_key: memcmp_encoding::encode_value(logical_order_value, order_type) + .expect("the data type is simple, should succeed"), + pk: if LEFT { + OwnedRow::empty() // empty row is minimal + } else { + OwnedRow::new(vec![None; cache_key_pk_len]) // all-NULL row is maximal in default order + } + .into(), + }); + + if LEFT { + let cursor = part_with_delta.lower_bound(Bound::Included(&search_key)); + if let Some((prev_key, _)) = cursor.peek_prev() + && prev_key.is_smallest() + { + // If the found lower bound of search key is right behind a smallest sentinel, + // we don't know if there's any other rows with the same order key in the state + // table but not in cache. We should conservatively return the sentinel key as + // the curr key. + prev_key + } else { + // If cursor is in ghost position, it simply means that the search key is larger + // than any existing key. Returning the last key in this case does no harm. Especially, + // if the last key is largest sentinel, the caller should extend the cache rightward + // to get possible entries with the same order value into the cache. + cursor.key().or_else(|| part_with_delta.last_key()).unwrap() + } + } else { + let cursor = part_with_delta.upper_bound(Bound::Included(&search_key)); + if let Some((next_key, _)) = cursor.peek_next() + && next_key.is_largest() + { + next_key + } else { + cursor + .key() + .or_else(|| part_with_delta.first_key()) + .unwrap() + } + } +} + #[cfg(test)] mod tests { - use risingwave_expr::window_function::{FrameBound, RowsFrameBounds}; + use std::collections::BTreeMap; + + use delta_btree_map::Change; + use risingwave_common::types::{ScalarImpl, Sentinelled}; + use risingwave_expr::window_function::FrameBound::*; + use risingwave_expr::window_function::{RowsFrameBounds, StateKey}; use super::*; @@ -326,170 +539,227 @@ mod tests { assert_equivalent( merge_rows_frames(&[]), RowsFrameBounds { - start: FrameBound::CurrentRow, - end: FrameBound::CurrentRow, + start: CurrentRow, + end: CurrentRow, }, ); let frames = [ &RowsFrameBounds { - start: FrameBound::Preceding(3), - end: FrameBound::Preceding(2), + start: Preceding(3), + end: Preceding(2), }, &RowsFrameBounds { - start: FrameBound::Preceding(1), - end: FrameBound::Preceding(4), + start: Preceding(1), + end: Preceding(4), }, ]; assert_equivalent( merge_rows_frames(&frames), RowsFrameBounds { - start: FrameBound::Preceding(4), - end: FrameBound::CurrentRow, + start: Preceding(4), + end: CurrentRow, }, ); let frames = [ &RowsFrameBounds { - start: FrameBound::Preceding(3), - end: FrameBound::Following(2), + start: Preceding(3), + end: Following(2), }, &RowsFrameBounds { - start: FrameBound::Preceding(2), - end: FrameBound::Following(3), + start: Preceding(2), + end: Following(3), }, ]; assert_equivalent( merge_rows_frames(&frames), RowsFrameBounds { - start: FrameBound::Preceding(3), - end: FrameBound::Following(3), + start: Preceding(3), + end: Following(3), }, ); let frames = [ &RowsFrameBounds { - start: FrameBound::UnboundedPreceding, - end: FrameBound::Following(2), + start: UnboundedPreceding, + end: Following(2), }, &RowsFrameBounds { - start: FrameBound::Preceding(2), - end: FrameBound::UnboundedFollowing, + start: Preceding(2), + end: UnboundedFollowing, }, ]; assert_equivalent( merge_rows_frames(&frames), RowsFrameBounds { - start: FrameBound::UnboundedPreceding, - end: FrameBound::UnboundedFollowing, + start: UnboundedPreceding, + end: UnboundedFollowing, }, ); let frames = [ &RowsFrameBounds { - start: FrameBound::UnboundedPreceding, - end: FrameBound::Following(2), + start: UnboundedPreceding, + end: Following(2), }, &RowsFrameBounds { - start: FrameBound::Following(5), - end: FrameBound::Following(2), + start: Following(5), + end: Following(2), }, ]; assert_equivalent( merge_rows_frames(&frames), RowsFrameBounds { - start: FrameBound::UnboundedPreceding, - end: FrameBound::Following(5), + start: UnboundedPreceding, + end: Following(5), }, ); } - mod rows_frame_tests { - use std::collections::BTreeMap; - - use delta_btree_map::Change; - use risingwave_common::types::{ScalarImpl, Sentinelled}; - use risingwave_expr::window_function::FrameBound::*; - use risingwave_expr::window_function::StateKey; - - use super::*; - - macro_rules! create_cache { - (..., $( $pk:literal ),* , ...) => { - { - let mut cache = create_cache!( $( $pk ),* ); - cache.insert(CacheKey::Smallest, OwnedRow::empty().into()); - cache.insert(CacheKey::Largest, OwnedRow::empty().into()); - cache - } - }; - (..., $( $pk:literal ),*) => { - { - let mut cache = create_cache!( $( $pk ),* ); - cache.insert(CacheKey::Smallest, OwnedRow::empty().into()); - cache - } - }; - ($( $pk:literal ),* , ...) => { - { - let mut cache = create_cache!( $( $pk ),* ); - cache.insert(CacheKey::Largest, OwnedRow::empty().into()); - cache - } - }; - ($( $pk:literal ),*) => { - { - #[allow(unused_mut)] - let mut cache = BTreeMap::new(); - $( - cache.insert( - CacheKey::Normal( - StateKey { - // order key doesn't matter here - order_key: vec![].into(), - pk: OwnedRow::new(vec![Some($pk.into())]).into(), - }, - ), - // value row doesn't matter here - OwnedRow::empty(), - ); - )* - cache - } - }; + macro_rules! create_cache { + (..., $( $pk:literal ),* , ...) => { + { + let mut cache = create_cache!( $( $pk ),* ); + cache.insert(CacheKey::Smallest, OwnedRow::empty().into()); + cache.insert(CacheKey::Largest, OwnedRow::empty().into()); + cache + } + }; + (..., $( $pk:literal ),*) => { + { + let mut cache = create_cache!( $( $pk ),* ); + cache.insert(CacheKey::Smallest, OwnedRow::empty().into()); + cache + } + }; + ($( $pk:literal ),* , ...) => { + { + let mut cache = create_cache!( $( $pk ),* ); + cache.insert(CacheKey::Largest, OwnedRow::empty().into()); + cache + } + }; + ($( $pk:literal ),*) => { + { + #[allow(unused_mut)] + let mut cache = BTreeMap::new(); + $( + cache.insert( + CacheKey::Normal( + StateKey { + // order key doesn't matter here + order_key: vec![].into(), + pk: OwnedRow::new(vec![Some(ScalarImpl::from($pk))]).into(), + }, + ), + // value row doesn't matter here + OwnedRow::empty(), + ); + )* + cache + } + }; + ($ord_type:expr, [..., $( ( $ord:literal, $pk:literal ) ),* , ...]) => { + { + let mut cache = create_cache!($ord_type, [$( ( $ord, $pk ) ),*]); + cache.insert(CacheKey::Smallest, OwnedRow::empty().into()); + cache.insert(CacheKey::Largest, OwnedRow::empty().into()); + cache + } + }; + ($ord_type:expr, [..., $( ( $ord:literal, $pk:literal ) ),*]) => { + { + let mut cache = create_cache!($ord_type, [$( ( $ord, $pk ) ),*]); + cache.insert(CacheKey::Smallest, OwnedRow::empty().into()); + cache + } + }; + ($ord_type:expr, [$( ( $ord:literal, $pk:literal ) ),* , ...]) => { + { + let mut cache = create_cache!($ord_type, [$( ( $ord, $pk ) ),*]); + cache.insert(CacheKey::Largest, OwnedRow::empty().into()); + cache + } + }; + ($ord_type:expr, [$( ( $ord:literal, $pk:literal ) ),*]) => { + { + #[allow(unused_mut)] + let mut cache = BTreeMap::new(); + $( + cache.insert( + CacheKey::Normal( + StateKey { + order_key: memcmp_encoding::encode_value( + Some(ScalarImpl::from($ord)), + $ord_type, + ).unwrap(), + pk: OwnedRow::new(vec![Some(ScalarImpl::from($pk))]).into(), + }, + ), + // value row doesn't matter here + OwnedRow::empty(), + ); + )* + cache + } } + } - macro_rules! create_change { - (Delete) => { - Change::Delete - }; - (Insert) => { - Change::Insert(OwnedRow::empty()) - }; - } + macro_rules! create_change { + (Delete) => { + Change::Delete + }; + (Insert) => { + Change::Insert(OwnedRow::empty()) + }; + } - macro_rules! create_delta { - ($(( $pk:literal, $change:ident )),+ $(,)?) => { - { - #[allow(unused_mut)] - let mut delta = BTreeMap::new(); - $( - delta.insert( - CacheKey::Normal( - StateKey { - // order key doesn't matter here - order_key: vec![].into(), - pk: OwnedRow::new(vec![Some($pk.into())]).into(), - }, - ), - // value row doesn't matter here - create_change!( $change ), - ); - )* - delta - } - }; - } + macro_rules! create_delta { + ($( ( $pk:literal, $change:ident ) ),+ $(,)?) => { + { + let mut delta = BTreeMap::new(); + $( + delta.insert( + CacheKey::Normal( + StateKey { + // order key doesn't matter here + order_key: vec![].into(), + pk: OwnedRow::new(vec![Some(ScalarImpl::from($pk))]).into(), + }, + ), + // value row doesn't matter here + create_change!( $change ), + ); + )* + delta + } + }; + ($ord_type:expr, [ $( ( $ord:literal, $pk:literal, $change:ident ) ),+ $(,)? ]) => { + { + let mut delta = BTreeMap::new(); + $( + delta.insert( + CacheKey::Normal( + StateKey { + order_key: memcmp_encoding::encode_value( + Some(ScalarImpl::from($ord)), + $ord_type, + ).unwrap(), + pk: OwnedRow::new(vec![Some(ScalarImpl::from($pk))]).into(), + }, + ), + // value row doesn't matter here + create_change!( $change ), + ); + )* + delta + } + }; + } + + mod rows_frame_tests { + use super::*; fn assert_cache_key_eq(given: &CacheKey, expected: impl Into) { assert_eq!( @@ -890,4 +1160,424 @@ mod tests { } } } + + mod range_frame_tests { + use risingwave_common::types::{data_types, DataType, Interval}; + use risingwave_common::util::sort_util::OrderType; + use risingwave_expr::window_function::RangeFrameOffset; + + use super::*; + + fn create_range_frame( + order_data_type: DataType, + order_type: OrderType, + start: FrameBound, + end: FrameBound, + ) -> RangeFrameBounds + where + T: Into, + { + let offset_data_type = match &order_data_type { + t @ data_types::range_frame_numeric!() => t.clone(), + data_types::range_frame_datetime!() => DataType::Interval, + _ => unreachable!(), + }; + + let map_fn = |x: T| { + RangeFrameOffset::new_for_test(x.into(), &order_data_type, &offset_data_type) + }; + let start = start.map(map_fn); + let end = end.map(map_fn); + + RangeFrameBounds { + order_data_type, + order_type, + offset_data_type, + start, + end, + } + } + + #[test] + fn test_calc_logical_for_int64_asc() { + let order_data_type = DataType::Int64; + let order_type = OrderType::ascending(); + + let range_frames = [ + &create_range_frame( + order_data_type.clone(), + order_type, + Preceding(3i64), + Preceding(2i64), + ), + &create_range_frame( + order_data_type.clone(), + order_type, + Preceding(1i64), + Following(2i64), + ), + ]; + + let ord_key_1 = StateKey { + order_key: memcmp_encoding::encode_value(&Some(ScalarImpl::Int64(1)), order_type) + .unwrap(), + pk: OwnedRow::empty().into(), + }; + let ord_key_2 = StateKey { + order_key: memcmp_encoding::encode_value(&Some(ScalarImpl::Int64(3)), order_type) + .unwrap(), + pk: OwnedRow::empty().into(), + }; + + let (logical_first_curr, logical_last_curr) = + calc_logical_curr_for_range_frames(&range_frames, &ord_key_1, &ord_key_2).unwrap(); + assert_eq!( + logical_first_curr.as_normal_expect(), + &Some(ScalarImpl::Int64(-1)) + ); + assert_eq!( + logical_last_curr.as_normal_expect(), + &Some(ScalarImpl::Int64(6)) + ); + + let (first_start, last_end) = + calc_logical_boundary_for_range_frames(&range_frames, &ord_key_1, &ord_key_2) + .unwrap(); + assert_eq!(first_start.as_normal_expect(), &Some(ScalarImpl::Int64(-2))); + assert_eq!(last_end.as_normal_expect(), &Some(ScalarImpl::Int64(5))); + } + + #[test] + fn test_calc_logical_for_timestamp_desc_nulls_first() { + let order_data_type = DataType::Timestamp; + let order_type = OrderType::descending_nulls_first(); + + let range_frames = [&create_range_frame( + order_data_type.clone(), + order_type, + Preceding(Interval::from_month_day_usec(1, 2, 3 * 1000 * 1000)), + Following(Interval::from_month_day_usec(0, 1, 0)), + )]; + + let ord_key_1 = StateKey { + order_key: memcmp_encoding::encode_value( + &Some(ScalarImpl::Timestamp( + "2024-01-28 00:30:00".parse().unwrap(), + )), + order_type, + ) + .unwrap(), + pk: OwnedRow::empty().into(), + }; + let ord_key_2 = StateKey { + order_key: memcmp_encoding::encode_value( + &Some(ScalarImpl::Timestamp( + "2024-01-26 15:47:00".parse().unwrap(), + )), + order_type, + ) + .unwrap(), + pk: OwnedRow::empty().into(), + }; + + let (logical_first_curr, logical_last_curr) = + calc_logical_curr_for_range_frames(&range_frames, &ord_key_1, &ord_key_2).unwrap(); + assert_eq!( + logical_first_curr.as_normal_expect(), + &Some(ScalarImpl::Timestamp( + "2024-01-29 00:30:00".parse().unwrap() + )) + ); + assert_eq!( + logical_last_curr.as_normal_expect(), + &Some(ScalarImpl::Timestamp( + "2023-12-24 15:46:57".parse().unwrap() + )) + ); + + let (first_start, last_end) = + calc_logical_boundary_for_range_frames(&range_frames, &ord_key_1, &ord_key_2) + .unwrap(); + assert_eq!( + first_start.as_normal_expect(), + &Some(ScalarImpl::Timestamp( + "2024-03-01 00:30:03".parse().unwrap() + )) + ); + assert_eq!( + last_end.as_normal_expect(), + &Some(ScalarImpl::Timestamp( + "2024-01-25 15:47:00".parse().unwrap() + )) + ); + } + + fn assert_find_left_right_result_eq( + order_data_type: DataType, + order_type: OrderType, + part_with_delta: DeltaBTreeMap<'_, CacheKey, OwnedRow>, + logical_order_value: ScalarImpl, + expected_left: Sentinelled, + expected_right: Sentinelled, + ) { + let frames = if matches!(order_data_type, DataType::Int32) { + [create_range_frame( + order_data_type.clone(), + order_type, + Preceding(0), // this doesn't matter here + Following(0), // this doesn't matter here + )] + } else { + panic!() + }; + let range_frames = frames.iter().collect::>(); + let logical_order_value = Some(logical_order_value); + let cache_key_pk_len = 1; + + let find_left_res = find_left_for_range_frames( + &range_frames, + part_with_delta, + &logical_order_value, + cache_key_pk_len, + ) + .clone(); + assert_eq!( + find_left_res.map(|x| x.pk.0.into_iter().next().unwrap().unwrap()), + expected_left + ); + + let find_right_res = find_right_for_range_frames( + &range_frames, + part_with_delta, + &logical_order_value, + cache_key_pk_len, + ) + .clone(); + assert_eq!( + find_right_res.map(|x| x.pk.0.into_iter().next().unwrap().unwrap()), + expected_right + ); + } + + #[test] + fn test_insert_delta_only() { + let order_data_type = DataType::Int32; + let order_type = OrderType::ascending(); + + let cache = create_cache!(); + let delta = create_delta!( + order_type, + [ + (1, 1, Insert), + (1, 11, Insert), + (3, 3, Insert), + (5, 5, Insert) + ] + ); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(2), + ScalarImpl::from(3).into(), + ScalarImpl::from(11).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(5), + ScalarImpl::from(5).into(), + ScalarImpl::from(5).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(6), + ScalarImpl::from(5).into(), + ScalarImpl::from(5).into(), + ); + } + + #[test] + fn test_simple() { + let order_data_type = DataType::Int32; + let order_type = OrderType::ascending(); + + let cache = create_cache!(order_type, [(2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]); + let delta = create_delta!( + order_type, + [(2, 2, Insert), (2, 22, Insert), (3, 3, Delete)] + ); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(0), + ScalarImpl::from(2).into(), + ScalarImpl::from(2).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(2), + ScalarImpl::from(2).into(), + ScalarImpl::from(22).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(3), + ScalarImpl::from(4).into(), + ScalarImpl::from(22).into(), + ); + } + + #[test] + fn test_empty_with_sentinels() { + let order_data_type = DataType::Int32; + let order_type = OrderType::ascending(); + + let cache = create_cache!(order_type, [..., , ...]); + let delta = create_delta!(order_type, [(1, 1, Insert), (2, 2, Insert)]); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(0), + Sentinelled::Smallest, + Sentinelled::Smallest, + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(1), + Sentinelled::Smallest, + ScalarImpl::from(1).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(2), + ScalarImpl::from(2).into(), + Sentinelled::Largest, + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(3), + Sentinelled::Largest, + Sentinelled::Largest, + ); + } + + #[test] + fn test_with_left_sentinels() { + let order_data_type = DataType::Int32; + let order_type = OrderType::ascending(); + + let cache = create_cache!(order_type, [..., (2, 2), (4, 4), (5, 5)]); + let delta = create_delta!( + order_type, + [ + (1, 1, Insert), + (2, 2, Insert), + (4, 44, Insert), + (5, 5, Delete) + ] + ); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(1), + Sentinelled::Smallest, + ScalarImpl::from(1).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(4), + ScalarImpl::from(4).into(), + ScalarImpl::from(44).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(5), + ScalarImpl::from(44).into(), + ScalarImpl::from(44).into(), + ); + } + + #[test] + fn test_with_right_sentinel() { + let order_data_type = DataType::Int32; + let order_type = OrderType::ascending(); + + let cache = create_cache!(order_type, [(2, 2), (4, 4), (5, 5), ...]); + let delta = create_delta!( + order_type, + [ + (1, 1, Insert), + (2, 2, Insert), + (4, 44, Insert), + (5, 5, Delete) + ] + ); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(1), + ScalarImpl::from(1).into(), + ScalarImpl::from(1).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(4), + ScalarImpl::from(4).into(), + Sentinelled::Largest, + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(5), + Sentinelled::Largest, + Sentinelled::Largest, + ); + } + } } diff --git a/src/stream/src/executor/over_window/general.rs b/src/stream/src/executor/over_window/general.rs index 0a56524ce4505..df4e3ac5c84c2 100644 --- a/src/stream/src/executor/over_window/general.rs +++ b/src/stream/src/executor/over_window/general.rs @@ -182,7 +182,7 @@ impl OverWindowExecutor { let order_key_data_types = args .order_key_indices .iter() - .map(|i| input_schema.fields()[*i].data_type.clone()) + .map(|i| input_schema[*i].data_type()) .collect(); let state_key_to_table_sub_pk_proj = RowConverter::calc_state_key_to_table_sub_pk_proj( diff --git a/src/stream/src/executor/over_window/over_partition.rs b/src/stream/src/executor/over_window/over_partition.rs index 8ce4cf1968ef2..a759e9e7334e3 100644 --- a/src/stream/src/executor/over_window/over_partition.rs +++ b/src/stream/src/executor/over_window/over_partition.rs @@ -26,8 +26,10 @@ use risingwave_common::array::stream_record::Record; use risingwave_common::estimate_size::collections::EstimatedBTreeMap; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::session_config::OverWindowCachePolicy as CachePolicy; -use risingwave_common::types::Sentinelled; -use risingwave_expr::window_function::{RowsFrameBounds, StateKey, WindowFuncCall}; +use risingwave_common::types::{Datum, Sentinelled}; +use risingwave_expr::window_function::{ + RangeFrameBounds, RowsFrameBounds, StateKey, WindowFuncCall, +}; use risingwave_storage::store::PrefetchOptions; use risingwave_storage::StateStore; @@ -268,6 +270,7 @@ pub(super) struct OverPartition<'a, S: StateStore> { /// The `ROWS` frame that is the union of all `ROWS` frames of all window functions in this /// over window executor. super_rows_frame_bounds: RowsFrameBounds, + range_frames: Vec<&'a RangeFrameBounds>, start_is_unbounded: bool, end_is_unbounded: bool, row_conv: RowConverter<'a>, @@ -294,6 +297,10 @@ impl<'a, S: StateStore> OverPartition<'a, S> { .collect::>(); // TODO(rc): maybe should avoid repeated merging let super_rows_frame_bounds = merge_rows_frames(&rows_frames); + let range_frames = calls + .iter() + .filter_map(|call| call.frame.bounds.as_range()) + .collect::>(); let start_is_unbounded = calls .iter() @@ -308,6 +315,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> { cache_policy, super_rows_frame_bounds, + range_frames, start_is_unbounded, end_is_unbounded, row_conv, @@ -423,10 +431,16 @@ impl<'a, S: StateStore> OverPartition<'a, S> { let delta_first = delta.first_key_value().unwrap().0.as_normal_expect(); let delta_last = delta.last_key_value().unwrap().0.as_normal_expect(); + let range_frame_logical_curr = + calc_logical_curr_for_range_frames(&self.range_frames, delta_first, delta_last); + if self.cache_policy.is_full() { // ensure everything is in the cache self.extend_cache_to_boundary(table).await?; } else { + // TODO(rc): later we should extend cache using `self.super_rows_frame_bounds` and + // `range_frame_logical_curr` as hints. + // ensure the cache covers all delta (if possible) self.extend_cache_by_range(table, delta_first..=delta_last) .await?; @@ -445,7 +459,8 @@ impl<'a, S: StateStore> OverPartition<'a, S> { let part_with_delta = DeltaBTreeMap::new(cache_inner, delta); self.stats.lookup_count += 1; - let res = self.find_affected_ranges_readonly(part_with_delta); + let res = self + .find_affected_ranges_readonly(part_with_delta, range_frame_logical_curr.as_ref()); let (need_extend_leftward, need_extend_rightward) = match res { Ok(ranges) => return Ok((part_with_delta, ranges)), @@ -479,6 +494,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> { fn find_affected_ranges_readonly<'cache>( &'_ self, part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + range_frame_logical_curr: Option<&(Sentinelled, Sentinelled)>, ) -> std::result::Result>, (bool, bool)> { if part_with_delta.first_key().is_none() { // nothing is left after applying the delta, meaning all entries are deleted @@ -487,6 +503,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> { let delta_first_key = part_with_delta.delta().first_key_value().unwrap().0; let delta_last_key = part_with_delta.delta().last_key_value().unwrap().0; + let cache_key_pk_len = delta_first_key.as_normal_expect().pk.len(); if part_with_delta.snapshot().is_empty() { // all existing keys are inserted in the delta @@ -506,22 +523,48 @@ impl<'a, S: StateStore> OverPartition<'a, S> { // to the first key is always affected. first_key } else { - find_first_curr_for_rows_frame( + let mut key = find_first_curr_for_rows_frame( &self.super_rows_frame_bounds, part_with_delta, delta_first_key, - ) + ); + + if let Some((logical_first_curr, _)) = range_frame_logical_curr { + let logical_curr = logical_first_curr.as_normal_expect(); // otherwise should go `end_is_unbounded` branch + let new_key = find_left_for_range_frames( + &self.range_frames, + part_with_delta, + logical_curr, + cache_key_pk_len, + ); + key = std::cmp::min(key, new_key); + } + + key }; let last_curr_key = if self.start_is_unbounded || delta_last_key == last_key { // similar to `first_curr_key` last_key } else { - find_last_curr_for_rows_frame( + let mut key = find_last_curr_for_rows_frame( &self.super_rows_frame_bounds, part_with_delta, delta_last_key, - ) + ); + + if let Some((_, logical_last_curr)) = range_frame_logical_curr { + let logical_curr = logical_last_curr.as_normal_expect(); // otherwise should go `start_is_unbounded` branch + let new_key = find_right_for_range_frames( + &self.range_frames, + part_with_delta, + logical_curr, + cache_key_pk_len, + ); + key = std::cmp::max(key, new_key); + } + + key }; { @@ -553,16 +596,35 @@ impl<'a, S: StateStore> OverPartition<'a, S> { return Ok(vec![]); } + let range_frame_logical_boundary = calc_logical_boundary_for_range_frames( + &self.range_frames, + first_curr_key.as_normal_expect(), + last_curr_key.as_normal_expect(), + ); + let first_frame_start = if self.start_is_unbounded || first_curr_key == first_key { // If the frame start is unbounded, or, the first curr key is the first key, then the first key // always need to be included in the affected range. first_key } else { - find_frame_start_for_rows_frame( + let mut key = find_frame_start_for_rows_frame( &self.super_rows_frame_bounds, part_with_delta, first_curr_key, - ) + ); + + if let Some((logical_first_start, _)) = range_frame_logical_boundary.as_ref() { + let logical_boundary = logical_first_start.as_normal_expect(); // otherwise should go `end_is_unbounded` branch + let new_key = find_left_for_range_frames( + &self.range_frames, + part_with_delta, + logical_boundary, + cache_key_pk_len, + ); + key = std::cmp::min(key, new_key); + } + + key }; assert!(first_frame_start <= first_curr_key); @@ -570,11 +632,24 @@ impl<'a, S: StateStore> OverPartition<'a, S> { // similar to `first_frame_start` last_key } else { - find_frame_end_for_rows_frame( + let mut key = find_frame_end_for_rows_frame( &self.super_rows_frame_bounds, part_with_delta, last_curr_key, - ) + ); + + if let Some((_, logical_last_end)) = range_frame_logical_boundary.as_ref() { + let logical_boundary = logical_last_end.as_normal_expect(); // otherwise should go `end_is_unbounded` branch + let new_key = find_right_for_range_frames( + &self.range_frames, + part_with_delta, + logical_boundary, + cache_key_pk_len, + ); + key = std::cmp::max(key, new_key); + } + + key }; assert!(last_frame_end >= last_curr_key); diff --git a/src/stream/src/executor/source/mod.rs b/src/stream/src/executor/source/mod.rs index 7df7cc2ea8373..8cdba698bec12 100644 --- a/src/stream/src/executor/source/mod.rs +++ b/src/stream/src/executor/source/mod.rs @@ -71,7 +71,7 @@ pub fn get_split_offset_col_idx( let mut split_idx = None; let mut offset_idx = None; for (idx, column) in column_descs.iter().enumerate() { - match column.additional_column_type { + match column.additional_column { AdditionalColumn { column_type: Some(ColumnType::Partition(_) | ColumnType::Filename(_)), } => { diff --git a/src/stream/src/from_proto/source/trad_source.rs b/src/stream/src/from_proto/source/trad_source.rs index a99be097e881d..28d923ffb69cc 100644 --- a/src/stream/src/from_proto/source/trad_source.rs +++ b/src/stream/src/from_proto/source/trad_source.rs @@ -96,7 +96,7 @@ impl ExecutorBuilder for SourceExecutorBuilder { // the column is from a legacy version && desc.version == ColumnDescVersion::Unspecified as i32 { - desc.additional_columns = Some(AdditionalColumn { + desc.additional_column = Some(AdditionalColumn { column_type: Some(AdditionalColumnType::Key( AdditionalColumnKey {}, )), @@ -110,7 +110,7 @@ impl ExecutorBuilder for SourceExecutorBuilder { { // compatible code: handle legacy column `_rw_kafka_timestamp` // the column is auto added for all kafka source to empower batch query on source - // solution: rewrite the column `additional_columns` to Timestamp + // solution: rewrite the column `additional_column` to Timestamp let _ = source_columns.iter_mut().map(|c| { let _ = c.column_desc.as_mut().map(|desc| { @@ -125,7 +125,7 @@ impl ExecutorBuilder for SourceExecutorBuilder { // the column is from a legacy version && desc.version == ColumnDescVersion::Unspecified as i32 { - desc.additional_columns = Some(AdditionalColumn { + desc.additional_column = Some(AdditionalColumn { column_type: Some(AdditionalColumnType::Timestamp( AdditionalColumnTimestamp {}, )),