Merge branch 'main' of https://github.com/risingwavelabs/risingwave i…

…nto wt-config-doc

.github/workflows/cherry-pick-to-release-branch.yml

@@ -3,7 +3,7 @@ on:
   pull_request:
     branches:
       - main
-    types: ["closed"]
+    types: ["closed", "labeled"]
 
 jobs:
   release_pull_request_1_7:

ci/docker-compose.yml

@@ -116,7 +116,7 @@ services:
     volumes:
       - ..:/risingwave
 
-  release-env:
+  release-env-x86:
     # build binaries on a earlier Linux distribution (therefore with earlier version GLIBC)
     # See https://github.com/risingwavelabs/risingwave/issues/4556 for more details.
     #
@@ -132,6 +132,12 @@ services:
     volumes:
       - ..:/mnt
 
+  release-env-arm:
+    image: quay.io/pypa/manylinux2014_aarch64
+    working_dir: /mnt
+    volumes:
+      - ..:/mnt
+
   elasticsearch:
     container_name: elasticsearch
     image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0

ci/scripts/release.sh

@@ -4,29 +4,30 @@
 set -euo pipefail
 
 REPO_ROOT=${PWD}
+ARCH="$(uname -m)"
 
 echo "--- Check env"
 if [ "${BUILDKITE_SOURCE}" != "schedule" ] && [ "${BUILDKITE_SOURCE}" != "webhook" ] && [[ -z "${BINARY_NAME+x}" ]]; then
   exit 0
 fi
 
 echo "--- Install aws cli"
-curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip"
 unzip -q awscliv2.zip && ./aws/install && mv /usr/local/bin/aws /bin/aws
 
 echo "--- Install lld"
 # The lld in the CentOS 7 repository is too old and contains a bug that causes a linker error.
 # So we install a newer version here. (17.0.6, latest version at the time of writing)
 # It is manually built in the same environent and uploaded to S3.
-aws s3 cp s3://ci-deps-dist/llvm-lld-manylinux2014_x86_64.tar.gz .
-tar -zxvf llvm-lld-manylinux2014_x86_64.tar.gz --directory=/usr/local
+aws s3 cp s3://ci-deps-dist/llvm-lld-manylinux2014_${ARCH}.tar.gz .
+tar -zxvf llvm-lld-manylinux2014_${ARCH}.tar.gz --directory=/usr/local
 ld.lld --version
 
 echo "--- Install dependencies for openssl"
 yum install -y perl-core
 
 echo "--- Install java and maven"
-yum install -y java-11-openjdk java-11-openjdk-devel wget python3 cyrus-sasl-devel
+yum install -y java-11-openjdk java-11-openjdk-devel wget python3 python3-devel cyrus-sasl-devel
 pip3 install toml-cli
 wget https://ci-deps-dist.s3.amazonaws.com/apache-maven-3.9.3-bin.tar.gz && tar -zxvf apache-maven-3.9.3-bin.tar.gz
 export PATH="${REPO_ROOT}/apache-maven-3.9.3/bin:$PATH"
@@ -40,8 +41,13 @@ source ci/scripts/common.sh
 unset RUSTC_WRAPPER # disable sccache
 
 echo "--- Install protoc3"
-curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
-unzip -o protoc-3.15.8-linux-x86_64.zip -d protoc
+PROTOC_ARCH=${ARCH}
+if [ ${ARCH} == "aarch64" ]; then
+  # shellcheck disable=SC1068
+  PROTOC_ARCH="aarch_64"
+fi
+curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-${PROTOC_ARCH}.zip
+unzip -o protoc-3.15.8-linux-${PROTOC_ARCH}.zip -d protoc
 mv ./protoc/bin/protoc /usr/local/bin/
 mv ./protoc/include/* /usr/local/include/
 
@@ -63,11 +69,11 @@ cd target/release && chmod +x risingwave risectl
 
 echo "--- Upload nightly binary to s3"
 if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then
-  tar -czvf risingwave-"$(date '+%Y%m%d')"-x86_64-unknown-linux.tar.gz risingwave
-  aws s3 cp risingwave-"$(date '+%Y%m%d')"-x86_64-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary
+  tar -czvf risingwave-"$(date '+%Y%m%d')"-${ARCH}-unknown-linux.tar.gz risingwave
+  aws s3 cp risingwave-"$(date '+%Y%m%d')"-${ARCH}-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary
 elif [[ -n "${BINARY_NAME+x}" ]]; then
-    tar -czvf risingwave-${BINARY_NAME}-x86_64-unknown-linux.tar.gz risingwave
-    aws s3 cp risingwave-${BINARY_NAME}-x86_64-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary
+    tar -czvf risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz risingwave
+    aws s3 cp risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary
 fi
 
 echo "--- Build connector node"
@@ -88,23 +94,29 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then
   dnf install -y gh
 
   echo "--- Release create"
-  gh release create "${BUILDKITE_TAG}" --notes "release ${BUILDKITE_TAG}" -d -p
+  response=$(gh api repos/risingwavelabs/risingwave/releases/tags/${BUILDKITE_TAG} 2>&1)
+  if [[ $response == *"Not Found"* ]]; then
+    echo "Tag ${BUILDKITE_TAG} does not exist. Creating release..."
+    gh release create "${BUILDKITE_TAG}" --notes "release ${BUILDKITE_TAG}" -d -p
+  else
+    echo "Tag ${BUILDKITE_TAG} already exists. Skipping release creation."
+  fi
 
   echo "--- Release upload risingwave asset"
-  tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz risingwave
-  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz
+  tar -czvf risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.tar.gz risingwave
+  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.tar.gz
 
   echo "--- Release upload risingwave debug info"
-  tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.dwp.tar.gz risingwave.dwp
-  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.dwp.tar.gz
+  tar -czvf risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.dwp.tar.gz risingwave.dwp
+  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.dwp.tar.gz
 
   echo "--- Release upload risectl asset"
-  tar -czvf risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz risectl
-  gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz
+  tar -czvf risectl-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.tar.gz risectl
+  gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.tar.gz
 
   echo "--- Release upload risingwave-all-in-one asset"
-  tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux-all-in-one.tar.gz risingwave libs
-  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux-all-in-one.tar.gz
+  tar -czvf risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux-all-in-one.tar.gz risingwave libs
+  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-{ARCH}-unknown-linux-all-in-one.tar.gz
 fi
 
 

ci/workflows/docker.yml

@@ -46,18 +46,36 @@ steps:
             DOCKER_TOKEN: docker-token
     retry: *auto-retry
 
-  - label: "pre build binary"
+  - label: "pre build binary: amd64"
     command: "ci/scripts/release.sh"
     plugins:
       - seek-oss/aws-sm#v2.3.1:
           env:
             GITHUB_TOKEN: github-token
       - docker-compose#v4.9.0:
-          run: release-env
+          run: release-env-x86
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
           propagate-environment: true
           environment:
             - BINARY_NAME
             - GITHUB_TOKEN
     retry: *auto-retry
+
+  - label: "pre build binary: aarch64 "
+    command: "ci/scripts/release.sh"
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GITHUB_TOKEN: github-token
+      - docker-compose#v4.9.0:
+          run: release-env-arm
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          propagate-environment: true
+          environment:
+            - BINARY_NAME
+            - GITHUB_TOKEN
+    agents:
+      queue: "linux-arm64"
+    retry: *auto-retry

ci/workflows/main-cron.yml

@@ -877,15 +877,15 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "release"
+  - label: "release amd64"
     command: "ci/scripts/release.sh"
     if: build.tag != null
     plugins:
       - seek-oss/aws-sm#v2.3.1:
           env:
             GITHUB_TOKEN: github-token
       - docker-compose#v4.9.0:
-          run: release-env
+          run: release-env-x86
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
           environment:
@@ -895,6 +895,26 @@ steps:
     timeout_in_minutes: 60
     retry: *auto-retry
 
+  - label: "release aarch64"
+    command: "ci/scripts/release.sh"
+    if: build.tag != null
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GITHUB_TOKEN: github-token
+      - docker-compose#v4.9.0:
+          run: release-env-arm
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - GITHUB_TOKEN
+            - BUILDKITE_TAG
+            - BUILDKITE_SOURCE
+    agents:
+      queue: "linux-arm64"
+    timeout_in_minutes: 60
+    retry: *auto-retry
+
   - label: "release docker image: amd64"
     command: "ci/scripts/docker.sh"
     key: "build-amd64"

e2e_test/batch/basic/logical_view.slt.part

@@ -40,7 +40,7 @@ SELECT * FROM v3;
 statement error
 DROP TABLE t;
 
-statement error other relation\(s\) depend on it
+statement error Permission denied
 DROP VIEW v2;
 
 statement ok

e2e_test/ddl/alter_rename.slt

@@ -229,7 +229,7 @@ DROP SCHEMA schema1;
 statement ok
 DROP SINK sink1;
 
-statement error other relation\(s\) depend on it
+statement error Permission denied
 DROP VIEW v5;
 
 statement ok

e2e_test/ddl/dependency_check.slt

@@ -16,7 +16,7 @@ create index i_b1 on b(b1);
 statement ok
 create materialized view mv1 as select * from a join b on a.a1 = b.b1;
 
-statement error other relation\(s\) depend on it
+statement error Permission denied
 drop index i_a1;
 
 statement ok
@@ -25,7 +25,7 @@ drop materialized view mv1;
 statement ok
 create materialized view mv2 as with ctx as (select a1 from a) select b1 from b;
 
-statement error other relation\(s\) depend on it
+statement error Permission denied
 drop table a;
 
 statement ok
@@ -48,13 +48,13 @@ create view v2 as select * from v;
 statement ok
 create materialized view mv3 as select * from v2;
 
-statement error other relation\(s\) depend on it
+statement error Permission denied
 drop source src;
 
-statement error other relation\(s\) depend on it
+statement error Permission denied
 drop view v;
 
-statement error other relation\(s\) depend on it
+statement error Permission denied
 drop view v2;
 
 statement ok

e2e_test/over_window/generated/batch/basic/cross_check.slt.part

@@ -21,6 +21,16 @@ select
 from v_a natural join v_c;
 ----
 
+query i
+select
+    id, out1, out10, out11
+from v_a_d
+except
+select
+    id, out1, out10, out11
+from v_a natural join v_d;
+----
+
 query i
 select
     id, out3, out4, out7, out8, out9
@@ -31,6 +41,26 @@ select
 from v_b natural join v_c;
 ----
 
+query i
+select
+    id, out3, out4, out10, out11
+from v_b_d
+except
+select
+    id, out3, out4, out10, out11
+from v_b natural join v_d;
+----
+
+query i
+select
+    id, out7, out8, out9, out10, out11
+from v_c_d
+except
+select
+    id, out7, out8, out9, out10, out11
+from v_c natural join v_d;
+----
+
 query i
 select
     id, out1, out2, out3, out4, out5, out6, out7, out8, out9
@@ -40,3 +70,13 @@ select
     id, out1, out2, out3, out4, out5, out6, out7, out8, out9
 from v_a natural join v_b natural join v_c;
 ----
+
+query i
+select
+    id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11
+from v_a_b_c_d
+except
+select
+    id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11
+from v_a natural join v_b natural join v_c natural join v_d;
+----

e2e_test/over_window/generated/batch/basic/mod.slt.part

@@ -35,6 +35,14 @@ select * from v_c order by id;
 100003  100  208  2  723  807  723  NULL  NULL  NULL  NULL
 100004  103  200  2  702  808  702  NULL  NULL  NULL  NULL
 
+query iiiiiiii
+select * from v_d order by id;
+----
+100001	100	200	1	701	805	1	2124
+100002	100	200	2	700	806	1	2124
+100003	100	208	2	723	807	1	2124
+100004	103	200	2	702	808	2	702
+
 include ./cross_check.slt.part
 
 statement ok
@@ -72,6 +80,16 @@ select * from v_c order by id;
 100005  100  200  3  717  810  717   700   700  NULL  NULL
 100006  105  204  5  703  828  703  NULL  NULL  NULL  NULL
 
+query iiiiiiii
+select * from v_d order by id;
+----
+100001	100	200	1	701	805	1	2124
+100002	100	200	2	700	806	1	2841
+100003	100	208	2	723	807	1	2841
+100004	103	200	2	702	808	2	702
+100005	100	200	3	717	810	1	2140
+100006	105	204	5	703	828	5	703
+
 include ./cross_check.slt.part
 
 statement ok
@@ -113,6 +131,16 @@ select * from v_c order by id;
 100005  100  200  1  717  810  717   723   701   806   806
 100006  105  204  5  703  828  703  NULL  NULL  NULL  NULL
 
+query iiiiiiii
+select * from v_d order by id;
+----
+100001	100	200	1	701	805	1	2940
+100002	100	200	2	799	806	1	2940
+100003	100	200	2	723	807	1	2940
+100004	103	200	2	702	808	2	702
+100005	100	200	1	717	810	1	2940
+100006	105	204	5	703	828	5	703
+
 include ./cross_check.slt.part
 
 statement ok
@@ -139,6 +167,13 @@ select * from v_c order by id;
 100005  100  200  1  717  810  717   701   701  NULL  NULL
 100006  105  204  5  703  828  703  NULL  NULL  NULL  NULL
 
+query iiiiiiii
+select * from v_d order by id;
+----
+100001	100	200	1	701	805	1	1418
+100005	100	200	1	717	810	1	1418
+100006	105	204	5	703	828	5	703
+
 include ./cross_check.slt.part
 
 include ./teardown.slt.part