Merge branch 'main' into sts/stream_key_sink_pk_mismatch

.dockerignore

@@ -48,3 +48,6 @@ risedev-components.user.env
 riselab-components.user.env
 
 .git/
+
+Dockerfile
+.dockerignore

Cargo.toml

@@ -106,7 +106,7 @@ hashbrown = { version = "0.14.0", features = [
 criterion = { version = "0.5", features = ["async_futures"] }
 tonic = { package = "madsim-tonic", version = "0.3.1" }
 tonic-build = { package = "madsim-tonic-build", version = "0.3.1" }
-icelake = { git = "https://github.com/icelake-io/icelake", rev = "85ca0e57bf059b2e84a8bca531f9a8f3bc2f8dfd" }
+icelake = { git = "https://github.com/icelake-io/icelake", rev = "166a36b1a40a64086db09a0e0f2ed6791cec548b" }
 arrow-array = "46"
 arrow-schema = "46"
 arrow-buffer = "46"
@@ -223,6 +223,6 @@ quanta = { git = "https://github.com/madsim-rs/quanta.git", rev = "948bdc3" }
 getrandom = { git = "https://github.com/madsim-rs/getrandom.git", rev = "8daf97e" }
 tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "fe39bb8e" }
 tokio-retry = { git = "https://github.com/madsim-rs/rust-tokio-retry.git", rev = "95e2fd3" }
-tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "4538cd6" }
+tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "ac00d88" }
 # patch: unlimit 4MB message size for grpc client
 etcd-client = { git = "https://github.com/risingwavelabs/etcd-client.git", rev = "d55550a" }

ci/scripts/docker.sh

@@ -7,9 +7,26 @@ ghcraddr="ghcr.io/risingwavelabs/risingwave"
 dockerhubaddr="risingwavelabs/risingwave"
 arch="$(uname -m)"
 
+echo "--- ghcr login"
+echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
+
+echo "--- dockerhub login"
+echo "$DOCKER_TOKEN" | docker login -u "risingwavelabs" --password-stdin
+
 # Build RisingWave docker image ${BUILDKITE_COMMIT}-${arch}
 echo "--- docker build and tag"
-docker build -f docker/Dockerfile --build-arg "GIT_SHA=${BUILDKITE_COMMIT}" -t "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" --target risingwave .
+docker buildx create \
+  --name container \
+  --driver=docker-container
+
+docker buildx build -f docker/Dockerfile \
+  --build-arg "GIT_SHA=${BUILDKITE_COMMIT}" -t "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" \
+  --progress plain \
+  --builder=container \
+  --load \
+  --cache-to "type=registry,ref=ghcr.io/risingwavelabs/risingwave-build-cache:${arch}" \
+  --cache-from "type=registry,ref=ghcr.io/risingwavelabs/risingwave-build-cache:${arch}" \
+  .
 
 echo "--- check the image can start correctly"
 container_id=$(docker run -d "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" playground)
@@ -25,12 +42,6 @@ fi
 echo "--- docker images"
 docker images
 
-echo "--- ghcr login"
-echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
-
-echo "--- dockerhub login"
-echo "$DOCKER_TOKEN" | docker login -u "risingwavelabs" --password-stdin
-
 echo "--- docker push to ghcr"
 docker push "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}"
 

ci/scripts/run-e2e-test.sh

@@ -65,7 +65,11 @@ echo "--- e2e, $mode, batch"
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
 cluster_start
 sqllogictest -p 4566 -d dev './e2e_test/ddl/**/*.slt' --junit "batch-ddl-${profile}"
-sqllogictest -p 4566 -d dev './e2e_test/background_ddl/**/*.slt' --junit "batch-ddl-${profile}"
+if [[ $mode != "standalone" ]]; then
+  sqllogictest -p 4566 -d dev './e2e_test/background_ddl/**/*.slt' --junit "batch-ddl-${profile}"
+else
+  echo "Skipping background_ddl test for $mode"
+fi
 sqllogictest -p 4566 -d dev './e2e_test/visibility_mode/*.slt' --junit "batch-${profile}"
 sqllogictest -p 4566 -d dev './e2e_test/database/prepare.slt'
 sqllogictest -p 4566 -d test './e2e_test/database/test.slt'

docker/Dockerfile

@@ -17,32 +17,56 @@ COPY ./proto  /risingwave/proto
 
 RUN cd /risingwave/dashboard && npm i && npm run build-static && rm -rf node_modules
 
-FROM base AS builder
+FROM base AS rust-base
 
-RUN apt-get update && apt-get -y install make cmake protobuf-compiler curl bash lld maven unzip
+RUN apt-get -y install make cmake protobuf-compiler curl bash lld unzip
 
 SHELL ["/bin/bash", "-c"]
 
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /root/.cargo/bin/:$PATH
+ENV CARGO_INCREMENTAL=0
 
-RUN mkdir -p /risingwave
+COPY rust-toolchain rust-toolchain
 
-WORKDIR /risingwave
+# We need to add the `rustfmt` dependency, otherwise `risingwave_pb` will not compile
+RUN rustup self update \
+  && rustup set profile minimal \
+  && rustup show \
+  && rustup component add rustfmt
 
-COPY ./ /risingwave
 
-ENV PATH /root/.cargo/bin/:$PATH
+# TODO: cargo-chef doesn't work well now, because we update Cargo.lock very often.
+# We may consider sccache instead.
 
-ENV IN_CONTAINER=1
+# RUN cargo install --git https://github.com/xxchan/cargo-chef cargo-chef --locked --rev 11f9fed
 
+# FROM rust-base AS rust-planner
+
+# RUN mkdir -p /risingwave
+# WORKDIR /risingwave
+# COPY ./ /risingwave
+
+# RUN cargo chef prepare --recipe-path recipe.json
+
+# FROM rust-base AS rust-builder
+
+# RUN mkdir -p /risingwave
+# WORKDIR /risingwave
+
+# COPY --from=rust-planner /risingwave/recipe.json recipe.json
+
+# # Build dependencies - this can be cached if the dependencies don't change
+# RUN cargo chef cook --release --recipe-path recipe.json
+
+FROM rust-base AS rust-builder
+
+# Build application
 ARG GIT_SHA
 ENV GIT_SHA=$GIT_SHA
 
-# We need to add the `rustfmt` dependency, otherwise `risingwave_pb` will not compile
-RUN rustup self update \
-  && rustup set profile minimal \
-  && rustup show \
-  && rustup component add rustfmt
+COPY ./ /risingwave
+WORKDIR /risingwave
 
 RUN cargo fetch && \
   cargo build -p risingwave_cmd_all --release --features "rw-static-link" && \
@@ -53,9 +77,39 @@ RUN cargo fetch && \
   chmod +x /risingwave/bin/jeprof && \
   mkdir -p /risingwave/lib && cargo clean
 
-RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true && \
-    mkdir -p /risingwave/bin/connector-node && \
-    tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node
+FROM base AS java-planner
+
+RUN mkdir -p /risingwave
+WORKDIR /risingwave
+
+COPY java /risingwave/java/
+
+# Move java/**/pom.xml to poms/**/pom.xml
+RUN find . -name pom.xml -exec bash -c 'mkdir -p poms/$(dirname {}); mv {} poms/{}' \;
+
+# We use rust-maven-plugin to build java-binding. So it's FROM rust-base
+FROM rust-base AS java-builder
+
+RUN apt-get -y install maven
+
+RUN mkdir -p /risingwave
+WORKDIR /risingwave/java
+
+# 1. copy only poms
+COPY --from=java-planner /risingwave/poms /risingwave/java/
+
+# 2. start downloading dependencies
+RUN mvn dependency:go-offline --fail-never
+
+# 3. add all source code and start compiling
+# TODO: only add java related code so that changing rust code won't recompile java code
+# Currently java-binding depends on the workspace Cargo.toml, which depends on the whole rust codebase
+# Besides, rust-maven-plugin sets --target-dir, so the dependencies are built twice. How to dedup?
+COPY ./ /risingwave
+
+RUN mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true && \
+  mkdir -p /risingwave/bin/connector-node && \
+  tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node
 
 FROM base AS risingwave
 
@@ -66,18 +120,19 @@ RUN apt-get -y install gdb \
 
 RUN mkdir -p /risingwave/bin/connector-node && mkdir -p /risingwave/lib
 
-COPY --from=builder /risingwave/bin/risingwave /risingwave/bin/risingwave
-COPY --from=builder /risingwave/bin/risingwave.dwp /risingwave/bin/risingwave.dwp
-COPY --from=builder /risingwave/bin/connector-node /risingwave/bin/connector-node
+COPY --from=rust-builder /risingwave/bin/risingwave /risingwave/bin/risingwave
+COPY --from=rust-builder /risingwave/bin/risingwave.dwp /risingwave/bin/risingwave.dwp
+COPY --from=java-builder /risingwave/bin/connector-node /risingwave/bin/connector-node
 COPY --from=dashboard-builder /risingwave/dashboard/out /risingwave/ui
-COPY --from=builder /risingwave/bin/jeprof /usr/local/bin/jeprof
+COPY --from=rust-builder /risingwave/bin/jeprof /usr/local/bin/jeprof
 
 # Set default playground mode to docker-playground profile
 ENV PLAYGROUND_PROFILE docker-playground
 # Set default dashboard UI to local path instead of github proxy
 ENV RW_DASHBOARD_UI_PATH /risingwave/ui
 # Set default connector libs path
 ENV CONNECTOR_LIBS_PATH /risingwave/bin/connector-node/libs
+ENV IN_CONTAINER=1
 
 ENTRYPOINT [ "/risingwave/bin/risingwave" ]
 CMD [ "playground" ]

e2e_test/background_ddl/basic.slt

@@ -8,7 +8,7 @@ statement ok
 CREATE TABLE t (v1 int);
 
 statement ok
-INSERT INTO t select * from generate_series(1, 500000);
+INSERT INTO t select * from generate_series(1, 300000);
 
 statement ok
 FLUSH;
@@ -22,12 +22,11 @@ CREATE MATERIALIZED VIEW m2 as SELECT * FROM t;
 statement ok
 CREATE MATERIALIZED VIEW m3 as SELECT * FROM t;
 
-sleep 3s
-
-query I
-select count(*) from rw_catalog.rw_ddl_progress;
-----
-3
+# Disable the flaky check:
+# query I
+# select count(*) from rw_catalog.rw_ddl_progress;
+# ----
+# 3
 
 statement error
 SELECT * FROM m1;
@@ -42,17 +41,17 @@ sleep 30s
 query I
 select count(*) from m1;
 ----
-500000
+300000
 
 query I
 select count(*) from m2;
 ----
-500000
+300000
 
 query I
 select count(*) from m3;
 ----
-500000
+300000
 
 statement ok
 DROP MATERIALIZED VIEW m1;

e2e_test/batch/catalog/metabase.slt.part

@@ -0,0 +1,24 @@
+query
+SELECT
+    NULL AS TABLE_CAT,
+    n.nspname AS TABLE_SCHEM,
+    ct.relname AS TABLE_NAME,
+    a.attname AS COLUMN_NAME,
+    (
+    information_schema._pg_expandarray(i.indkey)
+    ).n AS KEY_SEQ,
+    ci.relname AS PK_NAME,
+    information_schema._pg_expandarray(i.indkey) AS KEYS,
+    a.attnum AS A_ATTNUM
+FROM
+    pg_catalog.pg_class ct
+    JOIN pg_catalog.pg_attribute a ON (ct.oid = a.attrelid)
+    JOIN pg_catalog.pg_namespace n ON (ct.relnamespace = n.oid)
+    JOIN pg_catalog.pg_index i ON (a.attrelid = i.indrelid)
+    JOIN pg_catalog.pg_class ci ON (ci.oid = i.indexrelid)
+WHERE
+    true
+    AND n.nspname = 'public'
+    AND ct.relname = 'sentences'
+    AND i.indisprimary
+----