Merge branch 'main' into yiming/refine-sink-trait

.dockerignore

@@ -48,3 +48,6 @@ risedev-components.user.env
 riselab-components.user.env
 
 .git/
+
+Dockerfile
+.dockerignore

Cargo.toml

@@ -28,7 +28,6 @@ members = [
   "src/sqlparser/test_runner",
   "src/storage",
   "src/storage/backup",
-  "src/storage/backup/cmd",
   "src/storage/compactor",
   "src/storage/hummock_sdk",
   "src/storage/hummock_test",
@@ -106,7 +105,7 @@ hashbrown = { version = "0.14.0", features = [
 criterion = { version = "0.5", features = ["async_futures"] }
 tonic = { package = "madsim-tonic", version = "0.3.1" }
 tonic-build = { package = "madsim-tonic-build", version = "0.3.1" }
-icelake = { git = "https://github.com/icelake-io/icelake", rev = "85ca0e57bf059b2e84a8bca531f9a8f3bc2f8dfd" }
+icelake = { git = "https://github.com/icelake-io/icelake", rev = "166a36b1a40a64086db09a0e0f2ed6791cec548b" }
 arrow-array = "46"
 arrow-schema = "46"
 arrow-buffer = "46"

Makefile.toml

@@ -24,6 +24,7 @@ env_scripts = [
 #!@duckscript
 
 # only duckscript can modify env variables in cargo-make
+# duckscript doc: https://github.com/sagiegurari/duckscript/blob/master/docs/sdk.md
 
 set_env ENABLE_TELEMETRY "false"
 
@@ -70,6 +71,26 @@ else
   set_env BUILD_HUMMOCK_TRACE_CMD ""
 end
 
+is_ci = get_env RISINGWAVE_CI
+is_not_ci = not ${is_ci}
+
+if ${is_not_ci}
+  query_log_path = get_env RW_QUERY_LOG_PATH
+  no_query_log_path = not ${query_log_path}
+
+  if ${no_query_log_path}
+    set_env RW_QUERY_LOG_PATH "${PREFIX_LOG}"
+  fi
+
+  rust_log = get_env RUST_LOG
+  no_rust_log = not ${rust_log}
+
+  if ${no_rust_log}
+    set_env RUST_LOG "pgwire_query_log=info"
+  else
+    set_env RUST_LOG "pgwire_query_log=info,${rust_log}"
+  end
+end
 ''',
 ]
 
@@ -1292,7 +1313,7 @@ echo "All processes has exited."
 
 [tasks.slt]
 category = "RiseDev - SQLLogicTest"
-install_crate = { version = "0.15.3", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
+install_crate = { version = "0.17.0", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
   "--help",
 ], install_command = "binstall" }
 command = "sqllogictest"

ci/Dockerfile

@@ -42,7 +42,7 @@ ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \
     [email protected] \
-    sqllogictest-bin@0.15.3 \
+    sqllogictest-bin@0.17.0 \
     && cargo install sccache \
     && cargo cache -a \
     && rm -rf "/root/.cargo/registry/index" \

ci/build-ci-image.sh

@@ -13,7 +13,7 @@ cat ../rust-toolchain
 # !!! CHANGE THIS WHEN YOU WANT TO BUMP CI IMAGE !!! #
 #          AND ALSO docker-compose.yml               #
 ######################################################
-export BUILD_ENV_VERSION=v20230914
+export BUILD_ENV_VERSION=v20230919
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -71,7 +71,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230914
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230919
     depends_on:
       - mysql
       - db
@@ -81,22 +81,23 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230914
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230919
     depends_on:
       - mysql
       - db
       - elasticsearch
       - clickhouse-server
+      - pulsar
     volumes:
       - ..:/risingwave
 
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230914
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230919
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230914
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230919
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -107,7 +108,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230914
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230919
     depends_on:
       db:
         condition: service_healthy
@@ -182,3 +183,19 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,PLAINTEXT_INTERNAL://localhost:29093
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+
+  pulsar:
+    container_name: pulsar
+    image: apachepulsar/pulsar:latest
+    command: bin/pulsar standalone
+    ports:
+      - "6650:6650"
+      - "6651:8080"
+    expose:
+      - "8080"
+      - "6650"
+    healthcheck:
+      test: [ "CMD-SHELL", "bin/pulsar-admin brokers healthcheck"]
+      interval: 5s
+      timeout: 5s
+      retries: 5

ci/scripts/build.sh

@@ -51,13 +51,12 @@ cargo build \
     -p risingwave_regress_test \
     -p risingwave_sqlsmith \
     -p risingwave_compaction_test \
-    -p risingwave_backup_cmd \
     -p risingwave_e2e_extended_mode_test \
     $RISINGWAVE_FEATURE_FLAGS \
     --profile "$profile"
 
 
-artifacts=(risingwave sqlsmith compaction-test backup-restore risingwave_regress_test risingwave_e2e_extended_mode_test risedev-dev delete-range-test)
+artifacts=(risingwave sqlsmith compaction-test risingwave_regress_test risingwave_e2e_extended_mode_test risedev-dev delete-range-test)
 
 echo "--- Show link info"
 ldd target/"$profile"/risingwave

ci/scripts/check.sh

@@ -50,7 +50,4 @@ sccache --zero-stats
 
 echo "--- Run audit check"
 cargo audit \
-  --ignore RUSTSEC-2023-0052 --ignore RUSTSEC-2022-0093
- # https://github.com/risingwavelabs/risingwave/issues/11842
- # https://github.com/risingwavelabs/risingwave/issues/11986
-
+  --ignore RUSTSEC-2023-0052 # https://github.com/risingwavelabs/risingwave/issues/11842

ci/scripts/docker.sh

@@ -7,9 +7,26 @@ ghcraddr="ghcr.io/risingwavelabs/risingwave"
 dockerhubaddr="risingwavelabs/risingwave"
 arch="$(uname -m)"
 
+echo "--- ghcr login"
+echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
+
+echo "--- dockerhub login"
+echo "$DOCKER_TOKEN" | docker login -u "risingwavelabs" --password-stdin
+
 # Build RisingWave docker image ${BUILDKITE_COMMIT}-${arch}
 echo "--- docker build and tag"
-docker build -f docker/Dockerfile --build-arg "GIT_SHA=${BUILDKITE_COMMIT}" -t "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" --target risingwave .
+docker buildx create \
+  --name container \
+  --driver=docker-container
+
+docker buildx build -f docker/Dockerfile \
+  --build-arg "GIT_SHA=${BUILDKITE_COMMIT}" -t "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" \
+  --progress plain \
+  --builder=container \
+  --load \
+  --cache-to "type=registry,ref=ghcr.io/risingwavelabs/risingwave-build-cache:${arch}" \
+  --cache-from "type=registry,ref=ghcr.io/risingwavelabs/risingwave-build-cache:${arch}" \
+  .
 
 echo "--- check the image can start correctly"
 container_id=$(docker run -d "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" playground)
@@ -25,12 +42,6 @@ fi
 echo "--- docker images"
 docker images
 
-echo "--- ghcr login"
-echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
-
-echo "--- dockerhub login"
-echo "$DOCKER_TOKEN" | docker login -u "risingwavelabs" --password-stdin
-
 echo "--- docker push to ghcr"
 docker push "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}"
 

ci/scripts/e2e-pulsar-sink-test.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+source ci/scripts/common.sh
+
+while getopts 'p:' opt; do
+    case ${opt} in
+        p )
+            profile=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+download_and_prepare_rw "$profile" source
+
+echo "--- starting risingwave cluster"
+cargo make ci-start ci-pulsar-test
+sleep 1
+
+echo "--- waiting until pulsar is healthy"
+HTTP_CODE=404
+MAX_RETRY=20
+while [[ $HTTP_CODE -ne 200 && MAX_RETRY -gt 0 ]]
+do
+    HTTP_CODE=$(curl --connect-timeout 2 -s -o /dev/null -w ''%{http_code}'' http://pulsar:8080/admin/v2/clusters)
+    ((MAX_RETRY--))
+    sleep 5
+done
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+echo "--- testing pulsar sink"
+sqllogictest -p 4566 -d dev './e2e_test/sink/pulsar_sink.slt'
+
+sleep 1
+
+echo "--- Kill cluster"
+cargo make ci-kill