Merge branch 'main' into tab/deprecate-private-link-connection

Makefile.toml

@@ -27,7 +27,7 @@ env_scripts = [
 
   set_env ENABLE_TELEMETRY "false"
   set_env RW_TELEMETRY_TYPE "test"
-  set_env RW_SECRET_STORE_PRIVATE_KEY_HEX "0123456789abcdef"
+  set_env RW_SECRET_STORE_PRIVATE_KEY_HEX "0123456789abcdef0123456789abcdef"
   set_env RW_TEMP_SECRET_FILE_DIR "${PREFIX_SECRET}"
 
   is_sanitizer_enabled = get_env ENABLE_SANITIZER

ci/docker-compose.yml

@@ -1,4 +1,5 @@
 services:
+  # TODO: Rename this to `postgres`
   db:
     image: postgres:15-alpine
     environment:

ci/scripts/common.sh

@@ -15,7 +15,7 @@ export MCLI_DOWNLOAD_BIN=https://rw-ci-deps-dist.s3.amazonaws.com/mc
 export GCLOUD_DOWNLOAD_TGZ=https://rw-ci-deps-dist.s3.amazonaws.com/google-cloud-cli-475.0.0-linux-x86_64.tar.gz
 export NEXTEST_HIDE_PROGRESS_BAR=true
 export RW_TELEMETRY_TYPE=test
-export RW_SECRET_STORE_PRIVATE_KEY_HEX="0123456789abcdef"
+export RW_SECRET_STORE_PRIVATE_KEY_HEX="0123456789abcdef0123456789abcdef"
 export RUST_MIN_STACK=4194304
 
 unset LANG

ci/scripts/docker-scout-notify.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+buildkite-agent meta-data get SCOUT_REPORT > scout.report
+cat >> step.yaml << EOF
+steps:
+  - label: "docker scout slack notification"
+    command: "echo '--- notify the scout report'"
+    notify:
+      - slack:
+          channels:
+            - "#notification-buildkite"
+          message: |
+            ${report}
+EOF
+
+buildkite-agent pipeline upload step.yaml

ci/scripts/docker-scout.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+ghcraddr="ghcr.io/risingwavelabs/risingwave"
+arch="$(uname -m)"
+image="${ghcraddr}:${BUILDKITE_COMMIT}-${arch}"
+
+echo "--- ghcr login"
+echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
+
+echo "--- dockerhub login"
+echo "$DOCKER_TOKEN" | docker login -u "risingwavelabs" --password-stdin
+
+echo "--- pull docker image"
+echo "pulling ${image}"
+docker pull "${image}"
+
+echo "--- check vulnerabilities"
+mkdir -p scout
+function docker-scout {
+    docker run -it -e DOCKER_SCOUT_HUB_USER=risingwavelabs -e DOCKER_SCOUT_HUB_PASSWORD=$DOCKER_TOKEN -u root -v /var/run/docker.sock:/var/run/docker.sock -v $PWD/scout:/scout docker/scout-cli "$@"
+}
+
+echo "--- scout quickview"
+docker-scout quickview ${image} -o /scout/quickview.output
+cat scout/quickview.output
+read C H M L <<< $(grep 'Target'  scout/quickview.output | awk -F'[ │ ]+' '{print $4, $5, $6, $7}' | sed 's/[CHML]//g')
+cat >> scout/report.output << EOF
+Docker Scout Report:
+            - Critical: $C
+            - High: $H
+            - Medium: $M
+            - Low: $L
+EOF
+
+buildkite-agent meta-data set "SCOUT_REPORT" "$(cat scout/report.output)"
+
+echo "--- scout recommendations"
+docker-scout recommendations "${image}"
+
+echo "--- scout cves"
+docker-scout cves "${image}"

ci/scripts/docker.sh

@@ -29,13 +29,19 @@ docker buildx create \
   --name container \
   --driver=docker-container
 
+PULL_PARAM=""
+if [[ "${ALWAYS_PULL:-false}" = "true" ]]; then
+  PULL_PARAM="--pull"
+fi
+
 docker buildx build -f docker/Dockerfile \
   --build-arg "GIT_SHA=${BUILDKITE_COMMIT}" \
   --build-arg "CARGO_PROFILE=${CARGO_PROFILE}" \
   -t "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" \
   --progress plain \
   --builder=container \
   --load \
+  ${PULL_PARAM} \
   --cache-to "type=registry,ref=ghcr.io/risingwavelabs/risingwave-build-cache:${arch}" \
   --cache-from "type=registry,ref=ghcr.io/risingwavelabs/risingwave-build-cache:${arch}" \
   .

ci/scripts/e2e-kafka-sink-test.sh

@@ -150,7 +150,7 @@ echo "preparing confluent schema registry"
 python3 -m pip install --break-system-packages requests confluent-kafka
 
 echo "testing protobuf"
-sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/protobuf.slt'
+risedev slt 'e2e_test/sink/kafka/protobuf.slt'
 
 echo "testing avro"
-sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/avro.slt'
+risedev slt 'e2e_test/sink/kafka/avro.slt'

ci/scripts/e2e-source-test.sh

@@ -139,9 +139,6 @@ risedev slt './e2e_test/source_legacy/cdc/cdc.check_new_rows.slt'
 # drop relations
 risedev slt './e2e_test/source_legacy/cdc/cdc_share_stream_drop.slt'
 
-echo "--- postgres_query tvf test"
-risedev slt './e2e_test/source_legacy/tvf/postgres_query.slt'
-
 echo "--- Kill cluster"
 risedev ci-kill
 export RISINGWAVE_CI=true

ci/scripts/notify.py

@@ -31,8 +31,8 @@
     "recovery-test-deterministic": ["runji", "noelkwan"],
     "background-ddl-arrangement-backfill-recovery-test-deterministic": ["runji", "noelkwan"],
     "background-ddl-recovery-test-deterministic": ["runji", "noelkwan"],
-    "e2e-iceberg-sink-test": ["zilin"],
-    "e2e-iceberg-sink-v2-test": ["zilin"],
+    "e2e-iceberg-sink-test": ["zilin", "xinhao"],
+    "e2e-iceberg-sink-v2-test": ["zilin", "xinhao"],
     "e2e-java-binding-tests": ["yiming"],
     "s3-source-check-aws": ["bohan"],
     "s3-source-check-aws-json-parser": ["bohan"],
@@ -55,8 +55,9 @@
     "e2e-redis-sink-tests": ["xinhao"],
     "e2e-starrocks-sink-tests": ["xinhao"],
     "e2e-cassandra-sink-tests": ["xinhao"],
-    "e2e-clickhouse-sink-tests": ["bohan","xinhao"],
+    "e2e-clickhouse-sink-tests": ["bohan", "xinhao"],
     "e2e-pulsar-sink-tests": ["bohan"],
+    "e2e-mqtt-sink-tests": ["xinhao"],
     "connector-node-integration-test": ["siyuan"],
 }