Merge branch 'release-1.8' into wcy/cherry-pick

.github/labeler.yml

@@ -24,6 +24,10 @@ labels:
   title: "^doc.*"
 - label: "type/deprecate"
   title: "^deprecate.*"
+- label: "cherry-pick"
+  title: "^cherry-pick.*"
+- label: "cherry-pick"
+  title: "^cherry pick.*"
 
 - label: "breaking-change"
   body: '- \[x\] My PR contains breaking changes'

.github/pr-title-checker-config.json

@@ -4,7 +4,7 @@
     "color": "B60205"
   },
   "CHECKS": {
-    "regexp": "^(feat|fix|test|refactor|chore|style|doc|perf|build|ci|revert|deprecate)(\\(.*\\))?:.*",
+    "regexp": "^(feat|fix|test|refactor|chore|style|doc|perf|build|ci|revert|deprecate|cherry pick|cherry-pick)(\\(.*\\))?:.*",
     "ignoreLabels" : ["ignore-title"]
   },
   "MESSAGES": {

.typos.toml

@@ -10,6 +10,7 @@ steam = "stream"        # You played with Steam games too much.
 ot = "ot"
 bui = "bui"
 mosquitto = "mosquitto" # This is a MQTT broker.
+abd = "abd"
 
 [default.extend-identifiers]
 

Cargo.toml

@@ -160,6 +160,12 @@ opentelemetry = "0.21"
 opentelemetry-otlp = "0.14"
 opentelemetry_sdk = { version = "0.21", default-features = false }
 opentelemetry-semantic-conventions = "0.13"
+sea-orm = { version = "0.12.14", features = [
+  "sqlx-mysql",
+  "sqlx-postgres",
+  "sqlx-sqlite",
+  "runtime-tokio-native-tls",
+] }
 tokio-util = "0.7"
 tracing-opentelemetry = "0.22"
 

ci/scripts/e2e-sql-migration.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+export RW_PREFIX=$PWD/.risingwave
+export RW_PREFIX_DATA=$RW_PREFIX/data
+
+source ci/scripts/common.sh
+
+wait_for_recovery() {
+  set +e
+  timeout 20s bash -c '
+    while true; do
+      echo "Polling every 1s to check if the recovery is complete for 20s"
+      if psql -h localhost -p 4566 -d dev -U root -c "FLUSH;" </dev/null
+      then exit 0;
+      else sleep 1;
+      fi
+    done
+  '
+  STATUS=$?
+  set -e
+  if [[ $STATUS -ne 0 ]]; then
+    echo "Cluster failed to get recovered: $STATUS"
+    exit 1
+  else
+    echo "Cluster is recovered"
+  fi
+}
+
+while getopts 'p:' opt; do
+    case ${opt} in
+        p )
+            profile=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+download_and_prepare_rw "$profile" common
+
+echo "--- starting risingwave cluster, ci-1cn-1fe-with-recovery"
+RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
+cargo make ci-start ci-1cn-1fe-with-recovery
+
+echo "--- init cluster with some data & DDL"
+sqllogictest -d dev -h localhost -p 4566 './e2e_test/sql_migration/prepare.slt'
+
+echo "--- kill cluster"
+cargo make ci-kill
+
+echo "--- restart etcd"
+cargo make dev ci-meta-etcd-for-migration
+
+echo "--- run migration"
+mkdir -p "${RW_PREFIX_DATA}/sqlite/"
+./target/debug/risingwave risectl \
+meta \
+migration \
+--etcd-endpoints localhost:2388 \
+--sql-endpoint sqlite://"${RW_PREFIX_DATA}/sqlite/metadata.db"\?mode=rwc \
+-f
+
+echo "--- kill etcd"
+cargo make ci-kill
+
+echo "--- starting risingwave cluster, meta-1cn-1fe-sqlite"
+cargo make dev meta-1cn-1fe-sqlite
+
+echo "--- wait for recovery"
+wait_for_recovery
+
+echo "--- run check"
+sqllogictest -d dev -h localhost -p 4566 './e2e_test/sql_migration/check.slt'
+
+echo "--- kill cluster"
+cargo make kill
+
+echo "--- clean data"
+cargo make clean-data
+

ci/workflows/pull-request.yml

@@ -715,6 +715,22 @@ steps:
     timeout_in_minutes: 30
     retry: *auto-retry
 
+  - label: "e2e sql migration test"
+    command: "ci/scripts/e2e-sql-migration.sh -p ci-dev"
+    if: build.pull_request.labels includes "ci/run-e2e-sql-migration-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-sql-migration-tests?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+      - "docslt"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 10
+    retry: *auto-retry
+
   # FIXME(kwannoel): Let the github PR labeller label it, if sqlsmith source files has changes.
   - label: "fuzz test"
     command: "ci/scripts/pr-fuzz-test.sh -p ci-dev"