From 3f77d350eb8955897573ac1a44d648bfbaf5456c Mon Sep 17 00:00:00 2001
From: Noel Kwan <noelkwan1998@gmail.com>
Date: Tue, 16 Jan 2024 14:51:59 +0800
Subject: [PATCH] add tests which compare runtime

---
 ci/scripts/run-backfill-tests.sh              | 70 +++++++++++++++++++
 .../create_arrangement_backfill_mv.slt        |  5 ++
 .../backfill/runtime/create_no_shuffle_mv.slt |  5 ++
 e2e_test/backfill/runtime/create_table.slt    |  2 +
 e2e_test/backfill/runtime/insert.slt          |  5 ++
 e2e_test/backfill/runtime/validate_rows.slt   |  4 ++
 6 files changed, 91 insertions(+)
 create mode 100644 e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt
 create mode 100644 e2e_test/backfill/runtime/create_no_shuffle_mv.slt
 create mode 100644 e2e_test/backfill/runtime/create_table.slt
 create mode 100644 e2e_test/backfill/runtime/insert.slt
 create mode 100644 e2e_test/backfill/runtime/validate_rows.slt

diff --git a/ci/scripts/run-backfill-tests.sh b/ci/scripts/run-backfill-tests.sh
index 46bad790d21f7..f26a983d67050 100755
--- a/ci/scripts/run-backfill-tests.sh
+++ b/ci/scripts/run-backfill-tests.sh
@@ -200,12 +200,82 @@ test_sink_backfill_recovery() {
   wait
 }
 
+test_no_shuffle_backfill_runtime() {
+  echo "--- e2e, test_no_shuffle_backfill_runtime"
+  cargo make ci-start $CLUSTER_PROFILE
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_table.slt'
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt'
+
+  # Provide updates ...
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt' 2>&1 1>out.log &
+
+  # ... and concurrently create mv.
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_no_shuffle_mv.slt'
+
+  wait
+
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows.slt'
+
+  cargo make kill
+  cargo make wait-processes-exit
+}
+
+test_arrangement_backfill_runtime() {
+  echo "--- e2e, test_arrangement_backfill_runtime"
+  cargo make ci-start $CLUSTER_PROFILE
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_table.slt'
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt'
+
+  # Provide updates ...
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt' 2>&1 1>out.log &
+
+  # ... and concurrently create mv.
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt'
+
+  wait
+
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows.slt'
+
+  cargo make kill
+  cargo make wait-processes-exit
+}
+
+test_no_shuffle_backfill_snapshot_only_runtime() {
+  echo "--- e2e, test_no_shuffle_backfill_snapshot_only_runtime"
+  cargo make ci-start $CLUSTER_PROFILE
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_table.slt'
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt'
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt'
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_no_shuffle_mv.slt'
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows.slt'
+
+  cargo make kill
+  cargo make wait-processes-exit
+}
+
+test_arrangement_backfill_snapshot_only_runtime() {
+  echo "--- e2e, test_arrangement_backfill_snapshot_only_runtime"
+  cargo make ci-start $CLUSTER_PROFILE
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_table.slt'
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt'
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert.slt'
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt'
+  sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows.slt'
+
+  cargo make kill
+  cargo make wait-processes-exit
+}
+
 main() {
   set -euo pipefail
   test_snapshot_and_upstream_read
   test_backfill_tombstone
   test_replication_with_column_pruning
   test_sink_backfill_recovery
+  test_no_shuffle_backfill_runtime
+  test_arrangement_backfill_runtime
+  test_no_shuffle_backfill_snapshot_only_runtime
+  test_arrangement_backfill_snapshot_only_runtime
 }
 
 main
diff --git a/e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt b/e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt
new file mode 100644
index 0000000000000..abc7fe9f3392e
--- /dev/null
+++ b/e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt
@@ -0,0 +1,5 @@
+statement ok
+SET STREAMING_ENABLE_ARRANGEMENT_BACKFILL=true;
+
+statement ok
+CREATE MATERIALIZED VIEW m1 AS SELECT * FROM t;
\ No newline at end of file
diff --git a/e2e_test/backfill/runtime/create_no_shuffle_mv.slt b/e2e_test/backfill/runtime/create_no_shuffle_mv.slt
new file mode 100644
index 0000000000000..89806810649df
--- /dev/null
+++ b/e2e_test/backfill/runtime/create_no_shuffle_mv.slt
@@ -0,0 +1,5 @@
+statement ok
+SET STREAMING_ENABLE_ARRANGEMENT_BACKFILL=false;
+
+statement ok
+CREATE MATERIALIZED VIEW m1 AS SELECT * FROM t;
\ No newline at end of file
diff --git a/e2e_test/backfill/runtime/create_table.slt b/e2e_test/backfill/runtime/create_table.slt
new file mode 100644
index 0000000000000..fd71dffa36fb1
--- /dev/null
+++ b/e2e_test/backfill/runtime/create_table.slt
@@ -0,0 +1,2 @@
+statement ok
+CREATE TABLE t (v1 int, v2 varchar, v3 bigint);
\ No newline at end of file
diff --git a/e2e_test/backfill/runtime/insert.slt b/e2e_test/backfill/runtime/insert.slt
new file mode 100644
index 0000000000000..357ed56a61430
--- /dev/null
+++ b/e2e_test/backfill/runtime/insert.slt
@@ -0,0 +1,5 @@
+statement ok
+INSERT INTO t select 1, 'jakbj2khbe2', 22222222222 from generate_series(1, 500000);
+
+statement ok
+flush;
\ No newline at end of file
diff --git a/e2e_test/backfill/runtime/validate_rows.slt b/e2e_test/backfill/runtime/validate_rows.slt
new file mode 100644
index 0000000000000..ed3c9e1080458
--- /dev/null
+++ b/e2e_test/backfill/runtime/validate_rows.slt
@@ -0,0 +1,4 @@
+query I
+select count(*) from m1;
+----
+1000000
\ No newline at end of file