feat(backfill): add cluster config to enable arrangement_backfill (#…

…15692)

ci/scripts/sql/backfill/replication_with_column_pruning/create_base_table.sql

@@ -1,2 +1,2 @@
-SET STREAMING_ENABLE_ARRANGEMENT_BACKFILL=true;
+SET STREAMING_USE_ARRANGEMENT_BACKFILL=true;
 create table t1 (x int, y int, z int);

e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt

@@ -1,5 +1,5 @@
 statement ok
-SET STREAMING_ENABLE_ARRANGEMENT_BACKFILL=true;
+SET STREAMING_USE_ARRANGEMENT_BACKFILL=true;
 
 statement ok
 CREATE MATERIALIZED VIEW arrangement_backfill AS SELECT * FROM t;

e2e_test/backfill/runtime/create_no_shuffle_mv.slt

@@ -1,5 +1,5 @@
 statement ok
-SET STREAMING_ENABLE_ARRANGEMENT_BACKFILL=false;
+SET STREAMING_USE_ARRANGEMENT_BACKFILL=false;
 
 statement ok
 CREATE MATERIALIZED VIEW no_shuffle_backfill AS SELECT * FROM t;

e2e_test/batch/catalog/pg_settings.slt.part

@@ -1,62 +1,62 @@
 query TT
 SELECT context, name FROM pg_catalog.pg_settings ORDER BY (context, name);
 ----
-internal    block_size_kb
-internal    bloom_false_positive
-internal    data_directory
-internal    parallel_compact_size_mb
-internal    sstable_size_mb
-internal    state_store
-postmaster  backup_storage_directory
-postmaster  backup_storage_url
-postmaster  barrier_interval_ms
-postmaster  checkpoint_frequency
-postmaster  enable_tracing
-postmaster  max_concurrent_creating_streaming_jobs
-postmaster  pause_on_next_bootstrap
-user        application_name
-user        background_ddl
-user        batch_enable_distributed_dml
-user        batch_parallelism
-user        bytea_output
-user        client_encoding
-user        client_min_messages
-user        create_compaction_group_for_mv
-user        datestyle
-user        extra_float_digits
-user        idle_in_transaction_session_timeout
-user        intervalstyle
-user        lock_timeout
-user        max_split_range_gap
-user        query_epoch
-user        query_mode
-user        row_security
-user        rw_batch_enable_lookup_join
-user        rw_batch_enable_sort_agg
-user        rw_enable_join_ordering
-user        rw_enable_share_plan
-user        rw_enable_two_phase_agg
-user        rw_force_split_distinct_agg
-user        rw_force_two_phase_agg
-user        rw_implicit_flush
-user        rw_streaming_allow_jsonb_in_stream_key
-user        rw_streaming_enable_bushy_join
-user        rw_streaming_enable_delta_join
-user        rw_streaming_over_window_cache_policy
-user        search_path
-user        server_encoding
-user        server_version
-user        server_version_num
-user        sink_decouple
-user        standard_conforming_strings
-user        statement_timeout
-user        streaming_enable_arrangement_backfill
-user        streaming_parallelism
-user        streaming_rate_limit
-user        synchronize_seqscans
-user        timezone
-user        transaction_isolation
-user        visibility_mode
+internal block_size_kb
+internal bloom_false_positive
+internal data_directory
+internal parallel_compact_size_mb
+internal sstable_size_mb
+internal state_store
+postmaster backup_storage_directory
+postmaster backup_storage_url
+postmaster barrier_interval_ms
+postmaster checkpoint_frequency
+postmaster enable_tracing
+postmaster max_concurrent_creating_streaming_jobs
+postmaster pause_on_next_bootstrap
+user application_name
+user background_ddl
+user batch_enable_distributed_dml
+user batch_parallelism
+user bytea_output
+user client_encoding
+user client_min_messages
+user create_compaction_group_for_mv
+user datestyle
+user extra_float_digits
+user idle_in_transaction_session_timeout
+user intervalstyle
+user lock_timeout
+user max_split_range_gap
+user query_epoch
+user query_mode
+user row_security
+user rw_batch_enable_lookup_join
+user rw_batch_enable_sort_agg
+user rw_enable_join_ordering
+user rw_enable_share_plan
+user rw_enable_two_phase_agg
+user rw_force_split_distinct_agg
+user rw_force_two_phase_agg
+user rw_implicit_flush
+user rw_streaming_allow_jsonb_in_stream_key
+user rw_streaming_enable_bushy_join
+user rw_streaming_enable_delta_join
+user rw_streaming_over_window_cache_policy
+user search_path
+user server_encoding
+user server_version
+user server_version_num
+user sink_decouple
+user standard_conforming_strings
+user statement_timeout
+user streaming_parallelism
+user streaming_rate_limit
+user streaming_use_arrangement_backfill
+user synchronize_seqscans
+user timezone
+user transaction_isolation
+user visibility_mode
 
 query TT
 SELECT * FROM pg_catalog.pg_settings where name='dummy';

src/common/src/config.rs

@@ -868,6 +868,12 @@ pub struct StreamingDeveloperConfig {
 
     #[serde(default = "default::developer::memory_controller_threshold_stable")]
     pub memory_controller_threshold_stable: f64,
+
+    #[serde(default = "default::developer::stream_enable_arrangement_backfill")]
+    /// Enable arrangement backfill
+    /// If true, the arrangement backfill will be disabled,
+    /// even if session variable set.
+    pub enable_arrangement_backfill: bool,
 }
 
 /// The subsections `[batch.developer]`.
@@ -1487,6 +1493,9 @@ pub mod default {
         pub fn memory_controller_threshold_stable() -> f64 {
             0.7
         }
+        pub fn stream_enable_arrangement_backfill() -> bool {
+            true
+        }
     }
 
     pub use crate::system_param::default as system;

src/common/src/session_config/mod.rs

@@ -141,7 +141,7 @@ pub struct ConfigMap {
 
     /// Enable arrangement backfill for streaming queries. Defaults to false.
     #[parameter(default = false)]
-    streaming_enable_arrangement_backfill: bool,
+    streaming_use_arrangement_backfill: bool,
 
     /// Allow `jsonb` in stream key
     #[parameter(default = false, rename = "rw_streaming_allow_jsonb_in_stream_key")]

src/config/example.toml

@@ -105,6 +105,7 @@ stream_hash_agg_max_dirty_groups_heap_size = 67108864
 stream_memory_controller_threshold_aggressive = 0.9
 stream_memory_controller_threshold_graceful = 0.8
 stream_memory_controller_threshold_stable = 0.7
+stream_enable_arrangement_backfill = true
 
 [storage]
 share_buffers_sync_parallelism = 1