From 982de5627ef42a53f0244c8930742cedbcfd1a87 Mon Sep 17 00:00:00 2001 From: Dmitry Potepalov Date: Wed, 9 Oct 2024 10:49:01 +0200 Subject: [PATCH] Allow all possible experimental indexes on restore Presently we don't know which experimental features/indexes were enabled during table creation, because those could have been enabled on a per-session/per-query basis and not captured in the create_table_query. To avoid failures on restore, just enable all experimental features when creating the tables. Should be relatively harmless, since it affects only the restoration session; also most of those flags do not affect tables that don't have experimental indexes. --- .../coordinator/plugins/clickhouse/steps.py | 23 ++++++- .../plugins/clickhouse/test_steps.py | 60 +++++++++++++++++-- 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/astacus/coordinator/plugins/clickhouse/steps.py b/astacus/coordinator/plugins/clickhouse/steps.py index 098353a2..ccf5b288 100644 --- a/astacus/coordinator/plugins/clickhouse/steps.py +++ b/astacus/coordinator/plugins/clickhouse/steps.py @@ -631,11 +631,30 @@ def _create_dbs(client: ClickHouseClient) -> Iterator[Awaitable[None]]: # we need to re-enable these custom global settings when creating the table again. # We can enable these settings unconditionally because they are harmless # for tables not needing them. + # Upstream has introduced a similar list in the replica recovery context: + # https://github.com/ClickHouse/ClickHouse/commit/48ed54e822f6ed6f6bdd67db3df7a4a58e550bbb + # the two should be kept in sync until system.tables captures the query context. + b"SET allow_experimental_inverted_index=true", + b"SET allow_experimental_codecs=true", + b"SET allow_experimental_live_view=true", + b"SET allow_experimental_window_view=true", + b"SET allow_experimental_funnel_functions=true", + b"SET allow_experimental_nlp_functions=true", + b"SET allow_experimental_hash_functions=true", + b"SET allow_experimental_object_type=true", b"SET allow_experimental_annoy_index=true", + b"SET allow_experimental_usearch_index=true", + b"SET allow_experimental_bigint_types=true", + b"SET allow_experimental_window_functions=true", b"SET allow_experimental_geo_types=true", - b"SET allow_experimental_object_type=true", - b"SET allow_suspicious_codecs=true", + b"SET allow_experimental_map_type=true", b"SET allow_suspicious_low_cardinality_types=true", + b"SET allow_suspicious_fixed_string_types=true", + b"SET allow_suspicious_indices=true", + b"SET allow_suspicious_codecs=true", + b"SET allow_hyperscan=true", + b"SET allow_simdjson=true", + b"SET allow_deprecated_syntax_for_merge_tree=true", # If a table was created with flatten_nested=0, we must be careful to not re-create the # table with flatten_nested=1, since this would recreate the table with a different schema. # If a table was created with flatten_nested=1, the query in system.tables.create_table_query diff --git a/tests/unit/coordinator/plugins/clickhouse/test_steps.py b/tests/unit/coordinator/plugins/clickhouse/test_steps.py index 22c61313..0420e944 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_steps.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_steps.py @@ -859,11 +859,27 @@ async def test_creates_all_replicated_databases_and_tables_in_manifest() -> None b"SET receive_timeout=10.0", b"CREATE DATABASE `db-two` UUID '00000000-0000-0000-0000-000000000012'" b" ENGINE = Replicated('/clickhouse/databases/db%2Dtwo', '{my_shard}', '{my_replica}')", + b"SET allow_experimental_inverted_index=true", + b"SET allow_experimental_codecs=true", + b"SET allow_experimental_live_view=true", + b"SET allow_experimental_window_view=true", + b"SET allow_experimental_funnel_functions=true", + b"SET allow_experimental_nlp_functions=true", + b"SET allow_experimental_hash_functions=true", + b"SET allow_experimental_object_type=true", b"SET allow_experimental_annoy_index=true", + b"SET allow_experimental_usearch_index=true", + b"SET allow_experimental_bigint_types=true", + b"SET allow_experimental_window_functions=true", b"SET allow_experimental_geo_types=true", - b"SET allow_experimental_object_type=true", - b"SET allow_suspicious_codecs=true", + b"SET allow_experimental_map_type=true", b"SET allow_suspicious_low_cardinality_types=true", + b"SET allow_suspicious_fixed_string_types=true", + b"SET allow_suspicious_indices=true", + b"SET allow_suspicious_codecs=true", + b"SET allow_hyperscan=true", + b"SET allow_simdjson=true", + b"SET allow_deprecated_syntax_for_merge_tree=true", b"SET flatten_nested=0", b"CREATE TABLE db-one.table-uno ...", b"CREATE TABLE db-one.table-dos ...", @@ -917,11 +933,27 @@ async def test_creates_all_replicated_databases_and_tables_in_manifest_with_cust b"CREATE DATABASE `db-two` UUID '00000000-0000-0000-0000-000000000012'" b" ENGINE = Replicated('/clickhouse/databases/db%2Dtwo', '{my_shard}', '{my_replica}') " b"SETTINGS cluster_username='alice', cluster_password='alice_secret'", + b"SET allow_experimental_inverted_index=true", + b"SET allow_experimental_codecs=true", + b"SET allow_experimental_live_view=true", + b"SET allow_experimental_window_view=true", + b"SET allow_experimental_funnel_functions=true", + b"SET allow_experimental_nlp_functions=true", + b"SET allow_experimental_hash_functions=true", + b"SET allow_experimental_object_type=true", b"SET allow_experimental_annoy_index=true", + b"SET allow_experimental_usearch_index=true", + b"SET allow_experimental_bigint_types=true", + b"SET allow_experimental_window_functions=true", b"SET allow_experimental_geo_types=true", - b"SET allow_experimental_object_type=true", - b"SET allow_suspicious_codecs=true", + b"SET allow_experimental_map_type=true", b"SET allow_suspicious_low_cardinality_types=true", + b"SET allow_suspicious_fixed_string_types=true", + b"SET allow_suspicious_indices=true", + b"SET allow_suspicious_codecs=true", + b"SET allow_hyperscan=true", + b"SET allow_simdjson=true", + b"SET allow_deprecated_syntax_for_merge_tree=true", b"SET flatten_nested=0", b"CREATE TABLE db-one.table-uno ...", b"CREATE TABLE db-one.table-dos ...", @@ -960,11 +992,27 @@ async def test_drops_each_database_on_all_servers_before_recreating_it() -> None b" ENGINE = Replicated('/clickhouse/databases/db%2Dtwo', '{my_shard}', '{my_replica}')", ] queries_expected_on_a_single_node = [ + b"SET allow_experimental_inverted_index=true", + b"SET allow_experimental_codecs=true", + b"SET allow_experimental_live_view=true", + b"SET allow_experimental_window_view=true", + b"SET allow_experimental_funnel_functions=true", + b"SET allow_experimental_nlp_functions=true", + b"SET allow_experimental_hash_functions=true", + b"SET allow_experimental_object_type=true", b"SET allow_experimental_annoy_index=true", + b"SET allow_experimental_usearch_index=true", + b"SET allow_experimental_bigint_types=true", + b"SET allow_experimental_window_functions=true", b"SET allow_experimental_geo_types=true", - b"SET allow_experimental_object_type=true", - b"SET allow_suspicious_codecs=true", + b"SET allow_experimental_map_type=true", b"SET allow_suspicious_low_cardinality_types=true", + b"SET allow_suspicious_fixed_string_types=true", + b"SET allow_suspicious_indices=true", + b"SET allow_suspicious_codecs=true", + b"SET allow_hyperscan=true", + b"SET allow_simdjson=true", + b"SET allow_deprecated_syntax_for_merge_tree=true", b"SET flatten_nested=0", b"CREATE TABLE db-one.table-uno ...", b"CREATE TABLE db-one.table-dos ...",