From 21331d0cad768bcf1b778223dbc597e0646a63ba Mon Sep 17 00:00:00 2001 From: justindeguzman Date: Fri, 18 Oct 2024 16:36:43 -0700 Subject: [PATCH] Autogenerate settings --- .gitignore | 4 ++ copyClickhouseRepoDocs.sh | 32 +++++++--- package.json | 2 +- scripts/settings/autogenerate-settings.sh | 77 +++++++++++++++++++++++ 4 files changed, 106 insertions(+), 9 deletions(-) create mode 100755 scripts/settings/autogenerate-settings.sh diff --git a/.gitignore b/.gitignore index 451786a1037..a86b9a98d35 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,7 @@ sidebar_links.txt orphaned_pages.txt active_links.txt active_links.json + +# Files used by scripts to autogenerate settings +FormatFactorySettingsDeclaration.h +Settings.cpp diff --git a/copyClickhouseRepoDocs.sh b/copyClickhouseRepoDocs.sh index 2b35b148c68..ce6eed25675 100755 --- a/copyClickhouseRepoDocs.sh +++ b/copyClickhouseRepoDocs.sh @@ -1,8 +1,16 @@ #! ./bin/bash -echo "Start Cloning" + +SCRIPT_NAME=$(basename "$0") + +echo "[$SCRIPT_NAME] Start tasks for copying docs from ClickHouse repo" + +# Clone ClickHouse repo +echo "[$SCRIPT_NAME] Start cloning ClickHouse repo" git clone --depth 1 https://github.com/ClickHouse/ClickHouse.git -echo "Cloning completed" -echo "Start Copying" +echo "[$SCRIPT_NAME] Cloning completed" + +# Copy docs folders from ClickHouse repo to docs folder +echo "[$SCRIPT_NAME] Start copying docs" cp -r ClickHouse/docs/en/development docs/en/ cp -r ClickHouse/docs/en/engines docs/en/ cp -r ClickHouse/docs/en/getting-started docs/en/ @@ -11,14 +19,22 @@ cp -r ClickHouse/docs/en/operations docs/en/ cp -r ClickHouse/docs/en/sql-reference docs/en/ cp -r ClickHouse/docs/ru docs/ cp -r ClickHouse/docs/zh docs/ -echo "Copying completed" -echo "----Generate Changelog----" +# Necessary for autogenerating settings +cp ClickHouse/src/Core/FormatFactorySettingsDeclaration.h "$(dirname "$0")" +cp ClickHouse/src/Core/Settings.cpp "$(dirname "$0")" + +echo "[$SCRIPT_NAME] Copying completed" + +echo "[$SCRIPT_NAME] Generate changelog" cp docs/en/_placeholders/changelog/_index.md docs/en/whats-new/changelog/index.md sed "0,/^# $(date +%Y) Changelog/d" \ < ClickHouse/CHANGELOG.md \ >> docs/en/whats-new/changelog/index.md -echo "Start Deleting ClickHouse" + +# Delete ClickHouse repo +echo "[$SCRIPT_NAME] Start deleting ClickHouse repo" rm -r ClickHouse -echo "Deleting ClickHouse folder completed" -echo "----END----" +echo "[$SCRIPT_NAME] Deleting ClickHouse repo completed" + +echo "[$SCRIPT_NAME] Finish tasks for copying docs from ClickHouse repo" diff --git a/package.json b/package.json index 31deaddd3d0..9324a85abf9 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "serve": "docusaurus serve", "build-api-doc": "node clickhouseapi.js", "build-swagger": "npx @redocly/cli build-docs https://api.clickhouse.cloud/v1 --output build/en/cloud/manage/api/swagger.html", - "new-build": "bash ./copyClickhouseRepoDocs.sh && yarn build-api-doc && yarn build && yarn build-swagger", + "new-build": "bash ./copyClickhouseRepoDocs.sh && bash ./scripts/settings/autogenerate-settings.sh && yarn build-api-doc && yarn build && yarn build-swagger", "start": "docusaurus start", "swizzle": "docusaurus swizzle", "write-heading-ids": "docusaurus write-heading-ids" diff --git a/scripts/settings/autogenerate-settings.sh b/scripts/settings/autogenerate-settings.sh new file mode 100755 index 00000000000..7efbf98e1ad --- /dev/null +++ b/scripts/settings/autogenerate-settings.sh @@ -0,0 +1,77 @@ +#! ./bin/bash + +SCRIPT_NAME=$(basename "$0") + +echo "[$SCRIPT_NAME] Autogenerating settings" + +# Install ClickHouse +if [ ! -f ./clickhouse ]; then + echo -e "[$SCRIPT_NAME] Installing ClickHouse binary\n" + curl https://clickhouse.com/ | sh +fi + +# Autogenerate Format settings +./clickhouse -q " +WITH +'FormatFactorySettingsDeclaration.h' AS cpp_file, +settings_from_cpp AS +( + SELECT extract(line, 'M\\(\\w+, (\\w+),') AS name + FROM file(cpp_file, LineAsString) + WHERE match(line, '^\\s*M\\(') +), +main_content AS +( + SELECT format('## {} {}\\n\\nType: {}\\n\\nDefault value: {}\\n\\n{}\\n\\n', name, '{#'||name||'}', type, default, trim(BOTH '\\n' FROM description)) + FROM system.settings WHERE name IN settings_from_cpp + ORDER BY name +), +'--- +title: Format Settings +sidebar_label: Format Settings +slug: /en/operations/settings/formats +toc_max_heading_level: 2 +--- + +These settings are autogenerated from [source](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/FormatFactorySettings.h). +' AS prefix +SELECT prefix || (SELECT groupConcat(*) FROM main_content) +INTO OUTFILE 'docs/en/operations/settings/settings-formats.md' TRUNCATE FORMAT LineAsString +" + +# Autogenerate Format settings +./clickhouse -q " +WITH +'Settings.cpp' AS cpp_file, +settings_from_cpp AS +( + SELECT extract(line, 'M\\(\\w+, (\\w+),') AS name + FROM file(cpp_file, LineAsString) + WHERE match(line, '^\\s*M\\(') +), +main_content AS +( + SELECT format('## {} {}\\n\\nType: {}\\n\\nDefault value: {}\\n\\n{}\\n\\n', name, '{#'||name||'}', type, default, trim(BOTH '\\n' FROM description)) + FROM system.settings WHERE name IN settings_from_cpp + ORDER BY name +), +'--- +title: Core Settings +sidebar_label: Core Settings +slug: /en/operations/settings/settings +toc_max_heading_level: 2 +--- + +All below settings are also available in table [system.settings](/docs/en/operations/system-tables/settings). These settings are autogenerated from [source](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.cpp). +' AS prefix +SELECT prefix || (SELECT groupConcat(*) FROM main_content) +INTO OUTFILE 'docs/en/operations/settings/settings.md' TRUNCATE FORMAT LineAsString +" + +# Delete ClickHouse +if [ -f ./clickhouse ]; then + echo -e "\n[$SCRIPT_NAME] Deleting ClickHouse binary" + rm ./clickhouse +fi + +echo "[$SCRIPT_NAME] Autogenerating settings completed"