From 87a33127cfffc34b82eb613a980802211e3396dd Mon Sep 17 00:00:00 2001 From: Kamil Holubicki Date: Wed, 20 Mar 2024 10:36:27 +0100 Subject: [PATCH 1/4] Percona Telemetry Component implemented. --- CMakeLists.txt | 2 + components/percona_telemetry/CMakeLists.txt | 50 ++ components/percona_telemetry/README.md | 53 ++ components/percona_telemetry/common.h | 9 + components/percona_telemetry/component.cc | 102 ++++ components/percona_telemetry/config.cc | 130 +++++ components/percona_telemetry/config.h | 42 ++ components/percona_telemetry/data_provider.cc | 533 ++++++++++++++++++ components/percona_telemetry/data_provider.h | 89 +++ components/percona_telemetry/logger.cc | 54 ++ components/percona_telemetry/logger.h | 32 ++ .../percona_telemetry_component.cc | 53 ++ .../percona_telemetry_component.h | 58 ++ components/percona_telemetry/storage.cc | 150 +++++ components/percona_telemetry/storage.h | 24 + components/percona_telemetry/worker.cc | 84 +++ components/percona_telemetry/worker.h | 37 ++ config.h.cmake | 1 + configure.cmake | 4 + mysql-test/include/mysqld--help.inc | 3 +- mysql-test/r/percona_utility_user.result | 9 +- .../innodb/r/tablespace_encrypt_9.result | 3 - .../suite/innodb/t/tablespace_encrypt_9.test | 1 - .../suite/perfschema/t/error_log-master.opt | 1 + .../suite/perfschema/t/pfs_example-master.opt | 2 +- .../t/pfs_example_lifecycle-master.opt | 2 +- .../t/udf_reg_unreg_permissions-master.opt | 2 +- .../t/udf_reg_unreg_single_func-master.opt | 2 +- .../r/percona_telemetry_disable_basic.result | 1 + .../suite/sys_vars/t/all_vars-master.opt | 2 +- .../t/percona_telemetry_disable_basic.test | 1 + mysql-test/t/component-master.opt | 2 +- mysql-test/t/component-upgrade-master.opt | 2 +- mysql-test/t/component_debug-master.opt | 2 +- .../t/log_components_implicit-master.opt | 1 + mysql-test/t/log_options_cmdline-master.opt | 2 +- mysql-test/t/percona_utility_user.test | 8 +- sql/dd/impl/upgrade/server.cc | 79 +++ sql/dd/upgrade/server.h | 4 + sql/mysqld.cc | 17 + sql/mysqld.h | 4 + sql/sys_vars.cc | 8 + 42 files changed, 1641 insertions(+), 24 deletions(-) create mode 100644 components/percona_telemetry/CMakeLists.txt create mode 100644 components/percona_telemetry/README.md create mode 100644 components/percona_telemetry/common.h create mode 100644 components/percona_telemetry/component.cc create mode 100644 components/percona_telemetry/config.cc create mode 100644 components/percona_telemetry/config.h create mode 100644 components/percona_telemetry/data_provider.cc create mode 100644 components/percona_telemetry/data_provider.h create mode 100644 components/percona_telemetry/logger.cc create mode 100644 components/percona_telemetry/logger.h create mode 100644 components/percona_telemetry/percona_telemetry_component.cc create mode 100644 components/percona_telemetry/percona_telemetry_component.h create mode 100644 components/percona_telemetry/storage.cc create mode 100644 components/percona_telemetry/storage.h create mode 100644 components/percona_telemetry/worker.cc create mode 100644 components/percona_telemetry/worker.h create mode 100644 mysql-test/suite/sys_vars/r/percona_telemetry_disable_basic.result create mode 100644 mysql-test/suite/sys_vars/t/percona_telemetry_disable_basic.test create mode 100644 mysql-test/t/log_components_implicit-master.opt diff --git a/CMakeLists.txt b/CMakeLists.txt index 74854bba3367..6eb2b87ef427 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2189,6 +2189,8 @@ IF(WITH_ENCRYPTION_UDF) ADD_SUBDIRECTORY(extra/opensslpp) ENDIF() +option(WITH_PERCONA_TELEMETRY "Build Percona Telemetry component" OFF) + # Utility target to build every executable tagged with ADD_TEST. ADD_CUSTOM_TARGET(unittest_all) diff --git a/components/percona_telemetry/CMakeLists.txt b/components/percona_telemetry/CMakeLists.txt new file mode 100644 index 000000000000..7415fbb8a5a4 --- /dev/null +++ b/components/percona_telemetry/CMakeLists.txt @@ -0,0 +1,50 @@ +# Copyright (c) 2017, 2021, Oracle and/or its affiliates. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License, version 2.0, +# as published by the Free Software Foundation. +# +# This program is also distributed with certain software (including +# but not limited to OpenSSL) that is licensed under separate terms, +# as designated in a particular file or component or in included license +# documentation. The authors of MySQL hereby grant you an additional +# permission to link the program and your derivative works with the +# separately licensed software that they have included with MySQL. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License, version 2.0, for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +DISABLE_MISSING_PROFILE_WARNING() + +IF(WITH_PERCONA_TELEMETRY) + message(STATUS "Building Percona Telemetry component") +ELSE() + message(STATUS "Not building Percona Telemetry component") + RETURN() +ENDIF() + +MYSQL_ADD_COMPONENT(percona_telemetry + percona_telemetry_component.cc + config.cc + data_provider.cc + storage.cc + worker.cc + component.cc + logger.cc + MODULE_ONLY + TEST_ONLY + ) + +IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.1) + TARGET_LINK_LIBRARIES(component_percona_telemetry stdc++fs) +ENDIF() + +TARGET_INCLUDE_DIRECTORIES(component_percona_telemetry SYSTEM PRIVATE ${BOOST_PATCHES_DIR} ${BOOST_INCLUDE_DIR}) + +TARGET_LINK_LIBRARIES(component_percona_telemetry extra::rapidjson) diff --git a/components/percona_telemetry/README.md b/components/percona_telemetry/README.md new file mode 100644 index 000000000000..19e1b40928ed --- /dev/null +++ b/components/percona_telemetry/README.md @@ -0,0 +1,53 @@ +To compile and run: +1. configure project with cmake `-DWITH_PERCONA_TELEMETRY=ON`` +2. compile the server and the component +3. start mysqld +4. Percona Telemetry component is enabled by default +5. To disable without server restart execute `UNINSTALL COMPONENT "file://component_percona_telemetry";`` +6. To make disable changes permanent set `percona_telemetry_disable=1` in `my.cnf` +``` +mysql> show variables like '%percona_telemetry%'; ++-----------------------------------------+---------------------------------+ +| Variable_name | Value | ++-----------------------------------------+---------------------------------+ +| percona_telemetry.grace_interval | 86400 | +| percona_telemetry.history_keep_interval | 604800 | +| percona_telemetry.scrape_interval | 86400 | +| percona_telemetry.telemetry_root_dir | /usr/local/percona/telemetry/ps | +| percona_telemetry_disable | OFF | ++-----------------------------------------+---------------------------------+ +``` +Configurable in my.cnf: +``` +percona_telemetry.grace_interval = 20 +percona_telemetry.scrape_interval = 30 +percona_telemetry.history_keep_interval = 70 +percona_telemetry.telemetry_root_dir = /some/custom/dir +percona_telemetry_disable = ON/OFF +``` +Note that `percona_telemetry.telemetry_root_dir` has to exist and be writable. + +``` ++-----------------------------------------+----------------------------------------------------------------------+ +| Variable_name | Value | ++-----------------------------------------+----------------------------------------------------------------------+ +| percona_telemetry.grace_interval | 20 | +| percona_telemetry.history_keep_interval | 70 | +| percona_telemetry.scrape_interval | 30 | +| percona_telemetry.telemetry_root_dir | /usr/local/percona/telemetry/ps | +| percona_telemetry_disable | OFF | ++-----------------------------------------+----------------------------------------------------------------------+ + +``` +When the telemetry component is permanently disabled: +``` +mysql> show variables like '%percona_telemetry%'; ++---------------------------+-------+ +| Variable_name | Value | ++---------------------------+-------+ +| percona_telemetry_disable | ON | ++---------------------------+-------+ +1 row in set (0,00 sec) +``` + + diff --git a/components/percona_telemetry/common.h b/components/percona_telemetry/common.h new file mode 100644 index 000000000000..395e390c0da0 --- /dev/null +++ b/components/percona_telemetry/common.h @@ -0,0 +1,9 @@ +#ifndef PERCONA_TELEMETRY_COMMON_H +#define PERCONA_TELEMETRY_COMMON_H + +#include + +#define CURRENT_COMPONENT_NAME percona_telemetry +#define CURRENT_COMPONENT_NAME_STR BOOST_PP_STRINGIZE(CURRENT_COMPONENT_NAME) + +#endif /* PERCONA_TELEMETRY_COMMON_H */ diff --git a/components/percona_telemetry/component.cc b/components/percona_telemetry/component.cc new file mode 100644 index 000000000000..7ebeb83fdb7e --- /dev/null +++ b/components/percona_telemetry/component.cc @@ -0,0 +1,102 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "percona_telemetry_component.h" + +namespace { +REQUIRES_SERVICE_PLACEHOLDER_AS(mysql_thd_security_context, + thd_security_context_srv); +REQUIRES_SERVICE_PLACEHOLDER_AS(mysql_command_thread, command_thread_srv); +REQUIRES_SERVICE_PLACEHOLDER_AS(mysql_command_factory, command_factory_srv); +REQUIRES_SERVICE_PLACEHOLDER_AS(mysql_command_options, command_options_srv); +REQUIRES_SERVICE_PLACEHOLDER_AS(mysql_command_query, command_query_srv); +REQUIRES_SERVICE_PLACEHOLDER_AS(mysql_command_query_result, + command_query_result_srv); +REQUIRES_SERVICE_PLACEHOLDER_AS(mysql_command_field_info, + command_field_info_srv); +REQUIRES_SERVICE_PLACEHOLDER_AS(mysql_command_error_info, + command_error_info_srv); +REQUIRES_SERVICE_PLACEHOLDER_AS(registry, registry_srv); +REQUIRES_SERVICE_PLACEHOLDER_AS(log_builtins, log_builtins_srv); +REQUIRES_SERVICE_PLACEHOLDER_AS(log_builtins_string, log_builtins_string_srv); +REQUIRES_SERVICE_PLACEHOLDER_AS(component_sys_variable_register, + component_sys_variable_register_srv); +REQUIRES_SERVICE_PLACEHOLDER_AS(component_sys_variable_unregister, + component_sys_variable_unregister_srv); + +std::unique_ptr percona_telemetry_component; + +mysql_service_status_t component_init() { + auto services = std::make_unique(); + services->thread_security_context_service = thd_security_context_srv; + services->command_thread_service = command_thread_srv; + services->command_factory_service = command_factory_srv; + services->command_options_service = command_options_srv; + services->command_query_service = command_query_srv; + services->command_query_result_service = command_query_result_srv; + services->command_field_info_service = command_field_info_srv; + services->command_error_info_service = command_error_info_srv; + services->log_builtins_service = log_builtins_srv; + services->log_builtins_string = log_builtins_string_srv; + services->var_register_service = component_sys_variable_register_srv; + services->var_unregister_service = component_sys_variable_unregister_srv; + + percona_telemetry_component = + std::make_unique(std::move(services)); + + if (percona_telemetry_component->start()) { + return 1; + } + + return 0; +} + +mysql_service_status_t component_deinit() { + if (percona_telemetry_component->stop()) { + return 1; + } + + percona_telemetry_component.reset(); + return 0; +} + +BEGIN_COMPONENT_PROVIDES(CURRENT_COMPONENT_NAME) +END_COMPONENT_PROVIDES(); + +BEGIN_COMPONENT_REQUIRES(CURRENT_COMPONENT_NAME) +REQUIRES_SERVICE_AS(mysql_thd_security_context, thd_security_context_srv), + REQUIRES_SERVICE_AS(mysql_command_thread, command_thread_srv), + REQUIRES_SERVICE_AS(mysql_command_factory, command_factory_srv), + REQUIRES_SERVICE_AS(mysql_command_options, command_options_srv), + REQUIRES_SERVICE_AS(mysql_command_query, command_query_srv), + REQUIRES_SERVICE_AS(mysql_command_query_result, command_query_result_srv), + REQUIRES_SERVICE_AS(mysql_command_field_info, command_field_info_srv), + REQUIRES_SERVICE_AS(mysql_command_error_info, command_error_info_srv), + REQUIRES_SERVICE_AS(registry, registry_srv), + REQUIRES_SERVICE_AS(log_builtins, log_builtins_srv), + REQUIRES_SERVICE_AS(log_builtins_string, log_builtins_string_srv), + REQUIRES_SERVICE_AS(component_sys_variable_register, + component_sys_variable_register_srv), + REQUIRES_SERVICE_AS(component_sys_variable_unregister, + component_sys_variable_unregister_srv), + END_COMPONENT_REQUIRES(); + +BEGIN_COMPONENT_METADATA(CURRENT_COMPONENT_NAME) +METADATA("mysql.author", "Percona Corporation"), + METADATA("mysql.license", "GPL"), METADATA("mysql.version", "1"), + END_COMPONENT_METADATA(); + +DECLARE_COMPONENT(CURRENT_COMPONENT_NAME, CURRENT_COMPONENT_NAME_STR) +component_init, component_deinit END_DECLARE_COMPONENT(); + +} // namespace + +DECLARE_LIBRARY_COMPONENTS &COMPONENT_REF(CURRENT_COMPONENT_NAME) + END_DECLARE_LIBRARY_COMPONENTS diff --git a/components/percona_telemetry/config.cc b/components/percona_telemetry/config.cc new file mode 100644 index 000000000000..e04862f28f55 --- /dev/null +++ b/components/percona_telemetry/config.cc @@ -0,0 +1,130 @@ +#include + +#include "common.h" +#include "config.h" + +namespace { +constexpr char VAR_TELEMETRY_ROOT_DIR[] = "telemetry_root_dir"; +constexpr char VAR_SCRAPE_INTERVAL[] = "scrape_interval"; +constexpr char VAR_GRACE_INTERVAL[] = "grace_interval"; +constexpr char VAR_HISTORY_KEEP_INTERVAL[] = "history_keep_interval"; +constexpr char TELEMETRY_ROOT_DIR_DEFAULT[] = "/usr/local/percona/telemetry/ps"; + +constexpr uint SCRAPE_INTERVAL_DEFAULT = 60 * 60 * 24 * 1; // 1 day +constexpr uint SCRAPE_INTERVAL_MIN = 10; +constexpr uint SCRAPE_INTERVAL_MAX = 60 * 60 * 24 * 7; // 1 week +constexpr uint GRACE_INTERVAL_DEFAULT = 60 * 60 * 24 * 1; // 1 day +constexpr uint GRACE_INTERVAL_MIN = 20; +constexpr uint GRACE_INTERVAL_MAX = 60 * 60 * 24 * 2; // 2 days + +constexpr uint HISTORY_KEEP_INTERVAL_DEFAULT = 60 * 60 * 24 * 7; // 7 days +constexpr uint HISTORY_KEEP_INTERVAL_MIN = 60; +constexpr uint HISTORY_KEEP_INTERVAL_MAX = HISTORY_KEEP_INTERVAL_DEFAULT; +} // namespace + +Config::Config(SERVICE_TYPE(component_sys_variable_register) & + var_register_service, + SERVICE_TYPE(component_sys_variable_unregister) & + var_unregister_service) + : var_register_service_(var_register_service), + var_unregister_service_(var_unregister_service), + telemetry_root_dir_value_(nullptr) {} + +bool Config::init() { + STR_CHECK_ARG(str) telemetry_root_dir_arg; + telemetry_root_dir_arg.def_val = + const_cast(TELEMETRY_ROOT_DIR_DEFAULT); + if (var_register_service_.register_variable( + CURRENT_COMPONENT_NAME_STR, VAR_TELEMETRY_ROOT_DIR, + PLUGIN_VAR_STR | PLUGIN_VAR_MEMALLOC | PLUGIN_VAR_READONLY, + "Root path of the telemetry data for all mysqld servers", nullptr, + nullptr, &telemetry_root_dir_arg, &telemetry_root_dir_value_)) { + return true; + } + + INTEGRAL_CHECK_ARG(uint) + scrape_interval_arg, grace_interval_arg, history_keep_interval_arg; + scrape_interval_arg.def_val = SCRAPE_INTERVAL_DEFAULT; + scrape_interval_arg.min_val = SCRAPE_INTERVAL_MIN; + scrape_interval_arg.max_val = SCRAPE_INTERVAL_MAX; + scrape_interval_arg.blk_sz = 0; + if (var_register_service_.register_variable( + CURRENT_COMPONENT_NAME_STR, VAR_SCRAPE_INTERVAL, + PLUGIN_VAR_UNSIGNED | PLUGIN_VAR_INT | PLUGIN_VAR_READONLY, + "Telemetry scrape interval", nullptr, nullptr, &scrape_interval_arg, + &scrape_interval_value_)) { + return true; + } + + grace_interval_arg.def_val = GRACE_INTERVAL_DEFAULT; + grace_interval_arg.min_val = GRACE_INTERVAL_MIN; + grace_interval_arg.max_val = GRACE_INTERVAL_MAX; + grace_interval_arg.blk_sz = 0; + if (var_register_service_.register_variable( + CURRENT_COMPONENT_NAME_STR, VAR_GRACE_INTERVAL, + PLUGIN_VAR_UNSIGNED | PLUGIN_VAR_INT | PLUGIN_VAR_READONLY, + "Telemetry grace interval", nullptr, nullptr, &grace_interval_arg, + &grace_interval_value_)) { + return true; + } + + history_keep_interval_arg.def_val = HISTORY_KEEP_INTERVAL_DEFAULT; + history_keep_interval_arg.min_val = HISTORY_KEEP_INTERVAL_MIN; + history_keep_interval_arg.max_val = HISTORY_KEEP_INTERVAL_MAX; + history_keep_interval_arg.blk_sz = 0; + if (var_register_service_.register_variable( + CURRENT_COMPONENT_NAME_STR, VAR_HISTORY_KEEP_INTERVAL, + PLUGIN_VAR_UNSIGNED | PLUGIN_VAR_INT | PLUGIN_VAR_READONLY, + "Telemetry history keep interval", nullptr, nullptr, + &history_keep_interval_arg, &history_keep_interval_value_)) { + return true; + } + + return false; +} + +bool Config::deinit() { + bool res = false; + if (var_unregister_service_.unregister_variable(CURRENT_COMPONENT_NAME_STR, + VAR_TELEMETRY_ROOT_DIR)) { + res = true; + } + if (var_unregister_service_.unregister_variable(CURRENT_COMPONENT_NAME_STR, + VAR_SCRAPE_INTERVAL)) { + res = true; + } + if (var_unregister_service_.unregister_variable(CURRENT_COMPONENT_NAME_STR, + VAR_GRACE_INTERVAL)) { + res = true; + } + if (var_unregister_service_.unregister_variable(CURRENT_COMPONENT_NAME_STR, + VAR_HISTORY_KEEP_INTERVAL)) { + res = true; + } + + return res; +} + +const std::string &Config::telemetry_storage_dir_path() const noexcept { + assert(telemetry_root_dir_value_); + // it is read-only value + static std::string telemetry_root_dir_value_str(telemetry_root_dir_value_); + return telemetry_root_dir_value_str; +} + +std::chrono::seconds Config::scrape_interval() const noexcept { + return std::chrono::seconds(scrape_interval_value_); +} + +std::chrono::seconds Config::grace_interval() const noexcept { + return std::chrono::seconds(grace_interval_value_); +} + +std::chrono::seconds Config::history_keep_interval() const noexcept { + return std::chrono::seconds(history_keep_interval_value_); +} + +std::chrono::seconds Config::unconditional_history_cleanup_interval() + const noexcept { + return std::chrono::seconds(HISTORY_KEEP_INTERVAL_MAX); +} diff --git a/components/percona_telemetry/config.h b/components/percona_telemetry/config.h new file mode 100644 index 000000000000..b8b3b340b618 --- /dev/null +++ b/components/percona_telemetry/config.h @@ -0,0 +1,42 @@ +#ifndef PERCONA_TELEMETRY_CONFIG_H +#define PERCONA_TELEMETRY_CONFIG_H + +#include +#include +#include + +#include +#include + +class Config { + public: + Config(SERVICE_TYPE(component_sys_variable_register) & var_register_service, + SERVICE_TYPE(component_sys_variable_unregister) & + var_unregister_service); + ~Config() = default; + + Config(const Config &rhs) = delete; + Config(Config &&rhs) = delete; + Config &operator=(const Config &rhs) = delete; + Config &operator=(Config &&rhs) = delete; + + bool init(); + bool deinit(); + + const std::string &telemetry_storage_dir_path() const noexcept; + std::chrono::seconds scrape_interval() const noexcept; + std::chrono::seconds grace_interval() const noexcept; + std::chrono::seconds history_keep_interval() const noexcept; + std::chrono::seconds unconditional_history_cleanup_interval() const noexcept; + + private: + SERVICE_TYPE(component_sys_variable_register) & var_register_service_; + SERVICE_TYPE(component_sys_variable_unregister) & var_unregister_service_; + + const char *telemetry_root_dir_value_; + uint scrape_interval_value_; + uint grace_interval_value_; + uint history_keep_interval_value_; +}; + +#endif /* PERCONA_TELEMETRY_CONFIG_H */ diff --git a/components/percona_telemetry/data_provider.cc b/components/percona_telemetry/data_provider.cc new file mode 100644 index 000000000000..695ab4636e59 --- /dev/null +++ b/components/percona_telemetry/data_provider.cc @@ -0,0 +1,533 @@ +#include +#include + +#include "data_provider.h" +#include "logger.h" + +/* The list of metrics collected by Percona Telemetry Component: +Must have: +1. replication information (is it enabled, Galera vs. Group Replication) + If it is PXC, it is Galera + If there is replication (see below) -> async replication + If there is replication and semisync_master or semisync_slave plugin is +installed -> semi-sync replication For GR detection fool around GR plugin and +performance_schema.replication_group_members table (see Orchestrator) +2. product version with “…-pro” suffix for Pro Builds - SELECT VERSION(); + +Should have: +1. plugin information (list of active plugins) - + select plugin_name, plugin_status from information_schema.plugins; + select component_urn from mysql.component; + +Nice to have: +1. MySQL uptime - SHOW GLOBAL STATUS LIKE 'Uptime'; +2. number of databases - SELECT COUNT(*) FROM information_schema.SCHEMATA WHERE +SCHEMA_NAME NOT IN('mysql', 'information_schema', 'performance_schema', 'sys'); +3. size of databases - SELECT IFNULL(ROUND(SUM(data_length + index_length), 1), +"0") size_MB FROM information_schema.tables WHERE table_schema NOT IN('mysql', +'information_schema', 'performance_schema', 'sys'); +4. encryption methods applied (?) + encrypted tables count: SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE +CREATE_OPTIONS LIKE '%ENCRYPTION%'; encrypted databases count: SELECT COUNT(*) +FROM INFORMATION_SCHEMA.SCHEMATA WHERE DEFAULT_ENCRYPTION='YES'; encrypted +tablespaces count: SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES +WHERE (flag & 8192) != 0; +5. number of replication nodes, master/slave + source: + show replicas; + replica: + show replica status; -> Replica_IO_Running, Replica_SQL_Running +6. storage engine used (MyRocks, InnoDB) - SELECT DISTINCT ENGINE FROM +information_schema.tables WHERE table_schema NOT IN('mysql', +'information_schema', 'performance_schema', 'sys'); 7. +*/ + +namespace { +inline const char *b2s(bool val) { return val ? "1" : "0"; } + +namespace JSONKey { +const char *pillar_version = "pillar_version"; +const char *db_instance_id = "db_instance_id"; +const char *active_plugins = "active_plugins"; +const char *active_components = "active_components"; +const char *uptime = "uptime"; +const char *databases_count = "databases_count"; +const char *databases_size = "databases_size"; +const char *se_engines_in_use = "se_engines_in_use"; +const char *role = "role"; +const char *db_replication_id = "db_replication_id"; +const char *single_primary_mode = "single_primary_mode"; +const char *group_size = "group_size"; +const char *group_replication_info = "group_replication_info"; +const char *is_semisync_source = "is_semisync_source"; +const char *is_source = "is_source"; +const char *is_semisync_replica = "is_semisync_replica"; +const char *is_replica = "is_replica"; +const char *replication_info = "replication_info"; +} // namespace JSONKey +} // namespace + +DataProvider::DataProvider( + SERVICE_TYPE(mysql_command_factory) & command_factory_service, + SERVICE_TYPE(mysql_command_options) & command_options_service, + SERVICE_TYPE(mysql_command_query) & command_query_service, + SERVICE_TYPE(mysql_command_query_result) & command_query_result_service, + SERVICE_TYPE(mysql_command_field_info) & command_field_info_service, + SERVICE_TYPE(mysql_command_error_info) & command_error_info_service, + SERVICE_TYPE(mysql_command_thread) & command_thread_service, Logger &logger) + : command_factory_service_(command_factory_service), + command_options_service_(command_options_service), + command_query_service_(command_query_service), + command_query_result_service_(command_query_result_service), + command_field_info_service_(command_field_info_service), + command_error_info_service_(command_error_info_service), + command_thread_service_(command_thread_service), + logger_(logger) {} + +void DataProvider::thread_access_begin() { command_thread_service_.init(); } + +void DataProvider::thread_access_end() { command_thread_service_.end(); } + +bool DataProvider::do_query(const std::string &query, QueryResult *result, + unsigned int *err_no, + bool suppress_query_error_log) { + MYSQL_RES_H mysql_res = nullptr; + MYSQL_ROW_H row = nullptr; + uint64_t row_count = 0; + unsigned int num_column = 0; + std::string result_set; + MYSQL_H mysql_h = nullptr; + bool res = true; + + if (!result) { + return true; + } + result->clear(); + + mysql_service_status_t sstatus = command_factory_service_.init(&mysql_h); + if (!sstatus) + sstatus |= + command_options_service_.set(mysql_h, MYSQL_COMMAND_PROTOCOL, nullptr); + if (!sstatus) + sstatus |= + command_options_service_.set(mysql_h, MYSQL_COMMAND_USER_NAME, "root"); + if (!sstatus) + sstatus |= + command_options_service_.set(mysql_h, MYSQL_COMMAND_HOST_NAME, nullptr); + if (!sstatus) sstatus |= command_factory_service_.connect(mysql_h); + + // starting from this point, if the above succeeded we need to close mysql_h. + std::shared_ptr mysql_h_close_guard( + mysql_h, + [&srv = command_factory_service_, do_close = !sstatus](void *ptr) { + if (do_close && ptr) srv.close(static_cast(ptr)); + }); + + // if any of the above failed, just exit + if (sstatus) { + goto err; + } + + if (command_query_service_.query(mysql_h, query.data(), query.length())) { + if (err_no) { + command_error_info_service_.sql_errno(mysql_h, err_no); + if (command_error_info_service_.sql_errno(mysql_h, err_no)) { + logger_.warning("Failed to get the last query error"); + } + } + goto err; + } + + command_query_result_service_.store_result(mysql_h, &mysql_res); + if (mysql_res) { + std::shared_ptr query_result_free_guard( + mysql_res, [&srv = command_query_result_service_](void *ptr) { + if (ptr) srv.free_result(static_cast(ptr)); + }); + + if (command_query_service_.affected_rows(mysql_h, &row_count)) { + goto err; + } + if (command_field_info_service_.num_fields(mysql_res, &num_column)) { + goto err; + } + + for (uint64_t i = 0; i < row_count; i++) { + if (command_query_result_service_.fetch_row(mysql_res, &row)) { + goto err; + } + ulong *length = nullptr; + command_query_result_service_.fetch_lengths(mysql_res, &length); + + Row new_row; + + for (unsigned int j = 0; j < num_column; j++) { + new_row.emplace_back(std::string(row[j])); + } + result->push_back(std::move(new_row)); + } + } + + res = false; +err: + if (res && !suppress_query_error_log) { + logger_.info("do_query() failed. query: %s", query.c_str()); + } + return res; +} + +const std::string &DataProvider::get_database_instance_id() { + if (!database_instance_id_cache_.length()) { + QueryResult result; + if (do_query("SELECT @@server_uuid", &result)) { + static std::string empty; + return empty; + } + database_instance_id_cache_ = result[0][0]; + } + return database_instance_id_cache_; +} + +bool DataProvider::collect_db_instance_id_info(rapidjson::Document *document) { + const std::string &id = get_database_instance_id(); + + /* Id can be empty if: + 1. There is too low grace interval and the server didn't start yet, + so the SQL query failed. It will recover next time. + 2. Some other reason that caused selecting server_id to fail. */ + if (id.length() == 0) { + logger_.warning( + "Collecting db_instance_id failed. It may be caused by server still " + "initializing."); + return true; + } + rapidjson::Document::AllocatorType &allocator = document->GetAllocator(); + rapidjson::Value instance_id; + instance_id.SetString(id.c_str(), allocator); + document->AddMember(rapidjson::StringRef(JSONKey::db_instance_id), + instance_id, allocator); + return false; +} +bool DataProvider::collect_product_version_info(rapidjson::Document *document) { + // Version doesn't change during the lifetime, so query and cache it + if (version_cache_.empty()) { + QueryResult result; + if (do_query("SELECT @@VERSION, @@VERSION_COMMENT", &result)) { + return true; + } + + version_cache_ = result[0][0]; + + // Is it 'pro' build? + if (result[0][1].find("Pro") != std::string::npos) { + version_cache_ += "-pro"; + } + } + + rapidjson::Document::AllocatorType &allocator = document->GetAllocator(); + rapidjson::Value version; + version.SetString(version_cache_.c_str(), allocator); + // we use "pillar_version" as a key for backward compatibility with ph0 + document->AddMember(rapidjson::StringRef(JSONKey::pillar_version), version, + allocator); + return false; +} + +bool DataProvider::collect_plugins_info(rapidjson::Document *document) { + QueryResult result; + if (do_query("SELECT PLUGIN_NAME FROM information_schema.plugins WHERE " + "PLUGIN_STATUS='ACTIVE'", + &result)) { + return true; + } + + rapidjson::Document::AllocatorType &allocator = document->GetAllocator(); + + rapidjson::Value plugins(rapidjson::Type::kArrayType); + + for (auto &plugin_iter : result) { + rapidjson::Value plugin_name; + plugin_name.SetString(plugin_iter[0].c_str(), allocator); + plugins.PushBack(plugin_name, allocator); + } + document->AddMember(rapidjson::StringRef(JSONKey::active_plugins), plugins, + allocator); + + return false; +} + +bool DataProvider::collect_components_info(rapidjson::Document *document) { + QueryResult result; + if (do_query("SELECT component_urn FROM mysql.component", &result)) { + return true; + } + + rapidjson::Document::AllocatorType &allocator = document->GetAllocator(); + + rapidjson::Value components(rapidjson::Type::kArrayType); + + for (auto &component_iter : result) { + rapidjson::Value component_name; + component_name.SetString(component_iter[0].c_str(), allocator); + components.PushBack(component_name, allocator); + } + document->AddMember(rapidjson::StringRef(JSONKey::active_components), + components, allocator); + return false; +} + +bool DataProvider::collect_uptime_info(rapidjson::Document *document) { + QueryResult result; + if (do_query("SHOW GLOBAL STATUS LIKE 'Uptime'", &result)) { + return true; + } + + rapidjson::Document::AllocatorType &allocator = document->GetAllocator(); + rapidjson::Value uptime; + uptime.SetString(result[0][1].c_str(), allocator); + document->AddMember(rapidjson::StringRef(JSONKey::uptime), uptime, allocator); + return false; +} + +bool DataProvider::collect_dbs_number_info(rapidjson::Document *document) { + QueryResult result; + if (do_query( + "SELECT COUNT(*) FROM information_schema.SCHEMATA WHERE SCHEMA_NAME " + "NOT IN('mysql', 'information_schema', 'performance_schema', 'sys')", + &result)) { + return true; + } + + rapidjson::Document::AllocatorType &allocator = document->GetAllocator(); + rapidjson::Value db_cnt; + db_cnt.SetString(result[0][0].c_str(), allocator); + document->AddMember(rapidjson::StringRef(JSONKey::databases_count), db_cnt, + allocator); + return false; +} + +/* Note that this metric is update very X, so it may be inacurate. + We could make it accurate but that would need ANALYZE TABLE for every table + which would be overkill.*/ +bool DataProvider::collect_dbs_size_info(rapidjson::Document *document) { + QueryResult result; + if (do_query("SELECT IFNULL(ROUND(SUM(data_length + index_length), 1), '0') " + "size_MB FROM information_schema.tables WHERE table_schema NOT " + "IN('mysql', 'information_schema', 'performance_schema', 'sys')", + &result)) { + return true; + } + + rapidjson::Document::AllocatorType &allocator = document->GetAllocator(); + rapidjson::Value db_size; + db_size.SetString(result[0][0].c_str(), allocator); + document->AddMember(rapidjson::StringRef(JSONKey::databases_size), db_size, + allocator); + return false; +} + +bool DataProvider::collect_se_usage_info(rapidjson::Document *document) { + QueryResult result; + if (do_query("SELECT DISTINCT ENGINE FROM information_schema.tables WHERE " + "table_schema NOT IN('mysql', 'information_schema', " + "'performance_schema', 'sys');", + &result)) { + return true; + } + + rapidjson::Document::AllocatorType &allocator = document->GetAllocator(); + rapidjson::Value se_engines(rapidjson::Type::kArrayType); + + for (auto &engine_iter : result) { + rapidjson::Value engine_name; + engine_name.SetString(engine_iter[0].c_str(), allocator); + se_engines.PushBack(engine_name, allocator); + } + document->AddMember(rapidjson::StringRef(JSONKey::se_engines_in_use), + se_engines, allocator); + return false; +} + +bool DataProvider::collect_group_replication_info( + rapidjson::Document *document) { + // Do fast check if there is anything to learn about GR + static const std::string query_base( + "SELECT MEMBER_ROLE, @@global.group_replication_group_name, " + "@@global.group_replication_single_primary_mode FROM " + "performance_schema.replication_group_members WHERE MEMBER_STATE != " + "'OFFLINE'"); + std::ostringstream ss; + ss << query_base << " AND MEMBER_ID='" << get_database_instance_id() << "'"; + QueryResult result; + if (do_query(ss.str(), &result, nullptr, true)) { + /* Ideally we should get ER_UNKNOWN_SYSTEM_VARIABLE (1193) if GR plugin + is not installed. Unfortunately + mysql_command_error_info_service.sql_errno() returns 0 in case of such + failure. It returns proper string message, but we cannot rely on it as it + can be localized. Do the best possible: if the query failed, it means "no + GR replication" */ + /* + if (err_no == ER_UNKNOWN_SYSTEM_VARIABLE) { + return false; + } + */ + return false; + } + + if (result.size() > 0) { + // We've got some rows. Try to collect more details. + rapidjson::Document::AllocatorType &allocator = document->GetAllocator(); + rapidjson::Document gr_json(rapidjson::Type::kObjectType); + + rapidjson::Value role; + role.SetString(result[0][0].c_str(), allocator); + gr_json.AddMember(rapidjson::StringRef(JSONKey::role), role, allocator); + + rapidjson::Value replication_group_id; + replication_group_id.SetString(result[0][1].c_str(), allocator); + gr_json.AddMember(rapidjson::StringRef(JSONKey::db_replication_id), + replication_group_id, allocator); + + rapidjson::Value single_primary_mode; + single_primary_mode.SetString(result[0][2].c_str(), allocator); + gr_json.AddMember(rapidjson::StringRef(JSONKey::single_primary_mode), + single_primary_mode, allocator); + + /* replication group size */ + if (!do_query( + "SELECT COUNT(*) FROM performance_schema.replication_group_members", + &result)) { + rapidjson::Value group_size; + group_size.SetString(result[0][0].c_str(), allocator); + gr_json.AddMember(rapidjson::StringRef(JSONKey::group_size), group_size, + allocator); + } + + document->AddMember(rapidjson::StringRef(JSONKey::group_replication_info), + gr_json, allocator); + } + return false; +} + +bool DataProvider::collect_async_replication_info( + rapidjson::Document *document) { + bool is_source = false; + bool is_replica = false; + bool is_semisync_source = false; + bool is_semisync_replica = false; + + // If we are source + QueryResult result; + if (do_query("SHOW REPLICAS", &result)) { + return true; + } + is_source = result.size() > 0; + + // If we are replica + if (do_query("SHOW REPLICA STATUS", &result)) { + return true; + } + is_replica = result.size() > 0; + + // Name of the variable depends on what plugin was installed + // If we are semisync source + if (!do_query("SELECT @@global.rpl_semi_sync_source_enabled", &result, + nullptr, true) || + !do_query("SELECT @@global.rpl_semi_sync_master_enabled", &result, + nullptr, true)) { + is_semisync_source = !result[0][0].compare("1"); + is_semisync_source &= is_source; + } + // If we are semisync replica + if (!do_query("SELECT @@global.rpl_semi_sync_replica_enabled", &result, + nullptr, true) || + !do_query("SELECT @@global.rpl_semi_sync_slave_enabled", &result, nullptr, + true)) { + is_semisync_replica = !result[0][0].compare("1"); + is_semisync_replica &= is_replica; + } + + if (is_source || is_replica || is_semisync_source || is_semisync_replica) { + rapidjson::Document::AllocatorType &allocator = document->GetAllocator(); + rapidjson::Document r_json(rapidjson::Type::kObjectType); + + if (is_semisync_source) { + rapidjson::Value is_semisync_source_json; + is_semisync_source_json.SetString(b2s(is_semisync_source), allocator); + r_json.AddMember(rapidjson::StringRef(JSONKey::is_semisync_source), + is_semisync_source_json, allocator); + } else if (is_source) { + rapidjson::Value is_source_json; + is_source_json.SetString(b2s(is_source), allocator); + r_json.AddMember(rapidjson::StringRef(JSONKey::is_source), is_source_json, + allocator); + } + + if (is_semisync_replica) { + rapidjson::Value is_semisync_replica_json; + is_semisync_replica_json.SetString(b2s(is_semisync_replica), allocator); + r_json.AddMember(rapidjson::StringRef(JSONKey::is_semisync_replica), + is_semisync_replica_json, allocator); + } else if (is_replica) { + rapidjson::Value is_replica_json; + is_replica_json.SetString(b2s(is_replica), allocator); + r_json.AddMember(rapidjson::StringRef(JSONKey::is_replica), + is_replica_json, allocator); + } + + // we had to log something + assert(!r_json.ObjectEmpty()); + if (r_json.ObjectEmpty()) { + return true; + } + + document->AddMember(rapidjson::StringRef(JSONKey::replication_info), r_json, + allocator); + } + + return false; +} + +bool DataProvider::collect_metrics(rapidjson::Document *document) { + bool res = collect_db_instance_id_info(document); + + // If db_instance_id cannot be collected all other metrics are meaningless + if (res) { + logger_.info( + "Collecting db_instance_id failed. Skipping metrics scaping this time"); + return true; + } + + // Do the best we can, collect what is possible, even if something fails. + res |= collect_product_version_info(document); + res |= collect_plugins_info(document); + res |= collect_components_info(document); + res |= collect_uptime_info(document); + res |= collect_dbs_number_info(document); + res |= collect_dbs_size_info(document); + res |= collect_se_usage_info(document); + res |= collect_group_replication_info(document); + res |= collect_async_replication_info(document); + return res; +} + +std::string DataProvider::get_report() { + rapidjson::Document JSON(rapidjson::Type::kObjectType); + + if (collect_metrics(&JSON) && !JSON.ObjectEmpty()) { + logger_.info("Collecting of some metrics failed."); + } + + if (JSON.ObjectEmpty()) { + logger_.info("Collecting of all metrics failed."); + return std::string(); + } + + rapidjson::StringBuffer string_buffer; + string_buffer.Clear(); + RapidJsonWritterType string_writer(string_buffer); + JSON.Accept(string_writer); + std::string json(string_buffer.GetString(), string_buffer.GetSize()); + + return json; +} diff --git a/components/percona_telemetry/data_provider.h b/components/percona_telemetry/data_provider.h new file mode 100644 index 000000000000..c70de48d86b1 --- /dev/null +++ b/components/percona_telemetry/data_provider.h @@ -0,0 +1,89 @@ +#ifndef PERCONA_TELEMETRY_DATA_PROVIDER_H +#define PERCONA_TELEMETRY_DATA_PROVIDER_H + +#include +#include +#include + +#include +#include +#include +#include + +#ifdef RAPIDJSON_NO_SIZETYPEDEFINE +#include "my_rapidjson_size_t.h" +#endif + +#include + +#define COMPONENT_PERCONA_TELEMETRY_DEBUG_JSON_PRINTER +#ifndef COMPONENT_PERCONA_TELEMETRY_DEBUG_JSON_PRINTER +#include +using RapidJsonWritterType = rapidjson::Writer; +#else +#include +using RapidJsonWritterType = rapidjson::PrettyWriter; +#endif + +#include + +using Row = std::vector; +using QueryResult = std::vector; + +class Logger; +class DataProvider { + public: + DataProvider( + SERVICE_TYPE(mysql_command_factory) & command_factory_service, + SERVICE_TYPE(mysql_command_options) & command_options_service, + SERVICE_TYPE(mysql_command_query) & command_query_service, + SERVICE_TYPE(mysql_command_query_result) & command_query_result_service, + SERVICE_TYPE(mysql_command_field_info) & command_field_info_service, + SERVICE_TYPE(mysql_command_error_info) & command_error_info_service, + SERVICE_TYPE(mysql_command_thread) & command_thread_service, + Logger &logger); + + ~DataProvider() = default; + + DataProvider(const DataProvider &rhs) = delete; + DataProvider(DataProvider &&rhs) = delete; + DataProvider &operator=(const DataProvider &rhs) = delete; + DataProvider &operator=(DataProvider &&rhs) = delete; + + void thread_access_begin(); + void thread_access_end(); + std::string get_report(); + + private: + bool do_query(const std::string &query, QueryResult *result, + unsigned int *err_no = nullptr, + bool suppress_query_error_log = false); + bool collect_db_instance_id_info(rapidjson::Document *document); + bool collect_product_version_info(rapidjson::Document *document); + bool collect_plugins_info(rapidjson::Document *document); + bool collect_components_info(rapidjson::Document *document); + bool collect_uptime_info(rapidjson::Document *document); + bool collect_dbs_number_info(rapidjson::Document *document); + bool collect_dbs_size_info(rapidjson::Document *document); + bool collect_se_usage_info(rapidjson::Document *document); + bool collect_group_replication_info(rapidjson::Document *document); + bool collect_async_replication_info(rapidjson::Document *document); + bool collect_metrics(rapidjson::Document *document); + + const std::string &get_database_instance_id(); + + SERVICE_TYPE(mysql_command_factory) & command_factory_service_; + SERVICE_TYPE(mysql_command_options) & command_options_service_; + SERVICE_TYPE(mysql_command_query) & command_query_service_; + SERVICE_TYPE(mysql_command_query_result) & command_query_result_service_; + SERVICE_TYPE(mysql_command_field_info) & command_field_info_service_; + SERVICE_TYPE(mysql_command_error_info) & command_error_info_service_; + SERVICE_TYPE(mysql_command_thread) & command_thread_service_; + + Logger &logger_; + + std::string database_instance_id_cache_; + std::string version_cache_; +}; + +#endif /* PERCONA_TELEMETRY_DATA_PROVIDER_H */ diff --git a/components/percona_telemetry/logger.cc b/components/percona_telemetry/logger.cc new file mode 100644 index 000000000000..04047d8969b5 --- /dev/null +++ b/components/percona_telemetry/logger.cc @@ -0,0 +1,54 @@ +#include + +#include +#include "logger.h" + +SERVICE_TYPE(log_builtins) * log_bi; +SERVICE_TYPE(log_builtins_string) * log_bs; + +#define RETURN_IF_BLOCKED(level) \ + if (level > log_level_) return; + +namespace { +void log(loglevel level, const char *format, va_list *args) + MY_ATTRIBUTE((format(printf, 2, 0))); + +void log(loglevel level, const char *format, va_list *args) { + constexpr size_t buffer_len = 8 * 1024; + char buffer[buffer_len]; + vsnprintf(buffer, buffer_len, format, *args); + LogComponentErr(level, ER_LOG_PRINTF_MSG, buffer); +} +} // namespace + +Logger::Logger(SERVICE_TYPE(log_builtins) & log_builtins_service, + SERVICE_TYPE(log_builtins_string) & log_builtins_string_service, + loglevel log_level) + : log_level_(log_level) { + log_bi = &log_builtins_service; + log_bs = &log_builtins_string_service; +} + +void Logger::info(const char *format, ...) { + RETURN_IF_BLOCKED(INFORMATION_LEVEL); + va_list args; + va_start(args, format); + log(INFORMATION_LEVEL, format, &args); + va_end(args); +} + +void Logger::warning(const char *format, ...) { + RETURN_IF_BLOCKED(WARNING_LEVEL); + va_list args; + va_start(args, format); + log(WARNING_LEVEL, format, &args); + va_end(args); +} + +void Logger::error(const char *format, ...) { + RETURN_IF_BLOCKED(ERROR_LEVEL); + va_list args; + va_start(args, format); + log(ERROR_LEVEL, format, &args); + va_end(args); +} diff --git a/components/percona_telemetry/logger.h b/components/percona_telemetry/logger.h new file mode 100644 index 000000000000..68259d6902dc --- /dev/null +++ b/components/percona_telemetry/logger.h @@ -0,0 +1,32 @@ +#ifndef PERCONA_TELEMETRY_LOGGER_H +#define PERCONA_TELEMETRY_LOGGER_H + +#include + +#define LOG_COMPONENT_TAG CURRENT_COMPONENT_NAME_STR +#include +#include +#include "common.h" + +class Logger { + public: + Logger(SERVICE_TYPE(log_builtins) & log_builtins_service, + SERVICE_TYPE(log_builtins_string) & log_builtins_string_service, + loglevel log_level = SYSTEM_LEVEL); + + ~Logger() = default; + + Logger(const Logger &rhs) = delete; + Logger(Logger &&rhs) = delete; + Logger &operator=(const Logger &rhs) = delete; + Logger &operator=(Logger &&rhs) = delete; + + void info(const char *format, ...) MY_ATTRIBUTE((format(printf, 2, 3))); + void warning(const char *format, ...) MY_ATTRIBUTE((format(printf, 2, 3))); + void error(const char *format, ...) MY_ATTRIBUTE((format(printf, 2, 3))); + + private: + loglevel log_level_; +}; + +#endif /* PERCONA_TELEMETRY_LOGGER_H */ diff --git a/components/percona_telemetry/percona_telemetry_component.cc b/components/percona_telemetry/percona_telemetry_component.cc new file mode 100644 index 000000000000..01495c9ce21f --- /dev/null +++ b/components/percona_telemetry/percona_telemetry_component.cc @@ -0,0 +1,53 @@ +#include "percona_telemetry_component.h" +#include "config.h" +#include "data_provider.h" +#include "logger.h" +#include "storage.h" +#include "worker.h" + +PerconaTelemetryComponent::PerconaTelemetryComponent( + std::unique_ptr services) + : services_(std::move(services)) {} + +bool PerconaTelemetryComponent::start() { + logger_ = std::make_unique(*(services_->log_builtins_service), + *(services_->log_builtins_string), + INFORMATION_LEVEL); + + config_ = std::make_unique(*(services_->var_register_service), + *(services_->var_unregister_service)); + if (config_->init()) { + return true; + } + + storage_ = std::make_unique(*config_, *logger_); + + data_provider_ = std::make_unique( + *(services_->command_factory_service), + *(services_->command_options_service), + *(services_->command_query_service), + *(services_->command_query_result_service), + *(services_->command_field_info_service), + *(services_->command_error_info_service), + *(services_->command_thread_service), *logger_); + + worker_ = + std::make_unique(*config_, *storage_, *data_provider_, *logger_); + + return worker_->start(); +} + +bool PerconaTelemetryComponent::stop() { + if (worker_->stop()) { + logger_->warning("Unable to stop PerconaTelemetryComponent."); + return true; + } + + worker_.reset(); + data_provider_.reset(); + storage_.reset(); + config_->deinit(); + config_.reset(); + + return false; +} diff --git a/components/percona_telemetry/percona_telemetry_component.h b/components/percona_telemetry/percona_telemetry_component.h new file mode 100644 index 000000000000..fef030714a03 --- /dev/null +++ b/components/percona_telemetry/percona_telemetry_component.h @@ -0,0 +1,58 @@ +#ifndef PERCONA_TELEMETRY_COMPONENT_H +#define PERCONA_TELEMETRY_COMPONENT_H + +#include + +#include +#include +#include +#include +#include + +#include "config.h" +#include "data_provider.h" +#include "logger.h" +#include "storage.h" +#include "worker.h" + +class PerconaTelemetryComponent { + public: + struct Services { + SERVICE_TYPE(mysql_thd_security_context) * thread_security_context_service; + SERVICE_TYPE(mysql_command_thread) * command_thread_service; + SERVICE_TYPE(mysql_command_factory) * command_factory_service; + SERVICE_TYPE(mysql_command_options) * command_options_service; + SERVICE_TYPE(mysql_command_query) * command_query_service; + SERVICE_TYPE(mysql_command_query_result) * command_query_result_service; + SERVICE_TYPE(mysql_command_field_info) * command_field_info_service; + SERVICE_TYPE(mysql_command_error_info) * command_error_info_service; + SERVICE_TYPE(log_builtins) * log_builtins_service; + SERVICE_TYPE(log_builtins_string) * log_builtins_string; + SERVICE_TYPE(component_sys_variable_register) * var_register_service; + SERVICE_TYPE(component_sys_variable_unregister) * var_unregister_service; + }; + + PerconaTelemetryComponent(std::unique_ptr services); + ~PerconaTelemetryComponent() = default; + + PerconaTelemetryComponent(const PerconaTelemetryComponent &rhs) = delete; + PerconaTelemetryComponent(PerconaTelemetryComponent &&rhs) = delete; + PerconaTelemetryComponent &operator=(const PerconaTelemetryComponent &rhs) = + delete; + PerconaTelemetryComponent &operator=(PerconaTelemetryComponent &&rhs) = + delete; + + bool start(); + bool stop(); + + private: + std::unique_ptr services_; + + std::unique_ptr logger_; + std::unique_ptr config_; + std::unique_ptr storage_; + std::unique_ptr data_provider_; + std::unique_ptr worker_; +}; + +#endif /* PERCONA_TELEMETRY_COMPONENT_H */ diff --git a/components/percona_telemetry/storage.cc b/components/percona_telemetry/storage.cc new file mode 100644 index 000000000000..7e7e47097937 --- /dev/null +++ b/components/percona_telemetry/storage.cc @@ -0,0 +1,150 @@ +#include +#include +#include +#include + +#include +#include +#include + +#include "logger.h" +#include "storage.h" + +namespace fs = std::filesystem; + +namespace { +constexpr auto path_separator = std::filesystem::path::preferred_separator; + +// Let's keep it as std::string, because there is no way to string + string_vew +const std::string TMP_FILE_EXTENSION(".tmp"); +const std::string JSON_FILE_EXTENSION(".json"); + +class file_stream_exception : public std::exception { + public: + file_stream_exception(const char *what) : what_(what) {} + file_stream_exception(const std::string &what) : what_(what) {} + const char *what() const noexcept override { return what_.c_str(); } + + private: + std::string what_; +}; + +static std::string random_uuid() { + static thread_local boost::uuids::random_generator gen; + auto generated = gen(); + return boost::uuids::to_string(generated); +} + +} // namespace + +Storage::Storage(Config &config, Logger &logger) + : config_(config), logger_(logger), uuid_(random_uuid()) {} + +void Storage::clean_old_reports(std::chrono::seconds current_time) { + auto oldest_timestamp = current_time - config_.history_keep_interval(); + auto oldest_unconditional_cleanup_timestamp = + current_time - config_.unconditional_history_cleanup_interval(); + + std::vector files_to_delete; + + for (const auto &entry : + fs::directory_iterator(config_.telemetry_storage_dir_path())) { + if (entry.path().extension() == TMP_FILE_EXTENSION || + entry.path().extension() == JSON_FILE_EXTENSION) { + auto filename = entry.path().stem().string(); + // filename should be -.json/tmp + size_t delimiter_pos = filename.find("-"); + if (delimiter_pos == std::string::npos || delimiter_pos == 0 || + delimiter_pos == filename.length() - 1) { + logger_.warning("Skipping file deletion %s", filename.c_str()); + continue; + } + auto file_timestamp_str = filename.substr(0, delimiter_pos); + auto file_uuid = filename.substr(delimiter_pos + 1); + bool our_file = (file_uuid == uuid_); + + try { + auto file_timestamp = + std::chrono::seconds(std::stoul(file_timestamp_str)); + // We need to delete the file in 2 cases: + // 1. This is our file and it is older than this instance's + // history_keep_interval + // 2. This is someone's else file, but it is older than the highest + // possible history_keep_interval + if (our_file && (file_timestamp < oldest_timestamp)) { + logger_.info( + "Scheduling file %s owned by this server for deletion because it " + "is older than %ld seconds", + entry.path().filename().c_str(), + config_.history_keep_interval().count()); + files_to_delete.push_back(entry.path()); + } else if (!our_file && + (file_timestamp < oldest_unconditional_cleanup_timestamp)) { + logger_.info( + "Scheduling file %s owned by other server for deletion because " + "it is older than %ld seconds", + entry.path().filename().c_str(), + config_.unconditional_history_cleanup_interval().count()); + files_to_delete.push_back(entry.path()); + } + } catch (const std::invalid_argument &e) { + logger_.warning( + "Skipping file deletion %s. Can not determine timestamp.", + filename.c_str()); + } catch (const std::out_of_range &e) { + logger_.warning("Skipping file deletion %s. Timestamp out of range.", + filename.c_str()); + } + } + } + + for (const auto &path : files_to_delete) { + std::error_code ec; + logger_.info("Removing telemetry file: %s", path.c_str()); + if (!fs::remove(path, ec)) { + logger_.warning("Failed to remove file %s, ec: %d, msg: %s", path.c_str(), + ec.value(), ec.message().c_str()); + } + } +} + +bool Storage::store_report(const std::string &report) { + try { + auto now = std::chrono::system_clock::now(); + auto current_time = std::chrono::duration_cast( + now.time_since_epoch()); + + clean_old_reports(current_time); + + /* The report can be empty if we failed to collect all metrics. + In such a case skip creation of an empty file. */ + if (report.length() == 0) { + return false; + } + + std::string filename = config_.telemetry_storage_dir_path() + + path_separator + + std::to_string(current_time.count()) + "-" + uuid_; + std::string tmp_filename = filename + TMP_FILE_EXTENSION; + std::string json_filename = filename + JSON_FILE_EXTENSION; + std::ofstream file_stream(tmp_filename); + if (!file_stream.is_open()) throw file_stream_exception(strerror(errno)); + + if ((file_stream.write(report.c_str(), report.length())).fail()) + throw file_stream_exception(strerror(errno)); + + // close to flush + file_stream.close(); + std::error_code ec; + fs::rename(tmp_filename, json_filename, ec); + if (ec.value() != 0) { + throw file_stream_exception(ec.message()); + } + + logger_.info("Created telemetry file: %s", json_filename.c_str()); + return false; + } catch (const std::exception &e) { + logger_.warning("Problem during telemetry file write: %s", e.what()); + return true; + } +} diff --git a/components/percona_telemetry/storage.h b/components/percona_telemetry/storage.h new file mode 100644 index 000000000000..cd46c51b6b1b --- /dev/null +++ b/components/percona_telemetry/storage.h @@ -0,0 +1,24 @@ +#ifndef PERCONA_TELEMETRY_STORAGE_H +#define PERCONA_TELEMETRY_STORAGE_H + +#include +#include +#include +#include "config.h" + +class Logger; + +class Storage { + public: + Storage(Config &config, Logger &logger); + bool store_report(const std::string &report); + + private: + void clean_old_reports(std::chrono::seconds current_time); + + Config &config_; + Logger &logger_; + std::string uuid_; +}; + +#endif /* PERCONA_TELEMETRY_STORAGE_H */ diff --git a/components/percona_telemetry/worker.cc b/components/percona_telemetry/worker.cc new file mode 100644 index 000000000000..20aa5f599ed7 --- /dev/null +++ b/components/percona_telemetry/worker.cc @@ -0,0 +1,84 @@ +#include + +#include "logger.h" +#include "worker.h" + +Worker::Worker(Config &config, Storage &storage, DataProvider &data_provider, + Logger &logger) + : config_(config), + storage_(storage), + data_provider_(data_provider), + logger_(logger), + stop_worker_thd_(false), + caller_active_(ATOMIC_FLAG_INIT) {} + +bool Worker::start() { + std::thread thd(&Worker::worker_thd_fn, this); + std::swap(thd_, thd); + return false; +} + +bool Worker::stop() { + if (caller_active_.test_and_set()) { + logger_.info("worker active. unload prohibited"); + return true; + } + stop_worker_thd_.store(true); + cv_.notify_all(); + thd_.join(); + + return false; +} + +void Worker::worker_thd_fn() { + std::mutex m; + + auto grace_interval = config_.grace_interval(); + logger_.info("Applying Telemetry grace interval %ld seconds", + grace_interval.count()); + { + std::unique_lock lock(m); + if (std::cv_status::timeout != cv_.wait_for(lock, grace_interval)) { + // the thread was signaled to exit + return; + } + } + + /* If the server is still initializing after waiting for grace_period time, + data_provider will fail to scrape (all) metrics. It will provide empty + report which wont't cause file creation in storage. */ + + data_provider_.thread_access_begin(); + + logger_.info("Scrape interval: %ld seconds", + config_.scrape_interval().count()); + logger_.info("History keep interval: %ld seconds", + config_.history_keep_interval().count()); + logger_.info("Storage dir: %s", config_.telemetry_storage_dir_path().c_str()); + + while (true) { + if (stop_worker_thd_.load()) { + break; + } + + if (!caller_active_.test_and_set()) { + // Just in case, catch all exceptions. + // Telemetry component should not impact the server in any way. + try { + storage_.store_report(data_provider_.get_report()); + } catch (const std::exception &e) { + logger_.warning("%s", e.what()); + } catch (...) { + logger_.warning("%s", "unknown exception caught"); + } + + caller_active_.clear(); + + std::unique_lock lock(m); + cv_.wait_for(lock, config_.scrape_interval()); + // Timed out or signaled to exit. Go on anyway. + } + } + + data_provider_.thread_access_end(); +} diff --git a/components/percona_telemetry/worker.h b/components/percona_telemetry/worker.h new file mode 100644 index 000000000000..cdc423a44e76 --- /dev/null +++ b/components/percona_telemetry/worker.h @@ -0,0 +1,37 @@ +#ifndef PERCONA_TELEMETRY_WORKER_H +#define PERCONA_TELEMETRY_WORKER_H + +#include +#include +#include +#include +#include + +#include "config.h" +#include "data_provider.h" +#include "storage.h" + +class Logger; + +class Worker { + public: + Worker(Config &config, Storage &storage, DataProvider &data_provider, + Logger &logger); + + bool start(); + bool stop(); + + private: + void worker_thd_fn(); + Config &config_; + Storage &storage_; + DataProvider &data_provider_; + Logger &logger_; + std::atomic_bool stop_worker_thd_; + std::atomic_flag caller_active_; + std::condition_variable cv_; + + std::thread thd_; +}; + +#endif /* PERCONA_TELEMETRY_WORKER_H */ diff --git a/config.h.cmake b/config.h.cmake index d0ab39004a43..35def230d3b8 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -177,6 +177,7 @@ #cmakedefine HAVE_BUILTIN_STPCPY 1 #cmakedefine HAVE_GCC_SYNC_BUILTINS 1 #cmakedefine HAVE_VALGRIND +#cmakedefine HAVE_PERCONA_TELEMETRY #cmakedefine HAVE_SYS_GETTID 1 #cmakedefine HAVE_PTHREAD_GETTHREADID_NP 1 #cmakedefine HAVE_PTHREAD_THREADID_NP 1 diff --git a/configure.cmake b/configure.cmake index 4a593eec802a..c21c40d1b8db 100644 --- a/configure.cmake +++ b/configure.cmake @@ -552,6 +552,10 @@ IF(WITH_VALGRIND) ENDIF() ENDIF() +IF(WITH_PERCONA_TELEMETRY) + SET(HAVE_PERCONA_TELEMETRY 1) +ENDIF() + # Check for gettid() system call CHECK_C_SOURCE_COMPILES(" #include diff --git a/mysql-test/include/mysqld--help.inc b/mysql-test/include/mysqld--help.inc index 85161905249b..01a0dabca646 100644 --- a/mysql-test/include/mysqld--help.inc +++ b/mysql-test/include/mysqld--help.inc @@ -27,7 +27,8 @@ perl; # Variables which we don't want to display in result file since # they depend on type of build - @build_specific=qw/auto-generate-certs lock-order tls-version admin-tls-version debug-enable-extended-set-ops version-suffix/; + @build_specific=qw/auto-generate-certs lock-order tls-version admin-tls-version debug-enable-extended-set-ops version-suffix + percona-telemetry-disable/; # Plugins which may or may not be there: @plugins=qw/innodb ngram mecab ndb ndbinfo archive blackhole federated partition ndbcluster debug ssl des-key-file diff --git a/mysql-test/r/percona_utility_user.result b/mysql-test/r/percona_utility_user.result index 6fb0b78d5b49..9af14c4d344e 100644 --- a/mysql-test/r/percona_utility_user.result +++ b/mysql-test/r/percona_utility_user.result @@ -128,18 +128,12 @@ SET @@global.offline_mode = @global_saved_tmp; SHOW CREATE USER 'frank'@'localhost'; ERROR HY000: Operation SHOW CREATE USER failed for 'frank'@'localhost' # restart -SHOW GLOBAL STATUS LIKE "Connections"; -Variable_name Value -Connections 8 SHOW GLOBAL STATUS LIKE "Threads%"; Variable_name Value Threads_cached 0 Threads_connected 1 Threads_created 1 Threads_running 2 -SHOW GLOBAL STATUS LIKE "Connections"; -Variable_name Value -Connections 9 SHOW GLOBAL STATUS LIKE "Threads%"; Variable_name Value Threads_cached 0 @@ -168,8 +162,7 @@ COUNT(DISTINCT PROCESSLIST_ID) SELECT COUNT(*) FROM performance_schema.threads where type='FOREGROUND'; COUNT(*) 3 -KILL 9; -ERROR HY000: You are not owner of thread 9 +ERROR HY000: You are not owner of thread 10 REVOKE PROXY ON 'frank'@'%' FROM 'root'@'localhost'; CREATE ROLE r1; GRANT r1 TO frank@'%'; diff --git a/mysql-test/suite/innodb/r/tablespace_encrypt_9.result b/mysql-test/suite/innodb/r/tablespace_encrypt_9.result index c98c5bbf02e9..8db45d3fa125 100644 --- a/mysql-test/suite/innodb/r/tablespace_encrypt_9.result +++ b/mysql-test/suite/innodb/r/tablespace_encrypt_9.result @@ -101,9 +101,6 @@ ALTER TABLESPACE encrypt_ts ENCRYPTION='Y'; Connectin ids are same. CREATE TABLESPACE temp_ts ADD DATAFILE 'temp_ts.ibd'; # connection con3 -SELECT connection_id(); -connection_id() -11 # Run a DDL with this connection con3 ALTER TABLESPACE temp_ts ENCRYPTION='Y'; set global innodb_buf_flush_list_now = 1; diff --git a/mysql-test/suite/innodb/t/tablespace_encrypt_9.test b/mysql-test/suite/innodb/t/tablespace_encrypt_9.test index 784e80dc9d01..621ec1b1251e 100644 --- a/mysql-test/suite/innodb/t/tablespace_encrypt_9.test +++ b/mysql-test/suite/innodb/t/tablespace_encrypt_9.test @@ -149,7 +149,6 @@ CREATE TABLESPACE temp_ts ADD DATAFILE 'temp_ts.ibd'; --echo # connection con3 connect (con3,localhost,root,,); connection con3; -SELECT connection_id(); --echo # Run a DDL with this connection con3 ALTER TABLESPACE temp_ts ENCRYPTION='Y'; diff --git a/mysql-test/suite/perfschema/t/error_log-master.opt b/mysql-test/suite/perfschema/t/error_log-master.opt index 622b7752b324..aee36c20da98 100644 --- a/mysql-test/suite/perfschema/t/error_log-master.opt +++ b/mysql-test/suite/perfschema/t/error_log-master.opt @@ -1,2 +1,3 @@ +--loose-percona-telemetry-disable=ON $LOGGING_COMPONENTS_OPT --force-restart diff --git a/mysql-test/suite/perfschema/t/pfs_example-master.opt b/mysql-test/suite/perfschema/t/pfs_example-master.opt index 140c4471a792..55ffe60cbc18 100644 --- a/mysql-test/suite/perfschema/t/pfs_example-master.opt +++ b/mysql-test/suite/perfschema/t/pfs_example-master.opt @@ -1 +1 @@ -$PFS_EXAMPLE_OPT +--loose-percona-telemetry-disable=ON $PFS_EXAMPLE_OPT diff --git a/mysql-test/suite/perfschema/t/pfs_example_lifecycle-master.opt b/mysql-test/suite/perfschema/t/pfs_example_lifecycle-master.opt index 140c4471a792..61b096085885 100644 --- a/mysql-test/suite/perfschema/t/pfs_example_lifecycle-master.opt +++ b/mysql-test/suite/perfschema/t/pfs_example_lifecycle-master.opt @@ -1 +1 @@ -$PFS_EXAMPLE_OPT +--loose-percona-telemetry-disable $PFS_EXAMPLE_OPT diff --git a/mysql-test/suite/service_udf_registration/t/udf_reg_unreg_permissions-master.opt b/mysql-test/suite/service_udf_registration/t/udf_reg_unreg_permissions-master.opt index 1ff41bc61774..eb2cd3a870a5 100644 --- a/mysql-test/suite/service_udf_registration/t/udf_reg_unreg_permissions-master.opt +++ b/mysql-test/suite/service_udf_registration/t/udf_reg_unreg_permissions-master.opt @@ -1 +1 @@ -$UDF_REG_ONLY_3_FUNC_OPT +--loose-percona-telemetry-disable=ON $UDF_REG_ONLY_3_FUNC_OPT diff --git a/mysql-test/suite/service_udf_registration/t/udf_reg_unreg_single_func-master.opt b/mysql-test/suite/service_udf_registration/t/udf_reg_unreg_single_func-master.opt index 1ff41bc61774..eb2cd3a870a5 100644 --- a/mysql-test/suite/service_udf_registration/t/udf_reg_unreg_single_func-master.opt +++ b/mysql-test/suite/service_udf_registration/t/udf_reg_unreg_single_func-master.opt @@ -1 +1 @@ -$UDF_REG_ONLY_3_FUNC_OPT +--loose-percona-telemetry-disable=ON $UDF_REG_ONLY_3_FUNC_OPT diff --git a/mysql-test/suite/sys_vars/r/percona_telemetry_disable_basic.result b/mysql-test/suite/sys_vars/r/percona_telemetry_disable_basic.result new file mode 100644 index 000000000000..0a14b8b94dcf --- /dev/null +++ b/mysql-test/suite/sys_vars/r/percona_telemetry_disable_basic.result @@ -0,0 +1 @@ +Intentionally left empty diff --git a/mysql-test/suite/sys_vars/t/all_vars-master.opt b/mysql-test/suite/sys_vars/t/all_vars-master.opt index c885351013fe..b2cf3749e1c5 100644 --- a/mysql-test/suite/sys_vars/t/all_vars-master.opt +++ b/mysql-test/suite/sys_vars/t/all_vars-master.opt @@ -1 +1 @@ -$SEMISYNC_PLUGIN_OPT --loose-skip-ndbinfo +$SEMISYNC_PLUGIN_OPT --loose-skip-ndbinfo --loose-percona-telemetry-disable=ON diff --git a/mysql-test/suite/sys_vars/t/percona_telemetry_disable_basic.test b/mysql-test/suite/sys_vars/t/percona_telemetry_disable_basic.test new file mode 100644 index 000000000000..b7a245e3adfd --- /dev/null +++ b/mysql-test/suite/sys_vars/t/percona_telemetry_disable_basic.test @@ -0,0 +1 @@ +--echo Intentionally left empty diff --git a/mysql-test/t/component-master.opt b/mysql-test/t/component-master.opt index 79050f86cb6b..d63faf777479 100644 --- a/mysql-test/t/component-master.opt +++ b/mysql-test/t/component-master.opt @@ -1 +1 @@ -$EXAMPLE_COMPONENT_1_OPT +--loose-percona-telemetry-disable=ON $EXAMPLE_COMPONENT_1_OPT diff --git a/mysql-test/t/component-upgrade-master.opt b/mysql-test/t/component-upgrade-master.opt index 79050f86cb6b..d63faf777479 100644 --- a/mysql-test/t/component-upgrade-master.opt +++ b/mysql-test/t/component-upgrade-master.opt @@ -1 +1 @@ -$EXAMPLE_COMPONENT_1_OPT +--loose-percona-telemetry-disable=ON $EXAMPLE_COMPONENT_1_OPT diff --git a/mysql-test/t/component_debug-master.opt b/mysql-test/t/component_debug-master.opt index 79050f86cb6b..d63faf777479 100644 --- a/mysql-test/t/component_debug-master.opt +++ b/mysql-test/t/component_debug-master.opt @@ -1 +1 @@ -$EXAMPLE_COMPONENT_1_OPT +--loose-percona-telemetry-disable=ON $EXAMPLE_COMPONENT_1_OPT diff --git a/mysql-test/t/log_components_implicit-master.opt b/mysql-test/t/log_components_implicit-master.opt new file mode 100644 index 000000000000..03813db02e49 --- /dev/null +++ b/mysql-test/t/log_components_implicit-master.opt @@ -0,0 +1 @@ +--loose-percona-telemetry-disable=ON diff --git a/mysql-test/t/log_options_cmdline-master.opt b/mysql-test/t/log_options_cmdline-master.opt index e58c0914da96..7577ce430b30 100644 --- a/mysql-test/t/log_options_cmdline-master.opt +++ b/mysql-test/t/log_options_cmdline-master.opt @@ -1 +1 @@ -$LOGGING_COMPONENTS_OPT +--loose-percona-telemetry-disable=ON $LOGGING_COMPONENTS_OPT diff --git a/mysql-test/t/percona_utility_user.test b/mysql-test/t/percona_utility_user.test index cf9a2808e703..c4dfe9c1375c 100644 --- a/mysql-test/t/percona_utility_user.test +++ b/mysql-test/t/percona_utility_user.test @@ -209,13 +209,15 @@ SHOW CREATE USER 'frank'@'localhost'; # Restart to reset the global counters --source include/restart_mysqld.inc -SHOW GLOBAL STATUS LIKE "Connections"; +--let $connections_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Connections', Value, 1) SHOW GLOBAL STATUS LIKE "Threads%"; connect (frank,localhost,frank,password,mysql); # Invisibility issue: this is the thread_id, will increase with each connection -SHOW GLOBAL STATUS LIKE "Connections"; +--let $connections_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Connections', Value, 1) +--dec $connections_after +--assert($connections_before == $connections_after) # Invisibility issue: Threads_connected includes the utility user too # This is kept as is because this counter is used for logic within thread pool handling # E.g. fixing this would cause issues in threadpool_unix @@ -242,8 +244,10 @@ SELECT COUNT(*) FROM performance_schema.threads where type='FOREGROUND'; connection default; +--disable_query_log --error ER_KILL_DENIED_ERROR --eval KILL $conn_id +--enable_query_log REVOKE PROXY ON 'frank'@'%' FROM 'root'@'localhost'; # PS-5955: Must not be possible to grant or revoke a role to/from utility user diff --git a/sql/dd/impl/upgrade/server.cc b/sql/dd/impl/upgrade/server.cc index ef6c71eb823c..1ffef2bff8cc 100644 --- a/sql/dd/impl/upgrade/server.cc +++ b/sql/dd/impl/upgrade/server.cc @@ -66,6 +66,10 @@ #include "sql/trigger.h" // Trigger #include "sql/trigger_def.h" +#ifdef HAVE_PERCONA_TELEMETRY +#include "sql/dd/dd_utility.h" // check_if_server_ddse_readonly +#endif + typedef ulonglong sql_mode_t; extern const char *mysql_sys_schema[]; extern const char *fill_help_tables[]; @@ -951,6 +955,81 @@ bool upgrade_system_schemas(THD *thd) { return dd::end_transaction(thd, err); } +#ifdef HAVE_PERCONA_TELEMETRY +/* 1. We INSERT INTO, because prepared statements do not support + INSTALL COMPONENT + 2. We use stored procedure to be able to do conditional action. +*/ +static const char *percona_telemetry_install[] = { + "USE mysql;\n", + "SET @have_percona_telemetry= (SELECT COUNT(*) FROM mysql.component WHERE " + "component_urn='file://component_percona_telemetry');\n", + "SET @cmd= 'INSERT INTO mysql.component(component_group_id, component_urn) " + "VALUES (1, \"file://component_percona_telemetry\");';\n", + "SET @str = IF(@have_percona_telemetry = 0, @cmd, 'SET @dummy = 0');\n", + "PREPARE stmt FROM @str;\n", + "EXECUTE stmt;\n", + "DROP PREPARE stmt;\n", + "SET @group_id= (SELECT LAST_INSERT_ID());\n", + "SET @cmd= 'UPDATE mysql.component SET component_group_id=@group_id WHERE " + "component_id=@group_id;';\n", + "SET @str = IF(@have_percona_telemetry = 0, @cmd, 'SET @dummy = 0');\n", + "PREPARE stmt FROM @str;\n", + "EXECUTE stmt;\n", + "DROP PREPARE stmt;\n", + NULL}; + +static const char *percona_telemetry_uninstall[] = { + "USE mysql;\n", + "DELETE FROM mysql.component WHERE " + "component_urn=\"file://component_percona_telemetry\"\n;", + NULL}; + +bool setup_percona_telemetry(THD *thd [[maybe_unused]]) { + if (dd::check_if_server_ddse_readonly(thd, MYSQL_SCHEMA_NAME.str)) { + return false; + } + + Disable_autocommit_guard autocommit_guard(thd); + Bootstrap_error_handler bootstrap_error_handler; + + Server_option_guard acl_guard(&opt_noacl, true); + Server_option_guard general_log_guard(&opt_general_log, false); + Server_option_guard slow_log_guard(&opt_slow_log, false); + Disable_binlog_guard disable_binlog(thd); + Disable_sql_log_bin_guard disable_sql_log_bin(thd); + + bool err = false; + + bootstrap_error_handler.set_log_error(false); + + const char **query_ptr = opt_percona_telemetry_disable + ? &percona_telemetry_uninstall[0] + : &percona_telemetry_install[0]; + for (; *query_ptr != nullptr; query_ptr++) + if (ignore_error_and_execute(thd, *query_ptr)) { + LogErr(WARNING_LEVEL, ER_LOG_PRINTF_MSG, + "Failed during setup Percona Telemetry component. Ignoring."); + break; + }; + + bootstrap_error_handler.set_log_error(true); + + // TODO: needed? + close_thread_tables(thd); + close_cached_tables(nullptr, nullptr, false, LONG_TIMEOUT); + + if (dd::end_transaction(thd, err)) { + LogErr(WARNING_LEVEL, ER_LOG_PRINTF_MSG, + "Failed to setup Percona Telemetry component. Ignoring."); + } + + // Always return success, regardless of Percona Telemetry Component setup + // status. + return false; +} +#endif /* HAVE_PERCONA_TELEMETRY */ + bool no_server_upgrade_required() { return !( dd::bootstrap::DD_bootstrap_ctx::instance().is_server_upgrade() || diff --git a/sql/dd/upgrade/server.h b/sql/dd/upgrade/server.h index 8531a3b2d898..aa393cb75f50 100644 --- a/sql/dd/upgrade/server.h +++ b/sql/dd/upgrade/server.h @@ -72,6 +72,10 @@ namespace upgrade { */ bool upgrade_system_schemas(THD *thd); +#ifdef HAVE_PERCONA_TELEMETRY +bool setup_percona_telemetry(THD *thd); +#endif + bool no_server_upgrade_required(); bool I_S_upgrade_required(); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 6170b3070ba4..bca65b96867e 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1221,6 +1221,10 @@ bool opt_persist_sensitive_variables_in_plaintext{true}; int argc_cached; char **argv_cached; +#ifdef HAVE_PERCONA_TELEMETRY +bool opt_percona_telemetry_disable = false; +#endif + #if defined(_WIN32) /* Thread handle of shutdown event handler thread. @@ -7026,6 +7030,19 @@ static int init_server_components() { } } +#ifdef HAVE_PERCONA_TELEMETRY + if (!is_help_or_validate_option() && !opt_initialize) { + init_optimizer_cost_module(true); + if (bootstrap::run_bootstrap_thread(nullptr, nullptr, + &dd::upgrade::setup_percona_telemetry, + SYSTEM_THREAD_SERVER_UPGRADE)) { + LogErr(ERROR_LEVEL, ER_SERVER_UPGRADE_FAILED); + unireg_abort(MYSQLD_ABORT_EXIT); + } + delete_optimizer_cost_module(); + } +#endif + /* Re-create non DD based system views after a) if we upgraded system schemas b) I_S system view version is changed and server system views diff --git a/sql/mysqld.h b/sql/mysqld.h index f074165f45c8..f4dd2c2817d4 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -915,6 +915,10 @@ extern Deployed_components *g_deployed_components; extern bool opt_persist_sensitive_variables_in_plaintext; +#ifdef HAVE_PERCONA_TELEMETRY +extern bool opt_percona_telemetry_disable; +#endif + void persisted_variables_refresh_keyring_support(); /** diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index d74f0cb5c52b..17e8058b4dc9 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -2517,6 +2517,14 @@ static Sys_var_bool Sys_log_bin("log_bin", "Whether the binary log is enabled", READ_ONLY NON_PERSIST GLOBAL_VAR(opt_bin_log), NO_CMD_LINE, DEFAULT(true)); +#ifdef HAVE_PERCONA_TELEMETRY +static Sys_var_bool Sys_percona_telemetry_disable( + "percona_telemetry_disable", + "Whether Percona Telemetry component should be disabled after server start", + READ_ONLY NON_PERSIST GLOBAL_VAR(opt_percona_telemetry_disable), + CMD_LINE(OPT_ARG), DEFAULT(false)); +#endif + static bool transaction_write_set_check(sys_var *self, THD *thd, set_var *var) { if (check_session_admin(self, thd, var)) return true; // Can't change the algorithm when group replication is enabled. From cd95a4a92c0cc538a77c245db70f27610588cf4d Mon Sep 17 00:00:00 2001 From: Kamil Holubicki Date: Wed, 27 Mar 2024 11:19:17 +0100 Subject: [PATCH 2/4] TEL-27: moved db_replication_id to the top level of the JSON structure --- components/percona_telemetry/data_provider.cc | 73 +++++++++++++++++-- components/percona_telemetry/data_provider.h | 5 ++ 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/components/percona_telemetry/data_provider.cc b/components/percona_telemetry/data_provider.cc index 695ab4636e59..87c99bfc862e 100644 --- a/components/percona_telemetry/data_provider.cc +++ b/components/percona_telemetry/data_provider.cc @@ -67,6 +67,47 @@ const char *replication_info = "replication_info"; } // namespace JSONKey } // namespace +/* We need to provide db_replication_id key at the top level of the JSON +structure. Its value can potentially originate from different sources, so we +have to decide which one to use. This class solves the problem if there is +more than one source of the ID. */ +class DbReplicationIdSolver { +public: + DbReplicationIdSolver() = default; + ~DbReplicationIdSolver() = default; + DbReplicationIdSolver( const DbReplicationIdSolver&) = delete; + DbReplicationIdSolver( const DbReplicationIdSolver&&) = delete; + DbReplicationIdSolver& operator=(const DbReplicationIdSolver&) = delete; + DbReplicationIdSolver& operator=(const DbReplicationIdSolver&&) = delete; + + /* Voters in the order of their priorities. Lower number, lower priority. */ + enum Voter { + NONE, + GROUP_REPLICATION + }; + + void vote(const std::string& id, Voter voter) { + if (voter > id_voter_) { + db_replication_id_ = id; + id_voter_ = voter; + } + } + + const std::string& get_db_replication_id() const { + return db_replication_id_; + } + + void reset() { + db_replication_id_.clear(); + id_voter_ = Voter::NONE; + } + +private: + std::string db_replication_id_; + Voter id_voter_ {Voter::NONE}; +}; + + DataProvider::DataProvider( SERVICE_TYPE(mysql_command_factory) & command_factory_service, SERVICE_TYPE(mysql_command_options) & command_options_service, @@ -82,7 +123,8 @@ DataProvider::DataProvider( command_field_info_service_(command_field_info_service), command_error_info_service_(command_error_info_service), command_thread_service_(command_thread_service), - logger_(logger) {} + logger_(logger), + db_replication_id_solver_(std::make_shared()) {} void DataProvider::thread_access_begin() { command_thread_service_.init(); } @@ -383,10 +425,7 @@ bool DataProvider::collect_group_replication_info( role.SetString(result[0][0].c_str(), allocator); gr_json.AddMember(rapidjson::StringRef(JSONKey::role), role, allocator); - rapidjson::Value replication_group_id; - replication_group_id.SetString(result[0][1].c_str(), allocator); - gr_json.AddMember(rapidjson::StringRef(JSONKey::db_replication_id), - replication_group_id, allocator); + db_replication_id_solver_->vote(result[0][1], DbReplicationIdSolver::Voter::GROUP_REPLICATION); rapidjson::Value single_primary_mode; single_primary_mode.SetString(result[0][2].c_str(), allocator); @@ -488,7 +527,25 @@ bool DataProvider::collect_async_replication_info( return false; } +bool DataProvider::collect_db_replication_id(rapidjson::Document *document) { + + const std::string& id = db_replication_id_solver_->get_db_replication_id(); + if (id.length() > 0) { + rapidjson::Document::AllocatorType &allocator = document->GetAllocator(); + rapidjson::Value replication_group_id; + replication_group_id.SetString(id.c_str(), allocator); + document->AddMember(rapidjson::StringRef(JSONKey::db_replication_id), + replication_group_id, allocator); + } + + return false; +} + bool DataProvider::collect_metrics(rapidjson::Document *document) { + /* The configuration of this instance might have changed, so we need to colect + it every time. */ + db_replication_id_solver_->reset(); + bool res = collect_db_instance_id_info(document); // If db_instance_id cannot be collected all other metrics are meaningless @@ -508,6 +565,12 @@ bool DataProvider::collect_metrics(rapidjson::Document *document) { res |= collect_se_usage_info(document); res |= collect_group_replication_info(document); res |= collect_async_replication_info(document); + + /* The requirement is to have db_replication_id key at the top of JSON + structure. But it may originate from the different places. The above + collect_* methods may set their proposals to db_replication_id_solver_ + and it is up to db_replication_id_solver_ to decide which one to use. */ + res |= collect_db_replication_id(document); return res; } diff --git a/components/percona_telemetry/data_provider.h b/components/percona_telemetry/data_provider.h index c70de48d86b1..de23babf2999 100644 --- a/components/percona_telemetry/data_provider.h +++ b/components/percona_telemetry/data_provider.h @@ -31,6 +31,8 @@ using Row = std::vector; using QueryResult = std::vector; class Logger; +class DbReplicationIdSolver; + class DataProvider { public: DataProvider( @@ -68,8 +70,10 @@ class DataProvider { bool collect_se_usage_info(rapidjson::Document *document); bool collect_group_replication_info(rapidjson::Document *document); bool collect_async_replication_info(rapidjson::Document *document); + bool collect_db_replication_id(rapidjson::Document *document); bool collect_metrics(rapidjson::Document *document); + const std::string &get_database_instance_id(); SERVICE_TYPE(mysql_command_factory) & command_factory_service_; @@ -81,6 +85,7 @@ class DataProvider { SERVICE_TYPE(mysql_command_thread) & command_thread_service_; Logger &logger_; + std::shared_ptr db_replication_id_solver_; std::string database_instance_id_cache_; std::string version_cache_; From e4261bd2e410810bd421447e78a5238c4afb8047 Mon Sep 17 00:00:00 2001 From: Kamil Holubicki Date: Tue, 30 Apr 2024 14:42:26 +0200 Subject: [PATCH 3/4] https://perconadev.atlassian.net/browse/TEL-46 TEL-46: MySql telemetry component creates telemetry files with not sufficient r/w privileges for TA Problem: Depending on OS configuration, MySql component can create telemetry files with permissions not enough for TA to read their content (e.g. 600). It is required to have permissions mask at least 644. TA deletes files after processing, but deletion permission is a matter of the directory mask, not the file itself. Solution: Storage component adds needed permissions after file creation. --- components/percona_telemetry/data_provider.cc | 36 +++++++++---------- components/percona_telemetry/data_provider.h | 1 - components/percona_telemetry/storage.cc | 5 +++ 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/components/percona_telemetry/data_provider.cc b/components/percona_telemetry/data_provider.cc index 87c99bfc862e..ee696def6fc6 100644 --- a/components/percona_telemetry/data_provider.cc +++ b/components/percona_telemetry/data_provider.cc @@ -72,28 +72,25 @@ structure. Its value can potentially originate from different sources, so we have to decide which one to use. This class solves the problem if there is more than one source of the ID. */ class DbReplicationIdSolver { -public: - DbReplicationIdSolver() = default; - ~DbReplicationIdSolver() = default; - DbReplicationIdSolver( const DbReplicationIdSolver&) = delete; - DbReplicationIdSolver( const DbReplicationIdSolver&&) = delete; - DbReplicationIdSolver& operator=(const DbReplicationIdSolver&) = delete; - DbReplicationIdSolver& operator=(const DbReplicationIdSolver&&) = delete; + public: + DbReplicationIdSolver() = default; + ~DbReplicationIdSolver() = default; + DbReplicationIdSolver(const DbReplicationIdSolver &) = delete; + DbReplicationIdSolver(const DbReplicationIdSolver &&) = delete; + DbReplicationIdSolver &operator=(const DbReplicationIdSolver &) = delete; + DbReplicationIdSolver &operator=(const DbReplicationIdSolver &&) = delete; /* Voters in the order of their priorities. Lower number, lower priority. */ - enum Voter { - NONE, - GROUP_REPLICATION - }; + enum Voter { NONE, GROUP_REPLICATION }; - void vote(const std::string& id, Voter voter) { + void vote(const std::string &id, Voter voter) { if (voter > id_voter_) { db_replication_id_ = id; id_voter_ = voter; } } - const std::string& get_db_replication_id() const { + const std::string &get_db_replication_id() const { return db_replication_id_; } @@ -102,12 +99,11 @@ class DbReplicationIdSolver { id_voter_ = Voter::NONE; } -private: + private: std::string db_replication_id_; - Voter id_voter_ {Voter::NONE}; + Voter id_voter_{Voter::NONE}; }; - DataProvider::DataProvider( SERVICE_TYPE(mysql_command_factory) & command_factory_service, SERVICE_TYPE(mysql_command_options) & command_options_service, @@ -425,7 +421,8 @@ bool DataProvider::collect_group_replication_info( role.SetString(result[0][0].c_str(), allocator); gr_json.AddMember(rapidjson::StringRef(JSONKey::role), role, allocator); - db_replication_id_solver_->vote(result[0][1], DbReplicationIdSolver::Voter::GROUP_REPLICATION); + db_replication_id_solver_->vote( + result[0][1], DbReplicationIdSolver::Voter::GROUP_REPLICATION); rapidjson::Value single_primary_mode; single_primary_mode.SetString(result[0][2].c_str(), allocator); @@ -528,14 +525,13 @@ bool DataProvider::collect_async_replication_info( } bool DataProvider::collect_db_replication_id(rapidjson::Document *document) { - - const std::string& id = db_replication_id_solver_->get_db_replication_id(); + const std::string &id = db_replication_id_solver_->get_db_replication_id(); if (id.length() > 0) { rapidjson::Document::AllocatorType &allocator = document->GetAllocator(); rapidjson::Value replication_group_id; replication_group_id.SetString(id.c_str(), allocator); document->AddMember(rapidjson::StringRef(JSONKey::db_replication_id), - replication_group_id, allocator); + replication_group_id, allocator); } return false; diff --git a/components/percona_telemetry/data_provider.h b/components/percona_telemetry/data_provider.h index de23babf2999..97138531dc9b 100644 --- a/components/percona_telemetry/data_provider.h +++ b/components/percona_telemetry/data_provider.h @@ -73,7 +73,6 @@ class DataProvider { bool collect_db_replication_id(rapidjson::Document *document); bool collect_metrics(rapidjson::Document *document); - const std::string &get_database_instance_id(); SERVICE_TYPE(mysql_command_factory) & command_factory_service_; diff --git a/components/percona_telemetry/storage.cc b/components/percona_telemetry/storage.cc index 7e7e47097937..8769c497e4c7 100644 --- a/components/percona_telemetry/storage.cc +++ b/components/percona_telemetry/storage.cc @@ -141,6 +141,11 @@ bool Storage::store_report(const std::string &report) { throw file_stream_exception(ec.message()); } + fs::permissions(json_filename, + fs::perms::owner_read | fs::perms::owner_write | + fs::perms::group_read | fs::perms::others_read, + fs::perm_options::add); + logger_.info("Created telemetry file: %s", json_filename.c_str()); return false; } catch (const std::exception &e) { From 515468d2393c212f1a31a3b2608d9bfef76b083d Mon Sep 17 00:00:00 2001 From: Kamil Holubicki Date: Fri, 31 May 2024 13:16:05 +0200 Subject: [PATCH 4/4] PS-9165: Product Usage Tracking - phase 1 https://perconadev.atlassian.net/browse/PS-9165 Updated copyright info. --- components/percona_telemetry/CMakeLists.txt | 23 ++++---- components/percona_telemetry/common.h | 16 ++++++ components/percona_telemetry/component.cc | 16 ++++++ components/percona_telemetry/config.cc | 16 ++++++ components/percona_telemetry/config.h | 16 ++++++ components/percona_telemetry/data_provider.cc | 54 ++++++------------- components/percona_telemetry/data_provider.h | 16 ++++++ components/percona_telemetry/logger.cc | 16 ++++++ components/percona_telemetry/logger.h | 16 ++++++ .../percona_telemetry_component.cc | 16 ++++++ .../percona_telemetry_component.h | 16 ++++++ components/percona_telemetry/storage.cc | 16 ++++++ components/percona_telemetry/storage.h | 16 ++++++ components/percona_telemetry/worker.cc | 16 ++++++ components/percona_telemetry/worker.h | 16 ++++++ 15 files changed, 233 insertions(+), 52 deletions(-) diff --git a/components/percona_telemetry/CMakeLists.txt b/components/percona_telemetry/CMakeLists.txt index 7415fbb8a5a4..58a5949bb498 100644 --- a/components/percona_telemetry/CMakeLists.txt +++ b/components/percona_telemetry/CMakeLists.txt @@ -1,24 +1,19 @@ -# Copyright (c) 2017, 2021, Oracle and/or its affiliates. +# Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License, version 2.0, -# as published by the Free Software Foundation. -# -# This program is also distributed with certain software (including -# but not limited to OpenSSL) that is licensed under separate terms, -# as designated in a particular file or component or in included license -# documentation. The authors of MySQL hereby grant you an additional -# permission to link the program and your derivative works with the -# separately licensed software that they have included with MySQL. +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 of +# the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License, version 2.0, for more details. +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + DISABLE_MISSING_PROFILE_WARNING() diff --git a/components/percona_telemetry/common.h b/components/percona_telemetry/common.h index 395e390c0da0..216f378c20f0 100644 --- a/components/percona_telemetry/common.h +++ b/components/percona_telemetry/common.h @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #ifndef PERCONA_TELEMETRY_COMMON_H #define PERCONA_TELEMETRY_COMMON_H diff --git a/components/percona_telemetry/component.cc b/components/percona_telemetry/component.cc index 7ebeb83fdb7e..af28ef17dba4 100644 --- a/components/percona_telemetry/component.cc +++ b/components/percona_telemetry/component.cc @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #include #include #include diff --git a/components/percona_telemetry/config.cc b/components/percona_telemetry/config.cc index e04862f28f55..71879b953640 100644 --- a/components/percona_telemetry/config.cc +++ b/components/percona_telemetry/config.cc @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #include #include "common.h" diff --git a/components/percona_telemetry/config.h b/components/percona_telemetry/config.h index b8b3b340b618..8716f0efa721 100644 --- a/components/percona_telemetry/config.h +++ b/components/percona_telemetry/config.h @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #ifndef PERCONA_TELEMETRY_CONFIG_H #define PERCONA_TELEMETRY_CONFIG_H diff --git a/components/percona_telemetry/data_provider.cc b/components/percona_telemetry/data_provider.cc index ee696def6fc6..5b064554dc2b 100644 --- a/components/percona_telemetry/data_provider.cc +++ b/components/percona_telemetry/data_provider.cc @@ -1,47 +1,25 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #include #include #include "data_provider.h" #include "logger.h" -/* The list of metrics collected by Percona Telemetry Component: -Must have: -1. replication information (is it enabled, Galera vs. Group Replication) - If it is PXC, it is Galera - If there is replication (see below) -> async replication - If there is replication and semisync_master or semisync_slave plugin is -installed -> semi-sync replication For GR detection fool around GR plugin and -performance_schema.replication_group_members table (see Orchestrator) -2. product version with “…-pro” suffix for Pro Builds - SELECT VERSION(); - -Should have: -1. plugin information (list of active plugins) - - select plugin_name, plugin_status from information_schema.plugins; - select component_urn from mysql.component; - -Nice to have: -1. MySQL uptime - SHOW GLOBAL STATUS LIKE 'Uptime'; -2. number of databases - SELECT COUNT(*) FROM information_schema.SCHEMATA WHERE -SCHEMA_NAME NOT IN('mysql', 'information_schema', 'performance_schema', 'sys'); -3. size of databases - SELECT IFNULL(ROUND(SUM(data_length + index_length), 1), -"0") size_MB FROM information_schema.tables WHERE table_schema NOT IN('mysql', -'information_schema', 'performance_schema', 'sys'); -4. encryption methods applied (?) - encrypted tables count: SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE -CREATE_OPTIONS LIKE '%ENCRYPTION%'; encrypted databases count: SELECT COUNT(*) -FROM INFORMATION_SCHEMA.SCHEMATA WHERE DEFAULT_ENCRYPTION='YES'; encrypted -tablespaces count: SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES -WHERE (flag & 8192) != 0; -5. number of replication nodes, master/slave - source: - show replicas; - replica: - show replica status; -> Replica_IO_Running, Replica_SQL_Running -6. storage engine used (MyRocks, InnoDB) - SELECT DISTINCT ENGINE FROM -information_schema.tables WHERE table_schema NOT IN('mysql', -'information_schema', 'performance_schema', 'sys'); 7. -*/ - namespace { inline const char *b2s(bool val) { return val ? "1" : "0"; } diff --git a/components/percona_telemetry/data_provider.h b/components/percona_telemetry/data_provider.h index 97138531dc9b..2d017a25218f 100644 --- a/components/percona_telemetry/data_provider.h +++ b/components/percona_telemetry/data_provider.h @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #ifndef PERCONA_TELEMETRY_DATA_PROVIDER_H #define PERCONA_TELEMETRY_DATA_PROVIDER_H diff --git a/components/percona_telemetry/logger.cc b/components/percona_telemetry/logger.cc index 04047d8969b5..4e4127aa677d 100644 --- a/components/percona_telemetry/logger.cc +++ b/components/percona_telemetry/logger.cc @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #include #include diff --git a/components/percona_telemetry/logger.h b/components/percona_telemetry/logger.h index 68259d6902dc..91b391b4e887 100644 --- a/components/percona_telemetry/logger.h +++ b/components/percona_telemetry/logger.h @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #ifndef PERCONA_TELEMETRY_LOGGER_H #define PERCONA_TELEMETRY_LOGGER_H diff --git a/components/percona_telemetry/percona_telemetry_component.cc b/components/percona_telemetry/percona_telemetry_component.cc index 01495c9ce21f..03871e81e42b 100644 --- a/components/percona_telemetry/percona_telemetry_component.cc +++ b/components/percona_telemetry/percona_telemetry_component.cc @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #include "percona_telemetry_component.h" #include "config.h" #include "data_provider.h" diff --git a/components/percona_telemetry/percona_telemetry_component.h b/components/percona_telemetry/percona_telemetry_component.h index fef030714a03..c507c4160d26 100644 --- a/components/percona_telemetry/percona_telemetry_component.h +++ b/components/percona_telemetry/percona_telemetry_component.h @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #ifndef PERCONA_TELEMETRY_COMPONENT_H #define PERCONA_TELEMETRY_COMPONENT_H diff --git a/components/percona_telemetry/storage.cc b/components/percona_telemetry/storage.cc index 8769c497e4c7..0214f38ab194 100644 --- a/components/percona_telemetry/storage.cc +++ b/components/percona_telemetry/storage.cc @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #include #include #include diff --git a/components/percona_telemetry/storage.h b/components/percona_telemetry/storage.h index cd46c51b6b1b..646e25144e46 100644 --- a/components/percona_telemetry/storage.h +++ b/components/percona_telemetry/storage.h @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #ifndef PERCONA_TELEMETRY_STORAGE_H #define PERCONA_TELEMETRY_STORAGE_H diff --git a/components/percona_telemetry/worker.cc b/components/percona_telemetry/worker.cc index 20aa5f599ed7..f0cd387a258b 100644 --- a/components/percona_telemetry/worker.cc +++ b/components/percona_telemetry/worker.cc @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #include #include "logger.h" diff --git a/components/percona_telemetry/worker.h b/components/percona_telemetry/worker.h index cdc423a44e76..d4f2184e472b 100644 --- a/components/percona_telemetry/worker.h +++ b/components/percona_telemetry/worker.h @@ -1,3 +1,19 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + #ifndef PERCONA_TELEMETRY_WORKER_H #define PERCONA_TELEMETRY_WORKER_H