Merge pull request #5473 from inikep/TEL-26-trunk

TEL-26, TEL-29 [9.x]: Add unit tests and MTR tests for Percona Telemetry Component

components/percona_telemetry/data_provider.h

@@ -61,7 +61,7 @@ class DataProvider {
       SERVICE_TYPE(mysql_command_thread) & command_thread_service,
       Logger &logger);
 
-  ~DataProvider() = default;
+  virtual ~DataProvider() = default;
 
   DataProvider(const DataProvider &rhs) = delete;
   DataProvider(DataProvider &&rhs) = delete;
@@ -73,9 +73,9 @@ class DataProvider {
   std::string get_report();
 
  private:
-  bool do_query(const std::string &query, QueryResult *result,
-                unsigned int *err_no = nullptr,
-                bool suppress_query_error_log = false);
+  virtual bool do_query(const std::string &query, QueryResult *result,
+                        unsigned int *err_no = nullptr,
+                        bool suppress_query_error_log = false);
   bool collect_db_instance_id_info(rapidjson::Document *document);
   bool collect_product_version_info(rapidjson::Document *document);
   bool collect_plugins_info(rapidjson::Document *document);

mysql-test/suite/component_percona_telemetry/r/delete_obsolete_file.result

@@ -0,0 +1,6 @@
+CALL mtr.add_suppression("Component percona_telemetry reported: 'Skipping file deletion this_file_should_not_be_removed'");
+# restart:--percona_telemetry.grace_interval=30 --percona_telemetry.scrape_interval=30 --percona_telemetry.history_keep_interval=80 --percona_telemetry.telemetry_root_dir=<telemetry_root_dir>
+File with the name not conforming to the pattern still should be there
+1
+Obsolete file should be removed
+0

mysql-test/suite/component_percona_telemetry/r/invalid_path.result

@@ -0,0 +1,6 @@
+# restart:--percona_telemetry.grace_interval=30 --percona_telemetry.scrape_interval=30 --percona_telemetry.history_keep_interval=80 --percona_telemetry.telemetry_root_dir=<telemetry_root_dir>
+Server should still be alive
+CALL mtr.add_suppression("Component percona_telemetry reported: 'Problem during telemetry file write: filesystem error: directory iterator cannot open directory: No such file or directory");
+include/assert_grep.inc [Percona Telemetry Component warns about nonexistent directory]
+Telemetry root dir should contain 1 file
+1

mysql-test/suite/component_percona_telemetry/r/telemetry_disable.result

@@ -0,0 +1,11 @@
+include/assert.inc [Percona Telemetry Component should not be installed]
+UNINSTALL COMPONENT 'file://component_percona_telemetry';
+include/assert.inc [Percona Telemetry Component should not be installed]
+# restart:
+include/assert.inc [Percona Telemetry Component should be installed after server restart]
+# restart:--percona-telemetry-disable=1
+include/assert.inc [Percona Telemetry Component should not be installed when server is started with --percona-telemetry-disable=1]
+INSTALL COMPONENT 'file://component_percona_telemetry';
+include/assert.inc [It should be possible to manually install Percona Telemetry Component]
+# restart:--percona-telemetry-disable=1
+include/assert.inc [Percona Telemetry Component should not be installed when server is started with --percona-telemetry-disable=1]

mysql-test/suite/component_percona_telemetry/r/telemetry_file_creation.result

@@ -0,0 +1,11 @@
+# restart:--percona_telemetry.grace_interval=30 --percona_telemetry.scrape_interval=30 --percona_telemetry.history_keep_interval=80 --percona_telemetry.telemetry_root_dir=<telemetry_root_dir>
+Time passed: 10.0000. Still in grace_interval. Telemetry root dir should contain 0 files
+0
+Time passed: 40.0000. Time after grace_interval: 10.0000. Telemetry root dir should contain 1 file
+1
+Time passed: 70.0000. Time after grace_interval: 40.0000. Telemetry root dir should contain 2 files
+2
+Time passed: 100.0000. Time after grace_interval: 70.0000. Telemetry root dir should contain 3 files
+3
+Time passed: 130.0000. Time after grace_interval: 100.0000. Telemetry root dir should still contain 3 files
+3

mysql-test/suite/component_percona_telemetry/t/delete_obsolete_file.test

@@ -0,0 +1,40 @@
+# Test that obsolete files from another servers are removed,
+# but files with names not conforming to expected pattern are not.
+
+--source include/have_percona_telemetry.inc
+--source include/force_restart.inc
+
+CALL mtr.add_suppression("Component percona_telemetry reported: 'Skipping file deletion this_file_should_not_be_removed'");
+
+--let $telemetry_root_dir = $MYSQL_TMP_DIR/telemetry_dir
+--let $grace_interval = 30
+--let $scrape_interval = 30
+--let $history_keep_interval = 80
+
+--let $obsolete_file_name = 313671600-1af5d44c-81f9-4083-807d-e71ca7914f92.json
+--let $untouchable_file_name = this_file_should_not_be_removed.json
+--let $untouchable_file = $telemetry_root_dir/$untouchable_file_name
+--let $obsolete_file = $telemetry_root_dir/$obsolete_file_name
+
+--mkdir $telemetry_root_dir
+--exec touch $untouchable_file
+--exec touch $obsolete_file
+
+# restart the server with custom telemetry file path and timeouts
+--let $restart_parameters = "restart:--percona_telemetry.grace_interval=$grace_interval --percona_telemetry.scrape_interval=$scrape_interval --percona_telemetry.history_keep_interval=$history_keep_interval --percona_telemetry.telemetry_root_dir=$telemetry_root_dir"
+--replace_regex /telemetry_root_dir=.*telemetry_dir/telemetry_root_dir=<telemetry_root_dir>/
+--source include/restart_mysqld.inc
+
+# Wait for percona_telemetry.grace_interval to pass (+10sec)
+--let $timeout = `select $grace_interval + 10`
+--sleep $timeout
+
+--echo File with the name not conforming to the pattern still should be there
+--file_exists $untouchable_file
+--exec ls -1 $telemetry_root_dir | grep $untouchable_file_name | wc -l
+
+--echo Obsolete file should be removed
+--exec ls -1 $telemetry_root_dir | grep $obsolete_file_name | wc -l
+
+# cleanup
+--force-rmdir $telemetry_root_dir

mysql-test/suite/component_percona_telemetry/t/invalid_path.test

@@ -0,0 +1,42 @@
+# Test that Percona Telemetry Component doesn't do any harm to the server when it is not possible
+# to store telemetry file
+
+--source include/have_percona_telemetry.inc
+--source include/force_restart.inc
+
+--let $telemetry_root_dir = $MYSQL_TMP_DIR/telemetry_dir
+--let $grace_interval = 30
+--let $scrape_interval = 30
+--let $history_keep_interval = 80
+
+
+# restart the server with custom telemetry file path and timeouts
+--let $restart_parameters = "restart:--percona_telemetry.grace_interval=$grace_interval --percona_telemetry.scrape_interval=$scrape_interval --percona_telemetry.history_keep_interval=$history_keep_interval --percona_telemetry.telemetry_root_dir=$telemetry_root_dir"
+--replace_regex /telemetry_root_dir=.*telemetry_dir/telemetry_root_dir=<telemetry_root_dir>/
+--source include/restart_mysqld.inc
+
+# Wait for more than grace_interval. Component should try to create the telemetry file, but as the target dir
+# doesn't exist it will fail. It should complain with a warning in the log, but continue.
+--let $timeout = `select $grace_interval + 20`
+--sleep $timeout
+--echo Server should still be alive
+
+--let $warning_message = Component percona_telemetry reported: 'Problem during telemetry file write: filesystem error: directory iterator cannot open directory: No such file or directory
+--eval CALL mtr.add_suppression("$warning_message")
+--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.1.err
+--let $assert_select = $warning_message
+--let $assert_count=1
+--let $assert_text = Percona Telemetry Component warns about nonexistent directory
+--source include/assert_grep.inc
+
+
+# Now create the telemetry dir and wait for scrape_interval. Telemetry file should be created
+--mkdir $telemetry_root_dir
+
+--let $timeout = $scrape_interval
+--sleep $timeout
+--echo Telemetry root dir should contain 1 file
+--exec ls -1 $telemetry_root_dir | wc -l
+
+# cleanup
+--force-rmdir $telemetry_root_dir

mysql-test/suite/component_percona_telemetry/t/telemetry_disable.test

@@ -0,0 +1,50 @@
+# Component enable/disable
+
+--source include/have_percona_telemetry.inc
+--source include/force_restart.inc
+
+
+# Telemetry is enabled. It is checked already by have_percona_telemetry.inc
+--let $assert_text = Percona Telemetry Component should not be installed
+--let $assert_cond = [SELECT COUNT(*) FROM mysql.component WHERE component_urn = "file://component_percona_telemetry"] = 1
+--source include/assert.inc
+
+# Component can be unloaded
+UNINSTALL COMPONENT 'file://component_percona_telemetry';
+
+--let $assert_text = Percona Telemetry Component should not be installed
+--let $assert_cond = [SELECT COUNT(*) FROM mysql.component WHERE component_urn = "file://component_percona_telemetry"] = 0
+--source include/assert.inc
+
+# Restart the server. Percona Telemetry Component should be installed again
+--let $restart_parameters = "restart:"
+--source include/restart_mysqld.inc
+
+--let $assert_text = Percona Telemetry Component should be installed after server restart
+--let $assert_cond = [SELECT COUNT(*) FROM mysql.component WHERE component_urn = "file://component_percona_telemetry"] = 1
+--source include/assert.inc
+
+# Now restart the server with --percona-telemetry-disable=1. Component should not be loaded
+--let $restart_parameters = "restart:--percona-telemetry-disable=1"
+--source include/restart_mysqld.inc
+
+--let $assert_text = Percona Telemetry Component should not be installed when server is started with --percona-telemetry-disable=1
+--let $assert_cond = [SELECT COUNT(*) FROM mysql.component WHERE component_urn = "file://component_percona_telemetry"] = 0
+--source include/assert.inc
+
+# Component can be loaded
+INSTALL COMPONENT 'file://component_percona_telemetry';
+
+--let $assert_text = It should be possible to manually install Percona Telemetry Component
+--let $assert_cond = [SELECT COUNT(*) FROM mysql.component WHERE component_urn = "file://component_percona_telemetry"] = 1
+--source include/assert.inc
+
+# ... but after restart with --percona-telemetry-disable=1 is should not be there
+# Now restart the server with --percona-telemetry-disable=1. Component should not be loaded
+--let $restart_parameters = "restart:--percona-telemetry-disable=1"
+--source include/restart_mysqld.inc
+
+--let $assert_text = Percona Telemetry Component should not be installed when server is started with --percona-telemetry-disable=1
+--let $assert_cond = [SELECT COUNT(*) FROM mysql.component WHERE component_urn = "file://component_percona_telemetry"] = 0
+--source include/assert.inc
+

mysql-test/suite/component_percona_telemetry/t/telemetry_file_creation.test

@@ -0,0 +1,76 @@
+# Test the telemetry file creation and cleanup
+
+--source include/have_percona_telemetry.inc
+--source include/force_restart.inc
+
+--let $telemetry_root_dir = $MYSQL_TMP_DIR/telemetry_dir
+--let $grace_interval = 30
+--let $scrape_interval = 30
+--let $history_keep_interval = 80
+
+--mkdir $telemetry_root_dir
+
+# restart the server with custom telemetry file path and timeouts
+--let $restart_parameters = "restart:--percona_telemetry.grace_interval=$grace_interval --percona_telemetry.scrape_interval=$scrape_interval --percona_telemetry.history_keep_interval=$history_keep_interval --percona_telemetry.telemetry_root_dir=$telemetry_root_dir"
+--replace_regex /telemetry_root_dir=.*telemetry_dir/telemetry_root_dir=<telemetry_root_dir>/
+--source include/restart_mysqld.inc
+
+# time = 0
+--let $time_passed = 0
+--let $time_after_grace_interval=0
+# Right after the server start, due to percona_telemetry.grace_interval, the telemetry file should not be created
+--let $timeout = `select $grace_interval / 3`
+--sleep $timeout
+--let $time_passed = `select $time_passed + $timeout`
+
+### now we are withing grace_interval
+--echo Time passed: $time_passed. Still in grace_interval. Telemetry root dir should contain 0 files
+--exec ls -1 $telemetry_root_dir | wc -l
+
+
+
+# Wait for percona_telemetry.grace_interval to pass (+10sec) and check if telemetry file was created
+--let $timeout = `select $grace_interval - $time_passed + 10`
+--sleep $timeout
+--let $time_passed = `select $time_passed + $timeout`
+--let $time_after_grace_interval = `select $time_passed - $grace_interval`
+
+### now we are in 1st scrape interval
+--echo Time passed: $time_passed. Time after grace_interval: $time_after_grace_interval. Telemetry root dir should contain 1 file
+--exec ls -1 $telemetry_root_dir | wc -l
+
+
+
+# Wait a bit more. New telemetry file should be created
+--let $timeout = $scrape_interval
+--sleep $timeout
+--let $time_passed = `select $time_passed + $timeout`
+--let $time_after_grace_interval = `select $time_passed - $grace_interval`
+
+### now we are in 2nd scrape interval
+--echo Time passed: $time_passed. Time after grace_interval: $time_after_grace_interval. Telemetry root dir should contain 2 files
+--exec ls -1 $telemetry_root_dir | wc -l
+
+
+# Wait a bit more. New telemetry file should be created
+--let $timeout = $scrape_interval
+--sleep $timeout
+--let $time_passed = `select $time_passed + $timeout`
+--let $time_after_grace_interval = `select $time_passed - $grace_interval`
+
+### now we are in 3rd scrape interval
+--echo Time passed: $time_passed. Time after grace_interval: $time_after_grace_interval. Telemetry root dir should contain 3 files
+--exec ls -1 $telemetry_root_dir | wc -l
+
+
+# Due to history_keep_interval, new files should be created, but old ones should be deleted.
+--let $timeout = $scrape_interval
+--sleep $timeout
+--let $time_passed = `select $time_passed + $timeout`
+--let $time_after_grace_interval = `select $time_passed - $grace_interval`
+
+--echo Time passed: $time_passed. Time after grace_interval: $time_after_grace_interval. Telemetry root dir should still contain 3 files
+--exec ls -1 $telemetry_root_dir | wc -l
+
+# cleanup
+--force-rmdir $telemetry_root_dir

unittest/gunit/CMakeLists.txt

@@ -469,6 +469,7 @@ ADD_SUBDIRECTORY(innodb)
 ADD_SUBDIRECTORY(components/mysql_server)
 ADD_SUBDIRECTORY(components/keyring_common)
 ADD_SUBDIRECTORY(components/keyring_vault)
+ADD_SUBDIRECTORY(components/percona_telemetry)
 ADD_SUBDIRECTORY(xplugin)
 ADD_SUBDIRECTORY(group_replication)
 ADD_SUBDIRECTORY(libmysqlgcs)

unittest/gunit/components/percona_telemetry/CMakeLists.txt

@@ -0,0 +1,45 @@
+# Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2 of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+MY_CHECK_CXX_COMPILER_WARNING("-Wno-unused-local-typedefs" HAS_WARN_FLAG)
+IF(HAS_WARN_FLAG)
+  STRING_APPEND(CMAKE_CXX_FLAGS " ${HAS_WARN_FLAG}")
+ENDIF()
+
+INCLUDE_DIRECTORIES(SYSTEM
+  ${GMOCK_INCLUDE_DIRS}
+  ${BOOST_PATCHES_DIR}
+  ${BOOST_INCLUDE_DIR}
+)
+
+SET(PERCONA_TELEMETRY_COMPONENT_SRC
+  ${CMAKE_SOURCE_DIR}/components/percona_telemetry/data_provider.cc
+)
+
+SET(LOCAL_MOCK_SRC
+  logger.cc
+)
+
+# Add tests
+SET(TESTS
+  data_provider
+  )
+
+FOREACH(test ${TESTS})
+  MYSQL_ADD_EXECUTABLE(${test}-t ${PERCONA_TELEMETRY_COMPONENT_SRC} ${LOCAL_MOCK_SRC} ${test}-t.cc ADD_TEST ${test} LINK_LIBRARIES ${PERCONA_TELEMETRY_COMPONENT_LIBS})
+  TARGET_LINK_LIBRARIES(${test}-t mysys gunit_small)
+ENDFOREACH()
+