#347: Optimized algorithm for collecting the data of tables for query…

… optimization (#359)
Releem · Oct 8, 2024 · 1c69c33 · 1c69c33
1 parent a481930
commit 1c69c33
Show file tree

Hide file tree

Showing 7 changed files with 46 additions and 39 deletions.
diff --git a/config/config.go b/config/config.go
@@ -11,7 +11,7 @@ import (
 )
 
 const (
-	ReleemAgentVersion = "1.19.4.1"
+	ReleemAgentVersion = "1.19.4.2"
 )
 
 var (

diff --git a/current_version_agent b/current_version_agent
@@ -1 +1 @@
-1.19.4.1
+1.19.4.2
diff --git a/install.sh b/install.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# install.sh - Version 1.19.4.1
+# install.sh - Version 1.19.4.2
 # (C) Releem, Inc 2022
 # All rights reserved
 
@@ -9,7 +9,7 @@ export PATH=$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/
 # using the package manager.
 
 set -e -E
-install_script_version=1.19.4.1
+install_script_version=1.19.4.2
 logfile="/var/log/releem-install.log"
 
 WORKDIR="/opt/releem"

diff --git a/metrics/dbCollectQueries.go b/metrics/dbCollectQueries.go
@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"sort"
 	"strings"
+	"time"
 
 	u "github.com/Releem/mysqlconfigurer/utils"
 
@@ -118,19 +119,26 @@ func (DbCollectQueriesOptimization *DbCollectQueriesOptimization) GetMetrics(met
 			TABLE_COLLATION string
 		}
 		var information_schema_table information_schema_table_type
-
-		rows, err := config.DB.Query("SELECT IFNULL(TABLE_SCHEMA, 'NULL') as TABLE_SCHEMA, IFNULL(TABLE_NAME, 'NULL') as TABLE_NAME, IFNULL(TABLE_TYPE, 'NULL') as TABLE_TYPE,  IFNULL(ENGINE, 'NULL') as ENGINE, IFNULL(ROW_FORMAT, 'NULL') as ROW_FORMAT, IFNULL(TABLE_ROWS, 'NULL') as TABLE_ROWS, IFNULL(AVG_ROW_LENGTH, 'NULL') as AVG_ROW_LENGTH, IFNULL(MAX_DATA_LENGTH, 'NULL') as MAX_DATA_LENGTH,IFNULL(DATA_LENGTH, 'NULL') as DATA_LENGTH, IFNULL(INDEX_LENGTH, 'NULL') as INDEX_LENGTH, IFNULL(TABLE_COLLATION, 'NULL') as TABLE_COLLATION FROM information_schema.tables")
-		if err != nil {
-			DbCollectQueriesOptimization.logger.Error(err)
-		} else {
-			for rows.Next() {
-				err := rows.Scan(&information_schema_table.TABLE_SCHEMA, &information_schema_table.TABLE_NAME, &information_schema_table.TABLE_TYPE, &information_schema_table.ENGINE, &information_schema_table.ROW_FORMAT, &information_schema_table.TABLE_ROWS, &information_schema_table.AVG_ROW_LENGTH, &information_schema_table.MAX_DATA_LENGTH, &information_schema_table.DATA_LENGTH, &information_schema_table.INDEX_LENGTH, &information_schema_table.TABLE_COLLATION)
-				if err != nil {
-					DbCollectQueriesOptimization.logger.Error(err)
-					return err
+		i := 0
+		for _, database := range metrics.DB.Metrics.Databases {
+			rows, err := config.DB.Query(`SELECT IFNULL(TABLE_SCHEMA, 'NULL') as TABLE_SCHEMA, IFNULL(TABLE_NAME, 'NULL') as TABLE_NAME, IFNULL(TABLE_TYPE, 'NULL') as TABLE_TYPE,  IFNULL(ENGINE, 'NULL') as ENGINE, IFNULL(ROW_FORMAT, 'NULL') as ROW_FORMAT, IFNULL(TABLE_ROWS, 'NULL') as TABLE_ROWS, IFNULL(AVG_ROW_LENGTH, 'NULL') as AVG_ROW_LENGTH, IFNULL(MAX_DATA_LENGTH, 'NULL') as MAX_DATA_LENGTH,IFNULL(DATA_LENGTH, 'NULL') as DATA_LENGTH, IFNULL(INDEX_LENGTH, 'NULL') as INDEX_LENGTH, IFNULL(TABLE_COLLATION, 'NULL') as TABLE_COLLATION FROM information_schema.tables WHERE TABLE_SCHEMA = ? `, database)
+			if err != nil {
+				DbCollectQueriesOptimization.logger.Error(err)
+			} else {
+				for rows.Next() {
+					err := rows.Scan(&information_schema_table.TABLE_SCHEMA, &information_schema_table.TABLE_NAME, &information_schema_table.TABLE_TYPE, &information_schema_table.ENGINE, &information_schema_table.ROW_FORMAT, &information_schema_table.TABLE_ROWS, &information_schema_table.AVG_ROW_LENGTH, &information_schema_table.MAX_DATA_LENGTH, &information_schema_table.DATA_LENGTH, &information_schema_table.INDEX_LENGTH, &information_schema_table.TABLE_COLLATION)
+					if err != nil {
+						DbCollectQueriesOptimization.logger.Error(err)
+						return err
+					}
+					table := MetricGroupValue{"TABLE_SCHEMA": information_schema_table.TABLE_SCHEMA, "TABLE_NAME": information_schema_table.TABLE_NAME, "TABLE_TYPE": information_schema_table.TABLE_TYPE, "ENGINE": information_schema_table.ENGINE, "ROW_FORMAT": information_schema_table.ROW_FORMAT, "TABLE_ROWS": information_schema_table.TABLE_ROWS, "AVG_ROW_LENGTH": information_schema_table.AVG_ROW_LENGTH, "MAX_DATA_LENGTH": information_schema_table.MAX_DATA_LENGTH, "DATA_LENGTH": information_schema_table.DATA_LENGTH, "INDEX_LENGTH": information_schema_table.INDEX_LENGTH, "TABLE_COLLATION": information_schema_table.TABLE_COLLATION}
+					output = append(output, table)
 				}
-				table := MetricGroupValue{"TABLE_SCHEMA": information_schema_table.TABLE_SCHEMA, "TABLE_NAME": information_schema_table.TABLE_NAME, "TABLE_TYPE": information_schema_table.TABLE_TYPE, "ENGINE": information_schema_table.ENGINE, "ROW_FORMAT": information_schema_table.ROW_FORMAT, "TABLE_ROWS": information_schema_table.TABLE_ROWS, "AVG_ROW_LENGTH": information_schema_table.AVG_ROW_LENGTH, "MAX_DATA_LENGTH": information_schema_table.MAX_DATA_LENGTH, "DATA_LENGTH": information_schema_table.DATA_LENGTH, "INDEX_LENGTH": information_schema_table.INDEX_LENGTH, "TABLE_COLLATION": information_schema_table.TABLE_COLLATION}
-				output = append(output, table)
+			}
+			rows.Close()
+			i += 1
+			if i%25 == 0 {
+				time.Sleep(3 * time.Second)
 			}
 		}
 		QueriesOptimization["information_schema_tables"] = output

diff --git a/metrics/dbMetrics.go b/metrics/dbMetrics.go
@@ -40,26 +40,6 @@ func (DbMetrics *DbMetricsGatherer) GetMetrics(metrics *Metrics) error {
 		}
 		metrics.DB.Metrics.TotalTables = row
 	}
-	//list of databases
-	{
-		var database string
-		var output []string
-		rows, err := config.DB.Query("SELECT table_schema FROM INFORMATION_SCHEMA.tables group BY table_schema")
-		if err != nil {
-			DbMetrics.logger.Error(err)
-			return err
-		}
-		for rows.Next() {
-			err := rows.Scan(&database)
-			if err != nil {
-				DbMetrics.logger.Error(err)
-				return err
-			}
-			output = append(output, database)
-		}
-		rows.Close()
-		metrics.DB.Metrics.Databases = output
-	}
 	//Stat mysql Engine
 	{
 		var engine_db, engineenabled string

diff --git a/metrics/dbMetricsBase.go b/metrics/dbMetricsBase.go
@@ -142,9 +142,28 @@ func (DbMetricsBase *DbMetricsBaseGatherer) GetMetrics(metrics *Metrics) error {
 			metrics.DB.Metrics.InnoDBEngineStatus = status
 		}
 	}
+	//list of databases
+	{
+		var database string
+		var output []string
+		rows, err := config.DB.Query("SELECT table_schema FROM INFORMATION_SCHEMA.tables group BY table_schema")
+		if err != nil {
+			DbMetricsBase.logger.Error(err)
+			return err
+		}
+		for rows.Next() {
+			err := rows.Scan(&database)
+			if err != nil {
+				DbMetricsBase.logger.Error(err)
+				return err
+			}
+			output = append(output, database)
+		}
+		rows.Close()
+		metrics.DB.Metrics.Databases = output
+	}
 	DbMetricsBase.logger.Debug("collectMetrics ", metrics.DB.Metrics)
 	return nil
-
 }
 
 func contains(arr []int, num int) bool {

diff --git a/mysqlconfigurer.sh b/mysqlconfigurer.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# mysqlconfigurer.sh - Version 1.19.4.1
+# mysqlconfigurer.sh - Version 1.19.4.2
 # (C) Releem, Inc 2022
 # All rights reserved
 
@@ -15,7 +15,7 @@ MYSQLTUNER_REPORT=$MYSQLCONFIGURER_PATH"mysqltunerreport.json"
 RELEEM_MYSQL_VERSION=$MYSQLCONFIGURER_PATH"mysql_version"
 MYSQLCONFIGURER_CONFIGFILE="${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}"
 MYSQL_MEMORY_LIMIT=0
-VERSION="1.19.4.1"
+VERSION="1.19.4.2"
 RELEEM_INSTALL_PATH=$MYSQLCONFIGURER_PATH"install.sh"
 logfile="/var/log/releem-mysqlconfigurer.log"