Add archiving diagnostic commands (#21713)

* Add archiving diagnostic commands * Fix missing list commas * Code tidy, return failure errorlevel if email send fails * Add segment definition display to the archiving invalidation queue command for segment invalidations * Linter nitpicking fixes * Wrap symphony console classes, adjust tests * Update test results * Cast int values for consistency in json tests
matomo-org · Dec 28, 2023 · 55f6369 · 55f6369
1 parent 1630da7
commit 55f6369
Show file tree

Hide file tree

Showing 14 changed files with 1,058 additions and 0 deletions.
diff --git a/core/DataAccess/ArchiveTableDao.php b/core/DataAccess/ArchiveTableDao.php
@@ -9,7 +9,9 @@
 namespace Piwik\DataAccess;
 
 use Piwik\Common;
+use Piwik\Date;
 use Piwik\Db;
+use Piwik\Metrics\Formatter;
 
 /**
  * Data Access class for querying numeric & blob archive tables.
@@ -86,4 +88,57 @@ public function getArchiveTableAnalysis($tableDate)
 
         return $result;
     }
+
+    /**
+     * Return invalidation queue table data
+     *
+     * @param bool $prettyTime
+     *
+     * @return array
+     * @throws \Exception
+     */
+    public function getInvalidationQueueData(bool $prettyTime = false): array
+    {
+        $invalidationsTable = Common::prefixTable("archive_invalidations");
+        $segmentsTable = Common::prefixTable("segment");
+        $sql = "
+            SELECT ai.*, s.definition               
+            FROM `$invalidationsTable` ai 
+            LEFT JOIN `$segmentsTable` s ON SUBSTRING(ai.name, 5) = s.hash
+            GROUP BY ai.idinvalidation
+            ORDER BY ts_invalidated, idinvalidation ASC";
+        $invalidations = Db::fetchAll($sql);
+
+        $metricsFormatter = new Formatter();
+
+        $data = [];
+        foreach ($invalidations as $i) {
+
+            $waiting = (int) Date::now()->getTimestampUTC() - Date::factory($i['ts_invalidated'])->getTimestampUTC();
+            $processing = (int) $i['ts_started'] ? Date::now()->getTimestampUTC() - (int) $i['ts_started'] : '';
+
+            if ($prettyTime) {
+                $waiting = $metricsFormatter->getPrettyTimeFromSeconds($waiting, true);
+                if ($processing != '') {
+                    $processing = $metricsFormatter->getPrettyTimeFromSeconds($processing, true);
+                }
+            }
+
+            $d = [];
+            $d['Invalidation'] = (int) $i['idinvalidation'];
+            $d['Segment'] = $i['definition'];
+            $d['Site'] = (int) $i['idsite'];
+            $d['Period'] = ($i['period'] == 1 ? 'Day' : ($i['period'] == 2 ? 'Week' : ($i['period'] == 3 ? 'Month' :
+                ($i['period'] == 4 ? 'Year' : 'Range'))));
+            $d['Date'] = ($i['period'] == 1 ? $i['date1'] : ($i['period'] == 3 ? substr($i['date1'], 0, 7) :
+                ($i['period'] == 4 ? substr($i['date1'], 0, 4) : $i['date1'] . ' - ' . $i['date2'])));
+            $d['TimeQueued'] = $i['ts_invalidated'];
+            $d['Waiting'] = $waiting;
+            $d['Started'] = $i['ts_started'];
+            $d['Processing'] = $processing;
+            $d['Status'] = ($i['status'] == 1 ? 'Processing' : 'Queued');
+            $data[] = $d;
+        }
+        return $data;
+    }
 }
diff --git a/core/Plugin/ConsoleCommand.php b/core/Plugin/ConsoleCommand.php
@@ -322,6 +322,16 @@ protected function getOutput(): OutputInterface
         return $this->output;
     }
 
+    /**
+     * @param OutputInterface $ouput
+     *
+     * @return void
+     */
+    protected function setOutput(OutputInterface $output): void
+    {
+        $this->output = $output;
+    }
+
     /**
      * @return InputInterface
      */

diff --git a/core/Plugin/ConsoleCommand/ConsoleCommandBufferedOutput.php b/core/Plugin/ConsoleCommand/ConsoleCommandBufferedOutput.php
@@ -0,0 +1,21 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugin\ConsoleCommand;
+
+use Symfony\Component\Console\Output\BufferedOutput;
+
+/**
+ * Wrapper for Symphony buffered output class
+ */
+class ConsoleCommandBufferedOutput extends BufferedOutput
+{
+    //
+}
diff --git a/core/Plugin/ConsoleCommand/ConsoleCommandConsoleOutput.php b/core/Plugin/ConsoleCommand/ConsoleCommandConsoleOutput.php
@@ -0,0 +1,21 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugin\ConsoleCommand;
+
+use Symfony\Component\Console\Output\ConsoleOutput;
+
+/**
+ * Wrapper for Symphony console output class
+ */
+class ConsoleCommandConsoleOutput extends ConsoleOutput
+{
+    //
+}
diff --git a/plugins/Diagnostics/Commands/ArchivingConfig.php b/plugins/Diagnostics/Commands/ArchivingConfig.php
@@ -0,0 +1,112 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\Diagnostics\Commands;
+
+use Piwik\Config;
+use Piwik\Config\DatabaseConfig;
+use Piwik\Config\GeneralConfig;
+use Piwik\Plugin\ConsoleCommand;
+
+/**
+ * Diagnostic command that returns current configuration settings for archiving
+ */
+class ArchivingConfig extends ConsoleCommand
+{
+    protected function configure()
+    {
+        $this->setName('diagnostics:archiving-config');
+        $this->addNoValueOption('json', null,
+            "If supplied, the command will return data in json format");
+        $this->setDescription('Show configuration settings that can affect archiving performance');
+    }
+
+    protected function doExecute(): int
+    {
+        $input = $this->getInput();
+        $output = $this->getOutput();
+
+        $metrics = $this->getArchivingConfig();
+
+        if ($input->getOption('json')) {
+            $output->write(json_encode($metrics));
+        } else {
+            $headers = ['Section', 'Setting', 'Value'];
+            $this->renderTable($headers, $metrics);
+        }
+
+        return self::SUCCESS;
+    }
+
+    /**
+     * Retrieve various data statistics useful for diagnosing archiving performance
+     *
+     * @return array
+     */
+    public function getArchivingConfig(): array
+    {
+        $configs = [
+            'database' => [
+                'enable_segment_first_table_join_prefix',
+                'enable_first_table_join_prefix'
+                ],
+            'general' => [
+                'browser_archiving_disabled_enforce',
+                'enable_processing_unique_visitors_day',
+                'enable_processing_unique_visitors_week',
+                'enable_processing_unique_visitors_month',
+                'enable_processing_unique_visitors_year',
+                'enable_processing_unique_visitors_range',
+                'enable_processing_unique_visitors_multiple_sites',
+                'process_new_segments_from',
+                'time_before_today_archive_considered_outdated',
+                'time_before_week_archive_considered_outdated',
+                'time_before_month_archive_considered_outdated',
+                'time_before_year_archive_considered_outdated',
+                'time_before_range_archive_considered_outdated',
+                'enable_browser_archiving_triggering',
+                'archiving_range_force_on_browser_request',
+                'archiving_custom_ranges[]',
+                'archiving_query_max_execution_time',
+                'archiving_ranking_query_row_limit',
+                'disable_archiving_segment_for_plugins',
+                'disable_archive_actions_goals',
+                'datatable_archiving_maximum_rows_referrers',
+                'datatable_archiving_maximum_rows_subtable_referrers',
+                'datatable_archiving_maximum_rows_userid_users',
+                'datatable_archiving_maximum_rows_custom_dimensions',
+                'datatable_archiving_maximum_rows_subtable_custom_dimensions',
+                'datatable_archiving_maximum_rows_actions',
+                'datatable_archiving_maximum_rows_subtable_actions',
+                'datatable_archiving_maximum_rows_site_search',
+                'datatable_archiving_maximum_rows_events',
+                'datatable_archiving_maximum_rows_subtable_events',
+                'datatable_archiving_maximum_rows_products',
+                'datatable_archiving_maximum_rows_standard'
+                ]
+            ];
+
+        $data = [];
+        foreach ($configs as $section => $sectionConfigs) {
+            foreach ($sectionConfigs as $setting) {
+                switch ($section) {
+                    case 'general':
+                        $value = GeneralConfig::getConfigValue($setting);
+                        break;
+                    case 'database':
+                        $value = DatabaseConfig::getConfigValue($setting);
+                        break;
+                    default:
+                        $value = Config::getInstance()->{$section}[$setting] ?? '';
+                }
+                $data[] = ['Section' => $section, 'Setting' => $setting, 'Value' => $value];
+            }
+        }
+        return $data;
+    }
+}
diff --git a/plugins/Diagnostics/Commands/ArchivingInstanceStatistics.php b/plugins/Diagnostics/Commands/ArchivingInstanceStatistics.php
@@ -0,0 +1,73 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\Diagnostics\Commands;
+
+use Piwik\Common;
+use Piwik\Date;
+use Piwik\Plugin\ConsoleCommand;
+use Piwik\Db;
+
+/**
+ * Diagnostic command that returns instance statistics related to archiving
+ */
+class ArchivingInstanceStatistics extends ConsoleCommand
+{
+    protected function configure()
+    {
+        $this->setName('diagnostics:archiving-instance-statistics');
+        $this->addNoValueOption('json', null,
+            "If supplied, the command will return data in json format");
+        $this->setDescription('Show data statistics which can affect archiving performance');
+    }
+
+    protected function doExecute(): int
+    {
+        $input = $this->getInput();
+        $output = $this->getOutput();
+
+        $metrics = $this->getArchivingInstanceStatistics();
+
+        if ($input->getOption('json')) {
+            $output->write(json_encode($metrics));
+        } else {
+            $headers = ['Statistic Name', 'Value'];
+            $this->renderTable($headers, $metrics);
+        }
+
+        return self::SUCCESS;
+    }
+
+    /**
+     * Retrieve various data statistics useful for diagnosing archiving performance
+     *
+     * @return array
+     */
+    public function getArchivingInstanceStatistics(): array
+    {
+        $stats = [];
+        $stats[] = ['Site Count', (int) Db::fetchOne("SELECT COUNT(*) FROM " . Common::prefixTable("site"))];
+        $stats[] = ['Segment Count', (int) Db::fetchOne("SELECT COUNT(*) FROM " . Common::prefixTable("segment"))];
+        $stats[] = ['Database Version', defined('PIWIK_TEST_MODE') ? 'mysql-version-redacted' : Db::get()->getServerVersion()];
+        $stats[] = ['Last full Month Hits', (int) Db::fetchOne(
+            "SELECT COUNT(*) FROM " . Common::prefixTable("log_link_visit_action") . " WHERE server_time >= ? AND server_time <= ?",
+            [
+                Date::now()->setDay(1)->subMonth(1)->setTime('00:00:00')->toString('Y-m-d H:i:s'),
+                Date::now()->setDay(1)->subDay(1)->setTime('23:59:59')->toString('Y-m-d H:i:s')
+            ])
+        ];
+        $stats[] = ['Last 12 Month Hits', (int) Db::fetchOne(
+            "SELECT COUNT(*) FROM " . Common::prefixTable("log_link_visit_action") . " WHERE server_time >= ? AND server_time <= ?",
+            [
+                Date::now()->setDay(1)->subMonth(12)->setTime('00:00:00')->toString('Y-m-d H:i:s'),
+                Date::now()->setDay(1)->subDay(1)->setTime('23:59:59')->toString('Y-m-d H:i:s')
+            ])
+        ];
+        return $stats;
+    }
+}
diff --git a/plugins/Diagnostics/Commands/ArchivingMetrics.php b/plugins/Diagnostics/Commands/ArchivingMetrics.php
@@ -0,0 +1,68 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\Diagnostics\Commands;
+
+use Piwik\Container\StaticContainer;
+use Piwik\Plugin\ConsoleCommand;
+use Piwik\Plugins\Diagnostics\Diagnostic\ArchiveInvalidationsInformational;
+use Piwik\Translation\Translator;
+
+/**
+ * Diagnostic command that returns archiving invalidation metrics
+ */
+class ArchivingMetrics extends ConsoleCommand
+{
+    protected function configure()
+    {
+        $this->setName('diagnostics:archiving-metrics');
+        $this->addNoValueOption('json', null,
+            "If supplied, the command will return data in json format");
+        $this->setDescription('Show metrics describing the current archiving status');
+    }
+
+    protected function doExecute(): int
+    {
+        $input = $this->getInput();
+        $output = $this->getOutput();
+
+        $metrics = $this->getMetrics();
+
+        if ($input->getOption('json')) {
+            $output->write(json_encode($metrics));
+        } else {
+            $headers = ['Metric', 'Value'];
+            $this->renderTable($headers, $metrics);
+        }
+
+        return self::SUCCESS;
+    }
+
+    /**
+     * Get an archiving metrics array from the diagnostics class
+     *
+     * @return array
+     * @throws \Piwik\Exception\DI\DependencyException
+     * @throws \Piwik\Exception\DI\NotFoundException
+     */
+    public function getMetrics(): array
+    {
+        $metrics = [];
+        $informational = new ArchiveInvalidationsInformational(StaticContainer::get(Translator::class));
+        $diags[] = $informational->execute();
+        if (is_array($diags)) {
+            foreach (reset($diags) as $diag) {
+                $items = $diag->getItems();
+                if (count($items) > 0) {
+                    $metrics[] = [$diag->getLabel(), reset($items)->getComment()];
+                }
+            }
+        }
+        return $metrics;
+    }
+}