From ab958376f97ddc24535610f34c086476a9d12b7e Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Sat, 14 Apr 2018 10:54:06 +1200 Subject: [PATCH 1/3] Send email notification when queue reaches a certain size --- CHANGELOG.md | 1 + Configuration.php | 93 ++++++++++++++++++++++++++++++++ Queue/Backend/MySQL.php | 5 ++ QueuedTracking.php | 12 +++++ Tasks.php | 60 +++++++++++++++++++++ Updates/3.2.0.php | 4 ++ docs/faq.md | 11 ++++ tests/Unit/ConfigurationTest.php | 83 ++++++++++++++++++++++++++++ 8 files changed, 269 insertions(+) create mode 100644 Configuration.php create mode 100644 tests/Unit/ConfigurationTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 677bf98..c320cb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Added possibility to use a MySQL backend instead of redis - New option `queue-id` for the `queuedtracking:process` command which may improve processing speed as the command would only focus on one queue instead of trying to get the lock for a random queue. - Various other minor performance improvements +- New feature: Get notified by email when a single queue reaches a specific threshold 3.0.2 diff --git a/Configuration.php b/Configuration.php new file mode 100644 index 0000000..5d1d937 --- /dev/null +++ b/Configuration.php @@ -0,0 +1,93 @@ +getConfig(); + + if (empty($config->QueuedTracking)) { + $config->QueuedTracking = array(); + } + $reports = $config->QueuedTracking; + + // we make sure to set a value only if none has been configured yet, eg in common config. + if (empty($reports[self::KEY_NOTIFY_THRESHOLD])) { + $reports[self::KEY_NOTIFY_THRESHOLD] = self::DEFAULT_NOTIFY_THRESHOLD; + } + if (empty($reports[self::KEY_NOTIFY_EMAILS])) { + $reports[self::KEY_NOTIFY_EMAILS] = self::DEFAULT_NOTIFY_EMAILS; + } + $config->QueuedTracking = $reports; + + $config->forceSave(); + } + + public function uninstall() + { + $config = $this->getConfig(); + $config->QueuedTracking = array(); + $config->forceSave(); + } + + /** + * @return int + */ + public function getNotifyThreshold() + { + $value = $this->getConfigValue(self::KEY_NOTIFY_THRESHOLD, self::DEFAULT_NOTIFY_THRESHOLD); + + if ($value === false || $value === '' || $value === null) { + $value = self::DEFAULT_NOTIFY_THRESHOLD; + } + + return (int) $value; + } + + /** + * @return array + */ + public function getNotifyEmails() + { + $value = $this->getConfigValue(self::KEY_NOTIFY_EMAILS, self::DEFAULT_NOTIFY_EMAILS); + + if (empty($value)) { + $value = self::DEFAULT_NOTIFY_EMAILS; + } + if (!is_array($value)) { + $value = array($value); + } + + return $value; + } + + private function getConfig() + { + return Config::getInstance(); + } + + private function getConfigValue($name, $default) + { + $config = $this->getConfig(); + $attribution = $config->QueuedTracking; + if (isset($attribution[$name])) { + return $attribution[$name]; + } + return $default; + } +} diff --git a/Queue/Backend/MySQL.php b/Queue/Backend/MySQL.php index 73e5e9d..03d4900 100644 --- a/Queue/Backend/MySQL.php +++ b/Queue/Backend/MySQL.php @@ -36,6 +36,11 @@ public function install() UNIQUE unique_queue_key (`queue_key`)"); } + public function uninstall() + { + Db::query(sprintf('DROP TABLE IF EXISTS `%s`', $this->tablePrefixed)); + } + private function makePrefixedKeyListTableName($key) { return Common::prefixTable($this->tableListPrefix . $key); diff --git a/QueuedTracking.php b/QueuedTracking.php index af93e0b..925d729 100644 --- a/QueuedTracking.php +++ b/QueuedTracking.php @@ -27,6 +27,18 @@ public function install() { $mysql = new MySQL(); $mysql->install(); + + $configuration = new Configuration(); + $configuration->install(); + } + + public function uninstall() + { + $mysql = new MySQL(); + $mysql->uninstall(); + + $configuration = new Configuration(); + $configuration->uninstall(); } public function isTrackerPlugin() diff --git a/Tasks.php b/Tasks.php index e25fa63..7bcbe3b 100644 --- a/Tasks.php +++ b/Tasks.php @@ -9,14 +9,74 @@ namespace Piwik\Plugins\QueuedTracking; use Piwik\Common; +use Piwik\Config; use Piwik\Db; +use Piwik\Mail; use Piwik\Plugins\QueuedTracking\Queue\Backend\MySQL; +use Piwik\SettingsPiwik; class Tasks extends \Piwik\Plugin\Tasks { + /** + * @var Configuration + */ + private $config; + + public function __construct(Configuration $configuration) + { + $this->config = $configuration; + } + public function schedule() { $this->daily('optimizeQueueTable', null, self::LOWEST_PRIORITY); + $this->hourly('notifyQueueSize', null, self::LOWEST_PRIORITY); + } + + /** + * run eg using ./console core:run-scheduled-tasks "Piwik\Plugins\QueuedTracking\Tasks.notifyQueueSize" + */ + public function notifyQueueSize() + { + $settings = Queue\Factory::getSettings(); + + if (!$settings->queueEnabled->getValue()) { + // not needed to check anything + return; + } + + $emailsToNotify = $this->config->getNotifyEmails(); + $threshold = $this->config->getNotifyThreshold(); + + if (empty($emailsToNotify) || empty($threshold) || $threshold <= 0) { + // nobody to notify or no threshold defined + return; + } + + $backend = Queue\Factory::makeBackend(); + $queueManager = Queue\Factory::makeQueueManager($backend); + + $sendEmail = false; + $message = ""; + + foreach ($queueManager->getAllQueues() as $queue) { + $size = $queue->getNumberOfRequestSetsInQueue(); + $message .= sprintf("Queue ID %s has %s entries.
", $queue->getId(), $size); + if ($size >= $threshold) { + $sendEmail = true; + } + } + + if ($sendEmail) { + $message = sprintf("This is a notification that the threshold %s for a single queue has been reached.
The current queue sizes are as follows:

%s", $threshold, $message); + $message = $message . "

Sent from " . SettingsPiwik::getPiwikUrl(); + $mail = new Mail(); + $mail->setDefaultFromPiwik(); + $mail->addTo($emailsToNotify); + $mail->setSubject('Queued Tracking - queue size has reached your threshold'); + $mail->setBodyHtml($message); + $mail->send(); + } } /** diff --git a/Updates/3.2.0.php b/Updates/3.2.0.php index 52f64f7..6438428 100644 --- a/Updates/3.2.0.php +++ b/Updates/3.2.0.php @@ -9,6 +9,7 @@ namespace Piwik\Plugins\QueuedTracking; +use Piwik\Config; use Piwik\Updater; use Piwik\Updates as PiwikUpdates; use Piwik\Updater\Migration\Factory as MigrationFactory; @@ -42,5 +43,8 @@ public function getMigrations(Updater $updater) public function doUpdate(Updater $updater) { $updater->executeMigrations(__FILE__, $this->getMigrations($updater)); + + $config = new Configuration(); + $config->install(); } } diff --git a/docs/faq.md b/docs/faq.md index 17fb93c..87b222d 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -127,6 +127,17 @@ __Can I configure multiple Sentinel servers?__ Yes, once Sentinel is enabled you can configure multiple servers by specifying multiple hosts and ports comma separated via the UI. +__Can I be notified when a queue reaches a certain threshold?__ + +Yes, you can optionally receive an email when the number of requests queued in a single queue reaches a configured +threshold. You can configure this in your `config/config.ini.php` config file using the following configuration: + +``` +[QueuedTracking] +notify_queue_threshold_emails[] = example@example.org +notify_queue_threshold_single_queue = 250000 +``` + __Are there any known issues?__ * In case you are using bulk tracking the bulk tracking response varies compared to the regular one. We will always return diff --git a/tests/Unit/ConfigurationTest.php b/tests/Unit/ConfigurationTest.php new file mode 100644 index 0000000..4cfefac --- /dev/null +++ b/tests/Unit/ConfigurationTest.php @@ -0,0 +1,83 @@ +configuration = new Configuration(); + } + + public function test_shouldInstallConfig() + { + $this->configuration->install(); + + $QueuedTracking = Config::getInstance()->QueuedTracking; + $this->assertEquals(array( + 'notify_queue_threshold_single_queue' => Configuration::DEFAULT_NOTIFY_THRESHOLD, + 'notify_queue_threshold_emails' => Configuration::DEFAULT_NOTIFY_EMAILS + ), $QueuedTracking); + } + + public function test_getNotifyThreshold_shouldReturnDefaultThreshold() + { + $this->assertEquals(250000, $this->configuration->getNotifyThreshold()); + } + + public function test_getNotifyThreshold_shouldBePossibleToChangeValue() + { + Config::getInstance()->QueuedTracking = array( + Configuration::KEY_NOTIFY_THRESHOLD => 150 + ); + $this->assertEquals(150, $this->configuration->getNotifyThreshold()); + } + + public function test_getNotifyThreshold_noConfig_shouldReturnDefault() + { + Config::getInstance()->QueuedTracking = array(); + $this->assertEquals(Configuration::DEFAULT_NOTIFY_THRESHOLD, $this->configuration->getNotifyThreshold()); + } + + + public function test_getNotifyEmails_shouldReturnDefaultThreshold() + { + $this->assertEquals(array(), $this->configuration->getNotifyEmails()); + } + + public function test_getNotifyEmails_shouldBePossibleToChangeValue() + { + Config::getInstance()->QueuedTracking = array( + Configuration::KEY_NOTIFY_EMAILS => ['test@matomo.org'] + ); + $this->assertEquals(['test@matomo.org'], $this->configuration->getNotifyEmails()); + } + + public function test_getNotifyEmails_noConfig_shouldReturnDefault() + { + Config::getInstance()->QueuedTracking = array(); + $this->assertEquals(Configuration::DEFAULT_NOTIFY_EMAILS, $this->configuration->getNotifyEmails()); + } + + +} From dbfb9a8bc9f3edd7495a10205492d268bf0af795 Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Sun, 15 Apr 2018 19:07:58 +1200 Subject: [PATCH 2/3] fix tests --- Configuration.php | 8 ++++---- tests/Unit/ConfigurationTest.php | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Configuration.php b/Configuration.php index 5d1d937..7c16dba 100644 --- a/Configuration.php +++ b/Configuration.php @@ -12,7 +12,7 @@ class Configuration { - const DEFAULT_NOTIFY_EMAILS = []; + public static $DEFAULT_NOTIFY_EMAILS = []; const DEFAULT_NOTIFY_THRESHOLD = 250000; const KEY_NOTIFY_EMAILS = 'notify_queue_threshold_emails'; const KEY_NOTIFY_THRESHOLD = 'notify_queue_threshold_single_queue'; @@ -31,7 +31,7 @@ public function install() $reports[self::KEY_NOTIFY_THRESHOLD] = self::DEFAULT_NOTIFY_THRESHOLD; } if (empty($reports[self::KEY_NOTIFY_EMAILS])) { - $reports[self::KEY_NOTIFY_EMAILS] = self::DEFAULT_NOTIFY_EMAILS; + $reports[self::KEY_NOTIFY_EMAILS] = self::$DEFAULT_NOTIFY_EMAILS; } $config->QueuedTracking = $reports; @@ -64,10 +64,10 @@ public function getNotifyThreshold() */ public function getNotifyEmails() { - $value = $this->getConfigValue(self::KEY_NOTIFY_EMAILS, self::DEFAULT_NOTIFY_EMAILS); + $value = $this->getConfigValue(self::KEY_NOTIFY_EMAILS, self::$DEFAULT_NOTIFY_EMAILS); if (empty($value)) { - $value = self::DEFAULT_NOTIFY_EMAILS; + $value = self::$DEFAULT_NOTIFY_EMAILS; } if (!is_array($value)) { $value = array($value); diff --git a/tests/Unit/ConfigurationTest.php b/tests/Unit/ConfigurationTest.php index 4cfefac..73ff2ea 100644 --- a/tests/Unit/ConfigurationTest.php +++ b/tests/Unit/ConfigurationTest.php @@ -36,7 +36,7 @@ public function test_shouldInstallConfig() $QueuedTracking = Config::getInstance()->QueuedTracking; $this->assertEquals(array( 'notify_queue_threshold_single_queue' => Configuration::DEFAULT_NOTIFY_THRESHOLD, - 'notify_queue_threshold_emails' => Configuration::DEFAULT_NOTIFY_EMAILS + 'notify_queue_threshold_emails' => Configuration::$DEFAULT_NOTIFY_EMAILS ), $QueuedTracking); } @@ -76,7 +76,7 @@ public function test_getNotifyEmails_shouldBePossibleToChangeValue() public function test_getNotifyEmails_noConfig_shouldReturnDefault() { Config::getInstance()->QueuedTracking = array(); - $this->assertEquals(Configuration::DEFAULT_NOTIFY_EMAILS, $this->configuration->getNotifyEmails()); + $this->assertEquals(Configuration::$DEFAULT_NOTIFY_EMAILS, $this->configuration->getNotifyEmails()); } From cc47f4bb591effe13ad08aed48f72d8d24cc8b32 Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Mon, 16 Apr 2018 07:53:41 +1200 Subject: [PATCH 3/3] improve message --- Tasks.php | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Tasks.php b/Tasks.php index 7bcbe3b..aefe2a3 100644 --- a/Tasks.php +++ b/Tasks.php @@ -56,19 +56,26 @@ public function notifyQueueSize() $backend = Queue\Factory::makeBackend(); $queueManager = Queue\Factory::makeQueueManager($backend); - $sendEmail = false; - $message = ""; + $larger = ""; + $smaller = ""; foreach ($queueManager->getAllQueues() as $queue) { $size = $queue->getNumberOfRequestSetsInQueue(); - $message .= sprintf("Queue ID %s has %s entries.
", $queue->getId(), $size); + $entriesMessage = sprintf("Queue ID %s has %s entries.
", $queue->getId(), $size); if ($size >= $threshold) { - $sendEmail = true; + $larger .= $entriesMessage; + } else { + $smaller .= $entriesMessage; } } - if ($sendEmail) { - $message = sprintf("This is a notification that the threshold %s for a single queue has been reached.
The current queue sizes are as follows:

%s", $threshold, $message); + if (!empty($larger)) { + $message = sprintf("This is a notification that the threshold %s for a single queue has been reached.

The following queue sizes are greater than the threshold:
%s", $threshold, $larger); + + if (!empty($smaller)) { + $message .= sprintf("

The remaining queue sizes, which are below the threshold, are listed below:
%s", $smaller); + } + $message = $message . "

Sent from " . SettingsPiwik::getPiwikUrl(); $mail = new Mail(); $mail->setDefaultFromPiwik();