Merge pull request #121 from matomo-org/3xdev

Merge 3.x into 4.x
matomo-org · Aug 31, 2020 · fce443a · fce443a
2 parents d8d0cc6 + 3d72d9a
commit fce443a
Show file tree

Hide file tree

Showing 6 changed files with 82 additions and 14 deletions.
diff --git a/Controller.php b/Controller.php
@@ -153,7 +153,7 @@ public function processAuthCode()
             $client = $authorization->getConfiguredClient();
             $authorization->saveAccessToken($oauthCode, $client);
         } catch (\Exception $e) {
-            return $this->index($e->getMessage());
+            return $this->index($this->getNotificationExceptionText($e));
         }
 
         // reload index action to prove everything is configured
@@ -197,7 +197,8 @@ public function configureClient()
             $authorization->validateConfig($config);
             $authorization->saveConfig($config);
         } catch (\Exception $ex) {
-            $errorMessage = $ex->getMessage();
+            $errorMessage = $this->getNotificationExceptionText($ex);
+            $errorMessage = substr($errorMessage, 0, 1024);
         }
 
         Url::redirectToUrl(Url::getCurrentUrlWithoutQueryString() . Url::getCurrentQueryStringWithParametersModified([
@@ -224,7 +225,7 @@ public function deleteImportStatus()
 
             echo json_encode(['result' => 'ok']);
         } catch (\Exception $ex) {
-            $notification = new Notification($ex->getMessage());
+            $notification = new Notification($this->getNotificationExceptionText($ex));
             $notification->type = Notification::TYPE_TRANSIENT;
             $notification->context = Notification::CONTEXT_ERROR;
             $notification->title = Piwik::translate('General_Error');
@@ -259,7 +260,7 @@ public function changeImportEndDate()
 
             echo json_encode(['result' => 'ok']);
         } catch (\Exception $ex) {
-            $notification = new Notification($ex->getMessage());
+            $notification = new Notification($this->getNotificationExceptionText($ex));
             $notification->type = Notification::TYPE_TRANSIENT;
             $notification->context = Notification::CONTEXT_ERROR;
             $notification->title = Piwik::translate('General_Error');
@@ -338,7 +339,7 @@ public function startImport()
 
             echo json_encode([ 'result' => 'ok' ]);
         } catch (\Exception $ex) {
-            $notification = new Notification($ex->getMessage());
+            $notification = new Notification($this->getNotificationExceptionText($ex));
             $notification->type = Notification::TYPE_TRANSIENT;
             $notification->context = Notification::CONTEXT_ERROR;
             $notification->title = Piwik::translate('General_Error');
@@ -373,7 +374,7 @@ public function resumeImport()
 
             echo json_encode([ 'result' => 'ok' ]);
         } catch (\Exception $ex) {
-            $notification = new Notification($ex->getMessage());
+            $notification = new Notification($this->getNotificationExceptionText($ex));
             $notification->type = Notification::TYPE_TRANSIENT;
             $notification->context = Notification::CONTEXT_ERROR;
             $notification->title = Piwik::translate('General_Error');
@@ -415,12 +416,21 @@ public function scheduleReImport()
 
             echo json_encode([ 'result' => 'ok' ]);
         } catch (\Exception $ex) {
-            $notification = new Notification($ex->getMessage());
+            $notification = new Notification($this->getNotificationExceptionText($ex));
             $notification->type = Notification::TYPE_TRANSIENT;
             $notification->context = Notification::CONTEXT_ERROR;
             $notification->title = Piwik::translate('General_Error');
             $notification->hasNoClear();
             Notification\Manager::notify('GoogleAnalyticsImporter_rescheduleImport_failure', $notification);
         }
     }
+
+    private function getNotificationExceptionText(\Exception $e)
+    {
+        $message = $e->getMessage();
+        if (\Piwik_ShouldPrintBackTraceWithMessage()) {
+            $message .= "\n" . $e->getTraceAsString();
+        }
+        return $message;
+    }
 }
diff --git a/Google/GoogleAnalyticsQueryService.php b/Google/GoogleAnalyticsQueryService.php
@@ -16,6 +16,7 @@
 use Piwik\Date;
 use Piwik\Db;
 use Piwik\Metrics;
+use Piwik\Piwik;
 use Piwik\Site;
 use Piwik\Tracker\GoalManager;
 use Psr\Log\LoggerInterface;
@@ -180,6 +181,11 @@ private function gaBatchGet(Date $date, $metricNamesChunk, $options, $orderByMet
                     'message' => $ex->getMessage(),
                 ]);
 
+                /**
+                 * @ignore
+                 */
+                Piwik::postEvent('GoogleAnalyticsImporter.onApiError', [$ex]);
+
                 if ($ex->getCode() == 403 || $ex->getCode() == 429) {
                     if (strpos($ex->getMessage(), 'daily') !== false) {
                         throw new DailyRateLimitReached();
@@ -189,11 +195,17 @@ private function gaBatchGet(Date $date, $metricNamesChunk, $options, $orderByMet
 
                     $this->logger->debug("Waiting {$this->currentBackoffTime}s before trying again...");
 
+                    $this->backOff();
+                } else if ($this->isIgnorableException($ex)) {
+                    ++$attempts;
+
+                    $this->logger->info("Google Analytics API returned an ignorable or temporary error: {$ex->getMessage()}. Waiting {$this->currentBackoffTime}s before trying again...");
+
                     $this->backOff();
                 } else if ($ex->getCode() >= 500) {
                     ++$attempts;
 
-                    $this->logger->info("Google Analytics API returned error: {$ex->getMessage()}. Waiting one minute before trying again...");
+                    $this->logger->info("Google Analytics API returned error: {$ex->getMessage()}. Waiting {$this->currentBackoffTime}s before trying again...");
 
                     $messageContent = @json_decode($ex->getMessage(), true);
                     if (isset($messageContent['error']['message'])) {
@@ -256,4 +268,22 @@ private function backOff()
         $this->sleep($this->currentBackoffTime);
         $this->currentBackoffTime = min(self::MAX_BACKOFF_TIME, $this->currentBackoffTime * 2);
     }
+
+    private function isIgnorableException(\Exception $ex)
+    {
+        if ($ex->getCode() !== 400) {
+            return false;
+        }
+
+        $messageContent = @json_decode($ex->getMessage(), true);
+        if (empty($messageContent['error']['message'])) {
+            return false;
+        }
+
+        if (strpos($messageContent['error']['message'], 'Unknown metric') === 0) {
+            return true;
+        }
+
+        return false;
+    }
 }
diff --git a/Google/GoogleCustomDimensionMapper.php b/Google/GoogleCustomDimensionMapper.php
@@ -22,7 +22,7 @@ public function map(\Google_Service_Analytics_CustomDimension $gaCustomDimension
             'extractions' => [],
             'case_sensitive' => true,
             'scope' => $this->mapScope($gaCustomDimension),
-            'active' => $gaCustomDimension->getActive(),
+            'active' => (bool) $gaCustomDimension->getActive(),
         ];
 
         $blockedChars = Name::getBlockedCharacters();

diff --git a/Input/EndDate.php b/Input/EndDate.php
@@ -12,7 +12,6 @@
 use Piwik\Config;
 use Piwik\Date;
 use Piwik\SettingsServer;
-use Piwik\Site;
 
 class EndDate
 {
@@ -50,22 +49,36 @@ public function getMaxEndDate()
                 $installDate = get_option(\WpMatomo\Installer::OPTION_NAME_INSTALL_DATE);
             }
 
-            if (empty($installDate)) {
+            try {
+                $installDate = Date::factory($installDate);
+            } catch (\Exception $ex) {
+                // ignore
+            }
+
+            if (empty($installDate)
+                || !($installDate instanceof Date)
+            ) {
                 // matomo for WordPress was installed before this option was set
                 // we have to make sure there will be an end date otherwise it will always overwrite data
                 // we assume it was installed 2 days ago. It's not 100% accurate but best we can do
                 $installDate = Date::today()->subDay(2);
             } else {
                 // import up to 1 day before original install
-                $installDate = Date::factory($installDate)->subDay(1);
+                $installDate = $installDate->subDay(1);
             }
 
             return $installDate;
         }
 
         if ($this->forceMaxEndDate) {
-            return Date::factory($this->forceMaxEndDate);
+            try {
+                return Date::factory($this->forceMaxEndDate);
+            } catch (\Exception $ex) {
+                return null;
+            }
         }
+
+        return null;
     }
 
     public function limitMaxEndDateIfNeeded($endDate)

diff --git a/tests/Integration/Google/GoogleAnalyticsQueryServiceTest.php b/tests/Integration/Google/GoogleAnalyticsQueryServiceTest.php
@@ -34,6 +34,12 @@ public function getTestDataForGaErrorTest()
                 ],
             ]), 503), 'Failed to reach GA after 2 attempts. Restart the import later.'],
             [new \Exception('lakjdsflsdj', 503), 'Failed to reach GA after 2 attempts. Restart the import later.'],
+
+            [new \Exception(json_encode([
+                'error' => [
+                    'message' => 'Unknown metric(s): blah, blah and blah',
+                ],
+            ]), 400), 'Failed to reach GA after 2 attempts. Restart the import later. Last GA error message: this is a test exception'],
         ];
     }
 
@@ -50,7 +56,7 @@ public function test_query_returnsGaMessageWhenGaReturnsPersistentError($testEx,
         $builder = $this->getMockBuilder(\Google_Service_AnalyticsReporting_Resource_Reports::class);
         $mockReportingService->reports = $builder
             ->disableOriginalConstructor()
-            ->onlyMethods([ 'batchGet' ])
+            ->onlyMethods(['batchGet'])
             ->getMock();
         $mockReportingService->reports->method('batchGet')->willThrowException($testEx);
 

diff --git a/tests/Unit/Input/EndDateTest.php b/tests/Unit/Input/EndDateTest.php
@@ -35,6 +35,15 @@ public function test_getMaxEndDate_withConfigSectionWithValue()
         $this->assertEquals(Date::factory('today'), $endDate->getMaxEndDate());
     }
 
+    public function test_getMaxEndDate_withConfigSectionWithInvalidValue()
+    {
+        $mockConfig = $this->getMockConfig([
+            EndDate::CONFIG_NAME => 'tasdlfjsadf',
+        ]);
+        $endDate = new EndDate($mockConfig);
+        $this->assertNull($endDate->getMaxEndDate());
+    }
+
     public function test_getMaxEndDate_withConfigSectionWithValue_thatIsExplicitDate()
     {
         $mockConfig = $this->getMockConfig([