Update live for upcoming 5.1.4 release

CHANGELOG.md

@@ -1,5 +1,9 @@
 == Changelog ===
 
+= 5.1.4 =
+* Update Matomo core to version 5.1.2 (changes: https://matomo.org/changelog/matomo-5-1-2/)
+* Update some Matomo core dependencies.
+
 = 5.1.3 =
 * Update Matomo core to version 5.1.1 (changes: https://matomo.org/changelog/matomo-5-1-1/)
 * Moderate redesign to the feedback and get started admin pages.

app/config/global.ini.php

@@ -45,6 +45,14 @@
 ; Matomo should work correctly without this setting but we recommend to have a charset set.
 charset = utf8
 
+; In some database setups the collation used for queries and creating tables can have unexpected
+; values, or change after a database version upgrade.
+; If you encounter "Illegal mix of collation" errors, setting this config to the value matching
+; your existing database tables can help.
+; This setting will only be used if "charset" is also set.
+; Matomo should work correctly without this setting but we recommend to have a collation set.
+collation =
+
 ; Database error codes to ignore during updates
 ;
 ;ignore_error_codes[] = 1105
@@ -84,6 +92,7 @@
 type = InnoDB
 schema = Mysql
 charset = utf8mb4
+collation = utf8mb4_general_ci
 enable_ssl = 0
 ssl_ca =
 ssl_cert =

app/core/Db.php

@@ -160,6 +160,7 @@ public static function createReaderDatabaseObject($dbConfig = null)
         $dbConfig['type'] = $masterDbConfig['type'];
         $dbConfig['tables_prefix'] = $masterDbConfig['tables_prefix'];
         $dbConfig['charset'] = $masterDbConfig['charset'];
+        $dbConfig['collation'] = $masterDbConfig['collation'] ?? null;
         $db = @Adapter::factory($dbConfig['adapter'], $dbConfig);
         if (!empty($dbConfig['aurora_readonly_read_committed'])) {
             $db->exec('set session aurora_read_replica_read_committed = ON;set session transaction isolation level read committed;');

app/core/Db/Schema.php

@@ -75,6 +75,25 @@ private function getSchema() : \Piwik\Db\SchemaInterface
         }
         return $this->schema;
     }
+    /**
+     * Returns the default collation for a charset.
+     *
+     * @param string $charset
+     * @return string
+     */
+    public function getDefaultCollationForCharset(string $charset) : string
+    {
+        return $this->getSchema()->getDefaultCollationForCharset($charset);
+    }
+    /**
+     * Get the table options to use for a CREATE TABLE statement.
+     *
+     * @return string
+     */
+    public function getTableCreateOptions() : string
+    {
+        return $this->getSchema()->getTableCreateOptions();
+    }
     /**
      * Get the SQL to create a specific Piwik table
      *

app/core/Db/Schema/Tidb.php

@@ -23,8 +23,35 @@ public function supportsComplexColumnUpdates() : bool
     {
         return false;
     }
+    public function getDefaultCollationForCharset(string $charset) : string
+    {
+        $collation = parent::getDefaultCollationForCharset($charset);
+        if ('utf8mb4' === $charset && 'utf8mb4_bin' === $collation) {
+            // replace the TiDB default "utf8mb4_bin" with a better default
+            return 'utf8mb4_0900_ai_ci';
+        }
+        return $collation;
+    }
     public function getDefaultPort() : int
     {
         return 4000;
     }
+    public function getTableCreateOptions() : string
+    {
+        $engine = $this->getTableEngine();
+        $charset = $this->getUsedCharset();
+        $collation = $this->getUsedCollation();
+        $rowFormat = $this->getTableRowFormat();
+        if ('utf8mb4' === $charset && '' === $collation) {
+            $collation = 'utf8mb4_0900_ai_ci';
+        }
+        $options = "ENGINE={$engine} DEFAULT CHARSET={$charset}";
+        if ('' !== $collation) {
+            $options .= " COLLATE={$collation}";
+        }
+        if ('' !== $rowFormat) {
+            $options .= " {$rowFormat}";
+        }
+        return $options;
+    }
 }

app/core/Db/SchemaInterface.php

@@ -108,10 +108,24 @@ public function addMaxExecutionTimeHintToQuery(string $sql, float $limit) : stri
      * @return bool
      */
     public function supportsComplexColumnUpdates() : bool;
+    /**
+     * Returns the default collation for a charset used by this database engine.
+     *
+     * @param string $charset
+     *
+     * @return string
+     */
+    public function getDefaultCollationForCharset(string $charset) : string;
     /**
      * Return the default port used by this database engine
      *
      * @return int
      */
     public function getDefaultPort() : int;
+    /**
+     * Return the table options to use for a CREATE TABLE statement.
+     *
+     * @return string
+     */
+    public function getTableCreateOptions() : string;
 }

app/core/Db/Settings.php

@@ -27,6 +27,10 @@ public function getUsedCharset()
     {
         return strtolower($this->getDbSetting('charset'));
     }
+    public function getUsedCollation()
+    {
+        return strtolower($this->getDbSetting('collation') ?? '');
+    }
     public function getRowFormat()
     {
         return $this->getUsedCharset() === 'utf8mb4' ? 'ROW_FORMAT=DYNAMIC' : '';

app/core/DbHelper.php

@@ -192,7 +192,7 @@ public static function tableHasIndex($table, $indexName)
      * @return string
      * @throws Tracker\Db\DbException
      */
-    public static function getDefaultCharset()
+    public static function getDefaultCharset() : string
     {
         $result = \Piwik\Db::get()->fetchRow("SHOW CHARACTER SET LIKE 'utf8mb4'");
         if (empty($result)) {
@@ -211,6 +211,18 @@ public static function getDefaultCharset()
         }
         return 'utf8mb4';
     }
+    /**
+     * Returns the default collation for a charset.
+     *
+     * @param string $charset
+     *
+     * @return string
+     * @throws Exception
+     */
+    public static function getDefaultCollationForCharset(string $charset) : string
+    {
+        return Schema::getInstance()->getDefaultCollationForCharset($charset);
+    }
     /**
      * Returns sql queries to convert all installed tables to utf8mb4
      *

app/core/Period/Range.php

@@ -218,6 +218,10 @@ protected function generate()
                 $timezone = $this->timezone;
             }
             $endDate = Date::factory($strDateEnd, $timezone)->setTime("00:00:00");
+            $maxAllowedEndDate = Date::factory(self::getMaxAllowedEndTimestamp());
+            if ($endDate->isLater($maxAllowedEndDate)) {
+                $endDate = $maxAllowedEndDate;
+            }
         } else {
             throw new Exception($this->translator->translate('General_ExceptionInvalidDateRange', array($this->strDate, ' \'lastN\', \'previousN\', \'YYYY-MM-DD,YYYY-MM-DD\'')));
         }
@@ -463,4 +467,16 @@ public function getParentPeriodLabel()
     {
         return null;
     }
+    /**
+     * Returns the max allowed end timestamp for a range. If an enddate after this timestamp is provided, Matomo will
+     * automatically lower the end date to the date returned by this method.
+     * The max supported timestamp is always set to end of the current year plus 10 years.
+     *
+     * @return int
+     * @api
+     */
+    public static function getMaxAllowedEndTimestamp() : int
+    {
+        return strtotime(date('Y-12-31 00:00:00', strtotime('+10 year', Date::getNowTimestamp())));
+    }
 }

app/core/Tracker/Db/Mysqli.php

@@ -24,6 +24,7 @@ class Mysqli extends Db
     protected $username;
     protected $password;
     protected $charset;
+    protected $collation;
     protected $activeTransaction = false;
     protected $enable_ssl;
     protected $ssl_key;
@@ -56,7 +57,8 @@ public function __construct($dbInfo, $driverName = 'mysql')
         $this->dbname = $dbInfo['dbname'];
         $this->username = $dbInfo['username'];
         $this->password = $dbInfo['password'];
-        $this->charset = isset($dbInfo['charset']) ? $dbInfo['charset'] : null;
+        $this->charset = $dbInfo['charset'] ?? null;
+        $this->collation = $dbInfo['collation'] ?? null;
         if (!empty($dbInfo['enable_ssl'])) {
             $this->enable_ssl = $dbInfo['enable_ssl'];
         }
@@ -119,8 +121,16 @@ public function connect()
         if (!$this->connection || mysqli_connect_errno()) {
             throw new \Piwik\Tracker\Db\DbException("Connect failed: " . mysqli_connect_error());
         }
-        if ($this->charset && !mysqli_set_charset($this->connection, $this->charset)) {
-            throw new \Piwik\Tracker\Db\DbException("Set Charset failed: " . mysqli_error($this->connection));
+        if ($this->charset && $this->collation) {
+            // mysqli_set_charset does not support setting a collation
+            $query = "SET NAMES '" . $this->charset . "' COLLATE '" . $this->collation . "'";
+            if (!mysqli_query($this->connection, $query)) {
+                throw new \Piwik\Tracker\Db\DbException("Set charset/connection collation failed: " . mysqli_error($this->connection));
+            }
+        } elseif ($this->charset) {
+            if (!mysqli_set_charset($this->connection, $this->charset)) {
+                throw new \Piwik\Tracker\Db\DbException("Set Charset failed: " . mysqli_error($this->connection));
+            }
         }
         $this->password = '';
         if (self::$profiling && isset($timer)) {