From 471668b9406957d1cc9ba5a3af907fa53fd188de Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Mon, 30 Sep 2024 23:11:31 +0200 Subject: [PATCH] FIX Toolt to convert utf8mb4 --- htdocs/admin/system/database-tables.php | 18 +++++++--- htdocs/admin/system/database.php | 48 +++++++++++++++++++++++-- htdocs/admin/system/dbtable.php | 26 +++++++++++--- htdocs/core/db/mysqli.class.php | 5 +-- htdocs/install/repair.php | 16 +++++++-- 5 files changed, 98 insertions(+), 15 deletions(-) diff --git a/htdocs/admin/system/database-tables.php b/htdocs/admin/system/database-tables.php index fa7a86480baac..7ea4ba8336bcc 100644 --- a/htdocs/admin/system/database-tables.php +++ b/htdocs/admin/system/database-tables.php @@ -52,12 +52,17 @@ $db->query($sql); } if ($action == 'convertutf8') { - $sql = "ALTER TABLE ".$db->sanitize($table)." CHARACTER SET utf8 COLLATE utf8_unicode_ci"; // Set the default value on table + $collation = 'utf8_unicode_ci'; + $defaultcollation = $db->getDefaultCollationDatabase(); + if (preg_match('/general/', $defaultcollation)) { + $collation = 'utf8_general_ci'; + } + $sql = "ALTER TABLE ".$db->sanitize($table)." CHARACTER SET utf8 COLLATE ".$db->sanitize($collation); // Set the default value on table $resql1 = $db->query($sql); if (!$resql1) { setEventMessages($db->lasterror(), null, 'warnings'); } else { - $sql = "ALTER TABLE ".$db->sanitize($table)." CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci"; // Switch fields (may fails due to foreign key) + $sql = "ALTER TABLE ".$db->sanitize($table)." CONVERT TO CHARACTER SET utf8 COLLATE ".$db->sanitize($collation); // Switch fields (may fails due to foreign key) $resql2 = $db->query($sql); if (!$resql2) { setEventMessages($db->lasterror(), null, 'warnings'); @@ -65,12 +70,17 @@ } } if ($action == 'convertutf8mb4') { - $sql = "ALTER TABLE ".$db->sanitize($table)." CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"; // Set the default value on table + $collation = 'utf8mb4_unicode_ci'; + $defaultcollation = $db->getDefaultCollationDatabase(); + if (preg_match('/general/', $defaultcollation)) { + $collation = 'utf8mb4_general_ci'; + } + $sql = "ALTER TABLE ".$db->sanitize($table)." CHARACTER SET utf8mb4 COLLATE ".$db->sanitize($collation); // Set the default value on table $resql1 = $db->query($sql); if (!$resql1) { setEventMessages($db->lasterror(), null, 'warnings'); } else { - $sql = "ALTER TABLE ".$db->sanitize($table)." CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"; // Switch fields (may fails due to foreign key) + $sql = "ALTER TABLE ".$db->sanitize($table)." CONVERT TO CHARACTER SET utf8mb4 COLLATE ".$db->sanitize($collation); // Switch fields (may fails due to foreign key) $resql2 = $db->query($sql); if (!$resql2) { setEventMessages($db->lasterror(), null, 'warnings'); diff --git a/htdocs/admin/system/database.php b/htdocs/admin/system/database.php index 1c9bb6b2f93af..7d2990f77bebe 100644 --- a/htdocs/admin/system/database.php +++ b/htdocs/admin/system/database.php @@ -29,11 +29,34 @@ $langs->load("admin"); +$action = GETPOST('action', 'aZ09'); + if (!$user->admin) { accessforbidden(); } +/* + * Actions + */ + +if ($action == 'convertutf8unicode') { // Test on permission already done. + $sql = "ALTER DATABASE ".$db->sanitize($table[0])." CHARACTER SET utf8 COLLATE utf8_unicode_ci"; + $db->query($sql); +} +if ($action == 'convertutf8mb4unicode') { // Test on permission already done. + $sql = "ALTER DATABASE ".$db->sanitize($table[0])." CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"; + $db->query($sql); +} +if ($action == 'convertutf8general') { // Test on permission already done. + $sql = "ALTER DATABASE ".$db->sanitize($table[0])." CHARACTER SET utf8 COLLATE utf8_general_ci"; + $db->query($sql); +} +if ($action == 'convertutf8mb4general') { // Test on permission already done. + $sql = "ALTER DATABASE ".$db->sanitize($table[0])." CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"; + $db->query($sql); +} + /* * View @@ -59,14 +82,33 @@ print ''.$langs->trans("Password").''.preg_replace('/./i', '*', $dolibarr_main_db_pass).''."\n"; print ''.$langs->trans("DBStoringCharset").''.$db->getDefaultCharacterSetDatabase(); if ($db->type == 'mysqli') { - print ' '.$form->textwithpicto('', $langs->transnoentitiesnoconv("HelpMariaDBToGetValue", "SHOW VARIABLES LIKE 'character_set_database'").'
'.$langs->transnoentitiesnoconv("HelpMariaDBToGetPossibleValues", "SHOW CHARSET")); + print ' '.$form->textwithpicto('', $langs->transnoentitiesnoconv("HelpMariaDBToGetValue", "
SHOW VARIABLES LIKE 'character_set_database' (cached)
You can avoid cache effect with:
SELECT DEFAULT_CHARACTER_SET_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = '".$conf->db->name."'").'
'.$langs->transnoentitiesnoconv("HelpMariaDBToGetPossibleValues", "
SHOW CHARSET")); // We can use $db->getDefaultCharacterSetDatabase(), $db->getListOfCharacterSet(), } print ''."\n"; -print ''.$langs->trans("DBSortingCharset").''.$db->getDefaultCollationDatabase(); +print ''.$langs->trans("DBSortingCharset").''; +$defaultcollation = $db->getDefaultCollationDatabase(); +print dolPrintHTML($defaultcollation); if ($db->type == 'mysqli') { - print ' '.$form->textwithpicto('', $langs->transnoentitiesnoconv("HelpMariaDBToGetValue", "SHOW VARIABLES LIKE 'collation_database'").'
'.$langs->transnoentitiesnoconv("HelpMariaDBToGetPossibleValues", "SHOW COLLATION")); + if ($defaultcollation != $dolibarr_main_db_collation) { + print img_warning('The database default value of collation '.$defaultcollation.' differs from conf setup '.$dolibarr_main_db_collation); + } + print ' '.$form->textwithpicto('', $langs->transnoentitiesnoconv("HelpMariaDBToGetValue", "
SHOW VARIABLES LIKE 'collation_database' (cached)
You can avoid cache effect with:
SELECT DEFAULT_COLLATION_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = '".$conf->db->name."'").'
'.$langs->transnoentitiesnoconv("HelpMariaDBToGetPossibleValues", "
SHOW COLLATION")); // We can use $db->getDefaultCollationDatabase(), $db->getListOfCollation(); + + print '       '.$langs->trans("ConvertInto"); + if (!in_array($defaultcollation, array("utf8_unicode_ci"))) { + print '   utf8 unicode'; + } + if (!in_array($defaultcollation, array("utf8_general_ci"))) { + print '   utf8 general'; + } + if (!in_array($defaultcollation, array("utf8mb4_unicode_ci"))) { + print '   utf8mb4 unicode'; + } + if (!in_array($defaultcollation, array("utf8mb4_general_ci"))) { + print '   utf8mb4 general'; + } } print ''."\n"; print ''; diff --git a/htdocs/admin/system/dbtable.php b/htdocs/admin/system/dbtable.php index 95e738c250c3d..8167f1e6db97d 100644 --- a/htdocs/admin/system/dbtable.php +++ b/htdocs/admin/system/dbtable.php @@ -55,8 +55,17 @@ $sql = "ALTER TABLE ".$db->sanitize($table)." MODIFY ".$db->sanitize($row[0])." ".$row[1]." CHARACTER SET utf8"; // We must not sanitize the $row[1] $db->query($sql); - $sql = "ALTER TABLE ".$db->sanitize($table)." MODIFY ".$db->sanitize($row[0])." ".$row[1]." COLLATE utf8_unicode_ci"; // We must not sanitize the $row[1] - $db->query($sql); + $collation = 'utf8_unicode_ci'; + $defaultcollation = $db->getDefaultCollationDatabase(); + if (preg_match('/general/', $defaultcollation)) { + $collation = 'utf8_general_ci'; + } + + $sql = "ALTER TABLE ".$db->sanitize($table)." MODIFY ".$db->sanitize($row[0])." ".$row[1]." COLLATE ".$db->sanitize($collation); // We must not sanitize the $row[1] + $reslq2 = $db->query($sql); + if (!$resql2) { + setEventMessages($db->lasterror(), null, 'warnings'); + } break; } @@ -76,8 +85,17 @@ $sql = "ALTER TABLE ".$db->sanitize($table)." MODIFY ".$db->sanitize($row[0])." ".$row[1]." CHARACTER SET utf8mb4"; // We must not sanitize the $row[1] $db->query($sql); - $sql = "ALTER TABLE ".$db->sanitize($table)." MODIFY ".$db->sanitize($row[0])." ".$row[1]." COLLATE utf8mb4_unicode_ci"; // We must not sanitize the $row[1] - $db->query($sql); + $collation = 'utf8mb4_unicode_ci'; + $defaultcollation = $db->getDefaultCollationDatabase(); + if (preg_match('/general/', $defaultcollation)) { + $collation = 'utf8mb4_general_ci'; + } + + $sql = "ALTER TABLE ".$db->sanitize($table)." MODIFY ".$db->sanitize($row[0])." ".$row[1]." COLLATE ".$db->sanitize($collation); // We must not sanitize the $row[1] + $resql2 = $db->query($sql); + if (!$resql2) { + setEventMessages($db->lasterror(), null, 'warnings'); + } break; } diff --git a/htdocs/core/db/mysqli.class.php b/htdocs/core/db/mysqli.class.php index 7931374eb70cd..45d60332fdff3 100644 --- a/htdocs/core/db/mysqli.class.php +++ b/htdocs/core/db/mysqli.class.php @@ -105,6 +105,8 @@ public function __construct($type, $host, $user, $pass, $name = '', $port = 0) dol_syslog(get_class($this)."::DoliDBMysqli Connect error: ".$this->error, LOG_ERR); } + $disableforcecharset = 0; // Set to 1 to test without charset forcing + // If server connection is ok, we try to connect to the database if ($this->connected && $name) { if ($this->select_db($name)) { @@ -118,7 +120,6 @@ public function __construct($type, $host, $user, $pass, $name = '', $port = 0) $clientmustbe = 'utf8'; } - $disableforcecharset = 0; // Set to 1 to test without charset forcing if (empty($disableforcecharset) && $this->db->character_set_name() != $clientmustbe) { try { dol_syslog(get_class($this)."::DoliDBMysqli You should set the \$dolibarr_main_db_character_set and \$dolibarr_main_db_collation for the PHP to the same as the database default, so to ".$this->db->character_set_name(). " or upgrade database default to ".$clientmustbe.".", LOG_WARNING); @@ -173,7 +174,7 @@ public function __construct($type, $host, $user, $pass, $name = '', $port = 0) $clientmustbe = 'utf8'; } - if ($this->db->character_set_name() != $clientmustbe) { + if (empty($disableforcecharset) && $this->db->character_set_name() != $clientmustbe) { $this->db->set_charset($clientmustbe); // This set utf8_unicode_ci $collation = $conf->db->dolibarr_main_db_collation; diff --git a/htdocs/install/repair.php b/htdocs/install/repair.php index 9294990860ce8..20b6a9572bc1f 100644 --- a/htdocs/install/repair.php +++ b/htdocs/install/repair.php @@ -1347,10 +1347,16 @@ continue; } + $collation = 'utf8_unicode_ci'; + $defaultcollation = $db->getDefaultCollationDatabase(); + if (preg_match('/general/', $defaultcollation)) { + $collation = 'utf8_general_ci'; + } + print ''; print $table[0]; $sql1 = "ALTER TABLE ".$db->sanitize($table[0])." ROW_FORMAT=dynamic"; - $sql2 = "ALTER TABLE ".$db->sanitize($table[0])." CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci"; + $sql2 = "ALTER TABLE ".$db->sanitize($table[0])." CONVERT TO CHARACTER SET utf8 COLLATE ".$db->sanitize($collation); print ''; print ''; if ($force_utf8_on_tables == 'confirmed') { @@ -1471,10 +1477,16 @@ continue; } + $collation = 'utf8mb4_unicode_ci'; + $defaultcollation = $db->getDefaultCollationDatabase(); + if (preg_match('/general/', $defaultcollation)) { + $collation = 'utf8mb4_general_ci'; + } + print ''; print $table[0]; $sql1 = "ALTER TABLE ".$db->sanitize($table[0])." ROW_FORMAT=dynamic"; - $sql2 = "ALTER TABLE ".$db->sanitize($table[0])." CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"; + $sql2 = "ALTER TABLE ".$db->sanitize($table[0])." CONVERT TO CHARACTER SET utf8mb4 COLLATE ".$db->sanitize($collation); print ''; print ''; if ($force_utf8mb4_on_tables == 'confirmed') {