Skip to content

Commit

Permalink
FIX Toolt to convert utf8mb4
Browse files Browse the repository at this point in the history
  • Loading branch information
@eldy
eldy committed Sep 30, 2024
1 parent b5c6ead commit 471668b
Show file tree
Hide file tree
Showing 5 changed files with 98 additions and 15 deletions.
18 changes: 14 additions & 4 deletions htdocs/admin/system/database-tables.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,25 +52,35 @@
$db->query($sql);
}
if ($action == 'convertutf8') {
$sql = "ALTER TABLE ".$db->sanitize($table)." CHARACTER SET utf8 COLLATE utf8_unicode_ci"; // Set the default value on table
$collation = 'utf8_unicode_ci';
$defaultcollation = $db->getDefaultCollationDatabase();
if (preg_match('/general/', $defaultcollation)) {
$collation = 'utf8_general_ci';
}
$sql = "ALTER TABLE ".$db->sanitize($table)." CHARACTER SET utf8 COLLATE ".$db->sanitize($collation); // Set the default value on table
$resql1 = $db->query($sql);
if (!$resql1) {
setEventMessages($db->lasterror(), null, 'warnings');
} else {
$sql = "ALTER TABLE ".$db->sanitize($table)." CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci"; // Switch fields (may fails due to foreign key)
$sql = "ALTER TABLE ".$db->sanitize($table)." CONVERT TO CHARACTER SET utf8 COLLATE ".$db->sanitize($collation); // Switch fields (may fails due to foreign key)
$resql2 = $db->query($sql);
if (!$resql2) {
setEventMessages($db->lasterror(), null, 'warnings');
}
}
}
if ($action == 'convertutf8mb4') {
$sql = "ALTER TABLE ".$db->sanitize($table)." CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"; // Set the default value on table
$collation = 'utf8mb4_unicode_ci';
$defaultcollation = $db->getDefaultCollationDatabase();
if (preg_match('/general/', $defaultcollation)) {
$collation = 'utf8mb4_general_ci';
}
$sql = "ALTER TABLE ".$db->sanitize($table)." CHARACTER SET utf8mb4 COLLATE ".$db->sanitize($collation); // Set the default value on table
$resql1 = $db->query($sql);
if (!$resql1) {
setEventMessages($db->lasterror(), null, 'warnings');
} else {
$sql = "ALTER TABLE ".$db->sanitize($table)." CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"; // Switch fields (may fails due to foreign key)
$sql = "ALTER TABLE ".$db->sanitize($table)." CONVERT TO CHARACTER SET utf8mb4 COLLATE ".$db->sanitize($collation); // Switch fields (may fails due to foreign key)
$resql2 = $db->query($sql);
if (!$resql2) {
setEventMessages($db->lasterror(), null, 'warnings');
Expand Down
48 changes: 45 additions & 3 deletions htdocs/admin/system/database.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,34 @@

$langs->load("admin");

$action = GETPOST('action', 'aZ09');

if (!$user->admin) {
accessforbidden();
}


/*
* Actions
*/

if ($action == 'convertutf8unicode') { // Test on permission already done.
$sql = "ALTER DATABASE ".$db->sanitize($table[0])." CHARACTER SET utf8 COLLATE utf8_unicode_ci";
$db->query($sql);
}
if ($action == 'convertutf8mb4unicode') { // Test on permission already done.
$sql = "ALTER DATABASE ".$db->sanitize($table[0])." CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci";
$db->query($sql);
}
if ($action == 'convertutf8general') { // Test on permission already done.
$sql = "ALTER DATABASE ".$db->sanitize($table[0])." CHARACTER SET utf8 COLLATE utf8_general_ci";
$db->query($sql);
}
if ($action == 'convertutf8mb4general') { // Test on permission already done.
$sql = "ALTER DATABASE ".$db->sanitize($table[0])." CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci";
$db->query($sql);
}


/*
* View
Expand All @@ -59,14 +82,33 @@
print '<tr class="oddeven"><td width="300">'.$langs->trans("Password").'</td><td>'.preg_replace('/./i', '*', $dolibarr_main_db_pass).'</td></tr>'."\n";
print '<tr class="oddeven"><td width="300">'.$langs->trans("DBStoringCharset").'</td><td>'.$db->getDefaultCharacterSetDatabase();
if ($db->type == 'mysqli') {
print ' '.$form->textwithpicto('', $langs->transnoentitiesnoconv("HelpMariaDBToGetValue", "SHOW VARIABLES LIKE 'character_set_database'").'<br>'.$langs->transnoentitiesnoconv("HelpMariaDBToGetPossibleValues", "SHOW CHARSET"));
print ' '.$form->textwithpicto('', $langs->transnoentitiesnoconv("HelpMariaDBToGetValue", "<br>SHOW VARIABLES LIKE 'character_set_database' (cached)<br>You can avoid cache effect with:<br>SELECT DEFAULT_CHARACTER_SET_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = '".$conf->db->name."'").'<br>'.$langs->transnoentitiesnoconv("HelpMariaDBToGetPossibleValues", "<br>SHOW CHARSET"));
// We can use $db->getDefaultCharacterSetDatabase(), $db->getListOfCharacterSet(),
}
print '</td></tr>'."\n";
print '<tr class="oddeven"><td width="300">'.$langs->trans("DBSortingCharset").'</td><td>'.$db->getDefaultCollationDatabase();
print '<tr class="oddeven"><td width="300">'.$langs->trans("DBSortingCharset").'</td><td>';
$defaultcollation = $db->getDefaultCollationDatabase();
print dolPrintHTML($defaultcollation);
if ($db->type == 'mysqli') {
print ' '.$form->textwithpicto('', $langs->transnoentitiesnoconv("HelpMariaDBToGetValue", "SHOW VARIABLES LIKE 'collation_database'").'<br>'.$langs->transnoentitiesnoconv("HelpMariaDBToGetPossibleValues", "SHOW COLLATION"));
if ($defaultcollation != $dolibarr_main_db_collation) {
print img_warning('The database default value of collation '.$defaultcollation.' differs from conf setup '.$dolibarr_main_db_collation);
}
print ' '.$form->textwithpicto('', $langs->transnoentitiesnoconv("HelpMariaDBToGetValue", "<br>SHOW VARIABLES LIKE 'collation_database' (cached)<br>You can avoid cache effect with:<br>SELECT DEFAULT_COLLATION_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = '".$conf->db->name."'").'<br>'.$langs->transnoentitiesnoconv("HelpMariaDBToGetPossibleValues", "<br>SHOW COLLATION"));
// We can use $db->getDefaultCollationDatabase(), $db->getListOfCollation();

print ' &nbsp; &nbsp; &nbsp; <span class="opacitymedium small">'.$langs->trans("ConvertInto");
if (!in_array($defaultcollation, array("utf8_unicode_ci"))) {
print ' &nbsp; <a class="reposition" href="'.DOL_URL_ROOT.'/admin/system/database.php?action=convertutf8unicode&token='.newToken().'">utf8 unicode</a>';
}
if (!in_array($defaultcollation, array("utf8_general_ci"))) {
print ' &nbsp; <a class="reposition" href="'.DOL_URL_ROOT.'/admin/system/database.php?action=convertutf8general&token='.newToken().'">utf8 general</a>';
}
if (!in_array($defaultcollation, array("utf8mb4_unicode_ci"))) {
print ' &nbsp; <a class="reposition" href="'.DOL_URL_ROOT.'/admin/system/database.php?action=convertutf8mb4unicode&&token='.newToken().'">utf8mb4 unicode</a>';
}
if (!in_array($defaultcollation, array("utf8mb4_general_ci"))) {
print ' &nbsp; <a class="reposition" href="'.DOL_URL_ROOT.'/admin/system/database.php?action=convertutf8mb4general&&token='.newToken().'">utf8mb4 general</a>';
}
}
print '</td></tr>'."\n";
print '</table>';
Expand Down
26 changes: 22 additions & 4 deletions htdocs/admin/system/dbtable.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,17 @@
$sql = "ALTER TABLE ".$db->sanitize($table)." MODIFY ".$db->sanitize($row[0])." ".$row[1]." CHARACTER SET utf8"; // We must not sanitize the $row[1]
$db->query($sql);

$sql = "ALTER TABLE ".$db->sanitize($table)." MODIFY ".$db->sanitize($row[0])." ".$row[1]." COLLATE utf8_unicode_ci"; // We must not sanitize the $row[1]
$db->query($sql);
$collation = 'utf8_unicode_ci';
$defaultcollation = $db->getDefaultCollationDatabase();
if (preg_match('/general/', $defaultcollation)) {
$collation = 'utf8_general_ci';
}

$sql = "ALTER TABLE ".$db->sanitize($table)." MODIFY ".$db->sanitize($row[0])." ".$row[1]." COLLATE ".$db->sanitize($collation); // We must not sanitize the $row[1]
$reslq2 = $db->query($sql);
if (!$resql2) {
setEventMessages($db->lasterror(), null, 'warnings');
}

break;
}
Expand All @@ -76,8 +85,17 @@
$sql = "ALTER TABLE ".$db->sanitize($table)." MODIFY ".$db->sanitize($row[0])." ".$row[1]." CHARACTER SET utf8mb4"; // We must not sanitize the $row[1]
$db->query($sql);

$sql = "ALTER TABLE ".$db->sanitize($table)." MODIFY ".$db->sanitize($row[0])." ".$row[1]." COLLATE utf8mb4_unicode_ci"; // We must not sanitize the $row[1]
$db->query($sql);
$collation = 'utf8mb4_unicode_ci';
$defaultcollation = $db->getDefaultCollationDatabase();
if (preg_match('/general/', $defaultcollation)) {
$collation = 'utf8mb4_general_ci';
}

$sql = "ALTER TABLE ".$db->sanitize($table)." MODIFY ".$db->sanitize($row[0])." ".$row[1]." COLLATE ".$db->sanitize($collation); // We must not sanitize the $row[1]
$resql2 = $db->query($sql);
if (!$resql2) {
setEventMessages($db->lasterror(), null, 'warnings');
}

break;
}
Expand Down
5 changes: 3 additions & 2 deletions htdocs/core/db/mysqli.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ public function __construct($type, $host, $user, $pass, $name = '', $port = 0)
dol_syslog(get_class($this)."::DoliDBMysqli Connect error: ".$this->error, LOG_ERR);
}

$disableforcecharset = 0; // Set to 1 to test without charset forcing

// If server connection is ok, we try to connect to the database
if ($this->connected && $name) {
if ($this->select_db($name)) {
Expand All @@ -118,7 +120,6 @@ public function __construct($type, $host, $user, $pass, $name = '', $port = 0)
$clientmustbe = 'utf8';
}

$disableforcecharset = 0; // Set to 1 to test without charset forcing
if (empty($disableforcecharset) && $this->db->character_set_name() != $clientmustbe) {
try {
dol_syslog(get_class($this)."::DoliDBMysqli You should set the \$dolibarr_main_db_character_set and \$dolibarr_main_db_collation for the PHP to the same as the database default, so to ".$this->db->character_set_name(). " or upgrade database default to ".$clientmustbe.".", LOG_WARNING);
Expand Down Expand Up @@ -173,7 +174,7 @@ public function __construct($type, $host, $user, $pass, $name = '', $port = 0)
$clientmustbe = 'utf8';
}

if ($this->db->character_set_name() != $clientmustbe) {
if (empty($disableforcecharset) && $this->db->character_set_name() != $clientmustbe) {
$this->db->set_charset($clientmustbe); // This set utf8_unicode_ci

$collation = $conf->db->dolibarr_main_db_collation;
Expand Down
16 changes: 14 additions & 2 deletions htdocs/install/repair.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1347,10 +1347,16 @@
continue;
}

$collation = 'utf8_unicode_ci';
$defaultcollation = $db->getDefaultCollationDatabase();
if (preg_match('/general/', $defaultcollation)) {
$collation = 'utf8_general_ci';
}

print '<tr><td colspan="2">';
print $table[0];
$sql1 = "ALTER TABLE ".$db->sanitize($table[0])." ROW_FORMAT=dynamic";
$sql2 = "ALTER TABLE ".$db->sanitize($table[0])." CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci";
$sql2 = "ALTER TABLE ".$db->sanitize($table[0])." CONVERT TO CHARACTER SET utf8 COLLATE ".$db->sanitize($collation);
print '<!-- '.$sql1.' -->';
print '<!-- '.$sql2.' -->';
if ($force_utf8_on_tables == 'confirmed') {
Expand Down Expand Up @@ -1471,10 +1477,16 @@
continue;
}

$collation = 'utf8mb4_unicode_ci';
$defaultcollation = $db->getDefaultCollationDatabase();
if (preg_match('/general/', $defaultcollation)) {
$collation = 'utf8mb4_general_ci';
}

print '<tr><td colspan="2">';
print $table[0];
$sql1 = "ALTER TABLE ".$db->sanitize($table[0])." ROW_FORMAT=dynamic";
$sql2 = "ALTER TABLE ".$db->sanitize($table[0])." CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci";
$sql2 = "ALTER TABLE ".$db->sanitize($table[0])." CONVERT TO CHARACTER SET utf8mb4 COLLATE ".$db->sanitize($collation);
print '<!-- '.$sql1.' -->';
print '<!-- '.$sql2.' -->';
if ($force_utf8mb4_on_tables == 'confirmed') {
Expand Down

0 comments on commit 471668b

Please sign in to comment.