Database: Add utf8mb4 support

components/ILIAS/Database/classes/PDO/Internal.php

@@ -28,4 +28,5 @@ interface Internal extends ilDBInterface, ilDBPdoInterface
 {
     public function getFieldDefinition(): ?FieldDefinition;
     public function getIndexName(string $index_name_base): string;
+    public function getDbName(): string;
 }

components/ILIAS/Database/classes/PDO/ilDBPdo.php

@@ -38,7 +38,7 @@ class ilDBPdo implements Internal
 
     private string $host = '';
     private string $dbname = '';
-    private string $charset = 'utf8';
+    private string $charset = 'utf8mb4';
     private string $username = '';
     private string $password = '';
     private int $port = 3306;
@@ -52,6 +52,7 @@ class ilDBPdo implements Internal
     private string $db_type = '';
     private int $error_code = 0;
     private ?FieldDefinition $field_definition = null;
+    private ?bool $mb4_supported = null;
 
     private const SESSION_MODES = [
         'STRICT_TRANS_TABLES',
@@ -1475,16 +1476,11 @@ public function sanitizeMB4StringIfNotSupported(string $query): string
     #[\Override]
     public function doesCollationSupportMB4Strings(): bool
     {
-        // Currently ILIAS does not support utf8mb4, after that ilDB could check like this:
-        //		static $supported;
-        //		if (!isset($supported)) {
-        //			$q = "SELECT default_character_set_name FROM information_schema.SCHEMATA WHERE schema_name = %s;";
-        //			$res = $this->queryF($q, ['text'], [$this->getDbname()]);
-        //			$data = $this->fetchObject($res);
-        //			$supported = ($data->default_character_set_name === 'utf8mb4');
-        //		}
-
-        return false;
+        return $this->mb4_supported ??= ($this->fetchAssoc($this->queryF(
+            'SELECT default_character_set_name as charset FROM information_schema.SCHEMATA WHERE schema_name = %s',
+            [ilDBConstants::T_TEXT],
+            [$this->getDbname()]
+        ))['charset'] ?? '') === 'utf8mb4';
     }
 
     public function groupConcat(string $a_field_name, string $a_seperator = ",", ?string $a_order = null): string

components/ILIAS/Database/classes/Setup/MB4Migration.php

@@ -0,0 +1,161 @@
+<?php
+
+/**
+ * This file is part of ILIAS, a powerful learning management system
+ * published by ILIAS open source e-Learning e.V.
+ *
+ * ILIAS is licensed with the GPL-3.0,
+ * see https://www.gnu.org/licenses/gpl-3.0.en.html
+ * You should have received a copy of said license along with the
+ * source code, too.
+ *
+ * If this is not the case or you just want to try ILIAS, you'll find
+ * us at:
+ * https://www.ilias.de
+ * https://github.com/ILIAS-eLearning
+ *
+ *********************************************************************/
+
+declare(strict_types=1);
+
+namespace ILIAS\Database\Setup;
+
+use ilDBInterface;
+use ILIAS\Setup\Environment;
+use ILIAS\Setup\Migration;
+use ilIniFilesLoadedObjective;
+use ilDatabaseInitializedObjective;
+use ilDatabaseUpdatedObjective;
+use ilDBPdoInterface;
+use ilDBConstants;
+use ILIAS\Database\PDO\Internal;
+
+class MB4Migration implements Migration
+{
+    private const CHARSET = 'utf8mb4';
+
+    private Internal $db;
+    private string $dbName;
+    private string $collation;
+
+    public function prepare(Environment $environment): void
+    {
+        $this->db = $environment->getResource(Environment::RESOURCE_DATABASE);
+        $this->dbName = $this->db->getDbName();
+        $this->collation = $this->selectCollation();
+    }
+
+    public function getPreconditions(Environment $environment): array
+    {
+        return [
+            new ilIniFilesLoadedObjective(),
+            new ilDatabaseInitializedObjective(),
+            new ilDatabaseUpdatedObjective()
+        ];
+    }
+
+    public function step(Environment $environment): void
+    {
+        $this->convertCharsetDatabase();
+        $this->convertCharsetTables();
+    }
+
+    public function getDefaultAmountOfStepsPerRun(): int
+    {
+        return Migration::INFINITE;
+    }
+
+    public function getRemainingAmountOfSteps(): int
+    {
+        return 1;
+    }
+
+    public function getLabel(): string
+    {
+        return 'Migrate all tables from utf8mb3 to utf8mb4';
+    }
+
+    /**
+     * Returns utf8mb4_unicode_520_ci if the database supports it and falls back to utf8mb4_unicode_ci.
+     *
+     * @return string
+     */
+    private function selectCollation(): string
+    {
+        $result = $this->db->queryF(
+            'SHOW COLLATION WHERE COLLATION = %s',
+            [ilDBConstants::T_TEXT],
+            [ilDBConstants::MYSQL_COLLATION_UTF8MB4_520]
+        );
+        return $this->db->fetchAssoc($result) !== null ?
+            ilDBConstants::MYSQL_COLLATION_UTF8MB4_520 :
+            ilDBConstants::MYSQL_COLLATION_UTF8MB4_UNICODE;
+    }
+
+    /**
+     * ilDBInterface provides the method listTables, but it ignores sequence tables.
+     *
+     * @return string[]
+     */
+    private function findTables(): array
+    {
+        $s = $this->db->query('SHOW TABLES FROM ' . $this->db->quoteIdentifier($this->dbName));
+        return array_map('current', $this->db->fetchAll($s));
+    }
+
+    private function findDBCharset(): ?string
+    {
+        $q = 'SELECT DEFAULT_CHARACTER_SET_NAME as charset ' .
+             'FROM information_schema.SCHEMATA ' .
+             'WHERE SCHEMA_NAME = %s';
+        return $this->db->fetchAssoc($this->db->queryF($q, [ilDBConstants::T_TEXT], [$this->dbName]))['charset'] ?? null;
+    }
+
+    private function findTableCharset(string $table): ?string
+    {
+        $q = 'SELECT CCSA.CHARACTER_SET_NAME AS charset ' .
+             'FROM information_schema.TABLES AS T ' .
+             'JOIN information_schema.COLLATION_CHARACTER_SET_APPLICABILITY AS CCSA ' .
+             'WHERE T.TABLE_COLLATION = CCSA.COLLATION_NAME ' .
+             'AND TABLE_SCHEMA=%s AND TABLE_NAME=%s';
+        return $this->db->fetchAssoc($this->db->queryF(
+            $q,
+            [ilDBConstants::T_TEXT, ilDBConstants::T_TEXT],
+            [$this->dbName, $table]
+        ))['charset'] ?? null;
+    }
+
+    private function convertCharsetDatabase(): void
+    {
+        $charset = $this->findDBCharset();
+        if ($charset === null || $charset === self::CHARSET) {
+            return;
+        }
+        try {
+            $this->db->manipulate(sprintf(
+                'ALTER DATABASE %s CHARACTER SET = %s COLLATE = %s',
+                $this->db->quoteIdentifier($this->dbName),
+                $this->db->quoteIdentifier(self::CHARSET),
+                $this->db->quoteIdentifier($this->collation),
+            ));
+        } catch (\Exception $e) {
+            var_dump($e);
+        }
+    }
+
+    private function convertCharsetTables(): void
+    {
+        foreach ($this->findTables() as $table) {
+            $table_charset = $this->findTableCharset($table);
+            if ($table_charset === null || $table_charset === self::CHARSET) {
+                continue;
+            }
+            $this->db->manipulate(sprintf(
+                'ALTER TABLE %s CONVERT TO CHARACTER SET %s COLLATE %s',
+                $this->db->quoteIdentifier($table),
+                $this->db->quoteIdentifier(self::CHARSET),
+                $this->db->quoteIdentifier($this->collation),
+            ));
+        }
+    }
+}

components/ILIAS/Database/classes/Setup/ilDatabaseSetupAgent.php

@@ -30,6 +30,7 @@
 use ILIAS\Refinery\Factory as Refinery;
 use ILIAS\Refinery\Transformation;
 use ILIAS\Setup\Objective\CalledFromRootObjective;
+use ILIAS\Database\Setup\MB4Migration;
 
 class ilDatabaseSetupAgent implements Agent
 {
@@ -126,7 +127,8 @@ public function getStatusObjective(Storage $storage): Objective
     public function getMigrations(): array
     {
         return [
-            new ilMysqlMyIsamToInnoDbMigration()
+            new ilMysqlMyIsamToInnoDbMigration(),
+            new MB4Migration(),
         ];
     }
 

components/ILIAS/Database/classes/ilDBConstants.php

@@ -83,6 +83,10 @@ class ilDBConstants
     public const MYSQL_COLLATION_UTF8_SPANISH = "utf8_spanish_ci";
     public const MYSQL_COLLATION_UTF8_SWEDISH = "utf8_swedish_ci";
     public const MYSQL_COLLATION_UTF8_TURKISH = "utf8_turkish_ci";
+    public const MYSQL_COLLATION_UTF8MB4_UNICODE = 'utf8mb4_unicode_ci';
+    public const MYSQL_COLLATION_UTF8MB4_520 = 'utf8mb4_unicode_520_ci';
+    public const MYSQL_COLLATION_UTF8MB4_0900 = 'utf8mb4_0900_ai_ci';
+    public const MARIADB_COLLATION_UCA = 'uca1400_ai_ci';
 
     // Mapping AutoExec
     public const AUTOQUERY_INSERT = 1;
@@ -168,10 +172,25 @@ public static function getAvailableCollations(): array
             ilDBConstants::MYSQL_COLLATION_UTF8_SPANISH2,
             ilDBConstants::MYSQL_COLLATION_UTF8_SPANISH,
             ilDBConstants::MYSQL_COLLATION_UTF8_SWEDISH,
-            ilDBConstants::MYSQL_COLLATION_UTF8_TURKISH
+            ilDBConstants::MYSQL_COLLATION_UTF8_TURKISH,
+            ilDBConstants::MYSQL_COLLATION_UTF8MB4_UNICODE,
+            ilDBConstants::MYSQL_COLLATION_UTF8MB4_520,
+            ilDBConstants::MYSQL_COLLATION_UTF8MB4_0900,
+            ilDBConstants::MARIADB_COLLATION_UCA,
         ];
     }
 
+    public static function isUTF8MB4Collation(string $collation): bool
+    {
+        return in_array($collation, [
+            ilDBConstants::MYSQL_COLLATION_UTF8MB4,
+            ilDBConstants::MYSQL_COLLATION_UTF8MB4_UNICODE,
+            ilDBConstants::MYSQL_COLLATION_UTF8MB4_520,
+            ilDBConstants::MYSQL_COLLATION_UTF8MB4_0900,
+            ilDBConstants::MARIADB_COLLATION_UCA,
+        ], true);
+    }
+
     public static function describe(string $type): string
     {
         return self::$descriptions[$type];