diff --git a/components/ILIAS/Database/classes/PDO/Internal.php b/components/ILIAS/Database/classes/PDO/Internal.php index 47e5078d2f31..fd2ed847293a 100644 --- a/components/ILIAS/Database/classes/PDO/Internal.php +++ b/components/ILIAS/Database/classes/PDO/Internal.php @@ -28,4 +28,5 @@ interface Internal extends ilDBInterface, ilDBPdoInterface { public function getFieldDefinition(): ?FieldDefinition; public function getIndexName(string $index_name_base): string; + public function getDbName(): string; } diff --git a/components/ILIAS/Database/classes/PDO/ilDBPdo.php b/components/ILIAS/Database/classes/PDO/ilDBPdo.php index 108dfaeb1194..5104454499e5 100755 --- a/components/ILIAS/Database/classes/PDO/ilDBPdo.php +++ b/components/ILIAS/Database/classes/PDO/ilDBPdo.php @@ -38,7 +38,7 @@ class ilDBPdo implements Internal private string $host = ''; private string $dbname = ''; - private string $charset = 'utf8'; + private string $charset = 'utf8mb4'; private string $username = ''; private string $password = ''; private int $port = 3306; @@ -52,6 +52,7 @@ class ilDBPdo implements Internal private string $db_type = ''; private int $error_code = 0; private ?FieldDefinition $field_definition = null; + private ?bool $mb4_supported = null; private const SESSION_MODES = [ 'STRICT_TRANS_TABLES', @@ -1475,16 +1476,11 @@ public function sanitizeMB4StringIfNotSupported(string $query): string #[\Override] public function doesCollationSupportMB4Strings(): bool { - // Currently ILIAS does not support utf8mb4, after that ilDB could check like this: - // static $supported; - // if (!isset($supported)) { - // $q = "SELECT default_character_set_name FROM information_schema.SCHEMATA WHERE schema_name = %s;"; - // $res = $this->queryF($q, ['text'], [$this->getDbname()]); - // $data = $this->fetchObject($res); - // $supported = ($data->default_character_set_name === 'utf8mb4'); - // } - - return false; + return $this->mb4_supported ??= ($this->fetchAssoc($this->queryF( + 'SELECT default_character_set_name as charset FROM information_schema.SCHEMATA WHERE schema_name = %s', + [ilDBConstants::T_TEXT], + [$this->getDbname()] + ))['charset'] ?? '') === 'utf8mb4'; } public function groupConcat(string $a_field_name, string $a_seperator = ",", ?string $a_order = null): string diff --git a/components/ILIAS/Database/classes/Setup/MB4Migration.php b/components/ILIAS/Database/classes/Setup/MB4Migration.php new file mode 100644 index 000000000000..386275d3d159 --- /dev/null +++ b/components/ILIAS/Database/classes/Setup/MB4Migration.php @@ -0,0 +1,161 @@ +db = $environment->getResource(Environment::RESOURCE_DATABASE); + $this->dbName = $this->db->getDbName(); + $this->collation = $this->selectCollation(); + } + + public function getPreconditions(Environment $environment): array + { + return [ + new ilIniFilesLoadedObjective(), + new ilDatabaseInitializedObjective(), + new ilDatabaseUpdatedObjective() + ]; + } + + public function step(Environment $environment): void + { + $this->convertCharsetDatabase(); + $this->convertCharsetTables(); + } + + public function getDefaultAmountOfStepsPerRun(): int + { + return Migration::INFINITE; + } + + public function getRemainingAmountOfSteps(): int + { + return 1; + } + + public function getLabel(): string + { + return 'Migrate all tables from utf8mb3 to utf8mb4'; + } + + /** + * Returns utf8mb4_unicode_520_ci if the database supports it and falls back to utf8mb4_unicode_ci. + * + * @return string + */ + private function selectCollation(): string + { + $result = $this->db->queryF( + 'SHOW COLLATION WHERE COLLATION = %s', + [ilDBConstants::T_TEXT], + [ilDBConstants::MYSQL_COLLATION_UTF8MB4_520] + ); + return $this->db->fetchAssoc($result) !== null ? + ilDBConstants::MYSQL_COLLATION_UTF8MB4_520 : + ilDBConstants::MYSQL_COLLATION_UTF8MB4_UNICODE; + } + + /** + * ilDBInterface provides the method listTables, but it ignores sequence tables. + * + * @return string[] + */ + private function findTables(): array + { + $s = $this->db->query('SHOW TABLES FROM ' . $this->db->quoteIdentifier($this->dbName)); + return array_map('current', $this->db->fetchAll($s)); + } + + private function findDBCharset(): ?string + { + $q = 'SELECT DEFAULT_CHARACTER_SET_NAME as charset ' . + 'FROM information_schema.SCHEMATA ' . + 'WHERE SCHEMA_NAME = %s'; + return $this->db->fetchAssoc($this->db->queryF($q, [ilDBConstants::T_TEXT], [$this->dbName]))['charset'] ?? null; + } + + private function findTableCharset(string $table): ?string + { + $q = 'SELECT CCSA.CHARACTER_SET_NAME AS charset ' . + 'FROM information_schema.TABLES AS T ' . + 'JOIN information_schema.COLLATION_CHARACTER_SET_APPLICABILITY AS CCSA ' . + 'WHERE T.TABLE_COLLATION = CCSA.COLLATION_NAME ' . + 'AND TABLE_SCHEMA=%s AND TABLE_NAME=%s'; + return $this->db->fetchAssoc($this->db->queryF( + $q, + [ilDBConstants::T_TEXT, ilDBConstants::T_TEXT], + [$this->dbName, $table] + ))['charset'] ?? null; + } + + private function convertCharsetDatabase(): void + { + $charset = $this->findDBCharset(); + if ($charset === null || $charset === self::CHARSET) { + return; + } + try { + $this->db->manipulate(sprintf( + 'ALTER DATABASE %s CHARACTER SET = %s COLLATE = %s', + $this->db->quoteIdentifier($this->dbName), + $this->db->quoteIdentifier(self::CHARSET), + $this->db->quoteIdentifier($this->collation), + )); + } catch (\Exception $e) { + var_dump($e); + } + } + + private function convertCharsetTables(): void + { + foreach ($this->findTables() as $table) { + $table_charset = $this->findTableCharset($table); + if ($table_charset === null || $table_charset === self::CHARSET) { + continue; + } + $this->db->manipulate(sprintf( + 'ALTER TABLE %s CONVERT TO CHARACTER SET %s COLLATE %s', + $this->db->quoteIdentifier($table), + $this->db->quoteIdentifier(self::CHARSET), + $this->db->quoteIdentifier($this->collation), + )); + } + } +} diff --git a/components/ILIAS/Database/classes/Setup/ilDatabaseSetupAgent.php b/components/ILIAS/Database/classes/Setup/ilDatabaseSetupAgent.php index 224931937d77..321fff4ae3f1 100755 --- a/components/ILIAS/Database/classes/Setup/ilDatabaseSetupAgent.php +++ b/components/ILIAS/Database/classes/Setup/ilDatabaseSetupAgent.php @@ -30,6 +30,7 @@ use ILIAS\Refinery\Factory as Refinery; use ILIAS\Refinery\Transformation; use ILIAS\Setup\Objective\CalledFromRootObjective; +use ILIAS\Database\Setup\MB4Migration; class ilDatabaseSetupAgent implements Agent { @@ -126,7 +127,8 @@ public function getStatusObjective(Storage $storage): Objective public function getMigrations(): array { return [ - new ilMysqlMyIsamToInnoDbMigration() + new ilMysqlMyIsamToInnoDbMigration(), + new MB4Migration(), ]; } diff --git a/components/ILIAS/Database/classes/ilDBConstants.php b/components/ILIAS/Database/classes/ilDBConstants.php index 6c24ad31b869..c89b14d479b7 100755 --- a/components/ILIAS/Database/classes/ilDBConstants.php +++ b/components/ILIAS/Database/classes/ilDBConstants.php @@ -83,6 +83,10 @@ class ilDBConstants public const MYSQL_COLLATION_UTF8_SPANISH = "utf8_spanish_ci"; public const MYSQL_COLLATION_UTF8_SWEDISH = "utf8_swedish_ci"; public const MYSQL_COLLATION_UTF8_TURKISH = "utf8_turkish_ci"; + public const MYSQL_COLLATION_UTF8MB4_UNICODE = 'utf8mb4_unicode_ci'; + public const MYSQL_COLLATION_UTF8MB4_520 = 'utf8mb4_unicode_520_ci'; + public const MYSQL_COLLATION_UTF8MB4_0900 = 'utf8mb4_0900_ai_ci'; + public const MARIADB_COLLATION_UCA = 'uca1400_ai_ci'; // Mapping AutoExec public const AUTOQUERY_INSERT = 1; @@ -168,10 +172,25 @@ public static function getAvailableCollations(): array ilDBConstants::MYSQL_COLLATION_UTF8_SPANISH2, ilDBConstants::MYSQL_COLLATION_UTF8_SPANISH, ilDBConstants::MYSQL_COLLATION_UTF8_SWEDISH, - ilDBConstants::MYSQL_COLLATION_UTF8_TURKISH + ilDBConstants::MYSQL_COLLATION_UTF8_TURKISH, + ilDBConstants::MYSQL_COLLATION_UTF8MB4_UNICODE, + ilDBConstants::MYSQL_COLLATION_UTF8MB4_520, + ilDBConstants::MYSQL_COLLATION_UTF8MB4_0900, + ilDBConstants::MARIADB_COLLATION_UCA, ]; } + public static function isUTF8MB4Collation(string $collation): bool + { + return in_array($collation, [ + ilDBConstants::MYSQL_COLLATION_UTF8MB4, + ilDBConstants::MYSQL_COLLATION_UTF8MB4_UNICODE, + ilDBConstants::MYSQL_COLLATION_UTF8MB4_520, + ilDBConstants::MYSQL_COLLATION_UTF8MB4_0900, + ilDBConstants::MARIADB_COLLATION_UCA, + ], true); + } + public static function describe(string $type): string { return self::$descriptions[$type];