diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f9661d..f0ccfc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ Latest ------ ## Fixes +* [#303](https://github.com/cleverage/ui-process-bundle/issues/303) Logs list: the context filter also searches the value as encoded in JSON by Doctrine (internal `LogTextFilter`): PostgreSQL, MariaDB and SQLite keep the JSON as written (`/` written `\/`, `"` written `\"`, non-ASCII characters written `\uXXXX`), a search containing such characters (e.g. a path) found nothing there. Update documentation, add tests. * [#299](https://github.com/cleverage/ui-process-bundle/issues/299) `DoctrineProcessHandler`: below the `Error` level, a log record counted in the report is inserted right away at most every `process_execution.heartbeat_interval` seconds (new `setWriteRightAwayInterval()`), otherwise with the next ones: since 3.5, each warning was inserted with the report of the execution in a transaction of its own (5000 warnings: 60 s instead of 8 s). The records from the `Error` level are still inserted right away. Update documentation, add tests. * [#301](https://github.com/cleverage/ui-process-bundle/issues/301) The `pb_ui_file_filter` and `pb_ui_orm_filter` Monolog handlers accept the level of the handler they wrap (`file_level`, `database_level`, set by the new internal `MonologFilterLevelPass`) instead of every level: the Monolog processors no longer run for the records written by none of them, e.g. the debug record of each task execution. Update documentation, add tests. diff --git a/docs/reference/04-process_executions_and_logs.md b/docs/reference/04-process_executions_and_logs.md index 922fed8..aceb8b2 100644 --- a/docs/reference/04-process_executions_and_logs.md +++ b/docs/reference/04-process_executions_and_logs.md @@ -358,8 +358,10 @@ The command does not run them: they would block the running processes, which wri The "Process > Logs" menu lists the database logs (250 per page), with their level, message, date and a "Has context info ?" column. The detail page displays the log context. They can be filtered by process (or execution), level, message, context and date. The message and context filters are case-insensitive, on every database. The context -filter searches the JSON representation of the context, as stored by the database (e.g. with escaped non-ASCII -characters on PostgreSQL). +filter searches the JSON representation of the context (e.g. `"file":` finds the `file` keys), also with the +characters escaped by the JSON encoding of Doctrine, kept as they are by PostgreSQL, MariaDB and SQLite (`/` written +`\/`, `"` written `\"`, non-ASCII characters written `\uXXXX`): `/var/data` or `Élodie` are found on every database. +On these databases, the search of a non-ASCII character is case-sensitive (`élodie` does not find `Élodie`). The list counts the log records of the current filters, and the message and context filters read every log record (`LIKE '%...%'`, which no index can serve): on large tables they take seconds per page (with 3 million log records, diff --git a/src/Admin/Filter/LogTextFilter.php b/src/Admin/Filter/LogTextFilter.php index 2ecc67e..d8cb73c 100644 --- a/src/Admin/Filter/LogTextFilter.php +++ b/src/Admin/Filter/LogTextFilter.php @@ -13,6 +13,7 @@ namespace CleverAge\UiProcessBundle\Admin\Filter; +use Doctrine\DBAL\Types\Types; use Doctrine\ORM\QueryBuilder; use EasyCorp\Bundle\EasyAdminBundle\Contracts\Filter\FilterInterface; use EasyCorp\Bundle\EasyAdminBundle\Dto\EntityDto; @@ -25,7 +26,8 @@ * The TextFilter of EasyAdmin, for the message and the JSON context of the log records, case-insensitive on every * database: the column is converted to text (PostgreSQL has no LIKE operator for the json type: "operator does not * exist: json ~~ unknown", json || text is text) and lowercased, as the value. LIKE is case-sensitive on PostgreSQL, and - * on the JSON columns of MySQL and MariaDB (binary collation). + * on the JSON columns of MySQL and MariaDB (binary collation). On the JSON context, the value is also searched as encoded + * by Doctrine (see encodeAsStored()). * * @internal */ @@ -33,6 +35,9 @@ class LogTextFilter implements FilterInterface { use FilterTrait; + /** Escape character of the LIKE patterns of the JSON representation of the value, instead of the backslash */ + private const LIKE_ESCAPE = '!'; + public static function new(string $propertyName, ?string $label = null): self { return (new self()) @@ -45,14 +50,76 @@ public static function new(string $propertyName, ?string $label = null): self public function apply(QueryBuilder $queryBuilder, FilterDataDto $filterDataDto, ?FieldDto $fieldDto, EntityDto $entityDto): void { + $column = \sprintf("LOWER(CONCAT(%s.%s, ''))", $filterDataDto->getEntityAlias(), $filterDataDto->getProperty()); + $comparison = $filterDataDto->getComparison(); + $parameterName = $filterDataDto->getParameterName(); + $value = $filterDataDto->getValue(); + $condition = \sprintf('%s %s LOWER(:%s)', $column, $comparison, $parameterName); + $encodedValue = $this->isJson($entityDto, $filterDataDto->getProperty()) && \is_string($value) + ? $this->encodeAsStored($value, $comparison) + : null; + if (null === $encodedValue) { + $queryBuilder->andWhere($condition)->setParameter($parameterName, $value); + + return; + } + $encodedCondition = \sprintf( + '%s %s LOWER(:%s_json)%s', + $column, + $comparison, + $parameterName, + $this->isLike($comparison) ? " ESCAPE '".self::LIKE_ESCAPE."'" : '' + ); + // Matching either representation, or none of them for a negative comparison $queryBuilder ->andWhere(\sprintf( - "LOWER(CONCAT(%s.%s, '')) %s LOWER(:%s)", - $filterDataDto->getEntityAlias(), - $filterDataDto->getProperty(), - $filterDataDto->getComparison(), - $filterDataDto->getParameterName() + $this->isNegative($comparison) ? '(%s AND %s)' : '(%s OR %s)', + $condition, + $encodedCondition )) - ->setParameter($filterDataDto->getParameterName(), $filterDataDto->getValue()); + ->setParameter($parameterName, $value) + ->setParameter($parameterName.'_json', $encodedValue); + } + + /** + * The value as written in the JSON column by Doctrine (json_encode() of the DBAL "json" type: "/" written "\/", + * non-ASCII characters "\uXXXX"...), kept as it is by PostgreSQL (json type), MariaDB and SQLite (text): a search + * with such characters found nothing there. MySQL stores the JSON in a binary format, its text representation + * does not escape them. Null when the value is written the same. + */ + private function encodeAsStored(string $value, string $comparison): ?string + { + $encoded = json_encode($value, \JSON_PRESERVE_ZERO_FRACTION); + if (false === $encoded) { + return null; + } + $encoded = substr($encoded, 1, -1); + if ($encoded === $value) { + return null; + } + + // Escaped with its own character: the backslashes are then literal in the pattern (escape characters by default + // on MySQL, MariaDB and PostgreSQL, not on SQLite) + return $this->isLike($comparison) ? str_replace(self::LIKE_ESCAPE, self::LIKE_ESCAPE.self::LIKE_ESCAPE, $encoded) : $encoded; + } + + /** + * @param EntityDto $entityDto + */ + private function isJson(EntityDto $entityDto, string $property): bool + { + $classMetadata = $entityDto->getClassMetadata(); + + return $classMetadata->hasField($property) && Types::JSON === $classMetadata->getTypeOfField($property); + } + + private function isLike(string $comparison): bool + { + return str_contains(strtolower($comparison), 'like'); + } + + private function isNegative(string $comparison): bool + { + return str_starts_with(strtolower($comparison), 'not') || '!=' === $comparison || '<>' === $comparison; } } diff --git a/tests/Admin/Filter/LogTextFilterTest.php b/tests/Admin/Filter/LogTextFilterTest.php index f9fc135..e415477 100644 --- a/tests/Admin/Filter/LogTextFilterTest.php +++ b/tests/Admin/Filter/LogTextFilterTest.php @@ -28,6 +28,7 @@ use EasyCorp\Bundle\EasyAdminBundle\Form\Type\ComparisonType; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; #[CoversClass(LogTextFilter::class)] @@ -74,4 +75,78 @@ public function testApply(AbstractPlatform $platform, string $column): void self::assertSame('SELECT entity FROM '.LogRecord::class.' entity WHERE '.$column, $queryBuilder->getDQL()); self::assertSame('%a.csv%', $queryBuilder->getParameter('context_0')?->getValue()); } + + /** + * @return iterable + */ + public static function provideEscapedValues(): iterable + { + $column = "LOWER(CONCAT(entity.context, ''))"; + yield 'contains' => [ + ComparisonType::CONTAINS, '%data/a.csv%', + "({$column} like LOWER(:context_0) OR {$column} like LOWER(:context_0_json) ESCAPE '!')", + '%data\\/a.csv%', + ]; + yield 'not contains, escape character' => [ + ComparisonType::NOT_CONTAINS, '%a/b!%', + "({$column} not like LOWER(:context_0) AND {$column} not like LOWER(:context_0_json) ESCAPE '!')", + '%a\\/b!!%', + ]; + yield 'exactly' => [ + ComparisonType::EQ, '"é"', + "({$column} = LOWER(:context_0) OR {$column} = LOWER(:context_0_json))", + '\\"\\u00e9\\"', + ]; + yield 'not exactly' => [ + ComparisonType::NEQ, 'a/b', + "({$column} != LOWER(:context_0) AND {$column} != LOWER(:context_0_json))", + 'a\\/b', + ]; + } + + /** + * The JSON context is stored as encoded by Doctrine, kept as it is by PostgreSQL, MariaDB and SQLite: the value is + * also searched as encoded. + */ + #[DataProvider('provideEscapedValues')] + public function testApplyOnTheJsonContextWithEscapedCharacters(string $comparison, string $value, string $condition, string $encodedValue): void + { + $queryBuilder = $this->apply('context', $comparison, $value); + + self::assertSame('SELECT entity FROM '.LogRecord::class.' entity WHERE '.$condition, $queryBuilder->getDQL()); + self::assertSame($value, $queryBuilder->getParameter('context_0')?->getValue()); + self::assertSame($encodedValue, $queryBuilder->getParameter('context_0_json')?->getValue()); + } + + /** + * Not for the message (a string column), nor for a value written the same in JSON. + */ + #[TestWith(['message', '%a/b%'])] + #[TestWith(['context', '%a.csv%'])] + public function testApplyWithoutEscapedCharacters(string $property, string $value): void + { + $queryBuilder = $this->apply($property, ComparisonType::CONTAINS, $value); + + self::assertSame( + 'SELECT entity FROM '.LogRecord::class." entity WHERE LOWER(CONCAT(entity.{$property}, '')) like LOWER(:{$property}_0)", + $queryBuilder->getDQL() + ); + self::assertCount(1, $queryBuilder->getParameters()); + } + + private function apply(string $property, string $comparison, string $value): QueryBuilder + { + $queryBuilder = (new QueryBuilder($this->createStub(EntityManagerInterface::class)))->select('entity')->from(LogRecord::class, 'entity'); + $filter = LogTextFilter::new($property); + $filterDataDto = FilterDataDto::new(0, $filter->getAsDto(), 'entity', ['comparison' => $comparison, 'value' => $value]); + $metadata = new ClassMetadata(LogRecord::class); + $metadata->mapField(['fieldName' => 'message', 'type' => 'string']); + $metadata->mapField(['fieldName' => 'context', 'type' => 'json']); + /** @var EntityDto $entityDto */ + $entityDto = new EntityDto(LogRecord::class, $metadata); + + $filter->apply($queryBuilder, $filterDataDto, null, $entityDto); + + return $queryBuilder; + } } diff --git a/tests/Functional/LogRecordCrudTest.php b/tests/Functional/LogRecordCrudTest.php index 5783cbc..f57278b 100644 --- a/tests/Functional/LogRecordCrudTest.php +++ b/tests/Functional/LogRecordCrudTest.php @@ -144,6 +144,36 @@ public function testContextFilter(): void self::assertCount(2, $crawler->filter('table tbody tr')); } + /** + * The context is stored as encoded by Doctrine ("/" written "\/", non-ASCII characters "\uXXXX"...), kept as it + * is by PostgreSQL, MariaDB and SQLite (not by MySQL, binary format): a search with such characters is found too. + */ + public function testContextFilterWithCharactersEscapedInJson(): void + { + $this->login(); + $execution = $this->createExecution('test.process'); + $this->createLogRecord($execution, 'File message', ['file' => '/var/data/a.csv']); + $this->createLogRecord($execution, 'Name message', ['name' => 'Élodie "Lo" 100%']); + $this->createLogRecord($execution, 'Other message', ['file' => 'b.csv']); + + $crawler = $this->client->request('GET', '/process/log-record?filters[context][comparison]=like&filters[context][value]='.urlencode('data/a.csv')); + self::assertResponseIsSuccessful(); + self::assertCount(1, $crawler->filter('table tbody tr')); + self::assertStringContainsString('File message', $crawler->filter('table')->text()); + + $crawler = $this->client->request('GET', '/process/log-record?filters[context][comparison]=not%20like&filters[context][value]='.urlencode('data/a.csv')); + self::assertResponseIsSuccessful(); + self::assertCount(2, $crawler->filter('table tbody tr')); + self::assertStringNotContainsString('File message', $crawler->filter('table')->text()); + + foreach (['Élodie', '"Lo"'] as $value) { + $crawler = $this->client->request('GET', '/process/log-record?filters[context][comparison]=like&filters[context][value]='.urlencode($value)); + self::assertResponseIsSuccessful(); + self::assertCount(1, $crawler->filter('table tbody tr'), $value); + self::assertStringContainsString('Name message', $crawler->filter('table')->text()); + } + } + /** * Case-insensitive on every database: LIKE is case-sensitive on PostgreSQL (run there by the CI, migrations job). */