From c8d03d076b5e380ce4283b5c8172667e7216816d Mon Sep 17 00:00:00 2001 From: Nicolas Joubert Date: Fri, 6 Nov 2026 14:15:26 +0100 Subject: [PATCH] #305 - Database logs: limit the strings of the context to 65536 characters Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +++ .../04-process_executions_and_logs.md | 6 ++++- src/Entity/LogRecord.php | 25 +++++++++++++++++-- tests/Entity/LogRecordTest.php | 21 ++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54808110..d3b03aff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ Latest ------ +## Fixes +* [#305](https://github.com/cleverage/ui-process-bundle/issues/305) Database logs: the strings of the context of a log record are limited to 65536 characters (followed by ` [...] (over 65536 characters, total, truncated)`), as its arrays (#271): a larger string (e.g. the content of a file) over the maximum packet size of the database made the insertion of the log records written with it fail, and closed the connection shared with the process tasks. Update documentation, add tests. + v3.6 ------ diff --git a/docs/reference/04-process_executions_and_logs.md b/docs/reference/04-process_executions_and_logs.md index f5cafc55..3414a31d 100644 --- a/docs/reference/04-process_executions_and_logs.md +++ b/docs/reference/04-process_executions_and_logs.md @@ -193,7 +193,11 @@ and its values which cannot be encoded (e.g. a resource) replaced by `null`. As `Over 1000 items ( total), aborting normalization`) and to 9 levels deep, the context being the first one (the deeper values are replaced by `Over 9 levels deep, aborting normalization`): an unbounded context (e.g. the output of an aggregator) was stored whole, and over the maximum packet size of the database, it made the insertion of the log records -written with it fail. The objects of the context are stored with their public properties, without limit. +written with it fail. The strings of the context are limited to 65536 characters (Unicode code points), at any depth: +a longer one (e.g. the content of a file) is truncated and followed by ` [...] (over 65536 characters, total, +truncated)` (the log file has it whole); over the maximum packet size of the database (e.g. `max_allowed_packet` of +MySQL), its log record and the ones written with it were lost, and the connection, shared with the process tasks, was +closed. The objects of the context are stored with their public properties, without limit. Records are buffered and inserted every 500 records, and at the end (or failure) of the process. A record counted in the report of the execution (from `report_increment_level`, warnings by default) is inserted right away with the diff --git a/src/Entity/LogRecord.php b/src/Entity/LogRecord.php index ce114722..5524804c 100644 --- a/src/Entity/LogRecord.php +++ b/src/Entity/LogRecord.php @@ -27,6 +27,8 @@ class LogRecord /** Limits of the context, as in the log file (Monolog NormalizerFormatter) */ private const MAX_ITEMS = 1000; private const MAX_DEPTH = 9; + /** Characters (code points) of a string of the context */ + private const MAX_STRING_LENGTH = 65536; #[ORM\Id] #[ORM\GeneratedValue] @@ -70,7 +72,8 @@ public function __construct( /** * The limits of the log file (Monolog NormalizerFormatter): at most 1000 items per array, 9 levels deep. An unbounded * context (e.g. the output of an aggregator) was stored whole, and over the maximum packet size of the database, it - * made the insertion of the log records written with it fail. + * made the insertion of the log records written with it fail. The strings are limited to 65536 characters too (e.g. + * the content of a file): not by the log file. * * @param array $data * @@ -88,13 +91,31 @@ private function limit(array $data, int $depth): array if ($depth + 1 > self::MAX_DEPTH) { $limited[$key] = \sprintf('Over %d levels deep, aborting normalization', self::MAX_DEPTH); } else { - $limited[$key] = \is_array($value) ? $this->limit($value, $depth + 1) : $value; + $limited[$key] = match (true) { + \is_array($value) => $this->limit($value, $depth + 1), + \is_string($value) => $this->limitString($value), + default => $value, + }; } } return $limited; } + /** + * Truncated to MAX_STRING_LENGTH characters, followed by the number of characters of the whole string. Cut on a + * character: a valid UTF-8 string stays valid. + */ + private function limitString(string $value): string + { + // A string of at most MAX_STRING_LENGTH bytes has at most as many characters: not counted + if (\strlen($value) <= self::MAX_STRING_LENGTH || ($length = mb_strlen($value, 'UTF-8')) <= self::MAX_STRING_LENGTH) { + return $value; + } + + return mb_substr($value, 0, self::MAX_STRING_LENGTH, 'UTF-8').\sprintf(' [...] (over %d characters, %d total, truncated)', self::MAX_STRING_LENGTH, $length); + } + /** * Invalid UTF-8 sequences replaced (by "?"), NUL characters removed (rejected by PostgreSQL, or the value truncated * there), truncated to the length of the column: a number of characters (code points) in the database, not of diff --git a/tests/Entity/LogRecordTest.php b/tests/Entity/LogRecordTest.php index 9722231d..41eee87b 100644 --- a/tests/Entity/LogRecordTest.php +++ b/tests/Entity/LogRecordTest.php @@ -126,6 +126,27 @@ public function testEncodableContextIsUnchanged(): void self::assertSame($context, $this->createLogRecord($context)->context); } + /** + * The strings of the context are limited to 65536 characters (not bytes), at any depth: a larger one (e.g. the + * content of a file) could exceed the maximum packet size of the database, the log records were then lost. + */ + public function testContextStringsAreLimitedTo65536Characters(): void + { + $limit = str_repeat('é', 65536); + $context = $this->createLogRecord([ + 'content' => str_repeat('a', 1_000_000), + 'nested' => ['multibyte' => $limit.'éé', 'limit' => $limit, 'bytes' => str_repeat('a', 65536)], + 'number' => 1, + ])->context; + + self::assertSame(str_repeat('a', 65536).' [...] (over 65536 characters, 1000000 total, truncated)', $context['content']); + self::assertSame( + ['multibyte' => $limit.' [...] (over 65536 characters, 65538 total, truncated)', 'limit' => $limit, 'bytes' => str_repeat('a', 65536)], + $context['nested'] + ); + self::assertSame(1, $context['number']); + } + /** * The limits of the log file (Monolog NormalizerFormatter): at most 1000 items per array. */