Stable 25.8 Backport of #93663 - Fix trash in TokenExtractor

src/Interpreters/ITokenExtractor.cpp

@@ -55,6 +55,8 @@ bool NgramTokenExtractor::nextInString(const char * data, size_t length, size_t
         size_t sz = UTF8::seqLength(static_cast<UInt8>(data[*token_start + *token_length]));
         *token_length += sz;
     }
+    /// The sequence length is determined by the first character, but we should always check it does not go beyond the buffer length.
+    *token_length = std::min(*token_length, length - *token_start);
     *pos += UTF8::seqLength(static_cast<UInt8>(data[*pos]));
     return code_points == n;
 }
@@ -90,7 +92,8 @@ bool NgramTokenExtractor::nextInStringLike(const char * data, size_t length, siz
         else
         {
             const size_t sz = UTF8::seqLength(static_cast<UInt8>(data[i]));
-            for (size_t j = 0; j < sz; ++j)
+            /// The sequence length is determined by the first character, but we should always check it does not go beyond the buffer length.
+            for (size_t j = 0; j < sz && i + j < length; ++j)
                 token += data[i + j];
             i += sz;
             ++code_points;
@@ -254,7 +257,8 @@ bool DefaultTokenExtractor::nextInStringLike(const char * data, size_t length, s
         else
         {
             const size_t sz = UTF8::seqLength(static_cast<UInt8>(data[*pos]));
-            for (size_t j = 0; j < sz; ++j)
+            /// The sequence length is determined by the first character, but we should always check it does not go beyond the buffer length.
+            for (size_t j = 0; j < sz && *pos < length; ++j)
             {
                 token += data[*pos];
                 ++*pos;

tests/queries/0_stateless/03784_msan_token_iterator.sql

@@ -0,0 +1,4 @@
+DROP TABLE IF EXISTS t0;
+CREATE TABLE t0 (c0 Int, INDEX i1 substring(sipHash128(c0), 1, 4) TYPE ngrambf_v1(1, 8, 1, 0)) ENGINE = MergeTree ORDER BY ();
+INSERT INTO TABLE t0 (c0) SELECT number FROM numbers(28);
+DROP TABLE t0;

tests/queries/0_stateless/05317_ngrambf_truncated_utf8_overrun.reference

@@ -0,0 +1,2 @@
+0
+Granules: 0/2

tests/queries/0_stateless/05317_ngrambf_truncated_utf8_overrun.sql

@@ -0,0 +1,16 @@
+-- `ngrambf_v1` must not hash bytes past the end of a string that ends with a truncated UTF-8 sequence.
+-- With `index_granularity = 1` the bytes after the first row are the second row, so a buggy index
+-- contains the n-gram 'ab\xE4xx' and cannot skip the first granule for the search below.
+SET use_skip_indexes = 1;
+SET use_query_condition_cache = 0;
+
+DROP TABLE IF EXISTS t_ngram_truncated_utf8;
+CREATE TABLE t_ngram_truncated_utf8 (s String, INDEX idx s TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 1)
+ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1, index_granularity_bytes = '10Mi';
+INSERT INTO t_ngram_truncated_utf8 SELECT arrayJoin(['ab\xE4', 'xx']);
+
+SELECT count() FROM t_ngram_truncated_utf8 WHERE s LIKE '%ab\xE4xx%';
+SELECT trimBoth(explain) FROM (EXPLAIN indexes = 1 SELECT count() FROM t_ngram_truncated_utf8 WHERE s LIKE '%ab\xE4xx%')
+WHERE explain LIKE '%Granules:%';
+
+DROP TABLE t_ngram_truncated_utf8;