diff --git a/src/Interpreters/ITokenExtractor.cpp b/src/Interpreters/ITokenExtractor.cpp index fc77a7507cbcb..4faa7dd5578fe 100644 --- a/src/Interpreters/ITokenExtractor.cpp +++ b/src/Interpreters/ITokenExtractor.cpp @@ -55,6 +55,8 @@ bool NgramTokenExtractor::nextInString(const char * data, size_t length, size_t size_t sz = UTF8::seqLength(static_cast(data[*token_start + *token_length])); *token_length += sz; } + /// The sequence length is determined by the first character, but we should always check it does not go beyond the buffer length. + *token_length = std::min(*token_length, length - *token_start); *pos += UTF8::seqLength(static_cast(data[*pos])); return code_points == n; } @@ -90,7 +92,8 @@ bool NgramTokenExtractor::nextInStringLike(const char * data, size_t length, siz else { const size_t sz = UTF8::seqLength(static_cast(data[i])); - for (size_t j = 0; j < sz; ++j) + /// The sequence length is determined by the first character, but we should always check it does not go beyond the buffer length. + for (size_t j = 0; j < sz && i + j < length; ++j) token += data[i + j]; i += sz; ++code_points; @@ -254,7 +257,8 @@ bool DefaultTokenExtractor::nextInStringLike(const char * data, size_t length, s else { const size_t sz = UTF8::seqLength(static_cast(data[*pos])); - for (size_t j = 0; j < sz; ++j) + /// The sequence length is determined by the first character, but we should always check it does not go beyond the buffer length. + for (size_t j = 0; j < sz && *pos < length; ++j) { token += data[*pos]; ++*pos; diff --git a/tests/queries/0_stateless/03784_msan_token_iterator.reference b/tests/queries/0_stateless/03784_msan_token_iterator.reference new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/queries/0_stateless/03784_msan_token_iterator.sql b/tests/queries/0_stateless/03784_msan_token_iterator.sql new file mode 100644 index 0000000000000..925abbf847a8b --- /dev/null +++ b/tests/queries/0_stateless/03784_msan_token_iterator.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS t0; +CREATE TABLE t0 (c0 Int, INDEX i1 substring(sipHash128(c0), 1, 4) TYPE ngrambf_v1(1, 8, 1, 0)) ENGINE = MergeTree ORDER BY (); +INSERT INTO TABLE t0 (c0) SELECT number FROM numbers(28); +DROP TABLE t0; diff --git a/tests/queries/0_stateless/05317_ngrambf_truncated_utf8_overrun.reference b/tests/queries/0_stateless/05317_ngrambf_truncated_utf8_overrun.reference new file mode 100644 index 0000000000000..cd577cb0e1682 --- /dev/null +++ b/tests/queries/0_stateless/05317_ngrambf_truncated_utf8_overrun.reference @@ -0,0 +1,2 @@ +0 +Granules: 0/2 diff --git a/tests/queries/0_stateless/05317_ngrambf_truncated_utf8_overrun.sql b/tests/queries/0_stateless/05317_ngrambf_truncated_utf8_overrun.sql new file mode 100644 index 0000000000000..c7c9305be4b74 --- /dev/null +++ b/tests/queries/0_stateless/05317_ngrambf_truncated_utf8_overrun.sql @@ -0,0 +1,16 @@ +-- `ngrambf_v1` must not hash bytes past the end of a string that ends with a truncated UTF-8 sequence. +-- With `index_granularity = 1` the bytes after the first row are the second row, so a buggy index +-- contains the n-gram 'ab\xE4xx' and cannot skip the first granule for the search below. +SET use_skip_indexes = 1; +SET use_query_condition_cache = 0; + +DROP TABLE IF EXISTS t_ngram_truncated_utf8; +CREATE TABLE t_ngram_truncated_utf8 (s String, INDEX idx s TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 1) +ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1, index_granularity_bytes = '10Mi'; +INSERT INTO t_ngram_truncated_utf8 SELECT arrayJoin(['ab\xE4', 'xx']); + +SELECT count() FROM t_ngram_truncated_utf8 WHERE s LIKE '%ab\xE4xx%'; +SELECT trimBoth(explain) FROM (EXPLAIN indexes = 1 SELECT count() FROM t_ngram_truncated_utf8 WHERE s LIKE '%ab\xE4xx%') +WHERE explain LIKE '%Granules:%'; + +DROP TABLE t_ngram_truncated_utf8;