|
17 | 17 | */ |
18 | 18 |
|
19 | 19 | #pragma once |
| 20 | +#include <cstdint> |
20 | 21 | #include <functional> |
21 | 22 | #include <map> |
22 | 23 | #include <memory> |
23 | 24 | #include <optional> |
24 | 25 | #include <string> |
| 26 | +#include <utility> |
25 | 27 | #include <vector> |
26 | 28 |
|
27 | 29 | #include "paimon/predicate/predicate.h" |
28 | 30 | #include "paimon/utils/roaring_bitmap64.h" |
29 | 31 | #include "paimon/visibility.h" |
30 | 32 | namespace paimon { |
31 | | -/// A configuration structure for full-text search operations. |
| 33 | +/// A full-text search on a full-text indexed field, aligned with Java |
| 34 | +/// `org.apache.paimon.predicate.FullTextSearch`. |
32 | 35 | struct PAIMON_EXPORT FullTextSearch { |
33 | | - /// Enumeration of supported full-text search types. |
34 | | - enum class SearchType { |
35 | | - /// All terms in the query must be present (AND semantics). |
36 | | - MATCH_ALL = 1, |
37 | | - /// Any term in the query can match (OR semantics). |
38 | | - MATCH_ANY = 2, |
39 | | - /// Matches the exact sequence of words (with proximity). |
40 | | - PHRASE = 3, |
41 | | - /// Matches terms starting with the given string (e.g., "run*" → running, runner). |
42 | | - PREFIX = 4, |
43 | | - /// Supports wildcards * and ? (e.g., "ap*e", "app?e" -> "apple"). |
44 | | - WILDCARD = 5, |
45 | | - /// Default/fallback type for unrecognized or invalid queries. |
46 | | - UNKNOWN = 128 |
47 | | - }; |
48 | | - |
49 | | - FullTextSearch(const std::string& _field_name, std::optional<int32_t> _limit, |
50 | | - const std::string& _query, const SearchType& _search_type, |
51 | | - const std::optional<RoaringBitmap64>& _pre_filter, bool _with_score = false, |
52 | | - std::optional<float> _min_score = std::nullopt) |
| 36 | + FullTextSearch(const std::string& _field_name, const std::string& _query, int32_t _limit, |
| 37 | + std::optional<RoaringBitmap64> _include_row_ids = std::nullopt) |
53 | 38 | : field_name(_field_name), |
54 | | - limit(_limit), |
55 | 39 | query(_query), |
56 | | - search_type(_search_type), |
57 | | - pre_filter(_pre_filter), |
58 | | - with_score(_with_score), |
59 | | - min_score(_min_score) {} |
| 40 | + limit(_limit), |
| 41 | + include_row_ids(std::move(_include_row_ids)) {} |
60 | 42 |
|
61 | | - std::shared_ptr<FullTextSearch> ReplacePreFilter( |
62 | | - const std::optional<RoaringBitmap64>& _pre_filter) const { |
63 | | - return std::make_shared<FullTextSearch>(field_name, limit, query, search_type, _pre_filter, |
64 | | - with_score, min_score); |
| 43 | + /// Returns a copy of this search for an index shard covering the global row ids [`from`, `to`] |
| 44 | + /// (both inclusive), as Java `FullTextSearch#offsetRange` does: `include_row_ids` is clipped to |
| 45 | + /// the range and shifted to shard-local row ids by subtracting `from`. |
| 46 | + std::shared_ptr<FullTextSearch> OffsetRange(int64_t from, int64_t to) const { |
| 47 | + if (!include_row_ids) { |
| 48 | + return std::make_shared<FullTextSearch>(*this); |
| 49 | + } |
| 50 | + const RoaringBitmap64& global_row_ids = include_row_ids.value(); |
| 51 | + RoaringBitmap64 local_row_ids; |
| 52 | + for (auto iter = global_row_ids.EqualOrLarger(from), end = global_row_ids.End(); |
| 53 | + iter != end && *iter <= to; ++iter) { |
| 54 | + local_row_ids.Add(*iter - from); |
| 55 | + } |
| 56 | + return std::make_shared<FullTextSearch>(field_name, query, limit, std::move(local_row_ids)); |
65 | 57 | } |
66 | 58 |
|
67 | 59 | /// Name of the field to search within (must be a full-text indexed field). |
68 | 60 | std::string field_name; |
69 | | - /// Maximum number of documents to return. Purely a truncation switch, |
70 | | - /// orthogonal to `with_score`: set `with_score = true` to get relevance |
71 | | - /// scores; a non-empty `limit` does not by itself imply scoring. |
72 | | - /// |
73 | | - /// The `full-text` index requires a positive `limit` and always returns at most `limit` rows |
74 | | - /// with the highest scores. |
75 | | - std::optional<int32_t> limit; |
76 | | - /// The query string to search for. The interpretation depends on search_type: |
77 | | - /// |
78 | | - /// - For MATCH_ALL/MATCH_ANY: keywords are split into terms using the **same analyzer as |
79 | | - /// indexing**. |
80 | | - /// Example: "Hello World" → terms ["hello", "world"] (after lowercasing and tokenization). |
| 61 | + /// The JSON DSL query defined by the `paimon-full-text` engine (`core/src/query.rs` in |
| 62 | + /// apache/paimon-full-text). The root object holds exactly one query: |
81 | 63 | /// |
82 | | - /// - For PHRASE: matches the exact word sequence (with optional slop). Also be analyzed. |
| 64 | + /// - `match`: `query` (alias `terms`), optional `column`, `operator` (`Or` by default, or |
| 65 | + /// `And`), `boost` (1.0), `fuzziness` (0, a number, or `"auto"` or `null` for automatic), |
| 66 | + /// `max_expansions` (50) and `prefix_length` (0). The text is analyzed with the analyzer |
| 67 | + /// used at indexing time. |
| 68 | + /// - `multi_match`: `query`, `columns`, `boosts`, `operator`, `fuzziness`, `max_expansions` |
| 69 | + /// and `prefix_length`. |
| 70 | + /// - `match_phrase` (alias `phrase`): `query`, optional `column` and `slop` (0). |
| 71 | + /// - `boolean`: `must`, `should` and `must_not` lists of queries, and `queries` as |
| 72 | + /// `[occur, query]` pairs. |
| 73 | + /// - `boost`: `positive`, `negative` and `negative_boost` (0.5). |
83 | 74 | /// |
84 | | - /// - For PREFIX: matches terms starting with the given string (e.g., "run" → running, runner). |
85 | | - /// The query is not tokenized or filtered for stop words. The original prefix is retained, |
86 | | - /// and a prefix consisting entirely of ASCII letters and digits is also matched using the |
87 | | - /// lowercase case-normalization applied to pure ASCII terms at indexing time. |
| 75 | + /// Examples: `{"match":{"query":"paimon lake","operator":"And"}}` and |
| 76 | + /// `{"match_phrase":{"query":"paimon lake","slop":1}}`. |
88 | 77 | /// |
89 | | - /// - For WILDCARD: supports wildcards * and ? (e.g., "ap*e", "app?e"). |
90 | | - /// The query is not tokenized or filtered for stop words. The wildcard operators are |
91 | | - /// preserved. Both the original pattern and an alternative with each ASCII alphanumeric |
92 | | - /// fragment lowercased are matched, covering pure ASCII and mixed ASCII/non-ASCII terms. |
| 78 | + /// `field_name` selects the table column. DSL column names refer to fields inside the index: |
| 79 | + /// the `full-text` backend defaults to `text`, while `lucene-fts` uses `field_name`. |
93 | 80 | /// |
94 | | - /// - For the `full-text` index, `search_type` is ignored and the query is passed to the |
95 | | - /// native engine unchanged, so it must be a JSON DSL query such as |
96 | | - /// `{"match":{"query":"paimon lake"}}`. |
97 | | - /// |
98 | | - /// @note Analyzer consistency between indexing and querying is critical for correctness. |
| 81 | + /// The `full-text` index passes the query to the native engine unchanged. The `lucene-fts` |
| 82 | + /// index translates `match`, `multi_match`, `match_phrase` and `boolean` into Lucene queries |
| 83 | + /// and rejects `boost` queries and a `fuzziness` other than 0. An invalid query is reported as |
| 84 | + /// an error `Status`. |
99 | 85 | std::string query; |
100 | | - /// Type of search to perform. |
101 | | - SearchType search_type; |
102 | | - /// A pre-filter based on **global row IDs**, implemented by leveraging another global index. |
103 | | - /// Only rows whose global row ID is present in `pre_filter` will be included during search. |
104 | | - /// If not set, all rows will be included. |
105 | | - std::optional<RoaringBitmap64> pre_filter; |
106 | | - /// Whether to compute and return relevance scores (e.g. BM25). The 4-path matrix: |
107 | | - /// - `with_score=false, limit=nullopt` → BitmapGlobalIndexResult (all rows, no score) |
108 | | - /// - `with_score=false, limit=N` → BitmapGlobalIndexResult (any N matches, unscored) |
109 | | - /// - `with_score=true, limit=nullopt` → BitmapScoredGlobalIndexResult (all rows + all scores) |
110 | | - /// - `with_score=true, limit=N` → BitmapScoredGlobalIndexResult (top-N by score + |
111 | | - /// scores) |
112 | | - /// |
113 | | - /// For plain `LIMIT N` without ORDER BY (the common case when an online |
114 | | - /// engine, e.g. StarRocks, pushes down a predicate) set `with_score=false, |
115 | | - /// limit=N` — the unscored fast path, where the backend provides one. For top-N by relevance |
116 | | - /// use `with_score=true, limit=N` and drop the scores in the caller if unneeded. |
117 | | - /// |
118 | | - /// The built-in indexes deviate from this matrix: |
119 | | - /// - `lucene` ignores `with_score`: a search with `limit` returns the top `limit` rows with |
120 | | - /// their scores, and a search without `limit` returns all matching rows without scores. |
121 | | - /// - `full-text` requires `limit` and always ranks the matching rows by BM25 score, keeping the |
122 | | - /// top `limit` rows. `with_score` only selects whether their scores are returned. |
123 | | - /// |
124 | | - /// Default is `false`, which avoids score computation on backends with an unscored path. |
125 | | - bool with_score = false; |
126 | | - /// Minimum relevance-score threshold (exclusive); results with score ≤ this value are |
127 | | - /// excluded. The score is whatever the backend's similarity produces (e.g. BM25 for |
128 | | - /// full-text, classic TF-IDF for lucene), so a threshold is not directly comparable across |
129 | | - /// backends. |
130 | | - /// |
131 | | - /// Only the `full-text` index supports it; the `lucene` index rejects a search that sets it. |
132 | | - /// The `full-text` index applies it to the top `limit` rows returned by the engine, so a |
133 | | - /// search can return fewer than `limit` rows. |
134 | | - /// Default is nullopt (no threshold filtering). |
135 | | - std::optional<float> min_score; |
| 86 | + /// Maximum number of highest-scoring rows requested from each index shard; must be positive. |
| 87 | + int32_t limit; |
| 88 | + /// The **global row ids** to search within, aligned with Java `includeRowIds`. Only these rows |
| 89 | + /// are ranked, so an empty set matches no rows. If not set, all rows are searched. |
| 90 | + std::optional<RoaringBitmap64> include_row_ids; |
136 | 91 | }; |
137 | 92 | } // namespace paimon |
0 commit comments