Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 30 additions & 3 deletions java/lance-jni/src/blocking_scanner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ use lance_index::scalar::FullTextSearchQuery;
use lance_index::scalar::inverted::{
DocumentGranularity,
query::{
BooleanQuery as FtsBooleanQuery, BoostQuery as FtsBoostQuery, FtsQuery,
MatchQuery as FtsMatchQuery, MultiMatchQuery as FtsMultiMatchQuery, Occur as FtsOccur,
PhraseQuery as FtsPhraseQuery,
BooleanQuery as FtsBooleanQuery, BoostQuery as FtsBoostQuery,
CombinedFieldsQuery as FtsCombinedFieldsQuery, FtsQuery, MatchQuery as FtsMatchQuery,
MultiMatchQuery as FtsMultiMatchQuery, Occur as FtsOccur, PhraseQuery as FtsPhraseQuery,
},
};
use lance_io::ffi::to_ffi_arrow_array_stream;
Expand Down Expand Up @@ -174,6 +174,33 @@ pub(crate) fn build_full_text_search_query<'a>(

Ok(FtsQuery::MultiMatch(query))
}
"COMBINED_FIELDS" => {
let query_text = env.get_string_from_method(&java_obj, "getQueryText")?;
let columns: Vec<String> =
import_vec_from_method(env, &java_obj, "getColumns", |env, elem| {
let jstr = JString::from(elem);
let value: String = env.get_string(&jstr)?.into();
Ok(value)
})?;

let boosts: Option<Vec<f32>> =
env.get_optional_from_method(&java_obj, "getBoosts", |env, list_obj| {
import_vec_to_rust(env, &list_obj, |env, elem| {
env.get_f32_from_method(&elem, "floatValue")
})
})?;
let operator = env.get_fts_operator_from_method(&java_obj)?;

// Column uniqueness and boost (>= 1) validation live in the Rust core;
// `?` surfaces those errors across the JNI boundary.
let mut query = FtsCombinedFieldsQuery::try_new(query_text, columns)?;
if let Some(boosts) = boosts {
query = query.try_with_boosts(boosts)?;
}
query = query.with_operator(operator);

Ok(FtsQuery::CombinedFields(query))
}
"BOOST" => {
let positive_obj = env
.call_method(
Expand Down
97 changes: 97 additions & 0 deletions java/src/main/java/org/lance/ipc/FullTextQuery.java
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ public enum Type {
MATCH_PHRASE,
BOOST,
MULTI_MATCH,
COMBINED_FIELDS,
BOOLEAN
}

Expand Down Expand Up @@ -156,6 +157,15 @@ public static FullTextQuery multiMatch(
return new MultiMatchQuery(queryText, columns, boosts, operator);
}

public static FullTextQuery combinedFields(String queryText, List<String> columns) {
return combinedFields(queryText, columns, null, Operator.OR);
}

public static FullTextQuery combinedFields(
String queryText, List<String> columns, List<Float> boosts, Operator operator) {
return new CombinedFieldsQuery(queryText, columns, boosts, operator);
}

public static FullTextQuery boost(FullTextQuery positive, FullTextQuery negative) {
return boost(positive, negative, 0.5f);
}
Expand Down Expand Up @@ -431,6 +441,93 @@ public String toString() {
}
}

/**
* Combined-fields (BM25F) query across multiple columns.
*
* <p>Unlike {@link MultiMatchQuery}, which scores each column independently and keeps the best
* field, this query treats the target columns as a single virtual field so that term statistics
* are blended across fields (Lucene {@code CombinedFieldQuery}). A term that is rare in one field
* but common in another then scores consistently, and a single query term can match across fields
* (for example a first name in one column and a last name in another).
*
* <p>Target columns must be unique and share the same tokenizer/index configuration. When {@code
* boosts} is given, its length must equal the number of columns and each per-column weight must
* be finite and {@code >= 1} (fractional weights allowed); when {@code null}, every column
* defaults to {@code 1.0}. A {@code null} operator defaults to {@link Operator#OR}. These
* constraints are validated in the Rust core and surface as an exception when the query runs.
*/
public static final class CombinedFieldsQuery extends FullTextQuery {
private final String queryText;
private final List<String> columns;
private final Optional<List<Float>> boosts;
private final Operator operator;

CombinedFieldsQuery(
String queryText, List<String> columns, List<Float> boosts, Operator operator) {
Preconditions.checkArgument(
queryText != null && !queryText.isEmpty(), "queryText must not be null or empty");
Preconditions.checkArgument(
columns != null && !columns.isEmpty(), "columns must not be null or empty");

this.queryText = queryText;
this.columns =
Collections.unmodifiableList(new java.util.ArrayList<>(Objects.requireNonNull(columns)));
this.boosts =
boosts == null
? Optional.empty()
: Optional.of(Collections.unmodifiableList(new java.util.ArrayList<>(boosts)));
this.operator = operator == null ? Operator.OR : operator;
}

@Override
public Type getType() {
return Type.COMBINED_FIELDS;
}

public String getQueryText() {
return queryText;
}

public List<String> getColumns() {
return columns;
}

public Optional<List<Float>> getBoosts() {
return boosts;
}

public Operator getOperator() {
return operator;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof CombinedFieldsQuery)) return false;
CombinedFieldsQuery other = (CombinedFieldsQuery) o;
return operator == other.operator
&& Objects.equals(queryText, other.queryText)
&& Objects.equals(columns, other.columns)
&& Objects.equals(boosts, other.boosts);
}

@Override
public int hashCode() {
return Objects.hash(queryText, columns, operator, boosts);
}

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("type", getType())
.add("queryText", queryText)
.add("columns", columns)
.add("boosts", boosts)
.add("operator", operator)
.toString();
}
}

/** Boost query combining positive and negative queries. */
public static final class BoostQuery extends FullTextQuery {
private final FullTextQuery positive;
Expand Down
95 changes: 95 additions & 0 deletions java/src/test/java/org/lance/ipc/FullTextQueryTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,101 @@ void testMultiMatchQueryInequalityDifferentBoosts() {
assertFalse(a.equals(b), "MultiMatchQuery instances with different boosts must not be equal");
}

@Test
void testCombinedFieldsWithoutBoosts() {
FullTextQuery.CombinedFieldsQuery q =
(FullTextQuery.CombinedFieldsQuery)
FullTextQuery.combinedFields("hello", Arrays.asList("title", "body"));

assertEquals(FullTextQuery.Type.COMBINED_FIELDS, q.getType());
assertEquals("hello", q.getQueryText());
assertEquals(Arrays.asList("title", "body"), q.getColumns());
assertFalse(q.getBoosts().isPresent());
assertEquals(FullTextQuery.Operator.OR, q.getOperator());
}

@Test
void testCombinedFieldsWithBoosts() {
FullTextQuery.CombinedFieldsQuery q =
(FullTextQuery.CombinedFieldsQuery)
FullTextQuery.combinedFields(
"hello",
Arrays.asList("title", "body"),
Arrays.asList(2.0f, 1.0f),
FullTextQuery.Operator.AND);

assertEquals(FullTextQuery.Type.COMBINED_FIELDS, q.getType());
assertTrue(q.getBoosts().isPresent());
assertEquals(2, q.getBoosts().get().size());
assertEquals(2.0f, q.getBoosts().get().get(0));
assertEquals(1.0f, q.getBoosts().get().get(1));
assertEquals(FullTextQuery.Operator.AND, q.getOperator());
assertNotNull(q.toString());
}

@Test
void testCombinedFieldsQueryEquality() {
FullTextQuery a = FullTextQuery.combinedFields("hello", Arrays.asList("title", "body"));
FullTextQuery b = FullTextQuery.combinedFields("hello", Arrays.asList("title", "body"));
assertEquals(a, b, "Two CombinedFieldsQuery instances with the same fields must be equal");
assertEquals(a.hashCode(), b.hashCode());
}

@Test
void testCombinedFieldsQueryInequalityDifferentOperator() {
FullTextQuery a =
FullTextQuery.combinedFields(
"hello", Arrays.asList("title", "body"), null, FullTextQuery.Operator.AND);
FullTextQuery b =
FullTextQuery.combinedFields(
"hello", Arrays.asList("title", "body"), null, FullTextQuery.Operator.OR);
assertFalse(
a.equals(b), "CombinedFieldsQuery instances with different operator must not be equal");
}

@Test
void testCombinedFieldsQueryInequalityDifferentBoosts() {
FullTextQuery a =
FullTextQuery.combinedFields(
"hello",
Arrays.asList("title", "body"),
Arrays.asList(2.0f, 1.0f),
FullTextQuery.Operator.OR);
FullTextQuery b =
FullTextQuery.combinedFields(
"hello",
Arrays.asList("title", "body"),
Arrays.asList(1.0f, 1.0f),
FullTextQuery.Operator.OR);
assertFalse(
a.equals(b), "CombinedFieldsQuery instances with different boosts must not be equal");
}

@Test
void testCombinedFieldsNotEqualMultiMatch() {
FullTextQuery combined = FullTextQuery.combinedFields("hello", Arrays.asList("title", "body"));
FullTextQuery multi = FullTextQuery.multiMatch("hello", Arrays.asList("title", "body"));
assertFalse(
combined.equals(multi),
"CombinedFieldsQuery and MultiMatchQuery must not be equal even with same fields");
}

@Test
void testCombinedFieldsBoostsDefensivelyCopied() {
java.util.List<Float> boosts = new java.util.ArrayList<>(Arrays.asList(2.0f, 1.0f));
FullTextQuery.CombinedFieldsQuery q =
(FullTextQuery.CombinedFieldsQuery)
FullTextQuery.combinedFields(
"hello", Arrays.asList("title", "body"), boosts, FullTextQuery.Operator.OR);

// Mutating the caller-provided list must not change the stored query.
boosts.set(0, 9.0f);
boosts.clear();

assertTrue(q.getBoosts().isPresent());
assertEquals(Arrays.asList(2.0f, 1.0f), q.getBoosts().get());
}

@Test
void testDifferentSubtypesNotEqual() {
FullTextQuery match = FullTextQuery.match("hello", "body");
Expand Down
Loading
Loading