diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 0ce6ae7eb..0fc4425fa 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -423,11 +423,18 @@ static const tool_def_t TOOLS[] = { "property columns via " "fields (e.g. [\"complexity\",\"signature\",\"docstring\"]); format=\"json\" returns " "the SAME tree model as structured JSON. " - "PAGINATION: results are capped at limit (default 50). The response always includes " - "'total' (full match count before limit) and 'has_more' (true when total > " - "offset+returned). Detect truncation with has_more, then page by re-calling with " - "offset=offset+limit until has_more is false. Narrow first via label/file_pattern/" - "min_degree before paginating large result sets.", + "PAGINATION: Structural results are capped at limit (default 50); top-level 'total' is " + "the full structural match count and top-level 'has_more' reports whether structural rows " + "remain. Semantic results have no score threshold: they are ranked inside a fixed " + "250-candidate window before offset/limit slicing. The first semantic keyword selects that " + "bounded window; remaining keywords rerank it, so keyword order can affect saturated " + "results. JSON reports semantic pagination under " + "semantic.total, semantic.has_more, and semantic.truncated; tree output uses " + "semantic_total, semantic_has_more, and semantic_truncated. Semantic has_more only means " + "another page remains inside that window. Semantic truncated means the window filled and " + "corpus completeness is unknown. Advance offset by the number of semantic rows returned " + "while semantic has_more is true. If semantic truncated is true, narrow or rephrase the " + "query, or use another search mode, before claiming completeness.", "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}," "\"query\":{\"type\":\"string\",\"description\":\"Natural-language or keyword full-text " "search using BM25 ranking. Tokens are split on whitespace; camelCase identifiers are " @@ -443,13 +450,14 @@ static const tool_def_t TOOLS[] = { "\"type\":\"array\",\"items\":{\"type\":\"string\"},\"description\":\"MUST be an ARRAY of " "keyword strings (e.g. [\\\"send\\\",\\\"pubsub\\\",\\\"publish\\\"]) — NOT a single string. " "Each keyword is scored independently via per-keyword min-cosine; results reflect functions " - "that score well on ALL keywords. Requires moderate/full index mode. Results appear in the " - "'semantic_results' field (separate from 'results').\"},\"limit\":{\"type\":" - "\"integer\",\"description\":\"Max results per call. Default 50. Response carries " - "'total' (full match count) and 'has_more' (true if truncated) so callers can " - "detect the limit and paginate.\"},\"offset\":{\"type\":\"integer\",\"default\":0," - "\"description\":\"Skip the first N matching nodes. Combine with 'limit' to page: " - "increment offset by limit and re-call while has_more is true.\"}," + "that score well on ALL keywords; no score threshold is applied. Requires moderate/full " + "index mode. Results are ranked inside the fixed 250-candidate window before pagination; " + "see the tool description for semantic metadata and completeness guidance.\"},\"limit\":{" + "\"type\":\"integer\",\"description\":\"Max rows per result set per call. Default 50. " + "Structural and semantic pagination have separate metadata as described above.\"}," + "\"offset\":{\"type\":\"integer\",\"default\":0,\"description\":\"Skip the first N matching " + "nodes in each requested result set. Combine with 'limit' to page, advancing by the number " + "of rows returned while that result set's has_more is true.\"}," "\"format\":{\"type\":\"string\",\"enum\":[\"tree\",\"json\"],\"default\":\"tree\"," "\"description\":\"Response encoding. tree (default): prefix-grouped text rows. " "json: the SAME tree model as structured JSON (groups + column-ordered row arrays).\"}," @@ -3197,19 +3205,18 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu return json; } -/* Extract keyword strings from a yyjson array into `keywords`. Returns the - * number of strings copied (capped at `max_out`). */ +/* Extract keyword strings from a yyjson array into `keywords`. Returns the + * number copied (capped at `max_out`), or -1 when any element is not a string. */ static int extract_semantic_keywords(yyjson_val *sq_val, const char **keywords, int max_out) { - int kw_count = (int)yyjson_arr_size(sq_val); - if (kw_count > max_out) { - kw_count = max_out; - } size_t kw_idx = 0; size_t kw_max = 0; yyjson_val *kw_val; int ki = 0; yyjson_arr_foreach(sq_val, kw_idx, kw_max, kw_val) { - if (ki < kw_count && yyjson_is_str(kw_val)) { + if (!yyjson_is_str(kw_val)) { + return -1; + } + if (ki < max_out) { keywords[ki++] = yyjson_get_str(kw_val); } } @@ -3219,8 +3226,12 @@ static int extract_semantic_keywords(yyjson_val *sq_val, const char **keywords, /* Emit vector-search hits in the json-tree model: "semantic": {cols, rows} * — score order preserved (ranked output is never regrouped). */ static void emit_semantic_results(yyjson_mut_doc *doc, yyjson_mut_val *root, - cbm_vector_result_t *vresults, int vcount) { + const cbm_vector_result_t *vresults, int vcount, int total, + bool has_more, bool truncated) { yyjson_mut_val *sem = yyjson_mut_obj(doc); + yyjson_mut_obj_add_int(doc, sem, "total", total); + yyjson_mut_obj_add_bool(doc, sem, "has_more", has_more); + yyjson_mut_obj_add_bool(doc, sem, "truncated", truncated); yyjson_mut_val *scols = yyjson_mut_arr(doc); static const char *const sem_cols[] = {"qn", "label", "file", "score"}; for (size_t ci = 0; ci < sizeof(sem_cols) / sizeof(sem_cols[0]); ci++) { @@ -3240,13 +3251,111 @@ static void emit_semantic_results(yyjson_mut_doc *doc, yyjson_mut_val *root, yyjson_mut_obj_add_val(doc, root, "semantic", sem); } +typedef enum { + SEMANTIC_QUERY_OK = 0, + SEMANTIC_QUERY_TYPE_ERROR, + SEMANTIC_QUERY_STORE_ERROR, +} semantic_query_status_t; + +enum { SEMANTIC_CANDIDATE_WINDOW = 250 }; + +typedef struct { + int start; + int count; + bool has_more; + bool truncated; +} semantic_page_t; + +/* Semantic pagination treats oversized integers as exhausted pages instead of + * allowing narrowing to wrap them back to the first page. Keep this behavior + * local: other MCP integer arguments have their own established bounds. */ +static int semantic_page_int_arg(const char *args_json, const char *key, int default_val) { + yyjson_doc *doc = yyjson_read(args_json, strlen(args_json), 0); + if (!doc) { + return default_val; + } + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *val = yyjson_obj_get(root, key); + int result = default_val; + if (val && yyjson_is_int(val)) { + if (yyjson_is_uint(val)) { + uint64_t value = yyjson_get_uint(val); + result = value > INT_MAX ? INT_MAX : (int)value; + } else { + int64_t value = yyjson_get_sint(val); + if (value > INT_MAX) { + result = INT_MAX; + } else if (value < INT_MIN) { + result = INT_MIN; + } else { + result = (int)value; + } + } + } + yyjson_doc_free(doc); + return result; +} + +static void emit_semantic_results_toon(cbm_sb_t *sb, const cbm_vector_result_t *vresults, + int vcount, int total, bool has_more, bool truncated); + +static semantic_page_t semantic_page(int total, int offset, int limit) { + int start = offset < total ? offset : total; + int remaining = total - start; + int count = limit < remaining ? limit : remaining; + return (semantic_page_t){ + .start = start, + .count = count, + .has_more = count < remaining, + .truncated = total == SEMANTIC_CANDIDATE_WINDOW, + }; +} + +static char *combine_bm25_semantic(const char *bm25_output, bool json, + const cbm_vector_result_t *vresults, int vcount, int offset, + int limit) { + semantic_page_t page = semantic_page(vcount, offset, limit); + if (!json) { + cbm_sb_t sb; + cbm_sb_init(&sb); + cbm_sb_append(&sb, bm25_output); + emit_semantic_results_toon(&sb, page.count > 0 ? vresults + page.start : NULL, page.count, + vcount, page.has_more, page.truncated); + return cbm_sb_finish(&sb); + } + + yyjson_doc *source = yyjson_read(bm25_output, strlen(bm25_output), 0); + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = + source && doc ? yyjson_val_mut_copy(doc, yyjson_doc_get_root(source)) : NULL; + if (!source || !doc || !root) { + if (source) { + yyjson_doc_free(source); + } + if (doc) { + yyjson_mut_doc_free(doc); + } + return NULL; + } + yyjson_mut_doc_set_root(doc, root); + emit_semantic_results(doc, root, page.count > 0 ? vresults + page.start : NULL, page.count, + vcount, page.has_more, page.truncated); + char *combined = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + yyjson_doc_free(source); + return combined; +} + /* Run the semantic_query vector search from raw args. Sets *out_vresults / - * *out_vcount (caller frees via cbm_store_free_vector_results when vcount>0). - * Returns true if semantic_query was provided as a non-array (type error — - * caller should surface to the user). */ -static bool run_semantic_query_core(const char *args, cbm_store_t *store, const char *project, - int limit, cbm_vector_result_t **out_vresults, int *out_vcount, - bool *out_present) { + * *out_vcount (caller frees via cbm_store_free_vector_results). + * + * The 250-candidate window bounds vector scoring cost while giving callers a + * useful, stable result universe to paginate. It is intentionally independent + * of the requested page size so changing limit cannot change ranking. */ +static semantic_query_status_t run_semantic_query_core(const char *args, cbm_store_t *store, + const char *project, + cbm_vector_result_t **out_vresults, + int *out_vcount, bool *out_present) { enum { MAX_KW_SEARCH = 32 }; *out_vresults = NULL; *out_vcount = 0; @@ -3259,26 +3368,34 @@ static bool run_semantic_query_core(const char *args, cbm_store_t *store, const if (out_present && sq_val) { *out_present = true; } - bool type_error = false; if (sq_val && !yyjson_is_arr(sq_val)) { - type_error = true; + if (args_doc) { + yyjson_doc_free(args_doc); + } + return SEMANTIC_QUERY_TYPE_ERROR; } else if (sq_val && yyjson_arr_size(sq_val) > 0) { const char *keywords[MAX_KW_SEARCH]; int ki = extract_semantic_keywords(sq_val, keywords, MAX_KW_SEARCH); + if (ki < 0) { + yyjson_doc_free(args_doc); + return SEMANTIC_QUERY_TYPE_ERROR; + } cbm_vector_result_t *vresults = NULL; int vcount = 0; - int sem_limit = limit > 0 ? limit : CBM_SZ_16; - if (cbm_store_vector_search(store, project, keywords, ki, sem_limit, &vresults, &vcount) == - CBM_STORE_OK && - vcount > 0) { - *out_vresults = vresults; - *out_vcount = vcount; + int store_rc = cbm_store_vector_search(store, project, keywords, ki, + SEMANTIC_CANDIDATE_WINDOW, &vresults, &vcount); + if (store_rc != CBM_STORE_OK) { + cbm_store_free_vector_results(vresults, vcount); + yyjson_doc_free(args_doc); + return SEMANTIC_QUERY_STORE_ERROR; } + *out_vresults = vresults; + *out_vcount = vcount; } if (args_doc) { yyjson_doc_free(args_doc); } - return type_error; + return SEMANTIC_QUERY_OK; } /* ── Tree output for search_graph ─────────────────────────────────── @@ -3618,7 +3735,7 @@ static void emit_search_results_tree_json(yyjson_mut_doc *doc, yyjson_mut_val *r /* Emit semantic vector-search results as a TOON table. */ static void emit_semantic_results_toon(cbm_sb_t *sb, const cbm_vector_result_t *vresults, - int vcount) { + int vcount, int total, bool has_more, bool truncated) { static const char *const cols[] = {"qn", "label", "file", "score"}; cbm_tree_table_header(sb, "semantic", vcount, cols, 4); for (int v = 0; v < vcount; v++) { @@ -3629,6 +3746,9 @@ static void emit_semantic_results_toon(cbm_sb_t *sb, const cbm_vector_result_t * cbm_tree_cell_real(sb, vresults[v].score, false); cbm_tree_row_end(sb); } + cbm_tree_scalar_int(sb, "semantic_total", total); + cbm_tree_scalar_bool(sb, "semantic_has_more", has_more); + cbm_tree_scalar_bool(sb, "semantic_truncated", truncated); } static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { @@ -3654,10 +3774,11 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { bool legacy_json = format_arg && strcmp(format_arg, "json") == 0; free(format_arg); - /* BM25 path: if `query` is set, run FTS5 full-text search with ranking - * and return early. The regex/vector path below is untouched for all - * other callers. If FTS5 is unavailable or the query is empty after - * tokenization, fall through to the regex path. */ + /* BM25 path: if `query` is set, run FTS5 full-text search with ranking. + * BM25-only requests retain the fast return. A combined semantic request + * appends its independently paged sidecar before returning. If FTS5 is + * unavailable or the query is empty after tokenization, fall through to + * the regex/vector path. */ char *query = cbm_mcp_get_string_arg(args, "query"); if (query && query[0]) { int q_limit = cbm_mcp_get_int_arg(args, "limit", BM25_DEFAULT_LIMIT); @@ -3667,6 +3788,45 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { bm25_search(store, project, query, q_file_pattern, q_limit, q_offset, !legacy_json); free(q_file_pattern); if (bm25_json) { + cbm_vector_result_t *vresults = NULL; + int vcount = 0; + bool sq_present = false; + semantic_query_status_t semantic_status = + run_semantic_query_core(args, store, project, &vresults, &vcount, &sq_present); + if (semantic_status != SEMANTIC_QUERY_OK) { + cbm_store_free_vector_results(vresults, vcount); + free(bm25_json); + free(query); + free(project); + if (semantic_status == SEMANTIC_QUERY_STORE_ERROR) { + return cbm_mcp_text_result("semantic search failed", true); + } + return cbm_mcp_text_result( + "semantic_query must be an array of keyword strings, e.g. " + "[\"send\",\"pubsub\",\"publish\"] — not a single string. Split your query " + "into individual keywords; each is scored independently via per-keyword " + "min-cosine.", + true); + } + if (sq_present) { + int semantic_limit_arg = + semantic_page_int_arg(args, "limit", CBM_DEFAULT_SEARCH_LIMIT); + int semantic_offset_arg = semantic_page_int_arg(args, "offset", 0); + int semantic_limit = + semantic_limit_arg > 0 ? semantic_limit_arg : CBM_DEFAULT_SEARCH_LIMIT; + int semantic_offset = semantic_offset_arg > 0 ? semantic_offset_arg : 0; + char *combined = combine_bm25_semantic(bm25_json, legacy_json, vresults, vcount, + semantic_offset, semantic_limit); + cbm_store_free_vector_results(vresults, vcount); + free(bm25_json); + free(query); + free(project); + char *result = + cbm_mcp_text_result(combined ? combined : "out of memory", combined == NULL); + free(combined); + return result; + } + cbm_store_free_vector_results(vresults, vcount); free(query); free(project); char *result = cbm_mcp_text_result(bm25_json, false); @@ -3685,6 +3845,10 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { bool include_connected = cbm_mcp_get_bool_arg(args, "include_connected"); int limit = cbm_mcp_get_int_arg(args, "limit", CBM_DEFAULT_SEARCH_LIMIT); int offset = cbm_mcp_get_int_arg(args, "offset", 0); + int semantic_limit_arg = semantic_page_int_arg(args, "limit", CBM_DEFAULT_SEARCH_LIMIT); + int semantic_offset_arg = semantic_page_int_arg(args, "offset", 0); + int semantic_limit = semantic_limit_arg > 0 ? semantic_limit_arg : CBM_DEFAULT_SEARCH_LIMIT; + int semantic_offset = semantic_offset_arg > 0 ? semantic_offset_arg : 0; int min_degree = cbm_mcp_get_int_arg(args, "min_degree", CBM_NOT_FOUND); int max_degree = cbm_mcp_get_int_arg(args, "max_degree", CBM_NOT_FOUND); @@ -3729,9 +3893,9 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { cbm_vector_result_t *vresults = NULL; int vcount = 0; bool sq_present = false; - bool sq_type_error = - run_semantic_query_core(args, store, project, limit, &vresults, &vcount, &sq_present); - if (!sq_type_error) { + semantic_query_status_t semantic_status = + run_semantic_query_core(args, store, project, &vresults, &vcount, &sq_present); + if (semantic_status == SEMANTIC_QUERY_OK) { /* Semantic-only calls get semantic results only: the legacy * behavior also ran the UNFILTERED regex search and prepended * up to `limit` unrelated enriched nodes to the response. */ @@ -3774,18 +3938,17 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { } } } - if (vcount > 0) { - emit_semantic_results_toon(&sb, vresults, vcount); - } else if (semantic_only) { - static const char *const sem_cols[] = {"qn", "label", "file", "score"}; - cbm_tree_table_header(&sb, "semantic", 0, sem_cols, 4); + if (sq_present) { + semantic_page_t page = semantic_page(vcount, semantic_offset, semantic_limit); + emit_semantic_results_toon(&sb, page.count > 0 ? vresults + page.start : NULL, + page.count, vcount, page.has_more, page.truncated); + } + if (semantic_only && vcount == 0) { cbm_tree_scalar_str(&sb, "hint", "No semantic matches. semantic_query needs a moderate/full " "index; try broader or fewer keywords."); } - if (vcount > 0) { - cbm_store_free_vector_results(vresults, vcount); - } + cbm_store_free_vector_results(vresults, vcount); if (fields_owner) { yyjson_doc_free(fields_owner); } @@ -3801,7 +3964,8 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { free(text); return result; } - /* semantic_query type error: fall through to the shared error text. */ + /* Semantic validation and retrieval errors share cleanup, but retain + * distinct user-facing failures. */ if (fields_owner) { yyjson_doc_free(fields_owner); } @@ -3811,6 +3975,9 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { free(qn_pattern); free(file_pattern); free(relationship); + if (semantic_status == SEMANTIC_QUERY_STORE_ERROR) { + return cbm_mcp_text_result("semantic search failed", true); + } return cbm_mcp_text_result( "semantic_query must be an array of keyword strings, e.g. " "[\"send\",\"pubsub\",\"publish\"] — not a single string. Split your query " @@ -3822,15 +3989,18 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { cbm_vector_result_t *vresults = NULL; int vcount = 0; bool sq_present = false; - bool sq_type_error = - run_semantic_query_core(args, store, project, limit, &vresults, &vcount, &sq_present); - if (sq_type_error) { + semantic_query_status_t semantic_status = + run_semantic_query_core(args, store, project, &vresults, &vcount, &sq_present); + if (semantic_status != SEMANTIC_QUERY_OK) { free(project); free(label); free(name_pattern); free(qn_pattern); free(file_pattern); free(relationship); + if (semantic_status == SEMANTIC_QUERY_STORE_ERROR) { + return cbm_mcp_text_result("semantic search failed", true); + } return cbm_mcp_text_result( "semantic_query must be an array of keyword strings, e.g. " "[\"send\",\"pubsub\",\"publish\"] — not a single string. Split your query " @@ -3882,10 +4052,12 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { } } - if (vcount > 0) { - emit_semantic_results(doc, root, vresults, vcount); - cbm_store_free_vector_results(vresults, vcount); + if (sq_present) { + semantic_page_t page = semantic_page(vcount, semantic_offset, semantic_limit); + emit_semantic_results(doc, root, page.count > 0 ? vresults + page.start : NULL, page.count, + vcount, page.has_more, page.truncated); } + cbm_store_free_vector_results(vresults, vcount); char *json = yy_doc_to_str(doc); yyjson_mut_doc_free(doc); diff --git a/src/store/store.c b/src/store/store.c index ad0b7967b..2392348a2 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -8619,21 +8619,23 @@ static int vs_build_keyword_vectors(cbm_store_t *s, const char *project, const c * each of the query vectors. Returns 0.0 if the node vector is unavailable * or mis-sized. */ static double vs_min_cosine_score(const int8_t *node_vec, int node_vec_len, - const int8_t (*kw_vecs)[VS_VEC_DIM], int actual_kw) { + const int8_t (*kw_vecs)[VS_VEC_DIM], const double *kw_magnitudes, + int actual_kw) { if (!node_vec || node_vec_len != VS_VEC_DIM) { return 0.0; } + int32_t node_norm = 0; + for (int d = 0; d < VS_VEC_DIM; d++) { + node_norm += (int32_t)node_vec[d] * (int32_t)node_vec[d]; + } + double node_magnitude = sqrt((double)node_norm); double min_score = CBM_STORE_UNIT_POS_D; for (int k = 0; k < actual_kw; k++) { int32_t dot = 0; - int32_t ma = 0; - int32_t mb = 0; for (int d = 0; d < VS_VEC_DIM; d++) { dot += (int32_t)kw_vecs[k][d] * (int32_t)node_vec[d]; - ma += (int32_t)kw_vecs[k][d] * (int32_t)kw_vecs[k][d]; - mb += (int32_t)node_vec[d] * (int32_t)node_vec[d]; } - double denom = sqrt((double)ma) * sqrt((double)mb); + double denom = kw_magnitudes[k] * node_magnitude; double cos_k = denom > CBM_STORE_DENOM_EPS_D ? (double)dot / denom : 0.0; if (cos_k < min_score) { min_score = cos_k; @@ -8643,34 +8645,62 @@ static double vs_min_cosine_score(const int8_t *node_vec, int node_vec_len, } /* Append one candidate row read from the scan statement into the result - * vector. Grows the results array geometrically on demand. Returns the - * (possibly grown) results pointer, or NULL on allocation failure. */ -static cbm_vector_result_t *vs_append_result(cbm_vector_result_t *results, int *count, int *cap, - sqlite3_stmt *stmt, - const int8_t (*kw_vecs)[VS_VEC_DIM], int actual_kw) { + * vector. Grows the results array geometrically on demand. */ +static int vs_append_result(cbm_vector_result_t **results, int *count, int *cap, sqlite3_stmt *stmt, + const int8_t (*kw_vecs)[VS_VEC_DIM], const double *kw_magnitudes, + int actual_kw) { if (*count >= *cap) { int nc = *cap < CBM_SZ_16 ? CBM_SZ_16 : *cap * ST_COL_2; - cbm_vector_result_t *grown = realloc(results, (size_t)nc * sizeof(cbm_vector_result_t)); + cbm_vector_result_t *grown = realloc(*results, (size_t)nc * sizeof(cbm_vector_result_t)); if (!grown) { - return NULL; + return CBM_STORE_ERR; } - results = grown; + *results = grown; *cap = nc; } - int idx = (*count)++; - results[idx].node_id = sqlite3_column_int64(stmt, 0); + int idx = *count; + cbm_vector_result_t *result = &(*results)[idx]; + memset(result, 0, sizeof(*result)); + result->node_id = sqlite3_column_int64(stmt, 0); const char *name = (const char *)sqlite3_column_text(stmt, SKIP_ONE); const char *qn = (const char *)sqlite3_column_text(stmt, ST_COL_2); const char *fp = (const char *)sqlite3_column_text(stmt, ST_COL_3); const char *label = (const char *)sqlite3_column_text(stmt, ST_COL_4); - results[idx].name = name ? strdup(name) : strdup(""); - results[idx].qualified_name = qn ? strdup(qn) : strdup(""); - results[idx].file_path = fp ? strdup(fp) : strdup(""); - results[idx].label = label ? strdup(label) : strdup(""); + result->name = strdup(name ? name : ""); + result->qualified_name = strdup(qn ? qn : ""); + result->file_path = strdup(fp ? fp : ""); + result->label = strdup(label ? label : ""); + if (!result->name || !result->qualified_name || !result->file_path || !result->label) { + free(result->name); + free(result->qualified_name); + free(result->file_path); + free(result->label); + memset(result, 0, sizeof(*result)); + return CBM_STORE_ERR; + } const int8_t *node_vec = (const int8_t *)sqlite3_column_blob(stmt, ST_COL_6); int node_vec_len = sqlite3_column_bytes(stmt, ST_COL_6); - results[idx].score = vs_min_cosine_score(node_vec, node_vec_len, kw_vecs, actual_kw); - return results; + result->score = vs_min_cosine_score(node_vec, node_vec_len, kw_vecs, kw_magnitudes, actual_kw); + (*count)++; + return CBM_STORE_OK; +} + +static int vs_result_cmp(const void *a, const void *b) { + const cbm_vector_result_t *ra = a; + const cbm_vector_result_t *rb = b; + if (ra->score > rb->score) { + return -1; + } + if (ra->score < rb->score) { + return 1; + } + if (ra->node_id < rb->node_id) { + return -1; + } + if (ra->node_id > rb->node_id) { + return 1; + } + return 0; } int cbm_store_vector_search(cbm_store_t *s, const char *project, const char **keywords, @@ -8687,6 +8717,14 @@ int cbm_store_vector_search(cbm_store_t *s, const char *project, const char **ke if (actual_kw == 0) { return CBM_STORE_OK; } + double kw_magnitudes[VS_MAX_KW] = {0}; + for (int k = 0; k < actual_kw; k++) { + int32_t kw_norm = 0; + for (int d = 0; d < VS_VEC_DIM; d++) { + kw_norm += (int32_t)kw_vecs[k][d] * (int32_t)kw_vecs[k][d]; + } + kw_magnitudes[k] = sqrt((double)kw_norm); + } /* Scan all node vectors, compute per-keyword cosine, take min. * We use the FIRST keyword as the SQL sort (for top-K pre-filter), @@ -8697,7 +8735,7 @@ int cbm_store_vector_search(cbm_store_t *s, const char *project, const char **ke " INNER JOIN nodes n ON n.id = v.node_id" " WHERE v.project = ?2" " AND n.label IN (" CBM_SQL_CALLABLE_OR_TYPE_LABELS ")" - " ORDER BY score DESC" + " ORDER BY score DESC, n.id ASC" " LIMIT ?3"; sqlite3_stmt *stmt = NULL; @@ -8707,8 +8745,9 @@ int cbm_store_vector_search(cbm_store_t *s, const char *project, const char **ke return CBM_STORE_ERR; } - /* Use first keyword for SQL pre-filter, fetch more candidates for re-ranking */ - int fetch_limit = (limit > 0 ? limit : CBM_SZ_16) * ST_COL_5; + /* Use the first keyword for SQL pre-filtering and honor the caller's exact + * candidate bound. Window policy belongs to the caller. */ + int fetch_limit = limit > 0 ? limit : CBM_SZ_16; sqlite3_bind_blob(stmt, SKIP_ONE, kw_vecs[0], VS_VEC_DIM, SQLITE_STATIC); sqlite3_bind_text(stmt, ST_COL_2, project, SQLITE_AUTO_LEN, SQLITE_STATIC); sqlite3_bind_int(stmt, ST_COL_3, fetch_limit); @@ -8726,16 +8765,19 @@ int cbm_store_vector_search(cbm_store_t *s, const char *project, const char **ke int count = 0; int cap = 0; int step_rc = 0; + bool append_failed = false; while ((step_rc = sqlite3_step(stmt)) == SQLITE_ROW) { - cbm_vector_result_t *grown = - vs_append_result(results, &count, &cap, stmt, kw_vecs, actual_kw); - if (!grown) { + if (vs_append_result(&results, &count, &cap, stmt, kw_vecs, kw_magnitudes, actual_kw) != + CBM_STORE_OK) { + append_failed = true; break; } - results = grown; } - if (step_rc != SQLITE_DONE) { + if (append_failed) { + snprintf(s->errbuf, sizeof(s->errbuf), "vector_search: allocation failed"); + } else if (step_rc != SQLITE_DONE) { + store_set_error_sqlite(s, "vector_search: row scan aborted"); char rc_buf[VS_STR_BUF]; snprintf(rc_buf, sizeof(rc_buf), "%d", step_rc); cbm_log_warn("vector_search.step_error", "rc", rc_buf, "msg", sqlite3_errmsg(s->db)); @@ -8746,16 +8788,14 @@ int cbm_store_vector_search(cbm_store_t *s, const char *project, const char **ke cbm_log_info("vector_search.done", "candidates", cnt_buf); } sqlite3_finalize(stmt); + if (append_failed || step_rc != SQLITE_DONE) { + cbm_store_free_vector_results(results, count); + return CBM_STORE_ERR; + } /* Re-sort by min-score (SQL sorted by first keyword only) */ - for (int i = 0; i < count - SKIP_ONE; i++) { - for (int j = i + SKIP_ONE; j < count; j++) { - if (results[j].score > results[i].score) { - cbm_vector_result_t tmp = results[i]; - results[i] = results[j]; - results[j] = tmp; - } - } + if (count > 1) { + qsort(results, (size_t)count, sizeof(cbm_vector_result_t), vs_result_cmp); } /* Trim to requested limit */ diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 6b7af64ed..86728668d 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -893,6 +894,20 @@ TEST(mcp_tools_list_latest_metadata) { ASSERT_NOT_NULL(strstr(json, "in/out = selected degree across CALLS, USAGE, CALL_REFERENCE, " "INHERITS, and IMPLEMENTS")); ASSERT_NULL(strstr(json, "TOTAL degree across ALL edge types")); + /* #915: tools/list must teach callers that semantic pagination exhausts a + * bounded ranked window, not necessarily the corpus. */ + ASSERT_NOT_NULL(strstr(json, "ranked inside a fixed 250-candidate window before " + "offset/limit slicing")); + ASSERT_NOT_NULL(strstr(json, "semantic.total, semantic.has_more, and semantic.truncated")); + ASSERT_NOT_NULL(strstr(json, "semantic_total, semantic_has_more, and semantic_truncated")); + ASSERT_NOT_NULL(strstr(json, "Semantic has_more only means another page remains inside that " + "window")); + ASSERT_NOT_NULL(strstr(json, "Semantic truncated means the window filled and corpus " + "completeness is unknown")); + ASSERT_NOT_NULL(strstr(json, "Advance offset by the number of semantic rows returned")); + ASSERT_NOT_NULL(strstr(json, "no score threshold")); + ASSERT_NOT_NULL(strstr(json, "first semantic keyword selects that bounded window")); + ASSERT_NOT_NULL(strstr(json, "keyword order can affect saturated results")); free(json); PASS(); } @@ -1246,6 +1261,8 @@ TEST(mcp_get_int_arg) { ASSERT_EQ(val, 5); val = cbm_mcp_get_int_arg(args, "missing", 42); ASSERT_EQ(val, 42); + val = cbm_mcp_get_int_arg("{\"limit\":18446744073709551615}", "limit", 0); + ASSERT_LTE(val, 0); PASS(); } @@ -1698,6 +1715,13 @@ TEST(tool_search_graph_semantic_only_skips_structural_results_issue1295) { const char *proj = "semantic-only"; cbm_mcp_server_set_project(srv, proj); cbm_store_upsert_project(st, proj, "/tmp/semantic-only"); + ASSERT_EQ(cbm_store_exec(st, + "CREATE TABLE node_vectors(node_id INTEGER PRIMARY KEY,project TEXT " + "NOT NULL,vector BLOB NOT NULL);" + "CREATE TABLE token_vectors(id INTEGER PRIMARY KEY,project TEXT NOT " + "NULL,token TEXT NOT NULL,vector BLOB NOT NULL,idf INTEGER NOT " + "NULL);"), + CBM_STORE_OK); cbm_node_t unrelated = {.project = proj, .label = "Function", @@ -1735,6 +1759,656 @@ TEST(tool_search_graph_semantic_only_skips_structural_results_issue1295) { PASS(); } +enum { MCP_SEMANTIC_TEST_VECTOR_DIM = 768 }; + +static bool mcp_semantic_insert_vector(sqlite3_stmt *stmt, int64_t node_id, const char *project, + const int8_t *vector) { + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + return sqlite3_bind_int64(stmt, 1, node_id) == SQLITE_OK && + sqlite3_bind_text(stmt, 2, project, -1, SQLITE_STATIC) == SQLITE_OK && + sqlite3_bind_blob(stmt, 3, vector, MCP_SEMANTIC_TEST_VECTOR_DIM, SQLITE_STATIC) == + SQLITE_OK && + sqlite3_step(stmt) == SQLITE_DONE; +} + +static bool mcp_semantic_insert_token(sqlite3 *db, const char *project, const char *token, + const int8_t *vector) { + sqlite3_stmt *stmt = NULL; + bool ok = sqlite3_prepare_v2(db, + "INSERT INTO token_vectors(project,token,vector,idf) " + "VALUES(?1,?2,?3,1);", + -1, &stmt, NULL) == SQLITE_OK && + sqlite3_bind_text(stmt, 1, project, -1, SQLITE_STATIC) == SQLITE_OK && + sqlite3_bind_text(stmt, 2, token, -1, SQLITE_STATIC) == SQLITE_OK && + sqlite3_bind_blob(stmt, 3, vector, MCP_SEMANTIC_TEST_VECTOR_DIM, SQLITE_STATIC) == + SQLITE_OK && + sqlite3_step(stmt) == SQLITE_DONE; + sqlite3_finalize(stmt); + return ok; +} + +static bool mcp_semantic_create_vector_tables(cbm_store_t *store) { + return cbm_store_exec(store, + "CREATE TABLE node_vectors(node_id INTEGER PRIMARY KEY,project TEXT " + "NOT NULL,vector BLOB NOT NULL);" + "CREATE INDEX node_vectors_project_desc " + "ON node_vectors(project,node_id DESC);" + "CREATE TABLE token_vectors(id INTEGER PRIMARY KEY,project TEXT NOT " + "NULL,token TEXT NOT NULL,vector BLOB NOT NULL,idf INTEGER NOT NULL);") == + CBM_STORE_OK; +} + +static bool mcp_semantic_populate_saturated(cbm_store_t *st, const char *saturated) { + sqlite3 *db = cbm_store_get_db(st); + int8_t q1[MCP_SEMANTIC_TEST_VECTOR_DIM] = {0}; + int8_t q2[MCP_SEMANTIC_TEST_VECTOR_DIM] = {0}; + q1[0] = 127; + q2[1] = 127; + if (!db || + cbm_store_upsert_project(st, saturated, "/tmp/semantic-page-saturated") != CBM_STORE_OK || + !mcp_semantic_insert_token(db, saturated, "horizontal", q1) || + !mcp_semantic_insert_token(db, saturated, "vertical", q2)) { + return false; + } + + int64_t node_ids[260] = {0}; + for (int i = 1; i <= 260; i++) { + char name[32]; + char qn[64]; + snprintf(name, sizeof(name), "sat_%03d", i); + snprintf(qn, sizeof(qn), "%s.%s", saturated, name); + cbm_node_t node = {.project = saturated, + .label = "Function", + .name = name, + .qualified_name = qn, + .file_path = "semantic.c", + .start_line = i, + .end_line = i}; + node_ids[i - 1] = cbm_store_upsert_node(st, &node); + if (node_ids[i - 1] <= 0) { + return false; + } + } + + sqlite3_stmt *insert_vector = NULL; + if (sqlite3_prepare_v2(db, "INSERT INTO node_vectors(node_id,project,vector) VALUES(?1,?2,?3);", + -1, &insert_vector, NULL) != SQLITE_OK) { + return false; + } + for (int i = 260; i >= 1; i--) { + int8_t vector[MCP_SEMANTIC_TEST_VECTOR_DIM] = {0}; + if (i <= 10) { + vector[0] = 127; + } else if (i == 11) { + vector[0] = 80; + vector[1] = 100; + } else if (i == 12) { + vector[0] = 100; + vector[1] = 80; + } else { + vector[0] = 70; + vector[1] = 90; + } + if (!mcp_semantic_insert_vector(insert_vector, node_ids[i - 1], saturated, vector)) { + sqlite3_finalize(insert_vector); + return false; + } + } + sqlite3_finalize(insert_vector); + return true; +} + +static bool mcp_semantic_populate_uniform(cbm_store_t *st, const char *project, int node_count) { + sqlite3 *db = cbm_store_get_db(st); + int8_t query[MCP_SEMANTIC_TEST_VECTOR_DIM] = {0}; + query[0] = 127; + if (!db || + cbm_store_upsert_project(st, project, "/tmp/semantic-page-uniform") != CBM_STORE_OK || + !mcp_semantic_insert_token(db, project, "horizontal", query) || + !mcp_semantic_insert_token(db, project, "vertical", query)) { + return false; + } + + sqlite3_stmt *insert_vector = NULL; + if (sqlite3_prepare_v2(db, "INSERT INTO node_vectors(node_id,project,vector) VALUES(?1,?2,?3);", + -1, &insert_vector, NULL) != SQLITE_OK) { + return false; + } + for (int i = 1; i <= node_count; i++) { + char name[32]; + char qn[96]; + snprintf(name, sizeof(name), "uniform_%03d", i); + snprintf(qn, sizeof(qn), "%s.%s", project, name); + cbm_node_t node = {.project = project, + .label = "Function", + .name = name, + .qualified_name = qn, + .file_path = "uniform.c", + .start_line = i, + .end_line = i}; + int64_t node_id = cbm_store_upsert_node(st, &node); + if (node_id <= 0 || !mcp_semantic_insert_vector(insert_vector, node_id, project, query)) { + sqlite3_finalize(insert_vector); + return false; + } + } + sqlite3_finalize(insert_vector); + return true; +} + +static bool mcp_semantic_populate_underfull(cbm_store_t *st, const char *underfull) { + sqlite3 *db = cbm_store_get_db(st); + int8_t q1[MCP_SEMANTIC_TEST_VECTOR_DIM] = {0}; + q1[0] = 127; + if (!db || + cbm_store_upsert_project(st, underfull, "/tmp/semantic-page-underfull") != CBM_STORE_OK || + !mcp_semantic_insert_token(db, underfull, "horizontal", q1) || + !mcp_semantic_insert_token(db, underfull, "vertical", q1)) { + return false; + } + + sqlite3_stmt *insert_vector = NULL; + if (sqlite3_prepare_v2(db, "INSERT INTO node_vectors(node_id,project,vector) VALUES(?1,?2,?3);", + -1, &insert_vector, NULL) != SQLITE_OK) { + return false; + } + const int8_t underfull_vectors[3][MCP_SEMANTIC_TEST_VECTOR_DIM] = { + {[0] = 127}, {[1] = 127}, {[0] = -127}}; + static const char *const underfull_names[] = {"strong_match", "zero_match", "negative_match"}; + for (int i = 0; i < 3; i++) { + char qn[80]; + snprintf(qn, sizeof(qn), "%s.%s", underfull, underfull_names[i]); + cbm_node_t node = {.project = underfull, + .label = "Function", + .name = (char *)underfull_names[i], + .qualified_name = qn, + .file_path = "underfull.c", + .start_line = i + 1, + .end_line = i + 1}; + int64_t node_id = cbm_store_upsert_node(st, &node); + if (node_id <= 0 || + !mcp_semantic_insert_vector(insert_vector, node_id, underfull, underfull_vectors[i])) { + sqlite3_finalize(insert_vector); + return false; + } + } + cbm_node_t structural = {.project = underfull, + .label = "Function", + .name = "structural_only", + .qualified_name = "semantic-page-underfull.structural_only", + .file_path = "structural.c", + .start_line = 10, + .end_line = 11}; + if (cbm_store_upsert_node(st, &structural) <= 0) { + sqlite3_finalize(insert_vector); + return false; + } + sqlite3_finalize(insert_vector); + return cbm_store_exec(st, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');" + "INSERT INTO nodes_fts(rowid,name,qualified_name,label,file_path) " + "SELECT id,cbm_camel_split(name),qualified_name,label,file_path " + "FROM nodes;") == CBM_STORE_OK; +} + +static bool mcp_semantic_populate_empty(cbm_store_t *st, const char *empty) { + sqlite3 *db = cbm_store_get_db(st); + int8_t q1[MCP_SEMANTIC_TEST_VECTOR_DIM] = {0}; + int8_t q2[MCP_SEMANTIC_TEST_VECTOR_DIM] = {0}; + q1[0] = 127; + q2[1] = 127; + return db && cbm_store_upsert_project(st, empty, "/tmp/semantic-page-empty") == CBM_STORE_OK && + mcp_semantic_insert_token(db, empty, "horizontal", q1) && + mcp_semantic_insert_token(db, empty, "vertical", q2); +} + +typedef struct { + char cache[CBM_SZ_1K]; + char *saved_cache; + cbm_mcp_server_t *server; +} mcp_semantic_pagination_fixture_t; + +static cbm_store_t *mcp_semantic_open_project_store(const char *cache, const char *project) { + char path[CBM_SZ_1K]; + snprintf(path, sizeof(path), "%s/%s.db", cache, project); + cbm_store_t *store = cbm_store_open_path(path); + if (!store || !mcp_semantic_create_vector_tables(store)) { + cbm_store_close(store); + return NULL; + } + return store; +} + +static void mcp_semantic_pagination_fixture_close(mcp_semantic_pagination_fixture_t *fixture) { + if (!fixture) { + return; + } + cbm_mcp_server_free(fixture->server); + fixture->server = NULL; + cleanup_project_db(fixture->cache, "semantic-page-saturated"); + cleanup_project_db(fixture->cache, "semantic-page-underfull"); + cleanup_project_db(fixture->cache, "semantic-page-empty"); + cleanup_project_db(fixture->cache, "semantic-page-249"); + cleanup_project_db(fixture->cache, "semantic-page-250"); + restore_cache_dir(fixture->saved_cache); + free(fixture->saved_cache); + fixture->saved_cache = NULL; + if (fixture->cache[0]) { + (void)th_rmtree(fixture->cache); + } + memset(fixture, 0, sizeof(*fixture)); +} + +static bool mcp_semantic_pagination_fixture_open(mcp_semantic_pagination_fixture_t *fixture) { + memset(fixture, 0, sizeof(*fixture)); + snprintf(fixture->cache, sizeof(fixture->cache), "%s/cbm-semantic-pagination-XXXXXX", + cbm_tmpdir()); + if (!cbm_mkdtemp(fixture->cache)) { + return false; + } + const char *saved_cache = getenv("CBM_CACHE_DIR"); + fixture->saved_cache = saved_cache ? strdup(saved_cache) : NULL; + if ((saved_cache && !fixture->saved_cache) || + cbm_setenv("CBM_CACHE_DIR", fixture->cache, 1) != 0) { + mcp_semantic_pagination_fixture_close(fixture); + return false; + } + + cbm_store_t *store = mcp_semantic_open_project_store(fixture->cache, "semantic-page-saturated"); + bool ready = store && mcp_semantic_populate_saturated(store, "semantic-page-saturated"); + cbm_store_close(store); + store = + ready ? mcp_semantic_open_project_store(fixture->cache, "semantic-page-underfull") : NULL; + ready = store && mcp_semantic_populate_underfull(store, "semantic-page-underfull"); + cbm_store_close(store); + store = ready ? mcp_semantic_open_project_store(fixture->cache, "semantic-page-empty") : NULL; + ready = store && mcp_semantic_populate_empty(store, "semantic-page-empty"); + cbm_store_close(store); + store = ready ? mcp_semantic_open_project_store(fixture->cache, "semantic-page-249") : NULL; + ready = store && mcp_semantic_populate_uniform(store, "semantic-page-249", 249); + cbm_store_close(store); + store = ready ? mcp_semantic_open_project_store(fixture->cache, "semantic-page-250") : NULL; + ready = store && mcp_semantic_populate_uniform(store, "semantic-page-250", 250); + cbm_store_close(store); + fixture->server = ready ? cbm_mcp_server_new(NULL) : NULL; + if (!fixture->server) { + mcp_semantic_pagination_fixture_close(fixture); + return false; + } + return true; +} + +static cbm_mcp_server_t *setup_semantic_store_error_server(void) { + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + cbm_store_t *store = cbm_mcp_server_store(server); + if (!server || !store || !mcp_semantic_create_vector_tables(store) || + !mcp_semantic_populate_underfull(store, "semantic-page-underfull")) { + cbm_mcp_server_free(server); + return NULL; + } + cbm_mcp_server_set_project(server, "semantic-page-underfull"); + return server; +} + +static char *mcp_semantic_search(cbm_mcp_server_t *srv, const char *project, const char *extra) { + char args[512]; + snprintf(args, sizeof(args), + "{\"project\":\"%s\",\"semantic_query\":[\"horizontal\",\"vertical\"]%s}", project, + extra ? extra : ""); + char *response = cbm_mcp_handle_tool(srv, "search_graph", args); + char *inner = extract_text_content(response); + free(response); + return inner; +} + +static yyjson_doc *mcp_semantic_json_doc(cbm_mcp_server_t *srv, const char *project, + const char *extra) { + char *inner = mcp_semantic_search(srv, project, extra); + yyjson_doc *doc = inner ? yyjson_read(inner, strlen(inner), 0) : NULL; + free(inner); + return doc; +} + +static yyjson_val *mcp_semantic_rows(yyjson_doc *doc) { + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + yyjson_val *semantic = root ? yyjson_obj_get(root, "semantic") : NULL; + return semantic ? yyjson_obj_get(semantic, "rows") : NULL; +} + +static const char *mcp_semantic_row_qn(yyjson_val *rows, size_t index) { + yyjson_val *row = rows ? yyjson_arr_get(rows, index) : NULL; + yyjson_val *qn = row ? yyjson_arr_get(row, 0) : NULL; + return qn && yyjson_is_str(qn) ? yyjson_get_str(qn) : NULL; +} + +static void mcp_semantic_sql_error(sqlite3_context *ctx, int argc, sqlite3_value **argv) { + (void)argc; + (void)argv; + sqlite3_result_error(ctx, "forced semantic scan failure", -1); +} + +/* #915: semantic pagination must rank one named 250-candidate universe before + * slicing it. This handler fixture makes the first-keyword SQL order disagree + * with the two-keyword final order and puts equal-score nodes around the bound. */ +TEST(tool_search_graph_semantic_pagination_stable_window_issue915) { + mcp_semantic_pagination_fixture_t fixture; + ASSERT_TRUE(mcp_semantic_pagination_fixture_open(&fixture)); + cbm_mcp_server_t *srv = fixture.server; + + yyjson_doc *one = + mcp_semantic_json_doc(srv, "semantic-page-saturated", ",\"format\":\"json\",\"limit\":1"); + yyjson_val *one_rows = mcp_semantic_rows(one); + ASSERT_NOT_NULL(one_rows); + ASSERT_EQ(yyjson_arr_size(one_rows), 1); + ASSERT_STR_EQ(mcp_semantic_row_qn(one_rows, 0), "semantic-page-saturated.sat_011"); + + yyjson_doc *page1 = mcp_semantic_json_doc(srv, "semantic-page-saturated", + ",\"format\":\"json\",\"offset\":0,\"limit\":3"); + yyjson_doc *page2 = mcp_semantic_json_doc(srv, "semantic-page-saturated", + ",\"format\":\"json\",\"offset\":3,\"limit\":3"); + yyjson_doc *six = mcp_semantic_json_doc(srv, "semantic-page-saturated", + ",\"format\":\"json\",\"offset\":0,\"limit\":6"); + yyjson_val *page1_rows = mcp_semantic_rows(page1); + yyjson_val *page2_rows = mcp_semantic_rows(page2); + yyjson_val *six_rows = mcp_semantic_rows(six); + ASSERT_EQ(yyjson_arr_size(page1_rows), 3); + ASSERT_EQ(yyjson_arr_size(page2_rows), 3); + ASSERT_EQ(yyjson_arr_size(six_rows), 6); + for (size_t i = 0; i < 3; i++) { + ASSERT_STR_EQ(mcp_semantic_row_qn(page1_rows, i), mcp_semantic_row_qn(six_rows, i)); + ASSERT_STR_EQ(mcp_semantic_row_qn(page2_rows, i), mcp_semantic_row_qn(six_rows, i + 3)); + } + + yyjson_val *root = yyjson_doc_get_root(six); + yyjson_val *semantic = yyjson_obj_get(root, "semantic"); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(semantic, "total")), 250); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(semantic, "has_more"))); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(semantic, "truncated"))); + + yyjson_doc *whole = + mcp_semantic_json_doc(srv, "semantic-page-saturated", ",\"format\":\"json\",\"limit\":250"); + yyjson_val *whole_rows = mcp_semantic_rows(whole); + ASSERT_EQ(yyjson_arr_size(whole_rows), 250); + ASSERT_STR_EQ(mcp_semantic_row_qn(whole_rows, 239), "semantic-page-saturated.sat_250"); + ASSERT_STR_EQ(mcp_semantic_row_qn(whole_rows, 240), "semantic-page-saturated.sat_001"); + ASSERT_NULL(strstr(mcp_semantic_row_qn(whole_rows, 239), "sat_251")); + + yyjson_doc *exhausted = mcp_semantic_json_doc( + srv, "semantic-page-saturated", ",\"format\":\"json\",\"offset\":250,\"limit\":5"); + yyjson_doc *overflow = + mcp_semantic_json_doc(srv, "semantic-page-saturated", + ",\"format\":\"json\",\"offset\":2147483647,\"limit\":2147483647"); + yyjson_doc *oversized = mcp_semantic_json_doc( + srv, "semantic-page-saturated", ",\"format\":\"json\",\"offset\":2147483648,\"limit\":5"); + yyjson_doc *unsigned_max = + mcp_semantic_json_doc(srv, "semantic-page-saturated", + ",\"format\":\"json\",\"offset\":18446744073709551615,\"limit\":5"); + ASSERT_EQ(yyjson_arr_size(mcp_semantic_rows(exhausted)), 0); + ASSERT_EQ(yyjson_arr_size(mcp_semantic_rows(overflow)), 0); + ASSERT_EQ(yyjson_arr_size(mcp_semantic_rows(oversized)), 0); + ASSERT_EQ(yyjson_arr_size(mcp_semantic_rows(unsigned_max)), 0); + yyjson_val *exhausted_semantic = yyjson_obj_get(yyjson_doc_get_root(exhausted), "semantic"); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(exhausted_semantic, "total")), 250); + ASSERT_FALSE(yyjson_get_bool(yyjson_obj_get(exhausted_semantic, "has_more"))); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(exhausted_semantic, "truncated"))); + + yyjson_doc *final = mcp_semantic_json_doc(srv, "semantic-page-saturated", + ",\"format\":\"json\",\"offset\":249,\"limit\":1"); + yyjson_val *final_semantic = yyjson_obj_get(yyjson_doc_get_root(final), "semantic"); + ASSERT_EQ(yyjson_arr_size(mcp_semantic_rows(final)), 1); + ASSERT_FALSE(yyjson_get_bool(yyjson_obj_get(final_semantic, "has_more"))); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(final_semantic, "truncated"))); + char *final_tree = + mcp_semantic_search(srv, "semantic-page-saturated", ",\"offset\":249,\"limit\":1"); + ASSERT_NOT_NULL(strstr(final_tree, "semantic_has_more: false")); + ASSERT_NOT_NULL(strstr(final_tree, "semantic_truncated: true")); + free(final_tree); + + char *tree = mcp_semantic_search(srv, "semantic-page-saturated", ",\"limit\":3"); + ASSERT_NOT_NULL(tree); + ASSERT_NOT_NULL(strstr(tree, "semantic_total: 250")); + ASSERT_NOT_NULL(strstr(tree, "semantic_has_more: true")); + ASSERT_NOT_NULL(strstr(tree, "semantic_truncated: true")); + ASSERT_NOT_NULL(strstr(tree, "semantic-page-saturated.sat_011")); + + free(tree); + yyjson_doc_free(final); + yyjson_doc_free(unsigned_max); + yyjson_doc_free(oversized); + yyjson_doc_free(overflow); + yyjson_doc_free(exhausted); + yyjson_doc_free(whole); + yyjson_doc_free(six); + yyjson_doc_free(page2); + yyjson_doc_free(page1); + yyjson_doc_free(one); + mcp_semantic_pagination_fixture_close(&fixture); + PASS(); +} + +TEST(tool_search_graph_semantic_pagination_envelopes_issue915) { + mcp_semantic_pagination_fixture_t fixture; + ASSERT_TRUE(mcp_semantic_pagination_fixture_open(&fixture)); + cbm_mcp_server_t *srv = fixture.server; + + yyjson_doc *underfull = + mcp_semantic_json_doc(srv, "semantic-page-underfull", ",\"format\":\"json\",\"limit\":10"); + yyjson_val *rows = mcp_semantic_rows(underfull); + ASSERT_EQ(yyjson_arr_size(rows), 3); + ASSERT_STR_EQ(mcp_semantic_row_qn(rows, 0), "semantic-page-underfull.strong_match"); + ASSERT_STR_EQ(mcp_semantic_row_qn(rows, 1), "semantic-page-underfull.zero_match"); + ASSERT_STR_EQ(mcp_semantic_row_qn(rows, 2), "semantic-page-underfull.negative_match"); + yyjson_val *semantic = yyjson_obj_get(yyjson_doc_get_root(underfull), "semantic"); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(semantic, "total")), 3); + ASSERT_FALSE(yyjson_get_bool(yyjson_obj_get(semantic, "has_more"))); + ASSERT_FALSE(yyjson_get_bool(yyjson_obj_get(semantic, "truncated"))); + + yyjson_doc *negative_offset = mcp_semantic_json_doc( + srv, "semantic-page-underfull", ",\"format\":\"json\",\"offset\":-5,\"limit\":1"); + yyjson_doc *nonpositive_limit = + mcp_semantic_json_doc(srv, "semantic-page-underfull", ",\"format\":\"json\",\"limit\":0"); + ASSERT_STR_EQ(mcp_semantic_row_qn(mcp_semantic_rows(negative_offset), 0), + "semantic-page-underfull.strong_match"); + ASSERT_EQ(yyjson_arr_size(mcp_semantic_rows(nonpositive_limit)), 3); + + yyjson_doc *underfull_249 = + mcp_semantic_json_doc(srv, "semantic-page-249", ",\"format\":\"json\",\"limit\":250"); + yyjson_val *semantic_249 = yyjson_obj_get(yyjson_doc_get_root(underfull_249), "semantic"); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(semantic_249, "total")), 249); + ASSERT_FALSE(yyjson_get_bool(yyjson_obj_get(semantic_249, "truncated"))); + + yyjson_doc *full_250 = + mcp_semantic_json_doc(srv, "semantic-page-250", ",\"format\":\"json\",\"limit\":250"); + yyjson_val *semantic_250 = yyjson_obj_get(yyjson_doc_get_root(full_250), "semantic"); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(semantic_250, "total")), 250); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(semantic_250, "truncated"))); + char *tree_249 = mcp_semantic_search(srv, "semantic-page-249", ",\"offset\":249,\"limit\":1"); + ASSERT_NOT_NULL(strstr(tree_249, "semantic_total: 249")); + ASSERT_NOT_NULL(strstr(tree_249, "semantic_truncated: false")); + char *tree_250 = mcp_semantic_search(srv, "semantic-page-250", ",\"offset\":250,\"limit\":1"); + ASSERT_NOT_NULL(strstr(tree_250, "semantic_total: 250")); + ASSERT_NOT_NULL(strstr(tree_250, "semantic_truncated: true")); + free(tree_250); + free(tree_249); + + char *tree = mcp_semantic_search(srv, "semantic-page-underfull", ",\"limit\":10"); + ASSERT_NOT_NULL(tree); + ASSERT_NULL(strstr(tree, "structural_only")); + ASSERT_NOT_NULL(strstr(tree, "semantic_total: 3")); + ASSERT_NOT_NULL(strstr(tree, "semantic_has_more: false")); + ASSERT_NOT_NULL(strstr(tree, "semantic_truncated: false")); + ASSERT_NOT_NULL(strstr(tree, "semantic-page-underfull.strong_match")); + ASSERT_NOT_NULL(strstr(tree, "semantic-page-underfull.zero_match")); + ASSERT_NOT_NULL(strstr(tree, "semantic-page-underfull.negative_match")); + free(tree); + + yyjson_doc *combined = mcp_semantic_json_doc( + srv, "semantic-page-underfull", + ",\"format\":\"json\",\"name_pattern\":\"structural_only\",\"limit\":10"); + yyjson_val *combined_root = yyjson_doc_get_root(combined); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(combined_root, "total")), 1); + ASSERT_FALSE(yyjson_get_bool(yyjson_obj_get(combined_root, "has_more"))); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(yyjson_obj_get(combined_root, "semantic"), "total")), + 3); + char *combined_text = yyjson_val_write(combined_root, 0, NULL); + ASSERT_NOT_NULL(combined_text); + ASSERT_NOT_NULL(strstr(combined_text, "structural_only")); + free(combined_text); + + yyjson_doc *combined_offset = mcp_semantic_json_doc( + srv, "semantic-page-underfull", + ",\"format\":\"json\",\"name_pattern\":\"structural_only\",\"offset\":1,\"limit\":1"); + yyjson_val *combined_offset_root = yyjson_doc_get_root(combined_offset); + yyjson_val *combined_semantic = yyjson_obj_get(combined_offset_root, "semantic"); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(combined_offset_root, "total")), 1); + ASSERT_EQ(yyjson_arr_size(yyjson_obj_get(combined_offset_root, "groups")), 0); + ASSERT_STR_EQ(mcp_semantic_row_qn(yyjson_obj_get(combined_semantic, "rows"), 0), + "semantic-page-underfull.zero_match"); + + yyjson_doc *bm25_combined = + mcp_semantic_json_doc(srv, "semantic-page-underfull", + ",\"query\":\"strong_match\",\"format\":\"json\",\"limit\":2"); + yyjson_val *bm25_root = yyjson_doc_get_root(bm25_combined); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(bm25_root, "search_mode")), "bm25"); + ASSERT_EQ(yyjson_arr_size(yyjson_obj_get(bm25_root, "rows")), 1); + yyjson_val *bm25_semantic = yyjson_obj_get(bm25_root, "semantic"); + ASSERT_NOT_NULL(bm25_semantic); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(bm25_semantic, "total")), 3); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(bm25_semantic, "has_more"))); + ASSERT_STR_EQ(mcp_semantic_row_qn(yyjson_obj_get(bm25_semantic, "rows"), 0), + "semantic-page-underfull.strong_match"); + ASSERT_STR_EQ(mcp_semantic_row_qn(yyjson_obj_get(bm25_semantic, "rows"), 1), + "semantic-page-underfull.zero_match"); + char *bm25_tree = mcp_semantic_search(srv, "semantic-page-underfull", + ",\"query\":\"strong_match\",\"limit\":2"); + ASSERT_NOT_NULL(strstr(bm25_tree, "search_mode: bm25")); + ASSERT_NOT_NULL(strstr(bm25_tree, "semantic_total: 3")); + ASSERT_NOT_NULL(strstr(bm25_tree, "semantic_has_more: true")); + ASSERT_NOT_NULL(strstr(bm25_tree, "semantic-page-underfull.strong_match")); + ASSERT_NOT_NULL(strstr(bm25_tree, "semantic-page-underfull.zero_match")); + free(bm25_tree); + + yyjson_doc *empty = mcp_semantic_json_doc(srv, "semantic-page-empty", + ",\"format\":\"json\",\"offset\":0,\"limit\":5"); + yyjson_val *empty_semantic = yyjson_obj_get(yyjson_doc_get_root(empty), "semantic"); + ASSERT_NOT_NULL(empty_semantic); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(empty_semantic, "total")), 0); + ASSERT_FALSE(yyjson_get_bool(yyjson_obj_get(empty_semantic, "has_more"))); + ASSERT_FALSE(yyjson_get_bool(yyjson_obj_get(empty_semantic, "truncated"))); + ASSERT_EQ(yyjson_arr_size(yyjson_obj_get(empty_semantic, "rows")), 0); + + char *empty_array_response = cbm_mcp_handle_tool( + srv, "search_graph", + "{\"project\":\"semantic-page-empty\",\"semantic_query\":[],\"format\":\"json\"}"); + char *empty_array_inner = extract_text_content(empty_array_response); + yyjson_doc *empty_array = + empty_array_inner ? yyjson_read(empty_array_inner, strlen(empty_array_inner), 0) : NULL; + yyjson_val *empty_array_semantic = + empty_array ? yyjson_obj_get(yyjson_doc_get_root(empty_array), "semantic") : NULL; + ASSERT_NOT_NULL(empty_array_semantic); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(empty_array_semantic, "total")), 0); + ASSERT_EQ(yyjson_arr_size(yyjson_obj_get(empty_array_semantic, "rows")), 0); + + yyjson_doc_free(empty_array); + free(empty_array_inner); + free(empty_array_response); + yyjson_doc_free(empty); + yyjson_doc_free(bm25_combined); + yyjson_doc_free(combined_offset); + yyjson_doc_free(combined); + yyjson_doc_free(full_250); + yyjson_doc_free(underfull_249); + yyjson_doc_free(nonpositive_limit); + yyjson_doc_free(negative_offset); + yyjson_doc_free(underfull); + mcp_semantic_pagination_fixture_close(&fixture); + PASS(); +} + +TEST(tool_search_graph_semantic_store_error_has_no_partial_results_issue915) { + cbm_mcp_server_t *srv = setup_semantic_store_error_server(); + ASSERT_NOT_NULL(srv); + sqlite3 *db = cbm_store_get_db(cbm_mcp_server_store(srv)); + ASSERT_NOT_NULL(db); + ASSERT_EQ(sqlite3_create_function(db, "cbm_cosine_i8", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, + NULL, mcp_semantic_sql_error, NULL, NULL), + SQLITE_OK); + cbm_mcp_server_set_project(srv, "semantic-page-underfull"); + + char *response = cbm_mcp_handle_tool( + srv, "search_graph", + "{\"project\":\"semantic-page-underfull\",\"semantic_query\":[\"horizontal\"," + "\"vertical\"],\"format\":\"json\",\"limit\":10}"); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\"isError\":true")); + char *inner = extract_text_content(response); + ASSERT_STR_EQ(inner, "semantic search failed"); + ASSERT_NULL(strstr(response, "strong_match")); + ASSERT_NULL(strstr(response, "\"rows\"")); + ASSERT_NULL(strstr(response, "semantic_total")); + ASSERT_NULL(strstr(response, "semantic_has_more")); + ASSERT_NULL(strstr(response, "semantic_truncated")); + ASSERT_NULL(strstr(response, "\"has_more\"")); + ASSERT_NULL(strstr(response, "\"truncated\"")); + + char *combined_response = cbm_mcp_handle_tool( + srv, "search_graph", + "{\"project\":\"semantic-page-underfull\",\"query\":\"strong_match\"," + "\"semantic_query\":[\"horizontal\",\"vertical\"],\"format\":\"json\",\"limit\":10}"); + ASSERT_NOT_NULL(combined_response); + ASSERT_NOT_NULL(strstr(combined_response, "\"isError\":true")); + char *combined_inner = extract_text_content(combined_response); + ASSERT_STR_EQ(combined_inner, "semantic search failed"); + ASSERT_NULL(strstr(combined_response, "\"search_mode\":\"bm25\"")); + ASSERT_NULL(strstr(combined_response, "strong_match")); + ASSERT_NULL(strstr(combined_response, "\"rows\"")); + ASSERT_NULL(strstr(combined_response, "\"semantic\"")); + + free(combined_inner); + free(combined_response); + free(inner); + free(response); + cbm_mcp_server_free(srv); + PASS(); +} + +TEST(tool_search_graph_semantic_invalid_element_type_issue915) { + cbm_mcp_server_t *srv = setup_semantic_store_error_server(); + ASSERT_NOT_NULL(srv); + sqlite3 *db = cbm_store_get_db(cbm_mcp_server_store(srv)); + ASSERT_NOT_NULL(db); + ASSERT_EQ(sqlite3_create_function(db, "cbm_cosine_i8", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, + NULL, mcp_semantic_sql_error, NULL, NULL), + SQLITE_OK); + + char *response = cbm_mcp_handle_tool( + srv, "search_graph", + "{\"project\":\"semantic-page-underfull\",\"semantic_query\":[42],\"format\":\"json\"}"); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\"isError\":true")); + char *inner = extract_text_content(response); + ASSERT_NOT_NULL(strstr(inner, "semantic_query must be an array of keyword strings")); + ASSERT_NULL(strstr(inner, "semantic search failed")); + + char *combined_response = + cbm_mcp_handle_tool(srv, "search_graph", + "{\"project\":\"semantic-page-underfull\",\"query\":\"strong_match\"," + "\"semantic_query\":[\"horizontal\",42],\"format\":\"json\"}"); + ASSERT_NOT_NULL(combined_response); + ASSERT_NOT_NULL(strstr(combined_response, "\"isError\":true")); + char *combined_inner = extract_text_content(combined_response); + ASSERT_NOT_NULL(strstr(combined_inner, "semantic_query must be an array of keyword strings")); + ASSERT_NULL(strstr(combined_inner, "semantic search failed")); + ASSERT_NULL(strstr(combined_response, "\"search_mode\":\"bm25\"")); + + free(combined_inner); + free(combined_response); + free(inner); + free(response); + cbm_mcp_server_free(srv); + PASS(); +} + /* callers_total/callees_total must count what the caller can enumerate: with * include_tests=false (default) test-file rows are hidden from the table, so * the totals must apply the same filter — a raw visited_count overstated the @@ -2541,15 +3215,42 @@ TEST(mcp_resource_discovery_methods_return_empty_lists) { TEST(tool_query_graph_basic) { cbm_mcp_server_t *srv = setup_mcp_with_data(); + cbm_store_t *store = cbm_mcp_server_store(srv); + const char *project = "query-bounds"; + ASSERT_EQ(cbm_store_upsert_project(store, project, "/tmp/query-bounds"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, project); + cbm_node_t node = {.project = project, + .label = "Function", + .name = "bounded", + .qualified_name = "query-bounds.bounded", + .file_path = "bounded.c", + .start_line = 1, + .end_line = 1}; + ASSERT_GT(cbm_store_upsert_node(store, &node), 0); - char *resp = cbm_mcp_server_handle( - srv, "{\"jsonrpc\":\"2.0\",\"id\":14,\"method\":\"tools/call\"," - "\"params\":{\"name\":\"query_graph\"," - "\"arguments\":{\"query\":\"MATCH (f:Function) RETURN f.name\"}}}"); + char *resp = + cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":14,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"query_graph\"," + "\"arguments\":{\"project\":\"query-bounds\"," + "\"query\":\"MATCH (f:Function) RETURN f.name\"}}}"); ASSERT_NOT_NULL(resp); ASSERT_NOT_NULL(strstr(resp, "\"result\"")); free(resp); + /* Semantic pagination's overflow handling must not turn unrelated integer + * arguments into INT_MAX-sized work. Preserve query_graph's bounded default + * after the shared getter narrows UINT64_MAX to a nonpositive value. */ + resp = cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":15,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"query_graph\"," + "\"arguments\":{\"project\":\"query-bounds\"," + "\"query\":\"MATCH (f:Function) RETURN f.name\"," + "\"max_rows\":18446744073709551615}}}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "\"result\"")); + ASSERT_NULL(strstr(resp, "\"isError\":true")); + ASSERT_NOT_NULL(strstr(resp, "bounded")); + free(resp); + cbm_mcp_server_free(srv); PASS(); } @@ -11553,6 +12254,10 @@ SUITE(mcp) { RUN_TEST(tool_unknown_tool); RUN_TEST(tool_search_graph_basic); RUN_TEST(tool_search_graph_semantic_only_skips_structural_results_issue1295); + RUN_TEST(tool_search_graph_semantic_pagination_stable_window_issue915); + RUN_TEST(tool_search_graph_semantic_pagination_envelopes_issue915); + RUN_TEST(tool_search_graph_semantic_store_error_has_no_partial_results_issue915); + RUN_TEST(tool_search_graph_semantic_invalid_element_type_issue915); RUN_TEST(tool_trace_totals_respect_test_filter); RUN_TEST(tool_trace_totals_respect_test_filter_tests_root_subtree_issue1294); RUN_TEST(tool_get_architecture_cycles_detects_scc);