From ec28e66434bc03ec33dcd05563b3bfd7f58efe86 Mon Sep 17 00:00:00 2001 From: XIYBHK Date: Wed, 2 Sep 2026 15:57:28 +0800 Subject: [PATCH] fix(parsing): recover export-macro-hidden type definitions (#1989) Collect a bounded set of conventional export macro candidates and inject empty definitions into the existing C/C++ preprocessing second pass without overriding explicit caller definitions. Conservatively reconcile remapped definitions to recover hidden classes, structs, enums, free functions, and inline methods while suppressing matching base-class phantom callables. Add focused regression coverage for supported suffixes, ordinary all-caps negative controls, candidate limits, comments, strings, raw strings, overlong names, explicit define priority, and C/C++ extraction. Local MinGW extraction tests pass (344/344). ASan/UBSan were not available in the local MinGW toolchain and remain covered by upstream CI. Signed-off-by: XIYBHK --- internal/cbm/cbm.c | 250 +++++++++++++++++++++- internal/cbm/preprocessor.cpp | 305 ++++++++++++++++++++++++++- internal/cbm/preprocessor.h | 15 ++ tests/test_extraction.c | 384 ++++++++++++++++++++++++++++++++++ 4 files changed, 944 insertions(+), 10 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 97f00ca2a..3208bc986 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -1029,6 +1029,63 @@ static bool cbm_span_contains_callable_def(const char *src, int src_len, uint32_ return false; } +/* #1989: type-def counterpart of cbm_span_contains_callable_def. A rescued + * class/struct/enum definition has its name followed by a base clause (':') or + * a body ('{'), not a parameter list, so the callable validator would reject + * every type def pulled from the expanded tree. */ +static bool cbm_span_contains_type_def(const char *src, int src_len, uint32_t start_line, + uint32_t end_line, const char *name) { + if (!src || src_len <= 0 || !name || !name[0] || start_line == 0 || end_line < start_line) { + return false; + } + int span_start = 0; + uint32_t line = 1; + while (span_start < src_len && line < start_line) { + if (src[span_start++] == '\n') { + line++; + } + } + if (line != start_line) { + return false; + } + int span_end = span_start; + while (span_end < src_len && line <= end_line) { + if (src[span_end++] == '\n') { + line++; + } + } + size_t name_len = strlen(name); + for (int pos = span_start; pos + (int)name_len <= span_end; pos++) { + if (strncmp(src + pos, name, name_len) != 0 || + (pos > 0 && cbm_identifier_char(src[pos - 1])) || + (pos + (int)name_len < src_len && cbm_identifier_char(src[pos + name_len]))) { + continue; + } + int open = pos + (int)name_len; + while (open < span_end && isspace((unsigned char)src[open])) { + open++; + } + // Base clause (": public B"), body ("{"), or template args ("<..."). + if (open < span_end && (src[open] == ':' || src[open] == '{' || src[open] == '<')) { + return true; + } + } + return false; +} + +/* #1989: label buckets for the misparse-correction gates. A raw def extracted + * from a broken `class MOD_API Foo` shape lands under a callable label while + * the expanded tree yields a proper type label for the same name. */ +static bool cbm_def_label_is_type(const char *label) { + return label != NULL && (strcmp(label, "Class") == 0 || strcmp(label, "Enum") == 0 || + strcmp(label, "Interface") == 0 || strcmp(label, "Type") == 0 || + strcmp(label, "Struct") == 0 || strcmp(label, "Union") == 0); +} + +static bool cbm_def_label_is_callable(const char *label) { + return label != NULL && (strcmp(label, "Function") == 0 || strcmp(label, "Method") == 0); +} + /* Remap an expanded-source definition back to the original input file. Every * line in the definition must be attributable to the main file; generated * macro bodies, included headers, and ambiguous spans fail closed. */ @@ -1423,6 +1480,14 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C // Defs keep original-source line numbers; only CALLS are extracted from expanded source. if (language == CBM_LANG_C || language == CBM_LANG_CPP || language == CBM_LANG_CUDA) { uint64_t pp_start = now_ns(); + /* #1989: collect build-system export-macro candidates from the raw + * source. They are predefined empty in the expanded buffer (see + * preprocessor.cpp) so `class MOD_API Foo` parses with its real name, + * and the rescue loop below can adopt corrected type defs. Bounded: + * at most CBM_EXPORT_MACRO_MAX names per file. */ + char export_cands[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int export_cand_count = + cbm_export_macro_candidates(source, source_len, export_cands, CBM_EXPORT_MACRO_MAX); CBMPreprocessedSource *preprocessed = cbm_preprocess_with_map( source, source_len, rel_path, extra_defines, include_paths, language != CBM_LANG_C); if (preprocessed && preprocessed->source) { @@ -1503,17 +1568,51 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C * recover defs from it — adopting ONLY those that * intersect a raw ERROR region, whose name is visible on * the raw source line, and whose QN the raw pass did not - * already extract. */ - if (ts_node_has_error(root)) { + * already extract. + * + * #1989: build-system export macros (_API, + * _EXPORT, ...) are empty on the real compile line but + * opaque to tree-sitter, so `class MOD_API Foo` misparses + * with the macro as the type name — often with NO raw ERROR + * region at all (the class_specifier "succeeds" with the + * wrong name; enums and free functions do error out). When + * export-macro candidates were collected for this file, also + * adopt expanded defs that replace a raw def literally named + * as a candidate, or that correct a raw misparse label (the + * raw pass extracted the same name as Function/Method from + * the broken function_definition shape; the expanded tree + * yields a proper type def). Superseded raw defs are + * dropped so the corrected definition is not shadowed by + * the qualified-name dedup. With no candidates the block + * reduces to the original #961 behavior. */ + if (ts_node_has_error(root) || export_cand_count > 0) { cbm_error_regions_t raw_regs = {{0}, {0}, 0}; cbm_collect_error_regions(root, &raw_regs, source, source_len); - if (raw_regs.count > 0) { + if (raw_regs.count > 0 || export_cand_count > 0) { int defs_before = result->defs.count; cbm_extract_definitions(&pp_ctx); int w = defs_before; + char *superseded = NULL; + if (export_cand_count > 0 && defs_before > 0) { + superseded = (char *)calloc((size_t)defs_before, 1); + } + /* #1989: spans of defs already adopted from the + * expanded tree. A def nested inside one of these + * (an inline method of a rescued class) was dropped + * by the raw pass — the misparsed + * `class MOD_API Foo {...}` parsed as a + * function_definition, and nested defs in its body + * are never walked. Bounded: one span per adopted + * def, capped at 64. */ + struct { + uint32_t start; + uint32_t end; + } rescued_spans[64]; + int rescued_count = 0; for (int i = defs_before; i < result->defs.count; i++) { CBMDefinition *d = &result->defs.items[i]; bool adopt = false; + int supersede_j = -1; if (cbm_remap_preprocessed_def(d, preprocessed)) { for (int rj = 0; rj < raw_regs.count && !adopt; rj++) { if (d->start_line <= raw_regs.ends[rj] && @@ -1521,26 +1620,159 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C adopt = true; } } + if (!adopt && export_cand_count > 0 && d->name) { + for (int ci = 0; ci < export_cand_count && !adopt; ci++) { + for (int j = 0; j < defs_before && !adopt; j++) { + CBMDefinition *r = &result->defs.items[j]; + if (!r->name || + strcmp(r->name, export_cands[ci]) != 0) { + continue; + } + if (r->start_line <= d->end_line && + d->start_line <= r->end_line && + cbm_span_contains_type_def( + source, source_len, d->start_line, + d->end_line, d->name)) { + adopt = true; + supersede_j = j; + } + } + } + if (!adopt && d->label && cbm_def_label_is_type(d->label)) { + for (int j = 0; j < defs_before && !adopt; j++) { + CBMDefinition *r = &result->defs.items[j]; + /* Widened past callables (#1989): the + * misparse also mints value-label raw + * defs for types (`class X_API FEmpty {}` + * -> Variable FEmpty). Same name, + * overlapping span, raw label is NOT a + * type => raw is the misparse artifact. */ + if (!r->name || !r->label || !d->name || + strcmp(r->name, d->name) != 0 || + cbm_def_label_is_type(r->label)) { + continue; + } + if (r->start_line <= d->end_line && + d->start_line <= r->end_line) { + adopt = true; + supersede_j = j; + } + } + } + /* Nested rescue (#1989): a def nested inside + * an already-adopted def's span — the inline + * method of a rescued class, dropped by the + * raw pass because the class parsed as a + * function body. The same validation gates + * (name-on-line, shape, QN dedup) still run + * below before it lands. */ + if (!adopt && rescued_count > 0) { + for (int k = 0; k < rescued_count && !adopt; k++) { + if (d->start_line >= rescued_spans[k].start && + d->end_line <= rescued_spans[k].end) { + adopt = true; + } + } + } + } } - if (adopt && (!d->name || - !cbm_line_contains(source, source_len, d->start_line, - d->name) || - !cbm_span_contains_callable_def( - source, source_len, d->start_line, d->end_line, - d->name))) { + if (adopt && + (!d->name || + !cbm_line_contains(source, source_len, d->start_line, + d->name) || + (!cbm_span_contains_callable_def(source, source_len, + d->start_line, d->end_line, + d->name) && + !cbm_span_contains_type_def(source, source_len, d->start_line, + d->end_line, d->name)))) { adopt = false; } for (int j = 0; j < defs_before && adopt; j++) { + if (j == supersede_j || (superseded && superseded[j])) { + continue; // being replaced by this very def + } const char *q = result->defs.items[j].qualified_name; if (q && d->qualified_name && strcmp(q, d->qualified_name) == 0) { + /* #1989: a raw def holding the same QN is + * usually a reject — unless it is the + * misparse artifact this very def came to + * replace (raw "Function FCalc" from the + * broken `class MOD_API FCalc` shape vs the + * rescued "Class FCalc"). Same name, same + * span, raw label is not a type: supersede + * instead of dropping the correction. */ + CBMDefinition *r = &result->defs.items[j]; + if (d->label && cbm_def_label_is_type(d->label) && + r->label && r->name && d->name && + strcmp(r->name, d->name) == 0 && + !cbm_def_label_is_type(r->label) && + r->start_line == d->start_line && + r->end_line == d->end_line && superseded) { + superseded[j] = 1; + continue; + } adopt = false; } } if (adopt) { + if (supersede_j >= 0 && supersede_j < defs_before && + superseded) { + superseded[supersede_j] = 1; + } + /* Phantom-artifact suppression (#1989): on the + * raw tree `class MOD_API Foo : public Base + * {...}` parses as a function_definition whose + * declarator is the BASE-CLASS name, so the raw + * pass mints a bogus Function def named after + * the base spanning the whole class. Fingerprint: + * a raw callable def with an IDENTICAL span to + * the adopted type def whose name is one of the + * adopted def's base classes. A real method is + * a strict subset of the class span and never + * carries a base-class name. */ + if (d->label && cbm_def_label_is_type(d->label) && superseded && + d->base_classes) { + for (int j = 0; j < defs_before; j++) { + CBMDefinition *r = &result->defs.items[j]; + if (!r->label || !r->name || + !cbm_def_label_is_callable(r->label) || + r->start_line != d->start_line || + r->end_line != d->end_line) { + continue; + } + for (const char **b = d->base_classes; *b; b++) { + if (strcmp(*b, r->name) == 0) { + superseded[j] = 1; + break; + } + } + } + } + if (rescued_count < 64) { + rescued_spans[rescued_count].start = d->start_line; + rescued_spans[rescued_count].end = d->end_line; + rescued_count++; + } result->defs.items[w++] = *d; } } + if (superseded) { + int rw = 0; + for (int j = 0; j < defs_before; j++) { + if (!superseded[j]) { + result->defs.items[rw++] = result->defs.items[j]; + } + } + int shift = defs_before - rw; + if (shift > 0 && w > defs_before) { + memmove(&result->defs.items[rw], + &result->defs.items[defs_before], + (size_t)(w - defs_before) * sizeof(CBMDefinition)); + } + w -= shift; + free(superseded); + } result->defs.count = w; } } diff --git a/internal/cbm/preprocessor.cpp b/internal/cbm/preprocessor.cpp index 54cb073fe..9be6298f3 100644 --- a/internal/cbm/preprocessor.cpp +++ b/internal/cbm/preprocessor.cpp @@ -42,6 +42,283 @@ static bool has_preprocessor_work(const char *source, int source_len) { return false; } +// ── Export-macro candidates (#1989) ───────────────────────────────────────── +// Build systems define symbol-export macros empty on the compiler command line +// (UE's UBT: `/D "MODULE_API="`; CMake generate_export_header: `_EXPORT`), +// so they never appear as #define lines in the source. tree-sitter has no +// preprocessor state either, and `class MODULE_API Foo` misparses with the +// macro token as the type name (worse: enums and free functions are lost to +// ERROR regions entirely). To mirror the real compile line, the preprocessed +// second pass predefines the conventional export-macro-shaped identifiers found +// in the file as empty. The shape is deliberately narrow — ALL_CAPS identifier +// ending in a known export suffix, with a non-trivial prefix — and the list is +// capped, so ordinary all-caps identifiers cannot be swept in wholesale. +static const char *kExportMacroSuffixes[] = {"_API", "_EXPORT", "_IMPORT", + "_DLLEXPORT", "_DEPRECATED", NULL}; + +static bool is_export_macro_shape(const char *id, size_t len) { + if (id[0] < 'A' || id[0] > 'Z') { + return false; + } + for (size_t i = 0; i < len; i++) { + char c = id[i]; + bool upper = c >= 'A' && c <= 'Z'; + bool digit = c >= '0' && c <= '9'; + if (!upper && !digit && c != '_') { + return false; + } + } + for (int s = 0; kExportMacroSuffixes[s]; s++) { + size_t slen = strlen(kExportMacroSuffixes[s]); + // Require at least two prefix characters before the suffix so a bare + // "X_API"-style token (single-letter, easily a genuine symbol) stays out. + if (len >= slen + 2 && strncmp(id + len - slen, kExportMacroSuffixes[s], slen) == 0) { + return true; + } + } + return false; +} + +static bool is_identifier_start(char c) { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; +} + +static bool is_identifier_char(char c) { + return is_identifier_start(c) || (c >= '0' && c <= '9'); +} + +// C++ raw string literals (R"delim(...)delim"). Returns the index one past +// the closing quote, or -1 when the form cannot be confidently recognized +// (unterminated, malformed delimiter) — the caller then STOPS collecting so +// the mis-modeled literal cannot poison the scan state of the rest of the +// file (#1989 review). +static int skip_raw_string(const char *source, int source_len, int quote) { + int j = quote + 1; + int delim_start = j; + while (j < source_len) { + char c = source[j]; + if (c == '(') { + break; + } + // Delimiter chars: the standard d-char set excludes space, parens, + // backslash, and the line breaks — a double quote IS a legal d-char + // (R"""(...)""" compiles), so it must not end the delimiter (#1989 + // review round 3). + if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == ')' || c == '\\' || + j - delim_start >= 16) { + return -1; + } + j++; + } + if (j >= source_len || source[j] != '(') { + return -1; + } + int delim_len = j - delim_start; + j++; // past the opening "(" + while (j < source_len) { + if (source[j] == ')') { + int k = j + 1; + int m = 0; + while (m < delim_len && k + m < source_len && + source[k + m] == source[delim_start + m]) { + m++; + } + if (m == delim_len && k + m < source_len && source[k + m] == '"') { + return k + m + 1; + } + } + j++; + } + return -1; // unterminated raw string: uncertain +} + +// If the identifier just scanned is a raw-string prefix (R, LR, uR, UR, u8R) +// immediately followed by a quote, return skip_raw_string(...). Returns 0 for +// an ordinary identifier (including an identifier merely ENDING in R — only a +// standalone prefix token starts a raw string) and -1 on an unrecognizable +// raw form. +static int raw_string_after_prefix(const char *source, int source_len, int start, int end) { + static const char *prefixes[] = {"R", "LR", "uR", "UR", "u8R", NULL}; + size_t idlen = (size_t)(end - start); + bool is_prefix = false; + for (int p = 0; prefixes[p]; p++) { + if (idlen == strlen(prefixes[p]) && strncmp(source + start, prefixes[p], idlen) == 0) { + is_prefix = true; + break; + } + } + if (!is_prefix || end >= source_len || source[end] != '"') { + return 0; + } + return skip_raw_string(source, source_len, end); +} + +// Advance past comments and string/char literals so only real code tokens +// consume the candidate budget (#1989 review). Handles line-spliced // +// comments (a backslash before the newline continues the comment — line +// splicing happens before comment recognition) and, via the identifier-scan +// hook in the callers, C++ raw string literals. A digit separator (1'000'000) +// can still over-skip — that fails safe, toward NOT collecting a candidate. +static int skip_non_code(const char *source, int source_len, int i) { + while (i < source_len) { + char c = source[i]; + if (c == '/' && i + 1 < source_len && source[i + 1] == '/') { + i += 2; + for (;;) { + while (i < source_len && source[i] != '\n') { + i++; + } + if (i >= source_len) { + break; + } + // Line splice: a backslash right before the newline (allowing + // \r\n) means the comment continues on the next line. + int back = i - 1; + if (back >= 0 && source[back] == '\r') { + back--; + } + if (back >= 0 && source[back] == '\\') { + i++; // spliced: keep consuming inside the comment + continue; + } + i++; // real end-of-comment newline + break; + } + } else if (c == '/' && i + 1 < source_len && source[i + 1] == '*') { + i += 2; + while (i + 1 < source_len && !(source[i] == '*' && source[i + 1] == '/')) { + i++; + } + if (i + 1 < source_len) { + i += 2; // past the closing "*/" + } else { + i = source_len; // unterminated comment runs to EOF + } + } else if (c == '"' || c == '\'') { + char quote = c; + i++; + while (i < source_len) { + if (source[i] == '\\') { + i += 2; + continue; + } + if (source[i] == quote) { + i++; + break; + } + i++; + } + } else { + return i; + } + } + return i; +} + +// Count-only scan used by the early gate (no storage). +static bool has_export_macro_candidates(const char *source, int source_len) { + if (!source || source_len <= 0) { + return false; + } + int i = 0; + while (i < source_len) { + i = skip_non_code(source, source_len, i); + if (i >= source_len) { + break; + } + if (!is_identifier_start(source[i])) { + i++; + continue; + } + int start = i; + while (i < source_len && is_identifier_char(source[i])) { + i++; + } + // A raw-string prefix consumes everything up to its closing quote; + // an unrecognizable raw form poisons the scan state, so fail toward + // NOT running the second pass (raw behavior preserved). + int raw = raw_string_after_prefix(source, source_len, start, i); + if (raw < 0) { + return false; + } + if (raw > 0) { + i = raw; + continue; + } + if (is_export_macro_shape(source + start, (size_t)(i - start))) { + return true; + } + } + return false; +} + +static int collect_export_macro_candidates(const char *source, int source_len, + char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out) { + if (!source || source_len <= 0 || !out || max_out <= 0) { + return 0; + } + int stored = 0; + int i = 0; + while (i < source_len) { + i = skip_non_code(source, source_len, i); + if (i >= source_len) { + break; + } + if (!is_identifier_start(source[i])) { + i++; + continue; + } + int start = i; + while (i < source_len && is_identifier_char(source[i])) { + i++; + } + // Raw string (see has_export_macro_candidates): skip its body, or stop + // collecting entirely when the form cannot be confidently recognized — + // continuing would mis-tokenize the rest of the file and could both + // mint phantom candidates and mask real ones (#1989 review). + int raw = raw_string_after_prefix(source, source_len, start, i); + if (raw < 0) { + break; + } + if (raw > 0) { + i = raw; + continue; + } + size_t len = (size_t)(i - start); + /* Length gate FIRST: the dedup below reads out[k][0..len] against rows + * of CBM_EXPORT_MACRO_NAME_MAX bytes — an over-long candidate must be + * rejected before any compare touches them. */ + if (len >= CBM_EXPORT_MACRO_NAME_MAX) { + continue; + } + if (!is_export_macro_shape(source + start, len)) { + continue; + } + bool dup = false; + for (int k = 0; k < stored; k++) { + if (strncmp(out[k], source + start, len) == 0 && out[k][len] == '\0') { + dup = true; + break; + } + } + if (dup) { + continue; + } + if (stored >= max_out) { + break; // bounded: stop collecting once the cap is reached + } + memcpy(out[stored], source + start, len); + out[stored][len] = '\0'; + stored++; + } + return stored; +} + +int cbm_export_macro_candidates(const char *source, int source_len, + char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out) { + return collect_export_macro_candidates(source, source_len, out, max_out); +} + static int count_expanded_lines(const std::string &text) { int count = 1; for (char c : text) { @@ -146,7 +423,11 @@ static bool build_line_map(const std::string &expanded, const std::string &main_ CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_len, const char *filename, const char **extra_defines, const char **include_paths, int cpp_mode) { - if (!has_preprocessor_work(source, source_len)) { + // Run the second pass when there are directives to evaluate OR when the file + // carries export-macro-shaped identifiers to predefine empty (#1989) — a UE + // plugin header with only `#pragma once` + `#include` lines still needs it. + if (!has_preprocessor_work(source, source_len) && + !has_export_macro_candidates(source, source_len)) { return NULL; // NULL = no expansion needed, use original } @@ -156,6 +437,28 @@ CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_le for (int i = 0; extra_defines[i]; i++) dui.defines.push_back(extra_defines[i]); } + // Predefine collected export-macro candidates as empty, mirroring the + // real compile command line (`/D "MODULE_API="`). Names already provided + // by the caller win — never override an explicit define. + char export_cands[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int export_cand_count = + collect_export_macro_candidates(source, source_len, export_cands, CBM_EXPORT_MACRO_MAX); + for (int i = 0; i < export_cand_count; i++) { + bool provided = false; + for (std::list::const_iterator it = dui.defines.begin(); + it != dui.defines.end(); ++it) { + const std::string &def = *it; + size_t eq = def.find('='); + std::string name = (eq == std::string::npos) ? def : def.substr(0, eq); + if (name == export_cands[i]) { + provided = true; + break; + } + } + if (!provided) { + dui.defines.push_back(std::string(export_cands[i]) + "="); + } + } if (include_paths) { for (int i = 0; include_paths[i]; i++) dui.includePaths.push_back(include_paths[i]); diff --git a/internal/cbm/preprocessor.h b/internal/cbm/preprocessor.h index 72828c012..1b108b335 100644 --- a/internal/cbm/preprocessor.h +++ b/internal/cbm/preprocessor.h @@ -4,6 +4,15 @@ #include #include +/* #1989: build-system export macros (UE "_API", CMake generate_export_header + * "_EXPORT", ...) are defined empty on the real compile command line but are + * invisible to tree-sitter, so `class MOD_API Foo` misparses with the macro token + * as the type name. The preprocessed second pass predefines the conventional + * export-macro-shaped identifiers found in a source file as empty, mirroring the + * real build line. The candidate list is bounded. */ +#define CBM_EXPORT_MACRO_MAX 32 +#define CBM_EXPORT_MACRO_NAME_MAX 96 + #ifdef __cplusplus extern "C" { #endif @@ -15,6 +24,12 @@ typedef struct { int expanded_line_count; } CBMPreprocessedSource; +// Collect conventional export-macro-shaped identifiers (ALL_CAPS ending in +// _API/_EXPORT/_IMPORT/_DLLEXPORT/_DEPRECATED) from the source, up to max_out +// unique names. Returns the number of names stored in out. +int cbm_export_macro_candidates(const char *source, int source_len, + char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out); + // Preprocess C/C++ source: expand macros, evaluate #ifdef, resolve #include. // Returns malloc-allocated expanded source, or NULL if no expansion needed/on failure. // extra_defines: NULL-terminated array of "NAME=VALUE" strings (can be NULL). diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 7ea702c3a..0bcf03a66 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -8,6 +8,7 @@ #include "test_framework.h" #include "cbm.h" #include "foundation/constants.h" /* CBM_SZ_* */ +#include "preprocessor.h" /* cbm_export_macro_candidates (#1989) */ #include "../src/foundation/compat.h" /* cbm_clock_gettime (wide-flat scaling guard) */ #include "../src/foundation/compat_fs.h" #include @@ -5495,6 +5496,370 @@ TEST(extract_c_clean_file_no_recovery_duplicates_issue961) { PASS(); } +/* #1989: build-system export macros (UE "_API", CMake generate_export_header + * "_EXPORT") are empty on the real compile line but opaque to tree-sitter, + * so `class MOD_API Foo` misparses. The preprocessed second pass predefines the + * conventional export-macro-shaped identifiers found in the file as empty, and + * the recovery loop adopts the corrected type defs (superseding the raw + * misparse artifacts). */ +TEST(extract_cpp_export_macro_class_recovery_issue1989) { + CBMFileResult *r = extract("#pragma once\n" + "UCLASS()\n" + "class DUMMY_API UFoo : public UObject\n" + "{\n" + " GENERATED_BODY()\n" + "public:\n" + " int32 X = 0;\n" + "};\n", + CBM_LANG_CPP, "p", "ue.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "UFoo"); + ASSERT_NOT_NULL(cls); /* was missing entirely before the fix */ + ASSERT_EQ(cls->start_line, 3u); + ASSERT_NULL(find_def(r, "DUMMY_API")); /* macro-as-name artifact superseded */ + /* The raw misparse also mints a phantom Function def NAMED AFTER THE BASE + * CLASS spanning the whole class ("UObject" from the declarator of the + * broken function_definition shape) — it must be suppressed, not just the + * macro-named artifact. */ + ASSERT_NULL(find_def(r, "UObject")); + cbm_free_result(r); + PASS(); +} + +TEST(extract_cpp_export_macro_struct_recovery_issue1989) { + /* The struct case is the SILENT failure: the raw tree parses "successfully" + * with the macro token as the name and raises no error flag at all. */ + CBMFileResult *r = extract("#pragma once\n" + "USTRUCT()\n" + "struct DUMMY_API FBar\n" + "{\n" + " int32 Y;\n" + "};\n", + CBM_LANG_CPP, "p", "ue_struct.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *st = find_def(r, "FBar"); + ASSERT_NOT_NULL(st); + ASSERT(st->label && strcmp(st->label, "Class") == 0); /* struct_specifier -> Class */ + ASSERT_NULL(find_def(r, "DUMMY_API")); + cbm_free_result(r); + PASS(); +} + +TEST(extract_cpp_export_macro_enum_recovery_issue1989) { + /* `enum class MOD_API EKind` is lost to an ERROR region entirely. */ + CBMFileResult *r = extract("#pragma once\n" + "UENUM()\n" + "enum class DUMMY_API EKind : uint8\n" + "{\n" + " A,\n" + " B\n" + "};\n", + CBM_LANG_CPP, "p", "ue_enum.h"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(find_def(r, "EKind")); + cbm_free_result(r); + PASS(); +} + +TEST(extract_cpp_export_macro_free_function_recovery_issue1989) { + /* A free function DEFINED with the export macro is lost to an ERROR region + * on the raw tree (prototypes never mint defs — same as clean code). */ + CBMFileResult *r = extract("#pragma once\n" + "DUMMY_API int Add(int a, int b)\n" + "{\n" + " return a + b;\n" + "}\n", + CBM_LANG_CPP, "p", "ue_fn.h"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(find_def(r, "Add")); + cbm_free_result(r); + PASS(); +} + +TEST(extract_cpp_export_macro_inline_method_recovery_issue1989) { + /* The misparsed class parsed as a function body on the raw tree, so the + * raw walk never extracted its inline methods — the rescued class def must + * carry them in via the nested-span adoption. */ + CBMFileResult *r = extract("#pragma once\n" + "class DUMMY_API FCalc\n" + "{\n" + "public:\n" + " int Add(int a, int b) { return a + b; }\n" + "};\n", + CBM_LANG_CPP, "p", "ue_inline.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "FCalc"); + ASSERT_NOT_NULL(cls); + ASSERT(cls->label && strcmp(cls->label, "Class") == 0); /* not the raw "Function" mislabel */ + const CBMDefinition *m = find_def(r, "Add"); + ASSERT_NOT_NULL(m); + ASSERT(m->label && strcmp(m->label, "Method") == 0); + cbm_free_result(r); + PASS(); +} + +/* Negative control the maintainer asked for: ordinary ALL_CAPS identifiers + * (constants, enum values) must NOT be swept in as export-macro candidates and + * stripped from the graph. Only the narrow _API/_EXPORT/... suffix shape is. */ +TEST(extract_cpp_export_macro_negative_control_ordinary_caps_issue1989) { + CBMFileResult *r = extract("#pragma once\n" + "#define MAX_CONNECTIONS 16\n" + "const int TIMEOUT_MS = 250;\n" + "enum class State { IDLE, RUNNING };\n" + "struct Config { int MAX_RETRIES; };\n", + CBM_LANG_CPP, "p", "caps.h"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(find_def(r, "Config")); + ASSERT_TRUE(has_def(r, "Variable", "TIMEOUT_MS")); + cbm_free_result(r); + PASS(); +} + +/* C files get the same rescue (SQLITE_API-style prefixes are C, not C++). */ +TEST(extract_c_export_macro_recovery_issue1989) { + CBMFileResult *r = extract("typedef struct sqlite3 sqlite3;\n" + "SQLITE_API int sqlite3_open(const char *path)\n" + "{\n" + " return 0;\n" + "}\n", + CBM_LANG_C, "p", "db.c"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(find_def(r, "sqlite3_open")); + cbm_free_result(r); + PASS(); +} + +/* #1989 review (P1): an over-long candidate-shaped identifier (>= 96 chars) + * arriving AFTER a stored candidate must be rejected BEFORE the dedup + * compares — the pre-fix dedup ran strncmp(out[k], src, len) with + * len >= CBM_EXPORT_MACRO_NAME_MAX against 96-byte rows (out-of-bounds read; + * sanitizer lanes crash). Locally unsanitized, so this pins the behavior + * (normal candidate still recovers, no crash); CI's sanitized lanes guard + * the memory error itself. */ +TEST(extract_cpp_export_macro_overlong_candidate_safe_issue1989) { + char src[4096]; + int off = snprintf(src, sizeof(src), "class MOD_API First { int v; };\nclass "); + ASSERT_GTE(off, 0); + memset(src + off, 'A', 100); + off += 100; + ASSERT_GTE(snprintf(src + off, sizeof(src) - (size_t)off, "_API Long { int w; };\n"), 0); + CBMFileResult *r = extract(src, CBM_LANG_CPP, "p", "overlong.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *first = find_def(r, "First"); + ASSERT_NOT_NULL(first); + ASSERT(first->label && strcmp(first->label, "Class") == 0); + cbm_free_result(r); + PASS(); +} + +/* #1989 review (P2): candidate-shaped tokens inside comments and string + * literals must NOT consume the bounded budget — 32 of them ahead of the real + * class would otherwise exhaust the cap and leave the real macro undefined. */ +TEST(extract_cpp_export_macro_comment_string_budget_issue1989) { + char src[8192]; + int off = 0; + for (int n = 0; n < 31; n++) { + ASSERT_GTE(snprintf(src + off, sizeof(src) - (size_t)off, "// see DOC%02d_API notes\n", n), + 0); + off += (int)strlen(src + off); + } + ASSERT_GTE(snprintf(src + off, sizeof(src) - (size_t)off, + "static const char *kRef = \"STRING_DOC_API\";\n" + "class REALMOD_API Real { int v; };\n"), + 0); + CBMFileResult *r = extract(src, CBM_LANG_CPP, "p", "budget.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "Real"); + ASSERT_NOT_NULL(cls); + ASSERT(cls->label && strcmp(cls->label, "Class") == 0); + cbm_free_result(r); + PASS(); +} + +/* #1989 review: the bounded-budget contract — 33 distinct candidates collect + * only the first CBM_EXPORT_MACRO_MAX (32, in source order); the 33rd stays + * unpredefined and its class keeps the raw misparse (value label, not Class). */ +TEST(extract_cpp_export_macro_candidate_cap_issue1989) { + char src[8192]; + int off = 0; + for (int n = 0; n < 33; n++) { + ASSERT_GTE( + snprintf(src + off, sizeof(src) - (size_t)off, "class CAP%02d_API K%02d {};\n", n, n), + 0); + off += (int)strlen(src + off); + } + CBMFileResult *r = extract(src, CBM_LANG_CPP, "p", "cap.h"); + ASSERT_NOT_NULL(r); + int recovered = 0; + for (int n = 0; n < 33; n++) { + char name[8]; + snprintf(name, sizeof(name), "K%02d", n); + const CBMDefinition *d = find_def(r, name); + if (d && d->label && strcmp(d->label, "Class") == 0) { + recovered++; + } + } + ASSERT_EQ(recovered, 32); + /* The cap contract's other half: the 33rd candidate (K32) must STILL be + * present with its raw misparse — dropping the def entirely would be a + * regression, not a graceful cap. */ + const CBMDefinition *cap_tail = find_def(r, "K32"); + ASSERT_NOT_NULL(cap_tail); + ASSERT_TRUE(!cap_tail->label || strcmp(cap_tail->label, "Class") != 0); + cbm_free_result(r); + PASS(); +} + +/* #1989 review round 2: line-spliced // comments. A backslash before the + * newline continues the comment (splicing happens before comment + * recognition), so "FAKE_API" below is comment text and must NOT consume + * budget. 32 spliced fake comments would otherwise exhaust the cap and leave + * the real class unrecovered. */ +TEST(extract_cpp_export_macro_spliced_comment_budget_issue1989) { + char src[8192]; + int off = 0; + for (int n = 0; n < 31; n++) { + ASSERT_GTE( + snprintf(src + off, sizeof(src) - (size_t)off, "// fake \\\nFAKE%02d_API notes\n", n), + 0); + off += (int)strlen(src + off); + } + ASSERT_GTE(snprintf(src + off, sizeof(src) - (size_t)off, + "// last \\\r\nSPLICEFAKE_API notes\n" + "class REALMOD_API Real { int v; };\n"), + 0); + CBMFileResult *r = extract(src, CBM_LANG_CPP, "p", "spliced.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "Real"); + ASSERT_NOT_NULL(cls); + ASSERT(cls->label && strcmp(cls->label, "Class") == 0); + cbm_free_result(r); + PASS(); +} + +/* #1989 review round 2: C++ raw string literals. Tokens inside R"(...)" + * must not be collected, and code after the literal must still be scanned — + * an unmodeled raw string previously let "BAR_API" leak in AND masked the + * real candidate after the literal's closing quote. An unrecognizable raw + * form (unterminated/malformed delimiter) stops collection entirely so scan + * state cannot be poisoned. */ +TEST(extract_cpp_export_macro_raw_string_issue1989) { + CBMFileResult *r = extract("const char *s = R\"(foo \" BAR_API)\";\n" + "class REALMOD_API Real { int v; };\n", + CBM_LANG_CPP, "p", "raw.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "Real"); + ASSERT_NOT_NULL(cls); + ASSERT(cls->label && strcmp(cls->label, "Class") == 0); + cbm_free_result(r); + PASS(); +} + +TEST(extract_cpp_export_macro_raw_string_delimited_issue1989) { + CBMFileResult *r = extract("auto log = R\"log(FOO_API \"quote\")log\";\n" + "class DELIMMOD_API Delim { int v; };\n", + CBM_LANG_CPP, "p", "raw_delim.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "Delim"); + ASSERT_NOT_NULL(cls); + ASSERT(cls->label && strcmp(cls->label, "Class") == 0); + cbm_free_result(r); + PASS(); +} + +/* #1989 review: every conventional suffix variant must be collected and + * recovered, not just _API. */ +TEST(extract_cpp_export_macro_suffix_variants_issue1989) { + CBMFileResult *r = extract("class LIB_EXPORT A1 { int v; };\n" + "class LIB_IMPORT B2 { int v; };\n" + "class LIB_DLLEXPORT C3 { int v; };\n" + "class LIB_DEPRECATED D4 { int v; };\n", + CBM_LANG_CPP, "p", "suffix.h"); + ASSERT_NOT_NULL(r); + const char *names[] = {"A1", "B2", "C3", "D4"}; + for (int n = 0; n < 4; n++) { + const CBMDefinition *d = find_def(r, names[n]); + ASSERT_NOT_NULL(d); + ASSERT(d->label && strcmp(d->label, "Class") == 0); + } + cbm_free_result(r); + PASS(); +} + +/* #1989 review: caller-provided defines win — the collector must not push a + * define for a name the caller already provided (a duplicate could flip the + * expansion and silently change what parses). With an explicit empty define + * the class still recovers; with a caller define that corrupts the header, + * the injected empty define must NOT clobber it back into parsing. */ +TEST(extract_cpp_export_macro_explicit_define_priority_issue1989) { + const char *src = "class MYMOD_API Foo { int v; };\n"; + + const char *empty_defines[] = {"MYMOD_API=", NULL}; + CBMFileResult *r = cbm_extract_file(src, (int)strlen(src), CBM_LANG_CPP, "p", "prio_empty.h", 0, + empty_defines, NULL); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "Foo"); + ASSERT_NOT_NULL(cls); + ASSERT(cls->label && strcmp(cls->label, "Class") == 0); + cbm_free_result(r); + + const char *body_defines[] = {"MYMOD_API=Junk", NULL}; + CBMFileResult *r2 = cbm_extract_file(src, (int)strlen(src), CBM_LANG_CPP, "p", "prio_body.h", 0, + body_defines, NULL); + ASSERT_NOT_NULL(r2); + const CBMDefinition *foo2 = find_def(r2, "Foo"); + ASSERT_TRUE(foo2 == NULL || (foo2->label && strcmp(foo2->label, "Class") != 0)); + cbm_free_result(r2); + PASS(); +} + +/* #1989 review round 2: the collector CONTRACT itself, called directly (the + * reviewer's method): the raw string's inner token is not a candidate and the + * code after the literal IS. */ +TEST(extract_cpp_export_macro_collector_raw_string_contract_issue1989) { + const char *src = "const char *s = R\"(foo \" BAR_API)\";\n" + "class REALMOD_API Real { int v; };\n"; + char out[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int n = cbm_export_macro_candidates(src, (int)strlen(src), out, CBM_EXPORT_MACRO_MAX); + ASSERT_EQ(n, 1); + ASSERT(strcmp(out[0], "REALMOD_API") == 0); + PASS(); +} + +/* Delimiter form R"log(...)log" works the same way. */ +TEST(extract_cpp_export_macro_collector_raw_string_delim_contract_issue1989) { + const char *src = "auto log = R\"log(FOO_API \"q\")log\";\n" + "class DELIMMOD_API Delim { int v; };\n"; + char out[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int n = cbm_export_macro_candidates(src, (int)strlen(src), out, CBM_EXPORT_MACRO_MAX); + ASSERT_EQ(n, 1); + ASSERT(strcmp(out[0], "DELIMMOD_API") == 0); + PASS(); +} + +/* A double quote is a legal raw-string delimiter character. Keep scanning + * after R"""(...)""" so a later export macro is still collected. */ +TEST(extract_cpp_export_macro_collector_raw_string_quote_delim_issue1989) { + const char *src = "const char *s = R\"\"\"(FOO_API)\"\"\";\n" + "class QUOTEMOD_API Quoted { int v; };\n"; + char out[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int n = cbm_export_macro_candidates(src, (int)strlen(src), out, CBM_EXPORT_MACRO_MAX); + ASSERT_EQ(n, 1); + ASSERT(strcmp(out[0], "QUOTEMOD_API") == 0); + PASS(); +} + +/* Unterminated raw literal: the scan stops rather than mis-tokenizing the + * rest of the file (uncertain state -> fail toward raw behavior). */ +TEST(extract_cpp_export_macro_collector_raw_string_unterminated_issue1989) { + const char *src = "const char *s = R\"(never closed BAR_API\n" + "class REALMOD_API Real { int v; };\n"; + char out[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int n = cbm_export_macro_candidates(src, (int)strlen(src), out, CBM_EXPORT_MACRO_MAX); + ASSERT_EQ(n, 0); + PASS(); +} + /* #668: walk_defs used a fixed `walk_defs_frame_t stack[4096]` — a ~160 KB * C-stack frame that overflowed small thread stacks (the reporter's crash was in * the "definitions pass" on a large SQL file), and whose `top < 4096` push guards @@ -7243,6 +7608,25 @@ SUITE(extraction) { RUN_TEST(extract_cpp_preproc_macro_generated_callable_skipped_issue949); RUN_TEST(extract_c_ifdef_split_brace_after_include_remapped_issue949); RUN_TEST(extract_c_clean_file_no_recovery_duplicates_issue961); + RUN_TEST(extract_cpp_export_macro_class_recovery_issue1989); + RUN_TEST(extract_cpp_export_macro_struct_recovery_issue1989); + RUN_TEST(extract_cpp_export_macro_enum_recovery_issue1989); + RUN_TEST(extract_cpp_export_macro_free_function_recovery_issue1989); + RUN_TEST(extract_cpp_export_macro_inline_method_recovery_issue1989); + RUN_TEST(extract_cpp_export_macro_negative_control_ordinary_caps_issue1989); + RUN_TEST(extract_c_export_macro_recovery_issue1989); + RUN_TEST(extract_cpp_export_macro_overlong_candidate_safe_issue1989); + RUN_TEST(extract_cpp_export_macro_comment_string_budget_issue1989); + RUN_TEST(extract_cpp_export_macro_candidate_cap_issue1989); + RUN_TEST(extract_cpp_export_macro_spliced_comment_budget_issue1989); + RUN_TEST(extract_cpp_export_macro_raw_string_issue1989); + RUN_TEST(extract_cpp_export_macro_raw_string_delimited_issue1989); + RUN_TEST(extract_cpp_export_macro_collector_raw_string_contract_issue1989); + RUN_TEST(extract_cpp_export_macro_collector_raw_string_delim_contract_issue1989); + RUN_TEST(extract_cpp_export_macro_collector_raw_string_quote_delim_issue1989); + RUN_TEST(extract_cpp_export_macro_collector_raw_string_unterminated_issue1989); + RUN_TEST(extract_cpp_export_macro_suffix_variants_issue1989); + RUN_TEST(extract_cpp_export_macro_explicit_define_priority_issue1989); RUN_TEST(walk_defs_no_truncation_over_4096_issue668); RUN_TEST(extract_rust_test_attr_marks_is_test_issue855); RUN_TEST(extract_c_test_dir_marks_is_test_issue1294);