From 4aaf139444eeee458ea0f3ca7eeeb1e806b4bcc7 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Fri, 30 Oct 2026 12:03:41 +0700 Subject: [PATCH 1/2] fix: keep C and C++ functions that return pointers or references in the graph The C and C++ analyzers looked for a function's function_declarator as a direct child of its function_definition. When the return type is a pointer (`const char *f()`, `T **f()`) or a C++ reference (`T &f()`), tree-sitter wraps it in a pointer/reference declarator, so the function was left out of the dependency graph, together with every call made from inside it. A function returning a function pointer was also lost. An export macro before a typedef'd return type (`API lua_Integer f()`) was named after its return type: tree-sitter reads the type as the declarator and leaves the real name in an ERROR node before the parameters. Both analyzers now find the declarator through those wrappers (shared helper in analyzers/c_declarators.py) and read the macro case's name from the ERROR node. --- .../src/be/dependency_analyzer/analyzers/c.py | 38 ++++++----- .../analyzers/c_declarators.py | 51 ++++++++++++++ .../be/dependency_analyzer/analyzers/cpp.py | 35 +++++----- tests/test_c_cpp_declarators.py | 66 +++++++++++++++++++ 4 files changed, 154 insertions(+), 36 deletions(-) create mode 100644 codewiki/src/be/dependency_analyzer/analyzers/c_declarators.py create mode 100644 tests/test_c_cpp_declarators.py diff --git a/codewiki/src/be/dependency_analyzer/analyzers/c.py b/codewiki/src/be/dependency_analyzer/analyzers/c.py index fca53932..61493a1d 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/c.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/c.py @@ -5,6 +5,10 @@ from tree_sitter import Parser, Language import tree_sitter_c +from codewiki.src.be.dependency_analyzer.analyzers.c_declarators import ( + function_declarator, + macro_typed_name, +) from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) @@ -70,12 +74,7 @@ def _extract_nodes(self, node, top_level_nodes, lines): if node.type == "function_definition": node_type = "function" - # look for function_declarator - declarator = next((c for c in node.children if c.type == "function_declarator"), None) - if declarator: - identifier = next((c for c in declarator.children if c.type == "identifier"), None) - if identifier: - node_name = identifier.text.decode() + node_name = self._function_name(node) elif node.type == "struct_specifier": # Extract struct definitions: struct Name { ... } node_type = "struct" @@ -215,21 +214,24 @@ def _find_containing_function(self, node, top_level_nodes): current = node.parent while current: if current.type == "function_definition": - # Get function name - declarator = next( - (c for c in current.children if c.type == "function_declarator"), None - ) - if declarator: - identifier = next( - (c for c in declarator.children if c.type == "identifier"), None - ) - if identifier: - func_name = identifier.text.decode() - if func_name in top_level_nodes: - return func_name + func_name = self._function_name(current) + if func_name in top_level_nodes: + return func_name current = current.parent return None + def _function_name(self, definition): + """The name a function_definition defines, also when its return type is a pointer (`char *f()`) or an + export macro stands before a typedef'd return type (`API lua_Integer f()`).""" + declarator = function_declarator(definition) + if declarator is None: + return None + name = macro_typed_name(declarator) + if name: + return name + identifier = next((c for c in declarator.children if c.type == "identifier"), None) + return identifier.text.decode() if identifier else None + def analyze_c_file( file_path: str, content: str, repo_path: str = None diff --git a/codewiki/src/be/dependency_analyzer/analyzers/c_declarators.py b/codewiki/src/be/dependency_analyzer/analyzers/c_declarators.py new file mode 100644 index 00000000..ce6a59a9 --- /dev/null +++ b/codewiki/src/be/dependency_analyzer/analyzers/c_declarators.py @@ -0,0 +1,51 @@ +"""Declarator helpers shared by the C and C++ analyzers (tree-sitter-c and tree-sitter-cpp shape them alike).""" + +from typing import Optional + +# Declarators a return type wraps a function's name in: `char *f()`, `T **f()`, `T &f()`, `int (*f(void))(int)`. +WRAPPERS = ( + "pointer_declarator", + "reference_declarator", + "parenthesized_declarator", + "attributed_declarator", +) + + +def _inner(node): + named = node.child_by_field_name("declarator") + if named is not None: + return named + return next((c for c in node.named_children if c.type.endswith("declarator")), None) + + +def function_declarator(definition): + """The function_declarator that names a function_definition, through the declarators its return type wraps it + in. A function returning a function pointer has two: the innermost one holds the function's own name.""" + found = None + node = definition.child_by_field_name("declarator") + while node is not None: + if node.type == "function_declarator": + found = node + elif node.type not in WRAPPERS: + break + node = _inner(node) + return found + + +def macro_typed_name(declarator) -> Optional[str]: + """`API T name(...)`: an export macro before a typedef'd return type. The parser cannot tell the macro from a + type, so it takes T for the name and leaves the real name in an ERROR node before the parameters.""" + name = declarator.child_by_field_name("declarator") + params = declarator.child_by_field_name("parameters") + if name is None or name.type != "identifier": + return None + errors = [ + c + for c in declarator.children + if c.type == "ERROR" + and c.start_byte >= name.end_byte + and (params is None or c.end_byte <= params.start_byte) + ] + if len(errors) != 1 or [c.type for c in errors[0].named_children] != ["identifier"]: + return None + return errors[0].named_children[0].text.decode() diff --git a/codewiki/src/be/dependency_analyzer/analyzers/cpp.py b/codewiki/src/be/dependency_analyzer/analyzers/cpp.py index bd817e86..c94e882d 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/cpp.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/cpp.py @@ -6,6 +6,10 @@ from tree_sitter import Parser, Language import tree_sitter_cpp +from codewiki.src.be.dependency_analyzer.analyzers.c_declarators import ( + function_declarator, + macro_typed_name, +) from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship from codewiki.src.be.dependency_analyzer.utils.external_symbols import ( is_external_symbol, @@ -183,7 +187,7 @@ def _extract_nodes(self, node, top_level_nodes, lines): elif node.type == "function_definition": # Check if this is inside a class or function containing_class = self._find_containing_class_for_method(node) - declarator = next((c for c in node.children if c.type == "function_declarator"), None) + declarator = function_declarator(node) qualified_parts = self._get_qualified_declarator_parts(declarator) if declarator else [] if not containing_class and len(qualified_parts) > 1: containing_class = qualified_parts[-2] @@ -193,7 +197,10 @@ def _extract_nodes(self, node, top_level_nodes, lines): node_type = "function" if declarator: + node_name = macro_typed_name(declarator) for child in declarator.children: + if node_name: + break if child.type == "identifier": node_name = child.text.decode() break @@ -207,7 +214,7 @@ def _extract_nodes(self, node, top_level_nodes, lines): break elif node.type == "declaration": containing_class = self._find_containing_class_for_method(node) - declarator = next((c for c in node.children if c.type == "function_declarator"), None) + declarator = function_declarator(node) if containing_class and declarator: node_type = "method" node_name = self._get_declarator_name(declarator) @@ -304,6 +311,8 @@ def _is_global_variable(self, node) -> bool: def _get_declarator_name(self, declarator) -> Optional[str]: """Extract the declared function or method name from nested declarators.""" + if name := macro_typed_name(declarator): + return name for child in declarator.children: if child.type in ["identifier", "field_identifier"]: return child.text.decode() @@ -533,9 +542,7 @@ def _find_variable_type(self, node, variable_name: str) -> Optional[str]: if found: return found if current.type == "function_definition": - declarator = next( - (c for c in current.children if c.type == "function_declarator"), None - ) + declarator = function_declarator(current) found = self._search_parameter_declaration(declarator, variable_name) if found: return found @@ -617,17 +624,11 @@ def _find_containing_function(self, node, top_level_nodes): while current: if current.type == "function_definition": # Get function name - declarator = next( - (c for c in current.children if c.type == "function_declarator"), None - ) + declarator = function_declarator(current) if declarator: - identifier = next( - (c for c in declarator.children if c.type == "identifier"), None - ) - if identifier: - func_name = identifier.text.decode() - if func_name in top_level_nodes: - return func_name + func_name = self._get_declarator_name(declarator) + if func_name in top_level_nodes: + return func_name current = current.parent return None @@ -636,9 +637,7 @@ def _find_containing_function_or_method(self, node, top_level_nodes): current = node.parent while current: if current.type == "function_definition": - declarator = next( - (c for c in current.children if c.type == "function_declarator"), None - ) + declarator = function_declarator(current) if declarator: func_name = self._get_declarator_name(declarator) if func_name: diff --git a/tests/test_c_cpp_declarators.py b/tests/test_c_cpp_declarators.py new file mode 100644 index 00000000..eaa906c8 --- /dev/null +++ b/tests/test_c_cpp_declarators.py @@ -0,0 +1,66 @@ +"""C and C++ functions whose name sits inside the declarators of their return type. + +The analyzers looked for the function_declarator as a direct child of a function_definition. A function returning +a pointer (`const char *f()`, `T **f()`) or a C++ reference (`T &f()`) has it inside a pointer/reference declarator, +so it was left out of the graph, with every call made from inside it: 162 of lua's ~1,290 C functions, among them +lua_tolstring and lua_pushstring. And `API T f()` (an export macro before a typedef'd return type) was named T. +""" + +from codewiki.src.be.dependency_analyzer.analyzers.c import analyze_c_file +from codewiki.src.be.dependency_analyzer.analyzers.cpp import analyze_cpp_file + +C_SAMPLE = """\ +typedef struct State State; +typedef long lua_Integer; +static int counter; +static State *current; +static char **names; + +static State *find (int idx) { return current; } +static char **all_names (void) { counter++; return names; } +const char *tostring (State *L, int idx) { find(idx); return 0; } +API lua_Integer len (State *L) { return (lua_Integer)tostring(L, 1); } +int (*pick (int k)) (int) { all_names(); return 0; } +int plain (void) { return counter; } +""" + +CPP_SAMPLE = """\ +class Node { +public: + Node &parent(); + const char *name() const; +}; + +Node &Node::parent() { return *this; } +const char *Node::name() const { parent(); return "n"; } +static Node **roots(int n) { return 0; } +API size_t count(Node *n) { roots(1); return 0; } +""" + + +def _names(nodes): + return {n.name: n for n in nodes} + + +def test_c_functions_returning_pointers_are_in_the_graph(): + nodes, calls = analyze_c_file("/repo/lapi.c", C_SAMPLE, "/repo") + by_name = _names(nodes) + for fn in ("find", "all_names", "tostring", "len", "pick", "plain"): + assert by_name[fn].component_type == "function", fn + assert "lua_Integer" not in by_name # `API lua_Integer len(...)` is len, not its return type + edges = {(c.caller.split("::")[-1], c.callee) for c in calls} + assert {("tostring", "find"), ("len", "tostring"), ("pick", "all_names")} <= edges + + +def test_cpp_functions_returning_pointers_or_references_are_in_the_graph(): + nodes, calls = analyze_cpp_file("/repo/tree.cpp", CPP_SAMPLE, "/repo") + ids = {n.id for n in nodes} + assert { + "tree.cpp::Node.parent", + "tree.cpp::Node.name", + "tree.cpp::roots", + "tree.cpp::count", + } <= ids + assert not any(i.endswith("::size_t") for i in ids) + callers = {c.caller.split("::")[-1] for c in calls} + assert {"Node.name", "count"} <= callers From 8d6a56cdc366474a5ad6bb4084dcde865ba7ee93 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Fri, 30 Oct 2026 13:58:49 +0700 Subject: [PATCH 2/2] fix(c): a macro before the name of a function taking (void) is not its name `int CJSON_CDECL main(void) { ... }`: with a parameter list of one word, tree-sitter-c reads `int CJSON_CDECL` as a declaration missing its `;`, then a function_definition whose type is `main` and whose declarator is the parenthesized `(void)`. It has no function_declarator, so the function and every call made from it were left out. (With real parameters the macro lands in an ERROR node and the name is found, as before.) split_macro_function_name: a function_definition whose type is a type_identifier, whose declarator is a parenthesized single identifier, and whose previous sibling is a declaration ending in a missing `;` on the same line, defines a function named by that type. A real typedef'd return type (`Count two(void)`) follows a complete declaration, so it is not taken for a name. Measured (old branch head against this commit, components): cJSON 169 -> 170 (test.c::main); lz4, lua, inih, tinyxml2 and jq unchanged. --- .../src/be/dependency_analyzer/analyzers/c.py | 6 +++-- .../analyzers/c_declarators.py | 23 ++++++++++++++++ tests/test_c_cpp_declarators.py | 26 +++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/codewiki/src/be/dependency_analyzer/analyzers/c.py b/codewiki/src/be/dependency_analyzer/analyzers/c.py index 61493a1d..752d2680 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/c.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/c.py @@ -8,6 +8,7 @@ from codewiki.src.be.dependency_analyzer.analyzers.c_declarators import ( function_declarator, macro_typed_name, + split_macro_function_name, ) from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship @@ -222,10 +223,11 @@ def _find_containing_function(self, node, top_level_nodes): def _function_name(self, definition): """The name a function_definition defines, also when its return type is a pointer (`char *f()`) or an - export macro stands before a typedef'd return type (`API lua_Integer f()`).""" + export macro stands before a typedef'd return type (`API lua_Integer f()`) or before the name of a function + taking `(void)` (`int CJSON_CDECL main(void)`).""" declarator = function_declarator(definition) if declarator is None: - return None + return split_macro_function_name(definition) name = macro_typed_name(declarator) if name: return name diff --git a/codewiki/src/be/dependency_analyzer/analyzers/c_declarators.py b/codewiki/src/be/dependency_analyzer/analyzers/c_declarators.py index ce6a59a9..172a81dc 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/c_declarators.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/c_declarators.py @@ -49,3 +49,26 @@ def macro_typed_name(declarator) -> Optional[str]: if len(errors) != 1 or [c.type for c in errors[0].named_children] != ["identifier"]: return None return errors[0].named_children[0].text.decode() + + +def split_macro_function_name(definition) -> Optional[str]: + """`int API f(void) {...}`: a macro between the return type and the name, and a parameter list of one word. + tree-sitter-c reads `int API` as a declaration missing its `;`, then a function_definition whose type is the + name (`f`) and whose declarator is `(void)`. With real parameters the macro lands in an ERROR node instead and + `function_declarator` finds the name.""" + declarator = definition.child_by_field_name("declarator") + kind = definition.child_by_field_name("type") + before = definition.prev_named_sibling + if ( + declarator is None + or declarator.type != "parenthesized_declarator" + or [c.type for c in declarator.named_children] != ["identifier"] + or kind is None + or kind.type != "type_identifier" + or before is None + or before.type != "declaration" + or not (before.children and before.children[-1].is_missing) + or before.end_point[0] != definition.start_point[0] + ): + return None + return kind.text.decode() diff --git a/tests/test_c_cpp_declarators.py b/tests/test_c_cpp_declarators.py index eaa906c8..ec675b29 100644 --- a/tests/test_c_cpp_declarators.py +++ b/tests/test_c_cpp_declarators.py @@ -4,6 +4,8 @@ a pointer (`const char *f()`, `T **f()`) or a C++ reference (`T &f()`) has it inside a pointer/reference declarator, so it was left out of the graph, with every call made from inside it: 162 of lua's ~1,290 C functions, among them lua_tolstring and lua_pushstring. And `API T f()` (an export macro before a typedef'd return type) was named T. +`int CJSON_CDECL main(void)` (a macro before the name, one word of parameters) is read by tree-sitter-c as a +declaration missing its `;` followed by a definition whose type is `main`: it was left out too. """ from codewiki.src.be.dependency_analyzer.analyzers.c import analyze_c_file @@ -64,3 +66,27 @@ def test_cpp_functions_returning_pointers_or_references_are_in_the_graph(): assert not any(i.endswith("::size_t") for i in ids) callers = {c.caller.split("::")[-1] for c in calls} assert {"Node.name", "count"} <= callers + + +MACRO_SAMPLE = """\ +static void helper(void) { } +int CJSON_CDECL main(void) +{ + helper(); + return 0; +} +int EXPORT run(int argc) { helper(); return argc; } +typedef int Count; +int one; +Count two(void) { return 2; } +""" + + +def test_a_macro_before_the_name_of_a_void_function_is_not_its_name(): + nodes, calls = analyze_c_file("/repo/test.c", MACRO_SAMPLE, "/repo") + by_name = _names(nodes) + assert {"helper", "main", "run", "two"} <= set(by_name) + # `Count two(void)`: a real typedef'd return type, after a complete declaration + assert "Count" not in by_name + edges = {(c.caller.split("::")[-1], c.callee) for c in calls} + assert {("main", "helper"), ("run", "helper")} <= edges