diff --git a/codewiki/src/be/dependency_analyzer/analyzers/c.py b/codewiki/src/be/dependency_analyzer/analyzers/c.py index fca53932..752d2680 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/c.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/c.py @@ -5,6 +5,11 @@ from tree_sitter import Parser, Language import tree_sitter_c +from codewiki.src.be.dependency_analyzer.analyzers.c_declarators import ( + function_declarator, + macro_typed_name, + split_macro_function_name, +) from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) @@ -70,12 +75,7 @@ def _extract_nodes(self, node, top_level_nodes, lines): if node.type == "function_definition": node_type = "function" - # look for function_declarator - declarator = next((c for c in node.children if c.type == "function_declarator"), None) - if declarator: - identifier = next((c for c in declarator.children if c.type == "identifier"), None) - if identifier: - node_name = identifier.text.decode() + node_name = self._function_name(node) elif node.type == "struct_specifier": # Extract struct definitions: struct Name { ... } node_type = "struct" @@ -215,21 +215,25 @@ def _find_containing_function(self, node, top_level_nodes): current = node.parent while current: if current.type == "function_definition": - # Get function name - declarator = next( - (c for c in current.children if c.type == "function_declarator"), None - ) - if declarator: - identifier = next( - (c for c in declarator.children if c.type == "identifier"), None - ) - if identifier: - func_name = identifier.text.decode() - if func_name in top_level_nodes: - return func_name + func_name = self._function_name(current) + if func_name in top_level_nodes: + return func_name current = current.parent return None + def _function_name(self, definition): + """The name a function_definition defines, also when its return type is a pointer (`char *f()`) or an + export macro stands before a typedef'd return type (`API lua_Integer f()`) or before the name of a function + taking `(void)` (`int CJSON_CDECL main(void)`).""" + declarator = function_declarator(definition) + if declarator is None: + return split_macro_function_name(definition) + name = macro_typed_name(declarator) + if name: + return name + identifier = next((c for c in declarator.children if c.type == "identifier"), None) + return identifier.text.decode() if identifier else None + def analyze_c_file( file_path: str, content: str, repo_path: str = None diff --git a/codewiki/src/be/dependency_analyzer/analyzers/c_declarators.py b/codewiki/src/be/dependency_analyzer/analyzers/c_declarators.py new file mode 100644 index 00000000..172a81dc --- /dev/null +++ b/codewiki/src/be/dependency_analyzer/analyzers/c_declarators.py @@ -0,0 +1,74 @@ +"""Declarator helpers shared by the C and C++ analyzers (tree-sitter-c and tree-sitter-cpp shape them alike).""" + +from typing import Optional + +# Declarators a return type wraps a function's name in: `char *f()`, `T **f()`, `T &f()`, `int (*f(void))(int)`. +WRAPPERS = ( + "pointer_declarator", + "reference_declarator", + "parenthesized_declarator", + "attributed_declarator", +) + + +def _inner(node): + named = node.child_by_field_name("declarator") + if named is not None: + return named + return next((c for c in node.named_children if c.type.endswith("declarator")), None) + + +def function_declarator(definition): + """The function_declarator that names a function_definition, through the declarators its return type wraps it + in. A function returning a function pointer has two: the innermost one holds the function's own name.""" + found = None + node = definition.child_by_field_name("declarator") + while node is not None: + if node.type == "function_declarator": + found = node + elif node.type not in WRAPPERS: + break + node = _inner(node) + return found + + +def macro_typed_name(declarator) -> Optional[str]: + """`API T name(...)`: an export macro before a typedef'd return type. The parser cannot tell the macro from a + type, so it takes T for the name and leaves the real name in an ERROR node before the parameters.""" + name = declarator.child_by_field_name("declarator") + params = declarator.child_by_field_name("parameters") + if name is None or name.type != "identifier": + return None + errors = [ + c + for c in declarator.children + if c.type == "ERROR" + and c.start_byte >= name.end_byte + and (params is None or c.end_byte <= params.start_byte) + ] + if len(errors) != 1 or [c.type for c in errors[0].named_children] != ["identifier"]: + return None + return errors[0].named_children[0].text.decode() + + +def split_macro_function_name(definition) -> Optional[str]: + """`int API f(void) {...}`: a macro between the return type and the name, and a parameter list of one word. + tree-sitter-c reads `int API` as a declaration missing its `;`, then a function_definition whose type is the + name (`f`) and whose declarator is `(void)`. With real parameters the macro lands in an ERROR node instead and + `function_declarator` finds the name.""" + declarator = definition.child_by_field_name("declarator") + kind = definition.child_by_field_name("type") + before = definition.prev_named_sibling + if ( + declarator is None + or declarator.type != "parenthesized_declarator" + or [c.type for c in declarator.named_children] != ["identifier"] + or kind is None + or kind.type != "type_identifier" + or before is None + or before.type != "declaration" + or not (before.children and before.children[-1].is_missing) + or before.end_point[0] != definition.start_point[0] + ): + return None + return kind.text.decode() diff --git a/codewiki/src/be/dependency_analyzer/analyzers/cpp.py b/codewiki/src/be/dependency_analyzer/analyzers/cpp.py index bd817e86..c94e882d 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/cpp.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/cpp.py @@ -6,6 +6,10 @@ from tree_sitter import Parser, Language import tree_sitter_cpp +from codewiki.src.be.dependency_analyzer.analyzers.c_declarators import ( + function_declarator, + macro_typed_name, +) from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship from codewiki.src.be.dependency_analyzer.utils.external_symbols import ( is_external_symbol, @@ -183,7 +187,7 @@ def _extract_nodes(self, node, top_level_nodes, lines): elif node.type == "function_definition": # Check if this is inside a class or function containing_class = self._find_containing_class_for_method(node) - declarator = next((c for c in node.children if c.type == "function_declarator"), None) + declarator = function_declarator(node) qualified_parts = self._get_qualified_declarator_parts(declarator) if declarator else [] if not containing_class and len(qualified_parts) > 1: containing_class = qualified_parts[-2] @@ -193,7 +197,10 @@ def _extract_nodes(self, node, top_level_nodes, lines): node_type = "function" if declarator: + node_name = macro_typed_name(declarator) for child in declarator.children: + if node_name: + break if child.type == "identifier": node_name = child.text.decode() break @@ -207,7 +214,7 @@ def _extract_nodes(self, node, top_level_nodes, lines): break elif node.type == "declaration": containing_class = self._find_containing_class_for_method(node) - declarator = next((c for c in node.children if c.type == "function_declarator"), None) + declarator = function_declarator(node) if containing_class and declarator: node_type = "method" node_name = self._get_declarator_name(declarator) @@ -304,6 +311,8 @@ def _is_global_variable(self, node) -> bool: def _get_declarator_name(self, declarator) -> Optional[str]: """Extract the declared function or method name from nested declarators.""" + if name := macro_typed_name(declarator): + return name for child in declarator.children: if child.type in ["identifier", "field_identifier"]: return child.text.decode() @@ -533,9 +542,7 @@ def _find_variable_type(self, node, variable_name: str) -> Optional[str]: if found: return found if current.type == "function_definition": - declarator = next( - (c for c in current.children if c.type == "function_declarator"), None - ) + declarator = function_declarator(current) found = self._search_parameter_declaration(declarator, variable_name) if found: return found @@ -617,17 +624,11 @@ def _find_containing_function(self, node, top_level_nodes): while current: if current.type == "function_definition": # Get function name - declarator = next( - (c for c in current.children if c.type == "function_declarator"), None - ) + declarator = function_declarator(current) if declarator: - identifier = next( - (c for c in declarator.children if c.type == "identifier"), None - ) - if identifier: - func_name = identifier.text.decode() - if func_name in top_level_nodes: - return func_name + func_name = self._get_declarator_name(declarator) + if func_name in top_level_nodes: + return func_name current = current.parent return None @@ -636,9 +637,7 @@ def _find_containing_function_or_method(self, node, top_level_nodes): current = node.parent while current: if current.type == "function_definition": - declarator = next( - (c for c in current.children if c.type == "function_declarator"), None - ) + declarator = function_declarator(current) if declarator: func_name = self._get_declarator_name(declarator) if func_name: diff --git a/tests/test_c_cpp_declarators.py b/tests/test_c_cpp_declarators.py new file mode 100644 index 00000000..ec675b29 --- /dev/null +++ b/tests/test_c_cpp_declarators.py @@ -0,0 +1,92 @@ +"""C and C++ functions whose name sits inside the declarators of their return type. + +The analyzers looked for the function_declarator as a direct child of a function_definition. A function returning +a pointer (`const char *f()`, `T **f()`) or a C++ reference (`T &f()`) has it inside a pointer/reference declarator, +so it was left out of the graph, with every call made from inside it: 162 of lua's ~1,290 C functions, among them +lua_tolstring and lua_pushstring. And `API T f()` (an export macro before a typedef'd return type) was named T. +`int CJSON_CDECL main(void)` (a macro before the name, one word of parameters) is read by tree-sitter-c as a +declaration missing its `;` followed by a definition whose type is `main`: it was left out too. +""" + +from codewiki.src.be.dependency_analyzer.analyzers.c import analyze_c_file +from codewiki.src.be.dependency_analyzer.analyzers.cpp import analyze_cpp_file + +C_SAMPLE = """\ +typedef struct State State; +typedef long lua_Integer; +static int counter; +static State *current; +static char **names; + +static State *find (int idx) { return current; } +static char **all_names (void) { counter++; return names; } +const char *tostring (State *L, int idx) { find(idx); return 0; } +API lua_Integer len (State *L) { return (lua_Integer)tostring(L, 1); } +int (*pick (int k)) (int) { all_names(); return 0; } +int plain (void) { return counter; } +""" + +CPP_SAMPLE = """\ +class Node { +public: + Node &parent(); + const char *name() const; +}; + +Node &Node::parent() { return *this; } +const char *Node::name() const { parent(); return "n"; } +static Node **roots(int n) { return 0; } +API size_t count(Node *n) { roots(1); return 0; } +""" + + +def _names(nodes): + return {n.name: n for n in nodes} + + +def test_c_functions_returning_pointers_are_in_the_graph(): + nodes, calls = analyze_c_file("/repo/lapi.c", C_SAMPLE, "/repo") + by_name = _names(nodes) + for fn in ("find", "all_names", "tostring", "len", "pick", "plain"): + assert by_name[fn].component_type == "function", fn + assert "lua_Integer" not in by_name # `API lua_Integer len(...)` is len, not its return type + edges = {(c.caller.split("::")[-1], c.callee) for c in calls} + assert {("tostring", "find"), ("len", "tostring"), ("pick", "all_names")} <= edges + + +def test_cpp_functions_returning_pointers_or_references_are_in_the_graph(): + nodes, calls = analyze_cpp_file("/repo/tree.cpp", CPP_SAMPLE, "/repo") + ids = {n.id for n in nodes} + assert { + "tree.cpp::Node.parent", + "tree.cpp::Node.name", + "tree.cpp::roots", + "tree.cpp::count", + } <= ids + assert not any(i.endswith("::size_t") for i in ids) + callers = {c.caller.split("::")[-1] for c in calls} + assert {"Node.name", "count"} <= callers + + +MACRO_SAMPLE = """\ +static void helper(void) { } +int CJSON_CDECL main(void) +{ + helper(); + return 0; +} +int EXPORT run(int argc) { helper(); return argc; } +typedef int Count; +int one; +Count two(void) { return 2; } +""" + + +def test_a_macro_before_the_name_of_a_void_function_is_not_its_name(): + nodes, calls = analyze_c_file("/repo/test.c", MACRO_SAMPLE, "/repo") + by_name = _names(nodes) + assert {"helper", "main", "run", "two"} <= set(by_name) + # `Count two(void)`: a real typedef'd return type, after a complete declaration + assert "Count" not in by_name + edges = {(c.caller.split("::")[-1], c.callee) for c in calls} + assert {("main", "helper"), ("run", "helper")} <= edges