Keep C and C++ functions that return pointers or references in the graph

codewiki/src/be/dependency_analyzer/analyzers/c.py

@@ -5,6 +5,11 @@
 
 from tree_sitter import Parser, Language
 import tree_sitter_c
+from codewiki.src.be.dependency_analyzer.analyzers.c_declarators import (
+    function_declarator,
+    macro_typed_name,
+    split_macro_function_name,
+)
 from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship
 
 logger = logging.getLogger(__name__)
@@ -70,12 +75,7 @@ def _extract_nodes(self, node, top_level_nodes, lines):
 
         if node.type == "function_definition":
             node_type = "function"
-            # look for function_declarator
-            declarator = next((c for c in node.children if c.type == "function_declarator"), None)
-            if declarator:
-                identifier = next((c for c in declarator.children if c.type == "identifier"), None)
-                if identifier:
-                    node_name = identifier.text.decode()
+            node_name = self._function_name(node)
         elif node.type == "struct_specifier":
             # Extract struct definitions: struct Name { ... }
             node_type = "struct"
@@ -215,21 +215,25 @@ def _find_containing_function(self, node, top_level_nodes):
         current = node.parent
         while current:
             if current.type == "function_definition":
-                # Get function name
-                declarator = next(
-                    (c for c in current.children if c.type == "function_declarator"), None
-                )
-                if declarator:
-                    identifier = next(
-                        (c for c in declarator.children if c.type == "identifier"), None
-                    )
-                    if identifier:
-                        func_name = identifier.text.decode()
-                        if func_name in top_level_nodes:
-                            return func_name
+                func_name = self._function_name(current)
+                if func_name in top_level_nodes:
+                    return func_name
             current = current.parent
         return None
 
+    def _function_name(self, definition):
+        """The name a function_definition defines, also when its return type is a pointer (`char *f()`) or an
+        export macro stands before a typedef'd return type (`API lua_Integer f()`) or before the name of a function
+        taking `(void)` (`int CJSON_CDECL main(void)`)."""
+        declarator = function_declarator(definition)
+        if declarator is None:
+            return split_macro_function_name(definition)
+        name = macro_typed_name(declarator)
+        if name:
+            return name
+        identifier = next((c for c in declarator.children if c.type == "identifier"), None)
+        return identifier.text.decode() if identifier else None
+
 
 def analyze_c_file(
     file_path: str, content: str, repo_path: str = None

codewiki/src/be/dependency_analyzer/analyzers/c_declarators.py

@@ -0,0 +1,74 @@
+"""Declarator helpers shared by the C and C++ analyzers (tree-sitter-c and tree-sitter-cpp shape them alike)."""
+
+from typing import Optional
+
+# Declarators a return type wraps a function's name in: `char *f()`, `T **f()`, `T &f()`, `int (*f(void))(int)`.
+WRAPPERS = (
+    "pointer_declarator",
+    "reference_declarator",
+    "parenthesized_declarator",
+    "attributed_declarator",
+)
+
+
+def _inner(node):
+    named = node.child_by_field_name("declarator")
+    if named is not None:
+        return named
+    return next((c for c in node.named_children if c.type.endswith("declarator")), None)
+
+
+def function_declarator(definition):
+    """The function_declarator that names a function_definition, through the declarators its return type wraps it
+    in. A function returning a function pointer has two: the innermost one holds the function's own name."""
+    found = None
+    node = definition.child_by_field_name("declarator")
+    while node is not None:
+        if node.type == "function_declarator":
+            found = node
+        elif node.type not in WRAPPERS:
+            break
+        node = _inner(node)
+    return found
+
+
+def macro_typed_name(declarator) -> Optional[str]:
+    """`API T name(...)`: an export macro before a typedef'd return type. The parser cannot tell the macro from a
+    type, so it takes T for the name and leaves the real name in an ERROR node before the parameters."""
+    name = declarator.child_by_field_name("declarator")
+    params = declarator.child_by_field_name("parameters")
+    if name is None or name.type != "identifier":
+        return None
+    errors = [
+        c
+        for c in declarator.children
+        if c.type == "ERROR"
+        and c.start_byte >= name.end_byte
+        and (params is None or c.end_byte <= params.start_byte)
+    ]
+    if len(errors) != 1 or [c.type for c in errors[0].named_children] != ["identifier"]:
+        return None
+    return errors[0].named_children[0].text.decode()
+
+
+def split_macro_function_name(definition) -> Optional[str]:
+    """`int API f(void) {...}`: a macro between the return type and the name, and a parameter list of one word.
+    tree-sitter-c reads `int API` as a declaration missing its `;`, then a function_definition whose type is the
+    name (`f`) and whose declarator is `(void)`. With real parameters the macro lands in an ERROR node instead and
+    `function_declarator` finds the name."""
+    declarator = definition.child_by_field_name("declarator")
+    kind = definition.child_by_field_name("type")
+    before = definition.prev_named_sibling
+    if (
+        declarator is None
+        or declarator.type != "parenthesized_declarator"
+        or [c.type for c in declarator.named_children] != ["identifier"]
+        or kind is None
+        or kind.type != "type_identifier"
+        or before is None
+        or before.type != "declaration"
+        or not (before.children and before.children[-1].is_missing)
+        or before.end_point[0] != definition.start_point[0]
+    ):
+        return None
+    return kind.text.decode()

codewiki/src/be/dependency_analyzer/analyzers/cpp.py

@@ -6,6 +6,10 @@
 
 from tree_sitter import Parser, Language
 import tree_sitter_cpp
+from codewiki.src.be.dependency_analyzer.analyzers.c_declarators import (
+    function_declarator,
+    macro_typed_name,
+)
 from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship
 from codewiki.src.be.dependency_analyzer.utils.external_symbols import (
     is_external_symbol,
@@ -183,7 +187,7 @@ def _extract_nodes(self, node, top_level_nodes, lines):
         elif node.type == "function_definition":
             # Check if this is inside a class or function
             containing_class = self._find_containing_class_for_method(node)
-            declarator = next((c for c in node.children if c.type == "function_declarator"), None)
+            declarator = function_declarator(node)
             qualified_parts = self._get_qualified_declarator_parts(declarator) if declarator else []
             if not containing_class and len(qualified_parts) > 1:
                 containing_class = qualified_parts[-2]
@@ -193,7 +197,10 @@ def _extract_nodes(self, node, top_level_nodes, lines):
                 node_type = "function"
 
             if declarator:
+                node_name = macro_typed_name(declarator)
                 for child in declarator.children:
+                    if node_name:
+                        break
                     if child.type == "identifier":
                         node_name = child.text.decode()
                         break
@@ -207,7 +214,7 @@ def _extract_nodes(self, node, top_level_nodes, lines):
                             break
         elif node.type == "declaration":
             containing_class = self._find_containing_class_for_method(node)
-            declarator = next((c for c in node.children if c.type == "function_declarator"), None)
+            declarator = function_declarator(node)
             if containing_class and declarator:
                 node_type = "method"
                 node_name = self._get_declarator_name(declarator)
@@ -304,6 +311,8 @@ def _is_global_variable(self, node) -> bool:
 
     def _get_declarator_name(self, declarator) -> Optional[str]:
         """Extract the declared function or method name from nested declarators."""
+        if name := macro_typed_name(declarator):
+            return name
         for child in declarator.children:
             if child.type in ["identifier", "field_identifier"]:
                 return child.text.decode()
@@ -533,9 +542,7 @@ def _find_variable_type(self, node, variable_name: str) -> Optional[str]:
                 if found:
                     return found
             if current.type == "function_definition":
-                declarator = next(
-                    (c for c in current.children if c.type == "function_declarator"), None
-                )
+                declarator = function_declarator(current)
                 found = self._search_parameter_declaration(declarator, variable_name)
                 if found:
                     return found
@@ -617,17 +624,11 @@ def _find_containing_function(self, node, top_level_nodes):
         while current:
             if current.type == "function_definition":
                 # Get function name
-                declarator = next(
-                    (c for c in current.children if c.type == "function_declarator"), None
-                )
+                declarator = function_declarator(current)
                 if declarator:
-                    identifier = next(
-                        (c for c in declarator.children if c.type == "identifier"), None
-                    )
-                    if identifier:
-                        func_name = identifier.text.decode()
-                        if func_name in top_level_nodes:
-                            return func_name
+                    func_name = self._get_declarator_name(declarator)
+                    if func_name in top_level_nodes:
+                        return func_name
             current = current.parent
         return None
 
@@ -636,9 +637,7 @@ def _find_containing_function_or_method(self, node, top_level_nodes):
         current = node.parent
         while current:
             if current.type == "function_definition":
-                declarator = next(
-                    (c for c in current.children if c.type == "function_declarator"), None
-                )
+                declarator = function_declarator(current)
                 if declarator:
                     func_name = self._get_declarator_name(declarator)
                     if func_name:

tests/test_c_cpp_declarators.py

@@ -0,0 +1,92 @@
+"""C and C++ functions whose name sits inside the declarators of their return type.
+
+The analyzers looked for the function_declarator as a direct child of a function_definition. A function returning
+a pointer (`const char *f()`, `T **f()`) or a C++ reference (`T &f()`) has it inside a pointer/reference declarator,
+so it was left out of the graph, with every call made from inside it: 162 of lua's ~1,290 C functions, among them
+lua_tolstring and lua_pushstring. And `API T f()` (an export macro before a typedef'd return type) was named T.
+`int CJSON_CDECL main(void)` (a macro before the name, one word of parameters) is read by tree-sitter-c as a
+declaration missing its `;` followed by a definition whose type is `main`: it was left out too.
+"""
+
+from codewiki.src.be.dependency_analyzer.analyzers.c import analyze_c_file
+from codewiki.src.be.dependency_analyzer.analyzers.cpp import analyze_cpp_file
+
+C_SAMPLE = """\
+typedef struct State State;
+typedef long lua_Integer;
+static int counter;
+static State *current;
+static char **names;
+
+static State *find (int idx) { return current; }
+static char **all_names (void) { counter++; return names; }
+const char *tostring (State *L, int idx) { find(idx); return 0; }
+API lua_Integer len (State *L) { return (lua_Integer)tostring(L, 1); }
+int (*pick (int k)) (int) { all_names(); return 0; }
+int plain (void) { return counter; }
+"""
+
+CPP_SAMPLE = """\
+class Node {
+public:
+    Node &parent();
+    const char *name() const;
+};
+
+Node &Node::parent() { return *this; }
+const char *Node::name() const { parent(); return "n"; }
+static Node **roots(int n) { return 0; }
+API size_t count(Node *n) { roots(1); return 0; }
+"""
+
+
+def _names(nodes):
+    return {n.name: n for n in nodes}
+
+
+def test_c_functions_returning_pointers_are_in_the_graph():
+    nodes, calls = analyze_c_file("/repo/lapi.c", C_SAMPLE, "/repo")
+    by_name = _names(nodes)
+    for fn in ("find", "all_names", "tostring", "len", "pick", "plain"):
+        assert by_name[fn].component_type == "function", fn
+    assert "lua_Integer" not in by_name  # `API lua_Integer len(...)` is len, not its return type
+    edges = {(c.caller.split("::")[-1], c.callee) for c in calls}
+    assert {("tostring", "find"), ("len", "tostring"), ("pick", "all_names")} <= edges
+
+
+def test_cpp_functions_returning_pointers_or_references_are_in_the_graph():
+    nodes, calls = analyze_cpp_file("/repo/tree.cpp", CPP_SAMPLE, "/repo")
+    ids = {n.id for n in nodes}
+    assert {
+        "tree.cpp::Node.parent",
+        "tree.cpp::Node.name",
+        "tree.cpp::roots",
+        "tree.cpp::count",
+    } <= ids
+    assert not any(i.endswith("::size_t") for i in ids)
+    callers = {c.caller.split("::")[-1] for c in calls}
+    assert {"Node.name", "count"} <= callers
+
+
+MACRO_SAMPLE = """\
+static void helper(void) { }
+int CJSON_CDECL main(void)
+{
+    helper();
+    return 0;
+}
+int EXPORT run(int argc) { helper(); return argc; }
+typedef int Count;
+int one;
+Count two(void) { return 2; }
+"""
+
+
+def test_a_macro_before_the_name_of_a_void_function_is_not_its_name():
+    nodes, calls = analyze_c_file("/repo/test.c", MACRO_SAMPLE, "/repo")
+    by_name = _names(nodes)
+    assert {"helper", "main", "run", "two"} <= set(by_name)
+    # `Count two(void)`: a real typedef'd return type, after a complete declaration
+    assert "Count" not in by_name
+    edges = {(c.caller.split("::")[-1], c.callee) for c in calls}
+    assert {("main", "helper"), ("run", "helper")} <= edges