From 7537fbd7a7a46bd60531b53665cc987a4cafde37 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Sat, 1 Aug 2026 08:32:14 +0200 Subject: [PATCH 1/3] feat(scanner): Add Go parser fallback Parse Go source when ast-grep is unavailable. --- scanner/cargofallback.go | 31 ++++++- scanner/cargofallback_test.go | 149 ++++++++++++++++++++++++++++++++++ scanner/gofallback.go | 79 ++++++++++++++++++ scanner/gofallback_test.go | 111 +++++++++++++++++++++++++ 4 files changed, 367 insertions(+), 3 deletions(-) create mode 100644 scanner/gofallback.go create mode 100644 scanner/gofallback_test.go diff --git a/scanner/cargofallback.go b/scanner/cargofallback.go index 8ce7b07..a812e97 100644 --- a/scanner/cargofallback.go +++ b/scanner/cargofallback.go @@ -60,14 +60,39 @@ func scanForGraphOutcomeWithFilters(ctx context.Context, root string, filters Fi } return ScanOutcome{}, false, err } - fallback, fallbackErr := buildCargoFallbackOutcome(ctx, root, files, loader) - if fallbackErr != nil { + fallback := ScanOutcome{ + Sources: []ScanSourceOutcome{incomplete.Outcome}, + } + recovered := false + if goFallback, fallbackErr := buildGoFallbackOutcome(ctx, root, files); fallbackErr == nil { + mergeFallbackOutcome(&fallback, goFallback) + recovered = true + } else if ctx.Err() != nil { + return ScanOutcome{}, false, ctx.Err() + } + if err := ctx.Err(); err != nil { + return ScanOutcome{}, false, err + } + + if cargoFallback, fallbackErr := buildCargoFallbackOutcome(ctx, root, files, loader); fallbackErr == nil { + mergeFallbackOutcome(&fallback, cargoFallback) + recovered = true + } + if err := ctx.Err(); err != nil { + return ScanOutcome{}, false, err + } + if !recovered { return ScanOutcome{}, false, err } - fallback.Sources = append([]ScanSourceOutcome{incomplete.Outcome}, fallback.Sources...) return fallback, true, nil } +func mergeFallbackOutcome(dst *ScanOutcome, src ScanOutcome) { + dst.Analyses = append(dst.Analyses, src.Analyses...) + dst.Sources = append(dst.Sources, src.Sources...) + dst.precomputedEdges = append(dst.precomputedEdges, src.precomputedEdges...) +} + func buildCargoFallbackOutcome(ctx context.Context, root string, files []FileInfo, loader cargoMetadataLoader) (ScanOutcome, error) { manifests, err := discoverCargoManifests(ctx, root, files) if err != nil { diff --git a/scanner/cargofallback_test.go b/scanner/cargofallback_test.go index e4e3e5d..40f5339 100644 --- a/scanner/cargofallback_test.go +++ b/scanner/cargofallback_test.go @@ -63,6 +63,155 @@ func TestBuildFileGraphUsesCargoFallbackOnce(t *testing.T) { } } +func TestScanForGraphOutcomeUsesGoFallbackWithoutCargo(t *testing.T) { + root := t.TempDir() + writeRustCargoFixture(t, root, map[string]string{ + "main.go": "package main\n\nimport \"fmt\"\n\nfunc main() {}\n", + }) + loads := 0 + outcome, usedFallback, err := scanForGraphOutcome( + context.Background(), + root, + func(string) (ScanOutcome, error) { + return ScanOutcome{}, newIncompleteScanError("ast-grep", ScanSourceUnavailable, "ast-grep unavailable", ErrAstGrepNotFound) + }, + func(context.Context, string) ([]byte, error) { + loads++ + return nil, errors.New("unexpected Cargo fallback") + }, + ) + if err != nil { + t.Fatal(err) + } + if !usedFallback { + t.Fatal("Go-only recovery did not report fallback use") + } + if loads != 0 { + t.Fatalf("Cargo metadata loads = %d, want 0", loads) + } + want := []FileAnalysis{{ + Path: "main.go", + Language: "go", + Functions: []string{"main"}, + Imports: []string{"fmt"}, + }} + if !reflect.DeepEqual(outcome.Analyses, want) { + t.Fatalf("analyses = %#v, want %#v", outcome.Analyses, want) + } + if len(outcome.Sources) != 2 { + t.Fatalf("sources = %#v, want ast-grep and Go parser", outcome.Sources) + } + if outcome.Sources[0].Source != "ast-grep" || outcome.Sources[1].Source != "go-parser" { + t.Fatalf("sources = %#v, want ast-grep then Go parser", outcome.Sources) + } +} + +func TestScanForGraphOutcomeCombinesGoAndCargoFallbacks(t *testing.T) { + root, metadata := cargoFallbackFixture(t, map[string]any{ + "name": "core", "path": "core", "kind": nil, + }) + writeRustCargoFixture(t, root, map[string]string{ + "main.go": "package main\n\nfunc main() {}\n", + }) + + outcome, usedFallback, err := scanForGraphOutcome( + context.Background(), + root, + func(string) (ScanOutcome, error) { + return ScanOutcome{}, newIncompleteScanError("ast-grep", ScanSourceFailed, "ast-grep failed", errors.New("scan failure")) + }, + func(context.Context, string) ([]byte, error) { + return metadata, nil + }, + ) + if err != nil { + t.Fatal(err) + } + if !usedFallback { + t.Fatal("mixed recovery did not report fallback use") + } + if got, want := outcome.Analyses, []FileAnalysis{{ + Path: "main.go", + Language: "go", + Functions: []string{"main"}, + Imports: []string{}, + }}; !reflect.DeepEqual(got, want) { + t.Fatalf("analyses = %#v, want %#v", got, want) + } + if got, want := outcome.precomputedEdges, []fileEdge{{from: "app/src/lib.rs", to: "core/src/lib.rs"}}; !reflect.DeepEqual(got, want) { + t.Fatalf("edges = %#v, want %#v", got, want) + } + if len(outcome.Sources) != 3 || + outcome.Sources[0].Source != "ast-grep" || + outcome.Sources[1].Source != "go-parser" || + outcome.Sources[2].Source != "cargo-metadata" { + t.Fatalf("sources = %#v, want ast-grep, Go parser, and Cargo metadata", outcome.Sources) + } +} + +func TestScanForGraphOutcomeHonorsCancellationDuringFallback(t *testing.T) { + root := t.TempDir() + writeRustCargoFixture(t, root, map[string]string{ + "Cargo.toml": "package = \"broken\"\n", + "main.go": "package main\n\nfunc main() {}\n", + "src/lib.rs": "pub fn value() {}\n", + }) + ctx, cancel := context.WithCancel(context.Background()) + + _, _, err := scanForGraphOutcome( + ctx, + root, + func(string) (ScanOutcome, error) { + return ScanOutcome{}, newIncompleteScanError("ast-grep", ScanSourceFailed, "ast-grep failed", errors.New("scan failure")) + }, + func(context.Context, string) ([]byte, error) { + cancel() + return nil, errors.New("metadata canceled") + }, + ) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want %v", err, context.Canceled) + } +} + +func TestScanForGraphOutcomeHonorsPreCanceledContext(t *testing.T) { + root := t.TempDir() + writeRustCargoFixture(t, root, map[string]string{ + "main.go": "package main\n\nfunc main() {}\n", + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, _, err := scanForGraphOutcome( + ctx, + root, + func(string) (ScanOutcome, error) { + return ScanOutcome{}, newIncompleteScanError("ast-grep", ScanSourceFailed, "ast-grep failed", errors.New("scan failure")) + }, + nil, + ) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want %v", err, context.Canceled) + } +} + +func TestScanForGraphOutcomePreservesPrimaryErrorWhenFileScanFails(t *testing.T) { + root := filepath.Join(t.TempDir(), "missing") + primaryErr := newIncompleteScanError("ast-grep", ScanSourceFailed, "ast-grep failed", errors.New("scan failure")) + + _, _, err := scanForGraphOutcome( + context.Background(), + root, + func(string) (ScanOutcome, error) { + return ScanOutcome{}, primaryErr + }, + nil, + ) + if err != primaryErr { + t.Fatalf("error = %v, want original %v", err, primaryErr) + } +} + func TestBuildFileGraphDoesNotFallbackAfterAuthoritativeScan(t *testing.T) { root := t.TempDir() writeRustCargoFixture(t, root, map[string]string{"main.go": "package main\n"}) diff --git a/scanner/gofallback.go b/scanner/gofallback.go new file mode 100644 index 0000000..0fc0fa6 --- /dev/null +++ b/scanner/gofallback.go @@ -0,0 +1,79 @@ +package scanner + +import ( + "context" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "sort" + "strconv" + "strings" +) + +var errGoFallbackUnavailable = errors.New("go dependency fallback unavailable") + +func buildGoFallbackOutcome(ctx context.Context, root string, files []FileInfo) (ScanOutcome, error) { + var analyses []FileAnalysis + goFiles := 0 + skipped := 0 + fset := token.NewFileSet() + + for _, file := range files { + if !strings.EqualFold(filepath.Ext(file.Path), ".go") { + continue + } + goFiles++ + if err := ctx.Err(); err != nil { + return ScanOutcome{}, err + } + + path := filepath.Clean(file.Path) + parsed, err := parser.ParseFile(fset, filepath.Join(root, path), nil, parser.SkipObjectResolution) + if err != nil { + skipped++ + continue + } + + analysis := FileAnalysis{Path: path, Language: "go"} + for _, spec := range parsed.Imports { + importPath, err := strconv.Unquote(spec.Path.Value) + if err == nil && importPath != "" { + analysis.Imports = append(analysis.Imports, importPath) + } + } + for _, declaration := range parsed.Decls { + function, ok := declaration.(*ast.FuncDecl) + if ok && function.Name != nil && function.Name.Name != "" { + analysis.Functions = append(analysis.Functions, function.Name.Name) + } + } + analysis.Imports = dedupe(analysis.Imports) + analysis.Functions = dedupe(analysis.Functions) + if len(analysis.Imports) > 0 || len(analysis.Functions) > 0 { + analyses = append(analyses, analysis) + } + } + + if len(analyses) == 0 { + return ScanOutcome{}, errGoFallbackUnavailable + } + sort.Slice(analyses, func(i, j int) bool { + return analyses[i].Path < analyses[j].Path + }) + + detail := fmt.Sprintf("Go parser fallback recovered %d of %d Go files", len(analyses), goFiles) + if skipped > 0 { + detail += fmt.Sprintf(" (skipped %d with parse errors)", skipped) + } + return ScanOutcome{ + Analyses: analyses, + Sources: []ScanSourceOutcome{{ + Source: "go-parser", + Status: ScanSourceFallback, + Detail: detail, + }}, + }, nil +} diff --git a/scanner/gofallback_test.go b/scanner/gofallback_test.go new file mode 100644 index 0000000..01ded25 --- /dev/null +++ b/scanner/gofallback_test.go @@ -0,0 +1,111 @@ +package scanner + +import ( + "context" + "errors" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestBuildGoFallbackOutcome(t *testing.T) { + root := t.TempDir() + writeRustCargoFixture(t, root, map[string]string{ + "cmd/main.go": `package main + +import ( + alias "example.com/lib" + _ "example.com/side-effect" +) + +func main() {} + +type worker struct{} + +func (worker) Run() {} +`, + "cmd/main_test.go": `package main + +import "testing" + +func TestFeature(t *testing.T) {} +`, + "cmd/broken.go": "package main\nfunc broken(", + "notes.txt": "not Go", + }) + files := []FileInfo{ + {Path: filepath.FromSlash("cmd/main_test.go")}, + {Path: filepath.FromSlash("notes.txt")}, + {Path: filepath.FromSlash("cmd/broken.go")}, + {Path: filepath.FromSlash("cmd/main.go")}, + } + + outcome, err := buildGoFallbackOutcome(context.Background(), root, files) + if err != nil { + t.Fatal(err) + } + want := []FileAnalysis{ + { + Path: filepath.FromSlash("cmd/main.go"), + Language: "go", + Functions: []string{"main", "Run"}, + Imports: []string{"example.com/lib", "example.com/side-effect"}, + }, + { + Path: filepath.FromSlash("cmd/main_test.go"), + Language: "go", + Functions: []string{"TestFeature"}, + Imports: []string{"testing"}, + }, + } + if !reflect.DeepEqual(outcome.Analyses, want) { + t.Fatalf("analyses = %#v, want %#v", outcome.Analyses, want) + } + if len(outcome.Sources) != 1 || outcome.Sources[0].Source != "go-parser" || outcome.Sources[0].Status != ScanSourceFallback { + t.Fatalf("sources = %#v, want Go parser fallback", outcome.Sources) + } + if !strings.Contains(outcome.Sources[0].Detail, "2 of 3 Go files") || !strings.Contains(outcome.Sources[0].Detail, "skipped 1") { + t.Fatalf("detail = %q, want recovered and skipped counts", outcome.Sources[0].Detail) + } +} + +func TestBuildGoFallbackOutcomeUnavailable(t *testing.T) { + for _, tt := range []struct { + name string + files map[string]string + list []FileInfo + }{ + { + name: "no Go files", + files: map[string]string{"main.rs": "fn main() {}"}, + list: []FileInfo{{Path: "main.rs"}}, + }, + { + name: "no parseable Go files", + files: map[string]string{"broken.go": "package main\nfunc broken("}, + list: []FileInfo{{Path: "broken.go"}}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + writeRustCargoFixture(t, root, tt.files) + _, err := buildGoFallbackOutcome(context.Background(), root, tt.list) + if !errors.Is(err, errGoFallbackUnavailable) { + t.Fatalf("error = %v, want %v", err, errGoFallbackUnavailable) + } + }) + } +} + +func TestBuildGoFallbackOutcomeHonorsCancellation(t *testing.T) { + root := t.TempDir() + writeRustCargoFixture(t, root, map[string]string{"main.go": "package main\nfunc main() {}\n"}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := buildGoFallbackOutcome(ctx, root, []FileInfo{{Path: "main.go"}}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want %v", err, context.Canceled) + } +} From 8b6eb139c61f734697bb52dfd3c1f4e9450727d2 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:21:20 +0200 Subject: [PATCH 2/3] fix(scanner): Expose Go fallback to dependency scans Route public dependency scans through the Go parser fallback; keep cargo recovery graph-scoped, avoid duplicate cargo loads, and port tests to the post-#105 API. --- scanner/cargofallback.go | 31 +++++++++--- scanner/cargofallback_test.go | 90 +++++++++++++++++++++++++++++++++-- scanner/gofallback.go | 2 +- scanner/gofallback_test.go | 42 +++++++++++++++- scanner/walker.go | 22 ++++----- 5 files changed, 164 insertions(+), 23 deletions(-) diff --git a/scanner/cargofallback.go b/scanner/cargofallback.go index a812e97..ccc11c7 100644 --- a/scanner/cargofallback.go +++ b/scanner/cargofallback.go @@ -23,6 +23,12 @@ func buildFileGraphWithFallback(ctx context.Context, root string, scan dependenc } func buildFileGraphFromOutcomeWithCargoMetadataAndFilters(ctx context.Context, root string, outcome ScanOutcome, filters Filters, loader cargoMetadataLoader) (*FileGraph, error) { + for _, source := range outcome.Sources { + if source.Name == "cargo-metadata" && source.Status == ScanSourceFallback { + loader = nil + break + } + } fg, err := buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx, root, outcome.Analyses, filters, loader, outcome.Sources...) if err != nil { return nil, err @@ -32,7 +38,7 @@ func buildFileGraphFromOutcomeWithCargoMetadataAndFilters(ctx context.Context, r } func buildFileGraphWithFallbackWithFilters(ctx context.Context, root string, filters Filters, scan dependencyOutcomeScanner, loader cargoMetadataLoader) (*FileGraph, error) { - outcome, usedFallback, err := scanForGraphOutcomeWithFilters(ctx, root, filters, scan, loader) + outcome, usedFallback, err := scanForGraphOutcomeWithFilters(ctx, root, filters, scan, loader, true) if err != nil { return nil, err } @@ -43,9 +49,21 @@ func buildFileGraphWithFallbackWithFilters(ctx context.Context, root string, fil return buildFileGraphFromOutcomeWithCargoMetadataAndFilters(ctx, root, outcome, filters, graphLoader) } -func scanForGraphOutcomeWithFilters(ctx context.Context, root string, filters Filters, scan dependencyOutcomeScanner, loader cargoMetadataLoader) (ScanOutcome, bool, error) { +func scanForGraphOutcome(ctx context.Context, root string, scan dependencyOutcomeScanner, loader cargoMetadataLoader, allowCargoOnly ...bool) (ScanOutcome, bool, error) { + allow := true + if len(allowCargoOnly) > 0 { + allow = allowCargoOnly[0] + } + return scanForGraphOutcomeWithFilters(ctx, root, Filters{}, scan, loader, allow) +} + +func scanForGraphOutcomeWithFilters(ctx context.Context, root string, filters Filters, scan dependencyOutcomeScanner, loader cargoMetadataLoader, allowCargoOnly bool) (ScanOutcome, bool, error) { outcome, err := scan(root) if err == nil { + outcome.Analyses, err = filterAnalysesContext(ctx, outcome.Analyses, filters) + if err != nil { + return ScanOutcome{}, false, err + } return outcome, false, nil } @@ -73,10 +91,11 @@ func scanForGraphOutcomeWithFilters(ctx context.Context, root string, filters Fi if err := ctx.Err(); err != nil { return ScanOutcome{}, false, err } - - if cargoFallback, fallbackErr := buildCargoFallbackOutcome(ctx, root, files, loader); fallbackErr == nil { - mergeFallbackOutcome(&fallback, cargoFallback) - recovered = true + if recovered || allowCargoOnly { + if cargoFallback, fallbackErr := buildCargoFallbackOutcome(ctx, root, files, loader); fallbackErr == nil { + mergeFallbackOutcome(&fallback, cargoFallback) + recovered = true + } } if err := ctx.Err(); err != nil { return ScanOutcome{}, false, err diff --git a/scanner/cargofallback_test.go b/scanner/cargofallback_test.go index 40f5339..67c6d52 100644 --- a/scanner/cargofallback_test.go +++ b/scanner/cargofallback_test.go @@ -3,8 +3,10 @@ package scanner import ( "context" "errors" + "os" "path/filepath" "reflect" + "runtime" "testing" ) @@ -79,6 +81,7 @@ func TestScanForGraphOutcomeUsesGoFallbackWithoutCargo(t *testing.T) { loads++ return nil, errors.New("unexpected Cargo fallback") }, + false, ) if err != nil { t.Fatal(err) @@ -101,7 +104,7 @@ func TestScanForGraphOutcomeUsesGoFallbackWithoutCargo(t *testing.T) { if len(outcome.Sources) != 2 { t.Fatalf("sources = %#v, want ast-grep and Go parser", outcome.Sources) } - if outcome.Sources[0].Source != "ast-grep" || outcome.Sources[1].Source != "go-parser" { + if outcome.Sources[0].Name != "ast-grep" || outcome.Sources[1].Name != "go-parser" { t.Fatalf("sources = %#v, want ast-grep then Go parser", outcome.Sources) } } @@ -123,6 +126,7 @@ func TestScanForGraphOutcomeCombinesGoAndCargoFallbacks(t *testing.T) { func(context.Context, string) ([]byte, error) { return metadata, nil }, + false, ) if err != nil { t.Fatal(err) @@ -142,13 +146,88 @@ func TestScanForGraphOutcomeCombinesGoAndCargoFallbacks(t *testing.T) { t.Fatalf("edges = %#v, want %#v", got, want) } if len(outcome.Sources) != 3 || - outcome.Sources[0].Source != "ast-grep" || - outcome.Sources[1].Source != "go-parser" || - outcome.Sources[2].Source != "cargo-metadata" { + outcome.Sources[0].Name != "ast-grep" || + outcome.Sources[1].Name != "go-parser" || + outcome.Sources[2].Name != "cargo-metadata" { t.Fatalf("sources = %#v, want ast-grep, Go parser, and Cargo metadata", outcome.Sources) } } +func TestBuildFileGraphFromFallbackOutcomePreservesCargoEdgesWithoutReload(t *testing.T) { + root, metadata := cargoFallbackFixture(t, map[string]any{ + "name": "core", "path": "core", "kind": nil, + }) + writeRustCargoFixture(t, root, map[string]string{ + "main.go": "package main\n\nfunc main() {}\n", + }) + outcome, _, err := scanForGraphOutcome( + context.Background(), + root, + func(string) (ScanOutcome, error) { + return ScanOutcome{}, newIncompleteScanError("ast-grep", ScanSourceUnavailable, "ast-grep unavailable", ErrAstGrepNotFound) + }, + func(context.Context, string) ([]byte, error) { + return metadata, nil + }, + false, + ) + if err != nil { + t.Fatal(err) + } + + loads := 0 + graph, err := buildFileGraphFromOutcomeWithCargoMetadataAndFilters( + context.Background(), + root, + outcome, + Filters{}, + func(context.Context, string) ([]byte, error) { + loads++ + return metadata, nil + }, + ) + if err != nil { + t.Fatal(err) + } + if loads != 0 { + t.Fatalf("Cargo metadata reloads = %d, want 0", loads) + } + if got, want := graph.Imports["app/src/lib.rs"], []string{"core/src/lib.rs"}; !reflect.DeepEqual(got, want) { + t.Fatalf("fallback imports = %#v, want %#v", got, want) + } +} + +func TestScanForDepsOutcomeRejectsCargoOnlyEmptyRecovery(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("requires shell script execution") + } + + root, metadata := cargoFallbackFixture(t, map[string]any{ + "name": "core", "path": "core", "kind": nil, + }) + binDir := t.TempDir() + if err := os.WriteFile(filepath.Join(binDir, "sg"), []byte("#!/bin/sh\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(binDir, "cargo"), []byte("#!/bin/sh\n/bin/cat \"$CODEMAP_TEST_CARGO_METADATA\"\n"), 0o755); err != nil { + t.Fatal(err) + } + metadataPath := filepath.Join(t.TempDir(), "metadata.json") + if err := os.WriteFile(metadataPath, metadata, 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir) + t.Setenv("CODEMAP_TEST_CARGO_METADATA", metadataPath) + + outcome, err := ScanForDeps(context.Background(), root, Filters{}) + if !errors.Is(err, ErrAstGrepNotFound) { + t.Fatalf("error = %v, want %v", err, ErrAstGrepNotFound) + } + if len(outcome.Analyses) != 0 { + t.Fatalf("analyses = %#v, want no false successful analyses", outcome.Analyses) + } +} + func TestScanForGraphOutcomeHonorsCancellationDuringFallback(t *testing.T) { root := t.TempDir() writeRustCargoFixture(t, root, map[string]string{ @@ -168,6 +247,7 @@ func TestScanForGraphOutcomeHonorsCancellationDuringFallback(t *testing.T) { cancel() return nil, errors.New("metadata canceled") }, + false, ) if !errors.Is(err, context.Canceled) { t.Fatalf("error = %v, want %v", err, context.Canceled) @@ -189,6 +269,7 @@ func TestScanForGraphOutcomeHonorsPreCanceledContext(t *testing.T) { return ScanOutcome{}, newIncompleteScanError("ast-grep", ScanSourceFailed, "ast-grep failed", errors.New("scan failure")) }, nil, + false, ) if !errors.Is(err, context.Canceled) { t.Fatalf("error = %v, want %v", err, context.Canceled) @@ -206,6 +287,7 @@ func TestScanForGraphOutcomePreservesPrimaryErrorWhenFileScanFails(t *testing.T) return ScanOutcome{}, primaryErr }, nil, + false, ) if err != primaryErr { t.Fatalf("error = %v, want original %v", err, primaryErr) diff --git a/scanner/gofallback.go b/scanner/gofallback.go index 0fc0fa6..58f5dd4 100644 --- a/scanner/gofallback.go +++ b/scanner/gofallback.go @@ -71,7 +71,7 @@ func buildGoFallbackOutcome(ctx context.Context, root string, files []FileInfo) return ScanOutcome{ Analyses: analyses, Sources: []ScanSourceOutcome{{ - Source: "go-parser", + Name: "go-parser", Status: ScanSourceFallback, Detail: detail, }}, diff --git a/scanner/gofallback_test.go b/scanner/gofallback_test.go index 01ded25..85f63be 100644 --- a/scanner/gofallback_test.go +++ b/scanner/gofallback_test.go @@ -3,8 +3,10 @@ package scanner import ( "context" "errors" + "os" "path/filepath" "reflect" + "runtime" "strings" "testing" ) @@ -62,7 +64,7 @@ func TestFeature(t *testing.T) {} if !reflect.DeepEqual(outcome.Analyses, want) { t.Fatalf("analyses = %#v, want %#v", outcome.Analyses, want) } - if len(outcome.Sources) != 1 || outcome.Sources[0].Source != "go-parser" || outcome.Sources[0].Status != ScanSourceFallback { + if len(outcome.Sources) != 1 || outcome.Sources[0].Name != "go-parser" || outcome.Sources[0].Status != ScanSourceFallback { t.Fatalf("sources = %#v, want Go parser fallback", outcome.Sources) } if !strings.Contains(outcome.Sources[0].Detail, "2 of 3 Go files") || !strings.Contains(outcome.Sources[0].Detail, "skipped 1") { @@ -70,6 +72,44 @@ func TestFeature(t *testing.T) {} } } +func TestScanForDepsOutcomeUsesGoFallbackWithoutAstGrep(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("requires shell script execution") + } + + binDir := t.TempDir() + if err := os.WriteFile(filepath.Join(binDir, "sg"), []byte("#!/bin/sh\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir) + + root := t.TempDir() + writeRustCargoFixture(t, root, map[string]string{ + "cmd/main.go": "package main\n\nimport \"example.com/lib\"\n\nfunc main() {}\n", + }) + + outcome, err := ScanForDeps(context.Background(), root, Filters{}) + if err != nil { + t.Fatal(err) + } + want := []FileAnalysis{{ + Path: filepath.FromSlash("cmd/main.go"), + Language: "go", + Functions: []string{"main"}, + Imports: []string{"example.com/lib"}, + }} + if !reflect.DeepEqual(outcome.Analyses, want) { + t.Fatalf("analyses = %#v, want %#v", outcome.Analyses, want) + } + if len(outcome.Sources) != 2 || + outcome.Sources[0].Name != "ast-grep" || + outcome.Sources[0].Status != ScanSourceUnavailable || + outcome.Sources[1].Name != "go-parser" || + outcome.Sources[1].Status != ScanSourceFallback { + t.Fatalf("sources = %#v, want unavailable ast-grep followed by Go parser fallback", outcome.Sources) + } +} + func TestBuildGoFallbackOutcomeUnavailable(t *testing.T) { for _, tt := range []struct { name string diff --git a/scanner/walker.go b/scanner/walker.go index 5ef965e..fe2208c 100644 --- a/scanner/walker.go +++ b/scanner/walker.go @@ -364,8 +364,17 @@ func filterConfiguredAnalyses(root string, analyses []FileAnalysis) []FileAnalys } // ScanForDeps performs dependency analysis with explicit filters, provenance, -// and caller cancellation. +// and caller cancellation. When the primary ast-grep scan fails with an +// incomplete outcome, the Go parser fallback recovers what dependency +// references it can and the outcome stays provenance-annotated. func ScanForDeps(ctx context.Context, root string, filters Filters) (ScanOutcome, error) { + outcome, _, err := scanForGraphOutcomeWithFilters(ctx, root, filters, func(r string) (ScanOutcome, error) { + return scanForDepsPrimaryOutcome(ctx, r) + }, loadCargoFallbackMetadata, false) + return outcome, err +} + +func scanForDepsPrimaryOutcome(ctx context.Context, root string) (ScanOutcome, error) { if err := ctx.Err(); err != nil { return ScanOutcome{}, err } @@ -374,14 +383,5 @@ func ScanForDeps(ctx context.Context, root string, filters Filters) (ScanOutcome return ScanOutcome{}, err } defer astScanner.Close() - - outcome, err := astScanner.ScanDirectory(ctx, root) - if err != nil { - return ScanOutcome{}, err - } - outcome.Analyses, err = filterAnalysesContext(ctx, outcome.Analyses, filters) - if err != nil { - return ScanOutcome{}, err - } - return outcome, nil + return astScanner.ScanDirectory(ctx, root) } From 05640a9adcfb8fa9ac4bd6c9f62faa7109b305e3 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:38:17 +0200 Subject: [PATCH 3/3] fix(scanner): Close fallback honesty gaps from maintainer review - Emit coverage in the empty --importers CLI branch; scope the Go note to .go - Fire the Go fallback on degraded ast-grep outcomes (timeout/failure) - Dedup cargo-metadata sources in the --deps graph build - Drop the recovered-edge claim from the deps payload note - Count only dependency-bearing files in the recovery note; skip methods --- blast_radius.go | 9 ++- main_cli_polish_test.go | 25 ++++++++ scanner/cargofallback.go | 43 +++++++++++-- scanner/cargofallback_test.go | 111 ++++++++++++++++++++++++++++++++++ scanner/filegraph.go | 19 +++--- scanner/gofallback.go | 9 ++- scanner/gofallback_test.go | 40 ++++++++++-- scanner/walker.go | 6 +- 8 files changed, 236 insertions(+), 26 deletions(-) diff --git a/blast_radius.go b/blast_radius.go index 788429d..40e0d28 100644 --- a/blast_radius.go +++ b/blast_radius.go @@ -1314,8 +1314,13 @@ func renderImportersReportString(report scanner.ImportersReport) string { func renderImportersReportCLI(w io.Writer, report scanner.ImportersReport) { if len(report.Importers) == 0 && len(report.HubImports) == 0 { fmt.Fprintf(w, "No files import %s.\n", report.File) - fmt.Fprintln(w, " Note: files in the same package never import each other (Go resolves") - fmt.Fprintln(w, " imports at package level), so only cross-package importers appear here.") + if strings.EqualFold(filepath.Ext(report.File), ".go") { + fmt.Fprintln(w, " Note: files in the same package never import each other (Go resolves") + fmt.Fprintln(w, " imports at package level), so only cross-package importers appear here.") + } + // Show scan provenance even for an empty answer, so a partial scan + // isn't read as a confident negative. + renderCoverage(w, report.CoverageStatus, report.CoverageNotes) return } renderImportersReport(w, report) diff --git a/main_cli_polish_test.go b/main_cli_polish_test.go index 1c43de8..29574d3 100644 --- a/main_cli_polish_test.go +++ b/main_cli_polish_test.go @@ -31,6 +31,31 @@ func TestRenderImportersReportExplainsEmptyResult(t *testing.T) { } } +// A partial scan must not read as a confident negative: the empty importers +// branch shows coverage and skips the Go same-package note for non-Go files. +func TestRenderImportersReportCLIEmitsCoverageInEmptyBranch(t *testing.T) { + report := scanner.ImportersReport{ + Root: "/repo", + Mode: "importers", + File: "web/util.ts", + CoverageStatus: "partial", + CoverageNotes: []string{"ast-grep not found (checked bundled tools and PATH)", "Go parser fallback recovered 1 of 1 Go files"}, + } + + var buf strings.Builder + renderImportersReportCLI(&buf, report) + out := buf.String() + if !strings.Contains(out, "No files import web/util.ts") { + t.Fatalf("empty importers result must say so:\n%q", out) + } + if !strings.Contains(out, "Coverage: partial") { + t.Fatalf("empty result must carry the coverage provenance:\n%q", out) + } + if strings.Contains(out, "same package") { + t.Fatalf("Go same-package note must not be printed about a TypeScript file:\n%q", out) + } +} + func TestNonexistentPathGetsFriendlyError(t *testing.T) { _, stderr, err := runCodemapWithInput("", "drift") if err == nil { diff --git a/scanner/cargofallback.go b/scanner/cargofallback.go index ccc11c7..caae93c 100644 --- a/scanner/cargofallback.go +++ b/scanner/cargofallback.go @@ -59,12 +59,20 @@ func scanForGraphOutcome(ctx context.Context, root string, scan dependencyOutcom func scanForGraphOutcomeWithFilters(ctx context.Context, root string, filters Filters, scan dependencyOutcomeScanner, loader cargoMetadataLoader, allowCargoOnly bool) (ScanOutcome, bool, error) { outcome, err := scan(root) + degraded := false if err == nil { - outcome.Analyses, err = filterAnalysesContext(ctx, outcome.Analyses, filters) - if err != nil { - return ScanOutcome{}, false, err + // A degraded ast-grep outcome (timeout/failure) carries a nil error; + // treat it like an incomplete scan so the fallback still runs. + if incomplete := degradedAstGrepError(outcome); incomplete != nil { + err = incomplete + degraded = true + } else { + outcome.Analyses, err = filterAnalysesContext(ctx, outcome.Analyses, filters) + if err != nil { + return ScanOutcome{}, false, err + } + return outcome, false, nil } - return outcome, false, nil } var incomplete *IncompleteScanError @@ -101,11 +109,34 @@ func scanForGraphOutcomeWithFilters(ctx context.Context, root string, filters Fi return ScanOutcome{}, false, err } if !recovered { + if degraded { + // Nothing recovered from a degraded scan: fail closed with the + // degraded outcome, not a hard error. + return outcome, false, nil + } return ScanOutcome{}, false, err } return fallback, true, nil } +// degradedAstGrepError turns a fail-closed ast-grep outcome into the +// incomplete error the fallback gate recognizes, unless it already recovered +// analyses. +func degradedAstGrepError(outcome ScanOutcome) *IncompleteScanError { + if len(outcome.Analyses) > 0 { + return nil + } + for _, source := range outcome.Sources { + if source.Name != "ast-grep" { + continue + } + if source.Status == ScanSourceTimeout || source.Status == ScanSourceFailed { + return &IncompleteScanError{Outcome: source, Err: errors.New(source.Detail)} + } + } + return nil +} + func mergeFallbackOutcome(dst *ScanOutcome, src ScanOutcome) { dst.Analyses = append(dst.Analyses, src.Analyses...) dst.Sources = append(dst.Sources, src.Sources...) @@ -198,7 +229,9 @@ func buildCargoFallbackOutcome(ctx context.Context, root string, files []FileInf Sources: []ScanSourceOutcome{{ Name: "cargo-metadata", Status: ScanSourceFallback, - Detail: fmt.Sprintf("Cargo metadata fallback recovered %d dependency edges from %d of %d manifests", len(edges), handled, len(manifests)), + // Recovered edges only reach the file graph, so don't claim them + // in the `--deps` payload. + Detail: fmt.Sprintf("Cargo metadata fallback used for %d of %d manifests", handled, len(manifests)), }}, precomputedEdges: edges, }, nil diff --git a/scanner/cargofallback_test.go b/scanner/cargofallback_test.go index 67c6d52..38cecb1 100644 --- a/scanner/cargofallback_test.go +++ b/scanner/cargofallback_test.go @@ -197,6 +197,117 @@ func TestBuildFileGraphFromFallbackOutcomePreservesCargoEdgesWithoutReload(t *te } } +// A degraded ast-grep outcome (timeout/failure, nil error) must still fire +// the fallback: ScanDirectory fails closed, so the gate checks provenance. +func TestScanForGraphOutcomeFiresFallbackOnDegradedAstGrepOutcome(t *testing.T) { + root := t.TempDir() + writeRustCargoFixture(t, root, map[string]string{ + "main.go": "package main\n\nimport \"fmt\"\n\nfunc main() {}\n", + }) + outcome, usedFallback, err := scanForGraphOutcome( + context.Background(), + root, + func(string) (ScanOutcome, error) { + // Degraded outcome, nil error — what ScanDirectory returns on + // invalid JSON. The leading non-ast-grep source exercises the + // skip branch. + return ScanOutcome{ + Sources: []ScanSourceOutcome{ + {Name: "other", Status: ScanSourceFailed, Detail: "unrelated"}, + {Name: "ast-grep", Status: ScanSourceFailed, Detail: "ast-grep produced invalid JSON results"}, + }, + }, nil + }, + nil, + false, + ) + if err != nil { + t.Fatal(err) + } + if !usedFallback { + t.Fatal("degraded ast-grep outcome did not trigger the fallback") + } + if len(outcome.Analyses) != 1 || outcome.Analyses[0].Path != "main.go" { + t.Fatalf("analyses = %#v, want Go fallback recovery", outcome.Analyses) + } + if len(outcome.Sources) != 2 || + outcome.Sources[0].Name != "ast-grep" || outcome.Sources[0].Status != ScanSourceFailed || + outcome.Sources[1].Name != "go-parser" || outcome.Sources[1].Status != ScanSourceFallback { + t.Fatalf("sources = %#v, want degraded ast-grep followed by Go parser fallback", outcome.Sources) + } +} + +// An unrecoverable degraded primary stays fail-closed (nil error). +func TestScanForGraphOutcomeDegradedPrimaryStaysFailClosedWhenUnrecoverable(t *testing.T) { + root := t.TempDir() + writeRustCargoFixture(t, root, map[string]string{ + "main.ts": "export const value = 1;\n", + }) + primary := ScanOutcome{ + Sources: []ScanSourceOutcome{{Name: "ast-grep", Status: ScanSourceTimeout, Detail: "ast-grep timed out after 30s"}}, + } + outcome, usedFallback, err := scanForGraphOutcome( + context.Background(), + root, + func(string) (ScanOutcome, error) { return primary, nil }, + nil, + false, + ) + if err != nil { + t.Fatalf("unrecoverable degraded primary must fail closed, got error %v", err) + } + if usedFallback { + t.Fatal("unrecoverable degraded primary must not report fallback use") + } + if !reflect.DeepEqual(outcome, primary) { + t.Fatalf("outcome = %#v, want degraded primary preserved", outcome) + } +} + +// BuildFileGraphFromOutcome (the --deps graph path) must apply the Cargo +// dedup guard: one cargo-metadata source, no second cargo metadata run. +func TestBuildFileGraphFromOutcomeDedupesFallbackCargoSource(t *testing.T) { + root, metadata := cargoFallbackFixture(t, map[string]any{ + "name": "core", "path": "core", "kind": nil, + }) + writeRustCargoFixture(t, root, map[string]string{ + "main.go": "package main\n\nimport \"fmt\"\n\nfunc main() {}\n", + }) + outcome, _, err := scanForGraphOutcome( + context.Background(), + root, + func(string) (ScanOutcome, error) { + return ScanOutcome{}, newIncompleteScanError("ast-grep", ScanSourceUnavailable, "ast-grep unavailable", ErrAstGrepNotFound) + }, + func(context.Context, string) ([]byte, error) { return metadata, nil }, + false, + ) + if err != nil { + t.Fatal(err) + } + + loads := 0 + graph, err := BuildFileGraphFromOutcome(context.Background(), root, outcome, Filters{}) + if err != nil { + t.Fatal(err) + } + if loads != 0 { + t.Fatalf("cargo metadata loads = %d, want 0 (dedup guard must apply)", loads) + } + var cargoSources int + for _, source := range graph.Coverage.Sources { + if source.Name == "cargo-metadata" { + cargoSources++ + } + } + if cargoSources != 1 { + t.Fatalf("cargo-metadata sources = %d, want exactly 1 (got %#v)", cargoSources, graph.Coverage.Sources) + } + if got, want := graph.Imports["app/src/lib.rs"], []string{"core/src/lib.rs"}; !reflect.DeepEqual(got, want) { + t.Fatalf("fallback imports = %#v, want %#v", got, want) + } +} + func TestScanForDepsOutcomeRejectsCargoOnlyEmptyRecovery(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("requires shell script execution") diff --git a/scanner/filegraph.go b/scanner/filegraph.go index 3a72133..e183fc6 100644 --- a/scanner/filegraph.go +++ b/scanner/filegraph.go @@ -42,15 +42,10 @@ func BuildFileGraph(ctx context.Context, root string, filters Filters) (*FileGra }, loadCargoFallbackMetadata) } -// BuildFileGraphFromOutcome builds a file graph from a scan outcome without -// dropping scanner provenance. +// BuildFileGraphFromOutcome builds a file graph from a scan outcome, reusing +// the Cargo dedup guard so a fallback outcome doesn't re-run cargo metadata. func BuildFileGraphFromOutcome(ctx context.Context, root string, outcome ScanOutcome, filters Filters) (*FileGraph, error) { - fg, err := buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx, root, outcome.Analyses, filters, loadCargoMetadata, outcome.Sources...) - if err != nil { - return nil, err - } - applyPrecomputedFileEdges(fg, outcome.precomputedEdges) - return fg, nil + return buildFileGraphFromOutcomeWithCargoMetadataAndFilters(ctx, root, outcome, filters, loadCargoMetadata) } // BuildFileGraphFromAnalyses builds a file graph from pre-computed analyses @@ -92,8 +87,12 @@ func buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx context.Context, Packages: make(map[string][]string), PathAliases: make(map[string][]string), } + hasCargoSource := false for _, source := range sources { fg.Coverage.AddSource(source) + if source.Name == "cargo-metadata" { + hasCargoSource = true + } } // Detect module name from go.mod (for Go import resolution) @@ -125,7 +124,9 @@ func buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx context.Context, if err != nil { return nil, err } - if cargoOutcome != nil { + // The outcome already carries cargo provenance; keep exactly one + // cargo-metadata source per graph. + if cargoOutcome != nil && !hasCargoSource { fg.Coverage.AddSource(*cargoOutcome) } diff --git a/scanner/gofallback.go b/scanner/gofallback.go index 58f5dd4..c5cec66 100644 --- a/scanner/gofallback.go +++ b/scanner/gofallback.go @@ -17,7 +17,10 @@ var errGoFallbackUnavailable = errors.New("go dependency fallback unavailable") func buildGoFallbackOutcome(ctx context.Context, root string, files []FileInfo) (ScanOutcome, error) { var analyses []FileAnalysis + // Count only dependency-bearing files in the recovery note: type/const-only + // files produce no analysis, so including them would read as data loss. goFiles := 0 + dependencyFiles := 0 skipped := 0 fset := token.NewFileSet() @@ -46,7 +49,8 @@ func buildGoFallbackOutcome(ctx context.Context, root string, files []FileInfo) } for _, declaration := range parsed.Decls { function, ok := declaration.(*ast.FuncDecl) - if ok && function.Name != nil && function.Name.Name != "" { + // Skip methods; ast-grep only reports function_declaration. + if ok && function.Recv == nil && function.Name != nil && function.Name.Name != "" { analysis.Functions = append(analysis.Functions, function.Name.Name) } } @@ -54,6 +58,7 @@ func buildGoFallbackOutcome(ctx context.Context, root string, files []FileInfo) analysis.Functions = dedupe(analysis.Functions) if len(analysis.Imports) > 0 || len(analysis.Functions) > 0 { analyses = append(analyses, analysis) + dependencyFiles++ } } @@ -64,7 +69,7 @@ func buildGoFallbackOutcome(ctx context.Context, root string, files []FileInfo) return analyses[i].Path < analyses[j].Path }) - detail := fmt.Sprintf("Go parser fallback recovered %d of %d Go files", len(analyses), goFiles) + detail := fmt.Sprintf("Go parser fallback recovered %d of %d Go files with dependency references", len(analyses), dependencyFiles) if skipped > 0 { detail += fmt.Sprintf(" (skipped %d with parse errors)", skipped) } diff --git a/scanner/gofallback_test.go b/scanner/gofallback_test.go index 85f63be..bc03197 100644 --- a/scanner/gofallback_test.go +++ b/scanner/gofallback_test.go @@ -49,9 +49,10 @@ func TestFeature(t *testing.T) {} } want := []FileAnalysis{ { - Path: filepath.FromSlash("cmd/main.go"), - Language: "go", - Functions: []string{"main", "Run"}, + Path: filepath.FromSlash("cmd/main.go"), + Language: "go", + // `Run` is a method; ast-grep only reports function_declaration. + Functions: []string{"main"}, Imports: []string{"example.com/lib", "example.com/side-effect"}, }, { @@ -67,7 +68,7 @@ func TestFeature(t *testing.T) {} if len(outcome.Sources) != 1 || outcome.Sources[0].Name != "go-parser" || outcome.Sources[0].Status != ScanSourceFallback { t.Fatalf("sources = %#v, want Go parser fallback", outcome.Sources) } - if !strings.Contains(outcome.Sources[0].Detail, "2 of 3 Go files") || !strings.Contains(outcome.Sources[0].Detail, "skipped 1") { + if !strings.Contains(outcome.Sources[0].Detail, "2 of 2 Go files with dependency references") || !strings.Contains(outcome.Sources[0].Detail, "skipped 1") { t.Fatalf("detail = %q, want recovered and skipped counts", outcome.Sources[0].Detail) } } @@ -138,6 +139,37 @@ func TestBuildGoFallbackOutcomeUnavailable(t *testing.T) { } } +// The recovery note's denominator counts only dependency-bearing files, so +// type/const-only files (which ast-grep omits too) don't read as data loss. +func TestBuildGoFallbackOutcomeDenominatorExcludesTypeOnlyFiles(t *testing.T) { + root := t.TempDir() + writeRustCargoFixture(t, root, map[string]string{ + "cmd/main.go": "package main\n\nimport \"fmt\"\n\nfunc main() {}\n", + "cmd/types.go": "package main\n\ntype Worker struct{}\n\nconst Max = 3\n", + "cmd/broken.go": "package main\nfunc broken(", + "cmd/constants.go": "package main\n\nconst Version = \"v1\"\n", + }) + files := []FileInfo{ + {Path: "cmd/types.go"}, + {Path: "cmd/main.go"}, + {Path: "cmd/broken.go"}, + {Path: "cmd/constants.go"}, + } + outcome, err := buildGoFallbackOutcome(context.Background(), root, files) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(outcome.Sources[0].Detail, "1 of 1 Go files with dependency references") { + t.Fatalf("detail = %q, want 1 of 1 dependency-bearing files", outcome.Sources[0].Detail) + } + if !strings.Contains(outcome.Sources[0].Detail, "skipped 1") { + t.Fatalf("detail = %q, want skipped parse-error count", outcome.Sources[0].Detail) + } + if strings.Contains(outcome.Sources[0].Detail, "of 4 Go files") { + t.Fatalf("detail = %q, must not count type/const-only files in the denominator", outcome.Sources[0].Detail) + } +} + func TestBuildGoFallbackOutcomeHonorsCancellation(t *testing.T) { root := t.TempDir() writeRustCargoFixture(t, root, map[string]string{"main.go": "package main\nfunc main() {}\n"}) diff --git a/scanner/walker.go b/scanner/walker.go index fe2208c..ddba508 100644 --- a/scanner/walker.go +++ b/scanner/walker.go @@ -363,10 +363,8 @@ func filterConfiguredAnalyses(root string, analyses []FileAnalysis) []FileAnalys return filtered } -// ScanForDeps performs dependency analysis with explicit filters, provenance, -// and caller cancellation. When the primary ast-grep scan fails with an -// incomplete outcome, the Go parser fallback recovers what dependency -// references it can and the outcome stays provenance-annotated. +// ScanForDeps runs dependency analysis with explicit filters and cancellation, +// falling back to the Go parser when ast-grep fails. func ScanForDeps(ctx context.Context, root string, filters Filters) (ScanOutcome, error) { outcome, _, err := scanForGraphOutcomeWithFilters(ctx, root, filters, func(r string) (ScanOutcome, error) { return scanForDepsPrimaryOutcome(ctx, r)