diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6c60f65..23020cc 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -60,7 +60,7 @@ A `walkEcosystem` table declares how one ecosystem participates in the shared wa | any-depth name | `__pycache__`, `.venv` | matches the directory name anywhere under the nearest project root | | any-depth suffix | `*.egg-info` | matches a directory-name suffix anywhere under the nearest project root | -Per directory, the engine matches artifact rules against the **nearest** enclosing project root of each active ecosystem (table order, first match wins), emits the artifact and skips its subtree on a match, and otherwise reads the directory once to detect new project roots (pushed onto a recursion-scoped context stack) before descending into its child directories. Artifact matching runs **before** the hidden-directory check so compound rules ending in a hidden segment (`android/.gradle`) still match; unmatched hidden directories below the scan root are descended into only when an active ecosystem lists the name as an artifact. +Per directory, the engine matches artifact rules against the **nearest** enclosing project root of each active ecosystem (table order, first match wins), emits the artifact and skips its subtree on a match, and otherwise reads the directory once to detect new project roots (pushed onto a recursion-scoped context stack) before descending into its child directories. A child directory it would descend into is handed to its own goroutine, with a copy of the context stack, while one of 8 walk slots is free (separate from the sizing slots, so the walk is not starved while the stat scanners size); results are sorted by (table order, path) at the end, so they do not depend on the split. Artifact matching runs **before** the hidden-directory check so compound rules ending in a hidden segment (`android/.gradle`) still match; unmatched hidden directories below the scan root are descended into only when an active ecosystem lists the name as an artifact. Tables can also: diff --git a/internal/scanner/walk.go b/internal/scanner/walk.go index 7fa097a..2d44d28 100644 --- a/internal/scanner/walk.go +++ b/internal/scanner/walk.go @@ -154,98 +154,127 @@ func runWalk(ctx context.Context, root string, tables []walkEcosystem) ([]model. } } + var mu sync.Mutex // guards results var results []model.ScanResult - var stack []projectContext numTables := len(tables) + var wg sync.WaitGroup - // fstree does the traversal; the engine only decides what a directory is. - walker := fstree.Walker{ - FollowRoot: true, - // Artifact match first, before the hidden-dir check, against the - // ancestor contexts (dir's own context is pushed only if we descend). - // Size is filled in afterwards by sizePending. - Enter: func(dir string, info fs.FileInfo) bool { - name := filepath.Base(dir) - if rule, tableIdx, projRoot, ok := matchArtifact(stack, dir, name, numTables); ok { - result := model.ScanResult{ - Path: dir, - Ecosystem: tables[tableIdx].Eco, - Category: rule.Category, - LastMod: info.ModTime(), - Safety: rule.Safety, + // walk walks the tree under top with stack as the ancestors' project + // contexts. A directory it would descend into is handed to its own walk, + // with a copy of the stack, whenever a walkTokens slot is free; results + // are sorted below, so the order does not depend on the split. + var walk func(top string, stack []projectContext) + walk = func(top string, stack []projectContext) { + // fstree does the traversal; the engine only decides what a directory is. + walker := fstree.Walker{ + FollowRoot: top == root, + // Artifact match first, before the hidden-dir check, against the + // ancestor contexts (dir's own context is pushed only if we descend). + // Size is filled in afterwards by sizePending. + Enter: func(dir string, info fs.FileInfo) bool { + name := filepath.Base(dir) + if rule, tableIdx, projRoot, ok := matchArtifact(stack, dir, name, numTables); ok { + result := model.ScanResult{ + Path: dir, + Ecosystem: tables[tableIdx].Eco, + Category: rule.Category, + LastMod: info.ModTime(), + Safety: rule.Safety, + } + if tables[tableIdx].SetProjectRoot { + result.ProjectRoot = projRoot + } + if rule.Describe != nil { + n := rule.Describe(dir) + result.Recommendation = n.Recommendation + result.CloneAware, result.CloneSource = n.CloneAware, n.CloneSource + } + mu.Lock() + results = append(results, result) + ReportProgress(ctx, len(results)) + mu.Unlock() + return false // matched artifact: do not descend } - if tables[tableIdx].SetProjectRoot { - result.ProjectRoot = projRoot + if dir == top { + return true // checked by the walk that handed it over, or the scan root } - if rule.Describe != nil { - n := rule.Describe(dir) - result.Recommendation = n.Recommendation - result.CloneAware, result.CloneSource = n.CloneAware, n.CloneSource + // After artifact matching, so a project's own node_modules is + // still reported. + if installedPackageDirNames[name] { + return false } - results = append(results, result) - ReportProgress(ctx, len(results)) - return false // matched artifact: do not descend - } - // After artifact matching, so a project's own node_modules is - // still reported. - if installedPackageDirNames[name] { - return false - } - // The root is always entered: the hidden-dir rule applies below it. - return dir == root || !strings.HasPrefix(name, ".") || hiddenDescend[name] - }, - Entries: func(dir string, entries []fs.DirEntry) bool { - names := make(map[string]bool, len(entries)) - for _, e := range entries { - names[e.Name()] = true - } - - // Prune before establishing any context or descending, so nothing - // inside is ever a deletion target. - if isInstalledPackageTree(dir, names) { - return false - } - for i := range tables { - if p := tables[i].PruneRoot; p != nil && p(dir, names) { + // The hidden-dir rule applies below the root. + if strings.HasPrefix(name, ".") && !hiddenDescend[name] { return false } - } + select { + case walkTokens <- struct{}{}: + sub := slices.Clone(stack) + wg.Go(func() { + defer func() { <-walkTokens }() + walk(dir, sub) + }) + return false // the handed-over walk enters dir itself + default: + return true + } + }, + Entries: func(dir string, entries []fs.DirEntry) bool { + names := make(map[string]bool, len(entries)) + for _, e := range entries { + names[e.Name()] = true + } - // Marker check: establish the project contexts rooted at this directory. - for i := range tables { - t := &tables[i] - rooted := false - for _, m := range t.Markers { - if names[m] { - rooted = true - break - } + // Prune before establishing any context or descending, so nothing + // inside is ever a deletion target. + if isInstalledPackageTree(dir, names) { + return false } - if !rooted { - continue + for i := range tables { + if p := tables[i].PruneRoot; p != nil && p(dir, names) { + return false + } } - rules := t.Rules - if t.ExtraRules != nil { - if extra := t.ExtraRules(dir, names); len(extra) > 0 { - combined := make([]artifactRule, 0, len(t.Rules)+len(extra)) - combined = append(combined, t.Rules...) - combined = append(combined, extra...) - rules = combined + // Marker check: establish the project contexts rooted at this directory. + for i := range tables { + t := &tables[i] + rooted := false + for _, m := range t.Markers { + if names[m] { + rooted = true + break + } + } + if !rooted { + continue + } + + rules := t.Rules + if t.ExtraRules != nil { + if extra := t.ExtraRules(dir, names); len(extra) > 0 { + combined := make([]artifactRule, 0, len(t.Rules)+len(extra)) + combined = append(combined, t.Rules...) + combined = append(combined, extra...) + rules = combined + } } + stack = append(stack, projectContext{root: dir, tableIdx: i, rules: rules}) } - stack = append(stack, projectContext{root: dir, tableIdx: i, rules: rules}) - } - return true - }, - // The contexts dir pushed are the ones on top rooted at dir. - Leave: func(dir string) { - for len(stack) > 0 && stack[len(stack)-1].root == dir { - stack = stack[:len(stack)-1] - } - }, + return true + }, + // The contexts dir pushed are the ones on top rooted at dir. + Leave: func(dir string) { + for len(stack) > 0 && stack[len(stack)-1].root == dir { + stack = stack[:len(stack)-1] + } + }, + } + _, _ = walker.Walk(ctx, top) } - if _, err := walker.Walk(ctx, root); err != nil { + walk(root, nil) + wg.Wait() + if err := ctx.Err(); err != nil { return nil, err } @@ -268,6 +297,11 @@ func runWalk(ctx context.Context, root string, tables []walkEcosystem) ([]model. return results, nil } +// walkTokens bounds the extra goroutines runWalk starts to walk subtrees in +// parallel. It is separate from sizeTokens so the walk is not starved while +// the stat scanners size their caches alongside it. +var walkTokens = make(chan struct{}, sizeWorkerCap) + // installedPackageDirNames hold only installed packages when the walk reaches // them without matching an artifact rule (a global lib/node_modules, a tool's // bundled dependencies, a Python environment's site-packages). Enter prunes diff --git a/internal/scanner/walk_traversal_test.go b/internal/scanner/walk_traversal_test.go index b5be58e..97064bd 100644 --- a/internal/scanner/walk_traversal_test.go +++ b/internal/scanner/walk_traversal_test.go @@ -163,3 +163,46 @@ func TestWalkScan_HiddenNamedRootIsScanned(t *testing.T) { t.Errorf("expected only %s, got %+v", cache, results) } } + +// TestWalkScan_SplitMatchesSerial scans a tree of nested projects twice: with +// every walk slot taken (one goroutine walks it all) and with the slots free +// (subtrees are walked in parallel, each with a copy of its ancestors' +// project contexts). Both must report the same artifacts, attributed to the +// same project roots. +func TestWalkScan_SplitMatchesSerial(t *testing.T) { + root := t.TempDir() + for _, p := range []string{ + "mono/package.json", "mono/node_modules/x", + "mono/packages/a/package.json", "mono/packages/a/node_modules/x", "mono/packages/a/dist/x", + "mono/packages/b/src/__pycache__/x", "mono/packages/b/pyproject.toml", + "mono/packages/b/.venv/pyvenv.cfg", "mono/packages/b/sub/lib/__pycache__/x", + "rs/Cargo.toml", "rs/target/x", "rs/crates/c/Cargo.toml", "rs/crates/c/target/x", + "plain/deep/er/__pycache__/x", ".hidden/package.json", ".hidden/node_modules/x", + } { + touch(t, filepath.Join(root, filepath.FromSlash(p))) + } + scan := func() []string { + results, err := runWalk(context.Background(), root, walkEcosystemTable) + if err != nil { + t.Fatal(err) + } + var out []string + for _, r := range results { + out = append(out, r.Path+" "+string(r.Ecosystem)+" "+r.ProjectRoot) + } + return out + } + + for range cap(walkTokens) { + walkTokens <- struct{}{} + } + serial := scan() + for range cap(walkTokens) { + <-walkTokens + } + split := scan() + + if len(serial) == 0 || !slices.Equal(serial, split) { + t.Errorf("split:\n%v\nserial:\n%v", split, serial) + } +}