perf: 프로젝트 하위 디렉토리를 병렬로 탐색

docs/ARCHITECTURE.md

@@ -60,7 +60,7 @@ A `walkEcosystem` table declares how one ecosystem participates in the shared wa
 | any-depth name | `__pycache__`, `.venv` | matches the directory name anywhere under the nearest project root |
 | any-depth suffix | `*.egg-info` | matches a directory-name suffix anywhere under the nearest project root |
 
-Per directory, the engine matches artifact rules against the **nearest** enclosing project root of each active ecosystem (table order, first match wins), emits the artifact and skips its subtree on a match, and otherwise reads the directory once to detect new project roots (pushed onto a recursion-scoped context stack) before descending into its child directories. Artifact matching runs **before** the hidden-directory check so compound rules ending in a hidden segment (`android/.gradle`) still match; unmatched hidden directories below the scan root are descended into only when an active ecosystem lists the name as an artifact.
+Per directory, the engine matches artifact rules against the **nearest** enclosing project root of each active ecosystem (table order, first match wins), emits the artifact and skips its subtree on a match, and otherwise reads the directory once to detect new project roots (pushed onto a recursion-scoped context stack) before descending into its child directories. A child directory it would descend into is handed to its own goroutine, with a copy of the context stack, while one of 8 walk slots is free (separate from the sizing slots, so the walk is not starved while the stat scanners size); results are sorted by (table order, path) at the end, so they do not depend on the split. Artifact matching runs **before** the hidden-directory check so compound rules ending in a hidden segment (`android/.gradle`) still match; unmatched hidden directories below the scan root are descended into only when an active ecosystem lists the name as an artifact.
 
 Tables can also:
 

internal/scanner/walk.go

@@ -154,98 +154,127 @@ func runWalk(ctx context.Context, root string, tables []walkEcosystem) ([]model.
 		}
 	}
 
+	var mu sync.Mutex // guards results
 	var results []model.ScanResult
-	var stack []projectContext
 	numTables := len(tables)
+	var wg sync.WaitGroup
 
-	// fstree does the traversal; the engine only decides what a directory is.
-	walker := fstree.Walker{
-		FollowRoot: true,
-		// Artifact match first, before the hidden-dir check, against the
-		// ancestor contexts (dir's own context is pushed only if we descend).
-		// Size is filled in afterwards by sizePending.
-		Enter: func(dir string, info fs.FileInfo) bool {
-			name := filepath.Base(dir)
-			if rule, tableIdx, projRoot, ok := matchArtifact(stack, dir, name, numTables); ok {
-				result := model.ScanResult{
-					Path:      dir,
-					Ecosystem: tables[tableIdx].Eco,
-					Category:  rule.Category,
-					LastMod:   info.ModTime(),
-					Safety:    rule.Safety,
+	// walk walks the tree under top with stack as the ancestors' project
+	// contexts. A directory it would descend into is handed to its own walk,
+	// with a copy of the stack, whenever a walkTokens slot is free; results
+	// are sorted below, so the order does not depend on the split.
+	var walk func(top string, stack []projectContext)
+	walk = func(top string, stack []projectContext) {
+		// fstree does the traversal; the engine only decides what a directory is.
+		walker := fstree.Walker{
+			FollowRoot: top == root,
+			// Artifact match first, before the hidden-dir check, against the
+			// ancestor contexts (dir's own context is pushed only if we descend).
+			// Size is filled in afterwards by sizePending.
+			Enter: func(dir string, info fs.FileInfo) bool {
+				name := filepath.Base(dir)
+				if rule, tableIdx, projRoot, ok := matchArtifact(stack, dir, name, numTables); ok {
+					result := model.ScanResult{
+						Path:      dir,
+						Ecosystem: tables[tableIdx].Eco,
+						Category:  rule.Category,
+						LastMod:   info.ModTime(),
+						Safety:    rule.Safety,
+					}
+					if tables[tableIdx].SetProjectRoot {
+						result.ProjectRoot = projRoot
+					}
+					if rule.Describe != nil {
+						n := rule.Describe(dir)
+						result.Recommendation = n.Recommendation
+						result.CloneAware, result.CloneSource = n.CloneAware, n.CloneSource
+					}
+					mu.Lock()
+					results = append(results, result)
+					ReportProgress(ctx, len(results))
+					mu.Unlock()
+					return false // matched artifact: do not descend
 				}
-				if tables[tableIdx].SetProjectRoot {
-					result.ProjectRoot = projRoot
+				if dir == top {
+					return true // checked by the walk that handed it over, or the scan root
 				}
-				if rule.Describe != nil {
-					n := rule.Describe(dir)
-					result.Recommendation = n.Recommendation
-					result.CloneAware, result.CloneSource = n.CloneAware, n.CloneSource
+				// After artifact matching, so a project's own node_modules is
+				// still reported.
+				if installedPackageDirNames[name] {
+					return false
 				}
-				results = append(results, result)
-				ReportProgress(ctx, len(results))
-				return false // matched artifact: do not descend
-			}
-			// After artifact matching, so a project's own node_modules is
-			// still reported.
-			if installedPackageDirNames[name] {
-				return false
-			}
-			// The root is always entered: the hidden-dir rule applies below it.
-			return dir == root || !strings.HasPrefix(name, ".") || hiddenDescend[name]
-		},
-		Entries: func(dir string, entries []fs.DirEntry) bool {
-			names := make(map[string]bool, len(entries))
-			for _, e := range entries {
-				names[e.Name()] = true
-			}
-
-			// Prune before establishing any context or descending, so nothing
-			// inside is ever a deletion target.
-			if isInstalledPackageTree(dir, names) {
-				return false
-			}
-			for i := range tables {
-				if p := tables[i].PruneRoot; p != nil && p(dir, names) {
+				// The hidden-dir rule applies below the root.
+				if strings.HasPrefix(name, ".") && !hiddenDescend[name] {
 					return false
 				}
-			}
+				select {
+				case walkTokens <- struct{}{}:
+					sub := slices.Clone(stack)
+					wg.Go(func() {
+						defer func() { <-walkTokens }()
+						walk(dir, sub)
+					})
+					return false // the handed-over walk enters dir itself
+				default:
+					return true
+				}
+			},
+			Entries: func(dir string, entries []fs.DirEntry) bool {
+				names := make(map[string]bool, len(entries))
+				for _, e := range entries {
+					names[e.Name()] = true
+				}
 
-			// Marker check: establish the project contexts rooted at this directory.
-			for i := range tables {
-				t := &tables[i]
-				rooted := false
-				for _, m := range t.Markers {
-					if names[m] {
-						rooted = true
-						break
-					}
+				// Prune before establishing any context or descending, so nothing
+				// inside is ever a deletion target.
+				if isInstalledPackageTree(dir, names) {
+					return false
 				}
-				if !rooted {
-					continue
+				for i := range tables {
+					if p := tables[i].PruneRoot; p != nil && p(dir, names) {
+						return false
+					}
 				}
 
-				rules := t.Rules
-				if t.ExtraRules != nil {
-					if extra := t.ExtraRules(dir, names); len(extra) > 0 {
-						combined := make([]artifactRule, 0, len(t.Rules)+len(extra))
-						combined = append(combined, t.Rules...)
-						combined = append(combined, extra...)
-						rules = combined
+				// Marker check: establish the project contexts rooted at this directory.
+				for i := range tables {
+					t := &tables[i]
+					rooted := false
+					for _, m := range t.Markers {
+						if names[m] {
+							rooted = true
+							break
+						}
+					}
+					if !rooted {
+						continue
+					}
+
+					rules := t.Rules
+					if t.ExtraRules != nil {
+						if extra := t.ExtraRules(dir, names); len(extra) > 0 {
+							combined := make([]artifactRule, 0, len(t.Rules)+len(extra))
+							combined = append(combined, t.Rules...)
+							combined = append(combined, extra...)
+							rules = combined
+						}
 					}
+					stack = append(stack, projectContext{root: dir, tableIdx: i, rules: rules})
 				}
-				stack = append(stack, projectContext{root: dir, tableIdx: i, rules: rules})
-			}
-			return true
-		},
-		// The contexts dir pushed are the ones on top rooted at dir.
-		Leave: func(dir string) {
-			for len(stack) > 0 && stack[len(stack)-1].root == dir {
-				stack = stack[:len(stack)-1]
-			}
-		},
+				return true
+			},
+			// The contexts dir pushed are the ones on top rooted at dir.
+			Leave: func(dir string) {
+				for len(stack) > 0 && stack[len(stack)-1].root == dir {
+					stack = stack[:len(stack)-1]
+				}
+			},
+		}
+		_, _ = walker.Walk(ctx, top)
 	}
-	if _, err := walker.Walk(ctx, root); err != nil {
+	walk(root, nil)
+	wg.Wait()
+	if err := ctx.Err(); err != nil {
 		return nil, err
 	}
 
@@ -268,6 +297,11 @@ func runWalk(ctx context.Context, root string, tables []walkEcosystem) ([]model.
 	return results, nil
 }
 
+// walkTokens bounds the extra goroutines runWalk starts to walk subtrees in
+// parallel. It is separate from sizeTokens so the walk is not starved while
+// the stat scanners size their caches alongside it.
+var walkTokens = make(chan struct{}, sizeWorkerCap)
+
 // installedPackageDirNames hold only installed packages when the walk reaches
 // them without matching an artifact rule (a global lib/node_modules, a tool's
 // bundled dependencies, a Python environment's site-packages). Enter prunes

internal/scanner/walk_traversal_test.go

@@ -163,3 +163,46 @@ func TestWalkScan_HiddenNamedRootIsScanned(t *testing.T) {
 		t.Errorf("expected only %s, got %+v", cache, results)
 	}
 }
+
+// TestWalkScan_SplitMatchesSerial scans a tree of nested projects twice: with
+// every walk slot taken (one goroutine walks it all) and with the slots free
+// (subtrees are walked in parallel, each with a copy of its ancestors'
+// project contexts). Both must report the same artifacts, attributed to the
+// same project roots.
+func TestWalkScan_SplitMatchesSerial(t *testing.T) {
+	root := t.TempDir()
+	for _, p := range []string{
+		"mono/package.json", "mono/node_modules/x",
+		"mono/packages/a/package.json", "mono/packages/a/node_modules/x", "mono/packages/a/dist/x",
+		"mono/packages/b/src/__pycache__/x", "mono/packages/b/pyproject.toml",
+		"mono/packages/b/.venv/pyvenv.cfg", "mono/packages/b/sub/lib/__pycache__/x",
+		"rs/Cargo.toml", "rs/target/x", "rs/crates/c/Cargo.toml", "rs/crates/c/target/x",
+		"plain/deep/er/__pycache__/x", ".hidden/package.json", ".hidden/node_modules/x",
+	} {
+		touch(t, filepath.Join(root, filepath.FromSlash(p)))
+	}
+	scan := func() []string {
+		results, err := runWalk(context.Background(), root, walkEcosystemTable)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var out []string
+		for _, r := range results {
+			out = append(out, r.Path+" "+string(r.Ecosystem)+" "+r.ProjectRoot)
+		}
+		return out
+	}
+
+	for range cap(walkTokens) {
+		walkTokens <- struct{}{}
+	}
+	serial := scan()
+	for range cap(walkTokens) {
+		<-walkTokens
+	}
+	split := scan()
+
+	if len(serial) == 0 || !slices.Equal(serial, split) {
+		t.Errorf("split:\n%v\nserial:\n%v", split, serial)
+	}
+}