From 6f629797213dab7ece52557c9a3459eb1e58f72a Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Fri, 16 Oct 2026 12:08:15 -0300 Subject: [PATCH] mutate: -cache keeps the edits refused, for a run over the same code A pull request's gate run again after a fix re-broke every package of the change: spin#493's took eleven runs, each spending its ten to sixteen minutes of mutate on edits whose answer could not have moved. An edit refused is kept under a key of the edit, of every file in the directories of the packages its tests and its importers' tests are built from, their testdata, the module's pins, the toolchain and this tool's version; the next run asks only about the rest. Off unless named: a file a test reads from outside its package is not in the key, so a run from scratch stays the one to trust. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/mutate/main.go | 34 +++ internal/mutate/cache.go | 298 ++++++++++++++++++++++++ internal/mutate/cache_test.go | 121 ++++++++++ internal/mutate/runner.go | 4 +- internal/mutate/runner_internal_test.go | 3 + 5 files changed, 459 insertions(+), 1 deletion(-) create mode 100644 internal/mutate/cache.go create mode 100644 internal/mutate/cache_test.go diff --git a/cmd/mutate/main.go b/cmd/mutate/main.go index 1d35deb..c922b3f 100644 --- a/cmd/mutate/main.go +++ b/cmd/mutate/main.go @@ -31,6 +31,9 @@ func main() { // a loop append for ever has the machine long before it runs out. mem := flag.Float64("mem", 0, "how many GiB one test process and what it starts may hold, 0 for its share of the memory free when the run starts, negative for no bound (Linux)") staleOnly := flag.Bool("stale-exempts", false, "report every mutate-exempt reason in the tree that covers no edit this tool makes, and fail on one") + // Off unless named: a run from scratch is the one that can be trusted with a file a test + // reads from outside its package (mutate.Cache says why). + cacheDir := flag.String("cache", "", "a directory keeping the edits refused, so a run over the same code asks only about the rest; empty for none") flag.Parse() if *staleOnly { @@ -83,6 +86,12 @@ func main() { os.Exit(2) } + cache, chosen, known, err := fromCache(*cacheDir, chosen, importers) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(2) + } + mutate.Share(*jobs, int64(*mem*(1<<30))) started := time.Now() t, err := mutate.RunAll(chosen, importers, *timeout, *jobs, os.Stdout) @@ -90,6 +99,13 @@ func main() { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(2) } + if cache != nil { + if err := cache.Keep(t.Refused); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(2) + } + t.Killed += len(known) + } reportCosts(time.Since(started)) survived := t.Survived fmt.Printf("mutate: %d refused, %d survived, %d did not build, %d in packages with no tests\n", @@ -119,6 +135,24 @@ func main() { os.Exit(1) } +// fromCache splits chosen into the edits to ask about and those the cache in dir says a run over +// the same code refused; with no dir, there is no cache and every edit is asked. +func fromCache(dir string, chosen []mutate.Mutation, importers map[string][]string) (*mutate.Cache, []mutate.Mutation, []mutate.Mutation, error) { + if dir == "" { + return nil, chosen, nil, nil + } + cache, err := mutate.OpenCache(dir) + if err != nil { + return nil, nil, nil, err + } + fresh, known, err := cache.Known(chosen, importers) + if err != nil { + return nil, nil, nil, err + } + fmt.Printf("mutate: %d refused by a run over the same code already (%s); asking about %d\n", len(known), dir, len(fresh)) + return cache, fresh, known, nil +} + // reportCosts says where the run's time went, by what each process was for: the processes of // one purpose run side by side, so their wall is summed and can pass the run's own. func reportCosts(took time.Duration) { diff --git a/internal/mutate/cache.go b/internal/mutate/cache.go new file mode 100644 index 0000000..0efbbf7 --- /dev/null +++ b/internal/mutate/cache.go @@ -0,0 +1,298 @@ +package mutate + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "runtime/debug" + "sort" + "strings" + "sync" +) + +// Cache keeps the edits a run's tests refused, so the next run over the same code does not ask +// again: a gate run once more after a fix to one package re-broke every package of the change, +// and ten of its sixteen minutes went to edits whose answer could not have moved. +// +// An edit is kept under a key of everything its answer was read from: the edit itself; every +// file in the directory of each package its tests or its importers' tests are built from in this +// module, and those directories' testdata; the module's go.mod and go.sum, which pin the rest; +// the Go toolchain; and this tool's own version. A file a test reads from outside those - a path +// climbing out of its package - is not in it, which is why only a gate re-run uses a cache and a +// run from scratch (ci.yml's, on main) does not. +// +// Only refusals are kept. A survivor fails the run and is the next one's question; an edit that +// did not build costs a failed build, which is what asking the cache would save. +type Cache struct { + dir string + + mu sync.Mutex + modules map[string]*moduleGraph // by the module's root + digests map[string]string // a package directory's files, by directory + keys map[int]string // by Mutation.ID, from Known for Keep +} + +// OpenCache is the cache in dir, made if it is not there. +func OpenCache(dir string) (*Cache, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("mutate: the cache: %w", err) + } + return &Cache{dir: dir, modules: map[string]*moduleGraph{}, digests: map[string]string{}, keys: map[int]string{}}, nil +} + +// Known splits chosen into the edits to ask the tests about and those a run over the same code +// refused already. importers is Importers'. +func (c *Cache) Known(chosen []Mutation, importers map[string][]string) (fresh, refused []Mutation, err error) { + for _, m := range chosen { + key, err := c.key(m, importers[filepath.ToSlash(filepath.Dir(m.File))]) + if err != nil { + return nil, nil, err + } + c.mu.Lock() + c.keys[m.ID] = key + c.mu.Unlock() + if _, err := os.Stat(c.path(key)); err == nil { + refused = append(refused, m) + continue + } + fresh = append(fresh, m) + } + return fresh, refused, nil +} + +// Keep records each of refused, which Known was asked about, as refused. +func (c *Cache) Keep(refused []Mutation) error { + for _, m := range refused { + c.mu.Lock() + key, ok := c.keys[m.ID] + c.mu.Unlock() + if !ok { + return fmt.Errorf("mutate: %s was not looked up in the cache", m) + } + path := c.path(key) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("mutate: the cache: %w", err) + } + if err := os.WriteFile(path, []byte(m.String()+"\n"), 0o600); err != nil { + return fmt.Errorf("mutate: the cache: %w", err) + } + } + return nil +} + +func (c *Cache) path(key string) string { + return filepath.Join(c.dir, key[:2], key) // mutate-exempt: the store's layout, written and read by this function alone +} + +// key is m's, whose package's importers are importers (directories, as Importers names them). +func (c *Cache) key(m Mutation, importers []string) (string, error) { + dir, err := filepath.Abs(filepath.Dir(m.File)) + if err != nil { + return "", err + } + g, err := c.module(dir) + if err != nil { + return "", err + } + roots := []string{dir} + for _, imp := range importers { + abs, err := filepath.Abs(imp) + if err != nil { // mutate-exempt: Abs fails only when the working directory is gone, and dir's Abs above would first + return "", err + } + roots = append(roots, abs) + } + h := sha256.New() + file, err := filepath.Rel(g.root, filepath.Join(dir, filepath.Base(m.File))) + if err != nil { + return "", err + } + // A hash's Write never fails, so neither do these. + _, _ = fmt.Fprintf(h, "edit %s %d %d %q\n", filepath.ToSlash(file), m.Start, m.End, m.With) + _, _ = fmt.Fprintf(h, "tool %s\nmodule %s\n", toolVersion(), g.digest) + for _, d := range g.closure(roots) { + digest, err := c.dirDigest(d) + if err != nil { + return "", err + } + rel, err := filepath.Rel(g.root, d) + if err != nil { + return "", err + } + _, _ = fmt.Fprintf(h, "dir %s %s\n", filepath.ToSlash(rel), digest) + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// toolVersion is this binary's module version and sum, which a `go tool` build carries: a new +// version may break an edit differently or read an answer differently. +func toolVersion() string { + bi, ok := debug.ReadBuildInfo() + if !ok { + return "unknown" + } + return bi.Main.Path + "@" + bi.Main.Version + " " + bi.Main.Sum // mutate-exempt: a test binary's main module is the test's, whose version is the same from run to run +} + +// dirDigest is the files directly in dir, and everything under its testdata, by name and content. +func (c *Cache) dirDigest(dir string) (string, error) { + c.mu.Lock() + digest, ok := c.digests[dir] + c.mu.Unlock() + if ok { + return digest, nil + } + h := sha256.New() + entries, err := os.ReadDir(dir) + if err != nil { + return "", fmt.Errorf("mutate: the cache: %w", err) + } + for _, e := range entries { + if e.Type().IsRegular() { + if err := hashFile(h, dir, filepath.Join(dir, e.Name())); err != nil { + return "", err + } + } + } + testdata := filepath.Join(dir, "testdata") + err = filepath.WalkDir(testdata, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.Type().IsRegular() { + return hashFile(h, dir, path) + } + return nil + }) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("mutate: the cache: %w", err) + } + digest = hex.EncodeToString(h.Sum(nil)) + c.mu.Lock() + c.digests[dir] = digest + c.mu.Unlock() + return digest, nil +} + +func hashFile(h interface{ Write([]byte) (int, error) }, dir, path string) error { + body, err := os.ReadFile(path) //nolint:gosec // a file of the module being tested + if err != nil { + return fmt.Errorf("mutate: the cache: %w", err) + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + sum := sha256.Sum256(body) + _, err = fmt.Fprintf(h, "%s %x\n", filepath.ToSlash(rel), sum) + return err +} + +// moduleGraph is a module's packages, each by its directory, with the directories of the +// packages of this module it imports and those its tests import. +type moduleGraph struct { + root, digest string + imports, testDeps map[string][]string +} + +// module is the graph of the module dir is in, listed once. +func (c *Cache) module(dir string) (*moduleGraph, error) { + c.mu.Lock() + g, ok := c.modules[dir] + c.mu.Unlock() + if ok { + return g, nil + } + root, err := moduleRoot(dir) + if err != nil { + return nil, fmt.Errorf("mutate: the cache: %w", err) + } + c.mu.Lock() + g, ok = c.modules[root] + c.mu.Unlock() + if !ok { + if g, err = listModule(root); err != nil { + return nil, err + } + } + c.mu.Lock() + c.modules[root], c.modules[dir] = g, g + c.mu.Unlock() + return g, nil +} + +func listModule(root string) (*moduleGraph, error) { + // -e for the reason Importers has it: a package that does not build still names its imports. + list := exec.Command("go", "list", "-e", "-f", + "{{.ImportPath}}\t{{.Dir}}\t{{join .Imports \" \"}}\t{{join .TestImports \" \"}} {{join .XTestImports \" \"}}", "./...") + list.Dir = root + out, err := output("listing", list) + if err != nil { + return nil, fmt.Errorf("mutate: the cache: listing %s: %w", root, err) + } + dirOf := map[string]string{} + type row struct{ dir, imports, tests string } + var rows []row + for _, line := range strings.Split(string(out), "\n") { + fields := strings.Split(line, "\t") + if len(fields) != 4 { + continue + } + dirOf[fields[0]] = fields[1] + rows = append(rows, row{fields[1], fields[2], fields[3]}) + } + local := func(paths string) []string { + var dirs []string + for _, p := range strings.Fields(paths) { + if d, ok := dirOf[p]; ok { + dirs = append(dirs, d) + } + } + return dirs + } + g := &moduleGraph{root: root, imports: map[string][]string{}, testDeps: map[string][]string{}} + for _, r := range rows { + g.imports[r.dir] = local(r.imports) + g.testDeps[r.dir] = local(r.tests) + } + h := sha256.New() + for _, name := range []string{"go.mod", "go.sum"} { + if err := hashFile(h, root, filepath.Join(root, name)); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + } + g.digest = hex.EncodeToString(h.Sum(nil)) + return g, nil +} + +// closure is roots and every package of the module their tests are built from, sorted. +func (g *moduleGraph) closure(roots []string) []string { + seen := map[string]bool{} + var walk func(string) + walk = func(d string) { + if seen[d] { + return + } + seen[d] = true + for _, i := range g.imports[d] { + walk(i) + } + } + for _, r := range roots { + walk(r) + for _, t := range g.testDeps[r] { + walk(t) + } + } + out := make([]string, 0, len(seen)) + for d := range seen { + out = append(out, d) + } + sort.Strings(out) + return out +} diff --git a/internal/mutate/cache_test.go b/internal/mutate/cache_test.go new file mode 100644 index 0000000..7a99bef --- /dev/null +++ b/internal/mutate/cache_test.go @@ -0,0 +1,121 @@ +package mutate_test + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/spin-stack/go-tools/internal/mutate" +) + +// An edit the tests refused is not asked again of the same code, and is asked again once +// anything its tests are built from has changed: a test that stopped holding it is a survivor, +// not a cached refusal. A package its tests do not reach changes nothing. +func TestARefusalIsKeptForTheCodeItWasAskedOf(t *testing.T) { + dir := refuses(t) + lookup := cachedRun(t, doubling(t, dir)) + + fresh, known := lookup() + require.Len(t, fresh, 1, "an edit never asked was taken from the cache") + assert.Empty(t, known) + fresh, known = lookup() + assert.Empty(t, fresh, "an edit refused of this code was asked again") + assert.Len(t, known, 1) + + other := filepath.Join(dir, "other") + require.NoError(t, os.MkdirAll(other, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(other, "other.go"), []byte("package other\n\nfunc Other() int { return 1 }\n"), 0o600)) + fresh, _ = lookup() + assert.Empty(t, fresh, "a package the tests do not reach made a refusal stale") + + // Its testdata first, which leaves the test refusing the edit: kept again, under the new key. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "testdata"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "testdata", "in.txt"), []byte("read by a test\n"), 0o600)) + fresh, known = lookup() + assert.Len(t, fresh, 1, "a refusal was kept after its package's testdata changed") + assert.Empty(t, known) + + // Then the test, weakened until it lets the edit through. + test := filepath.Join(dir, "code_test.go") + body, err := os.ReadFile(test) + require.NoError(t, err) + require.NoError(t, os.WriteFile(test, []byte(strings.Replace(string(body), "Double(2) != 4", "Double(2) == 5", 1)), 0o600)) + fresh, known = lookup() + assert.Nil(t, fresh, "the weakened test was answered from the cache rather than found to let the edit through") + assert.Nil(t, known) +} + +// A refusal is of the packages the tests are built from too, however far down: one the tests +// import, which imports another that changes, can change what the tests answer. +func TestARefusalIsKeptForThePackagesItsTestsAreBuiltFrom(t *testing.T) { + // The package one directory into its module, as most are: the module's own files are then in + // no package's directory, and only the module's pins can say they changed. + dir := refuses(t) + root := filepath.Dir(dir) // the test's own temporary directory, which dir is made in + require.NoError(t, os.Remove(filepath.Join(dir, "go.mod"))) + for name, body := range map[string]string{ + "go.mod": "module example.com/m\n\ngo 1.24\n", + "dep/dep.go": "package dep\n\nimport \"example.com/m/dep/deeper\"\n\nfunc Two() int { return deeper.Two() }\n", + "dep/deeper/two.go": "package deeper\n\nfunc Two() int { return 2 }\n", + } { + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(root, name)), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, name), []byte(body), 0o600)) + } + test := filepath.Join(dir, "code_test.go") + body, err := os.ReadFile(test) + require.NoError(t, err) + body = []byte(strings.Replace(strings.Replace(string(body), "\t\"os\"\n", "\t\"os\"\n\n\t\"example.com/m/dep\"\n", 1), + "Double(2) != 4", "Double(dep.Two()) != 4", 1)) + require.NoError(t, os.WriteFile(test, body, 0o600)) + lookup := cachedRun(t, doubling(t, dir)) + + fresh, _ := lookup() + require.Len(t, fresh, 1) + fresh, known := lookup() + require.Empty(t, fresh, "an edit refused of this code was asked again") + require.Len(t, known, 1) + + require.NoError(t, os.WriteFile(filepath.Join(root, "dep", "deeper", "two.go"), []byte("package deeper\n\nfunc Two() int { return 1 + 1 }\n"), 0o600)) + fresh, known = lookup() + assert.Len(t, fresh, 1, "a refusal was kept after a package the tests are built from changed") + assert.Empty(t, known) + + // go.mod pins every module outside this one the tests are built from. + mod := filepath.Join(root, "go.mod") + pinned, err := os.ReadFile(mod) + require.NoError(t, err) + require.NoError(t, os.WriteFile(mod, append(pinned, "\ntoolchain go1.24.0\n"...), 0o600)) + fresh, known = lookup() + assert.Len(t, fresh, 1, "a refusal was kept after the module's pins changed") + assert.Empty(t, known) +} + +// cachedRun is a run over m with a cache that outlives it, as a gate run again keeps one: each +// call opens the cache anew, so nothing is remembered but what is on disk, asks the tests about m +// when the cache does not answer, and says which it was - nil, nil when the tests let m through. +func cachedRun(t *testing.T, m mutate.Mutation) func() (fresh, known []mutate.Mutation) { + t.Helper() + store := t.TempDir() + m.ID = 1 + return func() (fresh, known []mutate.Mutation) { + t.Helper() + c, err := mutate.OpenCache(store) + require.NoError(t, err) + fresh, known, err = c.Known([]mutate.Mutation{m}, nil) + require.NoError(t, err) + if len(fresh) > 0 { + tally, err := mutate.RunAll(fresh, nil, "2m", 1, io.Discard) + require.NoError(t, err) + require.NoError(t, c.Keep(tally.Refused)) + if len(tally.Survived) > 0 { + return nil, nil + } + } + return fresh, known + } +} diff --git a/internal/mutate/runner.go b/internal/mutate/runner.go index 0a94c32..e563f2c 100644 --- a/internal/mutate/runner.go +++ b/internal/mutate/runner.go @@ -14,7 +14,7 @@ import ( // Tally is what the tests did about every mutation. type Tally struct { - Survived []Mutation + Survived, Refused []Mutation Killed, Unbuildable, Untested int UntestedPackages map[string]bool } @@ -78,10 +78,12 @@ func RunAll(chosen []Mutation, importers map[string][]string, timeout string, jo switch r.outcome { case Killed: t.Killed++ + t.Refused = append(t.Refused, m) _, _ = fmt.Fprintf(w, " refused %s\n", m) case Exceeded: // Named with its bound: the edit is refused, and the loop it made is worth knowing of. t.Killed++ + t.Refused = append(t.Refused, m) _, _ = fmt.Fprintf(w, " refused %s (its tests held over %s, and were stopped)\n", m, gib(memLimit)) case Unbuildable: t.Unbuildable++ diff --git a/internal/mutate/runner_internal_test.go b/internal/mutate/runner_internal_test.go index 5171676..8a1eb5d 100644 --- a/internal/mutate/runner_internal_test.go +++ b/internal/mutate/runner_internal_test.go @@ -91,6 +91,9 @@ func TestAnEditThatAllocatesForEverIsStoppedAndRefused(t *testing.T) { if tally.Killed != 1 || !strings.Contains(said.String(), "refused "+plan[0].String()+" (its tests held over 0.2 GiB, and were stopped)") { t.Errorf("the edit in the package's binary was not refused at its bound: %s", said.String()) } + if len(tally.Refused) != 1 { + t.Errorf("an edit refused at its bound is not among the refusals a cache keeps: %v", tally.Refused) + } overlay, err := Overlay(plan[0], t.TempDir()) if err != nil {