diff --git a/cmd/mutate/main.go b/cmd/mutate/main.go index 1d35deb..c922b3f 100644 --- a/cmd/mutate/main.go +++ b/cmd/mutate/main.go @@ -31,6 +31,9 @@ func main() { // a loop append for ever has the machine long before it runs out. mem := flag.Float64("mem", 0, "how many GiB one test process and what it starts may hold, 0 for its share of the memory free when the run starts, negative for no bound (Linux)") staleOnly := flag.Bool("stale-exempts", false, "report every mutate-exempt reason in the tree that covers no edit this tool makes, and fail on one") + // Off unless named: a run from scratch is the one that can be trusted with a file a test + // reads from outside its package (mutate.Cache says why). + cacheDir := flag.String("cache", "", "a directory keeping the edits refused, so a run over the same code asks only about the rest; empty for none") flag.Parse() if *staleOnly { @@ -83,6 +86,12 @@ func main() { os.Exit(2) } + cache, chosen, known, err := fromCache(*cacheDir, chosen, importers) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(2) + } + mutate.Share(*jobs, int64(*mem*(1<<30))) started := time.Now() t, err := mutate.RunAll(chosen, importers, *timeout, *jobs, os.Stdout) @@ -90,6 +99,13 @@ func main() { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(2) } + if cache != nil { + if err := cache.Keep(t.Refused); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(2) + } + t.Killed += len(known) + } reportCosts(time.Since(started)) survived := t.Survived fmt.Printf("mutate: %d refused, %d survived, %d did not build, %d in packages with no tests\n", @@ -119,6 +135,24 @@ func main() { os.Exit(1) } +// fromCache splits chosen into the edits to ask about and those the cache in dir says a run over +// the same code refused; with no dir, there is no cache and every edit is asked. +func fromCache(dir string, chosen []mutate.Mutation, importers map[string][]string) (*mutate.Cache, []mutate.Mutation, []mutate.Mutation, error) { + if dir == "" { + return nil, chosen, nil, nil + } + cache, err := mutate.OpenCache(dir) + if err != nil { + return nil, nil, nil, err + } + fresh, known, err := cache.Known(chosen, importers) + if err != nil { + return nil, nil, nil, err + } + fmt.Printf("mutate: %d refused by a run over the same code already (%s); asking about %d\n", len(known), dir, len(fresh)) + return cache, fresh, known, nil +} + // reportCosts says where the run's time went, by what each process was for: the processes of // one purpose run side by side, so their wall is summed and can pass the run's own. func reportCosts(took time.Duration) { diff --git a/internal/mutate/cache.go b/internal/mutate/cache.go new file mode 100644 index 0000000..0efbbf7 --- /dev/null +++ b/internal/mutate/cache.go @@ -0,0 +1,298 @@ +package mutate + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "runtime/debug" + "sort" + "strings" + "sync" +) + +// Cache keeps the edits a run's tests refused, so the next run over the same code does not ask +// again: a gate run once more after a fix to one package re-broke every package of the change, +// and ten of its sixteen minutes went to edits whose answer could not have moved. +// +// An edit is kept under a key of everything its answer was read from: the edit itself; every +// file in the directory of each package its tests or its importers' tests are built from in this +// module, and those directories' testdata; the module's go.mod and go.sum, which pin the rest; +// the Go toolchain; and this tool's own version. A file a test reads from outside those - a path +// climbing out of its package - is not in it, which is why only a gate re-run uses a cache and a +// run from scratch (ci.yml's, on main) does not. +// +// Only refusals are kept. A survivor fails the run and is the next one's question; an edit that +// did not build costs a failed build, which is what asking the cache would save. +type Cache struct { + dir string + + mu sync.Mutex + modules map[string]*moduleGraph // by the module's root + digests map[string]string // a package directory's files, by directory + keys map[int]string // by Mutation.ID, from Known for Keep +} + +// OpenCache is the cache in dir, made if it is not there. +func OpenCache(dir string) (*Cache, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("mutate: the cache: %w", err) + } + return &Cache{dir: dir, modules: map[string]*moduleGraph{}, digests: map[string]string{}, keys: map[int]string{}}, nil +} + +// Known splits chosen into the edits to ask the tests about and those a run over the same code +// refused already. importers is Importers'. +func (c *Cache) Known(chosen []Mutation, importers map[string][]string) (fresh, refused []Mutation, err error) { + for _, m := range chosen { + key, err := c.key(m, importers[filepath.ToSlash(filepath.Dir(m.File))]) + if err != nil { + return nil, nil, err + } + c.mu.Lock() + c.keys[m.ID] = key + c.mu.Unlock() + if _, err := os.Stat(c.path(key)); err == nil { + refused = append(refused, m) + continue + } + fresh = append(fresh, m) + } + return fresh, refused, nil +} + +// Keep records each of refused, which Known was asked about, as refused. +func (c *Cache) Keep(refused []Mutation) error { + for _, m := range refused { + c.mu.Lock() + key, ok := c.keys[m.ID] + c.mu.Unlock() + if !ok { + return fmt.Errorf("mutate: %s was not looked up in the cache", m) + } + path := c.path(key) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("mutate: the cache: %w", err) + } + if err := os.WriteFile(path, []byte(m.String()+"\n"), 0o600); err != nil { + return fmt.Errorf("mutate: the cache: %w", err) + } + } + return nil +} + +func (c *Cache) path(key string) string { + return filepath.Join(c.dir, key[:2], key) // mutate-exempt: the store's layout, written and read by this function alone +} + +// key is m's, whose package's importers are importers (directories, as Importers names them). +func (c *Cache) key(m Mutation, importers []string) (string, error) { + dir, err := filepath.Abs(filepath.Dir(m.File)) + if err != nil { + return "", err + } + g, err := c.module(dir) + if err != nil { + return "", err + } + roots := []string{dir} + for _, imp := range importers { + abs, err := filepath.Abs(imp) + if err != nil { // mutate-exempt: Abs fails only when the working directory is gone, and dir's Abs above would first + return "", err + } + roots = append(roots, abs) + } + h := sha256.New() + file, err := filepath.Rel(g.root, filepath.Join(dir, filepath.Base(m.File))) + if err != nil { + return "", err + } + // A hash's Write never fails, so neither do these. + _, _ = fmt.Fprintf(h, "edit %s %d %d %q\n", filepath.ToSlash(file), m.Start, m.End, m.With) + _, _ = fmt.Fprintf(h, "tool %s\nmodule %s\n", toolVersion(), g.digest) + for _, d := range g.closure(roots) { + digest, err := c.dirDigest(d) + if err != nil { + return "", err + } + rel, err := filepath.Rel(g.root, d) + if err != nil { + return "", err + } + _, _ = fmt.Fprintf(h, "dir %s %s\n", filepath.ToSlash(rel), digest) + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// toolVersion is this binary's module version and sum, which a `go tool` build carries: a new +// version may break an edit differently or read an answer differently. +func toolVersion() string { + bi, ok := debug.ReadBuildInfo() + if !ok { + return "unknown" + } + return bi.Main.Path + "@" + bi.Main.Version + " " + bi.Main.Sum // mutate-exempt: a test binary's main module is the test's, whose version is the same from run to run +} + +// dirDigest is the files directly in dir, and everything under its testdata, by name and content. +func (c *Cache) dirDigest(dir string) (string, error) { + c.mu.Lock() + digest, ok := c.digests[dir] + c.mu.Unlock() + if ok { + return digest, nil + } + h := sha256.New() + entries, err := os.ReadDir(dir) + if err != nil { + return "", fmt.Errorf("mutate: the cache: %w", err) + } + for _, e := range entries { + if e.Type().IsRegular() { + if err := hashFile(h, dir, filepath.Join(dir, e.Name())); err != nil { + return "", err + } + } + } + testdata := filepath.Join(dir, "testdata") + err = filepath.WalkDir(testdata, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.Type().IsRegular() { + return hashFile(h, dir, path) + } + return nil + }) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("mutate: the cache: %w", err) + } + digest = hex.EncodeToString(h.Sum(nil)) + c.mu.Lock() + c.digests[dir] = digest + c.mu.Unlock() + return digest, nil +} + +func hashFile(h interface{ Write([]byte) (int, error) }, dir, path string) error { + body, err := os.ReadFile(path) //nolint:gosec // a file of the module being tested + if err != nil { + return fmt.Errorf("mutate: the cache: %w", err) + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + sum := sha256.Sum256(body) + _, err = fmt.Fprintf(h, "%s %x\n", filepath.ToSlash(rel), sum) + return err +} + +// moduleGraph is a module's packages, each by its directory, with the directories of the +// packages of this module it imports and those its tests import. +type moduleGraph struct { + root, digest string + imports, testDeps map[string][]string +} + +// module is the graph of the module dir is in, listed once. +func (c *Cache) module(dir string) (*moduleGraph, error) { + c.mu.Lock() + g, ok := c.modules[dir] + c.mu.Unlock() + if ok { + return g, nil + } + root, err := moduleRoot(dir) + if err != nil { + return nil, fmt.Errorf("mutate: the cache: %w", err) + } + c.mu.Lock() + g, ok = c.modules[root] + c.mu.Unlock() + if !ok { + if g, err = listModule(root); err != nil { + return nil, err + } + } + c.mu.Lock() + c.modules[root], c.modules[dir] = g, g + c.mu.Unlock() + return g, nil +} + +func listModule(root string) (*moduleGraph, error) { + // -e for the reason Importers has it: a package that does not build still names its imports. + list := exec.Command("go", "list", "-e", "-f", + "{{.ImportPath}}\t{{.Dir}}\t{{join .Imports \" \"}}\t{{join .TestImports \" \"}} {{join .XTestImports \" \"}}", "./...") + list.Dir = root + out, err := output("listing", list) + if err != nil { + return nil, fmt.Errorf("mutate: the cache: listing %s: %w", root, err) + } + dirOf := map[string]string{} + type row struct{ dir, imports, tests string } + var rows []row + for _, line := range strings.Split(string(out), "\n") { + fields := strings.Split(line, "\t") + if len(fields) != 4 { + continue + } + dirOf[fields[0]] = fields[1] + rows = append(rows, row{fields[1], fields[2], fields[3]}) + } + local := func(paths string) []string { + var dirs []string + for _, p := range strings.Fields(paths) { + if d, ok := dirOf[p]; ok { + dirs = append(dirs, d) + } + } + return dirs + } + g := &moduleGraph{root: root, imports: map[string][]string{}, testDeps: map[string][]string{}} + for _, r := range rows { + g.imports[r.dir] = local(r.imports) + g.testDeps[r.dir] = local(r.tests) + } + h := sha256.New() + for _, name := range []string{"go.mod", "go.sum"} { + if err := hashFile(h, root, filepath.Join(root, name)); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + } + g.digest = hex.EncodeToString(h.Sum(nil)) + return g, nil +} + +// closure is roots and every package of the module their tests are built from, sorted. +func (g *moduleGraph) closure(roots []string) []string { + seen := map[string]bool{} + var walk func(string) + walk = func(d string) { + if seen[d] { + return + } + seen[d] = true + for _, i := range g.imports[d] { + walk(i) + } + } + for _, r := range roots { + walk(r) + for _, t := range g.testDeps[r] { + walk(t) + } + } + out := make([]string, 0, len(seen)) + for d := range seen { + out = append(out, d) + } + sort.Strings(out) + return out +} diff --git a/internal/mutate/cache_test.go b/internal/mutate/cache_test.go new file mode 100644 index 0000000..7a99bef --- /dev/null +++ b/internal/mutate/cache_test.go @@ -0,0 +1,121 @@ +package mutate_test + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/spin-stack/go-tools/internal/mutate" +) + +// An edit the tests refused is not asked again of the same code, and is asked again once +// anything its tests are built from has changed: a test that stopped holding it is a survivor, +// not a cached refusal. A package its tests do not reach changes nothing. +func TestARefusalIsKeptForTheCodeItWasAskedOf(t *testing.T) { + dir := refuses(t) + lookup := cachedRun(t, doubling(t, dir)) + + fresh, known := lookup() + require.Len(t, fresh, 1, "an edit never asked was taken from the cache") + assert.Empty(t, known) + fresh, known = lookup() + assert.Empty(t, fresh, "an edit refused of this code was asked again") + assert.Len(t, known, 1) + + other := filepath.Join(dir, "other") + require.NoError(t, os.MkdirAll(other, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(other, "other.go"), []byte("package other\n\nfunc Other() int { return 1 }\n"), 0o600)) + fresh, _ = lookup() + assert.Empty(t, fresh, "a package the tests do not reach made a refusal stale") + + // Its testdata first, which leaves the test refusing the edit: kept again, under the new key. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "testdata"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "testdata", "in.txt"), []byte("read by a test\n"), 0o600)) + fresh, known = lookup() + assert.Len(t, fresh, 1, "a refusal was kept after its package's testdata changed") + assert.Empty(t, known) + + // Then the test, weakened until it lets the edit through. + test := filepath.Join(dir, "code_test.go") + body, err := os.ReadFile(test) + require.NoError(t, err) + require.NoError(t, os.WriteFile(test, []byte(strings.Replace(string(body), "Double(2) != 4", "Double(2) == 5", 1)), 0o600)) + fresh, known = lookup() + assert.Nil(t, fresh, "the weakened test was answered from the cache rather than found to let the edit through") + assert.Nil(t, known) +} + +// A refusal is of the packages the tests are built from too, however far down: one the tests +// import, which imports another that changes, can change what the tests answer. +func TestARefusalIsKeptForThePackagesItsTestsAreBuiltFrom(t *testing.T) { + // The package one directory into its module, as most are: the module's own files are then in + // no package's directory, and only the module's pins can say they changed. + dir := refuses(t) + root := filepath.Dir(dir) // the test's own temporary directory, which dir is made in + require.NoError(t, os.Remove(filepath.Join(dir, "go.mod"))) + for name, body := range map[string]string{ + "go.mod": "module example.com/m\n\ngo 1.24\n", + "dep/dep.go": "package dep\n\nimport \"example.com/m/dep/deeper\"\n\nfunc Two() int { return deeper.Two() }\n", + "dep/deeper/two.go": "package deeper\n\nfunc Two() int { return 2 }\n", + } { + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(root, name)), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, name), []byte(body), 0o600)) + } + test := filepath.Join(dir, "code_test.go") + body, err := os.ReadFile(test) + require.NoError(t, err) + body = []byte(strings.Replace(strings.Replace(string(body), "\t\"os\"\n", "\t\"os\"\n\n\t\"example.com/m/dep\"\n", 1), + "Double(2) != 4", "Double(dep.Two()) != 4", 1)) + require.NoError(t, os.WriteFile(test, body, 0o600)) + lookup := cachedRun(t, doubling(t, dir)) + + fresh, _ := lookup() + require.Len(t, fresh, 1) + fresh, known := lookup() + require.Empty(t, fresh, "an edit refused of this code was asked again") + require.Len(t, known, 1) + + require.NoError(t, os.WriteFile(filepath.Join(root, "dep", "deeper", "two.go"), []byte("package deeper\n\nfunc Two() int { return 1 + 1 }\n"), 0o600)) + fresh, known = lookup() + assert.Len(t, fresh, 1, "a refusal was kept after a package the tests are built from changed") + assert.Empty(t, known) + + // go.mod pins every module outside this one the tests are built from. + mod := filepath.Join(root, "go.mod") + pinned, err := os.ReadFile(mod) + require.NoError(t, err) + require.NoError(t, os.WriteFile(mod, append(pinned, "\ntoolchain go1.24.0\n"...), 0o600)) + fresh, known = lookup() + assert.Len(t, fresh, 1, "a refusal was kept after the module's pins changed") + assert.Empty(t, known) +} + +// cachedRun is a run over m with a cache that outlives it, as a gate run again keeps one: each +// call opens the cache anew, so nothing is remembered but what is on disk, asks the tests about m +// when the cache does not answer, and says which it was - nil, nil when the tests let m through. +func cachedRun(t *testing.T, m mutate.Mutation) func() (fresh, known []mutate.Mutation) { + t.Helper() + store := t.TempDir() + m.ID = 1 + return func() (fresh, known []mutate.Mutation) { + t.Helper() + c, err := mutate.OpenCache(store) + require.NoError(t, err) + fresh, known, err = c.Known([]mutate.Mutation{m}, nil) + require.NoError(t, err) + if len(fresh) > 0 { + tally, err := mutate.RunAll(fresh, nil, "2m", 1, io.Discard) + require.NoError(t, err) + require.NoError(t, c.Keep(tally.Refused)) + if len(tally.Survived) > 0 { + return nil, nil + } + } + return fresh, known + } +} diff --git a/internal/mutate/runner.go b/internal/mutate/runner.go index 0a94c32..e563f2c 100644 --- a/internal/mutate/runner.go +++ b/internal/mutate/runner.go @@ -14,7 +14,7 @@ import ( // Tally is what the tests did about every mutation. type Tally struct { - Survived []Mutation + Survived, Refused []Mutation Killed, Unbuildable, Untested int UntestedPackages map[string]bool } @@ -78,10 +78,12 @@ func RunAll(chosen []Mutation, importers map[string][]string, timeout string, jo switch r.outcome { case Killed: t.Killed++ + t.Refused = append(t.Refused, m) _, _ = fmt.Fprintf(w, " refused %s\n", m) case Exceeded: // Named with its bound: the edit is refused, and the loop it made is worth knowing of. t.Killed++ + t.Refused = append(t.Refused, m) _, _ = fmt.Fprintf(w, " refused %s (its tests held over %s, and were stopped)\n", m, gib(memLimit)) case Unbuildable: t.Unbuildable++ diff --git a/internal/mutate/runner_internal_test.go b/internal/mutate/runner_internal_test.go index 5171676..8a1eb5d 100644 --- a/internal/mutate/runner_internal_test.go +++ b/internal/mutate/runner_internal_test.go @@ -91,6 +91,9 @@ func TestAnEditThatAllocatesForEverIsStoppedAndRefused(t *testing.T) { if tally.Killed != 1 || !strings.Contains(said.String(), "refused "+plan[0].String()+" (its tests held over 0.2 GiB, and were stopped)") { t.Errorf("the edit in the package's binary was not refused at its bound: %s", said.String()) } + if len(tally.Refused) != 1 { + t.Errorf("an edit refused at its bound is not among the refusals a cache keeps: %v", tally.Refused) + } overlay, err := Overlay(plan[0], t.TempDir()) if err != nil {