diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ea3d7b..0456cd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Preserve existing file-cache snapshots when timestamps and names collide, including concurrent captures; report the collision instead of overwriting data. + - Let `crawlctl logs -- JOB` display logs for jobs whose names look like command-line flags. - Keep CLI-installed schedules tied to the selected config and log directory when `--config` is relative. diff --git a/cache/cache.go b/cache/cache.go index 62c148f..98afd73 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -23,6 +23,9 @@ type Snapshot struct { CreatedAt time.Time `json:"created_at"` } +// SnapshotFile copies a file without replacing an existing capture. The cache +// filesystem must support hard links for atomic publication. A name collision +// returns an error matching os.ErrExist. func SnapshotFile(opts SnapshotOptions) (Snapshot, error) { if opts.SourcePath == "" { return Snapshot{}, errors.New("source path is required") @@ -66,12 +69,7 @@ func SnapshotFile(opts SnapshotOptions) (Snapshot, error) { return Snapshot{}, fmt.Errorf("create snapshot: %w", err) } tmpPath := tmp.Name() - cleanup := true - defer func() { - if cleanup { - _ = os.Remove(tmpPath) - } - }() + defer os.Remove(tmpPath) if err := tmp.Chmod(0o600); err != nil { _ = tmp.Close() return Snapshot{}, fmt.Errorf("chmod snapshot: %w", err) @@ -87,9 +85,9 @@ func SnapshotFile(opts SnapshotOptions) (Snapshot, error) { if err := tmp.Close(); err != nil { return Snapshot{}, fmt.Errorf("close snapshot: %w", err) } - if err := os.Rename(tmpPath, target); err != nil { + // Link publishes the complete file only if this name is still available. + if err := os.Link(tmpPath, target); err != nil { return Snapshot{}, fmt.Errorf("commit snapshot: %w", err) } - cleanup = false return Snapshot{SourcePath: opts.SourcePath, Path: target, SizeBytes: copied, CreatedAt: createdAt}, nil } diff --git a/cache/cache_test.go b/cache/cache_test.go index d5b3883..db20de4 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -1,6 +1,7 @@ package cache import ( + "errors" "os" "path/filepath" "runtime" @@ -103,3 +104,80 @@ func TestSnapshotSQLiteValidation(t *testing.T) { } } } + +func TestSnapshotFilePreservesExistingCapture(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source.db") + opts := SnapshotOptions{SourcePath: source, CacheDir: filepath.Join(dir, "cache"), Now: func() time.Time { + return time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + }} + if err := os.WriteFile(source, []byte("first capture"), 0o600); err != nil { + t.Fatal(err) + } + first, err := SnapshotFile(opts) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("later capture"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := SnapshotFile(opts); !errors.Is(err, os.ErrExist) { + t.Errorf("colliding capture error = %v, want already exists", err) + } + data, err := os.ReadFile(first.Path) + if err != nil || string(data) != "first capture" { + t.Fatalf("first capture changed: %q, %v", data, err) + } + entries, err := os.ReadDir(opts.CacheDir) + if err != nil || len(entries) != 1 { + t.Fatalf("collision left extra cache files: %v, %v", entries, err) + } +} + +func TestSnapshotFileConcurrentCollision(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source.db") + if err := os.WriteFile(source, []byte("capture"), 0o600); err != nil { + t.Fatal(err) + } + opts := SnapshotOptions{SourcePath: source, CacheDir: filepath.Join(dir, "cache"), Now: func() time.Time { + return time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + }} + type result struct { + snapshot Snapshot + err error + } + const writers = 4 + results := make(chan result, writers) + start := make(chan struct{}) + for range writers { + go func() { + <-start + snapshot, err := SnapshotFile(opts) + results <- result{snapshot, err} + }() + } + close(start) + winners := 0 + var path string + for range writers { + r := <-results + if r.err == nil { + winners++ + path = r.snapshot.Path + } else if !errors.Is(r.err, os.ErrExist) { + t.Errorf("unexpected capture error: %v", r.err) + } + } + if winners != 1 { + t.Fatalf("successful publishers = %d, want 1", winners) + } + data, err := os.ReadFile(path) + if err != nil || string(data) != "capture" { + t.Fatalf("published capture = %q, %v", data, err) + } + entries, err := os.ReadDir(opts.CacheDir) + if err != nil || len(entries) != 1 { + t.Fatalf("publishers left extra cache files: %v, %v", entries, err) + } +} diff --git a/docs/packages.md b/docs/packages.md index 232c008..c68a264 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -17,7 +17,9 @@ their caller-provided parameters; `:memory:` keeps its existing behavior. `cache.SnapshotFile` accepts a single filename in `Name`, preserving its literal text, including whitespace. An empty name uses the source basename. Paths and parent-directory components are rejected before copying so captures stay inside -`CacheDir`. +`CacheDir`. Captures never replace an existing file. A timestamp/name collision +returns an error matching `os.ErrExist`, preserving the earlier capture. Atomic +publication requires a cache filesystem that supports hard links. `cache.SnapshotSQLite` preserves literal `SourcePath` and `DestinationDir` paths, including whitespace. On Windows, path components ending in spaces or dots are