Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

## Unreleased

- Preserve existing file-cache snapshots when timestamps and names collide, including concurrent captures; report the collision instead of overwriting data.

- Let `crawlctl logs -- JOB` display logs for jobs whose names look like command-line flags.

- Keep CLI-installed schedules tied to the selected config and log directory when `--config` is relative.
Expand Down
14 changes: 6 additions & 8 deletions cache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ type Snapshot struct {
CreatedAt time.Time `json:"created_at"`
}

// SnapshotFile copies a file without replacing an existing capture. The cache
// filesystem must support hard links for atomic publication. A name collision
// returns an error matching os.ErrExist.
func SnapshotFile(opts SnapshotOptions) (Snapshot, error) {
if opts.SourcePath == "" {
return Snapshot{}, errors.New("source path is required")
Expand Down Expand Up @@ -66,12 +69,7 @@ func SnapshotFile(opts SnapshotOptions) (Snapshot, error) {
return Snapshot{}, fmt.Errorf("create snapshot: %w", err)
}
tmpPath := tmp.Name()
cleanup := true
defer func() {
if cleanup {
_ = os.Remove(tmpPath)
}
}()
defer os.Remove(tmpPath)
if err := tmp.Chmod(0o600); err != nil {
_ = tmp.Close()
return Snapshot{}, fmt.Errorf("chmod snapshot: %w", err)
Expand All @@ -87,9 +85,9 @@ func SnapshotFile(opts SnapshotOptions) (Snapshot, error) {
if err := tmp.Close(); err != nil {
return Snapshot{}, fmt.Errorf("close snapshot: %w", err)
}
if err := os.Rename(tmpPath, target); err != nil {
// Link publishes the complete file only if this name is still available.
if err := os.Link(tmpPath, target); err != nil {
return Snapshot{}, fmt.Errorf("commit snapshot: %w", err)
}
cleanup = false
return Snapshot{SourcePath: opts.SourcePath, Path: target, SizeBytes: copied, CreatedAt: createdAt}, nil
}
78 changes: 78 additions & 0 deletions cache/cache_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cache

import (
"errors"
"os"
"path/filepath"
"runtime"
Expand Down Expand Up @@ -103,3 +104,80 @@ func TestSnapshotSQLiteValidation(t *testing.T) {
}
}
}

func TestSnapshotFilePreservesExistingCapture(t *testing.T) {
dir := t.TempDir()
source := filepath.Join(dir, "source.db")
opts := SnapshotOptions{SourcePath: source, CacheDir: filepath.Join(dir, "cache"), Now: func() time.Time {
return time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
}}
if err := os.WriteFile(source, []byte("first capture"), 0o600); err != nil {
t.Fatal(err)
}
first, err := SnapshotFile(opts)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(source, []byte("later capture"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := SnapshotFile(opts); !errors.Is(err, os.ErrExist) {
t.Errorf("colliding capture error = %v, want already exists", err)
}
data, err := os.ReadFile(first.Path)
if err != nil || string(data) != "first capture" {
t.Fatalf("first capture changed: %q, %v", data, err)
}
entries, err := os.ReadDir(opts.CacheDir)
if err != nil || len(entries) != 1 {
t.Fatalf("collision left extra cache files: %v, %v", entries, err)
}
}

func TestSnapshotFileConcurrentCollision(t *testing.T) {
dir := t.TempDir()
source := filepath.Join(dir, "source.db")
if err := os.WriteFile(source, []byte("capture"), 0o600); err != nil {
t.Fatal(err)
}
opts := SnapshotOptions{SourcePath: source, CacheDir: filepath.Join(dir, "cache"), Now: func() time.Time {
return time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
}}
type result struct {
snapshot Snapshot
err error
}
const writers = 4
results := make(chan result, writers)
start := make(chan struct{})
for range writers {
go func() {
<-start
snapshot, err := SnapshotFile(opts)
results <- result{snapshot, err}
}()
}
close(start)
winners := 0
var path string
for range writers {
r := <-results
if r.err == nil {
winners++
path = r.snapshot.Path
} else if !errors.Is(r.err, os.ErrExist) {
t.Errorf("unexpected capture error: %v", r.err)
}
}
if winners != 1 {
t.Fatalf("successful publishers = %d, want 1", winners)
}
data, err := os.ReadFile(path)
if err != nil || string(data) != "capture" {
t.Fatalf("published capture = %q, %v", data, err)
}
entries, err := os.ReadDir(opts.CacheDir)
if err != nil || len(entries) != 1 {
t.Fatalf("publishers left extra cache files: %v, %v", entries, err)
}
}
4 changes: 3 additions & 1 deletion docs/packages.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ their caller-provided parameters; `:memory:` keeps its existing behavior.
`cache.SnapshotFile` accepts a single filename in `Name`, preserving its literal
text, including whitespace. An empty name uses the source basename. Paths and
parent-directory components are rejected before copying so captures stay inside
`CacheDir`.
`CacheDir`. Captures never replace an existing file. A timestamp/name collision
returns an error matching `os.ErrExist`, preserving the earlier capture. Atomic
publication requires a cache filesystem that supports hard links.

`cache.SnapshotSQLite` preserves literal `SourcePath` and `DestinationDir` paths,
including whitespace. On Windows, path components ending in spaces or dots are
Expand Down
Loading