Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

## Unreleased

- Preserve literal table names during full snapshot imports so names with surrounding whitespace do not clear a different table or retain stale rows.

- Preserve existing file-cache snapshots when timestamps and names collide, including concurrent captures; report the collision instead of overwriting data.

- Let `crawlctl logs -- JOB` display logs for jobs whose names look like command-line flags.
Expand Down
4 changes: 4 additions & 0 deletions docs/packages.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,10 @@ must not commit or roll back. Legacy filter closures retain their own database
bindings. The driver's `ReadOnly` option is not an authorization boundary for
trusted callbacks.

Import uses literal table names for both deletion and insertion, including surrounding
whitespace. Full imports also pass those exact names to `DeleteTable` callbacks;
blank entries in `DeleteTables` remain ignored.

Snapshot rows remain v1 JSON objects; no new wire format is enabled. Import
planning compares unique column names independently of their order, so a
snapshot from an equivalent schema does not require replacement just because
Expand Down
4 changes: 2 additions & 2 deletions snapshot/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ func Import(ctx context.Context, opts ImportOptions) (Manifest, error) {
}
}
for i := len(deleteTables) - 1; i >= 0; i-- {
table := strings.TrimSpace(deleteTables[i])
if table == "" {
table := deleteTables[i]
if strings.TrimSpace(table) == "" {
continue
}
if opts.DeleteTable != nil {
Expand Down
56 changes: 56 additions & 0 deletions snapshot/snapshot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"database/sql"
"encoding/json"
"fmt"
"os"
"path/filepath"
"slices"
Expand Down Expand Up @@ -78,6 +79,61 @@ create table sync_state(source_name text, entity_type text, entity_id text, valu
}
}

func TestImportPreservesLiteralDeleteTableNames(t *testing.T) {
for _, table := range []string{" messages", "messages "} {
for _, explicit := range []bool{false, true} {
for _, custom := range []bool{false, true} {
t.Run(fmt.Sprintf("%q/explicit=%t/custom=%t", table, explicit, custom), func(t *testing.T) {
ctx := context.Background()
dst, err := store.Open(ctx, store.Options{
Path: filepath.Join(t.TempDir(), "dst.db"),
Schema: `create table messages(id text primary key);
create table ` + store.QuoteIdent(table) + `(id text primary key);`,
})
if err != nil {
t.Fatal(err)
}
defer dst.Close()
mustExec(t, dst.DB(), `insert into messages values ('unrelated')`)
mustExec(t, dst.DB(), `insert into `+store.QuoteIdent(table)+` values ('stale')`)
root := t.TempDir()
writeGzipJSONL(t, filepath.Join(root, "rows.jsonl.gz"), map[string]any{"id": "snapshot"})
if err := WriteManifest(root, Manifest{Version: 1, Tables: []TableManifest{{
Name: table, Columns: []string{"id"}, Files: []string{"rows.jsonl.gz"}, Rows: 1,
}}}); err != nil {
t.Fatal(err)
}
opts := ImportOptions{DB: dst.DB(), RootDir: root}
if explicit {
opts.DeleteTables = []string{"", " \t", table}
}
if custom {
opts.DeleteTable = func(ctx context.Context, tx *sql.Tx, name string) error {
if name != table {
t.Errorf("delete callback table = %q, want %q", name, table)
}
_, err := tx.ExecContext(ctx, "delete from "+store.QuoteIdent(name))
return err
}
}
if _, err := Import(ctx, opts); err != nil {
t.Fatal(err)
}
for name, want := range map[string]string{table: "snapshot", "messages": "unrelated"} {
var got string
if err := dst.DB().QueryRowContext(ctx, "select coalesce(group_concat(id), '') from "+store.QuoteIdent(name)).Scan(&got); err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("table %q rows = %q, want %q", name, got, want)
}
}
})
}
}
}
}

func TestExportRejectsUnsafeTablePath(t *testing.T) {
ctx := context.Background()
src, err := store.Open(ctx, store.Options{
Expand Down
Loading