From eaaf2a0d958411dad8890cfe7a17df10a9cf6849 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 19 Nov 2026 14:17:25 +0800 Subject: [PATCH] fix(snapshot): preserve literal table names during import --- CHANGELOG.md | 2 ++ docs/packages.md | 4 +++ snapshot/import.go | 4 +-- snapshot/snapshot_test.go | 56 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0456cd4..307e05d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Preserve literal table names during full snapshot imports so names with surrounding whitespace do not clear a different table or retain stale rows. + - Preserve existing file-cache snapshots when timestamps and names collide, including concurrent captures; report the collision instead of overwriting data. - Let `crawlctl logs -- JOB` display logs for jobs whose names look like command-line flags. diff --git a/docs/packages.md b/docs/packages.md index c68a264..d2e78a7 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -70,6 +70,10 @@ must not commit or roll back. Legacy filter closures retain their own database bindings. The driver's `ReadOnly` option is not an authorization boundary for trusted callbacks. +Import uses literal table names for both deletion and insertion, including surrounding +whitespace. Full imports also pass those exact names to `DeleteTable` callbacks; +blank entries in `DeleteTables` remain ignored. + Snapshot rows remain v1 JSON objects; no new wire format is enabled. Import planning compares unique column names independently of their order, so a snapshot from an equivalent schema does not require replacement just because diff --git a/snapshot/import.go b/snapshot/import.go index 63d7715..f55ae6e 100644 --- a/snapshot/import.go +++ b/snapshot/import.go @@ -47,8 +47,8 @@ func Import(ctx context.Context, opts ImportOptions) (Manifest, error) { } } for i := len(deleteTables) - 1; i >= 0; i-- { - table := strings.TrimSpace(deleteTables[i]) - if table == "" { + table := deleteTables[i] + if strings.TrimSpace(table) == "" { continue } if opts.DeleteTable != nil { diff --git a/snapshot/snapshot_test.go b/snapshot/snapshot_test.go index f130713..ccd4471 100644 --- a/snapshot/snapshot_test.go +++ b/snapshot/snapshot_test.go @@ -5,6 +5,7 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "os" "path/filepath" "slices" @@ -78,6 +79,61 @@ create table sync_state(source_name text, entity_type text, entity_id text, valu } } +func TestImportPreservesLiteralDeleteTableNames(t *testing.T) { + for _, table := range []string{" messages", "messages "} { + for _, explicit := range []bool{false, true} { + for _, custom := range []bool{false, true} { + t.Run(fmt.Sprintf("%q/explicit=%t/custom=%t", table, explicit, custom), func(t *testing.T) { + ctx := context.Background() + dst, err := store.Open(ctx, store.Options{ + Path: filepath.Join(t.TempDir(), "dst.db"), + Schema: `create table messages(id text primary key); +create table ` + store.QuoteIdent(table) + `(id text primary key);`, + }) + if err != nil { + t.Fatal(err) + } + defer dst.Close() + mustExec(t, dst.DB(), `insert into messages values ('unrelated')`) + mustExec(t, dst.DB(), `insert into `+store.QuoteIdent(table)+` values ('stale')`) + root := t.TempDir() + writeGzipJSONL(t, filepath.Join(root, "rows.jsonl.gz"), map[string]any{"id": "snapshot"}) + if err := WriteManifest(root, Manifest{Version: 1, Tables: []TableManifest{{ + Name: table, Columns: []string{"id"}, Files: []string{"rows.jsonl.gz"}, Rows: 1, + }}}); err != nil { + t.Fatal(err) + } + opts := ImportOptions{DB: dst.DB(), RootDir: root} + if explicit { + opts.DeleteTables = []string{"", " \t", table} + } + if custom { + opts.DeleteTable = func(ctx context.Context, tx *sql.Tx, name string) error { + if name != table { + t.Errorf("delete callback table = %q, want %q", name, table) + } + _, err := tx.ExecContext(ctx, "delete from "+store.QuoteIdent(name)) + return err + } + } + if _, err := Import(ctx, opts); err != nil { + t.Fatal(err) + } + for name, want := range map[string]string{table: "snapshot", "messages": "unrelated"} { + var got string + if err := dst.DB().QueryRowContext(ctx, "select coalesce(group_concat(id), '') from "+store.QuoteIdent(name)).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("table %q rows = %q, want %q", name, got, want) + } + } + }) + } + } + } +} + func TestExportRejectsUnsafeTablePath(t *testing.T) { ctx := context.Background() src, err := store.Open(ctx, store.Options{