From cfc20ced85bbf6dd2f429f12b76ee8aa3ccf05e5 Mon Sep 17 00:00:00 2001 From: GRIVN Date: Tue, 15 Sep 2026 01:55:31 +0800 Subject: [PATCH] fix(memory): preserve distinct facts during writes Limit remember and import deduplication to byte-identical active content. Keep fuzzy diff suggestions advisory, preserve distinct subjects and changed values, and resolve import duplicate indexes and edges to existing memories. Update canonical docs and host guides for deliberate forgetting. Validated with go build -o mnemon ., make test, and all 193 CLI E2E checks. Reproduced replacement and near-duplicate skipping on unmodified master, then verified the fixed CLI and SQLite records in isolated synthetic stores without embeddings. --- README.md | 2 +- cmd/memory/import.go | 48 +---- cmd/memory/import_diff_test.go | 157 ++++++++++++++++ cmd/memory/remember.go | 64 ++----- cmd/memory/remember_diff_test.go | 172 ++++++++++++++++++ docs/IMPORT.md | 15 +- docs/USAGE.md | 14 +- docs/design/05-pipelines.md | 61 ++++--- docs/zh/IMPORT.md | 12 +- docs/zh/README.md | 2 +- docs/zh/USAGE.md | 13 +- docs/zh/design/05-pipelines.md | 60 +++--- internal/memory/search/duplicate.go | 15 ++ internal/memory/setup/assets/claude/SKILL.md | 5 +- internal/memory/setup/assets/claude/guide.md | 4 +- .../memory/setup/assets/codebuddy/SKILL.md | 5 +- internal/memory/setup/assets/codex/SKILL.md | 5 +- internal/memory/setup/assets/cursor/SKILL.md | 5 +- internal/memory/setup/assets/kimi/SKILL.md | 5 +- internal/memory/setup/assets/nanobot/SKILL.md | 5 +- .../setup/assets/nanoclaw/container-skill.md | 9 +- .../memory/setup/assets/openclaw/SKILL.md | 5 +- .../memory/setup/assets/opencode/SKILL.md | 5 +- internal/memory/setup/assets/pi/SKILL.md | 5 +- internal/memory/setup/assets/qoder/SKILL.md | 5 +- .../memory/setup/assets/qoderwork/SKILL.md | 5 +- internal/memory/setup/assets/trae/SKILL.md | 5 +- .../memory/setup/assets/workbuddy/SKILL.md | 5 +- internal/memory/setup/assets/zcode/SKILL.md | 5 +- scripts/e2e_test.sh | 61 +++++++ 30 files changed, 595 insertions(+), 184 deletions(-) create mode 100644 cmd/memory/import_diff_test.go create mode 100644 cmd/memory/remember_diff_test.go create mode 100644 internal/memory/search/duplicate.go diff --git a/README.md b/README.md index 28df4225..052d21de 100644 --- a/README.md +++ b/README.md @@ -370,7 +370,7 @@ memory is useful. - **Four-graph architecture** — temporal, entity, causal, and semantic edges, not just vector similarity - **Intent-native protocol** — three primitives (`remember`, `link`, `recall`) map to the LLM's cognitive vocabulary, not database syntax; structured JSON output with signal transparency - **Intent-aware recall** — graph traversal + optional vector search (RRF fusion), enabled by default for all queries -- **Built-in deduplication** — `remember` auto-detects duplicates and conflicts; skips or auto-replaces +- **Built-in deduplication** — `remember` and `import` skip exact content repeats and preserve distinct facts; similarity suggestions guide review - **Retention lifecycle** — importance decay, access-count boosting, and garbage collection - **Privacy-safe receipts** — export hashed operation receipts for memory-boundary audits without raw memory contents or queries - **Optional embeddings** — works fully without an embedding provider; add local [Ollama](https://ollama.ai) or an OpenAI-compatible server for enhanced vector+keyword hybrid search diff --git a/cmd/memory/import.go b/cmd/memory/import.go index a20923fc..e684b006 100644 --- a/cmd/memory/import.go +++ b/cmd/memory/import.go @@ -56,7 +56,7 @@ exports are documented in docs/IMPORT.md.`, ec := embed.NewClientWithModel(resolveEmbedModel()) - // Build embed cache once for all diff and graph operations. + // Build embed cache once for all graph operations. var embedCache graph.EmbedCache if ec.Available() { dbEmbeds, err := db.GetAllEmbeddings() @@ -127,62 +127,30 @@ exports are documented in docs/IMPORT.md.`, } } - var action string - var replacedID string + action := "added" + var duplicateID string - if importNoDiff { - action = "added" - } else { + if !importNoDiff { allInsights, err := db.GetAllActiveInsights() if err != nil { results = append(results, importResult{Index: idx, ID: insight.ID, Content: insight.Content, Error: err.Error()}) continue } - opts := search.DiffOptions{Limit: 5, NewEmbedding: embeddingVec} - if embedCache != nil { - opts.ExistingEmbed = make([]search.EmbeddedItem, 0, len(embedCache)) - for id, v := range embedCache { - opts.ExistingEmbed = append(opts.ExistingEmbed, search.EmbeddedItem{ID: id, Embedding: v}) - } - } - result := search.Diff(allInsights, insight.Content, opts) - switch result.Suggestion { - case search.DiffDuplicate: + duplicateID = search.FindExactDuplicateID(allInsights, insight.Content) + if duplicateID != "" { action = "skipped" - if len(result.Matches) > 0 { - replacedID = result.Matches[0].ID - } - case search.DiffConflict, search.DiffUpdate: - action = "updated" - if len(result.Matches) > 0 { - replacedID = result.Matches[0].ID - } - default: - action = "added" } } if action == "skipped" { - db.LogOp("import-skip", insight.ID, fmt.Sprintf("duplicate of %s", replacedID)) - if replacedID != "" { - imported[idx] = replacedID - } else { - imported[idx] = insight.ID - } + db.LogOp("import-skip", insight.ID, fmt.Sprintf("duplicate of %s", duplicateID)) + imported[idx] = duplicateID results = append(results, importResult{Index: idx, ID: imported[idx], Content: insight.Content, Action: action}) continue } var writeErr error err = db.InTransaction(func() error { - if action == "updated" && replacedID != "" { - if err := db.SoftDeleteInsight(replacedID); err != nil { - fmt.Fprintf(os.Stderr, "warning: soft-delete %s: %v\n", replacedID, err) - } else { - db.LogOp("import-replace", replacedID, fmt.Sprintf("replaced by %s", insight.ID)) - delete(embedCache, replacedID) - } - } if err := db.InsertInsight(insight); err != nil { return fmt.Errorf("insert insight: %w", err) } diff --git a/cmd/memory/import_diff_test.go b/cmd/memory/import_diff_test.go new file mode 100644 index 00000000..2d34a51b --- /dev/null +++ b/cmd/memory/import_diff_test.go @@ -0,0 +1,157 @@ +package memory + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/mnemon-dev/mnemon/internal/memory/importdraft" + "github.com/mnemon-dev/mnemon/internal/memory/model" + "github.com/mnemon-dev/mnemon/internal/memory/store" +) + +func configureImportDiffTest(t *testing.T) { + t.Helper() + configureRememberDiffTest(t) + oldNoDiff, oldDryRun := importNoDiff, importDryRun + t.Cleanup(func() { importNoDiff, importDryRun = oldNoDiff, oldDryRun }) + importNoDiff, importDryRun = false, false +} + +type importDiffOutput struct { + Imported int `json:"imported"` + Updated int `json:"updated"` + Skipped int `json:"skipped"` + Errors int `json:"errors"` + Results []struct { + Index int `json:"index"` + ID string `json:"id"` + Content string `json:"content"` + Action string `json:"action"` + } `json:"results"` +} + +func importForDiffTest(t *testing.T, contents []string, edges []importdraft.DraftEdge) importDiffOutput { + t.Helper() + draft := importdraft.MemoryDraft{SchemaVersion: "1", Edges: edges} + for _, content := range contents { + draft.Insights = append(draft.Insights, importdraft.DraftInsight{ + Content: content, Category: "fact", Importance: 5, + }) + } + data, err := json.Marshal(draft) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "draft.json") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + var runErr error + out := captureStdout(t, func() { runErr = importCmd.RunE(importCmd, []string{path}) }) + if runErr != nil { + t.Fatalf("import: %v", runErr) + } + var result importDiffOutput + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("decode import output: %v\n%s", err, out) + } + if result.Errors != 0 || len(result.Results) != len(contents) { + t.Fatalf("incomplete import: %+v", result) + } + return result +} + +func TestImportPreservesDistinctContent(t *testing.T) { + const alpha = "Project Alpha uses PostgreSQL database for persistent application storage" + const details = " with indexed customer records, transaction history, audit events, replication, backups, failover, monitoring, access controls, migrations, connection pooling, and disaster recovery" + tests := []struct{ name, first, second string }{ + {"different subject", alpha, "Project Beta uses PostgreSQL database for persistent application storage"}, + {"changed value", alpha, "Project Alpha uses SQLite database for persistent application storage"}, + {"near duplicate", alpha + details, "Project Beta uses PostgreSQL database for persistent application storage" + details}, + {"conflict", alpha, "Project Alpha no longer uses PostgreSQL database for persistent application storage"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + configureImportDiffTest(t) + result := importForDiffTest(t, []string{tt.first, tt.second}, nil) + if result.Imported != 2 || result.Updated != 0 || result.Skipped != 0 { + t.Errorf("import = %+v, want two added insights", result) + } + assertActiveRememberContents(t, map[string]string{ + result.Results[0].ID: tt.first, result.Results[1].ID: tt.second, + }) + }) + } +} + +func TestImportPreservesExistingFact(t *testing.T) { + configureImportDiffTest(t) + const alpha = "Project Alpha uses PostgreSQL database for persistent application storage" + const beta = "Project Beta uses PostgreSQL database for persistent application storage" + first := rememberForDiffTest(t, alpha) + result := importForDiffTest(t, []string{beta}, nil) + if result.Imported != 1 || result.Updated != 0 || result.Skipped != 0 { + t.Errorf("import = %+v, want one added insight", result) + } + assertActiveRememberContents(t, map[string]string{first.ID: alpha, result.Results[0].ID: beta}) +} + +func TestImportExactDuplicateRetainsIndexAndEdgeMapping(t *testing.T) { + configureImportDiffTest(t) + const content = "Project Alpha uses PostgreSQL database for persistent application storage" + remImportance = 1 + first := rememberForDiffTest(t, content) + want := map[string]string{first.ID: content} + remImportance, remNoDiff = 5, true + for _, detail := range []string{"backups", "replication", "indexes", "migrations", "monitoring", "transactions"} { + extended := content + " with " + detail + result := rememberForDiffTest(t, extended) + want[result.ID] = extended + } + const linked = "Zebra migration survey notes" + result := importForDiffTest(t, []string{content, linked, linked}, []importdraft.DraftEdge{ + {SourceIndex: 0, TargetIndex: 2, EdgeType: "semantic", Weight: 0.75, Reason: "exact duplicate index mapping"}, + }) + if result.Imported != 1 || result.Updated != 0 || result.Skipped != 2 { + t.Errorf("import = %+v, want one added insight and two exact duplicates", result) + } + if result.Results[0].ID != first.ID || result.Results[0].Action != "skipped" { + t.Errorf("existing duplicate = %+v, want skipped %s", result.Results[0], first.ID) + } + linkedID := result.Results[1].ID + if result.Results[2].ID != linkedID || result.Results[2].Action != "skipped" { + t.Errorf("batch duplicate = %+v, want skipped %s", result.Results[2], linkedID) + } + want[linkedID] = linked + assertActiveRememberContents(t, want) + db, err := store.Open(store.StoreDir(dataDir, store.DefaultStoreName)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + edges, err := db.GetEdgesBySourceAndType(first.ID, model.EdgeSemantic) + if err != nil { + t.Fatal(err) + } + for _, edge := range edges { + if edge.TargetID == linkedID && edge.Metadata["reason"] == "exact duplicate index mapping" { + return + } + } + t.Fatalf("explicit edge from existing duplicate %s to batch duplicate %s is missing", first.ID, linkedID) +} + +func TestImportNoDiffStoresExactRepeats(t *testing.T) { + configureImportDiffTest(t) + importNoDiff = true + const content = "Project Alpha uses PostgreSQL database for persistent application storage" + result := importForDiffTest(t, []string{content, content}, nil) + if result.Imported != 2 || result.Updated != 0 || result.Skipped != 0 { + t.Errorf("import with --no-diff = %+v, want two added insights", result) + } + assertActiveRememberContents(t, map[string]string{ + result.Results[0].ID: content, result.Results[1].ID: content, + }) +} diff --git a/cmd/memory/remember.go b/cmd/memory/remember.go index 5f565757..0768f38a 100644 --- a/cmd/memory/remember.go +++ b/cmd/memory/remember.go @@ -118,9 +118,9 @@ var rememberCmd = &cobra.Command{ } // 2. Built-in diff: check for duplicates/conflicts (read-only, before transaction) - var diffAction string // "added", "updated", "skipped" - var replacedID string - var diffSuggestion search.DiffSuggestion + diffAction := "added" + diffSuggestion := search.DiffAdd + var duplicateID string // Build embed cache once — reused by diff, engine, and semantic candidates. var embedCache graph.EmbedCache @@ -136,10 +136,7 @@ var rememberCmd = &cobra.Command{ } } - if remNoDiff { - diffAction = "added" - diffSuggestion = search.DiffAdd - } else { + if !remNoDiff { allInsights, err := db.GetAllActiveInsights() if err != nil { return fmt.Errorf("load insights for diff: %w", err) @@ -159,45 +156,25 @@ var rememberCmd = &cobra.Command{ result := search.Diff(allInsights, content, opts) diffSuggestion = result.Suggestion - switch result.Suggestion { - case search.DiffDuplicate: + // Similarity cannot establish whether two facts have the same + // subject, value, or relationship. Keep diff suggestions advisory; + // only byte-identical content permits skipping a write. + duplicateID = search.FindExactDuplicateID(allInsights, content) + if duplicateID != "" { diffAction = "skipped" - if len(result.Matches) > 0 { - replacedID = result.Matches[0].ID - } - case search.DiffConflict: - // A CONFLICT means the two texts appear to disagree. Silently - // soft-deleting one side is destructive and has repeatedly - // clobbered unrelated same-domain memories (long technical - // notes share vocabulary at >=0.7 similarity, and change-log - // words like "replaced"/"no longer" appear in almost all of - // them). Keep both; the caller sees diff_suggestion=CONFLICT - // and can merge or delete deliberately. - diffAction = "added" - case search.DiffUpdate: - // Only auto-replace when the texts overlap heavily by TOKENS. - // Cosine similarity alone (same-domain embeddings cluster at - // 0.85+) is not enough evidence to destroy an existing memory. - if len(result.Matches) > 0 && result.Matches[0].TokenSimilarity >= 0.6 { - diffAction = "updated" - replacedID = result.Matches[0].ID - } else { - diffAction = "added" - } - default: - diffAction = "added" + diffSuggestion = search.DiffDuplicate } } // If duplicate, skip insert entirely if diffAction == "skipped" { - db.LogOp("diff-skip", insight.ID, fmt.Sprintf("duplicate of %s", replacedID)) + db.LogOp("diff-skip", insight.ID, fmt.Sprintf("duplicate of %s", duplicateID)) output := map[string]interface{}{ "id": insight.ID, "content": content, "action": "skipped", "diff_suggestion": string(diffSuggestion), - "replaced_id": replacedID, + "replaced_id": duplicateID, } enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") @@ -212,18 +189,6 @@ var rememberCmd = &cobra.Command{ embedded bool ) err = db.InTransaction(func() error { - // Soft-delete old insight if updating - if diffAction == "updated" && replacedID != "" { - if err := db.SoftDeleteInsight(replacedID); err != nil { - fmt.Fprintf(os.Stderr, "warning: soft-delete %s: %v\n", replacedID, err) - } else { - db.LogOp("diff-replace", replacedID, fmt.Sprintf("replaced by %s", insight.ID)) - // Remove deleted insight from embed cache to prevent - // creating edges to a soft-deleted node. - delete(embedCache, replacedID) - } - } - if err := db.InsertInsight(insight); err != nil { return fmt.Errorf("insert insight: %w", err) } @@ -268,7 +233,7 @@ var rememberCmd = &cobra.Command{ return nil }) if err != nil { - // Cache was mutated inside the transaction closure (delete/add entries). + // Cache was mutated inside the transaction closure (added entries). // On rollback those mutations don't match DB state, so discard the cache // to prevent any future code from accidentally using stale data. embedCache = nil @@ -306,9 +271,6 @@ var rememberCmd = &cobra.Command{ "auto_pruned": len(prunedIDs), "auto_pruned_ids": prunedIDs, } - if replacedID != "" { - output["replaced_id"] = replacedID - } enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") return enc.Encode(output) diff --git a/cmd/memory/remember_diff_test.go b/cmd/memory/remember_diff_test.go new file mode 100644 index 00000000..170bcdb2 --- /dev/null +++ b/cmd/memory/remember_diff_test.go @@ -0,0 +1,172 @@ +package memory + +import ( + "encoding/json" + "testing" + + "github.com/mnemon-dev/mnemon/internal/memory/search" + "github.com/mnemon-dev/mnemon/internal/memory/store" +) + +func configureRememberDiffTest(t *testing.T) { + t.Helper() + configureRememberTest(t) + storeName = store.DefaultStoreName + remNoDiff = false + remImportance = 5 + t.Setenv("MNEMON_MAX_INSIGHTS", "1000") +} + +type rememberDiffOutput struct { + ID string `json:"id"` + Action string `json:"action"` + DiffSuggestion search.DiffSuggestion `json:"diff_suggestion"` + ReplacedID string `json:"replaced_id"` +} + +func rememberForDiffTest(t *testing.T, content string) rememberDiffOutput { + t.Helper() + var runErr error + out := captureStdout(t, func() { + runErr = rememberCmd.RunE(rememberCmd, []string{content}) + }) + if runErr != nil { + t.Fatalf("remember: %v", runErr) + } + var result rememberDiffOutput + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("decode remember output: %v\n%s", err, out) + } + return result +} + +func assertActiveRememberContents(t *testing.T, want map[string]string) { + t.Helper() + db, err := store.Open(store.StoreDir(dataDir, store.DefaultStoreName)) + if err != nil { + t.Fatalf("reopen store: %v", err) + } + defer db.Close() + active, err := db.GetAllActiveInsights() + if err != nil { + t.Fatalf("get active insights: %v", err) + } + if len(active) != len(want) { + t.Errorf("active insight count = %d, want %d", len(active), len(want)) + } + for _, insight := range active { + if content, ok := want[insight.ID]; !ok || insight.Content != content { + t.Errorf("unexpected active insight %s: %q", insight.ID, insight.Content) + } + } +} + +func TestRememberPreservesDistinctContent(t *testing.T) { + const alpha = "Project Alpha uses PostgreSQL database for persistent application storage" + const details = " with indexed customer records, transaction history, audit events, replication, backups, failover, monitoring, access controls, migrations, connection pooling, and disaster recovery" + tests := []struct { + name string + first string + second string + suggestion search.DiffSuggestion + }{ + { + name: "different subject same property", + first: alpha, + second: "Project Beta uses PostgreSQL database for persistent application storage", + suggestion: search.DiffUpdate, + }, + { + name: "same subject changed value", + first: alpha, + second: "Project Alpha uses SQLite database for persistent application storage", + suggestion: search.DiffUpdate, + }, + { + name: "different subject above duplicate threshold", + first: alpha + details, + second: "Project Beta uses PostgreSQL database for persistent application storage" + details, + suggestion: search.DiffDuplicate, + }, + { + name: "same tokens different relationship", + first: "Project Alpha exports records to Project Beta", + second: "Project Beta exports records to Project Alpha", + suggestion: search.DiffDuplicate, + }, + { + name: "conflict remains advisory", + first: alpha, + second: "Project Alpha no longer uses PostgreSQL database for persistent application storage", + suggestion: search.DiffConflict, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + configureRememberDiffTest(t) + first := rememberForDiffTest(t, tt.first) + second := rememberForDiffTest(t, tt.second) + if first.Action != "added" || second.Action != "added" { + t.Errorf("actions = (%q, %q), want (added, added)", first.Action, second.Action) + } + if second.ReplacedID != "" { + t.Errorf("preserved content reports replaced_id = %q", second.ReplacedID) + } + if second.DiffSuggestion != tt.suggestion { + t.Errorf("diff_suggestion = %q, want %q", second.DiffSuggestion, tt.suggestion) + } + assertActiveRememberContents(t, map[string]string{first.ID: tt.first, second.ID: tt.second}) + }) + } +} + +func TestRememberSkipsExactDuplicate(t *testing.T) { + for _, content := range []string{ + "Project Alpha uses PostgreSQL database for persistent application storage", + "!!!", // Exact identity must not depend on searchable tokens. + } { + t.Run(content, func(t *testing.T) { + configureRememberDiffTest(t) + first := rememberForDiffTest(t, content) + repeat := rememberForDiffTest(t, content) + if repeat.Action != "skipped" || repeat.DiffSuggestion != search.DiffDuplicate || repeat.ReplacedID != first.ID { + t.Errorf("repeat = %+v, want skipped DUPLICATE of %s", repeat, first.ID) + } + assertActiveRememberContents(t, map[string]string{first.ID: content}) + }) + } +} + +func TestRememberExactDuplicateOutsideDiffCandidates(t *testing.T) { + configureRememberDiffTest(t) + const content = "Project Alpha uses PostgreSQL database for persistent application storage" + remImportance = 1 + first := rememberForDiffTest(t, content) + want := map[string]string{first.ID: content} + remImportance = 5 + remNoDiff = true + // Higher-importance keyword ties fill the five diff candidate slots. + for _, detail := range []string{"backups", "replication", "indexes", "migrations", "monitoring", "transactions"} { + extended := content + " with " + detail + result := rememberForDiffTest(t, extended) + want[result.ID] = extended + } + remNoDiff = false + repeat := rememberForDiffTest(t, content) + if repeat.Action != "skipped" || repeat.DiffSuggestion != search.DiffDuplicate || repeat.ReplacedID != first.ID { + t.Errorf("repeat = %+v, want skipped DUPLICATE of %s", repeat, first.ID) + } + assertActiveRememberContents(t, want) +} + +func TestRememberNoDiffStoresExactRepeat(t *testing.T) { + configureRememberDiffTest(t) + remNoDiff = true + const content = "Project Alpha uses PostgreSQL database for persistent application storage" + first := rememberForDiffTest(t, content) + repeat := rememberForDiffTest(t, content) + if repeat.Action != "added" || repeat.DiffSuggestion != search.DiffAdd || repeat.ReplacedID != "" { + t.Errorf("repeat with --no-diff = %+v, want added ADD without replaced_id", repeat) + } + assertActiveRememberContents(t, map[string]string{first.ID: content, repeat.ID: content}) +} diff --git a/docs/IMPORT.md b/docs/IMPORT.md index 024887c1..f45973b3 100644 --- a/docs/IMPORT.md +++ b/docs/IMPORT.md @@ -132,7 +132,7 @@ mnemon import memory_draft.json # Validate without writing mnemon import --dry-run memory_draft.json -# Skip duplicate/conflict detection and insert every entry as new +# Skip exact duplicate detection and insert every entry as new mnemon import --no-diff memory_draft.json # Import into a specific store @@ -141,10 +141,17 @@ mnemon import --store project-alpha memory_draft.json ### Output Example +Import uses the same exact-content deduplication rule as `remember`: only +byte-identical active content is skipped. Different facts and near-duplicates +are added, including conflicting or updated values. A skipped draft index maps +to the existing insight ID, so explicit edges still resolve to that memory. +To retire a superseded fact, verify the new memory and use `mnemon forget +` explicitly. Capacity-based auto-pruning remains separate. + ```json { "imported": 8, - "updated": 1, + "updated": 0, "skipped": 2, "errors": 0, "edges_inserted": 3, @@ -160,8 +167,8 @@ mnemon import --store project-alpha memory_draft.json | Field | Description | |---|---| | `imported` | Number of newly added memories | -| `updated` | Number of existing conflicting memories replaced | -| `skipped` | Number of duplicate memories skipped | +| `updated` | Always `0`; retained for output compatibility. Similarity does not replace existing memories | +| `skipped` | Number of byte-identical content repeats skipped | | `errors` | Number of failed writes. Import allows partial success; script callers should check this is `0` | | `edges_inserted` | Number of explicit edges inserted | | `auto_pruned` | Number of memories auto-pruned after capacity checks | diff --git a/docs/USAGE.md b/docs/USAGE.md index f7e7040d..98e7e2c3 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -99,7 +99,7 @@ mnemon setup --eject --target claude-code ### Core ```bash -# Remember — store a new insight (built-in diff: duplicates skipped, conflicts auto-replaced) +# Remember — store a new insight (exact repeats skipped; distinct content preserved) mnemon remember "Chose Qdrant over Milvus for vector search" \ --cat decision --imp 5 --entities "Qdrant,Milvus" --tags "architecture,search" --source agent @@ -138,6 +138,18 @@ mnemon import --no-diff memory_draft.json # skip deduplication mnemon forget ``` +`remember` and `import` skip only byte-identical content already present in an +active memory. Different subjects, changed values, reordered statements, and +near-duplicates are stored as new memories. `remember` still reports advisory +`diff_suggestion` values (`UPDATE`, `CONFLICT`, or `DUPLICATE`); read `action` to +see whether the write was `added` or `skipped`. On an exact repeat, the legacy +`replaced_id` field identifies the existing memory, which remains unchanged. +`--no-diff` also inserts exact repeats. + +To retire a superseded memory, store the new fact, verify it with `mnemon show +`, then explicitly run `mnemon forget `. Similarity alone never +authorizes replacement. Capacity-based auto-pruning still applies separately. + **Remember flags:** | Flag | Default | Description | diff --git a/docs/design/05-pipelines.md b/docs/design/05-pipelines.md index 996dec9f..519a9f26 100644 --- a/docs/design/05-pipelines.md +++ b/docs/design/05-pipelines.md @@ -8,8 +8,6 @@ `mnemon remember` is the core command for writing memories. It includes a built-in diff step that automatically detects duplicates and conflicts before storage. The write transaction executes atomically within a single SQLite transaction. -![Remember Pipeline](../diagrams/02-remember-pipeline.jpg) - ### Detailed Flow ``` @@ -29,18 +27,19 @@ mnemon remember "Chose Qdrant as the vector database" \ **Step 2.5: Built-in Diff (outside transaction, read-only)** -Compute similarity against all active insights: -- **DUPLICATE** (sim > 0.90) → skip insert entirely, return `action="skipped"` -- **CONFLICT/UPDATE** (sim 0.50–0.90) → soft-delete old insight, insert new as replacement -- **ADD** (sim < 0.50) → normal insert +Compute advisory similarity suggestions and check exact content against all active insights: +- **Byte-identical content** → skip insert, return `action="skipped"` and `diff_suggestion="DUPLICATE"` +- **Different content** → normal insert with `action="added"`, preserving the heuristic `diff_suggestion` for review -This step uses embedding cosine similarity when available, falling back to token overlap. The `--no-diff` flag disables this check. +Similarity uses embedding cosine when available, falling back to token overlap. +Neither similarity nor extracted entities establish that one fact supersedes +another. Exact-content lookup is independent of similarity candidate limits. +The `--no-diff` flag disables both checks and inserts even exact repeats. **Step 3: Atomic Transaction** ``` BEGIN TRANSACTION - ⓪ Soft-delete replaced insight (if diff found CONFLICT/UPDATE) ① INSERT insight (UUID, content, category, importance, tags, entities, source) ② UPDATE embedding (if vector is available) ③ Graph Engine: OnInsightCreated @@ -65,7 +64,6 @@ COMMIT "id": "abc-123", "action": "added", "diff_suggestion": "ADD", - "replaced_id": null, "edges_created": {"temporal": 2, "entity": 3, "causal": 1, "semantic": 1}, "semantic_candidates": [ {"id": "def-456", "content": "...", "cosine": 0.72, "auto_linked": false} @@ -80,7 +78,10 @@ COMMIT } ``` -The `action` field indicates what the built-in diff decided: `"added"` (new entry), `"replaced"` (conflict auto-replaced, `replaced_id` contains the old insight ID), or `"skipped"` (duplicate detected, no insert). +The `action` field is `"added"` (new entry) or `"skipped"` (byte-identical active +content, no insert). On a skipped repeat, the legacy `replaced_id` field names +the existing memory; it is not deleted or changed. A heuristic `UPDATE`, +`CONFLICT`, or near-`DUPLICATE` suggestion still produces `action="added"`. After receiving this output, the LLM can evaluate candidates and establish edges it considers appropriate via the `mnemon link` command. @@ -206,33 +207,45 @@ This is a unique innovation in Mnemon: **exposing the retrieval pipeline's inter ## 5.3 Deduplication & Conflict Detection: Diff -![Diff & Dedup Pipeline](../diagrams/07-diff-dedup-pipeline.jpg) +```mermaid +flowchart TD + Write[remember / import] --> Bypass{"--no-diff?"} + Bypass -->|Yes| Add[Insert new memory; preserve existing facts] + Bypass -->|No| Exact{Byte-identical active content?} + Exact -->|Yes| Skip[Skip insert; identify existing memory] + Exact -->|No| Add + Write -. remember only .-> Advisory[Similarity suggestions for review] +``` Diff is **built into `remember`** — no separate call needed. When `mnemon remember` is invoked, it automatically runs a diff check before inserting. When `remember` is called, the built-in diff runs before the transaction: 1. Compute similarity against all active insights (embedding cosine when available, token overlap as fallback) -2. Determine the action based on similarity thresholds: +2. Independently scan all active insights for byte-identical content. Similarity + suggestions describe possible relationships; exact equality alone permits a skip: -| Similarity | Action | Behavior | -|------------|--------|----------| -| > 0.90 | **DUPLICATE** | Skip insert entirely, return `action="skipped"` | -| 0.50 ~ 0.90 | **CONFLICT/UPDATE** | Soft-delete old insight, insert new as replacement | -| < 0.50 | **ADD** | Normal insert | +| Content | Diff suggestion | Behavior | +|---------|-----------------|----------| +| Byte-identical to an active memory | **DUPLICATE** | Skip insert, return `action="skipped"` | +| Different at any similarity | **ADD**, **UPDATE**, **CONFLICT**, or **DUPLICATE** | Insert with `action="added"`; retain the existing memory | -The `--no-diff` flag disables this check for cases where the caller wants unconditional insertion. +`import` uses the same exact-content lookup and preserves distinct content. +The `--no-diff` flag allows unconditional insertion on either command. +Capacity-based auto-pruning remains a separate lifecycle policy. ### Typical Workflow -A single `remember` call handles everything: +Store new content, then retire a specific old fact only when appropriate: ```bash -# Single command — diff is automatic +# Diff suggestions are automatic and advisory mnemon remember "Chose PostgreSQL to replace SQLite as the primary database" \ --cat decision --imp 5 --source agent -# → If conflict with existing "Chose SQLite as storage": -# auto-replaces old insight, returns action="replaced", replaced_id="" -# → If duplicate: returns action="skipped" -# → If new: returns action="added" +# → Exact repeat: action="skipped"; existing memory unchanged +# → Different content: action="added"; existing memories retained + +# If the new fact supersedes a specific old memory, verify before retiring it +mnemon show +mnemon forget ``` diff --git a/docs/zh/IMPORT.md b/docs/zh/IMPORT.md index 91a36d8a..63cdd84c 100644 --- a/docs/zh/IMPORT.md +++ b/docs/zh/IMPORT.md @@ -140,10 +140,16 @@ mnemon import --store project-alpha memory_draft.json ### 输出示例 +导入与 `remember` 使用相同的精确内容去重规则:仅跳过与活跃记忆逐字节 +完全相同的内容。不同事实、近似重复、冲突和变化后的属性值都会新增。 +被跳过的草稿索引映射到已有 insight ID,显式边仍可引用该记忆。 +如需淘汰已被取代的事实,请验证新记忆后显式执行 `mnemon forget `。 +基于容量的自动清理仍独立生效。 + ```json { "imported": 8, - "updated": 1, + "updated": 0, "skipped": 2, "errors": 0, "edges_inserted": 3, @@ -159,8 +165,8 @@ mnemon import --store project-alpha memory_draft.json | 字段 | 说明 | |---|---| | `imported` | 新增的记忆数量 | -| `updated` | 替换了已有冲突记忆的数量 | -| `skipped` | 检测为重复而跳过的数量 | +| `updated` | 固定为 `0`,保留此字段以兼容已有输出;相似度不会触发替换 | +| `skipped` | 因内容逐字节完全相同而跳过的数量 | | `errors` | 写入失败的数量;导入允许部分成功,脚本调用方应检查此字段是否为 0 | | `edges_inserted` | 成功插入的显式边数量 | | `auto_pruned` | 超出容量限制后自动删除的记忆数量 | diff --git a/docs/zh/README.md b/docs/zh/README.md index 9bb14eef..a42b0a03 100644 --- a/docs/zh/README.md +++ b/docs/zh/README.md @@ -329,7 +329,7 @@ store 可见。**Remind** 触发 recall 判断。**Nudge** 触发 writeback 判 - **四图架构** — 时序、实体、因果、语义四种边,不仅仅是向量相似度 - **意图原生协议** — 三个原语(`remember`、`link`、`recall`)映射到 LLM 的认知词汇而非数据库语法;结构化 JSON 输出,带信号透明度 - **意图感知召回** — 图遍历 + 可选向量搜索(RRF 融合),所有查询默认启用 -- **内置去重** — `remember` 自动检测重复和冲突;跳过或自动替换 +- **内置去重** — `remember` 和 `import` 仅跳过内容完全相同的记忆,保留不同事实;相似度建议供复核参考 - **保留度生命周期** — 重要性衰减、访问计数提升、免疫规则、垃圾回收 - **可选嵌入向量** — 可使用本地 [Ollama](https://ollama.ai) 或 OpenAI 兼容服务器,支持混合向量+关键词搜索 diff --git a/docs/zh/USAGE.md b/docs/zh/USAGE.md index 83bd5f9d..cdd21df6 100644 --- a/docs/zh/USAGE.md +++ b/docs/zh/USAGE.md @@ -94,7 +94,7 @@ mnemon setup --eject --target claude-code ### 核心命令 ```bash -# Remember — 存储新洞察(内置 diff:重复跳过,冲突自动替换) +# Remember — 存储新洞察(仅跳过内容完全相同的记忆,保留不同内容) mnemon remember "选择 Qdrant 而非 Milvus 做向量搜索" \ --cat decision --imp 5 --entities "Qdrant,Milvus" --tags "architecture,search" --source agent @@ -133,6 +133,17 @@ mnemon import --no-diff memory_draft.json # 跳过去重 mnemon forget ``` +`remember` 和 `import` 仅跳过与活跃记忆逐字节完全相同的内容。不同主体、 +变化后的属性值、调整语序的陈述和近似重复内容都会作为新记忆保存。 +`remember` 仍会返回建议性的 `diff_suggestion`(`UPDATE`、`CONFLICT` 或 +`DUPLICATE`);实际写入结果以 `action` 的 `added` 或 `skipped` 为准。 +完全重复时,兼容字段 `replaced_id` 指向保持不变的已有记忆。 +使用 `--no-diff` 则连完全重复的内容也会插入。 + +如需淘汰已被取代的记忆,先保存新事实,用 `mnemon show ` 验证, +再显式执行 `mnemon forget `。相似度本身不会授权替换; +基于容量的自动清理仍独立生效。 + **Remember 标志:** | 标志 | 默认值 | 说明 | diff --git a/docs/zh/design/05-pipelines.md b/docs/zh/design/05-pipelines.md index 06d591f1..908e3ece 100644 --- a/docs/zh/design/05-pipelines.md +++ b/docs/zh/design/05-pipelines.md @@ -6,8 +6,6 @@ `mnemon remember` 是写入记忆的核心命令。它包含内置的 diff 步骤,在存储前自动检测重复和冲突。写入事务在一个 SQLite 事务中原子执行。 -![Remember Pipeline](../../diagrams/02-remember-pipeline.jpg) - ### 流程详解 ``` @@ -27,18 +25,19 @@ mnemon remember "选择 Qdrant 作为向量数据库" \ **第 2.5 步:内置 Diff(事务外,只读)** -对所有活跃 insight 计算相似度: -- **DUPLICATE**(sim > 0.90)→ 跳过插入,返回 `action="skipped"` -- **CONFLICT/UPDATE**(sim 0.50–0.90)→ 软删除旧 insight,插入新的替换 -- **ADD**(sim < 0.50)→ 正常插入 +生成相似度建议,并与所有活跃 insight 进行精确内容比较: +- **内容逐字节完全相同** → 跳过插入,返回 `action="skipped"` 和 `diff_suggestion="DUPLICATE"` +- **内容不同** → 正常插入,返回 `action="added"`,保留启发式 `diff_suggestion` 供复核 -此步骤在有嵌入时使用余弦相似度,否则降级为 token 重叠。`--no-diff` 标志可禁用此检查。 +相似度在有嵌入时使用余弦计算,否则使用 token 重叠。 +相似度与自动抽取的实体都无法确定一个事实是否取代另一个。 +精确内容查找不受相似度候选数量限制。 +`--no-diff` 会禁用两项检查,连完全重复的内容也会插入。 **第三步:原子事务** ``` BEGIN TRANSACTION - ⓪ 软删除被替换的 insight(如果 diff 检测到 CONFLICT/UPDATE) ① INSERT insight(UUID, content, category, importance, tags, entities, source) ② UPDATE embedding(如果有向量) ③ Graph Engine: OnInsightCreated @@ -63,7 +62,6 @@ COMMIT "id": "abc-123", "action": "added", "diff_suggestion": "ADD", - "replaced_id": null, "edges_created": {"temporal": 2, "entity": 3, "causal": 1, "semantic": 1}, "semantic_candidates": [ {"id": "def-456", "content": "...", "cosine": 0.72, "auto_linked": false} @@ -78,7 +76,9 @@ COMMIT } ``` -`action` 字段表示内置 diff 的决定:`"added"`(新增)、`"replaced"`(冲突自动替换,`replaced_id` 包含旧 insight ID)或 `"skipped"`(检测到重复,未插入)。 +`action` 为 `"added"`(新增)或 `"skipped"`(与活跃记忆内容逐字节完全相同, +未插入)。跳过时,兼容字段 `replaced_id` 指向保持不变的已有记忆。 +启发式 `UPDATE`、`CONFLICT` 或近似 `DUPLICATE` 建议仍会返回 `action="added"`。 LLM 收到这个输出后,可以评估候选并通过 `mnemon link` 命令建立它认为合理的边。 @@ -204,33 +204,45 @@ final = w_kw·keyword + w_ent·entity + w_sim·similarity + w_gr·graph ## 5.3 去重与冲突检测:Diff -![Diff & Dedup Pipeline](../../diagrams/07-diff-dedup-pipeline.jpg) +```mermaid +flowchart TD + Write[remember / import] --> Bypass{"--no-diff?"} + Bypass -->|是| Add[插入新记忆,保留已有事实] + Bypass -->|否| Exact{与活跃记忆逐字节完全相同?} + Exact -->|是| Skip[跳过插入,返回已有记忆的标识] + Exact -->|否| Add + Write -. 仅 remember .-> Advisory[相似度建议供复核参考] +``` Diff 已**内置于 `remember`** — 无需单独调用。当调用 `mnemon remember` 时,它会自动在插入前运行 diff 检查。 调用 `remember` 时,内置 diff 在事务之前运行: 1. 对所有活跃 insight 计算相似度(有嵌入时使用余弦相似度,否则使用 token 重叠) -2. 根据相似度阈值判断动作: +2. 独立扫描所有活跃 insight,检查逐字节相同的内容。相似度仅提示潜在关系, + 只有精确相等才允许跳过写入: -| 相似度 | 动作 | 行为 | -|--------|------|------| -| > 0.90 | **DUPLICATE** | 跳过插入,返回 `action="skipped"` | -| 0.50 ~ 0.90 | **CONFLICT/UPDATE** | 软删除旧 insight,插入新的替换 | -| < 0.50 | **ADD** | 正常插入 | +| 内容 | Diff 建议 | 行为 | +|------|-----------|------| +| 与活跃记忆逐字节完全相同 | **DUPLICATE** | 跳过插入,返回 `action="skipped"` | +| 内容不同,任意相似度 | **ADD**、**UPDATE**、**CONFLICT** 或 **DUPLICATE** | 返回 `action="added"`,保留已有记忆 | -`--no-diff` 标志可禁用此检查,用于需要无条件插入的场景。 +`import` 使用相同的精确内容查找规则,保留不同内容。 +两条命令均可通过 `--no-diff` 无条件插入。 +基于容量的自动清理仍是独立的生命周期策略。 ### 典型工作流 -一条 `remember` 命令即可处理一切: +先保存新内容,仅在确有需要时淘汰指定的旧事实: ```bash -# 单条命令 — diff 自动执行 +# Diff 自动执行,结果仅供参考 mnemon remember "选择 PostgreSQL 替代 SQLite 作为主数据库" \ --cat decision --imp 5 --source agent -# → 如果与已有的 "选择 SQLite 作为存储" 冲突: -# 自动替换旧 insight,返回 action="replaced", replaced_id="" -# → 如果重复:返回 action="skipped" -# → 如果是新内容:返回 action="added" +# → 完全重复:action="skipped",已有记忆保持不变 +# → 内容不同:action="added",保留已有记忆 + +# 如果新事实取代了指定旧记忆,先验证,再淘汰 +mnemon show +mnemon forget ``` diff --git a/internal/memory/search/duplicate.go b/internal/memory/search/duplicate.go new file mode 100644 index 00000000..7ffff4f2 --- /dev/null +++ b/internal/memory/search/duplicate.go @@ -0,0 +1,15 @@ +package search + +import "github.com/mnemon-dev/mnemon/internal/memory/model" + +// FindExactDuplicateID returns the ID of an insight with byte-identical content, +// or an empty string when none exists. Callers supply all active insights: fuzzy +// diff candidates and similarity scores cannot establish content identity. +func FindExactDuplicateID(insights []*model.Insight, content string) string { + for _, insight := range insights { + if insight.Content == content { + return insight.ID + } + } + return "" +} diff --git a/internal/memory/setup/assets/claude/SKILL.md b/internal/memory/setup/assets/claude/SKILL.md index 7395758e..807221e1 100644 --- a/internal/memory/setup/assets/claude/SKILL.md +++ b/internal/memory/setup/assets/claude/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built-in: duplicates skipped, conflicts auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, `causal_candidates`. 2. **Link** (evaluate candidates from step 1 — use judgment, not mechanical rules): - Review `causal_candidates`: does a genuine cause-effect relationship exist? `causal_signal` is regex-based and prone to false positives — only link if the memories are truly causally related. - Review `semantic_candidates`: are these memories meaningfully related? High `similarity` alone is not sufficient — skip candidates that share keywords but discuss unrelated topics. diff --git a/internal/memory/setup/assets/claude/guide.md b/internal/memory/setup/assets/claude/guide.md index 841f8d2e..625be1e7 100644 --- a/internal/memory/setup/assets/claude/guide.md +++ b/internal/memory/setup/assets/claude/guide.md @@ -37,8 +37,8 @@ Tier C (importance 1, store only if genuinely reusable): → None of the above → STOP. **Step 2 — Does a highly overlapping memory already exist?** -→ Yes, incremental new info → UPDATE (merge into existing) -→ Yes, but contradicts/supersedes → REPLACE +→ Yes, incremental new info → STORE the new content; review the existing memory separately +→ Yes, but contradicts/supersedes → STORE and verify the new fact; explicitly forget the old ID only if superseded → No significant overlap → CREATE **Step 3 — Importance calibration** diff --git a/internal/memory/setup/assets/codebuddy/SKILL.md b/internal/memory/setup/assets/codebuddy/SKILL.md index def7eb51..13cdb3b6 100644 --- a/internal/memory/setup/assets/codebuddy/SKILL.md +++ b/internal/memory/setup/assets/codebuddy/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for CodeBuddy. Store facts, recall past knowl ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/codex/SKILL.md b/internal/memory/setup/assets/codex/SKILL.md index c6a0d03c..43dd28e0 100644 --- a/internal/memory/setup/assets/codex/SKILL.md +++ b/internal/memory/setup/assets/codex/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/cursor/SKILL.md b/internal/memory/setup/assets/cursor/SKILL.md index 6080caa4..26b3f738 100644 --- a/internal/memory/setup/assets/cursor/SKILL.md +++ b/internal/memory/setup/assets/cursor/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/kimi/SKILL.md b/internal/memory/setup/assets/kimi/SKILL.md index 00f756cb..f34dd6d9 100644 --- a/internal/memory/setup/assets/kimi/SKILL.md +++ b/internal/memory/setup/assets/kimi/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for Kimi Code. Store facts, recall past knowl ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/nanobot/SKILL.md b/internal/memory/setup/assets/nanobot/SKILL.md index d23cd90c..8393b2bb 100644 --- a/internal/memory/setup/assets/nanobot/SKILL.md +++ b/internal/memory/setup/assets/nanobot/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built-in: duplicates skipped, conflicts auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, `causal_candidates`. 2. **Link** (evaluate candidates from step 1 — use judgment, not mechanical rules): - Review `causal_candidates`: does a genuine cause-effect relationship exist? `causal_signal` is regex-based and prone to false positives — only link if the memories are truly causally related. - Review `semantic_candidates`: are these memories meaningfully related? High `similarity` alone is not sufficient — skip candidates that share keywords but discuss unrelated topics. diff --git a/internal/memory/setup/assets/nanoclaw/container-skill.md b/internal/memory/setup/assets/nanoclaw/container-skill.md index 2f6b17ed..2967e512 100644 --- a/internal/memory/setup/assets/nanoclaw/container-skill.md +++ b/internal/memory/setup/assets/nanoclaw/container-skill.md @@ -37,8 +37,8 @@ Run this decision tree after every substantive response: → No to all → STOP. **Step 2 — Does a highly overlapping memory already exist?** - → Yes, incremental new info → UPDATE (merge into existing) - → Yes, but contradicts/supersedes → REPLACE + → Yes, incremental new info → STORE the new content; review the existing memory separately + → Yes, but contradicts/supersedes → STORE and verify the new fact; explicitly forget the old ID only if superseded → No significant overlap → CREATE **Step 3 — Is it worth storing?** @@ -53,8 +53,9 @@ Run this decision tree after every substantive response: ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built-in: duplicates skipped, conflicts auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, `causal_candidates`. 2. **Link** (evaluate candidates from step 1 — use judgment, not mechanical rules): - Review `causal_candidates`: does a genuine cause-effect relationship exist? `causal_signal` is regex-based and prone to false positives — only link if the memories are truly causally related. - Review `semantic_candidates`: are these memories meaningfully related? High `similarity` alone is not sufficient — skip candidates that share keywords but discuss unrelated topics. diff --git a/internal/memory/setup/assets/openclaw/SKILL.md b/internal/memory/setup/assets/openclaw/SKILL.md index 2913ba97..c0574b49 100644 --- a/internal/memory/setup/assets/openclaw/SKILL.md +++ b/internal/memory/setup/assets/openclaw/SKILL.md @@ -84,8 +84,9 @@ mnemon setup --eject --target openclaw --yes ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built-in: duplicates skipped, conflicts auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, `causal_candidates`. 2. **Link** (evaluate candidates from step 1 — use judgment, not mechanical rules): - Review `causal_candidates`: does a genuine cause-effect relationship exist? `causal_signal` is regex-based and prone to false positives — only link if the memories are truly causally related. - Review `semantic_candidates`: are these memories meaningfully related? High `similarity` alone is not sufficient — skip candidates that share keywords but discuss unrelated topics. diff --git a/internal/memory/setup/assets/opencode/SKILL.md b/internal/memory/setup/assets/opencode/SKILL.md index f32ac2a9..3886fe16 100644 --- a/internal/memory/setup/assets/opencode/SKILL.md +++ b/internal/memory/setup/assets/opencode/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for OpenCode. Store facts, recall past knowle ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/pi/SKILL.md b/internal/memory/setup/assets/pi/SKILL.md index c6a0d03c..43dd28e0 100644 --- a/internal/memory/setup/assets/pi/SKILL.md +++ b/internal/memory/setup/assets/pi/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/qoder/SKILL.md b/internal/memory/setup/assets/qoder/SKILL.md index 2dba308d..5a053601 100644 --- a/internal/memory/setup/assets/qoder/SKILL.md +++ b/internal/memory/setup/assets/qoder/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/qoderwork/SKILL.md b/internal/memory/setup/assets/qoderwork/SKILL.md index d283a4c8..63772406 100644 --- a/internal/memory/setup/assets/qoderwork/SKILL.md +++ b/internal/memory/setup/assets/qoderwork/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for QoderWork. Store facts, recall past knowl ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/trae/SKILL.md b/internal/memory/setup/assets/trae/SKILL.md index 2dba308d..5a053601 100644 --- a/internal/memory/setup/assets/trae/SKILL.md +++ b/internal/memory/setup/assets/trae/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/workbuddy/SKILL.md b/internal/memory/setup/assets/workbuddy/SKILL.md index e0988b4d..6bd3f8f6 100644 --- a/internal/memory/setup/assets/workbuddy/SKILL.md +++ b/internal/memory/setup/assets/workbuddy/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for WorkBuddy. Store facts, recall past knowl ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/zcode/SKILL.md b/internal/memory/setup/assets/zcode/SKILL.md index b290e9c4..d413bd58 100644 --- a/internal/memory/setup/assets/zcode/SKILL.md +++ b/internal/memory/setup/assets/zcode/SKILL.md @@ -8,8 +8,9 @@ description: Use persistent memory when prior preferences, decisions, constraint ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/scripts/e2e_test.sh b/scripts/e2e_test.sh index a087c2aa..dc17a898 100755 --- a/scripts/e2e_test.sh +++ b/scripts/e2e_test.sh @@ -294,6 +294,67 @@ OUT=$($M --data-dir "$TESTDIR" status) assert_jq "total now 0" "$OUT" '.total_insights' '0' assert_jq "deleted now 1" "$OUT" '.deleted_insights' '1' +step "remember diff — distinct subjects retain independent facts without embeddings" +DIFF_DIR="$TESTDATA/distinct-facts" +diff_cli() { + MNEMON_EMBED_ENDPOINT="http://127.0.0.1:1" MNEMON_MAX_INSIGHTS=1000 \ + "$M" --data-dir "$DIFF_DIR" --store default "$@" +} +FACT_ALPHA="Project Alpha uses PostgreSQL database for persistent application storage" +FACT_BETA="Project Beta uses PostgreSQL database for persistent application storage" +ALPHA_OUT=$(diff_cli remember "$FACT_ALPHA" --cat fact --imp 5) +BETA_OUT=$(diff_cli remember "$FACT_BETA" --cat fact --imp 5) +ALPHA_ID=$(extract_id "$ALPHA_OUT") +BETA_ID=$(extract_id "$BETA_OUT") +assert_jq "first fact is added" "$ALPHA_OUT" '.action' 'added' +assert_jq "distinct subject is added" "$BETA_OUT" '.action' 'added' +assert_jq "UPDATE remains advisory" "$BETA_OUT" '.diff_suggestion' 'UPDATE' +assert_jq "distinct fact has no replaced id" "$BETA_OUT" 'has("replaced_id")' 'false' +assert_jq "embedding endpoint is unavailable" "$BETA_OUT" '.embedded' 'false' +assert_jq "Alpha remains addressable" "$(diff_cli show "$ALPHA_ID")" '.content' "$FACT_ALPHA" +assert_jq "Beta remains addressable" "$(diff_cli show "$BETA_ID")" '.content' "$FACT_BETA" +OUT=$(diff_cli recall "Project Alpha storage") +assert_contains "recall retains Alpha" "$OUT" "$FACT_ALPHA" +assert_contains "recall retains Beta" "$OUT" "$FACT_BETA" +OUT=$(diff_cli status) +assert_jq "both facts are active" "$OUT" '.total_insights' '2' +assert_jq "no fact was soft-deleted" "$OUT" '.deleted_insights' '0' + +step "remember diff — exact repeats skip, changed values need deliberate forgetting" +OUT=$(diff_cli remember "$FACT_BETA" --cat fact --imp 5) +assert_jq "exact repeat skips" "$OUT" '.action' 'skipped' +assert_jq "repeat identifies existing Beta" "$OUT" '.replaced_id' "$BETA_ID" +OUT=$(diff_cli remember "Project Alpha uses SQLite database for persistent application storage" --cat fact --imp 5) +CORRECTED_ID=$(extract_id "$OUT") +assert_jq "changed value is added" "$OUT" '.action' 'added' +assert_jq "all three facts are active" "$(diff_cli status)" '.total_insights' '3' +assert_jq "explicit forget deletes selected Alpha" "$(diff_cli forget "$ALPHA_ID")" '.status' 'deleted' +assert_jq "corrected fact remains addressable" "$(diff_cli show "$CORRECTED_ID")" '.id' "$CORRECTED_ID" +assert_jq "unrelated Beta remains addressable" "$(diff_cli show "$BETA_ID")" '.id' "$BETA_ID" + +step "remember diff — near duplicate subjects are both stored" +DIFF_DIR="$TESTDATA/distinct-long-facts" +FACT_DETAILS=" with indexed customer records, transaction history, audit events, replication, backups, failover, monitoring, access controls, migrations, connection pooling, and disaster recovery" +diff_cli remember "$FACT_ALPHA$FACT_DETAILS" --cat fact --imp 5 > /dev/null +OUT=$(diff_cli remember "$FACT_BETA$FACT_DETAILS" --cat fact --imp 5) +assert_jq "near duplicate is added" "$OUT" '.action' 'added' +assert_jq "heuristic duplicate remains advisory" "$OUT" '.diff_suggestion' 'DUPLICATE' +assert_jq "both long facts are active" "$(diff_cli status)" '.total_insights' '2' + +step "import diff — retain distinct subjects and reuse exact duplicate ids" +DIFF_DIR="$TESTDATA/distinct-import-facts" +jq -n --arg alpha "$FACT_ALPHA" --arg beta "$FACT_BETA" '{schema_version: "1", insights: [ + {content: $alpha, category: "fact", importance: 5}, + {content: $beta, category: "fact", importance: 5}, + {content: $alpha, category: "fact", importance: 5} +]}' > "$TESTDATA/distinct-import.json" +OUT=$(diff_cli import "$TESTDATA/distinct-import.json") +assert_jq "import adds both subjects" "$OUT" '.imported' '2' +assert_jq "import never auto-replaces" "$OUT" '.updated' '0' +assert_jq "import skips one exact repeat" "$OUT" '.skipped' '1' +assert_jq "import preserves exact duplicate index mapping" "$OUT" '.results[0].id == .results[2].id' 'true' +assert_jq "both imported facts remain active" "$(diff_cli status)" '.total_insights' '2' + # ══════════════════════════════════════════════════════════════════════ banner "Milestone 2: Graph Edge Auto-Generation" # ══════════════════════════════════════════════════════════════════════