diff --git a/README.md b/README.md index 28df4225..052d21de 100644 --- a/README.md +++ b/README.md @@ -370,7 +370,7 @@ memory is useful. - **Four-graph architecture** — temporal, entity, causal, and semantic edges, not just vector similarity - **Intent-native protocol** — three primitives (`remember`, `link`, `recall`) map to the LLM's cognitive vocabulary, not database syntax; structured JSON output with signal transparency - **Intent-aware recall** — graph traversal + optional vector search (RRF fusion), enabled by default for all queries -- **Built-in deduplication** — `remember` auto-detects duplicates and conflicts; skips or auto-replaces +- **Built-in deduplication** — `remember` and `import` skip exact content repeats and preserve distinct facts; similarity suggestions guide review - **Retention lifecycle** — importance decay, access-count boosting, and garbage collection - **Privacy-safe receipts** — export hashed operation receipts for memory-boundary audits without raw memory contents or queries - **Optional embeddings** — works fully without an embedding provider; add local [Ollama](https://ollama.ai) or an OpenAI-compatible server for enhanced vector+keyword hybrid search diff --git a/cmd/memory/import.go b/cmd/memory/import.go index a20923fc..e684b006 100644 --- a/cmd/memory/import.go +++ b/cmd/memory/import.go @@ -56,7 +56,7 @@ exports are documented in docs/IMPORT.md.`, ec := embed.NewClientWithModel(resolveEmbedModel()) - // Build embed cache once for all diff and graph operations. + // Build embed cache once for all graph operations. var embedCache graph.EmbedCache if ec.Available() { dbEmbeds, err := db.GetAllEmbeddings() @@ -127,62 +127,30 @@ exports are documented in docs/IMPORT.md.`, } } - var action string - var replacedID string + action := "added" + var duplicateID string - if importNoDiff { - action = "added" - } else { + if !importNoDiff { allInsights, err := db.GetAllActiveInsights() if err != nil { results = append(results, importResult{Index: idx, ID: insight.ID, Content: insight.Content, Error: err.Error()}) continue } - opts := search.DiffOptions{Limit: 5, NewEmbedding: embeddingVec} - if embedCache != nil { - opts.ExistingEmbed = make([]search.EmbeddedItem, 0, len(embedCache)) - for id, v := range embedCache { - opts.ExistingEmbed = append(opts.ExistingEmbed, search.EmbeddedItem{ID: id, Embedding: v}) - } - } - result := search.Diff(allInsights, insight.Content, opts) - switch result.Suggestion { - case search.DiffDuplicate: + duplicateID = search.FindExactDuplicateID(allInsights, insight.Content) + if duplicateID != "" { action = "skipped" - if len(result.Matches) > 0 { - replacedID = result.Matches[0].ID - } - case search.DiffConflict, search.DiffUpdate: - action = "updated" - if len(result.Matches) > 0 { - replacedID = result.Matches[0].ID - } - default: - action = "added" } } if action == "skipped" { - db.LogOp("import-skip", insight.ID, fmt.Sprintf("duplicate of %s", replacedID)) - if replacedID != "" { - imported[idx] = replacedID - } else { - imported[idx] = insight.ID - } + db.LogOp("import-skip", insight.ID, fmt.Sprintf("duplicate of %s", duplicateID)) + imported[idx] = duplicateID results = append(results, importResult{Index: idx, ID: imported[idx], Content: insight.Content, Action: action}) continue } var writeErr error err = db.InTransaction(func() error { - if action == "updated" && replacedID != "" { - if err := db.SoftDeleteInsight(replacedID); err != nil { - fmt.Fprintf(os.Stderr, "warning: soft-delete %s: %v\n", replacedID, err) - } else { - db.LogOp("import-replace", replacedID, fmt.Sprintf("replaced by %s", insight.ID)) - delete(embedCache, replacedID) - } - } if err := db.InsertInsight(insight); err != nil { return fmt.Errorf("insert insight: %w", err) } diff --git a/cmd/memory/import_diff_test.go b/cmd/memory/import_diff_test.go new file mode 100644 index 00000000..2d34a51b --- /dev/null +++ b/cmd/memory/import_diff_test.go @@ -0,0 +1,157 @@ +package memory + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/mnemon-dev/mnemon/internal/memory/importdraft" + "github.com/mnemon-dev/mnemon/internal/memory/model" + "github.com/mnemon-dev/mnemon/internal/memory/store" +) + +func configureImportDiffTest(t *testing.T) { + t.Helper() + configureRememberDiffTest(t) + oldNoDiff, oldDryRun := importNoDiff, importDryRun + t.Cleanup(func() { importNoDiff, importDryRun = oldNoDiff, oldDryRun }) + importNoDiff, importDryRun = false, false +} + +type importDiffOutput struct { + Imported int `json:"imported"` + Updated int `json:"updated"` + Skipped int `json:"skipped"` + Errors int `json:"errors"` + Results []struct { + Index int `json:"index"` + ID string `json:"id"` + Content string `json:"content"` + Action string `json:"action"` + } `json:"results"` +} + +func importForDiffTest(t *testing.T, contents []string, edges []importdraft.DraftEdge) importDiffOutput { + t.Helper() + draft := importdraft.MemoryDraft{SchemaVersion: "1", Edges: edges} + for _, content := range contents { + draft.Insights = append(draft.Insights, importdraft.DraftInsight{ + Content: content, Category: "fact", Importance: 5, + }) + } + data, err := json.Marshal(draft) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "draft.json") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + var runErr error + out := captureStdout(t, func() { runErr = importCmd.RunE(importCmd, []string{path}) }) + if runErr != nil { + t.Fatalf("import: %v", runErr) + } + var result importDiffOutput + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("decode import output: %v\n%s", err, out) + } + if result.Errors != 0 || len(result.Results) != len(contents) { + t.Fatalf("incomplete import: %+v", result) + } + return result +} + +func TestImportPreservesDistinctContent(t *testing.T) { + const alpha = "Project Alpha uses PostgreSQL database for persistent application storage" + const details = " with indexed customer records, transaction history, audit events, replication, backups, failover, monitoring, access controls, migrations, connection pooling, and disaster recovery" + tests := []struct{ name, first, second string }{ + {"different subject", alpha, "Project Beta uses PostgreSQL database for persistent application storage"}, + {"changed value", alpha, "Project Alpha uses SQLite database for persistent application storage"}, + {"near duplicate", alpha + details, "Project Beta uses PostgreSQL database for persistent application storage" + details}, + {"conflict", alpha, "Project Alpha no longer uses PostgreSQL database for persistent application storage"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + configureImportDiffTest(t) + result := importForDiffTest(t, []string{tt.first, tt.second}, nil) + if result.Imported != 2 || result.Updated != 0 || result.Skipped != 0 { + t.Errorf("import = %+v, want two added insights", result) + } + assertActiveRememberContents(t, map[string]string{ + result.Results[0].ID: tt.first, result.Results[1].ID: tt.second, + }) + }) + } +} + +func TestImportPreservesExistingFact(t *testing.T) { + configureImportDiffTest(t) + const alpha = "Project Alpha uses PostgreSQL database for persistent application storage" + const beta = "Project Beta uses PostgreSQL database for persistent application storage" + first := rememberForDiffTest(t, alpha) + result := importForDiffTest(t, []string{beta}, nil) + if result.Imported != 1 || result.Updated != 0 || result.Skipped != 0 { + t.Errorf("import = %+v, want one added insight", result) + } + assertActiveRememberContents(t, map[string]string{first.ID: alpha, result.Results[0].ID: beta}) +} + +func TestImportExactDuplicateRetainsIndexAndEdgeMapping(t *testing.T) { + configureImportDiffTest(t) + const content = "Project Alpha uses PostgreSQL database for persistent application storage" + remImportance = 1 + first := rememberForDiffTest(t, content) + want := map[string]string{first.ID: content} + remImportance, remNoDiff = 5, true + for _, detail := range []string{"backups", "replication", "indexes", "migrations", "monitoring", "transactions"} { + extended := content + " with " + detail + result := rememberForDiffTest(t, extended) + want[result.ID] = extended + } + const linked = "Zebra migration survey notes" + result := importForDiffTest(t, []string{content, linked, linked}, []importdraft.DraftEdge{ + {SourceIndex: 0, TargetIndex: 2, EdgeType: "semantic", Weight: 0.75, Reason: "exact duplicate index mapping"}, + }) + if result.Imported != 1 || result.Updated != 0 || result.Skipped != 2 { + t.Errorf("import = %+v, want one added insight and two exact duplicates", result) + } + if result.Results[0].ID != first.ID || result.Results[0].Action != "skipped" { + t.Errorf("existing duplicate = %+v, want skipped %s", result.Results[0], first.ID) + } + linkedID := result.Results[1].ID + if result.Results[2].ID != linkedID || result.Results[2].Action != "skipped" { + t.Errorf("batch duplicate = %+v, want skipped %s", result.Results[2], linkedID) + } + want[linkedID] = linked + assertActiveRememberContents(t, want) + db, err := store.Open(store.StoreDir(dataDir, store.DefaultStoreName)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + edges, err := db.GetEdgesBySourceAndType(first.ID, model.EdgeSemantic) + if err != nil { + t.Fatal(err) + } + for _, edge := range edges { + if edge.TargetID == linkedID && edge.Metadata["reason"] == "exact duplicate index mapping" { + return + } + } + t.Fatalf("explicit edge from existing duplicate %s to batch duplicate %s is missing", first.ID, linkedID) +} + +func TestImportNoDiffStoresExactRepeats(t *testing.T) { + configureImportDiffTest(t) + importNoDiff = true + const content = "Project Alpha uses PostgreSQL database for persistent application storage" + result := importForDiffTest(t, []string{content, content}, nil) + if result.Imported != 2 || result.Updated != 0 || result.Skipped != 0 { + t.Errorf("import with --no-diff = %+v, want two added insights", result) + } + assertActiveRememberContents(t, map[string]string{ + result.Results[0].ID: content, result.Results[1].ID: content, + }) +} diff --git a/cmd/memory/remember.go b/cmd/memory/remember.go index 5f565757..0768f38a 100644 --- a/cmd/memory/remember.go +++ b/cmd/memory/remember.go @@ -118,9 +118,9 @@ var rememberCmd = &cobra.Command{ } // 2. Built-in diff: check for duplicates/conflicts (read-only, before transaction) - var diffAction string // "added", "updated", "skipped" - var replacedID string - var diffSuggestion search.DiffSuggestion + diffAction := "added" + diffSuggestion := search.DiffAdd + var duplicateID string // Build embed cache once — reused by diff, engine, and semantic candidates. var embedCache graph.EmbedCache @@ -136,10 +136,7 @@ var rememberCmd = &cobra.Command{ } } - if remNoDiff { - diffAction = "added" - diffSuggestion = search.DiffAdd - } else { + if !remNoDiff { allInsights, err := db.GetAllActiveInsights() if err != nil { return fmt.Errorf("load insights for diff: %w", err) @@ -159,45 +156,25 @@ var rememberCmd = &cobra.Command{ result := search.Diff(allInsights, content, opts) diffSuggestion = result.Suggestion - switch result.Suggestion { - case search.DiffDuplicate: + // Similarity cannot establish whether two facts have the same + // subject, value, or relationship. Keep diff suggestions advisory; + // only byte-identical content permits skipping a write. + duplicateID = search.FindExactDuplicateID(allInsights, content) + if duplicateID != "" { diffAction = "skipped" - if len(result.Matches) > 0 { - replacedID = result.Matches[0].ID - } - case search.DiffConflict: - // A CONFLICT means the two texts appear to disagree. Silently - // soft-deleting one side is destructive and has repeatedly - // clobbered unrelated same-domain memories (long technical - // notes share vocabulary at >=0.7 similarity, and change-log - // words like "replaced"/"no longer" appear in almost all of - // them). Keep both; the caller sees diff_suggestion=CONFLICT - // and can merge or delete deliberately. - diffAction = "added" - case search.DiffUpdate: - // Only auto-replace when the texts overlap heavily by TOKENS. - // Cosine similarity alone (same-domain embeddings cluster at - // 0.85+) is not enough evidence to destroy an existing memory. - if len(result.Matches) > 0 && result.Matches[0].TokenSimilarity >= 0.6 { - diffAction = "updated" - replacedID = result.Matches[0].ID - } else { - diffAction = "added" - } - default: - diffAction = "added" + diffSuggestion = search.DiffDuplicate } } // If duplicate, skip insert entirely if diffAction == "skipped" { - db.LogOp("diff-skip", insight.ID, fmt.Sprintf("duplicate of %s", replacedID)) + db.LogOp("diff-skip", insight.ID, fmt.Sprintf("duplicate of %s", duplicateID)) output := map[string]interface{}{ "id": insight.ID, "content": content, "action": "skipped", "diff_suggestion": string(diffSuggestion), - "replaced_id": replacedID, + "replaced_id": duplicateID, } enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") @@ -212,18 +189,6 @@ var rememberCmd = &cobra.Command{ embedded bool ) err = db.InTransaction(func() error { - // Soft-delete old insight if updating - if diffAction == "updated" && replacedID != "" { - if err := db.SoftDeleteInsight(replacedID); err != nil { - fmt.Fprintf(os.Stderr, "warning: soft-delete %s: %v\n", replacedID, err) - } else { - db.LogOp("diff-replace", replacedID, fmt.Sprintf("replaced by %s", insight.ID)) - // Remove deleted insight from embed cache to prevent - // creating edges to a soft-deleted node. - delete(embedCache, replacedID) - } - } - if err := db.InsertInsight(insight); err != nil { return fmt.Errorf("insert insight: %w", err) } @@ -268,7 +233,7 @@ var rememberCmd = &cobra.Command{ return nil }) if err != nil { - // Cache was mutated inside the transaction closure (delete/add entries). + // Cache was mutated inside the transaction closure (added entries). // On rollback those mutations don't match DB state, so discard the cache // to prevent any future code from accidentally using stale data. embedCache = nil @@ -306,9 +271,6 @@ var rememberCmd = &cobra.Command{ "auto_pruned": len(prunedIDs), "auto_pruned_ids": prunedIDs, } - if replacedID != "" { - output["replaced_id"] = replacedID - } enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") return enc.Encode(output) diff --git a/cmd/memory/remember_diff_test.go b/cmd/memory/remember_diff_test.go new file mode 100644 index 00000000..170bcdb2 --- /dev/null +++ b/cmd/memory/remember_diff_test.go @@ -0,0 +1,172 @@ +package memory + +import ( + "encoding/json" + "testing" + + "github.com/mnemon-dev/mnemon/internal/memory/search" + "github.com/mnemon-dev/mnemon/internal/memory/store" +) + +func configureRememberDiffTest(t *testing.T) { + t.Helper() + configureRememberTest(t) + storeName = store.DefaultStoreName + remNoDiff = false + remImportance = 5 + t.Setenv("MNEMON_MAX_INSIGHTS", "1000") +} + +type rememberDiffOutput struct { + ID string `json:"id"` + Action string `json:"action"` + DiffSuggestion search.DiffSuggestion `json:"diff_suggestion"` + ReplacedID string `json:"replaced_id"` +} + +func rememberForDiffTest(t *testing.T, content string) rememberDiffOutput { + t.Helper() + var runErr error + out := captureStdout(t, func() { + runErr = rememberCmd.RunE(rememberCmd, []string{content}) + }) + if runErr != nil { + t.Fatalf("remember: %v", runErr) + } + var result rememberDiffOutput + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("decode remember output: %v\n%s", err, out) + } + return result +} + +func assertActiveRememberContents(t *testing.T, want map[string]string) { + t.Helper() + db, err := store.Open(store.StoreDir(dataDir, store.DefaultStoreName)) + if err != nil { + t.Fatalf("reopen store: %v", err) + } + defer db.Close() + active, err := db.GetAllActiveInsights() + if err != nil { + t.Fatalf("get active insights: %v", err) + } + if len(active) != len(want) { + t.Errorf("active insight count = %d, want %d", len(active), len(want)) + } + for _, insight := range active { + if content, ok := want[insight.ID]; !ok || insight.Content != content { + t.Errorf("unexpected active insight %s: %q", insight.ID, insight.Content) + } + } +} + +func TestRememberPreservesDistinctContent(t *testing.T) { + const alpha = "Project Alpha uses PostgreSQL database for persistent application storage" + const details = " with indexed customer records, transaction history, audit events, replication, backups, failover, monitoring, access controls, migrations, connection pooling, and disaster recovery" + tests := []struct { + name string + first string + second string + suggestion search.DiffSuggestion + }{ + { + name: "different subject same property", + first: alpha, + second: "Project Beta uses PostgreSQL database for persistent application storage", + suggestion: search.DiffUpdate, + }, + { + name: "same subject changed value", + first: alpha, + second: "Project Alpha uses SQLite database for persistent application storage", + suggestion: search.DiffUpdate, + }, + { + name: "different subject above duplicate threshold", + first: alpha + details, + second: "Project Beta uses PostgreSQL database for persistent application storage" + details, + suggestion: search.DiffDuplicate, + }, + { + name: "same tokens different relationship", + first: "Project Alpha exports records to Project Beta", + second: "Project Beta exports records to Project Alpha", + suggestion: search.DiffDuplicate, + }, + { + name: "conflict remains advisory", + first: alpha, + second: "Project Alpha no longer uses PostgreSQL database for persistent application storage", + suggestion: search.DiffConflict, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + configureRememberDiffTest(t) + first := rememberForDiffTest(t, tt.first) + second := rememberForDiffTest(t, tt.second) + if first.Action != "added" || second.Action != "added" { + t.Errorf("actions = (%q, %q), want (added, added)", first.Action, second.Action) + } + if second.ReplacedID != "" { + t.Errorf("preserved content reports replaced_id = %q", second.ReplacedID) + } + if second.DiffSuggestion != tt.suggestion { + t.Errorf("diff_suggestion = %q, want %q", second.DiffSuggestion, tt.suggestion) + } + assertActiveRememberContents(t, map[string]string{first.ID: tt.first, second.ID: tt.second}) + }) + } +} + +func TestRememberSkipsExactDuplicate(t *testing.T) { + for _, content := range []string{ + "Project Alpha uses PostgreSQL database for persistent application storage", + "!!!", // Exact identity must not depend on searchable tokens. + } { + t.Run(content, func(t *testing.T) { + configureRememberDiffTest(t) + first := rememberForDiffTest(t, content) + repeat := rememberForDiffTest(t, content) + if repeat.Action != "skipped" || repeat.DiffSuggestion != search.DiffDuplicate || repeat.ReplacedID != first.ID { + t.Errorf("repeat = %+v, want skipped DUPLICATE of %s", repeat, first.ID) + } + assertActiveRememberContents(t, map[string]string{first.ID: content}) + }) + } +} + +func TestRememberExactDuplicateOutsideDiffCandidates(t *testing.T) { + configureRememberDiffTest(t) + const content = "Project Alpha uses PostgreSQL database for persistent application storage" + remImportance = 1 + first := rememberForDiffTest(t, content) + want := map[string]string{first.ID: content} + remImportance = 5 + remNoDiff = true + // Higher-importance keyword ties fill the five diff candidate slots. + for _, detail := range []string{"backups", "replication", "indexes", "migrations", "monitoring", "transactions"} { + extended := content + " with " + detail + result := rememberForDiffTest(t, extended) + want[result.ID] = extended + } + remNoDiff = false + repeat := rememberForDiffTest(t, content) + if repeat.Action != "skipped" || repeat.DiffSuggestion != search.DiffDuplicate || repeat.ReplacedID != first.ID { + t.Errorf("repeat = %+v, want skipped DUPLICATE of %s", repeat, first.ID) + } + assertActiveRememberContents(t, want) +} + +func TestRememberNoDiffStoresExactRepeat(t *testing.T) { + configureRememberDiffTest(t) + remNoDiff = true + const content = "Project Alpha uses PostgreSQL database for persistent application storage" + first := rememberForDiffTest(t, content) + repeat := rememberForDiffTest(t, content) + if repeat.Action != "added" || repeat.DiffSuggestion != search.DiffAdd || repeat.ReplacedID != "" { + t.Errorf("repeat with --no-diff = %+v, want added ADD without replaced_id", repeat) + } + assertActiveRememberContents(t, map[string]string{first.ID: content, repeat.ID: content}) +} diff --git a/docs/IMPORT.md b/docs/IMPORT.md index 024887c1..f45973b3 100644 --- a/docs/IMPORT.md +++ b/docs/IMPORT.md @@ -132,7 +132,7 @@ mnemon import memory_draft.json # Validate without writing mnemon import --dry-run memory_draft.json -# Skip duplicate/conflict detection and insert every entry as new +# Skip exact duplicate detection and insert every entry as new mnemon import --no-diff memory_draft.json # Import into a specific store @@ -141,10 +141,17 @@ mnemon import --store project-alpha memory_draft.json ### Output Example +Import uses the same exact-content deduplication rule as `remember`: only +byte-identical active content is skipped. Different facts and near-duplicates +are added, including conflicting or updated values. A skipped draft index maps +to the existing insight ID, so explicit edges still resolve to that memory. +To retire a superseded fact, verify the new memory and use `mnemon forget +` explicitly. Capacity-based auto-pruning remains separate. + ```json { "imported": 8, - "updated": 1, + "updated": 0, "skipped": 2, "errors": 0, "edges_inserted": 3, @@ -160,8 +167,8 @@ mnemon import --store project-alpha memory_draft.json | Field | Description | |---|---| | `imported` | Number of newly added memories | -| `updated` | Number of existing conflicting memories replaced | -| `skipped` | Number of duplicate memories skipped | +| `updated` | Always `0`; retained for output compatibility. Similarity does not replace existing memories | +| `skipped` | Number of byte-identical content repeats skipped | | `errors` | Number of failed writes. Import allows partial success; script callers should check this is `0` | | `edges_inserted` | Number of explicit edges inserted | | `auto_pruned` | Number of memories auto-pruned after capacity checks | diff --git a/docs/USAGE.md b/docs/USAGE.md index f7e7040d..98e7e2c3 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -99,7 +99,7 @@ mnemon setup --eject --target claude-code ### Core ```bash -# Remember — store a new insight (built-in diff: duplicates skipped, conflicts auto-replaced) +# Remember — store a new insight (exact repeats skipped; distinct content preserved) mnemon remember "Chose Qdrant over Milvus for vector search" \ --cat decision --imp 5 --entities "Qdrant,Milvus" --tags "architecture,search" --source agent @@ -138,6 +138,18 @@ mnemon import --no-diff memory_draft.json # skip deduplication mnemon forget ``` +`remember` and `import` skip only byte-identical content already present in an +active memory. Different subjects, changed values, reordered statements, and +near-duplicates are stored as new memories. `remember` still reports advisory +`diff_suggestion` values (`UPDATE`, `CONFLICT`, or `DUPLICATE`); read `action` to +see whether the write was `added` or `skipped`. On an exact repeat, the legacy +`replaced_id` field identifies the existing memory, which remains unchanged. +`--no-diff` also inserts exact repeats. + +To retire a superseded memory, store the new fact, verify it with `mnemon show +`, then explicitly run `mnemon forget `. Similarity alone never +authorizes replacement. Capacity-based auto-pruning still applies separately. + **Remember flags:** | Flag | Default | Description | diff --git a/docs/design/05-pipelines.md b/docs/design/05-pipelines.md index 996dec9f..519a9f26 100644 --- a/docs/design/05-pipelines.md +++ b/docs/design/05-pipelines.md @@ -8,8 +8,6 @@ `mnemon remember` is the core command for writing memories. It includes a built-in diff step that automatically detects duplicates and conflicts before storage. The write transaction executes atomically within a single SQLite transaction. -![Remember Pipeline](../diagrams/02-remember-pipeline.jpg) - ### Detailed Flow ``` @@ -29,18 +27,19 @@ mnemon remember "Chose Qdrant as the vector database" \ **Step 2.5: Built-in Diff (outside transaction, read-only)** -Compute similarity against all active insights: -- **DUPLICATE** (sim > 0.90) → skip insert entirely, return `action="skipped"` -- **CONFLICT/UPDATE** (sim 0.50–0.90) → soft-delete old insight, insert new as replacement -- **ADD** (sim < 0.50) → normal insert +Compute advisory similarity suggestions and check exact content against all active insights: +- **Byte-identical content** → skip insert, return `action="skipped"` and `diff_suggestion="DUPLICATE"` +- **Different content** → normal insert with `action="added"`, preserving the heuristic `diff_suggestion` for review -This step uses embedding cosine similarity when available, falling back to token overlap. The `--no-diff` flag disables this check. +Similarity uses embedding cosine when available, falling back to token overlap. +Neither similarity nor extracted entities establish that one fact supersedes +another. Exact-content lookup is independent of similarity candidate limits. +The `--no-diff` flag disables both checks and inserts even exact repeats. **Step 3: Atomic Transaction** ``` BEGIN TRANSACTION - ⓪ Soft-delete replaced insight (if diff found CONFLICT/UPDATE) ① INSERT insight (UUID, content, category, importance, tags, entities, source) ② UPDATE embedding (if vector is available) ③ Graph Engine: OnInsightCreated @@ -65,7 +64,6 @@ COMMIT "id": "abc-123", "action": "added", "diff_suggestion": "ADD", - "replaced_id": null, "edges_created": {"temporal": 2, "entity": 3, "causal": 1, "semantic": 1}, "semantic_candidates": [ {"id": "def-456", "content": "...", "cosine": 0.72, "auto_linked": false} @@ -80,7 +78,10 @@ COMMIT } ``` -The `action` field indicates what the built-in diff decided: `"added"` (new entry), `"replaced"` (conflict auto-replaced, `replaced_id` contains the old insight ID), or `"skipped"` (duplicate detected, no insert). +The `action` field is `"added"` (new entry) or `"skipped"` (byte-identical active +content, no insert). On a skipped repeat, the legacy `replaced_id` field names +the existing memory; it is not deleted or changed. A heuristic `UPDATE`, +`CONFLICT`, or near-`DUPLICATE` suggestion still produces `action="added"`. After receiving this output, the LLM can evaluate candidates and establish edges it considers appropriate via the `mnemon link` command. @@ -206,33 +207,45 @@ This is a unique innovation in Mnemon: **exposing the retrieval pipeline's inter ## 5.3 Deduplication & Conflict Detection: Diff -![Diff & Dedup Pipeline](../diagrams/07-diff-dedup-pipeline.jpg) +```mermaid +flowchart TD + Write[remember / import] --> Bypass{"--no-diff?"} + Bypass -->|Yes| Add[Insert new memory; preserve existing facts] + Bypass -->|No| Exact{Byte-identical active content?} + Exact -->|Yes| Skip[Skip insert; identify existing memory] + Exact -->|No| Add + Write -. remember only .-> Advisory[Similarity suggestions for review] +``` Diff is **built into `remember`** — no separate call needed. When `mnemon remember` is invoked, it automatically runs a diff check before inserting. When `remember` is called, the built-in diff runs before the transaction: 1. Compute similarity against all active insights (embedding cosine when available, token overlap as fallback) -2. Determine the action based on similarity thresholds: +2. Independently scan all active insights for byte-identical content. Similarity + suggestions describe possible relationships; exact equality alone permits a skip: -| Similarity | Action | Behavior | -|------------|--------|----------| -| > 0.90 | **DUPLICATE** | Skip insert entirely, return `action="skipped"` | -| 0.50 ~ 0.90 | **CONFLICT/UPDATE** | Soft-delete old insight, insert new as replacement | -| < 0.50 | **ADD** | Normal insert | +| Content | Diff suggestion | Behavior | +|---------|-----------------|----------| +| Byte-identical to an active memory | **DUPLICATE** | Skip insert, return `action="skipped"` | +| Different at any similarity | **ADD**, **UPDATE**, **CONFLICT**, or **DUPLICATE** | Insert with `action="added"`; retain the existing memory | -The `--no-diff` flag disables this check for cases where the caller wants unconditional insertion. +`import` uses the same exact-content lookup and preserves distinct content. +The `--no-diff` flag allows unconditional insertion on either command. +Capacity-based auto-pruning remains a separate lifecycle policy. ### Typical Workflow -A single `remember` call handles everything: +Store new content, then retire a specific old fact only when appropriate: ```bash -# Single command — diff is automatic +# Diff suggestions are automatic and advisory mnemon remember "Chose PostgreSQL to replace SQLite as the primary database" \ --cat decision --imp 5 --source agent -# → If conflict with existing "Chose SQLite as storage": -# auto-replaces old insight, returns action="replaced", replaced_id="" -# → If duplicate: returns action="skipped" -# → If new: returns action="added" +# → Exact repeat: action="skipped"; existing memory unchanged +# → Different content: action="added"; existing memories retained + +# If the new fact supersedes a specific old memory, verify before retiring it +mnemon show +mnemon forget ``` diff --git a/docs/zh/IMPORT.md b/docs/zh/IMPORT.md index 91a36d8a..63cdd84c 100644 --- a/docs/zh/IMPORT.md +++ b/docs/zh/IMPORT.md @@ -140,10 +140,16 @@ mnemon import --store project-alpha memory_draft.json ### 输出示例 +导入与 `remember` 使用相同的精确内容去重规则:仅跳过与活跃记忆逐字节 +完全相同的内容。不同事实、近似重复、冲突和变化后的属性值都会新增。 +被跳过的草稿索引映射到已有 insight ID,显式边仍可引用该记忆。 +如需淘汰已被取代的事实,请验证新记忆后显式执行 `mnemon forget `。 +基于容量的自动清理仍独立生效。 + ```json { "imported": 8, - "updated": 1, + "updated": 0, "skipped": 2, "errors": 0, "edges_inserted": 3, @@ -159,8 +165,8 @@ mnemon import --store project-alpha memory_draft.json | 字段 | 说明 | |---|---| | `imported` | 新增的记忆数量 | -| `updated` | 替换了已有冲突记忆的数量 | -| `skipped` | 检测为重复而跳过的数量 | +| `updated` | 固定为 `0`,保留此字段以兼容已有输出;相似度不会触发替换 | +| `skipped` | 因内容逐字节完全相同而跳过的数量 | | `errors` | 写入失败的数量;导入允许部分成功,脚本调用方应检查此字段是否为 0 | | `edges_inserted` | 成功插入的显式边数量 | | `auto_pruned` | 超出容量限制后自动删除的记忆数量 | diff --git a/docs/zh/README.md b/docs/zh/README.md index 9bb14eef..a42b0a03 100644 --- a/docs/zh/README.md +++ b/docs/zh/README.md @@ -329,7 +329,7 @@ store 可见。**Remind** 触发 recall 判断。**Nudge** 触发 writeback 判 - **四图架构** — 时序、实体、因果、语义四种边,不仅仅是向量相似度 - **意图原生协议** — 三个原语(`remember`、`link`、`recall`)映射到 LLM 的认知词汇而非数据库语法;结构化 JSON 输出,带信号透明度 - **意图感知召回** — 图遍历 + 可选向量搜索(RRF 融合),所有查询默认启用 -- **内置去重** — `remember` 自动检测重复和冲突;跳过或自动替换 +- **内置去重** — `remember` 和 `import` 仅跳过内容完全相同的记忆,保留不同事实;相似度建议供复核参考 - **保留度生命周期** — 重要性衰减、访问计数提升、免疫规则、垃圾回收 - **可选嵌入向量** — 可使用本地 [Ollama](https://ollama.ai) 或 OpenAI 兼容服务器,支持混合向量+关键词搜索 diff --git a/docs/zh/USAGE.md b/docs/zh/USAGE.md index 83bd5f9d..cdd21df6 100644 --- a/docs/zh/USAGE.md +++ b/docs/zh/USAGE.md @@ -94,7 +94,7 @@ mnemon setup --eject --target claude-code ### 核心命令 ```bash -# Remember — 存储新洞察(内置 diff:重复跳过,冲突自动替换) +# Remember — 存储新洞察(仅跳过内容完全相同的记忆,保留不同内容) mnemon remember "选择 Qdrant 而非 Milvus 做向量搜索" \ --cat decision --imp 5 --entities "Qdrant,Milvus" --tags "architecture,search" --source agent @@ -133,6 +133,17 @@ mnemon import --no-diff memory_draft.json # 跳过去重 mnemon forget ``` +`remember` 和 `import` 仅跳过与活跃记忆逐字节完全相同的内容。不同主体、 +变化后的属性值、调整语序的陈述和近似重复内容都会作为新记忆保存。 +`remember` 仍会返回建议性的 `diff_suggestion`(`UPDATE`、`CONFLICT` 或 +`DUPLICATE`);实际写入结果以 `action` 的 `added` 或 `skipped` 为准。 +完全重复时,兼容字段 `replaced_id` 指向保持不变的已有记忆。 +使用 `--no-diff` 则连完全重复的内容也会插入。 + +如需淘汰已被取代的记忆,先保存新事实,用 `mnemon show ` 验证, +再显式执行 `mnemon forget `。相似度本身不会授权替换; +基于容量的自动清理仍独立生效。 + **Remember 标志:** | 标志 | 默认值 | 说明 | diff --git a/docs/zh/design/05-pipelines.md b/docs/zh/design/05-pipelines.md index 06d591f1..908e3ece 100644 --- a/docs/zh/design/05-pipelines.md +++ b/docs/zh/design/05-pipelines.md @@ -6,8 +6,6 @@ `mnemon remember` 是写入记忆的核心命令。它包含内置的 diff 步骤,在存储前自动检测重复和冲突。写入事务在一个 SQLite 事务中原子执行。 -![Remember Pipeline](../../diagrams/02-remember-pipeline.jpg) - ### 流程详解 ``` @@ -27,18 +25,19 @@ mnemon remember "选择 Qdrant 作为向量数据库" \ **第 2.5 步:内置 Diff(事务外,只读)** -对所有活跃 insight 计算相似度: -- **DUPLICATE**(sim > 0.90)→ 跳过插入,返回 `action="skipped"` -- **CONFLICT/UPDATE**(sim 0.50–0.90)→ 软删除旧 insight,插入新的替换 -- **ADD**(sim < 0.50)→ 正常插入 +生成相似度建议,并与所有活跃 insight 进行精确内容比较: +- **内容逐字节完全相同** → 跳过插入,返回 `action="skipped"` 和 `diff_suggestion="DUPLICATE"` +- **内容不同** → 正常插入,返回 `action="added"`,保留启发式 `diff_suggestion` 供复核 -此步骤在有嵌入时使用余弦相似度,否则降级为 token 重叠。`--no-diff` 标志可禁用此检查。 +相似度在有嵌入时使用余弦计算,否则使用 token 重叠。 +相似度与自动抽取的实体都无法确定一个事实是否取代另一个。 +精确内容查找不受相似度候选数量限制。 +`--no-diff` 会禁用两项检查,连完全重复的内容也会插入。 **第三步:原子事务** ``` BEGIN TRANSACTION - ⓪ 软删除被替换的 insight(如果 diff 检测到 CONFLICT/UPDATE) ① INSERT insight(UUID, content, category, importance, tags, entities, source) ② UPDATE embedding(如果有向量) ③ Graph Engine: OnInsightCreated @@ -63,7 +62,6 @@ COMMIT "id": "abc-123", "action": "added", "diff_suggestion": "ADD", - "replaced_id": null, "edges_created": {"temporal": 2, "entity": 3, "causal": 1, "semantic": 1}, "semantic_candidates": [ {"id": "def-456", "content": "...", "cosine": 0.72, "auto_linked": false} @@ -78,7 +76,9 @@ COMMIT } ``` -`action` 字段表示内置 diff 的决定:`"added"`(新增)、`"replaced"`(冲突自动替换,`replaced_id` 包含旧 insight ID)或 `"skipped"`(检测到重复,未插入)。 +`action` 为 `"added"`(新增)或 `"skipped"`(与活跃记忆内容逐字节完全相同, +未插入)。跳过时,兼容字段 `replaced_id` 指向保持不变的已有记忆。 +启发式 `UPDATE`、`CONFLICT` 或近似 `DUPLICATE` 建议仍会返回 `action="added"`。 LLM 收到这个输出后,可以评估候选并通过 `mnemon link` 命令建立它认为合理的边。 @@ -204,33 +204,45 @@ final = w_kw·keyword + w_ent·entity + w_sim·similarity + w_gr·graph ## 5.3 去重与冲突检测:Diff -![Diff & Dedup Pipeline](../../diagrams/07-diff-dedup-pipeline.jpg) +```mermaid +flowchart TD + Write[remember / import] --> Bypass{"--no-diff?"} + Bypass -->|是| Add[插入新记忆,保留已有事实] + Bypass -->|否| Exact{与活跃记忆逐字节完全相同?} + Exact -->|是| Skip[跳过插入,返回已有记忆的标识] + Exact -->|否| Add + Write -. 仅 remember .-> Advisory[相似度建议供复核参考] +``` Diff 已**内置于 `remember`** — 无需单独调用。当调用 `mnemon remember` 时,它会自动在插入前运行 diff 检查。 调用 `remember` 时,内置 diff 在事务之前运行: 1. 对所有活跃 insight 计算相似度(有嵌入时使用余弦相似度,否则使用 token 重叠) -2. 根据相似度阈值判断动作: +2. 独立扫描所有活跃 insight,检查逐字节相同的内容。相似度仅提示潜在关系, + 只有精确相等才允许跳过写入: -| 相似度 | 动作 | 行为 | -|--------|------|------| -| > 0.90 | **DUPLICATE** | 跳过插入,返回 `action="skipped"` | -| 0.50 ~ 0.90 | **CONFLICT/UPDATE** | 软删除旧 insight,插入新的替换 | -| < 0.50 | **ADD** | 正常插入 | +| 内容 | Diff 建议 | 行为 | +|------|-----------|------| +| 与活跃记忆逐字节完全相同 | **DUPLICATE** | 跳过插入,返回 `action="skipped"` | +| 内容不同,任意相似度 | **ADD**、**UPDATE**、**CONFLICT** 或 **DUPLICATE** | 返回 `action="added"`,保留已有记忆 | -`--no-diff` 标志可禁用此检查,用于需要无条件插入的场景。 +`import` 使用相同的精确内容查找规则,保留不同内容。 +两条命令均可通过 `--no-diff` 无条件插入。 +基于容量的自动清理仍是独立的生命周期策略。 ### 典型工作流 -一条 `remember` 命令即可处理一切: +先保存新内容,仅在确有需要时淘汰指定的旧事实: ```bash -# 单条命令 — diff 自动执行 +# Diff 自动执行,结果仅供参考 mnemon remember "选择 PostgreSQL 替代 SQLite 作为主数据库" \ --cat decision --imp 5 --source agent -# → 如果与已有的 "选择 SQLite 作为存储" 冲突: -# 自动替换旧 insight,返回 action="replaced", replaced_id="" -# → 如果重复:返回 action="skipped" -# → 如果是新内容:返回 action="added" +# → 完全重复:action="skipped",已有记忆保持不变 +# → 内容不同:action="added",保留已有记忆 + +# 如果新事实取代了指定旧记忆,先验证,再淘汰 +mnemon show +mnemon forget ``` diff --git a/internal/memory/search/duplicate.go b/internal/memory/search/duplicate.go new file mode 100644 index 00000000..7ffff4f2 --- /dev/null +++ b/internal/memory/search/duplicate.go @@ -0,0 +1,15 @@ +package search + +import "github.com/mnemon-dev/mnemon/internal/memory/model" + +// FindExactDuplicateID returns the ID of an insight with byte-identical content, +// or an empty string when none exists. Callers supply all active insights: fuzzy +// diff candidates and similarity scores cannot establish content identity. +func FindExactDuplicateID(insights []*model.Insight, content string) string { + for _, insight := range insights { + if insight.Content == content { + return insight.ID + } + } + return "" +} diff --git a/internal/memory/setup/assets/claude/SKILL.md b/internal/memory/setup/assets/claude/SKILL.md index 7395758e..807221e1 100644 --- a/internal/memory/setup/assets/claude/SKILL.md +++ b/internal/memory/setup/assets/claude/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built-in: duplicates skipped, conflicts auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, `causal_candidates`. 2. **Link** (evaluate candidates from step 1 — use judgment, not mechanical rules): - Review `causal_candidates`: does a genuine cause-effect relationship exist? `causal_signal` is regex-based and prone to false positives — only link if the memories are truly causally related. - Review `semantic_candidates`: are these memories meaningfully related? High `similarity` alone is not sufficient — skip candidates that share keywords but discuss unrelated topics. diff --git a/internal/memory/setup/assets/claude/guide.md b/internal/memory/setup/assets/claude/guide.md index 841f8d2e..625be1e7 100644 --- a/internal/memory/setup/assets/claude/guide.md +++ b/internal/memory/setup/assets/claude/guide.md @@ -37,8 +37,8 @@ Tier C (importance 1, store only if genuinely reusable): → None of the above → STOP. **Step 2 — Does a highly overlapping memory already exist?** -→ Yes, incremental new info → UPDATE (merge into existing) -→ Yes, but contradicts/supersedes → REPLACE +→ Yes, incremental new info → STORE the new content; review the existing memory separately +→ Yes, but contradicts/supersedes → STORE and verify the new fact; explicitly forget the old ID only if superseded → No significant overlap → CREATE **Step 3 — Importance calibration** diff --git a/internal/memory/setup/assets/codebuddy/SKILL.md b/internal/memory/setup/assets/codebuddy/SKILL.md index def7eb51..13cdb3b6 100644 --- a/internal/memory/setup/assets/codebuddy/SKILL.md +++ b/internal/memory/setup/assets/codebuddy/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for CodeBuddy. Store facts, recall past knowl ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/codex/SKILL.md b/internal/memory/setup/assets/codex/SKILL.md index c6a0d03c..43dd28e0 100644 --- a/internal/memory/setup/assets/codex/SKILL.md +++ b/internal/memory/setup/assets/codex/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/cursor/SKILL.md b/internal/memory/setup/assets/cursor/SKILL.md index 6080caa4..26b3f738 100644 --- a/internal/memory/setup/assets/cursor/SKILL.md +++ b/internal/memory/setup/assets/cursor/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/kimi/SKILL.md b/internal/memory/setup/assets/kimi/SKILL.md index 00f756cb..f34dd6d9 100644 --- a/internal/memory/setup/assets/kimi/SKILL.md +++ b/internal/memory/setup/assets/kimi/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for Kimi Code. Store facts, recall past knowl ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/nanobot/SKILL.md b/internal/memory/setup/assets/nanobot/SKILL.md index d23cd90c..8393b2bb 100644 --- a/internal/memory/setup/assets/nanobot/SKILL.md +++ b/internal/memory/setup/assets/nanobot/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built-in: duplicates skipped, conflicts auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, `causal_candidates`. 2. **Link** (evaluate candidates from step 1 — use judgment, not mechanical rules): - Review `causal_candidates`: does a genuine cause-effect relationship exist? `causal_signal` is regex-based and prone to false positives — only link if the memories are truly causally related. - Review `semantic_candidates`: are these memories meaningfully related? High `similarity` alone is not sufficient — skip candidates that share keywords but discuss unrelated topics. diff --git a/internal/memory/setup/assets/nanoclaw/container-skill.md b/internal/memory/setup/assets/nanoclaw/container-skill.md index 2f6b17ed..2967e512 100644 --- a/internal/memory/setup/assets/nanoclaw/container-skill.md +++ b/internal/memory/setup/assets/nanoclaw/container-skill.md @@ -37,8 +37,8 @@ Run this decision tree after every substantive response: → No to all → STOP. **Step 2 — Does a highly overlapping memory already exist?** - → Yes, incremental new info → UPDATE (merge into existing) - → Yes, but contradicts/supersedes → REPLACE + → Yes, incremental new info → STORE the new content; review the existing memory separately + → Yes, but contradicts/supersedes → STORE and verify the new fact; explicitly forget the old ID only if superseded → No significant overlap → CREATE **Step 3 — Is it worth storing?** @@ -53,8 +53,9 @@ Run this decision tree after every substantive response: ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built-in: duplicates skipped, conflicts auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, `causal_candidates`. 2. **Link** (evaluate candidates from step 1 — use judgment, not mechanical rules): - Review `causal_candidates`: does a genuine cause-effect relationship exist? `causal_signal` is regex-based and prone to false positives — only link if the memories are truly causally related. - Review `semantic_candidates`: are these memories meaningfully related? High `similarity` alone is not sufficient — skip candidates that share keywords but discuss unrelated topics. diff --git a/internal/memory/setup/assets/openclaw/SKILL.md b/internal/memory/setup/assets/openclaw/SKILL.md index 2913ba97..c0574b49 100644 --- a/internal/memory/setup/assets/openclaw/SKILL.md +++ b/internal/memory/setup/assets/openclaw/SKILL.md @@ -84,8 +84,9 @@ mnemon setup --eject --target openclaw --yes ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built-in: duplicates skipped, conflicts auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, `causal_candidates`. 2. **Link** (evaluate candidates from step 1 — use judgment, not mechanical rules): - Review `causal_candidates`: does a genuine cause-effect relationship exist? `causal_signal` is regex-based and prone to false positives — only link if the memories are truly causally related. - Review `semantic_candidates`: are these memories meaningfully related? High `similarity` alone is not sufficient — skip candidates that share keywords but discuss unrelated topics. diff --git a/internal/memory/setup/assets/opencode/SKILL.md b/internal/memory/setup/assets/opencode/SKILL.md index f32ac2a9..3886fe16 100644 --- a/internal/memory/setup/assets/opencode/SKILL.md +++ b/internal/memory/setup/assets/opencode/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for OpenCode. Store facts, recall past knowle ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/pi/SKILL.md b/internal/memory/setup/assets/pi/SKILL.md index c6a0d03c..43dd28e0 100644 --- a/internal/memory/setup/assets/pi/SKILL.md +++ b/internal/memory/setup/assets/pi/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/qoder/SKILL.md b/internal/memory/setup/assets/qoder/SKILL.md index 2dba308d..5a053601 100644 --- a/internal/memory/setup/assets/qoder/SKILL.md +++ b/internal/memory/setup/assets/qoder/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/qoderwork/SKILL.md b/internal/memory/setup/assets/qoderwork/SKILL.md index d283a4c8..63772406 100644 --- a/internal/memory/setup/assets/qoderwork/SKILL.md +++ b/internal/memory/setup/assets/qoderwork/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for QoderWork. Store facts, recall past knowl ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/trae/SKILL.md b/internal/memory/setup/assets/trae/SKILL.md index 2dba308d..5a053601 100644 --- a/internal/memory/setup/assets/trae/SKILL.md +++ b/internal/memory/setup/assets/trae/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for LLM agents. Store facts, recall past know ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/workbuddy/SKILL.md b/internal/memory/setup/assets/workbuddy/SKILL.md index e0988b4d..6bd3f8f6 100644 --- a/internal/memory/setup/assets/workbuddy/SKILL.md +++ b/internal/memory/setup/assets/workbuddy/SKILL.md @@ -8,8 +8,9 @@ description: Persistent memory CLI for WorkBuddy. Store facts, recall past knowl ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/internal/memory/setup/assets/zcode/SKILL.md b/internal/memory/setup/assets/zcode/SKILL.md index b290e9c4..d413bd58 100644 --- a/internal/memory/setup/assets/zcode/SKILL.md +++ b/internal/memory/setup/assets/zcode/SKILL.md @@ -8,8 +8,9 @@ description: Use persistent memory when prior preferences, decisions, constraint ## Workflow 1. **Remember**: `mnemon remember "" --cat --imp <1-5> --entities "e1,e2" --source agent` - - Diff is built in: duplicates are skipped, conflicts are auto-replaced. - - Output includes `action` (added/updated/skipped), `semantic_candidates`, and `causal_candidates`. + - Only exact content repeats are skipped; distinct content is stored and diff suggestions are advisory. + - To retire a superseded memory, store and verify the new fact, then explicitly run `mnemon forget `. + - Output includes `action` (added/skipped), `semantic_candidates`, and `causal_candidates`. 2. **Link** (evaluate candidates from step 1 using judgment): - Review `causal_candidates`: link only when the memories are genuinely causally related. - Review `semantic_candidates`: high `similarity` alone is not enough; skip unrelated keyword matches. diff --git a/scripts/e2e_test.sh b/scripts/e2e_test.sh index a087c2aa..dc17a898 100755 --- a/scripts/e2e_test.sh +++ b/scripts/e2e_test.sh @@ -294,6 +294,67 @@ OUT=$($M --data-dir "$TESTDIR" status) assert_jq "total now 0" "$OUT" '.total_insights' '0' assert_jq "deleted now 1" "$OUT" '.deleted_insights' '1' +step "remember diff — distinct subjects retain independent facts without embeddings" +DIFF_DIR="$TESTDATA/distinct-facts" +diff_cli() { + MNEMON_EMBED_ENDPOINT="http://127.0.0.1:1" MNEMON_MAX_INSIGHTS=1000 \ + "$M" --data-dir "$DIFF_DIR" --store default "$@" +} +FACT_ALPHA="Project Alpha uses PostgreSQL database for persistent application storage" +FACT_BETA="Project Beta uses PostgreSQL database for persistent application storage" +ALPHA_OUT=$(diff_cli remember "$FACT_ALPHA" --cat fact --imp 5) +BETA_OUT=$(diff_cli remember "$FACT_BETA" --cat fact --imp 5) +ALPHA_ID=$(extract_id "$ALPHA_OUT") +BETA_ID=$(extract_id "$BETA_OUT") +assert_jq "first fact is added" "$ALPHA_OUT" '.action' 'added' +assert_jq "distinct subject is added" "$BETA_OUT" '.action' 'added' +assert_jq "UPDATE remains advisory" "$BETA_OUT" '.diff_suggestion' 'UPDATE' +assert_jq "distinct fact has no replaced id" "$BETA_OUT" 'has("replaced_id")' 'false' +assert_jq "embedding endpoint is unavailable" "$BETA_OUT" '.embedded' 'false' +assert_jq "Alpha remains addressable" "$(diff_cli show "$ALPHA_ID")" '.content' "$FACT_ALPHA" +assert_jq "Beta remains addressable" "$(diff_cli show "$BETA_ID")" '.content' "$FACT_BETA" +OUT=$(diff_cli recall "Project Alpha storage") +assert_contains "recall retains Alpha" "$OUT" "$FACT_ALPHA" +assert_contains "recall retains Beta" "$OUT" "$FACT_BETA" +OUT=$(diff_cli status) +assert_jq "both facts are active" "$OUT" '.total_insights' '2' +assert_jq "no fact was soft-deleted" "$OUT" '.deleted_insights' '0' + +step "remember diff — exact repeats skip, changed values need deliberate forgetting" +OUT=$(diff_cli remember "$FACT_BETA" --cat fact --imp 5) +assert_jq "exact repeat skips" "$OUT" '.action' 'skipped' +assert_jq "repeat identifies existing Beta" "$OUT" '.replaced_id' "$BETA_ID" +OUT=$(diff_cli remember "Project Alpha uses SQLite database for persistent application storage" --cat fact --imp 5) +CORRECTED_ID=$(extract_id "$OUT") +assert_jq "changed value is added" "$OUT" '.action' 'added' +assert_jq "all three facts are active" "$(diff_cli status)" '.total_insights' '3' +assert_jq "explicit forget deletes selected Alpha" "$(diff_cli forget "$ALPHA_ID")" '.status' 'deleted' +assert_jq "corrected fact remains addressable" "$(diff_cli show "$CORRECTED_ID")" '.id' "$CORRECTED_ID" +assert_jq "unrelated Beta remains addressable" "$(diff_cli show "$BETA_ID")" '.id' "$BETA_ID" + +step "remember diff — near duplicate subjects are both stored" +DIFF_DIR="$TESTDATA/distinct-long-facts" +FACT_DETAILS=" with indexed customer records, transaction history, audit events, replication, backups, failover, monitoring, access controls, migrations, connection pooling, and disaster recovery" +diff_cli remember "$FACT_ALPHA$FACT_DETAILS" --cat fact --imp 5 > /dev/null +OUT=$(diff_cli remember "$FACT_BETA$FACT_DETAILS" --cat fact --imp 5) +assert_jq "near duplicate is added" "$OUT" '.action' 'added' +assert_jq "heuristic duplicate remains advisory" "$OUT" '.diff_suggestion' 'DUPLICATE' +assert_jq "both long facts are active" "$(diff_cli status)" '.total_insights' '2' + +step "import diff — retain distinct subjects and reuse exact duplicate ids" +DIFF_DIR="$TESTDATA/distinct-import-facts" +jq -n --arg alpha "$FACT_ALPHA" --arg beta "$FACT_BETA" '{schema_version: "1", insights: [ + {content: $alpha, category: "fact", importance: 5}, + {content: $beta, category: "fact", importance: 5}, + {content: $alpha, category: "fact", importance: 5} +]}' > "$TESTDATA/distinct-import.json" +OUT=$(diff_cli import "$TESTDATA/distinct-import.json") +assert_jq "import adds both subjects" "$OUT" '.imported' '2' +assert_jq "import never auto-replaces" "$OUT" '.updated' '0' +assert_jq "import skips one exact repeat" "$OUT" '.skipped' '1' +assert_jq "import preserves exact duplicate index mapping" "$OUT" '.results[0].id == .results[2].id' 'true' +assert_jq "both imported facts remain active" "$(diff_cli status)" '.total_insights' '2' + # ══════════════════════════════════════════════════════════════════════ banner "Milestone 2: Graph Edge Auto-Generation" # ══════════════════════════════════════════════════════════════════════