fix(memory): preserve distinct facts during remember and import

README.md

@@ -370,7 +370,7 @@ memory is useful.
 - **Four-graph architecture** — temporal, entity, causal, and semantic edges, not just vector similarity
 - **Intent-native protocol** — three primitives (`remember`, `link`, `recall`) map to the LLM's cognitive vocabulary, not database syntax; structured JSON output with signal transparency
 - **Intent-aware recall** — graph traversal + optional vector search (RRF fusion), enabled by default for all queries
-- **Built-in deduplication** — `remember` auto-detects duplicates and conflicts; skips or auto-replaces
+- **Built-in deduplication** — `remember` and `import` skip exact content repeats and preserve distinct facts; similarity suggestions guide review
 - **Retention lifecycle** — importance decay, access-count boosting, and garbage collection
 - **Privacy-safe receipts** — export hashed operation receipts for memory-boundary audits without raw memory contents or queries
 - **Optional embeddings** — works fully without an embedding provider; add local [Ollama](https://ollama.ai) or an OpenAI-compatible server for enhanced vector+keyword hybrid search

cmd/memory/import.go

@@ -56,7 +56,7 @@ exports are documented in docs/IMPORT.md.`,
 
 		ec := embed.NewClientWithModel(resolveEmbedModel())
 
-		// Build embed cache once for all diff and graph operations.
+		// Build embed cache once for all graph operations.
 		var embedCache graph.EmbedCache
 		if ec.Available() {
 			dbEmbeds, err := db.GetAllEmbeddings()
@@ -127,62 +127,30 @@ exports are documented in docs/IMPORT.md.`,
 				}
 			}
 
-			var action string
-			var replacedID string
+			action := "added"
+			var duplicateID string
 
-			if importNoDiff {
-				action = "added"
-			} else {
+			if !importNoDiff {
 				allInsights, err := db.GetAllActiveInsights()
 				if err != nil {
 					results = append(results, importResult{Index: idx, ID: insight.ID, Content: insight.Content, Error: err.Error()})
 					continue
 				}
-				opts := search.DiffOptions{Limit: 5, NewEmbedding: embeddingVec}
-				if embedCache != nil {
-					opts.ExistingEmbed = make([]search.EmbeddedItem, 0, len(embedCache))
-					for id, v := range embedCache {
-						opts.ExistingEmbed = append(opts.ExistingEmbed, search.EmbeddedItem{ID: id, Embedding: v})
-					}
-				}
-				result := search.Diff(allInsights, insight.Content, opts)
-				switch result.Suggestion {
-				case search.DiffDuplicate:
+				duplicateID = search.FindExactDuplicateID(allInsights, insight.Content)
+				if duplicateID != "" {
 					action = "skipped"
-					if len(result.Matches) > 0 {
-						replacedID = result.Matches[0].ID
-					}
-				case search.DiffConflict, search.DiffUpdate:
-					action = "updated"
-					if len(result.Matches) > 0 {
-						replacedID = result.Matches[0].ID
-					}
-				default:
-					action = "added"
 				}
 			}
 
 			if action == "skipped" {
-				db.LogOp("import-skip", insight.ID, fmt.Sprintf("duplicate of %s", replacedID))
-				if replacedID != "" {
-					imported[idx] = replacedID
-				} else {
-					imported[idx] = insight.ID
-				}
+				db.LogOp("import-skip", insight.ID, fmt.Sprintf("duplicate of %s", duplicateID))
+				imported[idx] = duplicateID
 				results = append(results, importResult{Index: idx, ID: imported[idx], Content: insight.Content, Action: action})
 				continue
 			}
 
 			var writeErr error
 			err = db.InTransaction(func() error {
-				if action == "updated" && replacedID != "" {
-					if err := db.SoftDeleteInsight(replacedID); err != nil {
-						fmt.Fprintf(os.Stderr, "warning: soft-delete %s: %v\n", replacedID, err)
-					} else {
-						db.LogOp("import-replace", replacedID, fmt.Sprintf("replaced by %s", insight.ID))
-						delete(embedCache, replacedID)
-					}
-				}
 				if err := db.InsertInsight(insight); err != nil {
 					return fmt.Errorf("insert insight: %w", err)
 				}

cmd/memory/import_diff_test.go

@@ -0,0 +1,157 @@
+package memory
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/mnemon-dev/mnemon/internal/memory/importdraft"
+	"github.com/mnemon-dev/mnemon/internal/memory/model"
+	"github.com/mnemon-dev/mnemon/internal/memory/store"
+)
+
+func configureImportDiffTest(t *testing.T) {
+	t.Helper()
+	configureRememberDiffTest(t)
+	oldNoDiff, oldDryRun := importNoDiff, importDryRun
+	t.Cleanup(func() { importNoDiff, importDryRun = oldNoDiff, oldDryRun })
+	importNoDiff, importDryRun = false, false
+}
+
+type importDiffOutput struct {
+	Imported int `json:"imported"`
+	Updated  int `json:"updated"`
+	Skipped  int `json:"skipped"`
+	Errors   int `json:"errors"`
+	Results  []struct {
+		Index   int    `json:"index"`
+		ID      string `json:"id"`
+		Content string `json:"content"`
+		Action  string `json:"action"`
+	} `json:"results"`
+}
+
+func importForDiffTest(t *testing.T, contents []string, edges []importdraft.DraftEdge) importDiffOutput {
+	t.Helper()
+	draft := importdraft.MemoryDraft{SchemaVersion: "1", Edges: edges}
+	for _, content := range contents {
+		draft.Insights = append(draft.Insights, importdraft.DraftInsight{
+			Content: content, Category: "fact", Importance: 5,
+		})
+	}
+	data, err := json.Marshal(draft)
+	if err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(t.TempDir(), "draft.json")
+	if err := os.WriteFile(path, data, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	var runErr error
+	out := captureStdout(t, func() { runErr = importCmd.RunE(importCmd, []string{path}) })
+	if runErr != nil {
+		t.Fatalf("import: %v", runErr)
+	}
+	var result importDiffOutput
+	if err := json.Unmarshal([]byte(out), &result); err != nil {
+		t.Fatalf("decode import output: %v\n%s", err, out)
+	}
+	if result.Errors != 0 || len(result.Results) != len(contents) {
+		t.Fatalf("incomplete import: %+v", result)
+	}
+	return result
+}
+
+func TestImportPreservesDistinctContent(t *testing.T) {
+	const alpha = "Project Alpha uses PostgreSQL database for persistent application storage"
+	const details = " with indexed customer records, transaction history, audit events, replication, backups, failover, monitoring, access controls, migrations, connection pooling, and disaster recovery"
+	tests := []struct{ name, first, second string }{
+		{"different subject", alpha, "Project Beta uses PostgreSQL database for persistent application storage"},
+		{"changed value", alpha, "Project Alpha uses SQLite database for persistent application storage"},
+		{"near duplicate", alpha + details, "Project Beta uses PostgreSQL database for persistent application storage" + details},
+		{"conflict", alpha, "Project Alpha no longer uses PostgreSQL database for persistent application storage"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			configureImportDiffTest(t)
+			result := importForDiffTest(t, []string{tt.first, tt.second}, nil)
+			if result.Imported != 2 || result.Updated != 0 || result.Skipped != 0 {
+				t.Errorf("import = %+v, want two added insights", result)
+			}
+			assertActiveRememberContents(t, map[string]string{
+				result.Results[0].ID: tt.first, result.Results[1].ID: tt.second,
+			})
+		})
+	}
+}
+
+func TestImportPreservesExistingFact(t *testing.T) {
+	configureImportDiffTest(t)
+	const alpha = "Project Alpha uses PostgreSQL database for persistent application storage"
+	const beta = "Project Beta uses PostgreSQL database for persistent application storage"
+	first := rememberForDiffTest(t, alpha)
+	result := importForDiffTest(t, []string{beta}, nil)
+	if result.Imported != 1 || result.Updated != 0 || result.Skipped != 0 {
+		t.Errorf("import = %+v, want one added insight", result)
+	}
+	assertActiveRememberContents(t, map[string]string{first.ID: alpha, result.Results[0].ID: beta})
+}
+
+func TestImportExactDuplicateRetainsIndexAndEdgeMapping(t *testing.T) {
+	configureImportDiffTest(t)
+	const content = "Project Alpha uses PostgreSQL database for persistent application storage"
+	remImportance = 1
+	first := rememberForDiffTest(t, content)
+	want := map[string]string{first.ID: content}
+	remImportance, remNoDiff = 5, true
+	for _, detail := range []string{"backups", "replication", "indexes", "migrations", "monitoring", "transactions"} {
+		extended := content + " with " + detail
+		result := rememberForDiffTest(t, extended)
+		want[result.ID] = extended
+	}
+	const linked = "Zebra migration survey notes"
+	result := importForDiffTest(t, []string{content, linked, linked}, []importdraft.DraftEdge{
+		{SourceIndex: 0, TargetIndex: 2, EdgeType: "semantic", Weight: 0.75, Reason: "exact duplicate index mapping"},
+	})
+	if result.Imported != 1 || result.Updated != 0 || result.Skipped != 2 {
+		t.Errorf("import = %+v, want one added insight and two exact duplicates", result)
+	}
+	if result.Results[0].ID != first.ID || result.Results[0].Action != "skipped" {
+		t.Errorf("existing duplicate = %+v, want skipped %s", result.Results[0], first.ID)
+	}
+	linkedID := result.Results[1].ID
+	if result.Results[2].ID != linkedID || result.Results[2].Action != "skipped" {
+		t.Errorf("batch duplicate = %+v, want skipped %s", result.Results[2], linkedID)
+	}
+	want[linkedID] = linked
+	assertActiveRememberContents(t, want)
+	db, err := store.Open(store.StoreDir(dataDir, store.DefaultStoreName))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	edges, err := db.GetEdgesBySourceAndType(first.ID, model.EdgeSemantic)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, edge := range edges {
+		if edge.TargetID == linkedID && edge.Metadata["reason"] == "exact duplicate index mapping" {
+			return
+		}
+	}
+	t.Fatalf("explicit edge from existing duplicate %s to batch duplicate %s is missing", first.ID, linkedID)
+}
+
+func TestImportNoDiffStoresExactRepeats(t *testing.T) {
+	configureImportDiffTest(t)
+	importNoDiff = true
+	const content = "Project Alpha uses PostgreSQL database for persistent application storage"
+	result := importForDiffTest(t, []string{content, content}, nil)
+	if result.Imported != 2 || result.Updated != 0 || result.Skipped != 0 {
+		t.Errorf("import with --no-diff = %+v, want two added insights", result)
+	}
+	assertActiveRememberContents(t, map[string]string{
+		result.Results[0].ID: content, result.Results[1].ID: content,
+	})
+}

cmd/memory/remember.go

@@ -118,9 +118,9 @@ var rememberCmd = &cobra.Command{
 		}
 
 		// 2. Built-in diff: check for duplicates/conflicts (read-only, before transaction)
-		var diffAction string // "added", "updated", "skipped"
-		var replacedID string
-		var diffSuggestion search.DiffSuggestion
+		diffAction := "added"
+		diffSuggestion := search.DiffAdd
+		var duplicateID string
 
 		// Build embed cache once — reused by diff, engine, and semantic candidates.
 		var embedCache graph.EmbedCache
@@ -136,10 +136,7 @@ var rememberCmd = &cobra.Command{
 			}
 		}
 
-		if remNoDiff {
-			diffAction = "added"
-			diffSuggestion = search.DiffAdd
-		} else {
+		if !remNoDiff {
 			allInsights, err := db.GetAllActiveInsights()
 			if err != nil {
 				return fmt.Errorf("load insights for diff: %w", err)
@@ -159,45 +156,25 @@ var rememberCmd = &cobra.Command{
 			result := search.Diff(allInsights, content, opts)
 			diffSuggestion = result.Suggestion
 
-			switch result.Suggestion {
-			case search.DiffDuplicate:
+			// Similarity cannot establish whether two facts have the same
+			// subject, value, or relationship. Keep diff suggestions advisory;
+			// only byte-identical content permits skipping a write.
+			duplicateID = search.FindExactDuplicateID(allInsights, content)
+			if duplicateID != "" {
 				diffAction = "skipped"
-				if len(result.Matches) > 0 {
-					replacedID = result.Matches[0].ID
-				}
-			case search.DiffConflict:
-				// A CONFLICT means the two texts appear to disagree. Silently
-				// soft-deleting one side is destructive and has repeatedly
-				// clobbered unrelated same-domain memories (long technical
-				// notes share vocabulary at >=0.7 similarity, and change-log
-				// words like "replaced"/"no longer" appear in almost all of
-				// them). Keep both; the caller sees diff_suggestion=CONFLICT
-				// and can merge or delete deliberately.
-				diffAction = "added"
-			case search.DiffUpdate:
-				// Only auto-replace when the texts overlap heavily by TOKENS.
-				// Cosine similarity alone (same-domain embeddings cluster at
-				// 0.85+) is not enough evidence to destroy an existing memory.
-				if len(result.Matches) > 0 && result.Matches[0].TokenSimilarity >= 0.6 {
-					diffAction = "updated"
-					replacedID = result.Matches[0].ID
-				} else {
-					diffAction = "added"
-				}
-			default:
-				diffAction = "added"
+				diffSuggestion = search.DiffDuplicate
 			}
 		}
 
 		// If duplicate, skip insert entirely
 		if diffAction == "skipped" {
-			db.LogOp("diff-skip", insight.ID, fmt.Sprintf("duplicate of %s", replacedID))
+			db.LogOp("diff-skip", insight.ID, fmt.Sprintf("duplicate of %s", duplicateID))
 			output := map[string]interface{}{
 				"id":              insight.ID,
 				"content":         content,
 				"action":          "skipped",
 				"diff_suggestion": string(diffSuggestion),
-				"replaced_id":     replacedID,
+				"replaced_id":     duplicateID,
 			}
 			enc := json.NewEncoder(os.Stdout)
 			enc.SetIndent("", "  ")
@@ -212,18 +189,6 @@ var rememberCmd = &cobra.Command{
 			embedded  bool
 		)
 		err = db.InTransaction(func() error {
-			// Soft-delete old insight if updating
-			if diffAction == "updated" && replacedID != "" {
-				if err := db.SoftDeleteInsight(replacedID); err != nil {
-					fmt.Fprintf(os.Stderr, "warning: soft-delete %s: %v\n", replacedID, err)
-				} else {
-					db.LogOp("diff-replace", replacedID, fmt.Sprintf("replaced by %s", insight.ID))
-					// Remove deleted insight from embed cache to prevent
-					// creating edges to a soft-deleted node.
-					delete(embedCache, replacedID)
-				}
-			}
-
 			if err := db.InsertInsight(insight); err != nil {
 				return fmt.Errorf("insert insight: %w", err)
 			}
@@ -268,7 +233,7 @@ var rememberCmd = &cobra.Command{
 			return nil
 		})
 		if err != nil {
-			// Cache was mutated inside the transaction closure (delete/add entries).
+			// Cache was mutated inside the transaction closure (added entries).
 			// On rollback those mutations don't match DB state, so discard the cache
 			// to prevent any future code from accidentally using stale data.
 			embedCache = nil
@@ -306,9 +271,6 @@ var rememberCmd = &cobra.Command{
 			"auto_pruned":          len(prunedIDs),
 			"auto_pruned_ids":      prunedIDs,
 		}
-		if replacedID != "" {
-			output["replaced_id"] = replacedID
-		}
 		enc := json.NewEncoder(os.Stdout)
 		enc.SetIndent("", "  ")
 		return enc.Encode(output)