diff --git a/build.mjs b/build.mjs index 99aaaf8..0be93c2 100644 --- a/build.mjs +++ b/build.mjs @@ -40,6 +40,8 @@ const libraryEntries = [ { in: "src/services/session.ts", out: "dist/services/session.js" }, { in: "src/services/tags.ts", out: "dist/services/tags.js" }, { in: "src/services/resultMerge.ts", out: "dist/services/resultMerge.js" }, + { in: "src/services/resultText.ts", out: "dist/services/resultText.js" }, + { in: "src/services/factCache.ts", out: "dist/services/factCache.js" }, ]; await Promise.all( diff --git a/src/services/client.ts b/src/services/client.ts index dfa032d..3b87e75 100644 --- a/src/services/client.ts +++ b/src/services/client.ts @@ -4,6 +4,7 @@ import { CONFIG, isConfigured, getApiKeyValue, getBaseUrl, PLUGIN_VERSION } from import { log } from "./logger.js"; import type { MemoryType } from "../types/index.js"; import { mergeProfileResults, mergeSearchResponses } from "./resultMerge.js"; +import { boundedMemoryText, recallProvenance } from "./resultText.js"; type ProfileParamsWithFilters = ProfileParams & { filters?: ReturnType; @@ -36,13 +37,15 @@ function withTimeout(promise: Promise, ms: number): Promise { /** Canonical search result item used across the codebase. */ export interface SearchResultItem { id?: string; - memory?: string; - content?: string; - chunk?: string; + memory?: unknown; + content?: unknown; + chunk?: unknown; + text?: unknown; context?: unknown; score?: number; similarity?: number; title?: string; + filepath?: string; updatedAt?: string; metadata?: Record | null; containerTag?: string; @@ -69,6 +72,8 @@ export interface ProfileWithSearchResult { memory: string; similarity?: number; title?: string; + filepath?: string; + metadata?: Record | null; updatedAt?: string; }>; total: number; @@ -154,13 +159,20 @@ export class SupermemoryClient { let searchResults: ProfileWithSearchResult["searchResults"]; if (result.searchResults) { - const mapped = (result.searchResults.results as SearchResultItem[]).map((r) => ({ - id: r.id, - memory: r.memory || r.content || String(r.context ?? ""), - similarity: r.similarity, - title: r.title, - updatedAt: r.updatedAt, - })); + const mapped = (result.searchResults.results as SearchResultItem[]) + .map((r) => { + const provenance = recallProvenance(r); + return { + id: r.id, + memory: boundedMemoryText(r), + similarity: r.similarity, + title: provenance.title, + filepath: provenance.filepath, + metadata: r.metadata, + updatedAt: r.updatedAt, + }; + }) + .filter((r) => r.memory.length > 0); searchResults = { results: dedupeWithSeen(mapped, (r) => r.memory), total: result.searchResults.total, diff --git a/src/services/context.ts b/src/services/context.ts index 5de6e74..97a24ab 100644 --- a/src/services/context.ts +++ b/src/services/context.ts @@ -1,5 +1,6 @@ import type { ProfileWithSearchResult, SearchResponse } from "./client.js"; -import { normalizeFact } from "./factCache.js"; +import { factKey } from "./factCache.js"; +import { memoryText } from "./resultText.js"; interface ProfileShape { static?: string[]; @@ -49,14 +50,23 @@ export function formatCombinedContext( ): FormattedContext { const parts: string[] = []; const newFacts: string[] = []; + const currentFactKeys = new Set(seenFacts); // Collect profile items, filtering out already-seen facts if (result.success && result.profile) { + const profileKeys = new Set(currentFactKeys); const items = [...(result.profile.static ?? []), ...(result.profile.dynamic ?? [])] .map((s) => s.trim()) - .filter((s) => s.length > 0 && !seenFacts.has(normalizeFact(s))) + .filter((s) => { + if (!s) return false; + const key = factKey(s); + if (profileKeys.has(key)) return false; + profileKeys.add(key); + return true; + }) .slice(0, maxProfileItems); if (items.length > 0) { + for (const item of items) currentFactKeys.add(factKey(item)); parts.push( `[Memory Profile]\n${items.map((s, i) => `${i + 1}. ${s}`).join("\n")}` ); @@ -73,15 +83,22 @@ export function formatCombinedContext( return id ? `id:${id}` : ""; } - const allMemories: string[] = []; + const allMemories: Array<{ text: string; display: string }> = []; if (result.searchResults && result.searchResults.results.length > 0) { for (const r of result.searchResults.results) { - const text = r.memory || ""; - if (!text || seenFacts.has(normalizeFact(text))) continue; + const text = memoryText(r); + const textKey = text ? factKey(text) : ""; + if (!text || currentFactKeys.has(textKey)) continue; const key = dedupKey(r.id, text); if (key && !seenKeys.has(key)) { seenKeys.add(key); - allMemories.push(text); + currentFactKeys.add(textKey); + const labels = [r.title, r.filepath] + .filter((label): label is string => typeof label === "string" && label.trim().length > 0); + allMemories.push({ + text, + display: labels.length > 0 ? `[${labels.join(" — ")}] ${text}` : text, + }); } } } @@ -89,12 +106,12 @@ export function formatCombinedContext( if (allMemories.length > 0) { const limitedMemories = allMemories.slice(0, maxMemories); const memories = limitedMemories - .map((m, i) => `${i + 1}. ${m}`) + .map((memory, i) => `${i + 1}. ${memory.display}`) .filter((m) => m.trim().length > 2) .join("\n"); if (memories) { parts.push(`[Relevant Memories]\n${memories}`); - newFacts.push(...limitedMemories); + newFacts.push(...limitedMemories.map((memory) => memory.text)); } } @@ -123,7 +140,7 @@ export function formatContextForPrompt( if (searchResult.success && searchResult.results && searchResult.results.length > 0) { const memories = searchResult.results .slice(0, maxMemories) - .map((r, i) => `${i + 1}. ${r.memory ?? r.chunk ?? r.content ?? ""}`) + .map((r, i) => `${i + 1}. ${memoryText(r)}`) .filter((m) => m.trim().length > 2) .join("\n"); if (memories) { diff --git a/src/services/factCache.ts b/src/services/factCache.ts index b236dce..6f0b4c8 100644 --- a/src/services/factCache.ts +++ b/src/services/factCache.ts @@ -1,8 +1,11 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs"; import { join } from "node:path"; import { homedir } from "node:os"; +import { createHash } from "node:crypto"; const CACHE_DIR = join(homedir(), ".codex-supermemory", "trackers"); +export const MAX_SEEN_FACTS = 500; +const HASH_PREFIX = "sha256:"; function ensureDir(): void { if (!existsSync(CACHE_DIR)) mkdirSync(CACHE_DIR, { recursive: true }); @@ -16,12 +19,19 @@ export function normalizeFact(s: string): string { return s.toLowerCase().replace(/\s+/g, " ").trim(); } +export function factKey(s: string): string { + return `${HASH_PREFIX}${createHash("sha256").update(normalizeFact(s)).digest("hex")}`; +} + export function getSeenFacts(sessionId: string): Set { const file = cacheFile(sessionId); if (!existsSync(file)) return new Set(); try { const parsed = JSON.parse(readFileSync(file, "utf-8")) as { facts?: string[] }; - return new Set(parsed.facts ?? []); + const facts = (parsed.facts ?? []) + .filter((fact): fact is string => typeof fact === "string" && fact.length > 0) + .map((fact) => fact.startsWith(HASH_PREFIX) ? fact : factKey(fact)); + return new Set(facts.slice(-MAX_SEEN_FACTS)); } catch { return new Set(); } @@ -31,6 +41,7 @@ export function addSeenFacts(sessionId: string, facts: string[]): void { if (facts.length === 0) return; ensureDir(); const seen = getSeenFacts(sessionId); - for (const f of facts) seen.add(normalizeFact(f)); - writeFileSync(cacheFile(sessionId), JSON.stringify({ facts: [...seen] })); + for (const f of facts) seen.add(factKey(f)); + const bounded = [...seen].slice(-MAX_SEEN_FACTS); + writeFileSync(cacheFile(sessionId), JSON.stringify({ facts: bounded })); } diff --git a/src/services/resultMerge.ts b/src/services/resultMerge.ts index 763fce7..5121e1f 100644 --- a/src/services/resultMerge.ts +++ b/src/services/resultMerge.ts @@ -3,6 +3,13 @@ import type { SearchResponse, SearchResultItem, } from "./client.js"; +import { + boundedMemoryText, + memoryText, + recallProvenance, + RECALL_MAX_RESULTS, + RECALL_MIN_SIMILARITY, +} from "./resultText.js"; function normalize(value: unknown): string { return String(value ?? "").toLowerCase().trim(); @@ -18,10 +25,6 @@ function dedupe(items: T[], getKey: (item: T) => string): T[] { }); } -function memoryText(result: SearchResultItem): string { - return result.memory ?? result.chunk ?? result.content ?? String(result.context ?? ""); -} - function searchKey(result: SearchResultItem): string { const content = normalize(memoryText(result)); if (content) return `content:${content}`; @@ -90,11 +93,14 @@ export function mergeProfileResults( const mergedSearch = mergeSearchResponses( successful.map((response) => ({ success: true, - results: response.searchResults?.results ?? [], + results: (response.searchResults?.results ?? []).filter( + (result) => + score(result) >= RECALL_MIN_SIMILARITY && memoryText(result).length > 0, + ), total: response.searchResults?.total ?? 0, timing: response.searchResults?.timing, })), - limit, + Math.min(limit, RECALL_MAX_RESULTS), ); return { @@ -103,13 +109,17 @@ export function mergeProfileResults( searchResults: mergedSearch.results && mergedSearch.results.length > 0 ? { - results: mergedSearch.results.map((result) => ({ - id: result.id, - memory: memoryText(result), - similarity: result.similarity, - title: result.title, - updatedAt: result.updatedAt, - })), + results: mergedSearch.results.map((result) => { + const provenance = recallProvenance(result); + return { + id: result.id, + memory: boundedMemoryText(result), + similarity: result.similarity, + title: provenance.title, + filepath: provenance.filepath, + updatedAt: result.updatedAt, + }; + }), total: mergedSearch.total ?? mergedSearch.results.length, timing: mergedSearch.timing, } diff --git a/src/services/resultText.ts b/src/services/resultText.ts new file mode 100644 index 0000000..9ebc3ea --- /dev/null +++ b/src/services/resultText.ts @@ -0,0 +1,64 @@ +export interface MemoryTextShape { + memory?: unknown; + chunk?: unknown; + content?: unknown; + text?: unknown; + context?: unknown; + title?: unknown; + filepath?: unknown; + filePath?: unknown; + path?: unknown; + metadata?: unknown; +} + +export const RECALL_MIN_SIMILARITY = 0.55; +export const RECALL_MAX_RESULTS = 5; +export const RECALL_MAX_RESULT_CHARS = 300; + +/** Return only a real string field; never stringify objects as `[object Object]`. */ +export function memoryText(result: MemoryTextShape): string { + for (const value of [ + result.memory, + result.chunk, + result.content, + result.text, + result.context, + ]) { + if (typeof value === "string" && value.trim()) return value.trim(); + } + return ""; +} + +export function boundedMemoryText( + result: MemoryTextShape, + maxChars = RECALL_MAX_RESULT_CHARS, +): string { + return memoryText(result).replace(/\s+/g, " ").slice(0, maxChars).trim(); +} + +function stringValue(...values: unknown[]): string | undefined { + const value = values.find( + (candidate) => typeof candidate === "string" && candidate.trim().length > 0, + ); + return typeof value === "string" ? value.trim() : undefined; +} + +export function recallProvenance( + result: MemoryTextShape, +): { title?: string; filepath?: string } { + const metadata = + result.metadata && typeof result.metadata === "object" + ? result.metadata as Record + : {}; + return { + title: stringValue(result.title, metadata.title), + filepath: stringValue( + result.filepath, + result.filePath, + result.path, + metadata.filepath, + metadata.filePath, + metadata.path, + ), + }; +} diff --git a/test/unit.mjs b/test/unit.mjs index 0e30a5f..c0dc9ab 100644 --- a/test/unit.mjs +++ b/test/unit.mjs @@ -244,6 +244,63 @@ describe("cross-container result merging", () => { assert.equal(result.status, 0, result.stderr); assert.deepEqual(JSON.parse(result.stdout), ["best", "new"]); }); + + test("normalizes and bounds high-quality profile recall hits", () => { + const script = ` + import { mergeProfileResults } from ${JSON.stringify(mergeModule)}; + const long = "x".repeat(360); + const merged = mergeProfileResults([{ + success: true, + profile: { static: [], dynamic: [] }, + searchResults: { results: [ + { id: "memory", memory: "memory value", similarity: 0.99, metadata: { title: "Decision", filePath: "src/a.ts" } }, + { id: "chunk", memory: {}, chunk: "chunk value", similarity: 0.95 }, + { id: "content", content: "content value", similarity: 0.9 }, + { id: "text", text: "text value", similarity: 0.85 }, + { id: "context", context: "context value", similarity: 0.8 }, + { id: "sixth", memory: long, similarity: 0.75 }, + { id: "low", memory: "too weak", similarity: 0.54 }, + { id: "object", context: { value: "never stringify" }, similarity: 1 } + ], total: 8 } + }], 20); + console.log(JSON.stringify(merged.searchResults.results)); + `; + const result = spawnSync("node", ["--input-type=module", "-e", script], { + encoding: "utf-8", + }); + assert.equal(result.status, 0, result.stderr); + const results = JSON.parse(result.stdout); + assert.equal(results.length, 5); + assert.deepEqual(results.map((item) => item.memory), [ + "memory value", "chunk value", "content value", "text value", "context value", + ]); + assert.equal(results[0].title, "Decision"); + assert.equal(results[0].filepath, "src/a.ts"); + assert.ok(results.every((item) => item.memory.length <= 300)); + }); +}); + +describe("session recall deduplication", () => { + const factCacheModule = new URL("../dist/services/factCache.js", import.meta.url).href; + + test("stores only a bounded set of hashed fact identities", (t) => { + const homeDir = makeTmpDir(); + t.after(() => rmSync(homeDir, { recursive: true, force: true })); + const script = ` + import { addSeenFacts, getSeenFacts } from ${JSON.stringify(factCacheModule)}; + addSeenFacts("session", Array.from({ length: 550 }, (_, i) => \`fact \${i}\`)); + console.log(JSON.stringify([...getSeenFacts("session")])); + `; + const result = spawnSync("node", ["--input-type=module", "-e", script], { + env: { ...process.env, HOME: homeDir, USERPROFILE: homeDir }, + encoding: "utf-8", + }); + assert.equal(result.status, 0, result.stderr); + const facts = JSON.parse(result.stdout); + assert.equal(facts.length, 500); + assert.ok(facts.every((fact) => /^sha256:[0-9a-f]{64}$/.test(fact))); + assert.ok(!facts.some((fact) => fact.includes("fact"))); + }); }); // ─── session ids ────────────────────────────────────────────────────────────