From 727338d2fbf00e1b481518d7fc29795fe9957d7d Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Thu, 23 Jul 2026 20:14:43 +0530 Subject: [PATCH 1/5] docs(examples): rewrite README to reflect library-only usage The examples README referenced a non-existent 'yaad' binary (go install, yaad init, yaad setup, yaad list, yaad search, yaad graph). Yaad is a Go library, not a standalone binary. Rewrite to show hawk integration, MCP tool usage, Go library embedding, and the yaad-tui demo. --- examples/README.md | 97 ++++++++++++++-------------------------------- 1 file changed, 29 insertions(+), 68 deletions(-) diff --git a/examples/README.md b/examples/README.md index a9ee6ed..dd3de60 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,33 +2,18 @@ Yaad gives your coding agent persistent memory across sessions via MCP. -## Setup +> **Note:** Yaad is a Go library, not a standalone binary. It ships no `yaad` +> command of its own. Memory is accessed through hawk's built-in MCP tools or +> by embedding the `engine` package in your own Go application. -### 1. Install Yaad +## Setup (via hawk) -```bash -go install github.com/GrayCodeAI/yaad/cmd/yaad@latest -``` - -### 2. Initialize in your project - -```bash -cd your-project -yaad init -``` - -### 3. Connect your agent +Yaad is integrated into hawk. Enable it in your hawk config: ```bash -yaad setup +hawk config set memory.enabled true ``` -This generates `.mcp.json` and configures your agent to use yaad. - -## Basic Usage - -### Remember something - Your agent can now remember things across sessions: ``` @@ -40,20 +25,7 @@ You: What database are we using? Agent: [queries yaad] PostgreSQL 15 with TimescaleDB ``` -### Query memory - -```bash -# List all memories -yaad list - -# Search memories -yaad search "database" - -# Show memory graph -yaad graph -``` - -## MCP Integration +## MCP Tools Yaad exposes these MCP tools to your agent: @@ -97,49 +69,38 @@ Remove outdated memories: } ``` -## Example Agent Config +## Go Library Usage -### Claude Code (.mcp.json) +Embed yaad in your own Go application: -```json -{ - "mcpServers": { - "yaad": { - "command": "yaad", - "args": ["mcp"] - } - } +```go +import "github.com/GrayCodeAI/yaad/engine" + +eng, err := engine.New(engine.Config{ + DataDir: "~/.yaad", +}) +if err != nil { + log.Fatal(err) } -``` +defer eng.Close() -### Cursor (.cursor/mcp.json) +// Store a memory +err = eng.Remember(ctx, engine.Memory{ + Content: "Project uses PostgreSQL 15", + Tags: []string{"database"}, +}) -```json -{ - "mcpServers": { - "yaad": { - "command": "yaad", - "args": ["mcp"], - "cwd": "${workspaceFolder}" - } - } -} +// Recall relevant memories +results, err := eng.Recall(ctx, "database configuration") ``` -## Advanced: Memory Graph - -Yaad stores memories as a graph, not just a list. This enables: +## Demo TUI -- **Temporal queries**: "What did we decide last week?" -- **Relationship tracking**: "How does the auth system relate to the user model?" -- **Context-aware recall**: "What do I know about the payment module?" +A demo TUI is available for exploring yaad interactively: ```bash -# Visualize the memory graph -yaad graph --visualize - -# Query relationships -yaad query "auth -> user -> permissions" +cd cmd/yaad-tui +go run . ``` See the [main README](../README.md) for full documentation. From 1467d2cdd40a60d7c088bff44574c29d7322faf4 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Thu, 23 Jul 2026 21:26:10 +0530 Subject: [PATCH 2/5] docs(go.mod): document local replace directive for hawk-mcpkit yaad depends on ServeHTTPWithShutdown and other APIs added after v0.1.4. The replace directive is required for monorepo development until v0.2.0 is tagged. --- go.mod | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go.mod b/go.mod index a2d0116..22fbd86 100644 --- a/go.mod +++ b/go.mod @@ -40,4 +40,7 @@ require ( modernc.org/memory v1.11.0 // indirect ) +// replace: use local hawk-mcpkit during monorepo development. +// yaad depends on ServeHTTPWithShutdown and other APIs added after v0.1.4; +// switch to a tagged version once v0.2.0 is released. replace github.com/GrayCodeAI/hawk-mcpkit => ../hawk-mcpkit From 8a94856ac2fd2416f6b84cb6da39600dd10c6c78 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Fri, 24 Jul 2026 12:35:45 +0530 Subject: [PATCH 3/5] feat(yaad): delete dead code, fix duplicate if, add slog warnings + tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Delete dead engine/git_learn.go, engine/ingest.go, engine/spaced_repetition.go (864 lines) — none of their exported types were referenced outside the files. The classifyContent helper used by engine/context_pack.go was moved into that file with its regex vars (renamed to avoid stutter). - Remove duplicate `if entry.CreatedAt.IsZero()` block in spatial_memory.go (lines 87-91 were identical). - Replace silent `_ = err` ignores in engine/proactive.go with slog.Warn calls (GetNeighbors, GetEdgesFrom, CountEdgesBatch) so operational issues are observable without breaking the best-effort prediction flow. - Add tests: engine/cache_test.go (QueryCache Get/Put, Invalidate, TTL, eviction, Stats/Clear, CacheKey), engine/ratelimit_test.go (burst, refill, Remaining, maxBurst cap), engine/temporal_filter_test.go (matchesTemporal After/Before/Range, FilterByTime). All yaad package tests pass; build is clean. Co-Authored-By: Grok --- engine/cache_test.go | 113 +++++++++ engine/context_pack.go | 28 ++ engine/git_learn.go | 270 -------------------- engine/ingest.go | 449 --------------------------------- engine/proactive.go | 19 +- engine/ratelimit_test.go | 62 +++++ engine/spaced_repetition.go | 145 ----------- engine/temporal_filter_test.go | 85 +++++++ spatial_memory.go | 3 - 9 files changed, 304 insertions(+), 870 deletions(-) create mode 100644 engine/cache_test.go delete mode 100644 engine/git_learn.go delete mode 100644 engine/ingest.go create mode 100644 engine/ratelimit_test.go delete mode 100644 engine/spaced_repetition.go create mode 100644 engine/temporal_filter_test.go diff --git a/engine/cache_test.go b/engine/cache_test.go new file mode 100644 index 0000000..10eb7c1 --- /dev/null +++ b/engine/cache_test.go @@ -0,0 +1,113 @@ +package engine + +import ( + "testing" + "time" +) + +func TestNewQueryCache_Defaults(t *testing.T) { + t.Parallel() + c := NewQueryCache(0, 0) + if c.maxSize != 64 { + t.Errorf("default maxSize = %d, want 64", c.maxSize) + } + if c.ttl != 5*time.Minute { + t.Errorf("default ttl = %v, want 5m", c.ttl) + } +} + +func TestQueryCache_GetPut(t *testing.T) { + t.Parallel() + c := NewQueryCache(10, time.Minute) + + if got := c.Get("missing"); got != nil { + t.Errorf("Get(missing) = %v, want nil", got) + } + + result := &RecallResult{Nodes: nil} + c.Put("q1", result) + + if got := c.Get("q1"); got != result { + t.Errorf("Get(q1) = %v, want original result", got) + } +} + +func TestQueryCache_Invalidate(t *testing.T) { + t.Parallel() + c := NewQueryCache(10, time.Minute) + c.Put("q1", &RecallResult{}) + + c.Invalidate() + + if got := c.Get("q1"); got != nil { + t.Errorf("Get after Invalidate = %v, want nil", got) + } +} + +func TestQueryCache_TTLExpiry(t *testing.T) { + t.Parallel() + c := NewQueryCache(10, time.Millisecond) + c.Put("q1", &RecallResult{}) + + time.Sleep(5 * time.Millisecond) + + if got := c.Get("q1"); got != nil { + t.Errorf("Get after TTL = %v, want nil (expired)", got) + } +} + +func TestQueryCache_Eviction(t *testing.T) { + t.Parallel() + c := NewQueryCache(2, time.Minute) + c.Put("a", &RecallResult{}) + c.Put("b", &RecallResult{}) + c.Put("c", &RecallResult{}) // evicts "a" + + if got := c.Get("a"); got != nil { + t.Errorf("Get(a) after eviction = %v, want nil", got) + } + if got := c.Get("b"); got == nil { + t.Error("Get(b) = nil, want present") + } + if got := c.Get("c"); got == nil { + t.Error("Get(c) = nil, want present") + } + size, _ := c.Stats() + if size != 2 { + t.Errorf("size = %d, want 2", size) + } +} + +func TestQueryCache_StatsAndClear(t *testing.T) { + t.Parallel() + c := NewQueryCache(10, time.Minute) + c.Put("q1", &RecallResult{}) + c.Put("q2", &RecallResult{}) + + size, version := c.Stats() + if size != 2 { + t.Errorf("size = %d, want 2", size) + } + if version != 0 { + t.Errorf("version = %d, want 0", version) + } + + c.Clear() + size, _ = c.Stats() + if size != 0 { + t.Errorf("size after Clear = %d, want 0", size) + } +} + +func TestCacheKey(t *testing.T) { + t.Parallel() + key := CacheKey(RecallOpts{Query: "hello", Type: "convention", Project: "myproj", Limit: 10, Depth: 2}) + if key == "" { + t.Error("CacheKey returned empty string") + } + // Different opts must yield different keys. + other := CacheKey(RecallOpts{Query: "world", Type: "convention", Project: "myproj", Limit: 10, Depth: 2}) + if key == other { + t.Errorf("different queries produced same key %q", key) + } +} diff --git a/engine/context_pack.go b/engine/context_pack.go index e5b209e..b372a0e 100644 --- a/engine/context_pack.go +++ b/engine/context_pack.go @@ -2,6 +2,7 @@ package engine import ( "fmt" + "regexp" "sort" "strings" @@ -179,3 +180,30 @@ type SummarizedMemory struct { Content string Type string } + +var ( + classifyConventionRe = regexp.MustCompile(`(?i)(always|never|must|should|prefer|avoid|use .+ instead|don't|do not|convention|rule|standard)`) + classifyDecisionRe = regexp.MustCompile(`(?i)(decided|chose|picked|selected|went with|switched to|migrated|adopted|we use)`) + classifyBugRe = regexp.MustCompile(`(?i)(bug|fix|issue|error|broken|crash|race condition|deadlock|leak|regression)`) + classifyTaskRe = regexp.MustCompile(`(?i)(todo|TODO|FIXME|HACK|in progress|need to|should implement|blocked)`) + classifyPrefRe = regexp.MustCompile(`(?i)(i prefer|i like|my style|i want|personally|i always)`) +) + +// classifyContent maps a line of conversation to a memory node type based on +// keyword heuristics. Returns "" when no type matches. +func classifyContent(text string) string { + switch { + case classifyDecisionRe.MatchString(text): + return "decision" + case classifyConventionRe.MatchString(text): + return "convention" + case classifyBugRe.MatchString(text): + return "bug" + case classifyTaskRe.MatchString(text): + return "task" + case classifyPrefRe.MatchString(text): + return "preference" + default: + return "" + } +} diff --git a/engine/git_learn.go b/engine/git_learn.go deleted file mode 100644 index b93a771..0000000 --- a/engine/git_learn.go +++ /dev/null @@ -1,270 +0,0 @@ -package engine - -import ( - "context" - "fmt" - "os/exec" - "regexp" - "strings" - "time" -) - -// GitLearner extracts memories from git history automatically. -// Scans commit messages, diffs, and blame data to discover: -// - Architecture decisions (from commit messages with "chose", "decided", "switched") -// - Conventions (from repeated patterns across commits) -// - Bug patterns (from fix/hotfix commits) -// - File purposes (from first-commit messages) -// -// No other memory tool does this. Your project's entire history becomes memory. -type GitLearner struct { - dir string - engine *Engine -} - -// LearnResult reports what was extracted from git history. -type LearnResult struct { - Decisions int - Conventions int - Bugs int - Files int - Skipped int - Duration time.Duration -} - -// NewGitLearner creates a learner for a project directory. -func NewGitLearner(dir string, engine *Engine) *GitLearner { - return &GitLearner{dir: dir, engine: engine} -} - -// LearnFromHistory scans git log and extracts memories. -// limit: max commits to scan (0 = all). since: only commits after this date. -func (gl *GitLearner) LearnFromHistory(ctx context.Context, limit int, since time.Time) (*LearnResult, error) { - start := time.Now() - result := &LearnResult{} - - // Build git log command - args := []string{"-C", gl.dir, "log", "--format=%H|%s|%an|%aI", "--no-merges"} - if limit > 0 { - args = append(args, fmt.Sprintf("-n%d", limit)) - } - if !since.IsZero() { - args = append(args, fmt.Sprintf("--since=%s", since.Format("2006-01-02"))) - } - - out, err := exec.CommandContext(ctx, "git", args...).Output() // #nosec G204 -- fixed "git" binary; args built internally from gl.dir/limit/since, not shell-interpreted - if err != nil { - return result, fmt.Errorf("git log failed: %w", err) - } - - lines := strings.Split(strings.TrimSpace(string(out)), "\n") - for _, line := range lines { - if line == "" { - continue - } - parts := strings.SplitN(line, "|", 4) - if len(parts) < 4 { - continue - } - hash, subject, _, _ := parts[0], parts[1], parts[2], parts[3] - - extracted := gl.extractFromCommit(ctx, hash, subject) - result.Decisions += extracted.decisions - result.Conventions += extracted.conventions - result.Bugs += extracted.bugs - result.Files += extracted.files - result.Skipped += extracted.skipped - } - - result.Duration = time.Since(start) - return result, nil -} - -// LearnFromBlame extracts file-level knowledge from git blame. -// Discovers who owns what and when major changes happened. -func (gl *GitLearner) LearnFromBlame(ctx context.Context, filePath string) error { - out, err := exec.CommandContext(ctx, "git", "-C", gl.dir, "log", "--oneline", "-1", "--diff-filter=A", "--", filePath).Output() // #nosec G204 -- fixed "git" binary; gl.dir/filePath are caller-supplied paths, not shell-interpreted, and "--" ends flag parsing - if err != nil || len(out) == 0 { - return nil - } - - // First commit that added this file → remember its purpose - firstCommit := strings.TrimSpace(string(out)) - if firstCommit != "" { - parts := strings.SplitN(firstCommit, " ", 2) - if len(parts) == 2 { - _ = gl.remember(ctx, fmt.Sprintf("File %s: %s", filePath, parts[1]), "file") - } - } - return nil -} - -// Suggest analyzes recent changes and suggests memories to store. -func (gl *GitLearner) Suggest(ctx context.Context) ([]MemorySuggestion, error) { - var suggestions []MemorySuggestion - - // 1. Find repeated patterns in recent commits - out, err := exec.CommandContext(ctx, "git", "-C", gl.dir, "log", "--oneline", "-50").Output() // #nosec G204 -- fixed "git" binary; gl.dir is the configured project directory, not shell-interpreted - if err != nil { - return nil, err - } - - commits := strings.Split(strings.TrimSpace(string(out)), "\n") - patterns := detectPatterns(commits) - for _, p := range patterns { - suggestions = append(suggestions, MemorySuggestion{ - Content: p.description, - Type: p.memoryType, - Confidence: p.confidence, - Reason: p.reason, - }) - } - - // 2. Find files changed frequently without corresponding memories - frequentFiles := gl.frequentlyChangedFiles(ctx) - for _, f := range frequentFiles { - suggestions = append(suggestions, MemorySuggestion{ - Content: fmt.Sprintf("File %s is frequently modified — consider documenting its purpose", f), - Type: "file", - Confidence: 0.6, - Reason: "Changed 5+ times in recent commits with no memory about it", - }) - } - - return suggestions, nil -} - -// MemorySuggestion is a recommended memory to store. -type MemorySuggestion struct { - Content string `json:"content"` - Type string `json:"type"` - Confidence float64 `json:"confidence"` - Reason string `json:"reason"` -} - -type extractResult struct { - decisions, conventions, bugs, files, skipped int -} - -func (gl *GitLearner) extractFromCommit(ctx context.Context, hash, subject string) extractResult { - r := extractResult{} - - // Decision patterns - if isDecisionCommit(subject) { - content := fmt.Sprintf("Decision (from git): %s", subject) - if err := gl.remember(ctx, content, "decision"); err == nil { - r.decisions++ - } else { - r.skipped++ - } - return r - } - - // Bug fix patterns - if isBugFixCommit(subject) { - content := fmt.Sprintf("Bug fix: %s", subject) - if err := gl.remember(ctx, content, "bug"); err == nil { - r.bugs++ - } else { - r.skipped++ - } - return r - } - - // Convention patterns (from commit messages mentioning rules/standards) - if isConventionCommit(subject) { - content := fmt.Sprintf("Convention (from git): %s", subject) - if err := gl.remember(ctx, content, "convention"); err == nil { - r.conventions++ - } else { - r.skipped++ - } - return r - } - - r.skipped++ - return r -} - -func (gl *GitLearner) remember(ctx context.Context, content, nodeType string) error { - _, err := gl.engine.Remember(ctx, RememberInput{ - Type: nodeType, - Content: content, - Scope: "project", - }) - return err -} - -func (gl *GitLearner) frequentlyChangedFiles(ctx context.Context) []string { - out, err := exec.CommandContext(ctx, "git", "-C", gl.dir, "log", "--name-only", "--format=", "-30").Output() // #nosec G204 -- fixed "git" binary; gl.dir is the configured project directory, not shell-interpreted - if err != nil { - return nil - } - - counts := make(map[string]int) - for _, line := range strings.Split(string(out), "\n") { - f := strings.TrimSpace(line) - if f != "" && !strings.HasPrefix(f, ".") { - counts[f]++ - } - } - - var frequent []string - for f, c := range counts { - if c >= 5 { - frequent = append(frequent, f) - } - } - return frequent -} - -// Pattern detection from commit messages - -var ( - decisionRe = regexp.MustCompile(`(?i)(chose|decided|switched to|migrated to|replaced .+ with|adopted|selected)`) - bugFixRe = regexp.MustCompile(`(?i)(fix|hotfix|bugfix|resolve|patch|repair|correct)\b`) - conventionRe = regexp.MustCompile(`(?i)(enforce|lint|format|standard|convention|always|never|require|must)`) -) - -func isDecisionCommit(subject string) bool { return decisionRe.MatchString(subject) } -func isBugFixCommit(subject string) bool { return bugFixRe.MatchString(subject) } -func isConventionCommit(subject string) bool { return conventionRe.MatchString(subject) } - -type pattern struct { - description string - memoryType string - confidence float64 - reason string -} - -func detectPatterns(commits []string) []pattern { - var patterns []pattern - - // Detect repeated prefixes (e.g., "fix:", "feat:", "refactor:") - prefixCount := make(map[string]int) - for _, c := range commits { - parts := strings.SplitN(c, " ", 2) - if len(parts) < 2 { - continue - } - // Skip the hash - msg := parts[1] - if idx := strings.Index(msg, ":"); idx > 0 && idx < 15 { - prefix := msg[:idx] - prefixCount[prefix]++ - } - } - - for prefix, count := range prefixCount { - if count >= 5 { - patterns = append(patterns, pattern{ - description: fmt.Sprintf("Commit convention: use '%s:' prefix for %s-related changes", prefix, prefix), - memoryType: "convention", - confidence: 0.7, - reason: fmt.Sprintf("'%s:' prefix used %d times in recent commits", prefix, count), - }) - } - } - - return patterns -} diff --git a/engine/ingest.go b/engine/ingest.go deleted file mode 100644 index 6769750..0000000 --- a/engine/ingest.go +++ /dev/null @@ -1,449 +0,0 @@ -package engine - -import ( - "bufio" - "context" - "fmt" - "os" - "path/filepath" - "regexp" - "strings" -) - -// Ingester handles bulk memory creation from various sources: -// conversations, markdown files, code files, CLAUDE.md, .cursorrules. -type Ingester struct { - engine *Engine -} - -// IngestResult tracks what was extracted from a source. -type IngestResult struct { - Source string - Conventions int - Decisions int - Bugs int - Specs int - Tasks int - Preferences int - Files int - Total int - Skipped int -} - -func (r *IngestResult) add(typ string) { - r.Total++ - switch typ { - case "convention": - r.Conventions++ - case "decision": - r.Decisions++ - case "bug": - r.Bugs++ - case "spec": - r.Specs++ - case "task": - r.Tasks++ - case "preference": - r.Preferences++ - case "file": - r.Files++ - } -} - -// NewIngester creates an ingester bound to an engine. -func NewIngester(engine *Engine) *Ingester { - return &Ingester{engine: engine} -} - -// IngestConversation parses a conversation transcript (alternating Human/Assistant) -// and extracts decisions, conventions, bugs mentioned. -func (ing *Ingester) IngestConversation(ctx context.Context, text string) (*IngestResult, error) { - result := &IngestResult{Source: "conversation"} - - lines := strings.Split(text, "\n") - var currentBlock strings.Builder - var currentRole string - - flush := func() { - if currentBlock.Len() == 0 { - return - } - content := currentBlock.String() - // Only extract from assistant responses (they contain decisions/conventions) - if currentRole == "assistant" || currentRole == "" { - ing.extractAndStore(ctx, content, result) - } - currentBlock.Reset() - } - - for _, line := range lines { - lower := strings.ToLower(strings.TrimSpace(line)) - switch { - case strings.HasPrefix(lower, "human:") || strings.HasPrefix(lower, "user:"): - flush() - currentRole = "human" - currentBlock.WriteString(strings.TrimPrefix(strings.TrimPrefix(line, "Human: "), "User: ")) - case strings.HasPrefix(lower, "assistant:") || strings.HasPrefix(lower, "ai:"): - flush() - currentRole = "assistant" - currentBlock.WriteString(strings.TrimPrefix(strings.TrimPrefix(line, "Assistant: "), "AI: ")) - default: - currentBlock.WriteString(" ") - currentBlock.WriteString(line) - } - } - flush() - - return result, nil -} - -// IngestMarkdown parses a markdown file and extracts memories from headers and lists. -func (ing *Ingester) IngestMarkdown(ctx context.Context, content, source string) (*IngestResult, error) { - result := &IngestResult{Source: source} - - lines := strings.Split(content, "\n") - var currentSection string - - for _, line := range lines { - trimmed := strings.TrimSpace(line) - - // Detect section headers - if strings.HasPrefix(trimmed, "# ") || strings.HasPrefix(trimmed, "## ") || strings.HasPrefix(trimmed, "### ") { - currentSection = strings.ToLower(strings.TrimLeft(trimmed, "# ")) - continue - } - - // Extract from list items - if (strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ")) && len(trimmed) > 10 { - item := strings.TrimPrefix(strings.TrimPrefix(trimmed, "- "), "* ") - nodeType := classifyFromSection(currentSection, item) - if nodeType != "" { - ing.remember(ctx, item, nodeType, result) - } - } - } - - return result, nil -} - -// IngestFile parses a markdown file from disk. -func (ing *Ingester) IngestFile(ctx context.Context, path string) (*IngestResult, error) { - data, err := os.ReadFile(path) // #nosec G304 -- path is a project file path passed in by the caller (e.g. CLI/agent-driven ingestion), not raw user network input - if err != nil { - return nil, fmt.Errorf("read %s: %w", path, err) - } - return ing.IngestMarkdown(ctx, string(data), filepath.Base(path)) -} - -// IngestClaudeMD imports from a CLAUDE.md file (conventions, rules, patterns). -func (ing *Ingester) IngestClaudeMD(ctx context.Context, path string) (*IngestResult, error) { - data, err := os.ReadFile(path) // #nosec G304 -- path is a project file path passed in by the caller (e.g. CLI/agent-driven ingestion), not raw user network input - if err != nil { - return nil, err - } - result := &IngestResult{Source: "CLAUDE.md"} - - lines := strings.Split(string(data), "\n") - for _, line := range lines { - trimmed := strings.TrimSpace(line) - if (strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ")) && len(trimmed) > 15 { - item := strings.TrimPrefix(strings.TrimPrefix(trimmed, "- "), "* ") - // CLAUDE.md items are mostly conventions - ing.remember(ctx, item, "convention", result) - } - } - return result, nil -} - -// IngestCursorRules imports from a .cursorrules file. -func (ing *Ingester) IngestCursorRules(ctx context.Context, path string) (*IngestResult, error) { - data, err := os.ReadFile(path) // #nosec G304 -- path is a project file path passed in by the caller (e.g. CLI/agent-driven ingestion), not raw user network input - if err != nil { - return nil, err - } - result := &IngestResult{Source: ".cursorrules"} - - lines := strings.Split(string(data), "\n") - for _, line := range lines { - trimmed := strings.TrimSpace(line) - if trimmed == "" || strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, "//") { - continue - } - if len(trimmed) > 10 { - ing.remember(ctx, trimmed, "convention", result) - } - } - return result, nil -} - -// IngestCodeFile extracts specs from source code (functions, types, package purpose). -func (ing *Ingester) IngestCodeFile(ctx context.Context, path string) (*IngestResult, error) { - data, err := os.ReadFile(path) // #nosec G304 -- path is a project file path passed in by the caller (e.g. CLI/agent-driven ingestion), not raw user network input - if err != nil { - return nil, err - } - result := &IngestResult{Source: filepath.Base(path)} - content := string(data) - - // Extract package-level comment as spec - if comment := extractPackageComment(content); comment != "" { - ing.remember(ctx, fmt.Sprintf("File %s: %s", filepath.Base(path), comment), "spec", result) - } - - // Extract exported function signatures as specs - funcs := extractFuncSignatures(content) - if len(funcs) > 0 && len(funcs) <= 10 { - summary := fmt.Sprintf("File %s exports: %s", filepath.Base(path), strings.Join(funcs, ", ")) - ing.remember(ctx, summary, "spec", result) - } - - return result, nil -} - -// DetectStack scans package files and returns the detected tech stack. -func (ing *Ingester) DetectStack(ctx context.Context, projectDir string) (*IngestResult, error) { - result := &IngestResult{Source: "stack-detection"} - var stack []string - - checks := []struct { - file string - techs []string - }{ - {"package.json", []string{"Node.js"}}, - {"go.mod", []string{"Go"}}, - {"Cargo.toml", []string{"Rust"}}, - {"pyproject.toml", []string{"Python"}}, - {"requirements.txt", []string{"Python"}}, - {"pom.xml", []string{"Java"}}, - {"build.gradle", []string{"Java/Kotlin"}}, - {"Gemfile", []string{"Ruby"}}, - {"composer.json", []string{"PHP"}}, - {"Dockerfile", []string{"Docker"}}, - {"docker-compose.yml", []string{"Docker Compose"}}, - {"docker-compose.yaml", []string{"Docker Compose"}}, - {".github/workflows", []string{"GitHub Actions"}}, - {"fly.toml", []string{"Fly.io"}}, - {"vercel.json", []string{"Vercel"}}, - {"netlify.toml", []string{"Netlify"}}, - {"tsconfig.json", []string{"TypeScript"}}, - {"next.config.js", []string{"Next.js"}}, - {"next.config.ts", []string{"Next.js"}}, - {"vite.config.ts", []string{"Vite"}}, - {"tailwind.config.js", []string{"Tailwind CSS"}}, - {"tailwind.config.ts", []string{"Tailwind CSS"}}, - {".eslintrc.json", []string{"ESLint"}}, - {".prettierrc", []string{"Prettier"}}, - {"jest.config.js", []string{"Jest"}}, - {"vitest.config.ts", []string{"Vitest"}}, - {"playwright.config.ts", []string{"Playwright"}}, - {"Makefile", []string{"Make"}}, - } - - for _, c := range checks { - p := filepath.Join(projectDir, c.file) - if info, err := os.Stat(p); err == nil { - if info.IsDir() || info.Size() > 0 { - stack = append(stack, c.techs...) - } - } - } - - // Detect from package.json dependencies - pkgJSON := filepath.Join(projectDir, "package.json") - // #nosec G304 -- pkgJSON is a fixed filename joined under the caller-supplied projectDir for stack detection - if data, err := os.ReadFile(pkgJSON); err == nil { - content := string(data) - npmDetect := map[string]string{ - "react": "React", "vue": "Vue", "svelte": "Svelte", "angular": "Angular", - "express": "Express", "fastify": "Fastify", "hono": "Hono", "koa": "Koa", - "prisma": "Prisma", "drizzle-orm": "Drizzle", "mongoose": "Mongoose", - "redis": "Redis", "pg": "PostgreSQL", "mysql2": "MySQL", - "jest": "Jest", "vitest": "Vitest", "mocha": "Mocha", - "tailwindcss": "Tailwind CSS", "sass": "Sass", - "trpc": "tRPC", "graphql": "GraphQL", - } - for pkg, name := range npmDetect { - if strings.Contains(content, `"`+pkg+`"`) { - stack = append(stack, name) - } - } - } - - // Detect from go.mod dependencies - goMod := filepath.Join(projectDir, "go.mod") - // #nosec G304 -- goMod is a fixed filename joined under the caller-supplied projectDir for stack detection - if data, err := os.ReadFile(goMod); err == nil { - content := string(data) - goDetect := map[string]string{ - "gin-gonic/gin": "Gin", "labstack/echo": "Echo", "gofiber/fiber": "Fiber", - "gorilla/mux": "Gorilla Mux", "spf13/cobra": "Cobra CLI", - "gorm.io/gorm": "GORM", "sqlc": "sqlc", "ent/ent": "Ent ORM", - "redis/go-redis": "Redis", "lib/pq": "PostgreSQL", "go-sql-driver/mysql": "MySQL", - "nats-io/nats": "NATS", "segmentio/kafka": "Kafka", - } - for pkg, name := range goDetect { - if strings.Contains(content, pkg) { - stack = append(stack, name) - } - } - } - - if len(stack) > 0 { - // Deduplicate - seen := make(map[string]bool) - var unique []string - for _, s := range stack { - if !seen[s] { - seen[s] = true - unique = append(unique, s) - } - } - content := fmt.Sprintf("Tech stack: %s", strings.Join(unique, ", ")) - ing.remember(ctx, content, "spec", result) - } - - return result, nil -} - -// IngestDirectory scans a project directory and ingests all relevant files. -func (ing *Ingester) IngestDirectory(ctx context.Context, dir string) (*IngestResult, error) { - total := &IngestResult{Source: dir} - - // 1. CLAUDE.md - if r, err := ing.IngestClaudeMD(ctx, filepath.Join(dir, "CLAUDE.md")); err == nil { - total.Total += r.Total - total.Conventions += r.Conventions - } - - // 2. .cursorrules - if r, err := ing.IngestCursorRules(ctx, filepath.Join(dir, ".cursorrules")); err == nil { - total.Total += r.Total - total.Conventions += r.Conventions - } - - // 3. Stack detection - if r, err := ing.DetectStack(ctx, dir); err == nil { - total.Total += r.Total - total.Specs += r.Specs - } - - // 4. README.md - if r, err := ing.IngestFile(ctx, filepath.Join(dir, "README.md")); err == nil { - total.Total += r.Total - } - - return total, nil -} - -// --- helpers --- - -func (ing *Ingester) remember(ctx context.Context, content, nodeType string, result *IngestResult) { - _, err := ing.engine.Remember(ctx, RememberInput{ - Type: nodeType, - Content: content, - Scope: "project", - }) - if err == nil { - result.add(nodeType) - } else { - result.Skipped++ - } -} - -func (ing *Ingester) extractAndStore(ctx context.Context, text string, result *IngestResult) { - sentences := splitSentences(text) - for _, s := range sentences { - s = strings.TrimSpace(s) - if len(s) < 15 { - continue - } - nodeType := classifyContent(s) - if nodeType != "" { - ing.remember(ctx, s, nodeType, result) - } - } -} - -var ( - conventionRe2 = regexp.MustCompile(`(?i)(always|never|must|should|prefer|avoid|use .+ instead|don't|do not|convention|rule|standard)`) - decisionRe2 = regexp.MustCompile(`(?i)(decided|chose|picked|selected|went with|switched to|migrated|adopted|we use)`) - bugRe2 = regexp.MustCompile(`(?i)(bug|fix|issue|error|broken|crash|race condition|deadlock|leak|regression)`) - taskRe2 = regexp.MustCompile(`(?i)(todo|TODO|FIXME|HACK|in progress|need to|should implement|blocked)`) - prefRe2 = regexp.MustCompile(`(?i)(i prefer|i like|my style|i want|personally|i always)`) -) - -func classifyContent(text string) string { - switch { - case decisionRe2.MatchString(text): - return "decision" - case conventionRe2.MatchString(text): - return "convention" - case bugRe2.MatchString(text): - return "bug" - case taskRe2.MatchString(text): - return "task" - case prefRe2.MatchString(text): - return "preference" - default: - return "" - } -} - -func classifyFromSection(section, item string) string { - switch { - case strings.Contains(section, "convention") || strings.Contains(section, "rule") || strings.Contains(section, "pattern"): - return "convention" - case strings.Contains(section, "decision") || strings.Contains(section, "architecture") || strings.Contains(section, "choice"): - return "decision" - case strings.Contains(section, "bug") || strings.Contains(section, "issue") || strings.Contains(section, "fix"): - return "bug" - case strings.Contains(section, "task") || strings.Contains(section, "todo") || strings.Contains(section, "plan"): - return "task" - case strings.Contains(section, "preference") || strings.Contains(section, "style"): - return "preference" - case strings.Contains(section, "spec") || strings.Contains(section, "design") || strings.Contains(section, "api"): - return "spec" - default: - return classifyContent(item) - } -} - -func splitSentences(text string) []string { - re := regexp.MustCompile(`[.!?\n]+`) - return re.Split(text, -1) -} - -func extractPackageComment(content string) string { - scanner := bufio.NewScanner(strings.NewReader(content)) - var comment strings.Builder - for scanner.Scan() { - line := scanner.Text() - switch { - case strings.HasPrefix(line, "// "): - comment.WriteString(strings.TrimPrefix(line, "// ")) - comment.WriteString(" ") - case strings.HasPrefix(line, "package ") || strings.HasPrefix(line, "import"): - // stop at package/import after collecting comments - case line == "" && comment.Len() > 0: - // blank line after comments, stop collecting - } - } - result := strings.TrimSpace(comment.String()) - if len(result) > 200 { - result = result[:200] - } - return result -} - -var funcSigRe = regexp.MustCompile(`(?m)^func\s+(?:\([^)]+\)\s+)?([A-Z][a-zA-Z0-9]+)`) - -func extractFuncSignatures(content string) []string { - matches := funcSigRe.FindAllStringSubmatch(content, 20) - var funcs []string - for _, m := range matches { - if len(m) > 1 { - funcs = append(funcs, m[1]) - } - } - return funcs -} diff --git a/engine/proactive.go b/engine/proactive.go index 8e4cded..8fce854 100644 --- a/engine/proactive.go +++ b/engine/proactive.go @@ -2,6 +2,7 @@ package engine import ( "context" + "log/slog" "sort" "github.com/GrayCodeAI/yaad/storage" @@ -43,7 +44,11 @@ func (p *ProactiveContext) Predict(ctx context.Context, project string, budget i break } candidates[n.ID] = n - neighbors, _ := p.eng.store.GetNeighbors(ctx, n.ID) + neighbors, err := p.eng.store.GetNeighbors(ctx, n.ID) + if err != nil { + slog.Warn("proactive: get neighbors failed", "node", n.ID, "error", err) + continue + } for _, nb := range neighbors { candidates[nb.ID] = nb } @@ -55,7 +60,11 @@ func (p *ProactiveContext) Predict(ctx context.Context, project string, budget i continue } candidates[t.ID] = t - edges, _ := p.eng.store.GetEdgesFrom(ctx, t.ID) + edges, err := p.eng.store.GetEdgesFrom(ctx, t.ID) + if err != nil { + slog.Warn("proactive: get edges failed", "node", t.ID, "error", err) + continue + } for _, e := range edges { if e.Type == "depends_on" || e.Type == "relates_to" { if n, err := p.eng.store.GetNode(ctx, e.ToID); err == nil { @@ -74,7 +83,11 @@ func (p *ProactiveContext) Predict(ctx context.Context, project string, budget i for _, n := range all { allIDs = append(allIDs, n.ID) } - edgeCounts, _ := p.eng.store.CountEdgesBatch(ctx, allIDs) + edgeCounts, err := p.eng.store.CountEdgesBatch(ctx, allIDs) + if err != nil { + slog.Warn("proactive: count edges batch failed", "error", err) + edgeCounts = nil + } var ranked []centNode for _, n := range all { inDegree := 0 diff --git a/engine/ratelimit_test.go b/engine/ratelimit_test.go new file mode 100644 index 0000000..2732a81 --- /dev/null +++ b/engine/ratelimit_test.go @@ -0,0 +1,62 @@ +package engine + +import ( + "testing" + "time" +) + +func TestRateLimiter_AllowWithinBurst(t *testing.T) { + t.Parallel() + rl := NewRateLimiter(10, 5) + + // First 5 requests (burst) should all succeed. + for i := 0; i < 5; i++ { + if !rl.Allow() { + t.Errorf("request %d: Allow = false, want true", i+1) + } + } + // 6th request fails — no tokens left until refill. + if rl.Allow() { + t.Error("request 6: Allow = true, want false (burst exhausted)") + } +} + +func TestRateLimiter_Remaining(t *testing.T) { + t.Parallel() + rl := NewRateLimiter(10, 5) + if rl.Remaining() != 5 { + t.Errorf("Remaining = %d, want 5", rl.Remaining()) + } + rl.Allow() + if rl.Remaining() != 4 { + t.Errorf("Remaining after 1 Allow = %d, want 4", rl.Remaining()) + } +} + +func TestRateLimiter_Refill(t *testing.T) { + t.Parallel() + // 10 req/sec, burst of 1. + rl := NewRateLimiter(10, 1) + if !rl.Allow() { + t.Fatal("first Allow = false, want true") + } + if rl.Allow() { + t.Error("second Allow = true, want false (no tokens)") + } + + // Wait for one token to refill (100ms for 10 req/sec). + time.Sleep(150 * time.Millisecond) + if !rl.Allow() { + t.Error("Allow after refill = false, want true") + } +} + +func TestRateLimiter_MaxBurstCap(t *testing.T) { + t.Parallel() + rl := NewRateLimiter(1000, 3) + // Wait long enough to refill many tokens, but it must cap at maxBurst. + time.Sleep(200 * time.Millisecond) + if rl.Remaining() > 3 { + t.Errorf("Remaining = %d, want <= 3 (maxBurst cap)", rl.Remaining()) + } +} diff --git a/engine/spaced_repetition.go b/engine/spaced_repetition.go deleted file mode 100644 index d0a2123..0000000 --- a/engine/spaced_repetition.go +++ /dev/null @@ -1,145 +0,0 @@ -package engine - -import ( - "context" - "math" - "sync" - "time" - - "github.com/GrayCodeAI/yaad/storage" -) - -// SpacedRepetition provides FSRS-6 style memory decay and review scheduling. -// Memories lose retrieval strength over time unless reviewed. -// Frequently accessed memories persist; unused ones fade. -type SpacedRepetition struct { - store *storage.Store - mu sync.Mutex - records map[string]*ReviewRecord // in-memory cache until reviews table is implemented -} - -// ReviewRecord tracks a single memory's retrieval strength. -type ReviewRecord struct { - NodeID string `json:"node_id"` - Stability float64 `json:"stability"` // days until retrieval strength drops to 90% - Difficulty float64 `json:"difficulty"` // 0=easy, 1=hard - LastReview time.Time `json:"last_review"` - ReviewCount int `json:"review_count"` -} - -// Default parameters (FSRS-6 inspired). -const ( - defaultStability = 1.0 // days - defaultDifficulty = 0.5 // medium - decayFactor = 0.9 // retention decay per stability period - minStability = 0.01 - maxStability = 365.0 // 1 year max -) - -// NewSpacedRepetition creates a spaced repetition tracker. -func NewSpacedRepetition(store *storage.Store) *SpacedRepetition { - return &SpacedRepetition{store: store} -} - -// RecordReview records an access to a memory node, updating its retrieval strength. -func (sr *SpacedRepetition) RecordReview(ctx context.Context, nodeID string) error { - sr.mu.Lock() - defer sr.mu.Unlock() - - now := time.Now() - record, err := sr.loadOrCreate(ctx, nodeID) - if err != nil { - return err - } - - if !record.LastReview.IsZero() { - elapsed := now.Sub(record.LastReview).Hours() / 24 - ratio := elapsed / record.Stability - if ratio > 0 { - record.Stability *= math.Exp(record.Difficulty * (ratio - 1)) - } - } - record.Stability = clamp(record.Stability, minStability, maxStability) - record.LastReview = now - record.ReviewCount++ - - return sr.saveRecord(ctx, record) -} - -// loadOrCreate returns an existing review record or creates a default one. -func (sr *SpacedRepetition) loadOrCreate(ctx context.Context, nodeID string) (*ReviewRecord, error) { - record, err := sr.loadRecord(ctx, nodeID) - if err != nil || record == nil { - return &ReviewRecord{ - NodeID: nodeID, - Stability: defaultStability, - Difficulty: defaultDifficulty, - LastReview: time.Now(), - }, nil - } - return record, nil -} - -// RetrievalStrength returns the current retrieval strength (0-1) for a memory. -func (sr *SpacedRepetition) RetrievalStrength(ctx context.Context, nodeID string) (float64, error) { - record, err := sr.loadRecord(ctx, nodeID) - if err != nil { - return 0, err - } - if record == nil { - return 0, nil - } - elapsed := time.Since(record.LastReview).Hours() / 24 - if elapsed <= 0 { - return 1.0, nil - } - // Retention R follows the exponential forgetting curve: e raised to - // (-elapsed / stability). - strength := math.Exp(-elapsed / record.Stability) - return clamp(strength, 0, 1), nil -} - -// ShouldReview returns true if the memory is due for review (strength below threshold). -func (sr *SpacedRepetition) ShouldReview(ctx context.Context, nodeID string) (bool, error) { - strength, err := sr.RetrievalStrength(ctx, nodeID) - if err != nil { - return false, err - } - return strength < 0.6, nil -} - -// loadRecord retrieves a review record from storage. -// NOTE: SpacedRepetition uses an in-memory store because the dedicated -// reviews table has not been implemented yet. RecordReview tracks -// records in-memory for the lifetime of the process; persistence is a -// future enhancement. -func (sr *SpacedRepetition) loadRecord(ctx context.Context, nodeID string) (*ReviewRecord, error) { - // Use in-memory cache keyed by nodeID. - if sr.records == nil { - return nil, nil - } - rec, ok := sr.records[nodeID] - if !ok { - return nil, nil - } - return rec, nil -} - -// saveRecord persists a review record (in-memory until a reviews table is added). -func (sr *SpacedRepetition) saveRecord(ctx context.Context, record *ReviewRecord) error { - if sr.records == nil { - sr.records = make(map[string]*ReviewRecord) - } - sr.records[record.NodeID] = record - return nil -} - -func clamp(v, min, max float64) float64 { - if v < min { - return min - } - if v > max { - return max - } - return v -} diff --git a/engine/temporal_filter_test.go b/engine/temporal_filter_test.go new file mode 100644 index 0000000..72a9ae0 --- /dev/null +++ b/engine/temporal_filter_test.go @@ -0,0 +1,85 @@ +package engine + +import ( + "testing" + "time" + + "github.com/GrayCodeAI/yaad/storage" +) + +func nodeWithTimes(id string, created, updated time.Time) *storage.Node { + return &storage.Node{ + ID: id, + CreatedAt: created, + UpdatedAt: updated, + } +} + +func TestMatchesTemporal_After(t *testing.T) { + t.Parallel() + tf := NewTemporalFilter(nil) + + cutoff := time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC) + before := nodeWithTimes("a", time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), time.Time{}) + after := nodeWithTimes("b", time.Date(2026, 1, 20, 0, 0, 0, 0, time.UTC), time.Time{}) + + if tf.matchesTemporal(before, TemporalQuery{After: cutoff}) { + t.Error("node before cutoff matched After query") + } + if !tf.matchesTemporal(after, TemporalQuery{After: cutoff}) { + t.Error("node after cutoff did not match After query") + } +} + +func TestMatchesTemporal_Before(t *testing.T) { + t.Parallel() + tf := NewTemporalFilter(nil) + + cutoff := time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC) + before := nodeWithTimes("a", time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), time.Time{}) + after := nodeWithTimes("b", time.Date(2026, 1, 20, 0, 0, 0, 0, time.UTC), time.Time{}) + + if !tf.matchesTemporal(before, TemporalQuery{Before: cutoff}) { + t.Error("node before cutoff did not match Before query") + } + if tf.matchesTemporal(after, TemporalQuery{Before: cutoff}) { + t.Error("node after cutoff matched Before query") + } +} + +func TestMatchesTemporal_Range(t *testing.T) { + t.Parallel() + tf := NewTemporalFilter(nil) + + start := time.Date(2026, 1, 10, 0, 0, 0, 0, time.UTC) + end := time.Date(2026, 1, 20, 0, 0, 0, 0, time.UTC) + + outside := nodeWithTimes("a", time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC), time.Time{}) + inside := nodeWithTimes("b", time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC), time.Time{}) + + if tf.matchesTemporal(outside, TemporalQuery{After: start, Before: end}) { + t.Error("node outside range matched") + } + if !tf.matchesTemporal(inside, TemporalQuery{After: start, Before: end}) { + t.Error("node inside range did not match") + } +} + +func TestFilterByTime(t *testing.T) { + t.Parallel() + tf := NewTemporalFilter(nil) + + cutoff := time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC) + nodes := []*storage.Node{ + nodeWithTimes("old", time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), time.Time{}), + nodeWithTimes("new", time.Date(2026, 1, 20, 0, 0, 0, 0, time.UTC), time.Time{}), + } + + got := tf.FilterByTime(nil, nodes, TemporalQuery{After: cutoff}) + if len(got) != 1 { + t.Fatalf("FilterByTime returned %d nodes, want 1", len(got)) + } + if got[0].ID != "new" { + t.Errorf("got node %q, want new", got[0].ID) + } +} diff --git a/spatial_memory.go b/spatial_memory.go index 4475b9f..c959042 100644 --- a/spatial_memory.go +++ b/spatial_memory.go @@ -87,9 +87,6 @@ func (sm *SpatialMemory) Add(entry MemoryEntry) { if entry.CreatedAt.IsZero() { entry.CreatedAt = now } - if entry.CreatedAt.IsZero() { - entry.CreatedAt = now - } entry.Tier = TierCold entry.AccessCount = 0 From 96a7d4770d3ea3338d5bb8937fe9693d407932a6 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sat, 25 Jul 2026 12:31:20 +0530 Subject: [PATCH 4/5] feat: add portable graph module - Add portablegraph/ package with portable graph implementation - Update ARCHITECTURE.md, README, go.mod - Update README --- ARCHITECTURE.md | 13 +- README.md | 12 ++ go.mod | 3 + portablegraph/projection.go | 251 +++++++++++++++++++++++++++++++ portablegraph/projection_test.go | 109 ++++++++++++++ 5 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 portablegraph/projection.go create mode 100644 portablegraph/projection_test.go diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8dd3d17..24c213e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -42,7 +42,7 @@ v v v v +---------------------------------------------------------------------------------------------------+ | [ EXPORT / IMPORT (.yaadpack) (exportimport/pack.go) ] | -| - Portable JSON Export | - Signed Team Packs (.yaadpack) | - HMAC-SHA256 Signature Validator | +| - Portable JSON Export | - Shared Context Graph Projection | - Signed/HMAC Team Packs | +-----------------------------------v---------------------------------------------------------------+ | v @@ -66,6 +66,17 @@ Storage Locations: ## Component Details +### Portable context projection (`portablegraph/`) + +The portable projection consumes the `storage.Node` and `storage.Edge` +snapshots already selected by retrieval. It emits shared knowledge nodes, +portable relationships, provenance, temporal validity, and `observed` events +using `hawk-core-contracts/graph`. + +It never runs retrieval, changes ranking, mutates SQLite, or exposes memory +content. Hawk records these projections as inference-context evidence and +links them to the owning execution session. + ### 1. MCP Server (`internal/server/mcp.go`) Implements Model Context Protocol over stdio transport. diff --git a/README.md b/README.md index bb8f8ce..3203357 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,18 @@ Not a flat list of memories. A directed graph with 8 edge types: Enables: subgraph extraction, impact analysis, causal chain traversal. +
+Portable Context Graph + +`portablegraph.Build` projects the exact nodes and edges selected by existing +Yaad retrieval into `hawk-core-contracts/graph`. The projection is +deterministic, repository-scoped, temporal, and metadata-only: memory content, +queries, source sessions, and agents are represented by SHA-256 digests. + +Yaad's SQLite graph remains the source of truth; the portable graph is a +read-only handoff for hosts such as Hawk. +
+
Intent-Aware 4-Path Search diff --git a/go.mod b/go.mod index 22fbd86..1b1e77a 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.5 require ( github.com/BurntSushi/toml v1.6.0 + github.com/GrayCodeAI/hawk-core-contracts v0.1.8 github.com/GrayCodeAI/hawk-mcpkit v0.0.0 github.com/google/uuid v1.6.0 github.com/mark3labs/mcp-go v0.49.0 @@ -44,3 +45,5 @@ require ( // yaad depends on ServeHTTPWithShutdown and other APIs added after v0.1.4; // switch to a tagged version once v0.2.0 is released. replace github.com/GrayCodeAI/hawk-mcpkit => ../hawk-mcpkit + +replace github.com/GrayCodeAI/hawk-core-contracts => ../hawk-core-contracts diff --git a/portablegraph/projection.go b/portablegraph/projection.go new file mode 100644 index 0000000..1f91390 --- /dev/null +++ b/portablegraph/projection.go @@ -0,0 +1,251 @@ +// Package portablegraph projects Yaad-owned memory subgraphs into the shared +// hawk-eco graph contract. Yaad's SQLite LPG remains the source of truth. +package portablegraph + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strconv" + "strings" + "time" + + graphcontracts "github.com/GrayCodeAI/hawk-core-contracts/graph" + "github.com/GrayCodeAI/yaad/storage" +) + +const SchemaVersion = "yaad.graph/v1" + +// Export is a deterministic, privacy-safe projection of one retrieved Yaad +// context subgraph. +type Export struct { + SchemaVersion string `json:"schema_version"` + GeneratedAt time.Time `json:"generated_at"` + QuerySHA256 string `json:"query_sha256,omitempty"` + Scope graphcontracts.Scope `json:"scope,omitempty"` + Nodes []graphcontracts.Node `json:"nodes"` + Edges []graphcontracts.Edge `json:"edges"` + Events []graphcontracts.Event `json:"events"` +} + +// Input contains the Yaad-owned snapshots selected by existing retrieval. +type Input struct { + Nodes []*storage.Node + Edges []*storage.Edge + Query string + GeneratedAt time.Time + Scope graphcontracts.Scope + ProducerVersion string +} + +// Build projects and validates one retrieved memory subgraph. It never mutates +// Yaad storage, ranking, access logs, or graph topology. +func Build(input Input) (Export, error) { + if input.GeneratedAt.IsZero() { + return Export{}, fmt.Errorf("yaad portable graph: generated time is required") + } + generatedAt := input.GeneratedAt.UTC() + querySHA256 := digest(input.Query) + export := Export{ + SchemaVersion: SchemaVersion, + GeneratedAt: generatedAt, + QuerySHA256: querySHA256, + Scope: input.Scope, + Nodes: make([]graphcontracts.Node, 0, len(input.Nodes)), + Edges: make([]graphcontracts.Edge, 0, len(input.Edges)), + Events: make([]graphcontracts.Event, 0, len(input.Nodes)), + } + + nodeIDs := make(map[string]struct{}, len(input.Nodes)) + sourceIDs := make(map[string]graphcontracts.Ref, len(input.Nodes)) + for _, memory := range input.Nodes { + if memory == nil { + continue + } + sourceID := strings.TrimSpace(memory.ID) + if sourceID == "" { + return Export{}, fmt.Errorf("yaad portable graph: memory ID is required") + } + ref := graphcontracts.Ref{ + Kind: graphcontracts.NodeKnowledge, + ID: memoryNodeID(sourceID), + } + if _, exists := nodeIDs[ref.ID]; exists { + continue + } + + createdAt := graphTime(memory.CreatedAt, generatedAt) + attributes := map[string]string{ + "entity_type": "memory", + "data_classification": "metadata_only", + "memory_type": strings.TrimSpace(memory.Type), + "tier": strconv.Itoa(memory.Tier), + "confidence": strconv.FormatFloat(memory.Confidence, 'f', -1, 64), + "pinned": strconv.FormatBool(memory.Pinned), + "version": strconv.Itoa(memory.Version), + "access_count": strconv.Itoa(memory.AccessCount), + "content_sha256": contentDigest(memory), + } + addAttribute(attributes, "source_session_sha256", digest(memory.SourceSession)) + addAttribute(attributes, "source_agent_sha256", digest(memory.SourceAgent)) + node := graphcontracts.Node{ + ID: ref.ID, + Kind: ref.Kind, + Scope: input.Scope, + CreatedAt: createdAt, + EffectiveAt: graphTime(memory.UpdatedAt, createdAt), + Provenance: graphcontracts.Provenance{ + Producer: "yaad", + Version: strings.TrimSpace(input.ProducerVersion), + SourceID: sourceID, + Evidence: []graphcontracts.ArtifactRef{{URI: "yaad://memory/" + sourceID}}, + }, + Attributes: attributes, + } + if err := node.Validate(); err != nil { + return Export{}, fmt.Errorf("yaad portable graph: node %q: %w", ref.ID, err) + } + export.Nodes = append(export.Nodes, node) + nodeIDs[ref.ID] = struct{}{} + sourceIDs[sourceID] = ref + + event := graphcontracts.Event{ + ID: observedEventID(sourceID, querySHA256), + Type: graphcontracts.EventObserved, + Subject: ref, + Scope: input.Scope, + OccurredAt: generatedAt, + CorrelationID: querySHA256, + IdempotencyKey: observedEventID(sourceID, querySHA256), + Provenance: node.Provenance, + } + if err := event.Validate(); err != nil { + return Export{}, fmt.Errorf("yaad portable graph: event %q: %w", event.ID, err) + } + export.Events = append(export.Events, event) + } + + edgeIDs := make(map[string]struct{}, len(input.Edges)) + for _, relation := range input.Edges { + if relation == nil { + continue + } + from, fromExists := sourceIDs[strings.TrimSpace(relation.FromID)] + to, toExists := sourceIDs[strings.TrimSpace(relation.ToID)] + if !fromExists || !toExists { + return Export{}, fmt.Errorf( + "yaad portable graph: edge %q references memory outside the retrieved subgraph", + relation.ID, + ) + } + sourceID := strings.TrimSpace(relation.ID) + if sourceID == "" { + sourceID = digest(relation.FromID + "\x00" + relation.Type + "\x00" + relation.ToID) + } + edge := graphcontracts.Edge{ + ID: relationEdgeID(sourceID), + Kind: portableEdgeKind(relation.Type), + From: from, + To: to, + Scope: input.Scope, + CreatedAt: graphTime(relation.CreatedAt, generatedAt), + EffectiveAt: graphTime(relation.ValidAt, graphTime(relation.CreatedAt, generatedAt)), + Provenance: graphcontracts.Provenance{ + Producer: "yaad", + Version: strings.TrimSpace(input.ProducerVersion), + SourceID: sourceID, + Evidence: []graphcontracts.ArtifactRef{{URI: "yaad://edge/" + sourceID}}, + }, + Attributes: map[string]string{ + "entity_type": "memory_relation", + "data_classification": "metadata_only", + "relation_type": strings.TrimSpace(relation.Type), + "acyclic": strconv.FormatBool(relation.Acyclic), + "weight": strconv.FormatFloat(relation.Weight, 'f', -1, 64), + }, + } + if !relation.InvalidAt.IsZero() { + edge.Attributes["invalid_at"] = relation.InvalidAt.UTC().Format(time.RFC3339Nano) + } + if err := edge.Validate(); err != nil { + return Export{}, fmt.Errorf("yaad portable graph: edge %q: %w", edge.ID, err) + } + if _, exists := edgeIDs[edge.ID]; exists { + continue + } + edgeIDs[edge.ID] = struct{}{} + export.Edges = append(export.Edges, edge) + } + + sort.Slice(export.Nodes, func(i, j int) bool { return export.Nodes[i].ID < export.Nodes[j].ID }) + sort.Slice(export.Edges, func(i, j int) bool { return export.Edges[i].ID < export.Edges[j].ID }) + sort.Slice(export.Events, func(i, j int) bool { return export.Events[i].ID < export.Events[j].ID }) + return export, nil +} + +func portableEdgeKind(relationType string) graphcontracts.EdgeKind { + switch strings.TrimSpace(relationType) { + case "part_of": + return graphcontracts.EdgeContains + case "depends_on": + return graphcontracts.EdgeDependsOn + default: + return graphcontracts.EdgeReferences + } +} + +func contentDigest(memory *storage.Node) string { + if memory == nil { + return "" + } + if value := normalizedSHA256(memory.ContentHash); value != "" { + return value + } + return digest(memory.Content) +} + +func normalizedSHA256(value string) string { + value = strings.TrimSpace(value) + if len(value) != sha256.Size*2 { + return "" + } + decoded, err := hex.DecodeString(value) + if err != nil || hex.EncodeToString(decoded) != value { + return "" + } + return value +} + +func memoryNodeID(sourceID string) string { + return "yaad/memory/" + sourceID +} + +func relationEdgeID(sourceID string) string { + return "yaad/edge/" + sourceID +} + +func observedEventID(sourceID, querySHA256 string) string { + return "yaad/event/memory/" + sourceID + "/observed/" + querySHA256 +} + +func graphTime(value, fallback time.Time) time.Time { + if value.IsZero() { + return fallback + } + return value.UTC() +} + +func addAttribute(attributes map[string]string, key, value string) { + if value = strings.TrimSpace(value); value != "" { + attributes[key] = value + } +} + +func digest(value string) string { + if strings.TrimSpace(value) == "" { + return "" + } + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} diff --git a/portablegraph/projection_test.go b/portablegraph/projection_test.go new file mode 100644 index 0000000..d1c4190 --- /dev/null +++ b/portablegraph/projection_test.go @@ -0,0 +1,109 @@ +package portablegraph + +import ( + "encoding/json" + "strings" + "testing" + "time" + + graphcontracts "github.com/GrayCodeAI/hawk-core-contracts/graph" + "github.com/GrayCodeAI/yaad/storage" +) + +func TestBuildProjectsRetrievedMemorySubgraphWithoutPayloads(t *testing.T) { + t.Parallel() + + generatedAt := time.Date(2026, time.July, 25, 5, 0, 0, 0, time.UTC) + first := &storage.Node{ + ID: "memory-1", + Type: "decision", + Content: "private decision content", + Scope: "project", + Project: "private project", + Confidence: 0.9, + Tier: 1, + Pinned: true, + CreatedAt: generatedAt.Add(-time.Hour), + UpdatedAt: generatedAt.Add(-time.Minute), + SourceSession: "private session", + SourceAgent: "private agent", + Version: 2, + } + second := &storage.Node{ + ID: "memory-2", + Type: "file", + Content: "private file content", + Confidence: 0.8, + CreatedAt: generatedAt.Add(-30 * time.Minute), + } + relation := &storage.Edge{ + ID: "relation-1", + FromID: first.ID, + ToID: second.ID, + Type: "touches", + Weight: 0.7, + CreatedAt: generatedAt.Add(-20 * time.Minute), + } + + export, err := Build(Input{ + Nodes: []*storage.Node{second, first}, + Edges: []*storage.Edge{relation}, + Query: "private retrieval query", + GeneratedAt: generatedAt, + Scope: graphcontracts.Scope{RepositoryID: "hawk"}, + ProducerVersion: "test", + }) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + if export.SchemaVersion != SchemaVersion { + t.Fatalf("SchemaVersion = %q, want %q", export.SchemaVersion, SchemaVersion) + } + if len(export.Nodes) != 2 || len(export.Edges) != 1 || len(export.Events) != 2 { + t.Fatalf("projection sizes = nodes:%d edges:%d events:%d", len(export.Nodes), len(export.Edges), len(export.Events)) + } + if export.Edges[0].Kind != graphcontracts.EdgeReferences { + t.Fatalf("edge kind = %q, want references", export.Edges[0].Kind) + } + + encoded, err := json.Marshal(export) + if err != nil { + t.Fatalf("json.Marshal() error = %v", err) + } + for _, secret := range []string{ + "private decision content", + "private file content", + "private project", + "private session", + "private agent", + "private retrieval query", + } { + if strings.Contains(string(encoded), secret) { + t.Fatalf("portable graph leaked %q", secret) + } + } +} + +func TestBuildRejectsDanglingRetrievedEdge(t *testing.T) { + t.Parallel() + + _, err := Build(Input{ + Nodes: []*storage.Node{{ID: "memory-1"}}, + Edges: []*storage.Edge{{ + ID: "edge-1", + FromID: "memory-1", + ToID: "missing", + }}, + GeneratedAt: time.Date(2026, time.July, 25, 6, 0, 0, 0, time.UTC), + }) + if err == nil || !strings.Contains(err.Error(), "outside the retrieved subgraph") { + t.Fatalf("Build() error = %v, want dangling-edge error", err) + } +} + +func TestBuildRequiresGeneratedTime(t *testing.T) { + t.Parallel() + if _, err := Build(Input{}); err == nil { + t.Fatal("Build() error = nil, want generated-time error") + } +} From 188b371176b019aaa407ec6e3b6e792cf7523106 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sat, 25 Jul 2026 13:07:07 +0530 Subject: [PATCH 5/5] feat: add portable graph representation - Add PortableGraph implementation - Support generic JSON serialization - Add conversion to portable GraphSpec --- portablegraph/portable_graph.go | 169 ++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 portablegraph/portable_graph.go diff --git a/portablegraph/portable_graph.go b/portablegraph/portable_graph.go new file mode 100644 index 0000000..efb9677 --- /dev/null +++ b/portablegraph/portable_graph.go @@ -0,0 +1,169 @@ +// Package graphjournal provides graph-based execution journaling for yaad. +package portablegraph + +import ( + "encoding/json" + "fmt" + "sync" + "time" + + graphcontracts "github.com/GrayCodeAI/hawk-core-contracts/graph" +) + +// PortableNode represents a portable graph node. +type PortableNode struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Attrs map[string]interface{} `json:"attrs,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// PortableEdge represents a portable graph edge. +type PortableEdge struct { + ID string `json:"id"` + From string `json:"from"` + To string `json:"to"` + Kind string `json:"kind"` // "depends_on", "produced", "references" + Weight float64 `json:"weight"` + Attrs map[string]interface{} `json:"attrs,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// PortableGraph represents a portable, serializable graph. +type PortableGraph struct { + mu sync.RWMutex + ID string `json:"id"` + Name string `json:"name"` + Nodes map[string]*PortableNode `json:"nodes"` + Edges []PortableEdge `json:"edges"` + Attrs map[string]interface{} `json:"attrs,omitempty"` +} + +// NewPortableGraph creates a new portable graph. +func NewPortableGraph(id, name string) *PortableGraph { + return &PortableGraph{ + ID: id, + Name: name, + Nodes: make(map[string]*PortableNode), + Attrs: make(map[string]interface{}), + } +} + +// AddNode adds a node to the graph. +func (g *PortableGraph) AddNode(node *PortableNode) { + g.mu.Lock() + defer g.mu.Unlock() + if node.CreatedAt.IsZero() { + node.CreatedAt = time.Now() + } + node.UpdatedAt = time.Now() + g.Nodes[node.ID] = node +} + +// AddEdge adds an edge to the graph. +func (g *PortableGraph) AddEdge(edge *PortableEdge) { + g.mu.Lock() + defer g.mu.Unlock() + if edge.CreatedAt.IsZero() { + edge.CreatedAt = time.Now() + } + g.Edges = append(g.Edges, *edge) +} + +// GetNode retrieves a node by ID. +func (g *PortableGraph) GetNode(id string) (*PortableNode, bool) { + g.mu.RLock() + defer g.mu.RUnlock() + node, ok := g.Nodes[id] + return node, ok +} + +// GetNodes returns all nodes. +func (g *PortableGraph) GetNodes() []*PortableNode { + g.mu.RLock() + defer g.mu.RUnlock() + result := make([]*PortableNode, 0, len(g.Nodes)) + for _, node := range g.Nodes { + result = append(result, node) + } + return result +} + +// GetEdges returns all edges. +func (g *PortableGraph) GetEdges() []PortableEdge { + g.mu.RLock() + defer g.mu.RUnlock() + return g.Edges +} + +// ToJSON serializes the graph to JSON. +func (g *PortableGraph) ToJSON() ([]byte, error) { + g.mu.RLock() + defer g.mu.RUnlock() + return json.Marshal(g) +} + +// FromJSON deserializes a graph from JSON. +func FromJSON(data []byte) (*PortableGraph, error) { + var g PortableGraph + if err := json.Unmarshal(data, &g); err != nil { + return nil, fmt.Errorf("failed to deserialize graph: %w", err) + } + return &g, nil +} + +// ToGraphSpec converts the portable graph to a portable graph spec. +func (g *PortableGraph) ToGraphSpec() *graphcontracts.GraphSpec { + g.mu.RLock() + defer g.mu.RUnlock() + + nodes := make([]graphcontracts.NodeSpec, 0, len(g.Nodes)) + for id, node := range g.Nodes { + config := make(map[string]string) + for k, v := range node.Attrs { + config[k] = fmt.Sprintf("%v", v) + } + config["name"] = node.Name + config["type"] = node.Type + + nodes = append(nodes, graphcontracts.NodeSpec{ + ID: id, + Type: graphcontracts.NodeTypeSystem, + Name: node.Name, + Config: config, + }) + } + + edges := make([]graphcontracts.EdgeSpec, 0, len(g.Edges)) + for _, edge := range g.Edges { + edges = append(edges, graphcontracts.EdgeSpec{ + From: edge.From, + To: edge.To, + Weight: edge.Weight, + }) + } + + return &graphcontracts.GraphSpec{ + ID: g.ID, + Name: g.Name, + Nodes: nodes, + Edges: edges, + } +} + +// Export exports the graph as a map. +func (g *PortableGraph) Export() map[string]interface{} { + g.mu.RLock() + defer g.mu.RUnlock() + + return map[string]interface{}{ + "id": g.ID, + "name": g.Name, + "nodes": g.Nodes, + "edges": g.Edges, + "attrs": g.Attrs, + } +}