feat: ecosystem architecture enhancements (Graph, Loop, Context, Harness)

README.md

@@ -18,7 +18,8 @@ tok is a **library**, not a CLI. It exposes token-efficiency primitives as a cle
 - **Output filtering** — a 31-layer pipeline (entropy pruning, perplexity filtering, AST-aware compression, H2O heavy-hitter, …) that strips noise from command output before it re-enters an LLM context.
 - **Token estimation** — `tok.EstimateTokens*` and `tok.EstimateCost` give model-aware token counts and pricing.
 - **Secrets scanning** — `tok.SecretDetector` and `tok.IsSensitiveFilename` catch credentials before they leak into prompts.
-- **Rate limiting & tracking** — persistent SQLite-backed gain tracking (`tok.NewTracker`).
+- **Usage limits & tracking** — thread-safe token/cost windows (`tok.NewUsageTracker`) plus persistent SQLite-backed gain tracking (`tok.NewTracker`).
+- **Runtime graph projection** — privacy-safe compression, usage, budget, and redaction summaries (`runtimegraph.Build`).
 
 It is consumed directly as a Go module, and it powers the `tok` commands inside [Hawk](https://github.com/GrayCodeAI/hawk) (`hawk tok ...`), which imports it as a library.
 
@@ -52,7 +53,7 @@ import "github.com/GrayCodeAI/tok"
 ### Compress a prompt
 
 ```go
-out := tok.PromptCompress("Please implement authentication", tok.IntensityUltra)
+out, _ := tok.PromptCompress("Please implement authentication", tok.IntensityUltra)
 // → "Implement auth."
 ```
 
@@ -78,6 +79,28 @@ findings := d.DetectSecrets(text)
 redacted := d.RedactSecrets(text)
 ```
 
+### Track usage and enforce a budget
+
+```go
+usage := tok.NewUsageTracker()
+usage.SetLimits(tok.UsageLimits{
+    HourlyTokens:  100_000,
+    SessionTokens: 500_000,
+    CostUSD:       10,
+})
+
+if allowed, _ := usage.CanProceed(); allowed {
+    usage.Record(promptTokens+completionTokens, requestCostUSD, provider, model)
+}
+
+allowed, reason := usage.CanProceed()
+snapshot := usage.GetUsage()
+```
+
+`SetLimits` and `GetLimits` are safe to use while requests are being recorded.
+Callers own request deduplication and decide whether a failed decision blocks
+the current request or the next one.
+
 ---
 
 ## Library API Highlights
@@ -88,6 +111,7 @@ redacted := d.RedactSecrets(text)
 - **`tok.SmartTruncate(content, maxLines, lang)`** — code truncation that preserves function signatures and **always reports the exact drop count** (`kept + dropped == total`).
 - **`tok.ExtractJSON / ExtractJSONArray / ExtractAllJSON`** — brace-balanced JSON extraction from LLM output with surrounding prose, markdown fences, and unterminated objects.
 - **`tok.NewTracker(ctx)`** — persistent gain tracker (SQLite + WAL, 90-day retention, pure-Go via `modernc.org/sqlite`). `Aggregate`, `Recent`, `Prune` queries.
+- **`tok.NewUsageTracker()`** — in-memory hourly, daily, session, and cost accounting with atomic limit snapshots and threshold decisions.
 - **`tok.EstimateTokensFast / WithEncoding / ForModel`** — model-aware token estimation.
 - **`filter.CompressWithRetry`** — validate-fix-retry loop: caller supplies a `Validator` and `AdjustFunc`; the loop escalates mode/intensity and retries up to N times.
 - **`filter.NewTOMLFilter` / `LoadTOMLFilterFile`** — full 8-stage TOML pipeline as a pluggable `Filter`.
@@ -168,6 +192,7 @@ Profile the hot paths with `./scripts/profile.sh [compress|tokens|filter|secrets
 ```
 tok
 ├── tok.go, *.go         Public library API (top-level package)
+├── runtimegraph/        Privacy-safe operations/policy/quality projection
 ├── internal/
 │   ├── compress/        Input compression engine (6 modes)
 │   ├── filter/          Output pipeline (31 layers)
@@ -181,6 +206,12 @@ tok
 
 Pure-Go, zero CGO, no runtime dependencies.
 
+`runtimegraph.Build` projects completed compression statistics, usage
+snapshots, budget decisions, and redaction summaries into `tok.graph/v1`.
+Input text, model names, budget reasons, secret types, and secret values are
+digest-only. Tok remains the source of truth for compression, limits, and
+redaction; graph consumers receive immutable summaries.
+
 ---
 
 ## Contributing

cost.go

@@ -2,6 +2,7 @@ package tok
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 	"sync"
 )
@@ -129,6 +130,20 @@ func EstimateCostSavings(stats Stats, model string) float64 {
 	return float64(stats.TokensSaved) / 1000 * pricing.InputPricePer1K
 }
 
+// EstimateCost estimates the USD cost of processing text with a model,
+// counting the text as input tokens. It is a convenience wrapper around
+// EstimateTokensForModel and GetModelPricing.
+//
+// Returns 0 if the model is unknown (no pricing registered).
+func EstimateCost(text string, model string) float64 {
+	pricing, ok := GetModelPricing(model)
+	if !ok {
+		return 0
+	}
+	tokens := EstimateTokensForModel(text, model)
+	return float64(tokens) / 1000 * pricing.InputPricePer1K
+}
+
 // FormatCostSavings returns a human-readable cost savings string.
 // Returns "$X.XX saved" if the model is known, or empty string if unknown.
 func FormatCostSavings(stats Stats, model string) string {
@@ -152,5 +167,6 @@ func ListModels() []string {
 	for name := range modelPricingRegistry {
 		models = append(models, name)
 	}
+	sort.Strings(models)
 	return models
 }

cost_test.go

@@ -1,6 +1,7 @@
 package tok
 
 import (
+	"sort"
 	"testing"
 )
 
@@ -331,6 +332,11 @@ func TestListModels(t *testing.T) {
 			t.Errorf("ListModels() missing expected model %q", m)
 		}
 	}
+
+	// ListModels must return models in sorted order (deterministic).
+	if !sort.StringsAreSorted(models) {
+		t.Errorf("ListModels() not sorted: %v", models)
+	}
 }
 
 func TestListModels_NoDuplicates(t *testing.T) {
@@ -345,6 +351,38 @@ func TestListModels_NoDuplicates(t *testing.T) {
 	}
 }
 
+func TestEstimateCost(t *testing.T) {
+	// gpt-4o input pricing is $0.0025 per 1K tokens.
+	pricing, ok := GetModelPricing("gpt-4o")
+	if !ok {
+		t.Fatal("expected gpt-4o pricing to be registered")
+	}
+
+	text := "Hello, world! This is a test of the EstimateCost function."
+	tokens := EstimateTokensForModel(text, "gpt-4o")
+	want := float64(tokens) / 1000 * pricing.InputPricePer1K
+
+	got := EstimateCost(text, "gpt-4o")
+	if got != want {
+		t.Errorf("EstimateCost = %v, want %v", got, want)
+	}
+	if got == 0 {
+		t.Error("expected non-zero cost for gpt-4o")
+	}
+}
+
+func TestEstimateCost_UnknownModel(t *testing.T) {
+	if got := EstimateCost("some text", "totally-fake-model-xyz"); got != 0 {
+		t.Errorf("EstimateCost for unknown model = %v, want 0", got)
+	}
+}
+
+func TestEstimateCost_EmptyText(t *testing.T) {
+	if got := EstimateCost("", "gpt-4o"); got != 0 {
+		t.Errorf("EstimateCost for empty text = %v, want 0", got)
+	}
+}
+
 func contains(s, substr string) bool {
 	return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsSubstring(s, substr))
 }

docs/AGENT_INTEGRATION.md

@@ -58,15 +58,14 @@ tok.Compress(text, tok.WithMode(tok.ModeAggressive))
 ### Compress a prompt
 
 ```go
-out := tok.PromptCompress(prompt, tok.IntensityFull)
+out, _ := tok.PromptCompress(prompt, tok.IntensityFull)
 // IntensityLite | IntensityFull | IntensityUltra
 ```
 
 ### Estimate tokens and cost
 
 ```go
-n := tok.EstimateTokensForModel(text, "gpt-4o")
-cost := tok.EstimateCost(n, "gpt-4o")
+cost := tok.EstimateCost(text, "gpt-4o")
 ```
 
 ### Redact secrets before sending context to a model

docs/DEPLOYMENT.md

@@ -38,7 +38,7 @@ import "github.com/GrayCodeAI/tok"
 
 ```go
 // Prompt compression (Lite / Full / Ultra intensities).
-out := tok.PromptCompress("Please implement authentication", tok.IntensityUltra)
+out, _ := tok.PromptCompress("Please implement authentication", tok.IntensityUltra)
 
 // Full output pipeline with options.
 out, _ := tok.Compress(verboseOutput, tok.Aggressive)

docs/PERFORMANCE.md

@@ -106,10 +106,10 @@ Tune the speed/compression trade-off through options passed to `tok.Compress`, o
 | Aggressive | `tok.Compress(text, tok.Aggressive)` | all | Strictest reduction |
 
 ```go
-out, err := tok.Compress(input, tok.Aggressive)
+out, _ := tok.Compress(input, tok.Aggressive)
 
 // Prompt-oriented helper with an intensity level:
-out := tok.PromptCompress(input, tok.IntensityFull)
+out, _ := tok.PromptCompress(input, tok.IntensityFull)
 ```
 
 Additional compression options:
@@ -137,7 +137,7 @@ rules, err := tok.LoadFilterRules("rules/custom.toml")
 if err != nil {
     log.Fatal(err)
 }
-compressed, err := tok.Compress(text, tok.WithCustomFilters(rules))
+compressed, _ := tok.Compress(text, tok.WithCustomFilters(rules))
 ```
 
 Profiles (presets of filter behavior) load via `tok.LoadProfile`.
@@ -147,11 +147,8 @@ Profiles (presets of filter behavior) load via `tok.LoadProfile`.
 ## Estimation, Cost & Tracking
 
 ```go
-// Token estimate for a specific model (fast path for short strings).
-n := tok.EstimateTokensForModel(text, "gpt-4o")
-
-// Estimated cost for a given token count / model.
-cost := tok.EstimateCost(tokens, "gpt-4o")
+// Estimated cost for a given model.
+cost := tok.EstimateCost(text, "gpt-4o")
 
 // Track usage over a context's lifetime.
 tracker := tok.NewTracker(ctx)
@@ -207,7 +204,7 @@ go test -bench=BenchmarkPipeline -benchmem
 Use a lighter intensity and let budget gates short-circuit work:
 
 ```go
-out := tok.PromptCompress(text, tok.IntensityLite)
+out, _ := tok.PromptCompress(text, tok.IntensityLite)
 ```
 
 ### Optimize for Memory
@@ -219,7 +216,7 @@ The pipeline uses pooled buffers and streams large inputs internally; prefer str
 Use the strongest settings when token budget matters most:
 
 ```go
-out := tok.PromptCompress(text, tok.IntensityUltra)
+out, _ := tok.PromptCompress(text, tok.IntensityUltra)
 // or
 out, _ := tok.Compress(text, tok.Aggressive)
 ```

docs/QUICK_START.md

@@ -21,7 +21,7 @@ import "github.com/GrayCodeAI/tok"
 ## Compress in One Call
 
 ```go
-out, err := tok.Compress(text)
+out, _ := tok.Compress(text)
 ```
 
 `Compress` uses an adaptive system that selects the right compression depth based
@@ -42,31 +42,30 @@ If you want explicit control, pass options to `Compress`:
 
 ```go
 // Aggressive mode: deepest available compression path
-out, err := tok.Compress(text, tok.Aggressive)
+out, _ := tok.Compress(text, tok.Aggressive)
 
 // Code-aware compression: preserve structure for a given language
-out, err := tok.Compress(src, tok.WithCodeAware("go"))
+out, _ := tok.Compress(src, tok.WithCodeAware("go"))
 
 // Perplexity-guided pruning at a target retention ratio
-out, err := tok.Compress(text, tok.WithPerplexityGuided(scorer, 0.5))
+out, _ := tok.Compress(text, tok.WithPerplexityGuided(scorer, 0.5))
 
 // Custom TOML filter rules (see "Custom Filters" below)
-out, err := tok.Compress(text, tok.WithCustomFilters(rules))
+out, _ := tok.Compress(text, tok.WithCustomFilters(rules))
 ```
 
 For prompt-style input there is a dedicated entry point with intensity levels:
 
 ```go
-out, err := tok.PromptCompress(prompt, tok.IntensityLite)
-out, err = tok.PromptCompress(prompt, tok.IntensityFull)
-out, err = tok.PromptCompress(prompt, tok.IntensityUltra)
+out, _ := tok.PromptCompress(prompt, tok.IntensityLite)
+out, _ = tok.PromptCompress(prompt, tok.IntensityFull)
+out, _ = tok.PromptCompress(prompt, tok.IntensityUltra)
 ```
 
 ## Token & Cost Estimation
 
 ```go
-tokens, err := tok.EstimateTokensForModel(text, "gpt-4o")
-cost, err := tok.EstimateCost(text, "gpt-4o")
+cost := tok.EstimateCost(text, "gpt-4o")
 ```
 
 ## Redaction & Extraction Helpers
@@ -98,7 +97,7 @@ rules, err := tok.LoadFilterRules("filters.toml")
 if err != nil {
     // handle error
 }
-out, err := tok.Compress(text, tok.WithCustomFilters(rules))
+out, _ := tok.Compress(text, tok.WithCustomFilters(rules))
 ```
 
 You can also load a named profile:

docs/SECURITY.md

@@ -47,7 +47,7 @@ rules, err := tok.LoadFilterRules("filters.toml")
 if err != nil {
     log.Fatal(err)
 }
-out, err := tok.Compress(text, tok.WithCustomFilters(rules))
+out, _ := tok.Compress(text, tok.WithCustomFilters(rules))
 ```
 
 Treat filter rule files as part of your trusted configuration. Set restrictive

docs/TROUBLESHOOTING.md

@@ -73,7 +73,7 @@ import "github.com/GrayCodeAI/tok"
 
 ```go
 // Prompt compression at higher intensity
-out := tok.PromptCompress(text, tok.IntensityFull)   // or tok.IntensityUltra
+out, _ := tok.PromptCompress(text, tok.IntensityFull)   // or tok.IntensityUltra
 
 // Generic compression with aggressive mode
 out, _ := tok.Compress(text, tok.Aggressive)
@@ -89,7 +89,7 @@ Available levels: `tok.IntensityLite`, `tok.IntensityFull`, `tok.IntensityUltra`
 
 1. Drop to a lighter intensity:
    ```go
-   out := tok.PromptCompress(text, tok.IntensityLite)
+   out, _ := tok.PromptCompress(text, tok.IntensityLite)
    ```
 
 2. For source code, use the code-aware option so language structure is preserved:
@@ -108,8 +108,7 @@ Use truncation and estimation helpers rather than guessing:
 
 ```go
 truncated := tok.SmartTruncate(text, maxTokens)
-n := tok.EstimateTokensForModel(truncated, "gpt-4o")
-cost := tok.EstimateCost(n, "gpt-4o")
+cost := tok.EstimateCost(truncated, "gpt-4o")
 ```
 
 ### Sensitive data appears in compressed output
@@ -146,7 +145,7 @@ inputs that don't need it.
 
 2. Use a lighter intensity for hot paths:
    ```go
-   out := tok.PromptCompress(text, tok.IntensityLite)
+   out, _ := tok.PromptCompress(text, tok.IntensityLite)
    ```
 
 3. For perplexity-guided compression, tune the keep ratio (lower = more

docs/architecture.md

@@ -46,15 +46,15 @@ tok/
 
 ```go
 // 🗜️ One-shot compression
-compressed, stats, err := tok.Compress(text,
+compressed, stats := tok.Compress(text,
     tok.WithTier(tok.TierCode),
     tok.WithBudget(4000),
     tok.WithQuery("implement OAuth flow"),
 )
 
 // 🔄 Reusable compressor (caches tokenizer state)
 c := tok.NewCompressor(tok.Aggressive)
-compressed, stats, err := c.Compress(text)
+compressed, stats := c.Compress(text)
 
 // 📊 Token estimation
 approx  := tok.EstimateTokens(text)         // fast, ±5%