From 25b396c7df9cf215ac88fe9de4107c74f6dcc7e0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 11 Oct 2026 22:10:18 +0000 Subject: [PATCH 01/14] feat(raw): define vector contracts and validation Keep caller-supplied vectors separate from legacy embedding APIs. Define raw DTOs, validation, strict JSON decoding, and bound cursors. Freeze legacy signatures and record the actual LocalAI consumer revision. Add an isolated contract runner with worktree-local caches and logs. Verified: sh test/raw/verify.sh contracts --- rag/raw/cursor.go | 112 ++++++++ rag/raw/cursor_test.go | 45 ++++ rag/raw/store.go | 91 +++++++ rag/raw/store_test.go | 27 ++ rag/raw/types.go | 196 ++++++++++++++ rag/raw/types_test.go | 108 ++++++++ rag/raw/validate.go | 449 +++++++++++++++++++++++++++++++ rag/raw/validate_test.go | 190 +++++++++++++ test/raw/downstream-lock.json | 15 ++ test/raw/legacy_contract_test.go | 69 +++++ test/raw/verify.sh | 23 ++ 11 files changed, 1325 insertions(+) create mode 100644 rag/raw/cursor.go create mode 100644 rag/raw/cursor_test.go create mode 100644 rag/raw/store.go create mode 100644 rag/raw/store_test.go create mode 100644 rag/raw/types.go create mode 100644 rag/raw/types_test.go create mode 100644 rag/raw/validate.go create mode 100644 rag/raw/validate_test.go create mode 100644 test/raw/downstream-lock.json create mode 100644 test/raw/legacy_contract_test.go create mode 100644 test/raw/verify.sh diff --git a/rag/raw/cursor.go b/rag/raw/cursor.go new file mode 100644 index 0000000..22169f7 --- /dev/null +++ b/rag/raw/cursor.go @@ -0,0 +1,112 @@ +package raw + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "strings" +) + +type cursor struct { + Version int `json:"version"` + Kind string `json:"kind"` + CollectionID string `json:"collection_id"` + LastID string `json:"last_id"` + Binding string `json:"binding"` +} + +func ValidateCollectionID(id string) error { + if len(id) != 36 || id == "00000000-0000-0000-0000-000000000000" { + return invalid() + } + for i, c := range id { + if i == 8 || i == 13 || i == 18 || i == 23 { + if c != '-' { + return invalid() + } + } else if !strings.ContainsRune("0123456789abcdef", c) { + return invalid() + } + } + return nil +} +func binding(f *Filter, v bool) (string, error) { + if e := ValidateFilter(f); e != nil { + return "", e + } + m := map[string]string{} + if f != nil { + for k, v := range f.MetadataEq { + m[k] = v + } + } + b, _ := json.Marshal(struct { + Metadata map[string]string + Vectors bool + }{m, v}) + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]), nil +} +func encode(c cursor) (string, error) { + b, e := json.Marshal(c) + if e != nil { + return "", NewError(CodeCursor) + } + s := base64.RawURLEncoding.EncodeToString(b) + if len(s) > MaxCursorBytes { + return "", NewError(CodeCursor) + } + return s, nil +} +func decode(s string) (cursor, error) { + var c cursor + if s == "" || len(s) > MaxCursorBytes { + return c, NewError(CodeCursor) + } + b, e := base64.RawURLEncoding.Strict().DecodeString(s) + if e != nil || DecodeRequest(b, &c) != nil || c.Version != 1 { + return c, NewError(CodeCursor) + } + return c, nil +} +func EncodeRecordCursor(cid, lastID string, f *Filter, includeVectors bool) (string, error) { + if ValidateCollectionID(cid) != nil || ValidateID(lastID) != nil { + return "", NewError(CodeCursor) + } + h, e := binding(f, includeVectors) + if e != nil { + return "", e + } + return encode(cursor{1, "records", cid, lastID, h}) +} +func DecodeRecordCursor(s, cid string, f *Filter, includeVectors bool) (string, error) { + c, e := decode(s) + if e != nil { + return "", e + } + h, e := binding(f, includeVectors) + if e != nil { + return "", e + } + if ValidateCollectionID(cid) != nil || c.Kind != "records" || c.CollectionID != cid || ValidateID(c.LastID) != nil || c.Binding != h { + return "", NewError(CodeCursor) + } + return c.LastID, nil +} +func EncodeCollectionCursor(lastID string) (string, error) { + if ValidateCollectionID(lastID) != nil { + return "", NewError(CodeCursor) + } + return encode(cursor{Version: 1, Kind: "collections", LastID: lastID}) +} +func DecodeCollectionCursor(s string) (string, error) { + c, e := decode(s) + if e != nil { + return "", e + } + if c.Kind != "collections" || c.CollectionID != "" || c.Binding != "" || ValidateCollectionID(c.LastID) != nil { + return "", NewError(CodeCursor) + } + return c.LastID, nil +} diff --git a/rag/raw/cursor_test.go b/rag/raw/cursor_test.go new file mode 100644 index 0000000..f6bb7bd --- /dev/null +++ b/rag/raw/cursor_test.go @@ -0,0 +1,45 @@ +package raw + +import ( + "encoding/base64" + "encoding/json" + "strings" + "testing" +) + +func TestRawCursorUntrusted(t *testing.T) { + cid := descriptor().CollectionID + c, _ := EncodeRecordCursor(cid, "last", nil, false) + b, _ := base64.RawURLEncoding.DecodeString(c) + var fields map[string]any + _ = json.Unmarshal(b, &fields) + for _, tc := range []struct { + key string + value any + }{{"version", 2}, {"version", nil}, {"kind", "collections"}, {"last_id", "\x00"}, {"last_id", ""}, {"binding", strings.Repeat("a", 64)}, {"collection_id", "not uuid"}, {"unexpected", true}} { + t.Run(tc.key, func(t *testing.T) { + m := map[string]any{} + for k, v := range fields { + m[k] = v + } + m[tc.key] = tc.value + b, _ := json.Marshal(m) + if _, e := DecodeRecordCursor(base64.RawURLEncoding.EncodeToString(b), cid, nil, false); e == nil { + t.Fatal(m) + } + }) + } + if _, e := DecodeCollectionCursor(c); e == nil { + t.Fatal("record cursor reused for collections") + } + a, _ := EncodeRecordCursor(cid, "last", &Filter{MetadataEq: map[string]string{"a": "b", "c": "d"}}, true) + if _, e := DecodeRecordCursor(a, cid, &Filter{MetadataEq: map[string]string{"c": "d", "a": "b"}}, true); e != nil { + t.Fatal("map ordering", e) + } + if _, e := EncodeRecordCursor(cid, "\n", nil, false); e == nil { + t.Fatal("invalid last ID") + } + if _, e := EncodeCollectionCursor("bad"); e == nil { + t.Fatal("invalid UUID") + } +} diff --git a/rag/raw/store.go b/rag/raw/store.go new file mode 100644 index 0000000..541ff12 --- /dev/null +++ b/rag/raw/store.go @@ -0,0 +1,91 @@ +package raw + +import "context" + +// Store owns raw persistence only, never embeddings, assets or sources. +// Create's bool reports newly created. Close drains/cancels owned resources. +type Store interface { + Capabilities(context.Context) (Capabilities, error) + CreateCollection(context.Context, CreateCollectionRequest) (Collection, bool, error) + ListCollections(context.Context, ListCollectionsRequest) (CollectionPage, error) + GetCollection(context.Context, string) (Collection, error) + Upsert(context.Context, string, UpsertRequest) (UpsertResponse, error) + GetRecords(context.Context, string, GetRecordsRequest) (GetRecordsResponse, error) + ListRecords(context.Context, string, ListRecordsRequest) (RecordPage, error) + DeleteRecords(context.Context, string, DeleteRecordsRequest) (DeleteRecordsResponse, error) + Reset(context.Context, string) error + DeleteCollection(context.Context, string) error + Search(context.Context, string, SearchRequest) (SearchResponse, error) + Close() +} + +const ( + CodeValidation = "VALIDATION_ERROR" + CodeCursor = "INVALID_CURSOR" + CodeUnauthorized = "UNAUTHORIZED" + CodeNotFound = "COLLECTION_NOT_FOUND" + CodeConflict = "COLLECTION_CONFIG_CONFLICT" + CodeSpaceMismatch = "EMBEDDING_SPACE_MISMATCH" + CodeBodyLimit = "BODY_TOO_LARGE" + CodeDeleteBound = "DELETE_BOUND_EXCEEDED" + CodeUnsupported = "NOT_IMPLEMENTED" + CodeUnavailable = "UNAVAILABLE" + CodeTimeout = "TIMEOUT" + CodeInternal = "INTERNAL_ERROR" +) + +// Error contains only safe public information; never place driver errors, +// SQL, credentials or caller payloads in Message. Retryable is not replay safety. +type Error struct { + Code string `json:"code"` + Status int `json:"status"` + Retryable bool `json:"retryable"` + Message string `json:"message"` +} + +func (e *Error) Error() string { return e.Message } +func NewError(code string) *Error { + e := &Error{Code: code} + switch code { + case CodeValidation: + e.Status = 400 + e.Message = "Invalid request" + case CodeCursor: + e.Status = 400 + e.Message = "Invalid or mismatched cursor" + case CodeUnauthorized: + e.Status = 401 + e.Message = "Authentication required" + case CodeNotFound: + e.Status = 404 + e.Message = "Collection not found" + case CodeConflict: + e.Status = 409 + e.Message = "Collection configuration conflicts" + case CodeSpaceMismatch: + e.Status = 409 + e.Message = "Embedding space does not match" + case CodeBodyLimit: + e.Status = 413 + e.Message = "Request body too large" + case CodeDeleteBound: + e.Status = 422 + e.Message = "Delete selection exceeds limit" + case CodeUnsupported: + e.Status = 501 + e.Message = "Capability not implemented" + case CodeUnavailable: + e.Status = 503 + e.Retryable = true + e.Message = "Required dependency unavailable" + case CodeTimeout: + e.Status = 504 + e.Retryable = true + e.Message = "Operation timed out" + default: + e.Code = CodeInternal + e.Status = 500 + e.Message = "Internal error" + } + return e +} diff --git a/rag/raw/store_test.go b/rag/raw/store_test.go new file mode 100644 index 0000000..0f203df --- /dev/null +++ b/rag/raw/store_test.go @@ -0,0 +1,27 @@ +package raw + +import "testing" + +func TestRawErrorStatuses(t *testing.T) { + for _, tc := range []struct { + code string + status int + retryable bool + }{ + {CodeValidation, 400, false}, {CodeCursor, 400, false}, {CodeUnauthorized, 401, false}, + {CodeNotFound, 404, false}, {CodeConflict, 409, false}, {CodeSpaceMismatch, 409, false}, + {CodeBodyLimit, 413, false}, {CodeDeleteBound, 422, false}, {CodeUnsupported, 501, false}, + {CodeUnavailable, 503, true}, {CodeTimeout, 504, true}, {CodeInternal, 500, false}, + } { + t.Run(tc.code, func(t *testing.T) { + e := NewError(tc.code) + if e.Code != tc.code || e.Status != tc.status || e.Retryable != tc.retryable || e.Error() == "" { + t.Fatal(e) + } + }) + } + e := NewError("secret database error") + if e.Code != CodeInternal || e.Error() != "Internal error" { + t.Fatal(e) + } +} diff --git a/rag/raw/types.go b/rag/raw/types.go new file mode 100644 index 0000000..9026bc1 --- /dev/null +++ b/rag/raw/types.go @@ -0,0 +1,196 @@ +// Package raw defines embedder-free caller-supplied vector contracts. +package raw + +import "time" + +const ( + MaxBodyBytes = 8 << 20 + MaxDimensions = 2000 + MaxBatch = 100 + MaxTextBytes = 64 << 10 + MaxMetadataBytes = 16 << 10 + MaxFilterPredicates = 8 + MaxDeleteMatches = 1000 + MaxCursorBytes = 2048 + SchemaVersion = 1 +) + +type LexicalConfig struct { + Enabled bool `json:"enabled"` + TextConfig string `json:"text_config"` +} + +// Nil fields mean omitted, not an explicit false or empty configuration. +type LexicalRequest struct { + Enabled *bool `json:"enabled,omitempty"` + TextConfig *string `json:"text_config,omitempty"` +} +type CreateCollectionRequest struct { + Name string `json:"name"` + Dimensions int `json:"dimensions"` + Metric string `json:"metric"` + EmbeddingSpaceID string `json:"embedding_space_id"` + Lexical *LexicalRequest `json:"lexical,omitempty"` +} +type Collection struct { + CollectionID string `json:"collection_id"` + Name string `json:"name"` + Kind string `json:"kind"` + Backend string `json:"backend"` + Dimensions int `json:"dimensions"` + Metric string `json:"metric"` + EmbeddingSpaceID string `json:"embedding_space_id"` + Lexical LexicalConfig `json:"lexical"` + SchemaVersion int `json:"schema_version"` + CreatedAt time.Time `json:"created_at"` +} + +// Text omission/null both clear text on replacement; metadata omission becomes {}. +type Record struct { + ID string `json:"id"` + Vector []float64 `json:"vector"` + Text *string `json:"text,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} +type CanonicalRecord struct { + ID string `json:"id"` + Vector []float32 `json:"vector,omitempty"` + Text *string `json:"text,omitempty"` + Metadata map[string]string `json:"metadata"` +} +type Filter struct { + MetadataEq map[string]string `json:"metadata_eq"` +} +type UpsertRequest struct { + EmbeddingSpaceID string `json:"embedding_space_id"` + Records []Record `json:"records"` +} +type UpsertResponse struct { + IDs []string `json:"ids"` +} +type GetRecordsRequest struct { + IDs []string `json:"ids"` + IncludeVectors bool `json:"include_vectors"` +} +type GetRecordsResponse struct { + Records []CanonicalRecord `json:"records"` + MissingIDs []string `json:"missing_ids"` +} +type ListCollectionsRequest struct { + Limit *int `json:"limit,omitempty"` + Cursor *string `json:"cursor,omitempty"` +} +type CollectionPage struct { + Collections []Collection `json:"collections"` + NextCursor *string `json:"next_cursor"` +} +type ListRecordsRequest struct { + Filter *Filter `json:"filter,omitempty"` + Limit *int `json:"limit,omitempty"` + Cursor *string `json:"cursor,omitempty"` + IncludeVectors bool `json:"include_vectors"` +} +type RecordPage struct { + Records []CanonicalRecord `json:"records"` + NextCursor *string `json:"next_cursor"` +} + +// A nil IDs slice means absent. An explicitly empty slice is an invalid selector. +type DeleteRecordsRequest struct { + IDs []string `json:"ids,omitempty"` + Filter *Filter `json:"filter,omitempty"` +} +type DeleteRecordsResponse struct { + DeletedCount int `json:"deleted_count"` +} +type ResetRequest struct{} +type ResetResponse struct { + Reset bool `json:"reset"` +} +type DeleteCollectionResponse struct { + Deleted bool `json:"deleted"` +} + +// Optional scalar pointers distinguish absence from explicit zero/empty values. +// DecodeRequest rejects null except where the contract permits it (text on +// records and cursors). Non-nil empty Vector is present for exclusivity checks. +type SearchRequest struct { + Mode string `json:"mode"` + Accuracy *string `json:"accuracy,omitempty"` + EmbeddingSpaceID *string `json:"embedding_space_id,omitempty"` + Vector []float64 `json:"vector,omitempty"` + Text *string `json:"text,omitempty"` + TopK *int `json:"top_k,omitempty"` + Filter *Filter `json:"filter,omitempty"` +} +type SearchResult struct { + ID string `json:"id"` + Text *string `json:"text,omitempty"` + Metadata map[string]string `json:"metadata"` + Score float64 `json:"score"` + VectorSimilarity *float64 `json:"vector_similarity,omitempty"` + VectorRank *int `json:"vector_rank,omitempty"` + LexicalRank *int `json:"lexical_rank,omitempty"` +} +type FusionParameters struct { + K int `json:"k"` + VectorWeight float64 `json:"vector_weight"` + LexicalWeight float64 `json:"lexical_weight"` + CandidatesPerArm int `json:"candidates_per_arm"` +} +type SearchResponse struct { + Mode string `json:"mode"` + Accuracy string `json:"accuracy"` + ScoreKind string `json:"score_kind"` + Results []SearchResult `json:"results"` + Count int `json:"count"` + Fusion *FusionParameters `json:"fusion,omitempty"` +} +type Readiness struct { + Ready bool `json:"ready"` + Reason string `json:"reason,omitempty"` +} +type IndexCapability struct { + Strategy string `json:"strategy"` + Installed bool `json:"installed"` + Readiness +} +type Limits struct { + BodyBytes int `json:"body_bytes"` + Dimensions int `json:"dimensions"` + NameBytes int `json:"name_bytes"` + SpaceIDBytes int `json:"space_id_bytes"` + IDBytes int `json:"id_bytes"` + TextBytes int `json:"text_bytes"` + MetadataKeys int `json:"metadata_keys"` + MetadataKeyBytes int `json:"metadata_key_bytes"` + MetadataValueBytes int `json:"metadata_value_bytes"` + MetadataBytes int `json:"metadata_bytes"` + Batch int `json:"batch"` + FilterPredicates int `json:"filter_predicates"` + DeleteMatches int `json:"delete_matches"` + ListDefault int `json:"list_default"` + ListMax int `json:"list_max"` + CursorBytes int `json:"cursor_bytes"` + TopKDefault int `json:"top_k_default"` + TopKMax int `json:"top_k_max"` + QueryTextBytes int `json:"query_text_bytes"` +} +type Capabilities struct { + Backend string `json:"backend"` + RawCollections bool `json:"raw_collections"` + Readiness + Modes []string `json:"modes"` + ModeReadiness map[string]Readiness `json:"mode_readiness"` + AccuracyOptions []string `json:"accuracy_options"` + DefaultAccuracy string `json:"default_accuracy"` + Metrics []string `json:"metrics"` + Limits Limits `json:"limits"` + AtomicBatch bool `json:"atomic_batch"` + Persistent bool `json:"persistent"` + Indexes []IndexCapability `json:"indexes"` +} + +func ContractLimits() Limits { + return Limits{MaxBodyBytes, MaxDimensions, 128, 256, 256, MaxTextBytes, 32, 64, 1024, MaxMetadataBytes, MaxBatch, MaxFilterPredicates, MaxDeleteMatches, 50, 100, MaxCursorBytes, 10, 100, 4096} +} diff --git a/rag/raw/types_test.go b/rag/raw/types_test.go new file mode 100644 index 0000000..9e9fe43 --- /dev/null +++ b/rag/raw/types_test.go @@ -0,0 +1,108 @@ +package raw + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestRawDTOs(t *testing.T) { + for _, tc := range []struct { + name string + value any + want string + }{ + {"get", GetRecordsResponse{Records: []CanonicalRecord{}, MissingIDs: []string{}}, `{"records":[],"missing_ids":[]}`}, + {"page", RecordPage{Records: []CanonicalRecord{}}, `{"records":[],"next_cursor":null}`}, + {"collection page", CollectionPage{Collections: []Collection{}}, `{"collections":[],"next_cursor":null}`}, + {"upsert", UpsertResponse{IDs: []string{"x"}}, `{"ids":["x"]}`}, + {"delete", DeleteRecordsResponse{1}, `{"deleted_count":1}`}, + {"reset", ResetResponse{true}, `{"reset":true}`}, + {"drop", DeleteCollectionResponse{true}, `{"deleted":true}`}, + {"record", CanonicalRecord{ID: "x", Metadata: map[string]string{}}, `{"id":"x","metadata":{}}`}, + {"empty text", CanonicalRecord{ID: "x", Text: ptr(""), Metadata: map[string]string{}}, `{"id":"x","text":"","metadata":{}}`}, + {"search", SearchResponse{Mode: "lexical", Accuracy: "not_applicable", ScoreKind: "bm25", Results: []SearchResult{}}, `{"mode":"lexical","accuracy":"not_applicable","score_kind":"bm25","results":[],"count":0}`}, + } { + t.Run(tc.name, func(t *testing.T) { + b, e := json.Marshal(tc.value) + if e != nil || string(b) != tc.want { + t.Fatal(string(b), e) + } + }) + } + if got := ResolveLexical(nil, ""); !got.Enabled || got.TextConfig != "english" { + t.Fatal(got) + } + if got := ResolveLexical(&LexicalRequest{Enabled: ptr(false)}, "simple"); got.Enabled || got.TextConfig != "simple" { + t.Fatal(got) + } +} +func TestRawStrictJSON(t *testing.T) { + for _, b := range []string{`{"ID":"x"}`, `{"id":"a","ID":"b"}`, `{"text":"\ud800"}`, `{"text":"\udc00"}`, `{"metadata":{"k":"a","k":"b"}}`, `{"metadata":{"text":null}}`, `{"vector":[null]}`, `null`, `[]`, `{"text":"` + string([]byte{255}) + `"}`} { + var r Record + if DecodeRequest([]byte(b), &r) == nil { + t.Fatal(b) + } + } + for _, b := range []string{`{"text":"\ud83d\ude00"}`, `{"text":null}`, `{"text":""}`, `{"text":"\\ud800"}`} { + var r Record + if e := DecodeRequest([]byte(b), &r); e != nil { + t.Fatal(b, e) + } + } + var r Record + if e := DecodeRequest([]byte(strings.Repeat(" ", MaxBodyBytes+1)), &r); e.(*Error).Status != 413 { + t.Fatal(e) + } +} +func TestRawLimitsAndReplacement(t *testing.T) { + d := descriptor() + r := UpsertRequest{EmbeddingSpaceID: "s", Records: []Record{{ID: strings.Repeat("é", 128), Vector: []float64{1, 0}, Text: ptr(strings.Repeat("x", 65536)), Metadata: map[string]string{strings.Repeat("k", 64): strings.Repeat("v", 1024)}}}} + out, e := ValidateUpsert(d, r) + if e != nil { + t.Fatal(e) + } + r.Records[0].Metadata["new"] = "v" + *r.Records[0].Text = "changed" + if len(out[0].Metadata) != 1 || len(*out[0].Text) != 65536 { + t.Fatal("canonical data aliases input") + } + m := map[string]string{} + for i := 0; i < 33; i++ { + m[string(rune('A'+i))] = "x" + } + r.Records[0].Metadata = m + if _, e := ValidateUpsert(d, r); e == nil { + t.Fatal("33 keys") + } + f := &Filter{MetadataEq: map[string]string{}} + for i := 0; i < 8; i++ { + f.MetadataEq[string(rune('a'+i))] = "x" + } + if ValidateFilter(f) != nil { + t.Fatal("eight predicates") + } + if ValidateDelete(DeleteRecordsRequest{Filter: f}) != nil { + t.Fatal("filter delete") + } + if ValidateDelete(DeleteRecordsRequest{IDs: []string{"x"}}) != nil { + t.Fatal("ID delete") + } + r.Records[0].Metadata = nil + r.Records[0].Text = nil + out, e = ValidateUpsert(d, r) + if e != nil || out[0].Text != nil || out[0].Metadata == nil || len(out[0].Metadata) != 0 { + t.Fatal(out, e) + } + for _, n := range []int{1, 100} { + s, e := ValidateSearch(d, SearchRequest{Mode: "lexical", Text: ptr("cat"), TopK: &n}) + if e != nil || *s.TopK != n { + t.Fatal(s, e) + } + } + for _, s := range []string{strings.Repeat("x", 4097), "\x00", "\xff"} { + if _, e := ValidateSearch(d, SearchRequest{Mode: "lexical", Text: &s}); e == nil { + t.Fatal("query accepted") + } + } +} diff --git a/rag/raw/validate.go b/rag/raw/validate.go new file mode 100644 index 0000000..e4c50f5 --- /dev/null +++ b/rag/raw/validate.go @@ -0,0 +1,449 @@ +package raw + +import ( + "bytes" + "encoding/json" + "io" + "math" + "reflect" + "strings" + "unicode" + "unicode/utf8" +) + +func invalid() error { return NewError(CodeValidation) } +func validString(s string, max int) bool { + return len(s) <= max && utf8.ValidString(s) && !strings.ContainsRune(s, 0) +} +func ValidateCreate(r CreateCollectionRequest) error { + if !validString(r.Name, 128) || r.Name == "" || r.Dimensions < 1 || r.Dimensions > MaxDimensions || r.Metric != "cosine" || !validString(r.EmbeddingSpaceID, 256) || r.EmbeddingSpaceID == "" { + return invalid() + } + if r.Lexical != nil && r.Lexical.TextConfig != nil && (!validString(*r.Lexical.TextConfig, MaxBodyBytes) || *r.Lexical.TextConfig == "") { + return invalid() + } + return nil +} + +// ResolveLexical snapshots the caller-supplied creation-time environment value. +// Database catalog resolution remains the persistence implementation's duty. +func ResolveLexical(r *LexicalRequest, configured string) LexicalConfig { + if configured == "" { + configured = "english" + } + v := LexicalConfig{true, configured} + if r != nil { + if r.Enabled != nil { + v.Enabled = *r.Enabled + } + if r.TextConfig != nil { + v.TextConfig = *r.TextConfig + } + } + return v +} +func NormalizeVector(v []float64, dimensions int) ([]float32, error) { + if dimensions < 1 || dimensions > MaxDimensions || len(v) != dimensions { + return nil, invalid() + } + var norm float64 + nonzero := false + for _, x := range v { + if math.IsNaN(x) || math.IsInf(x, 0) || math.Abs(x) > math.MaxFloat32 { + return nil, invalid() + } + f := float32(x) + if math.IsInf(float64(f), 0) { + return nil, invalid() + } + nonzero = nonzero || f != 0 + norm = math.Hypot(norm, x) + } + if !nonzero || norm == 0 || math.IsInf(norm, 0) { + return nil, invalid() + } + out := make([]float32, len(v)) + nonzero = false + for i, x := range v { + out[i] = float32(x / norm) + if math.IsNaN(float64(out[i])) || math.IsInf(float64(out[i]), 0) { + return nil, invalid() + } + nonzero = nonzero || out[i] != 0 + } + if !nonzero { + return nil, invalid() + } + return out, nil +} +func ValidateID(id string) error { + if id == "" || !validString(id, 256) { + return invalid() + } + for _, r := range id { + if unicode.IsControl(r) { + return invalid() + } + } + return nil +} +func ValidateIDs(ids []string) error { + if len(ids) < 1 || len(ids) > MaxBatch { + return invalid() + } + seen := map[string]bool{} + for _, id := range ids { + if ValidateID(id) != nil || seen[id] { + return invalid() + } + seen[id] = true + } + return nil +} +func validateMetadata(m map[string]string, maxKeys int) error { + if len(m) > maxKeys { + return invalid() + } + for k, v := range m { + if !validString(k, 64) || !validString(v, 1024) { + return invalid() + } + } + b, e := json.Marshal(m) + if e != nil || len(b) > MaxMetadataBytes { + return invalid() + } + return nil +} +func ValidateUpsert(d Collection, r UpsertRequest) ([]CanonicalRecord, error) { + if r.EmbeddingSpaceID == "" { + return nil, invalid() + } + if r.EmbeddingSpaceID != d.EmbeddingSpaceID { + return nil, NewError(CodeSpaceMismatch) + } + ids := make([]string, len(r.Records)) + for i, v := range r.Records { + ids[i] = v.ID + } + if e := ValidateIDs(ids); e != nil { + return nil, e + } + out := make([]CanonicalRecord, len(r.Records)) + for i, v := range r.Records { + if v.Text != nil && !validString(*v.Text, MaxTextBytes) { + return nil, invalid() + } + if e := validateMetadata(v.Metadata, 32); e != nil { + return nil, e + } + vec, e := NormalizeVector(v.Vector, d.Dimensions) + if e != nil { + return nil, e + } + m := map[string]string{} + for k, val := range v.Metadata { + m[k] = val + } + var text *string + if v.Text != nil { + s := *v.Text + text = &s + } + out[i] = CanonicalRecord{v.ID, vec, text, m} + } + return out, nil +} +func ValidateFilter(f *Filter) error { + if f == nil { + return nil + } + return validateMetadata(f.MetadataEq, MaxFilterPredicates) +} +func ValidateDelete(r DeleteRecordsRequest) error { + if (r.IDs != nil) == (r.Filter != nil) { + return invalid() + } + if r.IDs != nil { + return ValidateIDs(r.IDs) + } + if len(r.Filter.MetadataEq) == 0 { + return invalid() + } + return ValidateFilter(r.Filter) +} +func ValidateLimit(n *int) (int, error) { + if n == nil { + return 50, nil + } + if *n < 1 || *n > 100 { + return 0, invalid() + } + return *n, nil +} +func ValidateSearch(d Collection, r SearchRequest) (SearchRequest, error) { + if e := ValidateFilter(r.Filter); e != nil { + return r, e + } + k := 10 + if r.TopK != nil { + k = *r.TopK + } + if k < 1 || k > 100 { + return r, invalid() + } + r.TopK = &k + switch r.Mode { + case "vector": + if r.Text != nil { + return r, invalid() + } + case "lexical": + if r.Vector != nil || r.EmbeddingSpaceID != nil || r.Accuracy != nil { + return r, invalid() + } + case "hybrid": + default: + return r, invalid() + } + if r.Mode != "vector" { + if r.Text == nil || !validString(*r.Text, 4096) || strings.TrimSpace(*r.Text) == "" { + return r, invalid() + } + if !d.Lexical.Enabled { + return r, NewError(CodeUnsupported) + } + } + if r.Mode != "lexical" { + if r.EmbeddingSpaceID == nil || *r.EmbeddingSpaceID == "" { + return r, invalid() + } + if *r.EmbeddingSpaceID != d.EmbeddingSpaceID { + return r, NewError(CodeSpaceMismatch) + } + v, e := NormalizeVector(r.Vector, d.Dimensions) + if e != nil { + return r, e + } + r.Vector = make([]float64, len(v)) + for i, x := range v { + r.Vector[i] = float64(x) + } + a := "approximate" + if r.Accuracy != nil { + a = *r.Accuracy + } + if a != "approximate" && a != "exact" { + return r, invalid() + } + r.Accuracy = &a + } + return r, nil +} + +// DecodeRequest is the strict JSON boundary shared by future transports. It +// rejects duplicate keys at every depth, unknown fields, null scalars, malformed +// Unicode (including unpaired escaped surrogates), and trailing JSON. Library +// callers use the validators directly; nil and empty slices remain distinct. +func DecodeRequest(b []byte, dst any) error { + if len(b) > MaxBodyBytes { + return NewError(CodeBodyLimit) + } + if !utf8.Valid(b) || !validEscapes(b) { + return invalid() + } + dec := json.NewDecoder(bytes.NewReader(b)) + dec.UseNumber() + if e := checkJSON(dec, "", 0); e != nil { + return invalid() + } + if _, e := dec.Token(); e != io.EOF { + return invalid() + } + // Text null is allowed only for records, not a search field. + if _, ok := dst.(*SearchRequest); ok { + var fields map[string]json.RawMessage + if json.Unmarshal(b, &fields) != nil { + return invalid() + } + if v, ok := fields["text"]; ok && bytes.Equal(bytes.TrimSpace(v), []byte("null")) { + return invalid() + } + } + if !exactFields(b, reflect.TypeOf(dst)) { + return invalid() + } + dec = json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + if e := dec.Decode(dst); e != nil { + return invalid() + } + return nil +} +func checkJSON(d *json.Decoder, key string, depth int) error { + if depth > 64 { + return invalid() + } + tok, e := d.Token() + if e != nil { + return e + } + if tok == nil { + if key != "text" && key != "cursor" { + return invalid() + } + return nil + } + delim, ok := tok.(json.Delim) + if !ok { + return nil + } + switch delim { + case '{': + seen := map[string]bool{} + for d.More() { + t, e := d.Token() + if e != nil { + return e + } + k, ok := t.(string) + if !ok || seen[k] { + return invalid() + } + seen[k] = true + if key == "metadata" || key == "metadata_eq" { + v, e := d.Token() + if e != nil { + return e + } + if _, ok := v.(string); !ok { + return invalid() + } + } else if e := checkJSON(d, k, depth+1); e != nil { + return e + } + } + t, e := d.Token() + if e != nil || t != json.Delim('}') { + return invalid() + } + case '[': + for d.More() { + if e := checkJSON(d, "", depth+1); e != nil { + return e + } + } + t, e := d.Token() + if e != nil || t != json.Delim(']') { + return invalid() + } + default: + return invalid() + } + return nil +} + +// encoding/json replaces lone surrogate escapes; reject them before decoding. +func validEscapes(b []byte) bool { + hex := func(c byte) int { + switch { + case c >= '0' && c <= '9': + return int(c - '0') + case c >= 'a' && c <= 'f': + return int(c-'a') + 10 + case c >= 'A' && c <= 'F': + return int(c-'A') + 10 + } + return -1 + } + unit := func(i int) int { + if i+4 > len(b) { + return -1 + } + n := 0 + for _, c := range b[i : i+4] { + v := hex(c) + if v < 0 { + return -1 + } + n = n*16 + v + } + return n + } + for i := 0; i < len(b); i++ { + if b[i] != '\\' { + continue + } + i++ + if i >= len(b) { + return false + } + if b[i] != 'u' { + continue + } + n := unit(i + 1) + if n < 0 { + return false + } + i += 4 + if n >= 0xdc00 && n <= 0xdfff { + return false + } + if n >= 0xd800 && n <= 0xdbff { + if i+6 >= len(b) || b[i+1] != '\\' || b[i+2] != 'u' { + return false + } + m := unit(i + 3) + if m < 0xdc00 || m > 0xdfff { + return false + } + i += 6 + } + } + return true +} + +// encoding/json accepts case-insensitive field aliases; the wire contract does not. +func exactFields(b []byte, t reflect.Type) bool { + if t == nil { + return false + } + for t.Kind() == reflect.Pointer { + t = t.Elem() + } + if bytes.Equal(bytes.TrimSpace(b), []byte("null")) { + return true + } + switch t.Kind() { + case reflect.Struct: + var m map[string]json.RawMessage + if json.Unmarshal(b, &m) != nil { + return false + } + fields := map[string]reflect.Type{} + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + name := strings.Split(f.Tag.Get("json"), ",")[0] + if name != "" && name != "-" { + fields[name] = f.Type + } + } + for k, v := range m { + ft, ok := fields[k] + if !ok || !exactFields(v, ft) { + return false + } + } + case reflect.Slice: + var a []json.RawMessage + if json.Unmarshal(b, &a) != nil { + return false + } + for _, v := range a { + if !exactFields(v, t.Elem()) { + return false + } + } + } + return true +} diff --git a/rag/raw/validate_test.go b/rag/raw/validate_test.go new file mode 100644 index 0000000..8e9c6c0 --- /dev/null +++ b/rag/raw/validate_test.go @@ -0,0 +1,190 @@ +package raw + +import ( + "encoding/json" + "math" + "strings" + "testing" +) + +func ptr[T any](v T) *T { return &v } +func descriptor() Collection { + return Collection{CollectionID: "123e4567-e89b-12d3-a456-426614174000", Dimensions: 2, EmbeddingSpaceID: "s", Lexical: LexicalConfig{Enabled: true}} +} +func TestRawNormalize(t *testing.T) { + for _, tc := range []struct { + name string + v []float64 + dim int + ok bool + }{ + {"unit", []float64{3, 4}, 2, true}, {"tiny", []float64{1e-40, 2e-40}, 2, true}, {"huge", []float64{math.MaxFloat32, math.MaxFloat32}, 2, true}, {"zero", []float64{0, 0}, 2, false}, {"underflow", []float64{1e-300, 0}, 2, false}, {"overflow", []float64{math.MaxFloat64, 0}, 2, false}, {"nan", []float64{math.NaN(), 0}, 2, false}, {"inf", []float64{math.Inf(1), 0}, 2, false}, {"dimension", []float64{1}, 2, false}, {"invalid dimension", nil, 0, false}} { + t.Run(tc.name, func(t *testing.T) { + v, e := NormalizeVector(tc.v, tc.dim) + if (e == nil) != tc.ok { + t.Fatalf("%v", e) + } + if e == nil { + var n float64 + for _, x := range v { + n += float64(x) * float64(x) + } + if math.Abs(n-1) > 1e-6 { + t.Fatal(n) + } + } + }) + } + v, _ := NormalizeVector([]float64{3, 4}, 2) + if v[0] != float32(.6) || v[1] != float32(.8) { + t.Fatal(v) + } +} +func TestRawCreate(t *testing.T) { + base := CreateCollectionRequest{Name: "x", Dimensions: 2, Metric: "cosine", EmbeddingSpaceID: "s"} + for _, tc := range []struct { + name string + edit func(*CreateCollectionRequest) + ok bool + }{ + {"valid", func(*CreateCollectionRequest) {}, true}, {"name", func(r *CreateCollectionRequest) { r.Name = strings.Repeat("é", 65) }, false}, {"utf8", func(r *CreateCollectionRequest) { r.Name = "\xff" }, false}, {"dimensions", func(r *CreateCollectionRequest) { r.Dimensions = 2001 }, false}, {"metric", func(r *CreateCollectionRequest) { r.Metric = "dot" }, false}, {"space", func(r *CreateCollectionRequest) { r.EmbeddingSpaceID = "" }, false}, {"space limit", func(r *CreateCollectionRequest) { r.EmbeddingSpaceID = strings.Repeat("x", 257) }, false}} { + t.Run(tc.name, func(t *testing.T) { + r := base + tc.edit(&r) + if (ValidateCreate(r) == nil) != tc.ok { + t.Fatal(r) + } + }) + } +} +func TestRawUpsert(t *testing.T) { + for _, tc := range []struct { + name string + edit func(*UpsertRequest) + ok bool + }{ + {"valid", func(*UpsertRequest) {}, true}, {"duplicate", func(r *UpsertRequest) { r.Records = append(r.Records, r.Records[0]) }, false}, {"empty", func(r *UpsertRequest) { r.Records = nil }, false}, {"batch", func(r *UpsertRequest) { r.Records = make([]Record, 101) }, false}, {"control", func(r *UpsertRequest) { r.Records[0].ID = "a\n" }, false}, {"unicode control", func(r *UpsertRequest) { r.Records[0].ID = "a\u0085" }, false}, {"id limit", func(r *UpsertRequest) { r.Records[0].ID = strings.Repeat("a", 257) }, false}, {"id utf8", func(r *UpsertRequest) { r.Records[0].ID = "\xff" }, false}, {"text nul", func(r *UpsertRequest) { r.Records[0].Text = ptr("a\x00") }, false}, {"text utf8", func(r *UpsertRequest) { r.Records[0].Text = ptr("\xff") }, false}, {"text limit", func(r *UpsertRequest) { r.Records[0].Text = ptr(strings.Repeat("x", 65537)) }, false}, {"metadata key", func(r *UpsertRequest) { r.Records[0].Metadata = map[string]string{strings.Repeat("x", 65): "x"} }, false}, {"metadata value", func(r *UpsertRequest) { r.Records[0].Metadata = map[string]string{"x": strings.Repeat("x", 1025)} }, false}, {"metadata serialized", func(r *UpsertRequest) { + m := map[string]string{} + for i := 0; i < 32; i++ { + m[string(rune('A'+i))] = strings.Repeat("x", 1024) + } + r.Records[0].Metadata = m + }, false}, {"space conflict", func(r *UpsertRequest) { r.EmbeddingSpaceID = "other" }, false}} { + t.Run(tc.name, func(t *testing.T) { + r := UpsertRequest{EmbeddingSpaceID: "s", Records: []Record{{ID: "x", Vector: []float64{3, 4}}}} + tc.edit(&r) + out, e := ValidateUpsert(descriptor(), r) + if (e == nil) != tc.ok { + t.Fatalf("%v", e) + } + if e == nil && out[0].Metadata == nil { + t.Fatal("replacement must produce empty metadata") + } + }) + } +} +func TestRawSearch(t *testing.T) { + for _, tc := range []struct { + body string + ok bool + }{ + {`{"mode":"vector","vector":[3,4],"embedding_space_id":"s"}`, true}, + {`{"mode":"vector","vector":[3,4],"embedding_space_id":"s","text":""}`, false}, + {`{"mode":"vector","vector":[3,4],"embedding_space_id":"s","text":null}`, false}, + {`{"mode":"lexical","text":"cat"}`, true}, {`{"mode":"lexical","text":"cat","vector":[]}`, false}, {`{"mode":"lexical","text":"cat","vector":null}`, false}, {`{"mode":"lexical","text":"cat","accuracy":""}`, false}, {`{"mode":"lexical","text":"cat","embedding_space_id":""}`, false}, + {`{"mode":"hybrid","text":"cat","vector":[1,0],"embedding_space_id":"s","accuracy":"exact"}`, true}, {`{"mode":"lexical","text":" "}`, false}, {`{"mode":"lexical","text":"cat","top_k":0}`, false}, {`{"mode":"lexical","text":"cat","top_k":101}`, false}, {`{"text":"cat"}`, false}, {`{"mode":"vector","vector":[1,0],"embedding_space_id":"s","accuracy":""}`, false}, + } { + t.Run(tc.body, func(t *testing.T) { + var r SearchRequest + e := DecodeRequest([]byte(tc.body), &r) + if e == nil { + r, e = ValidateSearch(descriptor(), r) + } + if (e == nil) != tc.ok { + t.Fatalf("%v", e) + } + if e == nil && *r.TopK != 10 { + t.Fatal("default top_k") + } + if e == nil && r.Mode == "vector" && *r.Accuracy != "approximate" { + t.Fatal("default accuracy") + } + }) + } +} +func TestRawSelectorsAndJSON(t *testing.T) { + for _, ids := range [][]string{nil, {}, {"a", "a"}, {"\x00"}} { + if ValidateIDs(ids) == nil { + t.Fatal(ids) + } + } + if ValidateIDs([]string{"a"}) != nil { + t.Fatal("valid IDs") + } + for _, body := range []string{`{}`, `{"ids":[]}`, `{"filter":{"metadata_eq":{}}}`, `{"ids":["a"],"filter":{"metadata_eq":{"x":"y"}}}`} { + var r DeleteRecordsRequest + _ = DecodeRequest([]byte(body), &r) + if ValidateDelete(r) == nil { + t.Fatal(body) + } + } + for _, body := range []string{`{"id":"a","id":"b"}`, `{"unknown":1}`, `{} {}`, `{"metadata":{"a":2}}`, `{"metadata":{"a":null}}`, `{"metadata":{"a":{"b":"c"}}}`, `{"vector":null}`} { + var r Record + if DecodeRequest([]byte(body), &r) == nil { + t.Fatal(body) + } + } + f := &Filter{MetadataEq: map[string]string{}} + for i := 0; i < 9; i++ { + f.MetadataEq[string(rune('a'+i))] = "v" + } + if ValidateFilter(f) == nil { + t.Fatal("filter bound") + } + for _, n := range []int{0, -1, 101} { + if _, e := ValidateLimit(&n); e == nil { + t.Fatal(n) + } + } + if n, e := ValidateLimit(nil); e != nil || n != 50 { + t.Fatal(n, e) + } +} +func TestRawCursor(t *testing.T) { + cid := descriptor().CollectionID + f := &Filter{MetadataEq: map[string]string{"source": "a"}} + c, e := EncodeRecordCursor(cid, "last", f, false) + if e != nil { + t.Fatal(e) + } + last, e := DecodeRecordCursor(c, cid, f, false) + if e != nil || last != "last" { + t.Fatal(last, e) + } + for _, tc := range []struct { + c, cid string + f *Filter + v bool + }{{c, cid, nil, false}, {c, cid, f, true}, {c, "123e4567-e89b-12d3-a456-426614174001", f, false}, {"bad", cid, f, false}, {strings.Repeat("a", 2049), cid, f, false}} { + if _, e := DecodeRecordCursor(tc.c, tc.cid, tc.f, tc.v); e == nil { + t.Fatal(tc) + } + } + c, e = EncodeCollectionCursor(cid) + if e != nil { + t.Fatal(e) + } + if id, e := DecodeCollectionCursor(c); e != nil || id != cid { + t.Fatal(id, e) + } +} +func TestRawError(t *testing.T) { + e := NewError(CodeSpaceMismatch) + if e.Status != 409 || e.Retryable || e.Code != "EMBEDDING_SPACE_MISMATCH" { + t.Fatal(e) + } + b, _ := json.Marshal(e) + if !strings.Contains(string(b), `"code"`) { + t.Fatal(string(b)) + } +} diff --git a/test/raw/downstream-lock.json b/test/raw/downstream-lock.json new file mode 100644 index 0000000..b2b7228 --- /dev/null +++ b/test/raw/downstream-lock.json @@ -0,0 +1,15 @@ +{ + "repository": "https://github.com/mudler/LocalAI", + "revision": "d4a5af8bfed9c97e8cb01e584f1f9f8a572df844", + "localrecall_required_version": "v0.6.7-0.20261006203612-d7c99211a3c7", + "localrecall_baseline": "d7c99211a3c7d6721a841fe34938e934f9d8aeee", + "consuming_packages": ["github.com/mudler/LocalAI/core/services/agentpool"], + "imports": { + "core/services/agentpool/collections_shared.go": ["github.com/mudler/localrecall/rag/engine"], + "core/services/agentpool/collections_shared_test.go": ["github.com/mudler/localrecall/rag", "github.com/mudler/localrecall/rag/sources"] + }, + "production_calls": ["engine.PostgresCollectionExists", "engine.ListPostgresCollections"], + "test_calls": ["rag.NewSourceManager", "rag.PersistentKB", "sources.Config"], + "inspection": "Read-only git ls-remote HEAD and immutable GitHub go.mod/full source archive; all Go import call sites inspected. Actual LocalAI, not LocalAI-store. LocalAI also embeds collections through github.com/mudler/LocalAGI/webui/collections.", + "downstream_build_verified": false +} diff --git a/test/raw/legacy_contract_test.go b/test/raw/legacy_contract_test.go new file mode 100644 index 0000000..f509ac1 --- /dev/null +++ b/test/raw/legacy_contract_test.go @@ -0,0 +1,69 @@ +package raw_test + +import ( + "context" + "encoding/json" + "io" + "testing" + + "github.com/mudler/localrecall/pkg/client" + "github.com/mudler/localrecall/rag" + "github.com/mudler/localrecall/rag/engine" + "github.com/mudler/localrecall/rag/engine/localai" + "github.com/mudler/localrecall/rag/types" + "github.com/sashabaranov/go-openai" +) + +// Deliberately implements only the frozen legacy interface, without embedding it. +type customEngine struct{} + +func (customEngine) Store(string, map[string]string) (engine.Result, error) { + return engine.Result{}, nil +} +func (customEngine) StoreDocuments([]string, map[string]string) ([]engine.Result, error) { + return nil, nil +} +func (customEngine) GetEmbeddingDimensions() (int, error) { return 3, nil } +func (customEngine) Reset() error { return nil } +func (customEngine) Search(string, int) ([]types.Result, error) { return nil, nil } +func (customEngine) Count() int { return 0 } +func (customEngine) Delete(map[string]string, map[string]string, ...string) error { return nil } +func (customEngine) GetByID(string) (types.Result, error) { return types.Result{}, nil } +func (customEngine) GetBySource(string) ([]types.Result, error) { return nil, nil } + +var ( + _ rag.Engine = customEngine{} + _ func(string, string, rag.Engine, int, int, *openai.Client, string) (*rag.PersistentKB, error) = rag.NewPersistentCollectionKB + _ func(string, string, *openai.Client, string) (*engine.PostgresDB, error) = engine.NewPostgresDBCollection + _ func(string, string, *openai.Client, string) (*engine.ChromemDB, error) = engine.NewChromemDBCollection + _ func(*localai.StoreClient, *openai.Client, string) *engine.LocalAIRAGDB = engine.NewLocalAIRAGDB + _ func(string) *client.Client = client.NewClient + _ func(*client.Client, string) error = (*client.Client).CreateCollection + _ func(*client.Client) ([]string, error) = (*client.Client).ListCollections + _ func(*client.Client, string) ([]string, error) = (*client.Client).ListEntries + _ func(*client.Client, string, string) ([]client.EntryChunk, error) = (*client.Client).GetEntryContent + _ func(*client.Client, string, string) (io.ReadCloser, error) = (*client.Client).GetEntryRawFile + _ func(*client.Client, string, string) ([]string, error) = (*client.Client).DeleteEntry + _ func(*client.Client, string, string, int) ([]types.Result, error) = (*client.Client).Search + _ func(*client.Client, string) error = (*client.Client).Reset + _ func(*client.Client, string, string) (string, error) = (*client.Client).Store + _ func(context.Context, string, string) (bool, error) = engine.PostgresCollectionExists + _ func(context.Context, string) ([]string, error) = engine.ListPostgresCollections +) + +func TestLegacyContract(t *testing.T) { + // Unkeyed literal freezes field order/types as well as names and JSON shape. + r := types.Result{"id", map[string]string{"source": "s"}, []float32{1}, "text", float32(.5)} + b, e := json.Marshal(r) + if e != nil { + t.Fatal(e) + } + if string(b) != `{"ID":"id","Metadata":{"source":"s"},"Embedding":[1],"Content":"text","Similarity":0.5}` { + t.Fatal(string(b)) + } + var kb rag.PersistentKB + kb.Engine = customEngine{} + if kb.Engine.Count() != 0 { + t.Fatal("legacy engine") + } +} diff --git a/test/raw/verify.sh b/test/raw/verify.sh new file mode 100644 index 0000000..12432fc --- /dev/null +++ b/test/raw/verify.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu +ROOT=$(pwd -P) +[ -f "$ROOT/go.mod" ] && [ -d "$ROOT/rag" ] || { echo 'run from worktree root' >&2; exit 1; } +# Reject symlinks before creating any output, including nested cache paths. +for path in .sdd-work .sdd-work/tmp .sdd-work/cache .sdd-work/mod .sdd-work/gopath .sdd-work/collections .sdd-work/assets .sdd-work/logs; do + if [ -L "$path" ] || { [ -d "$path" ] && [ "$path" != .sdd-work ] && [ -n "$(find "$path" -type l -print -quit)" ]; }; then + echo "symlinks in output path $path are not permitted" >&2; exit 1 + fi +done +export TMPDIR="$ROOT/.sdd-work/tmp" GOCACHE="$ROOT/.sdd-work/cache" GOMODCACHE="$ROOT/.sdd-work/mod" GOPATH="$ROOT/.sdd-work/gopath" +export COLLECTION_DB_PATH="$ROOT/.sdd-work/collections" FILE_ASSETS="$ROOT/.sdd-work/assets" +export GOENV=off GOTOOLCHAIN=local +mkdir -p "$TMPDIR" "$GOCACHE" "$GOMODCACHE" "$GOPATH" "$COLLECTION_DB_PATH" "$FILE_ASSETS" "$ROOT/.sdd-work/logs" +case "${1:-}" in + contracts) + log="$ROOT/.sdd-work/logs/contracts-$(date +%s)-$$.log" + if go test -mod=readonly -race -count=1 ./rag/raw ./test/raw >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi + [ "$(grep -c '^ok[[:space:]]' "$log")" -eq 2 ] && ! grep -E '\[no test files\]|\[no tests to run\]' "$log" + git diff --check + ;; + *) echo 'usage: sh test/raw/verify.sh contracts' >&2; exit 2;; +esac From a5975c8424e3f01ec3fdfa23f1b109720e2a7823 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 11 Oct 2026 22:14:05 +0000 Subject: [PATCH 02/14] fix(raw): fit all valid IDs in record cursors HTML escaping makes some valid 256-byte IDs exceed the cursor cap.\nDisable it for base64-encoded cursor JSON without changing either limit. Cover maximum-length HTML characters, quotes, backslashes, and Unicode\nwith round-trip tests, including JSON-escaped line and paragraph separators. --- rag/raw/cursor.go | 11 ++++++++--- rag/raw/cursor_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/rag/raw/cursor.go b/rag/raw/cursor.go index 22169f7..4a3ecba 100644 --- a/rag/raw/cursor.go +++ b/rag/raw/cursor.go @@ -1,6 +1,7 @@ package raw import ( + "bytes" "crypto/sha256" "encoding/base64" "encoding/hex" @@ -49,11 +50,15 @@ func binding(f *Filter, v bool) (string, error) { return hex.EncodeToString(h[:]), nil } func encode(c cursor) (string, error) { - b, e := json.Marshal(c) - if e != nil { + var b bytes.Buffer + enc := json.NewEncoder(&b) + // Cursors are base64, not HTML. Without HTML escaping, valid IDs expand + // at most 2x (quotes, backslashes, or U+2028/U+2029), fitting the cap. + enc.SetEscapeHTML(false) + if e := enc.Encode(c); e != nil { return "", NewError(CodeCursor) } - s := base64.RawURLEncoding.EncodeToString(b) + s := base64.RawURLEncoding.EncodeToString(bytes.TrimSuffix(b.Bytes(), []byte{'\n'})) if len(s) > MaxCursorBytes { return "", NewError(CodeCursor) } diff --git a/rag/raw/cursor_test.go b/rag/raw/cursor_test.go index f6bb7bd..1281aba 100644 --- a/rag/raw/cursor_test.go +++ b/rag/raw/cursor_test.go @@ -7,6 +7,45 @@ import ( "testing" ) +func TestRawCursorMaxLengthIDRoundTrip(t *testing.T) { + cid := descriptor().CollectionID + for _, tc := range []struct { + name string + unit string + }{ + {"less_than", "<"}, + {"greater_than", ">"}, + {"ampersand", "&"}, + {"quote", "\""}, + {"backslash", "\\"}, + {"mixed_escapes", "<>&\"\\"}, + {"two_byte_unicode", "é"}, + {"three_byte_unicode", "界"}, + {"four_byte_unicode", "😀"}, + {"line_separator", "\u2028"}, + {"paragraph_separator", "\u2029"}, + } { + t.Run(tc.name, func(t *testing.T) { + id := strings.Repeat(tc.unit, 256/len(tc.unit)) + id += strings.Repeat("x", 256-len(id)) + if len(id) != 256 || ValidateID(id) != nil { + t.Fatal("fixture must be a valid maximum-length ID") + } + encoded, err := EncodeRecordCursor(cid, id, nil, false) + if err != nil { + t.Fatal("encode maximum-length ID:", err) + } + if len(encoded) > MaxCursorBytes { + t.Fatalf("cursor length %d exceeds %d", len(encoded), MaxCursorBytes) + } + decoded, err := DecodeRecordCursor(encoded, cid, nil, false) + if err != nil || decoded != id { + t.Fatalf("round trip: got %q, error %v", decoded, err) + } + }) + } +} + func TestRawCursorUntrusted(t *testing.T) { cid := descriptor().CollectionID c, _ := EncodeRecordCursor(cid, "last", nil, false) From 4f4be57634f6474b7bf8ecb84314ae849de5d6d0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 11 Oct 2026 22:25:44 +0000 Subject: [PATCH 03/14] feat(raw): add isolated PostgreSQL lifecycle Keep raw collections separate from legacy discovery and embedding paths. Use one owned pool, transactional lifecycle operations, and schema checks. Probe dependency usability instead of reporting installed indexes as ready. Share PostgreSQL helpers without changing legacy timeout or fallback policy. Reuse collection cursors and verify rollback, privileges, pool cleanup, configuration snapshots, and reset semantics with an owned database fixture. Verification: sh test/raw/verify.sh lifecycle Contracts, helper tests, and 16 lifecycle tests pass with race detection. --- rag/engine/postgres.go | 64 +------ rag/internal/pgutil/config.go | 60 +++++++ rag/internal/pgutil/config_test.go | 53 ++++++ rag/internal/pgutil/index.go | 74 ++++++++ rag/raw/postgres/acceptance_test.go | 211 +++++++++++++++++++++++ rag/raw/postgres/capabilities.go | 97 +++++++++++ rag/raw/postgres/lifecycle.go | 163 ++++++++++++++++++ rag/raw/postgres/lifecycle_test.go | 253 ++++++++++++++++++++++++++++ rag/raw/postgres/registry.go | 132 +++++++++++++++ rag/raw/postgres/service.go | 101 +++++++++++ test/raw/Dockerfile.postgres | 4 + test/raw/database.sh | 43 +++++ test/raw/verify.sh | 13 +- 13 files changed, 1211 insertions(+), 57 deletions(-) create mode 100644 rag/internal/pgutil/config.go create mode 100644 rag/internal/pgutil/config_test.go create mode 100644 rag/internal/pgutil/index.go create mode 100644 rag/raw/postgres/acceptance_test.go create mode 100644 rag/raw/postgres/capabilities.go create mode 100644 rag/raw/postgres/lifecycle.go create mode 100644 rag/raw/postgres/lifecycle_test.go create mode 100644 rag/raw/postgres/registry.go create mode 100644 rag/raw/postgres/service.go create mode 100644 test/raw/Dockerfile.postgres create mode 100644 test/raw/database.sh diff --git a/rag/engine/postgres.go b/rag/engine/postgres.go index eb97a5e..1e80d32 100644 --- a/rag/engine/postgres.go +++ b/rag/engine/postgres.go @@ -11,8 +11,8 @@ import ( "sync" "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" + "github.com/mudler/localrecall/rag/internal/pgutil" "github.com/mudler/localrecall/rag/types" "github.com/mudler/xlog" "github.com/sashabaranov/go-openai" @@ -187,31 +187,13 @@ const defaultPoolMaxConns = int32(4) // connection string has priority, then POSTGRES_POOL_MAX_CONNS, then // defaultPoolMaxConns. Invalid environment values are ignored. func applyPoolLimits(config *pgxpool.Config, getenv func(string) string) { - if hasPoolMaxConns(config.ConnString()) { - return - } - config.MaxConns = defaultPoolMaxConns - if v := getenv("POSTGRES_POOL_MAX_CONNS"); v != "" { - n, err := strconv.ParseInt(v, 10, 32) - if err != nil || n < 1 { - xlog.Warn("Ignoring invalid POSTGRES_POOL_MAX_CONNS", "value", v) - return - } - config.MaxConns = int32(n) - } + pgutil.ApplyPoolLimits(config, getenv) } // hasPoolMaxConns reports whether the connection string sets pool_max_conns. // pgxpool.ParseConfig removes the setting after it reads it, so parse the // string again at the connection level, where it is kept as a runtime param. -func hasPoolMaxConns(connString string) bool { - cfg, err := pgconn.ParseConfig(connString) - if err != nil { - return false - } - _, ok := cfg.RuntimeParams["pool_max_conns"] - return ok -} +func hasPoolMaxConns(connString string) bool { return pgutil.HasPoolMaxConns(connString) } // applyConnTimeouts sets per-connection safety timeouts on the pool config so // that a single wedged or corrupt index can never hang a backend forever and @@ -237,22 +219,7 @@ func hasPoolMaxConns(connString string) bool { // // Any value of "0" or "off" is treated as an explicit opt-out. func applyConnTimeouts(config *pgxpool.Config, getenv func(string) string) { - if config.ConnConfig.RuntimeParams == nil { - config.ConnConfig.RuntimeParams = map[string]string{} - } - set := func(param, env, def string) { - v := def - if e := getenv(env); e != "" { - v = e - } - if v == "" || v == "0" || strings.EqualFold(v, "off") { - return - } - config.ConnConfig.RuntimeParams[param] = v - } - set("lock_timeout", "POSTGRES_LOCK_TIMEOUT", "30s") - set("idle_in_transaction_session_timeout", "POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT", "300s") - set("statement_timeout", "POSTGRES_STATEMENT_TIMEOUT", "") + pgutil.ApplyConnTimeouts(config, getenv) } // execNoStatementTimeout runs a DDL statement (typically a CREATE INDEX) with @@ -362,10 +329,7 @@ func (p *PostgresDB) setupDatabase() error { if err := p.ensureBM25IndexConfig(ctx, indexName); err != nil { return fmt.Errorf("failed to ensure BM25 index text_config: %w", err) } - err = p.execNoStatementTimeout(ctx, fmt.Sprintf(` - CREATE INDEX IF NOT EXISTS %s ON %s - USING bm25(full_text) WITH (text_config='%s') - `, indexName, p.tableName, p.bm25TextConfig)) + err = p.execNoStatementTimeout(ctx, pgutil.BM25IndexSQL(p.tableName, indexName, "full_text", p.bm25TextConfig)) if err != nil { return fmt.Errorf("failed to create BM25 index (required for hybrid search): %w", err) } @@ -453,10 +417,7 @@ func (p *PostgresDB) createVectorIndex(ctx context.Context, vectorscaleInstalled indexName := fmt.Sprintf("idx_%s_embedding", p.tableName) if vectorscaleInstalled { - err := p.execNoStatementTimeout(ctx, fmt.Sprintf(` - CREATE INDEX IF NOT EXISTS %s ON %s - USING diskann(embedding) - `, indexName, p.tableName)) + err := p.execNoStatementTimeout(ctx, pgutil.VectorIndexSQL(p.tableName, indexName, "embedding", true)) if err == nil { xlog.Info("Created DiskANN index for vector search") return nil @@ -464,10 +425,7 @@ func (p *PostgresDB) createVectorIndex(ctx context.Context, vectorscaleInstalled xlog.Warn("Failed to create DiskANN index, trying HNSW", "error", err) } - err := p.execNoStatementTimeout(ctx, fmt.Sprintf(` - CREATE INDEX IF NOT EXISTS %s ON %s - USING hnsw(embedding vector_cosine_ops) - `, indexName, p.tableName)) + err := p.execNoStatementTimeout(ctx, pgutil.VectorIndexSQL(p.tableName, indexName, "embedding", false)) if err != nil { xlog.Warn("Failed to create HNSW index", "error", err) return nil @@ -653,13 +611,7 @@ func (p *PostgresDB) migrateEmbeddingDimensions(ctx context.Context) error { return nil } -func formatVector(vec []float32) string { - parts := make([]string, len(vec)) - for i, v := range vec { - parts[i] = fmt.Sprintf("%.6f", v) - } - return "[" + strings.Join(parts, ",") + "]" -} +func formatVector(vec []float32) string { return pgutil.FormatVector(vec) } func (p *PostgresDB) Count() int { ctx := context.Background() diff --git a/rag/internal/pgutil/config.go b/rag/internal/pgutil/config.go new file mode 100644 index 0000000..a065bac --- /dev/null +++ b/rag/internal/pgutil/config.go @@ -0,0 +1,60 @@ +package pgutil + +import ( + "fmt" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/mudler/xlog" + "strconv" + "strings" +) + +const defaultPoolMaxConns = int32(4) + +func ApplyPoolLimits(config *pgxpool.Config, getenv func(string) string) { + if HasPoolMaxConns(config.ConnString()) { + return + } + config.MaxConns = defaultPoolMaxConns + if v := getenv("POSTGRES_POOL_MAX_CONNS"); v != "" { + n, err := strconv.ParseInt(v, 10, 32) + if err != nil || n < 1 { + xlog.Warn("Ignoring invalid POSTGRES_POOL_MAX_CONNS", "value", v) + return + } + config.MaxConns = int32(n) + } +} +func HasPoolMaxConns(connString string) bool { + cfg, err := pgconn.ParseConfig(connString) + if err != nil { + return false + } + _, ok := cfg.RuntimeParams["pool_max_conns"] + return ok +} +func ApplyConnTimeouts(config *pgxpool.Config, getenv func(string) string) { + if config.ConnConfig.RuntimeParams == nil { + config.ConnConfig.RuntimeParams = map[string]string{} + } + set := func(param, env, def string) { + v := def + if e := getenv(env); e != "" { + v = e + } + if v == "" || v == "0" || strings.EqualFold(v, "off") { + return + } + config.ConnConfig.RuntimeParams[param] = v + } + set("lock_timeout", "POSTGRES_LOCK_TIMEOUT", "30s") + set("idle_in_transaction_session_timeout", "POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT", "300s") + set("statement_timeout", "POSTGRES_STATEMENT_TIMEOUT", "") +} +func FormatVector(vec []float32) string { + parts := make([]string, len(vec)) + for i, v := range vec { + parts[i] = fmt.Sprintf("%.6f", v) + } + return "[" + strings.Join(parts, ",") + "]" +} diff --git a/rag/internal/pgutil/config_test.go b/rag/internal/pgutil/config_test.go new file mode 100644 index 0000000..74e97d5 --- /dev/null +++ b/rag/internal/pgutil/config_test.go @@ -0,0 +1,53 @@ +package pgutil + +import ( + "github.com/jackc/pgx/v5/pgxpool" + "testing" +) + +func TestPoolCompatibility(t *testing.T) { + for _, tc := range []struct { + dsn, env string + want int32 + }{ + {"postgres://fixture/db", "", 4}, + {"postgres://fixture/db", "7", 7}, + {"postgres://fixture/db", "invalid", 4}, + {"postgres://fixture/db?pool_max_conns=9", "7", 9}, + } { + c, e := pgxpool.ParseConfig(tc.dsn) + if e != nil { + t.Fatal(e) + } + ApplyPoolLimits(c, func(string) string { return tc.env }) + if c.MaxConns != tc.want { + t.Fatalf("%+v: %d", tc, c.MaxConns) + } + } + c, e := pgxpool.ParseConfig("postgres://fixture/db") + if e != nil { + t.Fatal(e) + } + ApplyConnTimeouts(c, func(string) string { return "" }) + if c.ConnConfig.RuntimeParams["lock_timeout"] != "30s" || c.ConnConfig.RuntimeParams["idle_in_transaction_session_timeout"] != "300s" || c.ConnConfig.RuntimeParams["statement_timeout"] != "" { + t.Fatal(c.ConnConfig.RuntimeParams) + } +} + +func TestIndexSQLCompatibility(t *testing.T) { + if got := VectorIndexSQL("documents", "idx_embedding", "embedding", false); got != "CREATE INDEX IF NOT EXISTS idx_embedding ON documents USING hnsw (embedding vector_cosine_ops)" { + t.Fatal(got) + } + if got := VectorIndexSQL("documents", "idx_embedding", "embedding", true); got != "CREATE INDEX IF NOT EXISTS idx_embedding ON documents USING diskann (embedding)" { + t.Fatal(got) + } + if got := BM25IndexSQL("documents", "idx_bm25", "full_text", "english"); got != "CREATE INDEX IF NOT EXISTS idx_bm25 ON documents USING bm25 (full_text) WITH (text_config='english')" { + t.Fatal(got) + } + if got := Vector([]float32{0.12345678}); got != "[0.12345678]" { + t.Fatal(got) + } + if got := FormatVector([]float32{0.12345678}); got != "[0.123457]" { + t.Fatal(got) + } +} diff --git a/rag/internal/pgutil/index.go b/rag/internal/pgutil/index.go new file mode 100644 index 0000000..f947ba7 --- /dev/null +++ b/rag/internal/pgutil/index.go @@ -0,0 +1,74 @@ +// Package pgutil contains PostgreSQL primitives with no embedding dependencies. +package pgutil + +import ( + "context" + "fmt" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "strconv" + "strings" +) + +type DB interface { + Exec(context.Context, string, ...any) (pgconn.CommandTag, error) + QueryRow(context.Context, string, ...any) pgx.Row +} + +// Vector preserves float32 round-trip precision; FormatVector preserves legacy six decimals. +func Vector(v []float32) string { + p := make([]string, len(v)) + for i, x := range v { + p[i] = strconv.FormatFloat(float64(x), 'g', -1, 32) + } + return "[" + strings.Join(p, ",") + "]" +} +func ResolveTextConfig(ctx context.Context, db DB, name string) (string, error) { + var v string + err := db.QueryRow(ctx, `SELECT n.nspname || '.' || c.cfgname FROM pg_ts_config c JOIN pg_namespace n ON n.oid=c.cfgnamespace WHERE c.oid=$1::regconfig`, name).Scan(&v) + return v, err +} +func HasAccessMethod(ctx context.Context, db DB, name string) bool { + var yes bool + err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_am WHERE amname=$1)`, name).Scan(&yes) + return err == nil && yes +} +func VectorIndexSQL(table, index, column string, diskANN bool) string { + method := "hnsw (" + column + " vector_cosine_ops)" + if diskANN { + method = "diskann (" + column + ")" + } + return fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s USING %s", index, table, method) +} +func BM25IndexSQL(table, index, column, config string) string { + return fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s USING bm25 (%s) WITH (text_config='%s')", index, table, column, strings.ReplaceAll(config, "'", "''")) +} +func CreateBM25(ctx context.Context, db DB, table, index, column, config string) error { + _, err := db.Exec(ctx, BM25IndexSQL(table, index, column, config)) + return err +} + +// CreateVectorIndex uses a savepoint so a failed DiskANN build can safely try HNSW. +func CreateVectorIndex(ctx context.Context, tx pgx.Tx, table, index, column string, dimensions int) error { + if dimensions < 1 || dimensions > 2000 { + return fmt.Errorf("unsupported dimensions") + } + if HasAccessMethod(ctx, tx, "diskann") { + if _, err := tx.Exec(ctx, "SAVEPOINT raw_ann"); err != nil { + return err + } + _, err := tx.Exec(ctx, VectorIndexSQL(table, index, column, true)) + if err == nil { + _, err = tx.Exec(ctx, "RELEASE SAVEPOINT raw_ann") + return err + } + if _, err = tx.Exec(ctx, "ROLLBACK TO SAVEPOINT raw_ann"); err != nil { + return err + } + } + if !HasAccessMethod(ctx, tx, "hnsw") { + return nil + } // exact-only readiness, never an implicit search fallback + _, err := tx.Exec(ctx, VectorIndexSQL(table, index, column, false)) + return err +} diff --git a/rag/raw/postgres/acceptance_test.go b/rag/raw/postgres/acceptance_test.go new file mode 100644 index 0000000..4522322 --- /dev/null +++ b/rag/raw/postgres/acceptance_test.go @@ -0,0 +1,211 @@ +package postgres + +import ( + "context" + "os" + "strings" + "testing" + + "github.com/mudler/localrecall/rag/internal/pgutil" + "github.com/mudler/localrecall/rag/raw" +) + +func TestRawLifecycleSharedCursor(t *testing.T) { + s, ctx := fixture(t) + for i := 0; i < 2; i++ { + c, _, e := s.CreateCollection(ctx, request()) + if e != nil { + t.Fatal(e) + } + defer s.DeleteCollection(ctx, c.CollectionID) + } + n := 1 + p, e := s.ListCollections(ctx, raw.ListCollectionsRequest{Limit: &n}) + if e != nil || p.NextCursor == nil { + t.Fatalf("%+v %v", p, e) + } + last, e := raw.DecodeCollectionCursor(*p.NextCursor) + if e != nil || last != p.Collections[0].CollectionID { + t.Fatalf("shared cursor: %s %v", last, e) + } + p, e = s.ListCollections(ctx, raw.ListCollectionsRequest{Limit: &n, Cursor: p.NextCursor}) + if e != nil || len(p.Collections) != 1 || p.Collections[0].CollectionID <= last { + t.Fatalf("next: %+v %v", p, e) + } +} + +func TestRawCapabilitiesSchemaDegradation(t *testing.T) { + s, ctx := fixture(t) + exec(t, s, "UPDATE lr_raw_schema SET version=2") + defer exec(t, s, "UPDATE lr_raw_schema SET version=1") + c, e := s.Capabilities(ctx) + if e != nil || c.Ready || c.ModeReadiness["vector"].Ready { + t.Fatalf("corrupt schema ready: %+v %v", c, e) + } +} + +func TestRawLifecycleWithoutBM25(t *testing.T) { + s, ctx := fixture(t) + exec(t, s, "DROP EXTENSION pg_textsearch") + defer exec(t, s, "CREATE EXTENSION pg_textsearch VERSION '1.3.1'") + c, e := s.Capabilities(ctx) + if e != nil || !c.Ready || c.ModeReadiness["lexical"].Ready { + t.Fatalf("degraded: %+v %v", c, e) + } + var before, after int + count := `SELECT count(*) FROM pg_class WHERE relname LIKE 'lr_raw_%'` + if e = s.pool.QueryRow(ctx, count).Scan(&before); e != nil { + t.Fatal(e) + } + _, _, e = s.CreateCollection(ctx, request()) + checkCode(t, e, raw.CodeUnavailable) + if e = s.pool.QueryRow(ctx, count).Scan(&after); e != nil || before != after { + t.Fatalf("DDL leaked %d %d %v", before, after, e) + } + r := request() + no := false + r.Lexical = &raw.LexicalRequest{Enabled: &no} + got, _, e := s.CreateCollection(ctx, r) + if e != nil { + t.Fatal(e) + } + defer s.DeleteCollection(ctx, got.CollectionID) +} + +func TestRawLifecycleSnapshotAndRestart(t *testing.T) { + s, ctx := fixture(t) + t.Setenv("BM25_TEXT_CONFIG", "simple") + c, _, e := s.CreateCollection(ctx, request()) + if e != nil { + t.Fatal(e) + } + t.Setenv("BM25_TEXT_CONFIG", "english") + s.Close() + reopened, e := New(ctx, Config{os.Getenv("LOCALRECALL_TEST_DATABASE_URL")}) + if e != nil { + t.Fatal(e) + } + defer reopened.Close() + defer reopened.DeleteCollection(ctx, c.CollectionID) + got, e := reopened.GetCollection(ctx, c.CollectionID) + if e != nil || got.Lexical.TextConfig != "pg_catalog.simple" { + t.Fatalf("snapshot %+v %v", got, e) + } + var legacy bool + if e = reopened.pool.QueryRow(ctx, `SELECT to_regclass('collection_config') IS NOT NULL`).Scan(&legacy); e != nil || legacy { + t.Fatalf("legacy registry touched: %v %v", legacy, e) + } +} + +func TestRawPoolCorruptTableCleanup(t *testing.T) { + s, ctx := fixture(t) + c, _, e := s.CreateCollection(ctx, request()) + if e != nil { + t.Fatal(e) + } + table, _ := tableName(c.CollectionID) + exec(t, s, "ALTER TABLE "+table+" ALTER COLUMN metadata DROP NOT NULL") + defer exec(t, s, "DELETE FROM lr_raw_collections WHERE collection_id=$1", c.CollectionID) + defer exec(t, s, "DROP TABLE "+table) + url := os.Getenv("LOCALRECALL_TEST_DATABASE_URL") + "&application_name=raw_failed_constructor" + for i := 0; i < 3; i++ { + broken, e := New(ctx, Config{url}) + if broken != nil { + broken.Close() + t.Fatal("corruption admitted") + } + checkCode(t, e, raw.CodeUnavailable) + } + var n int + if e = s.pool.QueryRow(ctx, `SELECT count(*) FROM pg_stat_activity WHERE application_name='raw_failed_constructor'`).Scan(&n); e != nil || n != 0 { + t.Fatalf("constructor connections leaked: %d %v", n, e) + } + _, e = s.GetCollection(ctx, c.CollectionID) + checkCode(t, e, raw.CodeUnavailable) +} + +func TestRawLegacyHelpers(t *testing.T) { + s, ctx := fixture(t) + tx, e := s.pool.Begin(ctx) + if e != nil { + t.Fatal(e) + } + defer tx.Rollback(context.Background()) + _, e = tx.Exec(ctx, `CREATE TABLE legacy_helper_fixture (embedding vector(3),full_text text)`) + if e != nil { + t.Fatal(e) + } + for _, q := range []string{pgutil.BM25IndexSQL("legacy_helper_fixture", "legacy_bm25", "full_text", "english"), pgutil.VectorIndexSQL("legacy_helper_fixture", "legacy_ann", "embedding", false)} { + if _, e = tx.Exec(ctx, q); e != nil { + t.Fatal(e) + } + } + _, e = tx.Exec(ctx, `INSERT INTO legacy_helper_fixture VALUES ('[1,0,0]','legacy regression')`) + if e != nil { + t.Fatal(e) + } + var distance float64 + if e = tx.QueryRow(ctx, `SELECT embedding <=> '[1,0,0]'::vector FROM legacy_helper_fixture`).Scan(&distance); e != nil || distance != 0 { + t.Fatal(distance, e) + } + if e = tx.QueryRow(ctx, `SELECT full_text <@> 'regression' FROM legacy_helper_fixture ORDER BY full_text <@> 'regression' LIMIT 1`).Scan(&distance); e != nil { + t.Fatal(e) + } + if pgutil.FormatVector([]float32{0.12345678}) != "[0.123457]" || !strings.Contains(pgutil.VectorIndexSQL("t", "i", "embedding", true), "diskann (embedding)") { + t.Fatal("legacy formatting changed") + } +} + +func TestRawCapabilitiesMissingCreatePrivilege(t *testing.T) { + s, ctx := fixture(t) + role := "raw_probe_denied" + exec(t, s, "CREATE ROLE "+role) + defer exec(t, s, "DROP ROLE "+role) + exec(t, s, "GRANT SELECT ON lr_raw_schema,lr_raw_collections TO "+role) + defer exec(t, s, "REVOKE SELECT ON lr_raw_schema,lr_raw_collections FROM "+role) + // Bootstrap reads an empty registry successfully, but CREATE capability must + // be false even though pgvector and BM25 remain installed and selectable. + denied, e := New(ctx, Config{os.Getenv("LOCALRECALL_TEST_DATABASE_URL") + "&options=-c%20role%3D" + role}) + if e != nil { + t.Fatal(e) + } + defer denied.Close() + c, e := denied.Capabilities(ctx) + if e != nil || c.Ready { + t.Fatalf("unprivileged ready: %+v %v", c, e) + } + if len(c.Indexes) != 3 || !c.Indexes[1].Installed || c.Indexes[1].Ready { + t.Fatalf("installed vs usable: %+v", c.Indexes) + } + _, _, e = denied.CreateCollection(ctx, request()) + checkCode(t, e, raw.CodeUnavailable) +} + +func TestRawLifecycleResetStatementSnapshot(t *testing.T) { + s, ctx := fixture(t) + c, _, e := s.CreateCollection(ctx, request()) + if e != nil { + t.Fatal(e) + } + defer s.DeleteCollection(ctx, c.CollectionID) + table, _ := tableName(c.CollectionID) + exec(t, s, "INSERT INTO "+table+" (id,vector) VALUES ('before','[1,0,0]')") + writer, e := s.pool.Begin(ctx) + if e != nil { + t.Fatal(e) + } + defer writer.Rollback(ctx) + if _, e = writer.Exec(ctx, "INSERT INTO "+table+" (id,vector) VALUES ('concurrent','[0,1,0]')"); e != nil { + t.Fatal(e) + } + if e = s.Reset(ctx, c.CollectionID); e != nil { + t.Fatal(e) + } + if e = writer.Commit(ctx); e != nil { + t.Fatal(e) + } + var id string + if e = s.pool.QueryRow(ctx, "SELECT id FROM "+table).Scan(&id); e != nil || id != "concurrent" { + t.Fatalf("reset became a writer barrier: %s %v", id, e) + } +} diff --git a/rag/raw/postgres/capabilities.go b/rag/raw/postgres/capabilities.go new file mode 100644 index 0000000..e6affdd --- /dev/null +++ b/rag/raw/postgres/capabilities.go @@ -0,0 +1,97 @@ +package postgres + +import ( + "context" + "os" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/mudler/localrecall/rag/internal/pgutil" + "github.com/mudler/localrecall/rag/raw" +) + +func readiness(ok bool) raw.Readiness { + if ok { + return raw.Readiness{Ready: true} + } + return raw.Readiness{Reason: "Required dependency unavailable"} +} + +// Capabilities probes actual DDL and operators under the service role. All probe +// objects are rolled back; installed access methods alone do not imply usability. +func (s *Service) Capabilities(ctx context.Context) (raw.Capabilities, error) { + c := raw.Capabilities{Backend: "postgres", RawCollections: true, Modes: []string{"vector", "lexical", "hybrid"}, AccuracyOptions: []string{"approximate", "exact"}, DefaultAccuracy: "approximate", Metrics: []string{"cosine"}, Limits: raw.ContractLimits(), AtomicBatch: true, Persistent: true, ModeReadiness: map[string]raw.Readiness{}} + for _, mode := range c.Modes { + c.ModeReadiness[mode] = readiness(false) + } + c.Readiness = readiness(false) + ctx, done, e := s.operation(ctx) + if e != nil { + return c, nil + } + defer done() + tx, e := s.pool.Begin(ctx) + if e != nil { + return c, nil + } + defer tx.Rollback(context.Background()) + installed := map[string]bool{} + for _, name := range []string{"diskann", "hnsw", "bm25"} { + installed[name] = pgutil.HasAccessMethod(ctx, tx, name) + c.Indexes = append(c.Indexes, raw.IndexCapability{Strategy: name, Installed: installed[name], Readiness: readiness(false)}) + } + if schemaVersion(ctx, tx) != nil { + return c, nil + } + if _, e = tx.Exec(ctx, "SELECT "+columns+" FROM lr_raw_collections LIMIT 0"); e != nil { + return c, nil + } + table := pgx.Identifier{"lr_probe_" + uuid.NewString()}.Sanitize() + if _, e = tx.Exec(ctx, "CREATE TABLE "+table+" (id text PRIMARY KEY, vector vector(3), text text)"); e != nil { + return c, nil + } + if _, e = tx.Exec(ctx, "INSERT INTO "+table+" VALUES ('probe','[1,0,0]','readiness probe')"); e != nil { + return c, nil + } + var score float64 + if e = tx.QueryRow(ctx, "SELECT vector <=> '[1,0,0]'::vector FROM "+table).Scan(&score); e != nil { + return c, nil + } + c.Readiness = readiness(true) + c.ModeReadiness["vector"] = readiness(true) + for i, name := range []string{"diskann", "hnsw", "bm25"} { + if !installed[name] { + continue + } + if _, e = tx.Exec(ctx, "SAVEPOINT capability"); e != nil { + return c, nil + } + index := pgx.Identifier{"lr_probe_index_" + uuid.NewString()}.Sanitize() + if name == "bm25" { + var config string + config, e = pgutil.ResolveTextConfig(ctx, tx, raw.ResolveLexical(nil, os.Getenv("BM25_TEXT_CONFIG")).TextConfig) + if e == nil { + e = pgutil.CreateBM25(ctx, tx, table, index, "text", config) + } + if e == nil { + e = tx.QueryRow(ctx, "SELECT text <@> 'probe' FROM "+table+" ORDER BY text <@> 'probe' LIMIT 1").Scan(&score) + } + } else { + _, e = tx.Exec(ctx, pgutil.VectorIndexSQL(table, index, "vector", name == "diskann")) + } + ok := e == nil + // Isolate each failed dependency and remove each successful probe index too. + if _, e = tx.Exec(ctx, "ROLLBACK TO SAVEPOINT capability"); e != nil { + return c, nil + } + if _, e = tx.Exec(ctx, "RELEASE SAVEPOINT capability"); e != nil { + return c, nil + } + c.Indexes[i].Readiness = readiness(ok) + if name == "bm25" { + c.ModeReadiness["lexical"] = readiness(ok) + c.ModeReadiness["hybrid"] = readiness(ok) + } + } + return c, nil +} diff --git a/rag/raw/postgres/lifecycle.go b/rag/raw/postgres/lifecycle.go new file mode 100644 index 0000000..94b78ca --- /dev/null +++ b/rag/raw/postgres/lifecycle.go @@ -0,0 +1,163 @@ +package postgres + +import ( + "context" + "fmt" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/mudler/localrecall/rag/internal/pgutil" + "github.com/mudler/localrecall/rag/raw" + "os" +) + +func (s *Service) CreateCollection(ctx context.Context, r raw.CreateCollectionRequest) (c raw.Collection, created bool, err error) { + if err = raw.ValidateCreate(r); err != nil { + return + } + lex := raw.ResolveLexical(r.Lexical, os.Getenv("BM25_TEXT_CONFIG")) + ctx, done, e := s.operation(ctx) + if e != nil { + return c, false, e + } + defer done() + // READ COMMITTED gives the post-advisory-lock lookup a fresh snapshot. + tx, e := s.pool.Begin(ctx) + if e != nil { + return c, false, dbError(e) + } + defer tx.Rollback(context.Background()) + _, e = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,763214992))`, r.Name) + if e != nil { + return c, false, dbError(e) + } + if e = schemaVersion(ctx, tx); e != nil { + return c, false, dbError(e) + } + if lex.Enabled { + lex.TextConfig, e = pgutil.ResolveTextConfig(ctx, tx, lex.TextConfig) + if e != nil { + return c, false, raw.NewError(raw.CodeValidation) + } + } + c, e = scan(tx.QueryRow(ctx, "SELECT "+columns+" FROM lr_raw_collections WHERE name=$1", r.Name)) + if e == nil { + if c.Dimensions != r.Dimensions || c.Metric != r.Metric || c.EmbeddingSpaceID != r.EmbeddingSpaceID || c.Lexical != lex { + return c, false, raw.NewError(raw.CodeConflict) + } + if e = validateTable(ctx, tx, c); e != nil { + return raw.Collection{}, false, dbError(e) + } + return c, false, dbError(tx.Commit(ctx)) + } + if e != pgx.ErrNoRows { + return c, false, dbError(e) + } + id := uuid.NewString() + table, _ := tableName(id) + _, e = tx.Exec(ctx, fmt.Sprintf(`CREATE TABLE %s (id text COLLATE "C" PRIMARY KEY,vector vector(%d) NOT NULL,text text,metadata jsonb NOT NULL DEFAULT '{}')`, table, r.Dimensions)) + if e != nil { + return c, false, dbError(e) + } + if lex.Enabled { + if e = pgutil.CreateBM25(ctx, tx, table, pgx.Identifier{"lr_bm25_" + id}.Sanitize(), "text", lex.TextConfig); e != nil { + return c, false, dbError(e) + } + } + if e = pgutil.CreateVectorIndex(ctx, tx, table, pgx.Identifier{"lr_ann_" + id}.Sanitize(), "vector", r.Dimensions); e != nil { + return c, false, dbError(e) + } + c, e = scan(tx.QueryRow(ctx, `INSERT INTO lr_raw_collections(collection_id,name,dimensions,metric,embedding_space_id,lexical_enabled,text_config,schema_version) VALUES($1,$2,$3,$4,$5,$6,$7,1) RETURNING `+columns, id, r.Name, r.Dimensions, r.Metric, r.EmbeddingSpaceID, lex.Enabled, lex.TextConfig)) + if e != nil { + return raw.Collection{}, false, dbError(e) + } + if e = tx.Commit(ctx); e != nil { + return raw.Collection{}, false, dbError(e) + } + return c, true, nil +} +func (s *Service) GetCollection(ctx context.Context, id string) (c raw.Collection, err error) { + err = s.transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { var e error; c, e = lookup(ctx, tx, id); return e }) + if err != nil { + c = raw.Collection{} + } + return +} +func (s *Service) Reset(ctx context.Context, id string) error { + return s.transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { + c, e := lookup(ctx, tx, id) + if e != nil { + return e + } + table, _ := tableName(c.CollectionID) + _, e = tx.Exec(ctx, "DELETE FROM "+table) + return e + }) +} +func (s *Service) DeleteCollection(ctx context.Context, id string) error { + return s.transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { + c, e := lookup(ctx, tx, id) + if e != nil { + return e + } + table, _ := tableName(c.CollectionID) + if _, e = tx.Exec(ctx, "DROP TABLE "+table); e != nil { + return e + } + _, e = tx.Exec(ctx, "DELETE FROM lr_raw_collections WHERE collection_id=$1", id) + return e + }) +} +func (s *Service) ListCollections(ctx context.Context, r raw.ListCollectionsRequest) (p raw.CollectionPage, err error) { + limit, e := raw.ValidateLimit(r.Limit) + if e != nil { + return p, e + } + after := "" + if r.Cursor != nil { + after, e = raw.DecodeCollectionCursor(*r.Cursor) + if e != nil { + return p, e + } + } + err = s.transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { + if e := schemaVersion(ctx, tx); e != nil { + return e + } + rows, e := tx.Query(ctx, "SELECT "+columns+" FROM lr_raw_collections WHERE collection_id::text > $1 COLLATE \"C\" ORDER BY collection_id::text COLLATE \"C\" LIMIT $2", after, limit+1) + if e != nil { + return e + } + p.Collections = []raw.Collection{} + for rows.Next() { + c, e := scan(rows) + if e != nil { + rows.Close() + return e + } + p.Collections = append(p.Collections, c) + } + e = rows.Err() + rows.Close() + if e != nil { + return e + } + for _, c := range p.Collections { + if e = validateTable(ctx, tx, c); e != nil { + return e + } + } + if len(p.Collections) > limit { + p.Collections = p.Collections[:limit] + v, e := raw.EncodeCollectionCursor(p.Collections[limit-1].CollectionID) + if e != nil { + return e + } + p.NextCursor = &v + } + return nil + }) + if err != nil { + p = raw.CollectionPage{} + } + return +} diff --git a/rag/raw/postgres/lifecycle_test.go b/rag/raw/postgres/lifecycle_test.go new file mode 100644 index 0000000..474877e --- /dev/null +++ b/rag/raw/postgres/lifecycle_test.go @@ -0,0 +1,253 @@ +package postgres + +import ( + "context" + "errors" + "os" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/mudler/localrecall/rag/raw" +) + +func fixture(t *testing.T) (*Service, context.Context) { + t.Helper() + if os.Getenv("LOCALRECALL_TEST_DATABASE_URL") == "" || os.Getenv("LOCALRECALL_TEST_DATABASE_OWNER") == "" { + t.Fatal("owned database fixture required; run sh test/raw/verify.sh lifecycle") + } + ctx := context.Background() + s, e := New(ctx, Config{os.Getenv("LOCALRECALL_TEST_DATABASE_URL")}) + if e != nil { + t.Fatal(e) + } + t.Cleanup(s.Close) + return s, ctx +} +func request() raw.CreateCollectionRequest { + return raw.CreateCollectionRequest{Name: uuid.NewString(), Dimensions: 3, Metric: "cosine", EmbeddingSpaceID: "space"} +} +func checkCode(t *testing.T, e error, code string) { + t.Helper() + var r *raw.Error + if !errors.As(e, &r) || r.Code != code { + t.Fatalf("error %v; want %s", e, code) + } +} +func exec(t *testing.T, s *Service, q string, args ...any) { + t.Helper() + if _, e := s.pool.Exec(context.Background(), q, args...); e != nil { + t.Fatal(e) + } +} + +func TestRawLifecycle(t *testing.T) { + s, ctx := fixture(t) + r := request() + r.Name = "quoted ' ; " + r.Name + c, created, e := s.CreateCollection(ctx, r) + if e != nil || !created { + t.Fatalf("create %v %v", created, e) + } + defer s.DeleteCollection(ctx, c.CollectionID) + if c.Kind != "raw" || c.Backend != "postgres" || c.SchemaVersion != 1 || raw.ValidateCollectionID(c.CollectionID) != nil { + t.Fatal(c) + } + again, created, e := s.CreateCollection(ctx, r) + if e != nil || created || again.CollectionID != c.CollectionID { + t.Fatalf("idempotency %v %v", created, e) + } + r.Dimensions++ + _, _, e = s.CreateCollection(ctx, r) + checkCode(t, e, raw.CodeConflict) + other, e := New(ctx, Config{os.Getenv("LOCALRECALL_TEST_DATABASE_URL")}) + if e != nil { + t.Fatal(e) + } + defer other.Close() + got, e := other.GetCollection(ctx, c.CollectionID) + if e != nil || got.CollectionID != c.CollectionID { + t.Fatalf("reopen %v", e) + } + page, e := other.ListCollections(ctx, raw.ListCollectionsRequest{}) + if e != nil || len(page.Collections) == 0 { + t.Fatalf("list %v", e) + } + table, _ := tableName(c.CollectionID) + exec(t, s, "INSERT INTO "+table+" (id,vector,text,metadata) VALUES ('a','[1,0,0]','hello','{}')") + if e = s.Reset(ctx, c.CollectionID); e != nil { + t.Fatal(e) + } + var n int + if e = s.pool.QueryRow(ctx, "SELECT count(*) FROM "+table).Scan(&n); e != nil || n != 0 { + t.Fatalf("reset %d %v", n, e) + } + if e = s.DeleteCollection(ctx, c.CollectionID); e != nil { + t.Fatal(e) + } + _, e = s.GetCollection(ctx, c.CollectionID) + checkCode(t, e, raw.CodeNotFound) + _, e = s.GetCollection(ctx, "'; drop table lr_raw_collections;") + checkCode(t, e, raw.CodeValidation) +} +func TestRawLifecycleConcurrentCreate(t *testing.T) { + s, ctx := fixture(t) + r := request() + var wg sync.WaitGroup + ids := make(chan string, 8) + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c, _, e := s.CreateCollection(ctx, r) + if e != nil { + t.Error(e) + return + } + ids <- c.CollectionID + }() + } + wg.Wait() + close(ids) + first := "" + for id := range ids { + if first == "" { + first = id + } + if first != id { + t.Fatal("different IDs") + } + } + if first != "" { + defer s.DeleteCollection(ctx, first) + } +} +func TestRawLifecycleLexicalRollback(t *testing.T) { + s, ctx := fixture(t) + r := request() + bad := "not_a_config'; --" + r.Lexical = &raw.LexicalRequest{TextConfig: &bad} + _, _, e := s.CreateCollection(ctx, r) + if e == nil { + t.Fatal("invalid catalog config accepted") + } + var n int + s.pool.QueryRow(ctx, "SELECT count(*) FROM lr_raw_collections WHERE name=$1", r.Name).Scan(&n) + if n != 0 { + t.Fatal("partial descriptor") + } + no := false + r.Lexical.Enabled = &no + c, _, e := s.CreateCollection(ctx, r) + if e != nil { + t.Fatal(e) + } + defer s.DeleteCollection(ctx, c.CollectionID) + if c.Lexical.Enabled { + t.Fatal("lexical enabled") + } +} +func TestRawBootstrapRejectCorruption(t *testing.T) { + s, ctx := fixture(t) + r := request() + c, _, e := s.CreateCollection(ctx, r) + if e != nil { + t.Fatal(e) + } + table, _ := tableName(c.CollectionID) + exec(t, s, "DROP TABLE "+table) + _, e = s.GetCollection(ctx, c.CollectionID) + checkCode(t, e, raw.CodeUnavailable) + broken, e := New(ctx, Config{os.Getenv("LOCALRECALL_TEST_DATABASE_URL")}) + if broken != nil { + broken.Close() + } + if e == nil { + t.Fatal("reconstructed missing table") + } + exec(t, s, "DELETE FROM lr_raw_collections WHERE collection_id=$1", c.CollectionID) + exec(t, s, "UPDATE lr_raw_schema SET version=2") + broken, e = New(ctx, Config{os.Getenv("LOCALRECALL_TEST_DATABASE_URL")}) + if broken != nil { + broken.Close() + } + exec(t, s, "UPDATE lr_raw_schema SET version=1") + if e == nil { + t.Fatal("accepted future schema") + } +} +func TestRawLifecycleBoundedLocks(t *testing.T) { + s, ctx := fixture(t) + c, _, e := s.CreateCollection(ctx, request()) + if e != nil { + t.Fatal(e) + } + defer s.DeleteCollection(ctx, c.CollectionID) + tx, e := s.pool.Begin(ctx) + if e != nil { + t.Fatal(e) + } + defer tx.Rollback(ctx) + table, _ := tableName(c.CollectionID) + if _, e = tx.Exec(ctx, "LOCK TABLE "+table+" IN ACCESS EXCLUSIVE MODE"); e != nil { + t.Fatal(e) + } + start := time.Now() + e = s.Reset(ctx, c.CollectionID) + if e == nil || time.Since(start) > 7*time.Second { + t.Fatalf("unbounded wait %s %v", time.Since(start), e) + } +} +func TestRawCapabilities(t *testing.T) { + s, ctx := fixture(t) + c, e := s.Capabilities(ctx) + if e != nil || !c.Ready || !c.ModeReadiness["lexical"].Ready { + t.Fatalf("capabilities %+v %v", c, e) + } +} +func TestRawPool(t *testing.T) { + s, ctx := fixture(t) + p := s.pool + for i := 0; i < 3; i++ { + c, _, e := s.CreateCollection(ctx, request()) + if e != nil { + t.Fatal(e) + } + if s.pool != p { + t.Fatal("pool changed") + } + if e = s.DeleteCollection(ctx, c.CollectionID); e != nil { + t.Fatal(e) + } + } + s.Close() + s.Close() + _, e := s.GetCollection(ctx, uuid.NewString()) + if e == nil { + t.Fatal("closed service accepted operation") + } +} +func TestRawBootstrapPrivileges(t *testing.T) { + s, ctx := fixture(t) + role := "raw_denied_" + uuid.New().String()[:8] + ident := pgx.Identifier{role}.Sanitize() + exec(t, s, "CREATE ROLE "+ident+" LOGIN PASSWORD 'fixture-only'") + defer exec(t, s, "DROP ROLE "+ident) + cfg, e := pgx.ParseConfig(os.Getenv("LOCALRECALL_TEST_DATABASE_URL")) + if e != nil { + t.Fatal(e) + } + cfg.User = role + cfg.Password = "fixture-only" + // SET ROLE on the connection string preserves the fixture's endpoint. + url := os.Getenv("LOCALRECALL_TEST_DATABASE_URL") + "&options=-c%20role%3D" + role + denied, e := New(ctx, Config{url}) + if denied != nil { + denied.Close() + } + if e == nil { + t.Fatal("restricted role admitted") + } +} diff --git a/rag/raw/postgres/registry.go b/rag/raw/postgres/registry.go new file mode 100644 index 0000000..a3aee38 --- /dev/null +++ b/rag/raw/postgres/registry.go @@ -0,0 +1,132 @@ +package postgres + +import ( + "context" + "fmt" + "github.com/jackc/pgx/v5" + "github.com/mudler/localrecall/rag/raw" + "strings" +) + +const columns = `collection_id::text,name,dimensions,metric,embedding_space_id,lexical_enabled,text_config,schema_version,created_at` + +func tableName(id string) (string, error) { + if e := raw.ValidateCollectionID(id); e != nil { + return "", e + } + return pgx.Identifier{"lr_raw_" + strings.ReplaceAll(id, "-", "")}.Sanitize(), nil +} +func scan(row pgx.Row) (c raw.Collection, e error) { + c.Kind = "raw" + c.Backend = "postgres" + e = row.Scan(&c.CollectionID, &c.Name, &c.Dimensions, &c.Metric, &c.EmbeddingSpaceID, &c.Lexical.Enabled, &c.Lexical.TextConfig, &c.SchemaVersion, &c.CreatedAt) + return +} +func schemaVersion(ctx context.Context, tx pgx.Tx) error { + var n, v int + if e := tx.QueryRow(ctx, "SELECT count(*),coalesce(min(version),0) FROM lr_raw_schema").Scan(&n, &v); e != nil { + return e + } + if n != 1 || v != 1 { + return raw.NewError(raw.CodeUnavailable) + } + return nil +} +func (s *Service) bootstrap(ctx context.Context) error { + tx, e := s.pool.Begin(ctx) + if e != nil { + return e + } + defer tx.Rollback(ctx) + if _, e = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(763214991)`); e != nil { + return e + } + var vector bool + if e = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname='vector')`).Scan(&vector); e != nil || !vector { + return raw.NewError(raw.CodeUnavailable) + } + var marker, registry bool + if e = tx.QueryRow(ctx, `SELECT to_regclass('lr_raw_schema') IS NOT NULL,to_regclass('lr_raw_collections') IS NOT NULL`).Scan(&marker, ®istry); e != nil { + return e + } + if marker != registry { + return raw.NewError(raw.CodeUnavailable) + } + if !marker { + _, e = tx.Exec(ctx, `CREATE TABLE lr_raw_schema (version integer NOT NULL); INSERT INTO lr_raw_schema VALUES (1); + CREATE TABLE lr_raw_collections (collection_id uuid PRIMARY KEY,name text COLLATE "C" NOT NULL UNIQUE,dimensions integer NOT NULL CHECK(dimensions BETWEEN 1 AND 2000),metric text NOT NULL CHECK(metric='cosine'),embedding_space_id text NOT NULL,lexical_enabled boolean NOT NULL,text_config text NOT NULL,schema_version integer NOT NULL CHECK(schema_version=1),created_at timestamptz NOT NULL DEFAULT now())`) + if e != nil { + return e + } + } + if e = schemaVersion(ctx, tx); e != nil { + return e + } + rows, e := tx.Query(ctx, "SELECT "+columns+" FROM lr_raw_collections") + if e != nil { + return e + } + var cs []raw.Collection + for rows.Next() { + c, e := scan(rows) + if e != nil { + rows.Close() + return e + } + cs = append(cs, c) + } + e = rows.Err() + rows.Close() + if e != nil { + return e + } + for _, c := range cs { + if e = validateTable(ctx, tx, c); e != nil { + return e + } + } + return tx.Commit(ctx) +} +func validateTable(ctx context.Context, tx pgx.Tx, c raw.Collection) error { + if c.SchemaVersion != 1 || raw.ValidateCreate(raw.CreateCollectionRequest{Name: c.Name, Dimensions: c.Dimensions, Metric: c.Metric, EmbeddingSpaceID: c.EmbeddingSpaceID}) != nil { + return raw.NewError(raw.CodeUnavailable) + } + table, e := tableName(c.CollectionID) + if e != nil { + return e + } + // Hold a relation lock until transaction end, preventing concurrent DROP/ALTER. + if _, e = tx.Exec(ctx, "SELECT id,vector,text,metadata FROM "+table+" LIMIT 0"); e != nil { + return e + } + var valid bool + e = tx.QueryRow(ctx, `SELECT count(*)=4 AND bool_and(CASE attname + WHEN 'id' THEN atttypid='text'::regtype AND attnotnull AND attcollation='"C"'::regcollation AND EXISTS(SELECT 1 FROM pg_index WHERE indrelid=attrelid AND indisprimary AND indkey::text=attnum::text) + WHEN 'vector' THEN atttypid='vector'::regtype AND atttypmod=$2 AND attnotnull + WHEN 'text' THEN atttypid='text'::regtype AND NOT attnotnull + WHEN 'metadata' THEN atttypid='jsonb'::regtype AND attnotnull ELSE false END) + FROM pg_attribute WHERE attrelid=$1::regclass AND attnum>0 AND NOT attisdropped`, table, c.Dimensions).Scan(&valid) + if e != nil { + return e + } + if !valid { + return fmt.Errorf("invalid raw table structure") + } + return nil +} +func lookup(ctx context.Context, tx pgx.Tx, id string) (raw.Collection, error) { + if e := raw.ValidateCollectionID(id); e != nil { + return raw.Collection{}, e + } + if e := schemaVersion(ctx, tx); e != nil { + return raw.Collection{}, e + } + c, e := scan(tx.QueryRow(ctx, "SELECT "+columns+" FROM lr_raw_collections WHERE collection_id=$1", id)) + if e == pgx.ErrNoRows { + return c, raw.NewError(raw.CodeNotFound) + } + if e != nil { + return c, e + } + return c, validateTable(ctx, tx, c) +} diff --git a/rag/raw/postgres/service.go b/rag/raw/postgres/service.go new file mode 100644 index 0000000..91c573e --- /dev/null +++ b/rag/raw/postgres/service.go @@ -0,0 +1,101 @@ +// Package postgres implements isolated, embedder-free raw PostgreSQL storage. +package postgres + +import ( + "context" + "errors" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/mudler/localrecall/rag/internal/pgutil" + "github.com/mudler/localrecall/rag/raw" + "os" + "sync" + "time" +) + +type Config struct{ DatabaseURL string } +type Service struct { + pool *pgxpool.Pool + mu sync.Mutex + closed bool + wg sync.WaitGroup + stop context.Context + cancel context.CancelFunc + once sync.Once +} + +func New(ctx context.Context, cfg Config) (*Service, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + pc, e := pgxpool.ParseConfig(cfg.DatabaseURL) + if e != nil { + return nil, dbError(e) + } + pgutil.ApplyPoolLimits(pc, os.Getenv) + pgutil.ApplyConnTimeouts(pc, os.Getenv) + pc.ConnConfig.RuntimeParams["lock_timeout"] = "5s" + pc.ConnConfig.RuntimeParams["statement_timeout"] = "30s" + p, e := pgxpool.NewWithConfig(ctx, pc) + if e != nil { + return nil, dbError(e) + } + stop, c := context.WithCancel(context.Background()) + s := &Service{pool: p, stop: stop, cancel: c} + if e = s.bootstrap(ctx); e != nil { + s.Close() + return nil, dbError(e) + } + return s, nil +} +func (s *Service) operation(ctx context.Context) (context.Context, func(), error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return nil, nil, raw.NewError(raw.CodeUnavailable) + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + stop := context.AfterFunc(s.stop, cancel) + s.wg.Add(1) + return ctx, func() { stop(); cancel(); s.wg.Done() }, nil +} +func (s *Service) Close() { + s.once.Do(func() { + s.mu.Lock() + s.closed = true + s.cancel() + s.mu.Unlock() + s.wg.Wait() + s.pool.Close() + }) +} +func dbError(e error) error { + if e == nil { + return nil + } + var r *raw.Error + if errors.As(e, &r) { + return e + } + var p *pgconn.PgError + if errors.Is(e, context.DeadlineExceeded) || errors.Is(e, context.Canceled) || (errors.As(e, &p) && (p.Code == "57014" || p.Code == "55P03")) { + return raw.NewError(raw.CodeTimeout) + } + return raw.NewError(raw.CodeUnavailable) +} +func (s *Service) transaction(ctx context.Context, fn func(context.Context, pgx.Tx) error) error { + ctx, done, e := s.operation(ctx) + if e != nil { + return e + } + defer done() + tx, e := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead}) + if e != nil { + return dbError(e) + } + defer tx.Rollback(context.Background()) + if e = fn(ctx, tx); e != nil { + return dbError(e) + } + return dbError(tx.Commit(ctx)) +} diff --git a/test/raw/Dockerfile.postgres b/test/raw/Dockerfile.postgres new file mode 100644 index 0000000..69b1db5 --- /dev/null +++ b/test/raw/Dockerfile.postgres @@ -0,0 +1,4 @@ +FROM postgres:18-bookworm +RUN apt-get update && apt-get install -y --no-install-recommends build-essential git ca-certificates postgresql-server-dev-18 postgresql-18-pgvector && rm -rf /var/lib/apt/lists/* +RUN git clone --depth 1 --branch v1.3.1 https://github.com/timescale/pg_textsearch /tmp/pg_textsearch && make -C /tmp/pg_textsearch && make -C /tmp/pg_textsearch install && rm -rf /tmp/pg_textsearch +CMD ["postgres", "-c", "shared_preload_libraries=pg_textsearch"] diff --git a/test/raw/database.sh b/test/raw/database.sh new file mode 100644 index 0000000..ed0250b --- /dev/null +++ b/test/raw/database.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Execute a command against a new, private fixture; never accept an external URL. +set -eu +[ -f test/raw/Dockerfile.postgres ] || { echo 'run from worktree root' >&2; exit 1; } +[ -z "${LOCALRECALL_TEST_DATABASE_URL:-}" ] || { echo 'external database targets refused' >&2; exit 1; } +ROOT=$(pwd -P) +[ ! -L .sdd-work ] && [ ! -L .sdd-work/docker ] || exit 1 +mkdir -p .sdd-work/docker .sdd-work/tmp .sdd-work/logs +export DOCKER_CONFIG="$ROOT/.sdd-work/docker" TMPDIR="$ROOT/.sdd-work/tmp" +token=$(od -An -N12 -tx1 /dev/urandom | tr -d ' \n') +name="lr-raw-$token" +image="localrecall-raw-task2-a5975c8" +cleanup() { + if [ "$(docker inspect -f '{{index .Config.Labels "localrecall.raw.owner"}}' "$name" 2>/dev/null || true)" = "$token" ]; then + docker rm -fv "$name" >/dev/null + fi + printf "Owned fixture %s removed\n" "$name" +} +trap cleanup EXIT HUP INT TERM +docker image inspect "$image" >/dev/null +# PG18 declares /var/lib/postgresql as a volume; a private tmpfs avoids persistent volumes. +docker run -d --name "$name" --label "localrecall.raw.owner=$token" --tmpfs /var/lib/postgresql:rw -p 127.0.0.1::5432 -e POSTGRES_PASSWORD="$token" -e POSTGRES_DB=raw_fixture "$image" >/dev/null +n=0 +until docker exec "$name" pg_isready -U postgres -d raw_fixture >/dev/null 2>&1; do + n=$((n+1)); [ "$n" -lt 90 ] || { docker logs "$name"; exit 1; }; sleep 1 +done +docker exec -i "$name" psql -v ON_ERROR_STOP=1 -U postgres -d raw_fixture <<'SQL' +CREATE EXTENSION vector; +CREATE EXTENSION pg_textsearch VERSION '1.3.1'; +CREATE TABLE fixture_probe (id text PRIMARY KEY, vector vector(3), text text); +CREATE INDEX fixture_hnsw ON fixture_probe USING hnsw (vector vector_cosine_ops); +CREATE INDEX fixture_bm25 ON fixture_probe USING bm25 (text) WITH (text_config='english'); +INSERT INTO fixture_probe VALUES ('one','[1,0,0]','fixture readiness'), ('two','[0,1,0]','other'); +SELECT vector <=> '[1,0,0]'::vector FROM fixture_probe ORDER BY vector <=> '[1,0,0]'::vector LIMIT 1; +SELECT text <@> 'readiness' FROM fixture_probe ORDER BY text <@> 'readiness' LIMIT 1; +DROP TABLE fixture_probe; +SQL +port=$(docker port "$name" 5432/tcp | sed -n 's/^127\.0\.0\.1://p') +case "$port" in ''|*[!0-9]*) exit 1;; esac +export LOCALRECALL_TEST_DATABASE_URL="postgres://postgres:$token@127.0.0.1:$port/raw_fixture?sslmode=disable" +export LOCALRECALL_TEST_DATABASE_OWNER="$token" +printf 'Owned fixture %s at loopback port %s (removed on exit)\n' "$name" "$port" +"$@" diff --git a/test/raw/verify.sh b/test/raw/verify.sh index 12432fc..fe21387 100644 --- a/test/raw/verify.sh +++ b/test/raw/verify.sh @@ -13,11 +13,22 @@ export COLLECTION_DB_PATH="$ROOT/.sdd-work/collections" FILE_ASSETS="$ROOT/.sdd- export GOENV=off GOTOOLCHAIN=local mkdir -p "$TMPDIR" "$GOCACHE" "$GOMODCACHE" "$GOPATH" "$COLLECTION_DB_PATH" "$FILE_ASSETS" "$ROOT/.sdd-work/logs" case "${1:-}" in + lifecycle) + sh test/raw/verify.sh contracts + go test -v -mod=readonly -race -count=1 ./rag/internal/pgutil/... + log="$ROOT/.sdd-work/logs/lifecycle-$(date +%s)-$$.log" + if sh test/raw/database.sh go test -v -mod=readonly -race -count=1 -timeout=10m ./rag/raw/postgres/... -run "^(TestRaw(Lifecycle|Bootstrap|Capabilities|Pool|LegacyHelpers))" >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi + for suite in Lifecycle Bootstrap Capabilities Pool LegacyHelpers; do + grep -q -- "--- PASS: TestRaw${suite}" "$log" + done + ! grep -E "SKIP|no tests to run|no test files" "$log" + git diff --check + ;; contracts) log="$ROOT/.sdd-work/logs/contracts-$(date +%s)-$$.log" if go test -mod=readonly -race -count=1 ./rag/raw ./test/raw >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi [ "$(grep -c '^ok[[:space:]]' "$log")" -eq 2 ] && ! grep -E '\[no test files\]|\[no tests to run\]' "$log" git diff --check ;; - *) echo 'usage: sh test/raw/verify.sh contracts' >&2; exit 2;; + *) echo 'usage: sh test/raw/verify.sh contracts|lifecycle' >&2; exit 2;; esac From 28f5705e8fafbda710d6c34dc1cad62af834587e Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 11 Oct 2026 22:30:51 +0000 Subject: [PATCH 04/14] fix(raw): honor reset and text config semantics Use a fresh statement snapshot for reset while protecting table lifetime. Preserve quoted text configuration names through creation and reopen. Keep operational lookup failures distinct from invalid configuration. Add deterministic regressions for the reset boundary and lookup failures. --- rag/internal/pgutil/index.go | 8 +- rag/raw/postgres/lifecycle.go | 21 +++-- rag/raw/postgres/review_test.go | 137 ++++++++++++++++++++++++++++++++ rag/raw/postgres/service.go | 4 +- 4 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 rag/raw/postgres/review_test.go diff --git a/rag/internal/pgutil/index.go b/rag/internal/pgutil/index.go index f947ba7..16aa028 100644 --- a/rag/internal/pgutil/index.go +++ b/rag/internal/pgutil/index.go @@ -25,7 +25,7 @@ func Vector(v []float32) string { } func ResolveTextConfig(ctx context.Context, db DB, name string) (string, error) { var v string - err := db.QueryRow(ctx, `SELECT n.nspname || '.' || c.cfgname FROM pg_ts_config c JOIN pg_namespace n ON n.oid=c.cfgnamespace WHERE c.oid=$1::regconfig`, name).Scan(&v) + err := db.QueryRow(ctx, `SELECT format('%I.%I', n.nspname, c.cfgname) FROM pg_ts_config c JOIN pg_namespace n ON n.oid=c.cfgnamespace WHERE c.oid=$1::regconfig`, name).Scan(&v) return v, err } func HasAccessMethod(ctx context.Context, db DB, name string) bool { @@ -33,6 +33,9 @@ func HasAccessMethod(ctx context.Context, db DB, name string) bool { err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_am WHERE amname=$1)`, name).Scan(&yes) return err == nil && yes } + +// VectorIndexSQL requires trusted SQL identifier fragments for table, index and +// column (for dynamic identifiers, use pgx.Identifier.Sanitize). func VectorIndexSQL(table, index, column string, diskANN bool) string { method := "hnsw (" + column + " vector_cosine_ops)" if diskANN { @@ -40,6 +43,9 @@ func VectorIndexSQL(table, index, column string, diskANN bool) string { } return fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s USING %s", index, table, method) } + +// BM25IndexSQL requires trusted SQL identifier fragments for table, index and +// column. config is a value and is escaped as a SQL string literal. func BM25IndexSQL(table, index, column, config string) string { return fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s USING bm25 (%s) WITH (text_config='%s')", index, table, column, strings.ReplaceAll(config, "'", "''")) } diff --git a/rag/raw/postgres/lifecycle.go b/rag/raw/postgres/lifecycle.go index 94b78ca..1c210d8 100644 --- a/rag/raw/postgres/lifecycle.go +++ b/rag/raw/postgres/lifecycle.go @@ -2,9 +2,11 @@ package postgres import ( "context" + "errors" "fmt" "github.com/google/uuid" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/mudler/localrecall/rag/internal/pgutil" "github.com/mudler/localrecall/rag/raw" "os" @@ -36,7 +38,13 @@ func (s *Service) CreateCollection(ctx context.Context, r raw.CreateCollectionRe if lex.Enabled { lex.TextConfig, e = pgutil.ResolveTextConfig(ctx, tx, lex.TextConfig) if e != nil { - return c, false, raw.NewError(raw.CodeValidation) + // Only malformed or unknown catalog names are caller errors. + // Cancellation, privileges and dependency failures retain dbError mapping. + var pe *pgconn.PgError + if errors.As(e, &pe) && (pe.Code == "42704" || pe.Code == "42602" || pe.Code == "42601" || pe.Code == "3F000") { + return c, false, raw.NewError(raw.CodeValidation) + } + return c, false, dbError(e) } } c, e = scan(tx.QueryRow(ctx, "SELECT "+columns+" FROM lr_raw_collections WHERE name=$1", r.Name)) @@ -76,14 +84,17 @@ func (s *Service) CreateCollection(ctx context.Context, r raw.CreateCollectionRe return c, true, nil } func (s *Service) GetCollection(ctx context.Context, id string) (c raw.Collection, err error) { - err = s.transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { var e error; c, e = lookup(ctx, tx, id); return e }) + err = s.transaction(ctx, pgx.RepeatableRead, func(ctx context.Context, tx pgx.Tx) error { var e error; c, e = lookup(ctx, tx, id); return e }) if err != nil { c = raw.Collection{} } return } func (s *Service) Reset(ctx context.Context, id string) error { - return s.transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { + // Each statement needs a fresh snapshot: inserts committed after lookup + // but before DELETE must be cleared. lookup holds the relation lock until + // commit, protecting table lifetime without blocking independent writers. + return s.transaction(ctx, pgx.ReadCommitted, func(ctx context.Context, tx pgx.Tx) error { c, e := lookup(ctx, tx, id) if e != nil { return e @@ -94,7 +105,7 @@ func (s *Service) Reset(ctx context.Context, id string) error { }) } func (s *Service) DeleteCollection(ctx context.Context, id string) error { - return s.transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { + return s.transaction(ctx, pgx.ReadCommitted, func(ctx context.Context, tx pgx.Tx) error { c, e := lookup(ctx, tx, id) if e != nil { return e @@ -119,7 +130,7 @@ func (s *Service) ListCollections(ctx context.Context, r raw.ListCollectionsRequ return p, e } } - err = s.transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { + err = s.transaction(ctx, pgx.RepeatableRead, func(ctx context.Context, tx pgx.Tx) error { if e := schemaVersion(ctx, tx); e != nil { return e } diff --git a/rag/raw/postgres/review_test.go b/rag/raw/postgres/review_test.go new file mode 100644 index 0000000..5092c67 --- /dev/null +++ b/rag/raw/postgres/review_test.go @@ -0,0 +1,137 @@ +package postgres + +import ( + "context" + "strings" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/mudler/localrecall/rag/raw" +) + +// TraceQueryStart runs before PostgreSQL receives the statement, giving these +// tests a deterministic boundary without sleeps or production test hooks. +type queryBoundary func(context.Context, *pgx.Conn, string) context.Context + +func (f queryBoundary) TraceQueryStart(ctx context.Context, c *pgx.Conn, d pgx.TraceQueryStartData) context.Context { + return f(ctx, c, d.SQL) +} +func (f queryBoundary) TraceQueryEnd(context.Context, *pgx.Conn, pgx.TraceQueryEndData) {} +func tracedPool(t *testing.T, s *Service, f queryBoundary) { + t.Helper() + cfg := s.pool.Config() + cfg.ConnConfig.Tracer = f + p, err := pgxpool.NewWithConfig(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + s.pool.Close() + s.pool = p +} + +func TestRawLifecycleResetFreshDeleteSnapshot(t *testing.T) { + s, ctx := fixture(t) + writer, _ := fixture(t) + c, _, err := s.CreateCollection(ctx, request()) + if err != nil { + t.Fatal(err) + } + defer writer.DeleteCollection(ctx, c.CollectionID) + table, _ := tableName(c.CollectionID) + reached := false + tracedPool(t, s, func(ctx context.Context, conn *pgx.Conn, q string) context.Context { + if q == "DELETE FROM "+table { + reached = true + // Lookup and its table validation have finished; DELETE has not started. + exec(t, writer, "INSERT INTO "+table+" (id,vector) VALUES ('between','[1,0,0]')") + } + return ctx + }) + if err := s.Reset(ctx, c.CollectionID); err != nil { + t.Fatal(err) + } + var n int + if err := writer.pool.QueryRow(ctx, "SELECT count(*) FROM "+table).Scan(&n); err != nil { + t.Fatal(err) + } + if !reached || n != 0 { + t.Fatalf("DELETE boundary reached=%v; surviving committed records=%d", reached, n) + } +} + +func TestRawLifecycleQuotedTextConfig(t *testing.T) { + s, ctx := fixture(t) + exec(t, s, `CREATE SCHEMA "Mixed.Schema space"`) + defer exec(t, s, `DROP SCHEMA "Mixed.Schema space" CASCADE`) + exec(t, s, `CREATE TEXT SEARCH CONFIGURATION "Mixed.Schema space"."Config.Name space" (COPY=pg_catalog.english)`) + for _, name := range []string{`"Mixed.Schema space"."Config.Name space"`, "english"} { + t.Run(name, func(t *testing.T) { + r := request() + r.Lexical = &raw.LexicalRequest{TextConfig: &name} + c, created, err := s.CreateCollection(ctx, r) + if err != nil || !created { + t.Fatalf("create: %v %v", created, err) + } + defer s.DeleteCollection(ctx, c.CollectionID) + want := name + if name == "english" { + want = "pg_catalog.english" + } + if c.Lexical.TextConfig != want { + t.Fatalf("config=%q want=%q", c.Lexical.TextConfig, want) + } + reopened, _ := fixture(t) + got, err := reopened.GetCollection(ctx, c.CollectionID) + if err != nil || got.Lexical != c.Lexical { + t.Fatalf("reopen: %+v %v", got, err) + } + again, created, err := reopened.CreateCollection(ctx, r) + if err != nil || created || again.CollectionID != c.CollectionID { + t.Fatalf("idempotent reopen: %+v %v %v", again, created, err) + } + }) + } +} + +func TestRawLifecycleTextConfigErrors(t *testing.T) { + for _, tc := range []struct{ name, code string }{ + {"unknown", raw.CodeValidation}, {"malformed", raw.CodeValidation}, + {"cancel", raw.CodeTimeout}, {"dependency", raw.CodeUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + s, ctx := fixture(t) + reached := false + tracedPool(t, s, func(ctx context.Context, c *pgx.Conn, q string) context.Context { + if strings.Contains(q, "FROM pg_ts_config") { + reached = true + if tc.name == "cancel" { + canceled, cancel := context.WithCancel(ctx) + cancel() + return canceled + } + if tc.name == "dependency" { + if err := c.Close(ctx); err != nil { + t.Fatal(err) + } + } + } + return ctx + }) + name := "english" + if tc.name == "unknown" { + name = "no_such_raw_config" + } + if tc.name == "malformed" { + name = `"unterminated` + } + r := request() + r.Lexical = &raw.LexicalRequest{TextConfig: &name} + _, _, err := s.CreateCollection(ctx, r) + if !reached { + t.Fatal("configuration lookup not reached") + } + checkCode(t, err, tc.code) + }) + } +} diff --git a/rag/raw/postgres/service.go b/rag/raw/postgres/service.go index 91c573e..3a97b3d 100644 --- a/rag/raw/postgres/service.go +++ b/rag/raw/postgres/service.go @@ -83,13 +83,13 @@ func dbError(e error) error { } return raw.NewError(raw.CodeUnavailable) } -func (s *Service) transaction(ctx context.Context, fn func(context.Context, pgx.Tx) error) error { +func (s *Service) transaction(ctx context.Context, isolation pgx.TxIsoLevel, fn func(context.Context, pgx.Tx) error) error { ctx, done, e := s.operation(ctx) if e != nil { return e } defer done() - tx, e := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead}) + tx, e := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: isolation}) if e != nil { return dbError(e) } From 8798cfe9b1834300c888ecc60f6c7c2e3f4c0cb3 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 11 Oct 2026 22:36:37 +0000 Subject: [PATCH 05/14] feat(raw): add atomic PostgreSQL records Store caller-supplied vectors without entering the embedding lifecycle. Replace each batch atomically and preserve request order in responses. Bound filtered deletion and bind weakly consistent cursors to projections. Verify with sh test/raw/verify.sh records. --- rag/raw/postgres/cursors.go | 21 ++ rag/raw/postgres/filters.go | 28 +++ rag/raw/postgres/records.go | 206 ++++++++++++++++++ rag/raw/postgres/records_concurrency_test.go | 160 ++++++++++++++ rag/raw/postgres/records_test.go | 210 +++++++++++++++++++ test/raw/verify.sh | 12 +- 6 files changed, 636 insertions(+), 1 deletion(-) create mode 100644 rag/raw/postgres/cursors.go create mode 100644 rag/raw/postgres/filters.go create mode 100644 rag/raw/postgres/records.go create mode 100644 rag/raw/postgres/records_concurrency_test.go create mode 100644 rag/raw/postgres/records_test.go diff --git a/rag/raw/postgres/cursors.go b/rag/raw/postgres/cursors.go new file mode 100644 index 0000000..e2f31f9 --- /dev/null +++ b/rag/raw/postgres/cursors.go @@ -0,0 +1,21 @@ +package postgres + +import "github.com/mudler/localrecall/rag/raw" + +// Record cursors share the contract's untrusted-input validation and binding. +// They do not encode a revision: mutations never globally invalidate paging. +func recordPageInput(id string, r raw.ListRecordsRequest) (limit int, after string, err error) { + if err = raw.ValidateCollectionID(id); err != nil { + return + } + if err = raw.ValidateFilter(r.Filter); err != nil { + return + } + if limit, err = raw.ValidateLimit(r.Limit); err != nil { + return + } + if r.Cursor != nil { + after, err = raw.DecodeRecordCursor(*r.Cursor, id, r.Filter, r.IncludeVectors) + } + return +} diff --git a/rag/raw/postgres/filters.go b/rag/raw/postgres/filters.go new file mode 100644 index 0000000..e91850c --- /dev/null +++ b/rag/raw/postgres/filters.go @@ -0,0 +1,28 @@ +package postgres + +import ( + "fmt" + "github.com/mudler/localrecall/rag/raw" + "sort" + "strings" +) + +// metadataPredicate only accepts validated equality filters. Both keys and +// values are parameters, including keys that resemble SQL or JSON paths. +func metadataPredicate(f *raw.Filter, args *[]any) string { + if f == nil || len(f.MetadataEq) == 0 { + return "TRUE" + } + keys := make([]string, 0, len(f.MetadataEq)) + for k := range f.MetadataEq { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + n := len(*args) + 1 + *args = append(*args, k, f.MetadataEq[k]) + parts = append(parts, fmt.Sprintf("metadata ->> $%d::text = $%d::text", n, n+1)) + } + return strings.Join(parts, " AND ") +} diff --git a/rag/raw/postgres/records.go b/rag/raw/postgres/records.go new file mode 100644 index 0000000..42d2902 --- /dev/null +++ b/rag/raw/postgres/records.go @@ -0,0 +1,206 @@ +package postgres + +import ( + "context" + "encoding/json" + "fmt" + "github.com/jackc/pgx/v5" + "github.com/mudler/localrecall/rag/internal/pgutil" + "github.com/mudler/localrecall/rag/raw" + "sort" +) + +// Upsert fully replaces records. Stable IDs prevent duplicates, not stale +// retries: a delayed retry can overwrite a newer committed value. +func (s *Service) Upsert(ctx context.Context, id string, r raw.UpsertRequest) (out raw.UpsertResponse, err error) { + err = s.transaction(ctx, pgx.ReadCommitted, func(ctx context.Context, tx pgx.Tx) error { + c, e := lookup(ctx, tx, id) + if e != nil { + return e + } + records, e := raw.ValidateUpsert(c, r) + if e != nil { + return e + } + out.IDs = make([]string, len(records)) + for i, v := range records { + out.IDs[i] = v.ID + } + // Every writer takes row locks in C order, independent of request order. + sort.Slice(records, func(i, j int) bool { return records[i].ID < records[j].ID }) + table, _ := tableName(id) + for _, v := range records { + metadata, e := json.Marshal(v.Metadata) + if e != nil { + return e + } + _, e = tx.Exec(ctx, "INSERT INTO "+table+" (id,vector,text,metadata) VALUES ($1,$2::vector,$3,$4::jsonb) ON CONFLICT (id) DO UPDATE SET vector=EXCLUDED.vector,text=EXCLUDED.text,metadata=EXCLUDED.metadata", v.ID, pgutil.Vector(v.Vector), v.Text, string(metadata)) + if e != nil { + return e + } + } + return nil + }) + if err != nil { + out = raw.UpsertResponse{} + } + return +} + +func recordColumns(vectors bool) string { + if vectors { + return "id,text,metadata,vector::text" + } + return "id,text,metadata" +} +func readRecords(rows pgx.Rows, vectors bool) ([]raw.CanonicalRecord, error) { + defer rows.Close() + out := []raw.CanonicalRecord{} + for rows.Next() { + var r raw.CanonicalRecord + var vector string + dst := []any{&r.ID, &r.Text, &r.Metadata} + if vectors { + dst = append(dst, &vector) + } + if e := rows.Scan(dst...); e != nil { + return nil, e + } + if vectors { + if e := json.Unmarshal([]byte(vector), &r.Vector); e != nil { + return nil, e + } + } + out = append(out, r) + } + return out, rows.Err() +} +func (s *Service) GetRecords(ctx context.Context, id string, r raw.GetRecordsRequest) (out raw.GetRecordsResponse, err error) { + if err = raw.ValidateIDs(r.IDs); err != nil { + return + } + err = s.transaction(ctx, pgx.RepeatableRead, func(ctx context.Context, tx pgx.Tx) error { + if _, e := lookup(ctx, tx, id); e != nil { + return e + } + table, _ := tableName(id) + rows, e := tx.Query(ctx, "SELECT "+recordColumns(r.IncludeVectors)+" FROM "+table+" WHERE id=ANY($1::text[])", r.IDs) + if e != nil { + return e + } + records, e := readRecords(rows, r.IncludeVectors) + if e != nil { + return e + } + byID := make(map[string]raw.CanonicalRecord, len(records)) + for _, v := range records { + byID[v.ID] = v + } + out.Records = []raw.CanonicalRecord{} + out.MissingIDs = []string{} + for _, id := range r.IDs { + if v, ok := byID[id]; ok { + out.Records = append(out.Records, v) + } else { + out.MissingIDs = append(out.MissingIDs, id) + } + } + return nil + }) + if err != nil { + out = raw.GetRecordsResponse{} + } + return +} +func (s *Service) ListRecords(ctx context.Context, id string, r raw.ListRecordsRequest) (out raw.RecordPage, err error) { + limit, after, e := recordPageInput(id, r) + if e != nil { + return out, e + } + err = s.transaction(ctx, pgx.RepeatableRead, func(ctx context.Context, tx pgx.Tx) error { + if _, e := lookup(ctx, tx, id); e != nil { + return e + } + table, _ := tableName(id) + args := []any{after, limit + 1} + predicate := metadataPredicate(r.Filter, &args) + rows, e := tx.Query(ctx, "SELECT "+recordColumns(r.IncludeVectors)+" FROM "+table+` WHERE id > $1 COLLATE "C" AND `+predicate+` ORDER BY id COLLATE "C" LIMIT $2`, args...) + if e != nil { + return e + } + out.Records, e = readRecords(rows, r.IncludeVectors) + if e != nil { + return e + } + if len(out.Records) > limit { + out.Records = out.Records[:limit] + cursor, e := raw.EncodeRecordCursor(id, out.Records[limit-1].ID, r.Filter, r.IncludeVectors) + if e != nil { + return e + } + out.NextCursor = &cursor + } + return nil + }) + if err != nil { + out = raw.RecordPage{} + } + return +} +func (s *Service) DeleteRecords(ctx context.Context, id string, r raw.DeleteRecordsRequest) (out raw.DeleteRecordsResponse, err error) { + if err = raw.ValidateDelete(r); err != nil { + return + } + err = s.transaction(ctx, pgx.ReadCommitted, func(ctx context.Context, tx pgx.Tx) error { + if _, e := lookup(ctx, tx, id); e != nil { + return e + } + table, _ := tableName(id) + args := []any{} + predicate := "" + limit := raw.MaxDeleteMatches + 1 + if r.IDs != nil { + args = append(args, r.IDs) + predicate = "id=ANY($1::text[])" + limit = raw.MaxBatch + } else { + predicate = metadataPredicate(r.Filter, &args) + } + // Lock a bounded, deterministically ordered selection. Delete only these + // IDs: later matching inserts are allowed to survive this request. + rows, e := tx.Query(ctx, "SELECT id FROM "+table+" WHERE "+predicate+fmt.Sprintf(` ORDER BY id COLLATE "C" LIMIT %d FOR UPDATE`, limit), args...) + if e != nil { + return e + } + ids := []string{} + for rows.Next() { + var id string + if e = rows.Scan(&id); e != nil { + rows.Close() + return e + } + ids = append(ids, id) + } + e = rows.Err() + rows.Close() + if e != nil { + return e + } + if len(ids) > raw.MaxDeleteMatches { + return raw.NewError(raw.CodeDeleteBound) + } + if len(ids) == 0 { + return nil + } + tag, e := tx.Exec(ctx, "DELETE FROM "+table+" WHERE id=ANY($1::text[])", ids) + if e != nil { + return e + } + out.DeletedCount = int(tag.RowsAffected()) + return nil + }) + if err != nil { + out = raw.DeleteRecordsResponse{} + } + return +} diff --git a/rag/raw/postgres/records_concurrency_test.go b/rag/raw/postgres/records_concurrency_test.go new file mode 100644 index 0000000..473f252 --- /dev/null +++ b/rag/raw/postgres/records_concurrency_test.go @@ -0,0 +1,160 @@ +package postgres + +import ( + "context" + "github.com/jackc/pgx/v5" + "github.com/mudler/localrecall/rag/raw" + "strings" + "sync" + "testing" + "time" +) + +func TestRawConcurrencyOverlapping(t *testing.T) { + s, ctx, id := recordFixture(t) + var wg sync.WaitGroup + errs := make(chan error, 12) + for i := 0; i < 12; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + a, b := rec("a"), rec("b") + text := string(rune('A' + i)) + a.Text = &text + b.Text = &text + rs := []raw.Record{a, b} + if i%2 == 0 { + rs = []raw.Record{b, a} + } + _, e := s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: rs}) + errs <- e + }(i) + } + wg.Wait() + close(errs) + for e := range errs { + if e != nil { + t.Fatal(e) + } + } + got, e := s.GetRecords(ctx, id, raw.GetRecordsRequest{IDs: []string{"a", "b"}}) + if e != nil || *got.Records[0].Text != *got.Records[1].Text { + t.Fatal(got, e) + } +} +func TestRawConcurrencyDisjointAndBounds(t *testing.T) { + s, ctx, id := recordFixture(t) + put(t, s, ctx, id, rec("a")) + table, _ := tableName(id) + tx, e := s.pool.Begin(ctx) + if e != nil { + t.Fatal(e) + } + defer tx.Rollback(ctx) + if _, e = tx.Exec(ctx, "SELECT id FROM "+table+" WHERE id='a' FOR UPDATE"); e != nil { + t.Fatal(e) + } + // A locked record must not impose a registry-wide writer lock or block reads. + fast, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + put(t, s, fast, id, rec("disjoint")) + if _, e = s.ListRecords(fast, id, raw.ListRecordsRequest{}); e != nil { + t.Fatal(e) + } + short, stop := context.WithTimeout(ctx, 100*time.Millisecond) + defer stop() + _, e = s.Upsert(short, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{rec("a")}}) + checkCode(t, e, raw.CodeTimeout) + start := time.Now() + _, e = s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{rec("a")}}) + checkCode(t, e, raw.CodeTimeout) + if elapsed := time.Since(start); elapsed < 4*time.Second || elapsed > 8*time.Second { + t.Fatal(elapsed) + } + op, done, e := s.operation(ctx) + if e != nil { + t.Fatal(e) + } + defer done() + deadline, ok := op.Deadline() + if !ok || time.Until(deadline) > 30*time.Second { + t.Fatal("request deadline absent") + } + canceled, c := context.WithCancel(ctx) + c() + _, e = s.GetRecords(canceled, id, raw.GetRecordsRequest{IDs: []string{"a"}}) + checkCode(t, e, raw.CodeTimeout) +} +func TestRawConcurrencyLockedDeleteLaterInsert(t *testing.T) { + s, ctx, id := recordFixture(t) + writer, _ := fixture(t) + table, _ := tableName(id) + f := &raw.Filter{MetadataEq: map[string]string{"source": "x"}} + r := rec("a") + r.Metadata = f.MetadataEq + put(t, s, ctx, id, r) + reached := false + tracedPool(t, s, func(ctx context.Context, c *pgx.Conn, q string) context.Context { + if strings.HasPrefix(q, "DELETE FROM "+table+" WHERE") { + reached = true + r.ID = "later" + put(t, writer, ctx, id, r) + } + return ctx + }) + got, e := s.DeleteRecords(ctx, id, raw.DeleteRecordsRequest{Filter: f}) + if e != nil || got.DeletedCount != 1 || !reached { + t.Fatal(got, e, reached) + } + p, e := writer.ListRecords(ctx, id, raw.ListRecordsRequest{}) + if e != nil || len(p.Records) != 1 || p.Records[0].ID != "later" { + t.Fatal(p, e) + } +} +func TestRawConcurrencyResetWriter(t *testing.T) { + s, ctx, id := recordFixture(t) + writer, _ := fixture(t) + put(t, s, ctx, id, rec("a")) + // Pause reset after its DELETE statement (at commit). A later insert can + // commit while reset still holds its row locks and is allowed to survive. + reached := false + tracedPool(t, s, func(ctx context.Context, c *pgx.Conn, q string) context.Context { + if q == "commit" && !reached { + reached = true + put(t, writer, ctx, id, rec("later")) + } + return ctx + }) + if e := s.Reset(ctx, id); e != nil { + t.Fatal(e) + } + p, e := writer.ListRecords(ctx, id, raw.ListRecordsRequest{}) + if e != nil || !reached || len(p.Records) != 1 || p.Records[0].ID != "later" { + t.Fatal(p, e, reached) + } +} +func TestRawConcurrencyReadSnapshotAndDDL(t *testing.T) { + s, ctx, id := recordFixture(t) + writer, _ := fixture(t) + put(t, s, ctx, id, rec("a")) + table, _ := tableName(id) + reached := false + tracedPool(t, s, func(ctx context.Context, c *pgx.Conn, q string) context.Context { + if strings.HasPrefix(q, "SELECT id,text,metadata FROM "+table) { + reached = true + put(t, writer, ctx, id, rec("b")) + short, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + checkCode(t, writer.DeleteCollection(short, id), raw.CodeTimeout) + } + return ctx + }) + p, e := s.ListRecords(ctx, id, raw.ListRecordsRequest{}) + if e != nil || !reached || len(p.Records) != 1 { + t.Fatal(p, e, reached) + } + p, e = writer.ListRecords(ctx, id, raw.ListRecordsRequest{}) + if e != nil || len(p.Records) != 2 { + t.Fatal(p, e) + } +} diff --git a/rag/raw/postgres/records_test.go b/rag/raw/postgres/records_test.go new file mode 100644 index 0000000..0f09f19 --- /dev/null +++ b/rag/raw/postgres/records_test.go @@ -0,0 +1,210 @@ +package postgres + +import ( + "context" + "fmt" + "github.com/mudler/localrecall/rag/raw" + "math" + "reflect" + "sort" + "strings" + "testing" +) + +func recordFixture(t *testing.T) (*Service, context.Context, string) { + t.Helper() + s, ctx := fixture(t) + c, _, e := s.CreateCollection(ctx, request()) + if e != nil { + t.Fatal(e) + } + t.Cleanup(func() { s.DeleteCollection(ctx, c.CollectionID) }) + return s, ctx, c.CollectionID +} +func rec(id string) raw.Record { return raw.Record{ID: id, Vector: []float64{1, 2, 3}} } +func put(t *testing.T, s *Service, ctx context.Context, id string, rs ...raw.Record) { + t.Helper() + got, e := s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: rs}) + if e != nil { + t.Fatal(e) + } + for i, r := range rs { + if got.IDs[i] != r.ID { + t.Fatal(got) + } + } +} +func TestRawRecordsReplacement(t *testing.T) { + s, ctx, id := recordFixture(t) + text := "bicycle" + r := rec("b") + r.Text = &text + r.Metadata = map[string]string{"source": "file:///not/resolved"} + put(t, s, ctx, id, r, rec("a")) + got, e := s.GetRecords(ctx, id, raw.GetRecordsRequest{IDs: []string{"b", "missing", "a"}, IncludeVectors: true}) + if e != nil || len(got.Records) != 2 || !reflect.DeepEqual(got.MissingIDs, []string{"missing"}) { + t.Fatalf("%+v %v", got, e) + } + v, _ := raw.NormalizeVector(r.Vector, 3) + if !reflect.DeepEqual(got.Records[0].Vector, v) { + t.Fatal(got) + } + put(t, s, ctx, id, rec("b")) + got, e = s.GetRecords(ctx, id, raw.GetRecordsRequest{IDs: []string{"b"}}) + if e != nil || got.Records[0].Text != nil || len(got.Records[0].Metadata) != 0 || got.Records[0].Vector != nil { + t.Fatalf("%+v %v", got, e) + } + // Stable IDs avoid duplicates, but a delayed retry intentionally overwrites newer data. + put(t, s, ctx, id, r) + got, e = s.GetRecords(ctx, id, raw.GetRecordsRequest{IDs: []string{"b"}}) + if e != nil || *got.Records[0].Text != text { + t.Fatal(got, e) + } + d, e := s.DeleteRecords(ctx, id, raw.DeleteRecordsRequest{IDs: []string{"missing", "a"}}) + if e != nil || d.DeletedCount != 1 { + t.Fatal(d, e) + } +} +func TestRawRecordsValidationRollback(t *testing.T) { + s, ctx, id := recordFixture(t) + for _, v := range [][]float64{{0, 0, 0}, {1, 2}, {math.NaN(), 0, 1}, {math.Inf(1), 0, 1}, {math.MaxFloat64, 0, 1}, {1e-100, 0, 0}} { + bad := rec("z") + bad.Vector = v + _, e := s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{rec("a"), bad}}) + checkCode(t, e, raw.CodeValidation) + } + _, e := s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{rec("a"), rec("a")}}) + checkCode(t, e, raw.CodeValidation) + _, e = s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "wrong", Records: []raw.Record{rec("a")}}) + checkCode(t, e, raw.CodeSpaceMismatch) + table, _ := tableName(id) + exec(t, s, "ALTER TABLE "+table+" ADD CHECK (id <> 'z')") + text := "rollbackuniqueterm" + r := rec("a") + r.Text = &text + _, e = s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{r, rec("z")}}) + if e == nil { + t.Fatal("injected failure accepted") + } + got, e := s.GetRecords(ctx, id, raw.GetRecordsRequest{IDs: []string{"a", "z"}}) + if e != nil || len(got.Records) != 0 { + t.Fatal(got, e) + } + // Force the BM25 access path after rollback; no aborted tuple may escape. + tx, e := s.pool.Begin(ctx) + if e != nil { + t.Fatal(e) + } + defer tx.Rollback(ctx) + if _, e = tx.Exec(ctx, "SET LOCAL enable_seqscan=off"); e != nil { + t.Fatal(e) + } + rows, e := tx.Query(ctx, "SELECT id FROM "+table+" ORDER BY text <@> 'rollbackuniqueterm' LIMIT 10") + if e != nil { + t.Fatal(e) + } + defer rows.Close() + if rows.Next() { + t.Fatal("aborted BM25 record visible") + } + if rows.Err() != nil { + t.Fatal(rows.Err()) + } +} +func TestRawCursorPaging(t *testing.T) { + s, ctx, id := recordFixture(t) + ids := []string{"é", "中", "😀", "z", "A", "\\\"", "\u2028"} + for _, v := range ids { + put(t, s, ctx, id, rec(v)) + } + sort.Strings(ids) + n := 2 + req := raw.ListRecordsRequest{Limit: &n} + var seen []string + for { + p, e := s.ListRecords(ctx, id, req) + if e != nil { + t.Fatal(e) + } + for _, r := range p.Records { + seen = append(seen, r.ID) + if r.Vector != nil { + t.Fatal(r) + } + } + if p.NextCursor == nil { + break + } + req.Cursor = p.NextCursor + } + if !reflect.DeepEqual(seen, ids) { + t.Fatal(seen, ids) + } + p, e := s.ListRecords(ctx, id, raw.ListRecordsRequest{Limit: &n}) + if e != nil { + t.Fatal(e) + } + for _, r := range []raw.ListRecordsRequest{{Cursor: p.NextCursor, IncludeVectors: true}, {Cursor: p.NextCursor, Filter: &raw.Filter{MetadataEq: map[string]string{"x": "y"}}}, {Cursor: ptr("bad")}, {Cursor: ptr(strings.Repeat("x", 2049))}} { + _, e = s.ListRecords(ctx, id, r) + checkCode(t, e, raw.CodeCursor) + } + other, _, cid := recordFixture(t) + _, e = other.ListRecords(ctx, cid, raw.ListRecordsRequest{Cursor: p.NextCursor}) + checkCode(t, e, raw.CodeCursor) + put(t, s, ctx, id, rec("0"), rec("zz")) + _, e = s.DeleteRecords(ctx, id, raw.DeleteRecordsRequest{IDs: []string{ids[0]}}) + if e != nil { + t.Fatal(e) + } + next, e := s.ListRecords(ctx, id, raw.ListRecordsRequest{Cursor: p.NextCursor}) + if e != nil || len(next.Records) == 0 { + t.Fatal(next, e) + } + for _, r := range next.Records { + if r.ID <= ids[1] { + t.Fatal("cursor invalidated", r) + } + } +} +func ptr(s string) *string { return &s } +func TestRawDeleteBound(t *testing.T) { + s, ctx, id := recordFixture(t) + f := &raw.Filter{MetadataEq: map[string]string{"x' OR true --": "v'"}} + for i := 0; i < 1001; i += 100 { + rs := []raw.Record{} + for j := i; j < i+100 && j < 1001; j++ { + r := rec(fmt.Sprintf("%04d", j)) + r.Metadata = f.MetadataEq + rs = append(rs, r) + } + put(t, s, ctx, id, rs...) + } + _, e := s.DeleteRecords(ctx, id, raw.DeleteRecordsRequest{Filter: f}) + checkCode(t, e, raw.CodeDeleteBound) + table, _ := tableName(id) + var count int + if e = s.pool.QueryRow(ctx, "SELECT count(*) FROM "+table).Scan(&count); e != nil || count != 1001 { + t.Fatal(count, e) + } + _, e = s.DeleteRecords(ctx, id, raw.DeleteRecordsRequest{IDs: []string{"1000"}}) + if e != nil { + t.Fatal(e) + } + d, e := s.DeleteRecords(ctx, id, raw.DeleteRecordsRequest{Filter: f}) + if e != nil || d.DeletedCount != 1000 { + t.Fatal(d, e) + } + for _, r := range []raw.DeleteRecordsRequest{{}, {IDs: []string{}}, {Filter: &raw.Filter{}}, {IDs: []string{"x"}, Filter: f}} { + _, e = s.DeleteRecords(ctx, id, r) + checkCode(t, e, raw.CodeValidation) + } + f.MetadataEq = map[string]string{} + for i := 0; i < 9; i++ { + f.MetadataEq[fmt.Sprint(i)] = "x" + } + _, e = s.ListRecords(ctx, id, raw.ListRecordsRequest{Filter: f}) + checkCode(t, e, raw.CodeValidation) + zero := 0 + _, e = s.ListRecords(ctx, id, raw.ListRecordsRequest{Limit: &zero}) + checkCode(t, e, raw.CodeValidation) +} diff --git a/test/raw/verify.sh b/test/raw/verify.sh index fe21387..c436b7e 100644 --- a/test/raw/verify.sh +++ b/test/raw/verify.sh @@ -13,6 +13,16 @@ export COLLECTION_DB_PATH="$ROOT/.sdd-work/collections" FILE_ASSETS="$ROOT/.sdd- export GOENV=off GOTOOLCHAIN=local mkdir -p "$TMPDIR" "$GOCACHE" "$GOMODCACHE" "$GOPATH" "$COLLECTION_DB_PATH" "$FILE_ASSETS" "$ROOT/.sdd-work/logs" case "${1:-}" in + records) + sh test/raw/verify.sh lifecycle + log="$ROOT/.sdd-work/logs/records-$(date +%s)-$$.log" + if sh test/raw/database.sh go test -v -mod=readonly -race -count=1 -timeout=10m ./rag/raw/postgres/... -run "^(TestRaw(Records|Concurrency|Cursor|DeleteBound))" >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi + for suite in Records Concurrency Cursor DeleteBound; do + grep -q -- "--- PASS: TestRaw${suite}" "$log" + done + ! grep -E "SKIP|no tests to run|no test files" "$log" + git diff --check + ;; lifecycle) sh test/raw/verify.sh contracts go test -v -mod=readonly -race -count=1 ./rag/internal/pgutil/... @@ -30,5 +40,5 @@ case "${1:-}" in [ "$(grep -c '^ok[[:space:]]' "$log")" -eq 2 ] && ! grep -E '\[no test files\]|\[no tests to run\]' "$log" git diff --check ;; - *) echo 'usage: sh test/raw/verify.sh contracts|lifecycle' >&2; exit 2;; + *) echo 'usage: sh test/raw/verify.sh contracts|lifecycle|records' >&2; exit 2;; esac From c527cba37cf164fd8a66a8d228e675db153cb54d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 11 Oct 2026 22:47:53 +0000 Subject: [PATCH 06/14] feat(raw): add explicit PostgreSQL search modes Keep caller-supplied vectors independent of embeddings and legacy fallback. Fence exact ranking from ANN and fuse bounded hybrid arms in one snapshot. Expose scores, component ranks, and fusion parameters without vectors. Preserve unsupported errors for immutable disabled lexical capability. Return unavailable errors for missing runtime indexes instead of fallback. Verify generated plans, filtered recall, candidate bounds, and cancellation. Validation: sh test/raw/verify.sh search. --- rag/internal/pgutil/search.go | 21 ++ rag/raw/postgres/explain_test.go | 128 ++++++++++++ rag/raw/postgres/search.go | 160 +++++++++++++++ rag/raw/postgres/search_snapshot_test.go | 87 ++++++++ rag/raw/postgres/search_test.go | 242 +++++++++++++++++++++++ test/raw/verify.sh | 12 +- 6 files changed, 649 insertions(+), 1 deletion(-) create mode 100644 rag/internal/pgutil/search.go create mode 100644 rag/raw/postgres/explain_test.go create mode 100644 rag/raw/postgres/search.go create mode 100644 rag/raw/postgres/search_snapshot_test.go create mode 100644 rag/raw/postgres/search_test.go diff --git a/rag/internal/pgutil/search.go b/rag/internal/pgutil/search.go new file mode 100644 index 0000000..64e75b8 --- /dev/null +++ b/rag/internal/pgutil/search.go @@ -0,0 +1,21 @@ +package pgutil + +import ( + "context" + "strings" +) + +// BM25Distance binds a query to its index's declared text configuration. SQL +// fragments are trusted identifiers/parameters, never caller text. Parentheses +// matter: PostgreSQL otherwise binds unary minus before the <@> operator. +func BM25Distance(column, query, index string) string { + return column + " <@> to_bm25query(" + query + ", '" + strings.ReplaceAll(index, "'", "''") + "')" +} + +// IndexStrategy reports a configured valid, ready, non-partial index, NOT the +// plan executed by PostgreSQL. Callers hold a relation lock while using it. +func IndexStrategy(ctx context.Context, db DB, table, index, column string) (string, error) { + var strategy string + err := db.QueryRow(ctx, `SELECT am.amname FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid JOIN pg_am am ON am.oid=c.relam JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=i.indkey[0] WHERE i.indrelid=$1::regclass AND c.relname=$2 AND a.attname=$3 AND i.indisvalid AND i.indisready AND i.indpred IS NULL AND i.indexprs IS NULL`, table, index, column).Scan(&strategy) + return strategy, err +} diff --git a/rag/raw/postgres/explain_test.go b/rag/raw/postgres/explain_test.go new file mode 100644 index 0000000..a0858d9 --- /dev/null +++ b/rag/raw/postgres/explain_test.go @@ -0,0 +1,128 @@ +package postgres + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strings" + "testing" + + "github.com/mudler/localrecall/rag/internal/pgutil" + "github.com/mudler/localrecall/rag/raw" +) + +// Deterministic synthetic corpus, including many nearer excluded records. +func rankedCorpus(t *testing.T, s *Service, id string) []raw.Record { + t.Helper() + rs := make([]raw.Record, 600) + for i := range rs { + group := "other" + if i%17 == 0 { + group = "keep" + } + text := "bicycle" + rs[i] = raw.Record{ID: fmt.Sprintf("%04d", i), Vector: []float64{math.Cos(float64(i) * 0.017), math.Sin(float64(i) * 0.017), 0.1}, Text: &text, Metadata: map[string]string{"group": group}} + } + for i := 0; i < len(rs); i += 100 { + put(t, s, t.Context(), id, rs[i:i+100]...) + } + return rs +} +func TestRawExplain(t *testing.T) { + s, ctx, id := recordFixture(t) + rankedCorpus(t, s, id) + table, _ := tableName(id) + exec(t, s, "ANALYZE "+table) + tx, e := s.pool.Begin(ctx) + if e != nil { + t.Fatal(e) + } + defer tx.Rollback(ctx) + strategy, e := pgutil.IndexStrategy(ctx, tx, table, "lr_ann_"+id, "vector") + if e != nil || (strategy != "hnsw" && strategy != "diskann") { + t.Fatalf("ANN unavailable: %q %v", strategy, e) + } + // Bias toward indexes to make the exact fence assertion meaningful. + if _, e = tx.Exec(ctx, "SET LOCAL enable_seqscan=off"); e != nil { + t.Fatal(e) + } + for _, exact := range []bool{true, false} { + args := []any{"[1,0,0]", 10} + predicate := metadataPredicate(&raw.Filter{MetadataEq: map[string]string{"group": "keep"}}, &args) + var plan []byte + if e = tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "+vectorQuery(table, predicate, exact), args...).Scan(&plan); e != nil { + t.Fatal(e) + } + if !json.Valid(plan) { + t.Fatal(string(plan)) + } + t.Logf("exact=%v dimensions=3 corpus=600 selective=1/17 plan=%s", exact, plan) + if exact && (strings.Contains(string(plan), "lr_ann_") || !strings.Contains(string(plan), "CTE Scan")) { + t.Fatal("exact ranking must be fenced from ANN", string(plan)) + } + if !exact && !strings.Contains(string(plan), "lr_ann_"+id) { + t.Fatal("fixture must demonstrate generated approximate query has an ANN access path", string(plan)) + } + } +} +func TestRawRecallBruteForce(t *testing.T) { + s, ctx, id := recordFixture(t) + rs := rankedCorpus(t, s, id) + type hit struct { + id string + score float64 + } + want := []hit{} + for _, r := range rs { + if r.Metadata["group"] != "keep" { + continue + } + v, e := raw.NormalizeVector(r.Vector, 3) + if e != nil { + t.Fatal(e) + } + norm := 0.0 + for _, x := range v { + norm += float64(x) * float64(x) + } + want = append(want, hit{r.ID, float64(v[0]) / math.Sqrt(norm)}) + } + sort.Slice(want, func(i, j int) bool { + if want[i].score == want[j].score { + return want[i].id < want[j].id + } + return want[i].score > want[j].score + }) + r := searchRequest("vector") + r.Filter = &raw.Filter{MetadataEq: map[string]string{"group": "keep"}} + exact, e := s.Search(ctx, id, r) + if e != nil || exact.Count != 10 { + t.Fatal(exact, e) + } + for i, h := range exact.Results { + if h.ID != want[i].id || math.Abs(h.Score-want[i].score) > 1e-6 || h.VectorSimilarity == nil || *h.VectorSimilarity != h.Score { + t.Fatalf("rank %d got %+v want %+v", i, h, want[i]) + } + } + r.Accuracy = nil + approx, e := s.Search(ctx, id, r) + if e != nil { + t.Fatal(e) + } + hits := 0 + for _, h := range approx.Results { + if h.Metadata["group"] != "keep" { + t.Fatal("filter escaped", h) + } + for _, w := range want[:10] { + if w.id == h.ID { + hits++ + } + } + } + if approx.Count > 10 || approx.Accuracy != "approximate" { + t.Fatal(approx) + } + t.Logf("synthetic dimensions=3 corpus=600 eligible=%d k=10 returned=%d recall@10=%.2f underfill=%d (planner-dependent, not an ANN quality guarantee)", len(want), approx.Count, float64(hits)/10, 10-approx.Count) +} diff --git a/rag/raw/postgres/search.go b/rag/raw/postgres/search.go new file mode 100644 index 0000000..2707f7e --- /dev/null +++ b/rag/raw/postgres/search.go @@ -0,0 +1,160 @@ +package postgres + +import ( + "context" + "sort" + + "github.com/jackc/pgx/v5" + "github.com/mudler/localrecall/rag/internal/pgutil" + "github.com/mudler/localrecall/rag/raw" +) + +var _ raw.Store = (*Service)(nil) + +func candidateN(k int) int { return min(1000, max(100, 10*k)) } + +// vectorQuery fences exact ranking behind materialization: no ANN access path +// can implement an ORDER BY on this CTE. Approximate keeps the bare operator. +// SQL predicates precede LIMIT, but ANN traversal can still underfill or lose +// filtered recall. Never expand candidates without bounds or retry as exact. +func vectorQuery(table, predicate string, exact bool) string { + if exact { + return `WITH eligible AS MATERIALIZED (SELECT id,text,metadata,vector FROM ` + table + ` WHERE ` + predicate + `) SELECT id,text,metadata,1-(vector <=> $1::vector) FROM eligible ORDER BY vector <=> $1::vector,id COLLATE "C" LIMIT $2` + } + return `SELECT id,text,metadata,1-(vector <=> $1::vector) FROM ` + table + ` WHERE ` + predicate + ` ORDER BY vector <=> $1::vector,id COLLATE "C" LIMIT $2` +} +func lexicalQuery(table, index, predicate string) string { + distance := pgutil.BM25Distance("text", "$1", index) + return `SELECT id,text,metadata,-(` + distance + `) FROM ` + table + ` WHERE ` + predicate + ` AND (` + distance + `) < 0 ORDER BY ` + distance + `,id COLLATE "C" LIMIT $2` +} +func searchRows(ctx context.Context, tx pgx.Tx, q string, args []any) ([]raw.SearchResult, error) { + rows, e := tx.Query(ctx, q, args...) + if e != nil { + return nil, e + } + defer rows.Close() + out := []raw.SearchResult{} + for rows.Next() { + var r raw.SearchResult + if e = rows.Scan(&r.ID, &r.Text, &r.Metadata, &r.Score); e != nil { + return nil, e + } + out = append(out, r) + } + return out, rows.Err() +} + +// Search never accepts an embedder: opaque caller vectors identify their space. +// Both arms and their payloads use one repeatable-read snapshot. Errors, including +// cancellation/commit failures, discard all results; BM25 never falls back. +func (s *Service) Search(ctx context.Context, id string, r raw.SearchRequest) (out raw.SearchResponse, err error) { + err = s.transaction(ctx, pgx.RepeatableRead, func(ctx context.Context, tx pgx.Tx) error { + c, e := lookup(ctx, tx, id) + if e != nil { + return e + } + r, e = raw.ValidateSearch(c, r) + if e != nil { + return e + } + table, _ := tableName(id) + k := *r.TopK + n := k + if r.Mode == "hybrid" { + n = candidateN(k) + } + out = raw.SearchResponse{Mode: r.Mode, Accuracy: "not_applicable", ScoreKind: "bm25", Results: []raw.SearchResult{}} + var vs, ls []raw.SearchResult + if r.Mode != "lexical" { + out.Accuracy = *r.Accuracy + out.ScoreKind = "cosine_similarity" + if *r.Accuracy == "approximate" { + strategy, e := pgutil.IndexStrategy(ctx, tx, table, "lr_ann_"+id, "vector") + if e != nil { + return e + } + if strategy != "hnsw" && strategy != "diskann" { + return raw.NewError(raw.CodeUnavailable) + } + } + v := make([]float32, len(r.Vector)) + for i, x := range r.Vector { + v[i] = float32(x) + } + args := []any{pgutil.Vector(v), n} + predicate := metadataPredicate(r.Filter, &args) + vs, e = searchRows(ctx, tx, vectorQuery(table, predicate, *r.Accuracy == "exact"), args) + if e != nil { + return e + } + for i := range vs { + rank, sim := i+1, vs[i].Score + vs[i].VectorRank = &rank + vs[i].VectorSimilarity = &sim + } + } + if r.Mode != "vector" { + strategy, e := pgutil.IndexStrategy(ctx, tx, table, "lr_bm25_"+id, "text") + if e != nil { + return e + } + if strategy != "bm25" { + return raw.NewError(raw.CodeUnavailable) + } + args := []any{*r.Text, n} + predicate := metadataPredicate(r.Filter, &args) + // pg_textsearch resolves the literal index name, not a regclass literal. + ls, e = searchRows(ctx, tx, lexicalQuery(table, "lr_bm25_"+id, predicate), args) + if e != nil { + return e + } + for i := range ls { + rank := i + 1 + ls[i].LexicalRank = &rank + } + } + switch r.Mode { + case "vector": + out.Results = vs + case "lexical": + out.Results = ls + case "hybrid": + out.ScoreKind = "rrf" + out.Fusion = &raw.FusionParameters{K: 60, VectorWeight: 0.5, LexicalWeight: 0.5, CandidatesPerArm: n} + merged := map[string]raw.SearchResult{} + for _, v := range vs { + v.Score = 0.5 / float64(60+*v.VectorRank) + merged[v.ID] = v + } + for _, l := range ls { + v, ok := merged[l.ID] + if !ok { + v = l + v.Score = 0 + } + v.LexicalRank = l.LexicalRank + v.Score += 0.5 / float64(60+*l.LexicalRank) + merged[l.ID] = v + } + for _, v := range merged { + out.Results = append(out.Results, v) + } + sort.Slice(out.Results, func(i, j int) bool { + a, b := out.Results[i], out.Results[j] + if a.Score == b.Score { + return a.ID < b.ID + } + return a.Score > b.Score + }) + if len(out.Results) > k { + out.Results = out.Results[:k] + } + } + out.Count = len(out.Results) + return ctx.Err() + }) + if err != nil { + out = raw.SearchResponse{} + } + return +} diff --git a/rag/raw/postgres/search_snapshot_test.go b/rag/raw/postgres/search_snapshot_test.go new file mode 100644 index 0000000..8839467 --- /dev/null +++ b/rag/raw/postgres/search_snapshot_test.go @@ -0,0 +1,87 @@ +package postgres + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/mudler/localrecall/rag/raw" +) + +// Query tracing supplies a deterministic arm boundary without production hooks. +type armBarrier struct { + entered chan struct{} + release chan struct{} +} + +func (b *armBarrier) TraceQueryStart(ctx context.Context, _ *pgx.Conn, d pgx.TraceQueryStartData) context.Context { + if strings.HasPrefix(d.SQL, "SELECT id,text,metadata,-(") { + close(b.entered) + select { + case <-b.release: + case <-ctx.Done(): + } + } + return ctx +} +func (*armBarrier) TraceQueryEnd(context.Context, *pgx.Conn, pgx.TraceQueryEndData) {} +func TestRawSearchSnapshotAndCancellation(t *testing.T) { + for _, cancelArm := range []bool{false, true} { + t.Run(map[bool]string{false: "snapshot", true: "cancel_after_vector"}[cancelArm], func(t *testing.T) { + s, ctx, id := recordFixture(t) + text := "bicycle" + put(t, s, ctx, id, raw.Record{ID: "a", Vector: []float64{1, 0, 0}, Text: &text}) + writer, e := New(ctx, Config{s.pool.Config().ConnString()}) + if e != nil { + t.Fatal(e) + } + defer writer.Close() + barrier := &armBarrier{make(chan struct{}), make(chan struct{})} + cfg := s.pool.Config() + cfg.ConnConfig.Tracer = barrier + s.pool.Close() + s.pool, e = pgxpool.NewWithConfig(ctx, cfg) + if e != nil { + t.Fatal(e) + } + requestCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + type answer struct { + out raw.SearchResponse + err error + } + done := make(chan answer, 1) + go func() { out, e := s.Search(requestCtx, id, searchRequest("hybrid")); done <- answer{out, e} }() + select { + case <-barrier.entered: + case <-requestCtx.Done(): + t.Fatal("lexical arm not reached") + } + if cancelArm { + cancel() + } else { + other := "ocean" + put(t, writer, ctx, id, raw.Record{ID: "a", Vector: []float64{0, 1, 0}, Text: &other}) + } + close(barrier.release) + got := <-done + if cancelArm { + checkCode(t, got.err, raw.CodeTimeout) + if got.out.Results != nil || got.out.Count != 0 { + t.Fatal("partial response", got) + } + return + } + if got.err != nil || got.out.Count != 1 { + t.Fatal(got) + } + hit := got.out.Results[0] + if hit.Text == nil || *hit.Text != "bicycle" || hit.LexicalRank == nil || hit.VectorRank == nil || hit.Score != 1.0/61 { + t.Fatalf("mixed snapshot: %+v", hit) + } + }) + } +} diff --git a/rag/raw/postgres/search_test.go b/rag/raw/postgres/search_test.go new file mode 100644 index 0000000..41c2041 --- /dev/null +++ b/rag/raw/postgres/search_test.go @@ -0,0 +1,242 @@ +package postgres + +import ( + "context" + "fmt" + "github.com/mudler/localrecall/rag/raw" + "math" + "testing" +) + +func searchRequest(mode string) raw.SearchRequest { + space, text, exact := "space", "bicycle", "exact" + r := raw.SearchRequest{Mode: mode} + if mode != "lexical" { + r.Vector = []float64{1, 0, 0} + r.EmbeddingSpaceID = &space + r.Accuracy = &exact + } + if mode != "vector" { + r.Text = &text + } + return r +} +func TestRawSearchModes(t *testing.T) { + s, ctx, id := recordFixture(t) + text := "bicycle bicycle" + other := "ocean" + put(t, s, ctx, id, raw.Record{ID: "b", Vector: []float64{1, 0, 0}, Text: &text, Metadata: map[string]string{"keep": "yes"}}, raw.Record{ID: "a", Vector: []float64{1, 0, 0}, Text: &text, Metadata: map[string]string{"keep": "yes"}}, raw.Record{ID: "z", Vector: []float64{0, 1, 0}, Text: &other}) + for _, mode := range []string{"vector", "lexical", "hybrid"} { + r := searchRequest(mode) + r.Filter = &raw.Filter{MetadataEq: map[string]string{"keep": "yes"}} + got, e := s.Search(ctx, id, r) + if e != nil || got.Count != 2 || got.Results[0].ID != "a" { + t.Fatalf("%s: %+v %v", mode, got, e) + } + if got.Mode != mode || got.Count != len(got.Results) { + t.Fatal(got) + } + wantAccuracy, wantKind := "exact", "cosine_similarity" + if mode == "lexical" { + wantAccuracy, wantKind = "not_applicable", "bm25" + } + if mode == "hybrid" { + wantKind = "rrf" + } + if got.Accuracy != wantAccuracy || got.ScoreKind != wantKind { + t.Fatal(got) + } + if mode == "hybrid" { + if got.Fusion == nil || got.Fusion.K != 60 || got.Fusion.VectorWeight != 0.5 || got.Fusion.LexicalWeight != 0.5 { + t.Fatal(got) + } + if got.Results[0].VectorRank == nil || *got.Results[0].VectorRank != 1 || got.Results[0].LexicalRank == nil || *got.Results[0].LexicalRank != 1 { + t.Fatal(got) + } + if got.Fusion.CandidatesPerArm != 100 || math.Abs(got.Results[0].Score-1.0/61) > 1e-12 { + t.Fatal(got) + } + } + if mode == "lexical" && got.Results[0].Score <= 0 { + t.Fatal("BM25 must be higher-is-better", got) + } + } + r := searchRequest("lexical") + empty := "nomatchingterm" + r.Text = &empty + got, e := s.Search(ctx, id, r) + if e != nil || got.Count != 0 { + t.Fatal(got, e) + } + r = searchRequest("vector") + wrong := "wrong" + r.EmbeddingSpaceID = &wrong + _, e = s.Search(ctx, id, r) + checkCode(t, e, raw.CodeSpaceMismatch) + r = searchRequest("vector") + r.Vector = []float64{1, 0} + _, e = s.Search(ctx, id, r) + checkCode(t, e, raw.CodeValidation) + canceled, cancel := context.WithCancel(ctx) + cancel() + got, e = s.Search(canceled, id, searchRequest("hybrid")) + checkCode(t, e, raw.CodeTimeout) + if got.Results != nil { + t.Fatal("partial response") + } +} +func TestRawSearchMissingIndexes(t *testing.T) { + s, ctx, id := recordFixture(t) + put(t, s, ctx, id, rec("a")) + exec(t, s, `DROP INDEX "lr_ann_`+id+`"`) + r := searchRequest("vector") + r.Accuracy = nil + _, e := s.Search(ctx, id, r) + checkCode(t, e, raw.CodeUnavailable) + if _, e = s.Search(ctx, id, searchRequest("vector")); e != nil { + t.Fatal(e) + } + exec(t, s, `DROP INDEX "lr_bm25_`+id+`"`) + _, e = s.Search(ctx, id, searchRequest("lexical")) + checkCode(t, e, raw.CodeUnavailable) +} + +// Regression: pg_textsearch takes an index *name*, not SQL-quoted regclass +// syntax. UUID hyphens make this distinction observable on every collection. +func TestRawSearchBM25LiteralIndex(t *testing.T) { + s, ctx, id := recordFixture(t) + text := "bicycle" + put(t, s, ctx, id, raw.Record{ID: "a", Vector: []float64{1, 0, 0}, Text: &text}) + table, _ := tableName(id) + var distance float64 + if e := s.pool.QueryRow(ctx, `SELECT text <@> to_bm25query($1,$2) FROM `+table, text, "lr_bm25_"+id).Scan(&distance); e != nil { + t.Fatal(e) + } + got, e := s.Search(ctx, id, searchRequest("lexical")) + if e != nil || got.Count != 1 || got.Results[0].Score != -distance || distance >= 0 { + t.Fatalf("literal index distance=%g response=%+v error=%v", distance, got, e) + } +} +func TestRawSearchCandidateBounds(t *testing.T) { + for _, c := range []struct{ k, n int }{{1, 100}, {10, 100}, {11, 110}, {100, 1000}} { + if candidateN(c.k) != c.n { + t.Fatal(c) + } + } + s, ctx, id := recordFixture(t) + // Globally higher-ranked excluded rows exceed either candidate limit. + for batch := 0; batch < 3; batch++ { + rs := []raw.Record{} + for i := 0; i < 100; i++ { + text := "bicycle" + rs = append(rs, raw.Record{ID: fmt.Sprintf("a%03d", batch*100+i), Vector: []float64{1, 0, 0}, Text: &text}) + } + put(t, s, ctx, id, rs...) + } + text := "bicycle" + put(t, s, ctx, id, raw.Record{ID: "z", Vector: []float64{0, 1, 0}, Text: &text, Metadata: map[string]string{"keep": "yes"}}) + for _, mode := range []string{"vector", "lexical", "hybrid"} { + r := searchRequest(mode) + k := 1 + r.TopK = &k + r.Filter = &raw.Filter{MetadataEq: map[string]string{"keep": "yes"}} + got, e := s.Search(ctx, id, r) + if e != nil || got.Count != 1 || got.Results[0].ID != "z" { + t.Fatal(mode, got, e) + } + if mode == "hybrid" && (got.Results[0].VectorRank == nil || got.Results[0].LexicalRank == nil || got.Results[0].Score != 1.0/61) { + t.Fatal(got) + } + } + // Without a filter, z lies outside BOTH bounded arms, even exact hybrid. + r := searchRequest("hybrid") + k := 10 + r.TopK = &k + got, e := s.Search(ctx, id, r) + if e != nil { + t.Fatal(e) + } + for _, h := range got.Results { + if h.ID == "z" { + t.Fatal("unexpected boundary result") + } + } +} +func TestRawSearchDisabledLexical(t *testing.T) { + s, ctx := fixture(t) + r := request() + disabled := false + r.Lexical = &raw.LexicalRequest{Enabled: &disabled} + c, _, e := s.CreateCollection(ctx, r) + if e != nil { + t.Fatal(e) + } + defer s.DeleteCollection(ctx, c.CollectionID) + for _, mode := range []string{"lexical", "hybrid"} { + got, e := s.Search(ctx, c.CollectionID, searchRequest(mode)) + // Immutable configuration does not implement this collection capability (501). + // Unlike a configured capability with a missing runtime index (503), + // retrying cannot restore it. Preserve the Task1 validator contract. + checkCode(t, e, raw.CodeUnsupported) + if got.Results != nil { + t.Fatal(got) + } + } + if _, e = s.Search(ctx, c.CollectionID, searchRequest("vector")); e != nil { + t.Fatal(e) + } +} + +// A lexical-only hit beyond the bounded vector arm must not acquire a global +// vector rank. Approximate candidates may differ, but remain bounded and tied +// by ID within each returned arm; no exact membership is promised for ANN. +func TestRawSearchCandidateUnion(t *testing.T) { + s, ctx, id := recordFixture(t) + for batch := 0; batch < 2; batch++ { + records := []raw.Record{} + for i := batch * 100; i < min(110, (batch+1)*100); i++ { + text := "ocean" + if i >= 99 { + text = "bicycle" + } + records = append(records, raw.Record{ID: fmt.Sprintf("a%03d", i), Vector: []float64{1, 0, 0}, Text: &text}) + } + put(t, s, ctx, id, records...) + } + for _, accuracy := range []string{"exact", "approximate"} { + r := searchRequest("hybrid") + r.Accuracy = &accuracy + got, e := s.Search(ctx, id, r) + if e != nil || got.Count != 10 || got.Fusion.CandidatesPerArm != 100 { + t.Fatal(got, e) + } + foundBoundary := false + for i, h := range got.Results { + score := 0.0 + if h.VectorRank != nil { + if *h.VectorRank < 1 || *h.VectorRank > 100 { + t.Fatal(h) + } + score += 0.5 / float64(60+*h.VectorRank) + } + if h.LexicalRank != nil { + score += 0.5 / float64(60+*h.LexicalRank) + } + if math.Abs(score-h.Score) > 1e-12 { + t.Fatal(h) + } + if i > 0 && (got.Results[i-1].Score < h.Score || (got.Results[i-1].Score == h.Score && got.Results[i-1].ID > h.ID)) { + t.Fatal("unstable fusion order", got) + } + if accuracy == "exact" && h.ID == "a100" { + foundBoundary = true + if h.VectorRank != nil || h.LexicalRank == nil || *h.LexicalRank != 2 { + t.Fatal("candidate boundary ignored", h) + } + } + } + if accuracy == "exact" && (!foundBoundary || got.Results[0].ID != "a099") { + t.Fatal("expected bounded union", got) + } + } +} diff --git a/test/raw/verify.sh b/test/raw/verify.sh index c436b7e..dd9d00e 100644 --- a/test/raw/verify.sh +++ b/test/raw/verify.sh @@ -13,6 +13,16 @@ export COLLECTION_DB_PATH="$ROOT/.sdd-work/collections" FILE_ASSETS="$ROOT/.sdd- export GOENV=off GOTOOLCHAIN=local mkdir -p "$TMPDIR" "$GOCACHE" "$GOMODCACHE" "$GOPATH" "$COLLECTION_DB_PATH" "$FILE_ASSETS" "$ROOT/.sdd-work/logs" case "${1:-}" in + search) + sh test/raw/verify.sh records + log="$ROOT/.sdd-work/logs/search-$(date +%s)-$$.log" + if sh test/raw/database.sh go test -v -mod=readonly -race -count=1 -timeout=10m ./rag/raw/postgres/... -run "^(TestRaw(Search|Explain|Recall))" >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi + for suite in Search Explain Recall; do + grep -q -- "--- PASS: TestRaw${suite}" "$log" + done + ! grep -E "SKIP|no tests to run|no test files" "$log" + git diff --check + ;; records) sh test/raw/verify.sh lifecycle log="$ROOT/.sdd-work/logs/records-$(date +%s)-$$.log" @@ -40,5 +50,5 @@ case "${1:-}" in [ "$(grep -c '^ok[[:space:]]' "$log")" -eq 2 ] && ! grep -E '\[no test files\]|\[no tests to run\]' "$log" git diff --check ;; - *) echo 'usage: sh test/raw/verify.sh contracts|lifecycle|records' >&2; exit 2;; + *) echo 'usage: sh test/raw/verify.sh contracts|lifecycle|records|search' >&2; exit 2;; esac From 2f65bd20a62c0a3512022c028e0344462c13ceae Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 11 Oct 2026 22:54:48 +0000 Subject: [PATCH 07/14] fix(raw): require cosine ordering in ANN indexes An L2 ANN index cannot serve cosine ordering. Reject approximate search\ninstead of silently accepting an incompatible index. Check the selected opclass family for pgvector cosine ordering support.\nBind type and operator identity to pgvector and query search-path visibility.\nKeep BM25 checks unchanged. Preserve the regression for L2 rejection, exact search, and cosine success.\nVerify with sh test/raw/verify.sh search. --- rag/internal/pgutil/search.go | 36 ++++++++++++++++++++++++++++++++- rag/raw/postgres/search_test.go | 31 ++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/rag/internal/pgutil/search.go b/rag/internal/pgutil/search.go index 64e75b8..9a90196 100644 --- a/rag/internal/pgutil/search.go +++ b/rag/internal/pgutil/search.go @@ -14,8 +14,42 @@ func BM25Distance(column, query, index string) string { // IndexStrategy reports a configured valid, ready, non-partial index, NOT the // plan executed by PostgreSQL. Callers hold a relation lock while using it. +// ANN indexes must support the query's pgvector cosine ordering, not merely +// have an ANN access method. Extension membership binds type/operator identity; +// visibility ensures the unqualified query resolves to those same objects. +// BM25 retains its existing method-only check. func IndexStrategy(ctx context.Context, db DB, table, index, column string) (string, error) { var strategy string - err := db.QueryRow(ctx, `SELECT am.amname FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid JOIN pg_am am ON am.oid=c.relam JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=i.indkey[0] WHERE i.indrelid=$1::regclass AND c.relname=$2 AND a.attname=$3 AND i.indisvalid AND i.indisready AND i.indpred IS NULL AND i.indexprs IS NULL`, table, index, column).Scan(&strategy) + err := db.QueryRow(ctx, ` +SELECT am.amname +FROM pg_catalog.pg_index i +JOIN pg_catalog.pg_class c ON c.oid=i.indexrelid +JOIN pg_catalog.pg_am am ON am.oid=c.relam +JOIN pg_catalog.pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=i.indkey[0] +WHERE i.indrelid=$1::regclass AND c.relname=$2 AND a.attname=$3 + AND i.indisvalid AND i.indisready AND i.indpred IS NULL AND i.indexprs IS NULL + AND (am.amname NOT IN ('hnsw', 'diskann') OR EXISTS ( + SELECT 1 + FROM pg_catalog.pg_opclass oc + JOIN pg_catalog.pg_amop ao ON ao.amopfamily=oc.opcfamily AND ao.amopmethod=am.oid + JOIN pg_catalog.pg_operator op ON op.oid=ao.amopopr + JOIN pg_catalog.pg_type vt ON vt.oid=a.atttypid + JOIN pg_catalog.pg_extension ext ON ext.extname='vector' + JOIN pg_catalog.pg_depend td ON td.classid='pg_catalog.pg_type'::regclass + AND td.objid=vt.oid AND td.refclassid='pg_catalog.pg_extension'::regclass + AND td.refobjid=ext.oid AND td.deptype='e' + JOIN pg_catalog.pg_depend od ON od.classid='pg_catalog.pg_operator'::regclass + AND od.objid=op.oid AND od.refclassid='pg_catalog.pg_extension'::regclass + AND od.refobjid=ext.oid AND od.deptype='e' + JOIN pg_catalog.pg_opfamily sf ON sf.oid=ao.amopsortfamily + JOIN pg_catalog.pg_am sam ON sam.oid=sf.opfmethod + WHERE oc.oid=i.indclass[0] AND oc.opcmethod=am.oid AND oc.opcintype=vt.oid + AND vt.typname='vector' AND pg_catalog.pg_type_is_visible(vt.oid) + AND op.oprname='<=>' AND op.oprleft=vt.oid AND op.oprright=vt.oid + AND pg_catalog.pg_operator_is_visible(op.oid) + AND ao.amoplefttype=vt.oid AND ao.amoprighttype=vt.oid AND ao.amoppurpose='o' + AND sf.opfnamespace='pg_catalog'::regnamespace AND sf.opfname='float_ops' + AND sam.amname='btree' + ))`, table, index, column).Scan(&strategy) return strategy, err } diff --git a/rag/raw/postgres/search_test.go b/rag/raw/postgres/search_test.go index 41c2041..e211fd1 100644 --- a/rag/raw/postgres/search_test.go +++ b/rag/raw/postgres/search_test.go @@ -101,6 +101,37 @@ func TestRawSearchMissingIndexes(t *testing.T) { checkCode(t, e, raw.CodeUnavailable) } +// An ANN access method alone is insufficient: L2 cannot order a cosine query. +func TestRawSearchANNOperatorClass(t *testing.T) { + s, ctx, id := recordFixture(t) + text := "bicycle" + put(t, s, ctx, id, raw.Record{ID: "a", Vector: []float64{1, 0, 0}, Text: &text}) + table, _ := tableName(id) + for _, opclass := range []string{"vector_l2_ops", "vector_cosine_ops"} { + exec(t, s, `DROP INDEX "lr_ann_`+id+`"`) + exec(t, s, `CREATE INDEX "lr_ann_`+id+`" ON `+table+` USING hnsw (vector `+opclass+`)`) + for _, mode := range []string{"vector", "hybrid"} { + for _, accuracy := range []string{"approximate", "exact"} { + t.Run(opclass+"/"+mode+"/"+accuracy, func(t *testing.T) { + r := searchRequest(mode) + r.Accuracy = &accuracy + got, e := s.Search(ctx, id, r) + if opclass == "vector_l2_ops" && accuracy == "approximate" { + checkCode(t, e, raw.CodeUnavailable) // HTTP 503, not silent exact fallback. + if got.Results != nil { + t.Fatal("partial response", got) + } + return + } + if e != nil || got.Count != 1 || got.Results[0].ID != "a" || got.Accuracy != accuracy { + t.Fatalf("response=%+v error=%v", got, e) + } + }) + } + } + } +} + // Regression: pg_textsearch takes an index *name*, not SQL-quoted regclass // syntax. UUID hyphens make this distinction observable on every collection. func TestRawSearchBM25LiteralIndex(t *testing.T) { From 30fd1d72fd9dff9485204d114320f16dfe72b085 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 11 Oct 2026 23:01:47 +0000 Subject: [PATCH 08/14] feat(raw): expose authenticated API and client Expose caller-supplied vectors without entering legacy embedding paths. Own the raw pool independently and drain or cancel requests on shutdown. Verify HTTP and client contracts against the owned PostgreSQL fixture. --- main.go | 46 ++++++- pkg/client/raw.go | 177 +++++++++++++++++++++++++ pkg/client/raw_test.go | 68 ++++++++++ raw_routes.go | 232 ++++++++++++++++++++++++++++++++ raw_routes_test.go | 291 +++++++++++++++++++++++++++++++++++++++++ test/raw/verify.sh | 14 +- 6 files changed, 826 insertions(+), 2 deletions(-) create mode 100644 pkg/client/raw.go create mode 100644 pkg/client/raw_test.go create mode 100644 raw_routes.go create mode 100644 raw_routes_test.go diff --git a/main.go b/main.go index 1bc4b91..d1475ef 100644 --- a/main.go +++ b/main.go @@ -1,9 +1,18 @@ package main import ( + "context" + "errors" + "github.com/mudler/localrecall/rag/raw" + rawpostgres "github.com/mudler/localrecall/rag/raw/postgres" + "net" + "net/http" "os" + "os/signal" "strconv" "strings" + "syscall" + "time" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" @@ -89,7 +98,42 @@ func startAPI(listenAddress string) { registerAPIRoutes(e, openAIClient, chunkingSize, overlap, keys) - e.Logger.Fatal(e.Start(listenAddress)) + // The raw pool belongs to this server, independently of legacy collections. + var store raw.Store + if vectorEngine == "postgres" { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + service, err := rawpostgres.New(ctx, rawpostgres.Config{DatabaseURL: os.Getenv("DATABASE_URL")}) + cancel() + if err != nil { + e.Logger.Error("Raw PostgreSQL unavailable") + } else { + store = service + } + } + registerRawAPIRoutes(e, store) + requests, cancelRequests := context.WithCancel(context.Background()) + e.Server.BaseContext = func(_ net.Listener) context.Context { return requests } + stop, cancelSignal := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancelSignal() + exited := make(chan error, 1) + go func() { exited <- e.Start(listenAddress) }() + select { + case err := <-exited: + if err != nil && !errors.Is(err, http.ErrServerClosed) { + e.Logger.Error("HTTP server stopped") + } + case <-stop.Done(): + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + if err := e.Shutdown(ctx); err != nil { + cancelRequests() + _ = e.Close() + } + cancel() + } + cancelRequests() + if store != nil { + store.Close() + } } func main() { diff --git a/pkg/client/raw.go b/pkg/client/raw.go new file mode 100644 index 0000000..bfcfae1 --- /dev/null +++ b/pkg/client/raw.go @@ -0,0 +1,177 @@ +// Package client provides additive context-aware access to raw vectors. +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/google/uuid" + "github.com/mudler/localrecall/rag/raw" + "io" + "net/http" + "net/url" + "strconv" + "strings" +) + +type RawClient struct { + baseURL, apiKey string + httpClient *http.Client +} + +func NewRawClient(baseURL, apiKey string, httpClient *http.Client) *RawClient { + if httpClient == nil { + httpClient = http.DefaultClient + } + return &RawClient{baseURL, apiKey, httpClient} +} +func (c *RawClient) do(ctx context.Context, method, path string, q url.Values, body, out any) (int, error) { + u, err := url.Parse(c.baseURL) + if err != nil { + return 0, err + } + if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return 0, fmt.Errorf("invalid raw API base URL") + } + u.Path = strings.TrimRight(u.Path, "/") + "/api/raw/v1" + path + u.RawPath = "" + u.RawQuery = q.Encode() + var reader io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return 0, err + } + reader = bytes.NewReader(b) + } + req, err := http.NewRequestWithContext(ctx, method, u.String(), reader) + if err != nil { + return 0, err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if c.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+c.apiKey) + } + resp, err := c.httpClient.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + var envelope struct { + Success bool `json:"success"` + Data json.RawMessage `json:"data"` + Error *raw.Error `json:"error"` + } + b, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20)) + if err != nil { + return resp.StatusCode, err + } + if err = json.Unmarshal(b, &envelope); err != nil { + return resp.StatusCode, fmt.Errorf("invalid raw API response (HTTP %d)", resp.StatusCode) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 || !envelope.Success { + if envelope.Error != nil { + e := raw.NewError(envelope.Error.Code) + e.Status = resp.StatusCode + return resp.StatusCode, e + } + return resp.StatusCode, fmt.Errorf("raw API request failed (HTTP %d)", resp.StatusCode) + } + if out != nil { + err = json.Unmarshal(envelope.Data, out) + } + return resp.StatusCode, err +} +func rawCollectionPath(cid string) (string, error) { + id, err := uuid.Parse(cid) + if err != nil || id.String() != cid { + return "", raw.NewError(raw.CodeValidation) + } + return "/collections/" + cid, nil +} +func (c *RawClient) Capabilities(ctx context.Context) (out raw.Capabilities, err error) { + _, err = c.do(ctx, "GET", "/capabilities", nil, nil, &out) + return +} +func (c *RawClient) CreateCollection(ctx context.Context, req raw.CreateCollectionRequest) (out raw.Collection, created bool, err error) { + status, err := c.do(ctx, "POST", "/collections", nil, req, &out) + return out, status == 201, err +} +func (c *RawClient) ListCollections(ctx context.Context, req raw.ListCollectionsRequest) (out raw.CollectionPage, err error) { + q := url.Values{} + if req.Limit != nil { + q.Set("limit", strconv.Itoa(*req.Limit)) + } + if req.Cursor != nil { + q.Set("cursor", *req.Cursor) + } + _, err = c.do(ctx, "GET", "/collections", q, nil, &out) + return +} +func (c *RawClient) GetCollection(ctx context.Context, cid string) (out raw.Collection, err error) { + path, err := rawCollectionPath(cid) + if err != nil { + return + } + _, err = c.do(ctx, "GET", path, nil, nil, &out) + return +} +func (c *RawClient) Upsert(ctx context.Context, cid string, req raw.UpsertRequest) (out raw.UpsertResponse, err error) { + path, err := rawCollectionPath(cid) + if err != nil { + return + } + _, err = c.do(ctx, "POST", path+"/records/upsert", nil, req, &out) + return +} +func (c *RawClient) GetRecords(ctx context.Context, cid string, req raw.GetRecordsRequest) (out raw.GetRecordsResponse, err error) { + path, err := rawCollectionPath(cid) + if err != nil { + return + } + _, err = c.do(ctx, "POST", path+"/records/get", nil, req, &out) + return +} +func (c *RawClient) ListRecords(ctx context.Context, cid string, req raw.ListRecordsRequest) (out raw.RecordPage, err error) { + path, err := rawCollectionPath(cid) + if err != nil { + return + } + _, err = c.do(ctx, "POST", path+"/records/list", nil, req, &out) + return +} +func (c *RawClient) DeleteRecords(ctx context.Context, cid string, req raw.DeleteRecordsRequest) (out raw.DeleteRecordsResponse, err error) { + path, err := rawCollectionPath(cid) + if err != nil { + return + } + _, err = c.do(ctx, "POST", path+"/records/delete", nil, req, &out) + return +} +func (c *RawClient) Search(ctx context.Context, cid string, req raw.SearchRequest) (out raw.SearchResponse, err error) { + path, err := rawCollectionPath(cid) + if err != nil { + return + } + _, err = c.do(ctx, "POST", path+"/search", nil, req, &out) + return +} +func (c *RawClient) Reset(ctx context.Context, cid string) error { + path, err := rawCollectionPath(cid) + if err != nil { + return err + } + _, err = c.do(ctx, "POST", path+"/reset", nil, raw.ResetRequest{}, nil) + return err +} +func (c *RawClient) DeleteCollection(ctx context.Context, cid string) error { + path, err := rawCollectionPath(cid) + if err != nil { + return err + } + _, err = c.do(ctx, "DELETE", path, nil, nil, nil) + return err +} diff --git a/pkg/client/raw_test.go b/pkg/client/raw_test.go new file mode 100644 index 0000000..f751164 --- /dev/null +++ b/pkg/client/raw_test.go @@ -0,0 +1,68 @@ +package client + +import ( + "context" + "errors" + "github.com/mudler/localrecall/rag/raw" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestRawClient(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer key" { + t.Error("auth") + } + if r.URL.Path != "/api/raw/v1/collections" { + t.Error(r.URL) + } + w.WriteHeader(201) + w.Write([]byte(`{"success":true,"data":{"collection_id":"id"}}`)) + })) + defer srv.Close() + c := NewRawClient(srv.URL, "key", srv.Client()) + d, created, err := c.CreateCollection(context.Background(), raw.CreateCollectionRequest{}) + if err != nil || !created || d.CollectionID != "id" { + t.Fatal(d, created, err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, _, err = c.CreateCollection(ctx, raw.CreateCollectionRequest{}) + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} + +func TestRawClientSafety(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.WriteHeader(503) + w.Write([]byte(`{"success":false,"error":{"code":"UNAVAILABLE","message":"secret SQL"}}`)) + })) + defer srv.Close() + c := NewRawClient(srv.URL, "", srv.Client()) + for _, cid := range []string{"../capabilities", "foo/bar", "?x=1", "%2f", "."} { + if _, err := c.GetCollection(context.Background(), cid); err == nil { + t.Fatal(cid) + } + } + if calls != 0 { + t.Fatal(calls) + } + _, err := c.Capabilities(context.Background()) + var re *raw.Error + if !errors.As(err, &re) || re.Status != 503 || !re.Retryable || strings.Contains(err.Error(), "secret") { + t.Fatal(err) + } + for _, base := range []string{srv.URL + "?bad=1", srv.URL + "#fragment", "file:///tmp/foo", "http://user:password@localhost"} { + if _, err := NewRawClient(base, "key", srv.Client()).Capabilities(context.Background()); err == nil { + t.Fatal(base) + } + } + if calls != 1 { + t.Fatal(calls) + } +} diff --git a/raw_routes.go b/raw_routes.go new file mode 100644 index 0000000..2d2791c --- /dev/null +++ b/raw_routes.go @@ -0,0 +1,232 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "github.com/labstack/echo/v4" + "github.com/mudler/localrecall/rag/raw" + "io" + "net/http" + "strconv" + "strings" + "time" +) + +func rawFailure(c echo.Context, err error) error { + e := raw.NewError(raw.CodeInternal) + var re *raw.Error + if errors.As(err, &re) { + e = raw.NewError(re.Code) + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + e = raw.NewError(raw.CodeTimeout) + } + return c.JSON(e.Status, struct { + Success bool `json:"success"` + Error *raw.Error `json:"error"` + }{false, e}) +} +func rawBody(c echo.Context, dst any) error { + b, err := io.ReadAll(http.MaxBytesReader(c.Response(), c.Request().Body, raw.MaxBodyBytes)) + if c.Request().Context().Err() != nil { + return c.Request().Context().Err() + } + if err != nil { + var limit *http.MaxBytesError + if errors.As(err, &limit) { + return raw.NewError(raw.CodeBodyLimit) + } + return raw.NewError(raw.CodeValidation) + } + if err := raw.DecodeRequest(b, dst); err != nil { + return err + } + // NUL is invalid in every PostgreSQL-bound string, including map keys. + decoder := json.NewDecoder(bytes.NewReader(b)) + for { + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return raw.NewError(raw.CodeValidation) + } + if text, ok := token.(string); ok && strings.ContainsRune(text, 0) { + return raw.NewError(raw.CodeValidation) + } + } + return nil +} + +// registerRawAPIRoutes inherits the instance middleware installed by registerAPIRoutes. +// A nil store denotes a failed bootstrap, never a fallback to legacy persistence. +func registerRawAPIRoutes(e *echo.Echo, store raw.Store) { + g := e.Group("/api/raw/v1") + g.Use(func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + ctx, cancel := context.WithTimeout(c.Request().Context(), 30*time.Second) + defer cancel() + c.SetRequest(c.Request().WithContext(ctx)) + if store == nil && c.Path() != "/api/raw/v1/capabilities" { + if vectorEngine != "postgres" { + return rawFailure(c, raw.NewError(raw.CodeUnsupported)) + } + return rawFailure(c, raw.NewError(raw.CodeUnavailable)) + } + return next(c) + } + }) + g.GET("/capabilities", func(c echo.Context) error { + if store == nil { + caps := raw.Capabilities{Backend: vectorEngine, RawCollections: vectorEngine == "postgres", Readiness: raw.Readiness{Reason: "Required dependency unavailable"}, Limits: raw.ContractLimits()} + if caps.RawCollections { + caps.Modes = []string{"vector", "lexical", "hybrid"} + caps.AccuracyOptions = []string{"approximate", "exact"} + caps.DefaultAccuracy = "approximate" + caps.Metrics = []string{"cosine"} + caps.AtomicBatch = true + caps.Persistent = true + } + return c.JSON(200, successResponse("", caps)) + } + data, err := store.Capabilities(c.Request().Context()) + if err != nil { + return rawFailure(c, err) + } + return c.JSON(200, successResponse("", data)) + }) + g.POST("/collections", func(c echo.Context) error { + var req raw.CreateCollectionRequest + if err := rawBody(c, &req); err != nil { + return rawFailure(c, err) + } + data, created, err := store.CreateCollection(c.Request().Context(), req) + if err != nil { + return rawFailure(c, err) + } + status := 200 + if created { + status = 201 + } + return c.JSON(status, successResponse("", data)) + }) + g.GET("/collections", func(c echo.Context) error { + var req raw.ListCollectionsRequest + for k, v := range c.QueryParams() { + if len(v) != 1 { + return rawFailure(c, raw.NewError(raw.CodeValidation)) + } + switch k { + case "limit": + n, err := strconv.Atoi(v[0]) + if err != nil { + return rawFailure(c, raw.NewError(raw.CodeValidation)) + } + req.Limit = &n + case "cursor": + req.Cursor = &v[0] + default: + return rawFailure(c, raw.NewError(raw.CodeValidation)) + } + } + data, err := store.ListCollections(c.Request().Context(), req) + if err != nil { + return rawFailure(c, err) + } + return c.JSON(200, successResponse("", data)) + }) + g.GET("/collections/:cid", func(c echo.Context) error { + data, err := store.GetCollection(c.Request().Context(), c.Param("cid")) + if err != nil { + return rawFailure(c, err) + } + return c.JSON(200, successResponse("", data)) + }) + g.POST("/collections/:cid/records/upsert", func(c echo.Context) error { + var req raw.UpsertRequest + if err := rawBody(c, &req); err != nil { + return rawFailure(c, err) + } + data, err := store.Upsert(c.Request().Context(), c.Param("cid"), req) + if err != nil { + return rawFailure(c, err) + } + return c.JSON(200, successResponse("", data)) + }) + g.POST("/collections/:cid/records/get", func(c echo.Context) error { + var req raw.GetRecordsRequest + if err := rawBody(c, &req); err != nil { + return rawFailure(c, err) + } + data, err := store.GetRecords(c.Request().Context(), c.Param("cid"), req) + if err != nil { + return rawFailure(c, err) + } + return c.JSON(200, successResponse("", data)) + }) + g.POST("/collections/:cid/records/list", func(c echo.Context) error { + var req raw.ListRecordsRequest + if err := rawBody(c, &req); err != nil { + return rawFailure(c, err) + } + data, err := store.ListRecords(c.Request().Context(), c.Param("cid"), req) + if err != nil { + return rawFailure(c, err) + } + return c.JSON(200, successResponse("", data)) + }) + g.POST("/collections/:cid/records/delete", func(c echo.Context) error { + var req raw.DeleteRecordsRequest + if err := rawBody(c, &req); err != nil { + return rawFailure(c, err) + } + data, err := store.DeleteRecords(c.Request().Context(), c.Param("cid"), req) + if err != nil { + return rawFailure(c, err) + } + return c.JSON(200, successResponse("", data)) + }) + g.POST("/collections/:cid/search", func(c echo.Context) error { + var req raw.SearchRequest + if err := rawBody(c, &req); err != nil { + return rawFailure(c, err) + } + data, err := store.Search(c.Request().Context(), c.Param("cid"), req) + if err != nil { + return rawFailure(c, err) + } + return c.JSON(200, successResponse("", data)) + }) + g.POST("/collections/:cid/reset", func(c echo.Context) error { + var req raw.ResetRequest + if err := rawBody(c, &req); err != nil { + return rawFailure(c, err) + } + if err := store.Reset(c.Request().Context(), c.Param("cid")); err != nil { + return rawFailure(c, err) + } + return c.JSON(200, successResponse("", raw.ResetResponse{Reset: true})) + }) + g.DELETE("/collections/:cid", func(c echo.Context) error { + b, err := io.ReadAll(http.MaxBytesReader(c.Response(), c.Request().Body, raw.MaxBodyBytes)) + if c.Request().Context().Err() != nil { + return rawFailure(c, c.Request().Context().Err()) + } + if err != nil { + var limit *http.MaxBytesError + if errors.As(err, &limit) { + return rawFailure(c, raw.NewError(raw.CodeBodyLimit)) + } + return rawFailure(c, raw.NewError(raw.CodeValidation)) + } + if len(b) != 0 { + return rawFailure(c, raw.NewError(raw.CodeValidation)) + } + if err := store.DeleteCollection(c.Request().Context(), c.Param("cid")); err != nil { + return rawFailure(c, err) + } + return c.JSON(200, successResponse("", raw.DeleteCollectionResponse{Deleted: true})) + }) +} diff --git a/raw_routes_test.go b/raw_routes_test.go new file mode 100644 index 0000000..c9ee1ec --- /dev/null +++ b/raw_routes_test.go @@ -0,0 +1,291 @@ +package main + +import ( + "context" + "errors" + "github.com/jackc/pgx/v5" + "github.com/labstack/echo/v4" + "github.com/mudler/localrecall/pkg/client" + "github.com/mudler/localrecall/rag/engine" + "github.com/mudler/localrecall/rag/raw" + rawpostgres "github.com/mudler/localrecall/rag/raw/postgres" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync/atomic" + "testing" +) + +type rawSpy struct { + raw.Store + calls int + err error +} + +func (s *rawSpy) CreateCollection(context.Context, raw.CreateCollectionRequest) (raw.Collection, bool, error) { + s.calls++ + return raw.Collection{}, true, s.err +} +func TestRawHTTPValidation(t *testing.T) { + s := &rawSpy{} + e := echo.New() + registerRawAPIRoutes(e, s) + for _, body := range []string{`null`, `{} {}`, `{"name":"a","name":"b"}`, `{"unknown":1}`, `{"lexical":{"enabled":true,"enabled":false}}`, `{"name":"\u0000"}`, "{\"name\":\"\xff\"}"} { + r := httptest.NewRecorder() + e.ServeHTTP(r, httptest.NewRequest("POST", "/api/raw/v1/collections", strings.NewReader(body))) + if r.Code != 400 { + t.Fatalf("%q: %d %s", body, r.Code, r.Body) + } + } + r := httptest.NewRecorder() + e.ServeHTTP(r, httptest.NewRequest("POST", "/api/raw/v1/collections", strings.NewReader(strings.Repeat(" ", raw.MaxBodyBytes+1)))) + if r.Code != 413 || s.calls != 0 { + t.Fatalf("limit %d calls %d", r.Code, s.calls) + } + s.err = errors.New("postgres://secret SQL payload") + r = httptest.NewRecorder() + e.ServeHTTP(r, httptest.NewRequest("POST", "/api/raw/v1/collections", strings.NewReader(`{}`))) + if r.Code != 500 || strings.Contains(r.Body.String(), "secret") { + t.Fatal(r.Code, r.Body) + } +} + +func TestRawHTTPAuthAndReadiness(t *testing.T) { + old := vectorEngine + defer func() { vectorEngine = old }() + for _, backend := range []string{"postgres", "chromem"} { + vectorEngine = backend + for _, keys := range [][]string{nil, {"key"}} { + // Avoid legacy discovery against an external service: register with chromem + // and an empty fixture directory, then restore the raw backend. + vectorEngine = "chromem" + e := echo.New() + registerAPIRoutes(e, nil, 400, 0, keys) + vectorEngine = backend + registerRawAPIRoutes(e, nil) + for _, path := range []string{"/api/raw/v1/capabilities", "/api/raw/v1/collections"} { + r := httptest.NewRecorder() + q := httptest.NewRequest("GET", path, nil) + e.ServeHTTP(r, q) + if len(keys) > 0 { + if r.Code != 401 { + t.Fatal(path, r.Code) + } + continue + } + if path == "/api/raw/v1/collections" { + want := 501 + if backend == "postgres" { + want = 503 + } + if r.Code != want { + t.Fatal(r.Code, r.Body) + } + } else if path == "/api/raw/v1/capabilities" && r.Code != 200 { + t.Fatal(r.Code) + } + } + if len(keys) > 0 { + r := httptest.NewRecorder() + q := httptest.NewRequest("GET", "/api/raw/v1/capabilities", nil) + q.Header.Set("Authorization", "Bearer key") + e.ServeHTTP(r, q) + if r.Code != 200 { + t.Fatal(r.Code) + } + } + } + } +} + +func TestRawAPILifecycle(t *testing.T) { + if os.Getenv("LOCALRECALL_TEST_DATABASE_OWNER") == "" { + t.Fatal("owned database fixture required") + } + var calls atomic.Int64 + trap := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(500) })) + defer trap.Close() + t.Setenv("OPENAI_BASE_URL", trap.URL) + t.Setenv("LOCALAI_ENDPOINT", trap.URL) + assetTrap := t.TempDir() + "/not-a-directory" + if err := os.WriteFile(assetTrap, []byte("trap"), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("FILE_ASSETS", assetTrap) + t.Setenv("COLLECTION_DB_PATH", assetTrap) + ctx := context.Background() + cfg := rawpostgres.Config{DatabaseURL: os.Getenv("LOCALRECALL_TEST_DATABASE_URL")} + service, err := rawpostgres.New(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer service.Close() + e := echo.New() + registerRawAPIRoutes(e, service) + server := httptest.NewServer(e) + defer server.Close() + c := client.NewRawClient(server.URL, "", server.Client()) + caps, err := c.Capabilities(ctx) + if err != nil || !caps.Ready { + t.Fatal(caps, err) + } + req := raw.CreateCollectionRequest{Name: "api-owned", Dimensions: 3, Metric: "cosine", EmbeddingSpaceID: "space"} + d, created, err := c.CreateCollection(ctx, req) + if err != nil || !created { + t.Fatal(d, created, err) + } + _, created, err = c.CreateCollection(ctx, req) + if err != nil || created { + t.Fatal(created, err) + } + bad := req + bad.Dimensions = 4 + _, _, err = c.CreateCollection(ctx, bad) + assertRawStatus(t, err, 409) + if _, err = c.GetCollection(ctx, d.CollectionID); err != nil { + t.Fatal(err) + } + page, err := c.ListCollections(ctx, raw.ListCollectionsRequest{}) + if err != nil || len(page.Collections) == 0 { + t.Fatal(page, err) + } + text := "bicycle" + _, err = c.Upsert(ctx, d.CollectionID, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{{ID: "segment", Vector: []float64{1, 0, 0}, Text: &text, Metadata: map[string]string{"source": trap.URL}}}}) + if err != nil { + t.Fatal(err) + } + got, err := c.GetRecords(ctx, d.CollectionID, raw.GetRecordsRequest{IDs: []string{"segment", "missing"}, IncludeVectors: true}) + if err != nil || len(got.Records) != 1 || len(got.MissingIDs) != 1 { + t.Fatal(got, err) + } + records, err := c.ListRecords(ctx, d.CollectionID, raw.ListRecordsRequest{}) + if err != nil || len(records.Records) != 1 { + t.Fatal(records, err) + } + space := "space" + exact := "exact" + for _, mode := range []string{"vector", "lexical", "hybrid"} { + q := raw.SearchRequest{Mode: mode} + if mode != "lexical" { + q.Vector = []float64{1, 0, 0} + q.EmbeddingSpaceID = &space + q.Accuracy = &exact + } + if mode != "vector" { + q.Text = &text + } + result, err := c.Search(ctx, d.CollectionID, q) + if err != nil || result.Count != 1 { + t.Fatal(result, err) + } + } + service.Close() + service, err = rawpostgres.New(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer service.Close() + reopened := echo.New() + registerRawAPIRoutes(reopened, service) + srv := httptest.NewServer(reopened) + defer srv.Close() + c = client.NewRawClient(srv.URL, "", srv.Client()) + if _, err = c.GetCollection(ctx, d.CollectionID); err != nil { + t.Fatal(err) + } + del, err := c.DeleteRecords(ctx, d.CollectionID, raw.DeleteRecordsRequest{IDs: []string{"segment"}}) + if err != nil || del.DeletedCount != 1 { + t.Fatal(del, err) + } + if err = c.Reset(ctx, d.CollectionID); err != nil { + t.Fatal(err) + } + if err = c.DeleteCollection(ctx, d.CollectionID); err != nil { + t.Fatal(err) + } + _, err = c.GetCollection(ctx, d.CollectionID) + assertRawStatus(t, err, 404) + names, err := engine.ListPostgresCollections(ctx, cfg.DatabaseURL) + if err != nil { + t.Fatal(err) + } + for _, name := range names { + if name == req.Name { + t.Fatal("raw leaked into legacy discovery") + } + } + conn, err := pgx.Connect(ctx, cfg.DatabaseURL) + if err != nil { + t.Fatal(err) + } + defer conn.Close(ctx) + if _, err = conn.Exec(ctx, "UPDATE lr_raw_schema SET version=999"); err != nil { + t.Fatal(err) + } + failed, err := rawpostgres.New(ctx, cfg) + if err == nil { + failed.Close() + t.Fatal("unknown schema accepted") + } + if _, err = conn.Exec(ctx, "UPDATE lr_raw_schema SET version=1"); err != nil { + t.Fatal(err) + } + if calls.Load() != 0 { + t.Fatalf("embedding/source calls %d", calls.Load()) + } +} +func assertRawStatus(t *testing.T, err error, status int) { + t.Helper() + var re *raw.Error + if !errors.As(err, &re) || re.Status != status { + t.Fatalf("wanted %d: %v", status, err) + } +} + +type rawBrokenBody struct{ err error } + +func (b rawBrokenBody) Read([]byte) (int, error) { return 0, b.err } +func (b rawBrokenBody) Close() error { return nil } +func TestRawHTTPErrors(t *testing.T) { + s := &rawSpy{} + e := echo.New() + registerRawAPIRoutes(e, s) + for _, code := range []string{raw.CodeValidation, raw.CodeNotFound, raw.CodeConflict, raw.CodeSpaceMismatch, raw.CodeBodyLimit, raw.CodeDeleteBound, raw.CodeUnsupported, raw.CodeUnavailable, raw.CodeTimeout, raw.CodeInternal} { + s.err = raw.NewError(code) + r := httptest.NewRecorder() + e.ServeHTTP(r, httptest.NewRequest("POST", "/api/raw/v1/collections", strings.NewReader(`{}`))) + if r.Code != raw.NewError(code).Status { + t.Fatal(code, r.Code) + } + } + s.err = context.Canceled + r := httptest.NewRecorder() + e.ServeHTTP(r, httptest.NewRequest("POST", "/api/raw/v1/collections", strings.NewReader(`{}`))) + if r.Code != 504 { + t.Fatal(r.Code) + } + before := s.calls + q := httptest.NewRequest("POST", "/api/raw/v1/collections", nil) + q.Body = rawBrokenBody{errors.New("read secret")} + r = httptest.NewRecorder() + e.ServeHTTP(r, q) + if r.Code != 400 || s.calls != before { + t.Fatal(r.Code, s.calls) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + q = httptest.NewRequest("POST", "/api/raw/v1/collections", strings.NewReader(`{}`)).WithContext(ctx) + r = httptest.NewRecorder() + e.ServeHTTP(r, q) + if r.Code != 504 || s.calls != before { + t.Fatal(r.Code, s.calls) + } + for _, body := range []string{`{"records":[{"metadata":{"a":{"nested":"no"}}}]}`, `{"records":[{"metadata":{"a":"one","a":"two"}}]}`, `{"records":null}`, `{"records":[{"metadata":{"a":"\u0000"}}]}`} { + r = httptest.NewRecorder() + e.ServeHTTP(r, httptest.NewRequest("POST", "/api/raw/v1/collections/cid/records/upsert", strings.NewReader(body))) + if r.Code != 400 { + t.Fatal(r.Code, r.Body) + } + } +} diff --git a/test/raw/verify.sh b/test/raw/verify.sh index dd9d00e..76b0a65 100644 --- a/test/raw/verify.sh +++ b/test/raw/verify.sh @@ -13,6 +13,18 @@ export COLLECTION_DB_PATH="$ROOT/.sdd-work/collections" FILE_ASSETS="$ROOT/.sdd- export GOENV=off GOTOOLCHAIN=local mkdir -p "$TMPDIR" "$GOCACHE" "$GOMODCACHE" "$GOPATH" "$COLLECTION_DB_PATH" "$FILE_ASSETS" "$ROOT/.sdd-work/logs" case "${1:-}" in + api) + sh test/raw/verify.sh search + log="$ROOT/.sdd-work/logs/api-$(date +%s)-$$.log" + if sh test/raw/database.sh go test -v -mod=readonly -race -count=1 -timeout=10m -run '^TestRaw' . >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi + for suite in HTTPValidation HTTPAuthAndReadiness APILifecycle; do grep -q -- "--- PASS: TestRaw${suite}" "$log"; done + ! grep -E "SKIP|no tests to run|no test files" "$log" + log="$ROOT/.sdd-work/logs/api-client-$(date +%s)-$$.log" + if go test -v -mod=readonly -race -count=1 ./pkg/client/... >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi + grep -q -- '--- PASS: TestRawClient' "$log" + ! grep -E "SKIP|no tests to run|no test files" "$log" + git diff --check + ;; search) sh test/raw/verify.sh records log="$ROOT/.sdd-work/logs/search-$(date +%s)-$$.log" @@ -50,5 +62,5 @@ case "${1:-}" in [ "$(grep -c '^ok[[:space:]]' "$log")" -eq 2 ] && ! grep -E '\[no test files\]|\[no tests to run\]' "$log" git diff --check ;; - *) echo 'usage: sh test/raw/verify.sh contracts|lifecycle|records|search' >&2; exit 2;; + *) echo 'usage: sh test/raw/verify.sh contracts|lifecycle|records|search|api' >&2; exit 2;; esac From c607ffb30f7ad86dc0fc2481a32975e20619138b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 11 Oct 2026 23:08:06 +0000 Subject: [PATCH 09/14] fix(raw): bound HTTP reads and retain failures Context cancellation alone cannot interrupt an incoming HTTP body read. Interrupt raw transport reads and preserve timeout responses without changing legacy read timeouts. Test stalled POST and DELETE requests. Return startup failures after cleanup so bind errors exit nonzero. Reject oversized client responses and preserve encoded URL prefixes. Check legacy envelopes and discovery while raw collections remain live. Verify with sh test/raw/verify.sh api, including prior suites and race checks against private database fixtures. --- main.go | 19 +++- pkg/client/raw.go | 13 ++- pkg/client/raw_test.go | 24 +++++ raw_routes.go | 18 ++++ raw_routes_test.go | 213 +++++++++++++++++++++++++++++++++++++++-- 5 files changed, 271 insertions(+), 16 deletions(-) diff --git a/main.go b/main.go index d1475ef..4b446f2 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,7 @@ func init() { sourceManager.Start() } -func startAPI(listenAddress string) { +func startAPI(listenAddress string) error { e := echo.New() e.Use(middleware.Logger()) e.Use(middleware.Recover()) @@ -111,16 +111,23 @@ func startAPI(listenAddress string) { } } registerRawAPIRoutes(e, store) - requests, cancelRequests := context.WithCancel(context.Background()) - e.Server.BaseContext = func(_ net.Listener) context.Context { return requests } stop, cancelSignal := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancelSignal() + return serveAPI(e, listenAddress, store, stop) +} + +// serveAPI returns startup failures only after releasing server-owned resources. +func serveAPI(e *echo.Echo, listenAddress string, store raw.Store, stop context.Context) error { + requests, cancelRequests := context.WithCancel(context.Background()) + e.Server.BaseContext = func(_ net.Listener) context.Context { return requests } + var serveErr error exited := make(chan error, 1) go func() { exited <- e.Start(listenAddress) }() select { case err := <-exited: if err != nil && !errors.Is(err, http.ErrServerClosed) { e.Logger.Error("HTTP server stopped") + serveErr = err } case <-stop.Done(): ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) @@ -129,13 +136,17 @@ func startAPI(listenAddress string) { _ = e.Close() } cancel() + <-exited } cancelRequests() if store != nil { store.Close() } + return serveErr } func main() { - startAPI(listeningAddress) + if err := startAPI(listeningAddress); err != nil { + os.Exit(1) + } } diff --git a/pkg/client/raw.go b/pkg/client/raw.go index bfcfae1..32f406a 100644 --- a/pkg/client/raw.go +++ b/pkg/client/raw.go @@ -34,8 +34,12 @@ func (c *RawClient) do(ctx context.Context, method, path string, q url.Values, b if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { return 0, fmt.Errorf("invalid raw API base URL") } - u.Path = strings.TrimRight(u.Path, "/") + "/api/raw/v1" + path - u.RawPath = "" + escapedPath := strings.TrimRight(u.EscapedPath(), "/") + "/api/raw/v1" + path + u.Path, err = url.PathUnescape(escapedPath) + if err != nil { + return 0, err + } + u.RawPath = escapedPath u.RawQuery = q.Encode() var reader io.Reader if body != nil { @@ -65,10 +69,13 @@ func (c *RawClient) do(ctx context.Context, method, path string, q url.Values, b Data json.RawMessage `json:"data"` Error *raw.Error `json:"error"` } - b, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20)) + b, err := io.ReadAll(io.LimitReader(resp.Body, (32<<20)+1)) if err != nil { return resp.StatusCode, err } + if len(b) > 32<<20 { + return resp.StatusCode, fmt.Errorf("raw API response exceeds 32 MiB") + } if err = json.Unmarshal(b, &envelope); err != nil { return resp.StatusCode, fmt.Errorf("invalid raw API response (HTTP %d)", resp.StatusCode) } diff --git a/pkg/client/raw_test.go b/pkg/client/raw_test.go index f751164..c917d35 100644 --- a/pkg/client/raw_test.go +++ b/pkg/client/raw_test.go @@ -66,3 +66,27 @@ func TestRawClientSafety(t *testing.T) { t.Fatal(calls) } } + +func TestRawClientResponseOverflow(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // The first 32 MiB are valid JSON plus whitespace; truncation hides junk. + prefix := `{"success":true,"data":{}}` + w.Write([]byte(prefix + strings.Repeat(" ", (32<<20)-len(prefix)) + "x")) + })) + defer srv.Close() + if _, err := NewRawClient(srv.URL, "", srv.Client()).Capabilities(context.Background()); err == nil { + t.Fatal("oversized response accepted") + } +} +func TestRawClientEncodedPrefix(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.EscapedPath() != "/tenant%2Fprivate/api/raw/v1/capabilities" { + t.Errorf("prefix semantics changed: %s", r.URL.EscapedPath()) + } + w.Write([]byte(`{"success":true,"data":{}}`)) + })) + defer srv.Close() + if _, err := NewRawClient(srv.URL+"/tenant%2Fprivate", "", srv.Client()).Capabilities(context.Background()); err != nil { + t.Fatal(err) + } +} diff --git a/raw_routes.go b/raw_routes.go index 2d2791c..8785cb0 100644 --- a/raw_routes.go +++ b/raw_routes.go @@ -22,6 +22,8 @@ func rawFailure(c echo.Context, err error) error { } if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { e = raw.NewError(raw.CodeTimeout) + // Do not drain an incomplete timed-out body before sending the error. + c.Response().Header().Set("Connection", "close") } return c.JSON(e.Status, struct { Success bool `json:"success"` @@ -69,6 +71,22 @@ func registerRawAPIRoutes(e *echo.Echo, store raw.Store) { ctx, cancel := context.WithTimeout(c.Request().Context(), 30*time.Second) defer cancel() c.SetRequest(c.Request().WithContext(ctx)) + // Incoming Body.Read does not observe context cancellation. Interrupt + // the transport only for raw requests, including early cancellation. + controller := http.NewResponseController(c.Response().Writer) + interrupted := make(chan struct{}) + stop := context.AfterFunc(ctx, func() { + _ = controller.SetReadDeadline(time.Now()) + close(interrupted) + }) + defer func() { + if !stop() { + <-interrupted + } + // Do not leak a raw deadline into a subsequent keep-alive request. + _ = controller.SetReadDeadline(time.Time{}) + }() + if store == nil && c.Path() != "/api/raw/v1/capabilities" { if vectorEngine != "postgres" { return rawFailure(c, raw.NewError(raw.CodeUnsupported)) diff --git a/raw_routes_test.go b/raw_routes_test.go index c9ee1ec..3b02708 100644 --- a/raw_routes_test.go +++ b/raw_routes_test.go @@ -1,20 +1,26 @@ package main import ( + "bufio" "context" "errors" + "fmt" "github.com/jackc/pgx/v5" "github.com/labstack/echo/v4" "github.com/mudler/localrecall/pkg/client" "github.com/mudler/localrecall/rag/engine" "github.com/mudler/localrecall/rag/raw" rawpostgres "github.com/mudler/localrecall/rag/raw/postgres" + "io" + "net" "net/http" "net/http/httptest" "os" + "os/exec" "strings" "sync/atomic" "testing" + "time" ) type rawSpy struct { @@ -64,6 +70,9 @@ func TestRawHTTPAuthAndReadiness(t *testing.T) { registerAPIRoutes(e, nil, 400, 0, keys) vectorEngine = backend registerRawAPIRoutes(e, nil) + if len(keys) == 0 { + assertLegacyEnvelope(t, e) + } for _, path := range []string{"/api/raw/v1/capabilities", "/api/raw/v1/collections"} { r := httptest.NewRecorder() q := httptest.NewRequest("GET", path, nil) @@ -121,8 +130,13 @@ func TestRawAPILifecycle(t *testing.T) { t.Fatal(err) } defer service.Close() + oldBackend, oldPath := vectorEngine, collectionDBPath + vectorEngine, collectionDBPath = "chromem", t.TempDir() + defer func() { vectorEngine, collectionDBPath = oldBackend, oldPath }() e := echo.New() + registerAPIRoutes(e, nil, 400, 0, nil) registerRawAPIRoutes(e, service) + assertLegacyEnvelope(t, e) server := httptest.NewServer(e) defer server.Close() c := client.NewRawClient(server.URL, "", server.Client()) @@ -150,6 +164,18 @@ func TestRawAPILifecycle(t *testing.T) { if err != nil || len(page.Collections) == 0 { t.Fatal(page, err) } + assertLiveIsolation := func() { + names, err := engine.ListPostgresCollections(ctx, cfg.DatabaseURL) + if err != nil { + t.Fatal(err) + } + for _, name := range names { + if name == req.Name { + t.Fatal("raw leaked into legacy discovery") + } + } + } + assertLiveIsolation() text := "bicycle" _, err = c.Upsert(ctx, d.CollectionID, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{{ID: "segment", Vector: []float64{1, 0, 0}, Text: &text, Metadata: map[string]string{"source": trap.URL}}}}) if err != nil { @@ -187,6 +213,7 @@ func TestRawAPILifecycle(t *testing.T) { } defer service.Close() reopened := echo.New() + registerAPIRoutes(reopened, nil, 400, 0, nil) registerRawAPIRoutes(reopened, service) srv := httptest.NewServer(reopened) defer srv.Close() @@ -194,6 +221,8 @@ func TestRawAPILifecycle(t *testing.T) { if _, err = c.GetCollection(ctx, d.CollectionID); err != nil { t.Fatal(err) } + assertLiveIsolation() + assertLegacyEnvelope(t, reopened) del, err := c.DeleteRecords(ctx, d.CollectionID, raw.DeleteRecordsRequest{IDs: []string{"segment"}}) if err != nil || del.DeletedCount != 1 { t.Fatal(del, err) @@ -206,15 +235,6 @@ func TestRawAPILifecycle(t *testing.T) { } _, err = c.GetCollection(ctx, d.CollectionID) assertRawStatus(t, err, 404) - names, err := engine.ListPostgresCollections(ctx, cfg.DatabaseURL) - if err != nil { - t.Fatal(err) - } - for _, name := range names { - if name == req.Name { - t.Fatal("raw leaked into legacy discovery") - } - } conn, err := pgx.Connect(ctx, cfg.DatabaseURL) if err != nil { t.Fatal(err) @@ -289,3 +309,178 @@ func TestRawHTTPErrors(t *testing.T) { } } } + +// A context deadline alone cannot unblock net/http's incoming Body.Read. +func TestRawStreamingBodyCancellation(t *testing.T) { + for _, method := range []string{"POST", "DELETE"} { + for _, early := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/cancel=%v", method, early), func(t *testing.T) { + s := &rawSpy{} + e := echo.New() + registerRawAPIRoutes(e, s) + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + if early { + cancel() + ctx, cancel = context.WithTimeout(context.Background(), time.Hour) + } + defer cancel() + entered, done := make(chan struct{}), make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(entered) + e.ServeHTTP(w, r.WithContext(ctx)) + close(done) + })) + defer server.Close() + conn, err := net.Dial("tcp", strings.TrimPrefix(server.URL, "http://")) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + path := "/api/raw/v1/collections" + if method == "DELETE" { + path += "/00000000-0000-0000-0000-000000000000" + } + fmt.Fprintf(conn, "%s %s HTTP/1.1\r\nHost: fixture\r\nContent-Length: 100\r\n\r\n{", method, path) + <-entered + if early { + cancel() + } + select { + case <-done: + case <-time.After(time.Second): + conn.Close() // unblock and clean up even against the broken implementation + <-done + t.Fatal("handler remained blocked reading body after cancellation") + } + if s.calls != 0 { + t.Fatalf("store calls: %d", s.calls) + } + conn.SetReadDeadline(time.Now().Add(time.Second)) + response, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + b, _ := io.ReadAll(response.Body) + if response.StatusCode != 504 || !strings.Contains(string(b), `"code":"TIMEOUT"`) { + t.Fatalf("%d %s", response.StatusCode, b) + } + }) + } + } +} + +func TestRawServerBindFailureExit(t *testing.T) { + if os.Getenv("RAW_BIND_CHILD") == "1" { + vectorEngine = "chromem" + listeningAddress = "127.0.0.1:not-a-port" + main() + return + } + cmd := exec.Command(os.Args[0], "-test.run=^TestRawServerBindFailureExit$") + cmd.Env = append(os.Environ(), "RAW_BIND_CHILD=1") + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("bind failure exited successfully: %s", output) + } + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 1 { + t.Fatalf("unexpected exit: %v %s", err, output) + } +} + +func assertLegacyEnvelope(t *testing.T, e *echo.Echo) { + t.Helper() + old := vectorEngine + vectorEngine = "chromem" + defer func() { vectorEngine = old }() + for _, tc := range []struct { + method, path string + status int + body string + }{ + {"GET", "/api/collections", 200, `{"success":true,"message":"Collections retrieved successfully","data":{"collections":null,"count":0}}`}, + {"POST", "/api/collections/absent/reset", 404, `{"success":false,"error":{"code":"NOT_FOUND","message":"Collection not found","details":"Collection 'absent' does not exist"}}`}, + } { + r := httptest.NewRecorder() + e.ServeHTTP(r, httptest.NewRequest(tc.method, tc.path, nil)) + if r.Code != tc.status || strings.TrimSpace(r.Body.String()) != tc.body { + t.Fatalf("legacy %s: %d %s", tc.path, r.Code, r.Body) + } + } +} + +type rawCloseSpy struct { + raw.Store + closed bool +} + +func (s *rawCloseSpy) Close() { s.closed = true } +func TestRawServerCleanup(t *testing.T) { + for _, fail := range []bool{false, true} { + e := echo.New() + e.HideBanner = true + store := &rawCloseSpy{} + stop, cancel := context.WithCancel(context.Background()) + address := "127.0.0.1:not-a-port" + if !fail { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + e.Listener = listener + cancel() // injectable signal context, never signal the shared test process + } + err := serveAPI(e, address, store, stop) + cancel() + if (err != nil) != fail || !store.closed { + t.Fatalf("fail=%v error=%v closed=%v", fail, err, store.closed) + } + } +} + +func (s *rawSpy) DeleteCollection(context.Context, string) error { s.calls++; return s.err } + +func TestRawDeadlineDoesNotLimitLegacyBody(t *testing.T) { + e := echo.New() + registerRawAPIRoutes(e, &rawSpy{}) + e.POST("/legacy-stream", func(c echo.Context) error { + b, err := io.ReadAll(c.Request().Body) + if err != nil { + return err + } + return c.String(200, string(b)) + }) + server := httptest.NewServer(e) + defer server.Close() + conn, err := net.Dial("tcp", strings.TrimPrefix(server.URL, "http://")) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + // First exercise successful raw middleware on this same connection. + fmt.Fprint(conn, "POST /api/raw/v1/collections HTTP/1.1\r\nHost: fixture\r\nContent-Length: 2\r\n\r\n{}") + reader := bufio.NewReader(conn) + response, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + io.Copy(io.Discard, response.Body) + response.Body.Close() + fmt.Fprint(conn, "POST /legacy-stream HTTP/1.1\r\nHost: fixture\r\nContent-Length: 2\r\n\r\na") + time.Sleep(200 * time.Millisecond) + fmt.Fprint(conn, "b") + conn.SetReadDeadline(time.Now().Add(time.Second)) + response, err = http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + b, _ := io.ReadAll(response.Body) + if response.StatusCode != 200 || string(b) != "ab" { + t.Fatalf("%d %s", response.StatusCode, b) + } + if e.Server.ReadTimeout != 0 { + t.Fatal("global read timeout changed") + } +} From 8c31f799fe52be28264c9d9fcdde78c906daa9a4 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 11 Oct 2026 23:20:00 +0000 Subject: [PATCH 10/14] test: exercise pinned LocalAI consumer with genuine embeddings --- test/raw/LOCALAI-SMOKE.md | 72 ++++++++++++++ test/raw/localai-smoke.sh | 97 +++++++++++++++++++ .../templates/localai_consumer_test.go.txt | 69 +++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 test/raw/LOCALAI-SMOKE.md create mode 100644 test/raw/localai-smoke.sh create mode 100644 test/raw/templates/localai_consumer_test.go.txt diff --git a/test/raw/LOCALAI-SMOKE.md b/test/raw/LOCALAI-SMOKE.md new file mode 100644 index 0000000..8f1562d --- /dev/null +++ b/test/raw/LOCALAI-SMOKE.md @@ -0,0 +1,72 @@ +# Actual LocalAI consumer smoke + +Run from the LocalRecall worktree root: + +```sh +LOCALAI_SMOKE_BACKEND=/absolute/path/to/llama.cpp/grpc-server \ +LOCALAI_SMOKE_LIBS=/absolute/path/to/native/libraries \ +sh test/raw/localai-smoke.sh > .sdd-work/logs/localai-real.log 2>&1 +``` + +This is a required-dependency test: missing inputs fail, never skip. It requires +Go matching the pinned consumer, Python 3, Docker, curl for provisioning, and +approximately 1 GiB of additional headroom with the existing Go caches populated. +A clean Go cache needs substantially more. Do not blindly pull large images. + +## Provisioning + +The runner intentionally does not download native dependencies or images: + +* Download `https://codeload.github.com/mudler/LocalAI/tar.gz/d4a5af8bfed9c97e8cb01e584f1f9f8a572df844` + into `.sdd-work/localai.tar.gz`; extract with `--strip-components=1` into + `.sdd-work/localai`. Read its `AGENTS.md` and build instructions. Generate Go + protobuf files with `make protogen-go`, using private GOPATH/GOCACHE/GOMODCACHE + and TMPDIR under `.sdd-work`. The runner verifies every archive file against + the extracted tree, while allowing extra generated files and the smoke fixture. +* Download `https://huggingface.co/gaianet/All-MiniLM-L6-v2-Embedding-GGUF/resolve/main/all-MiniLM-L6-v2-ggml-model-f16.gguf` + into `.sdd-work/model-smoke/minilm.gguf` (44 MiB). The runner requires SHA-256 + `797b70c4edf85907fe0a49eb85811256f65fa0f7bf52166b147fd16be2be4662`: + upstream `main` is mutable, but accepted file content is pinned. +* Have cached provider image + `sha256:f1bc435659b9da49180e868fdd0361915171a740cd947bf081b62a896a7ea8d0` + and the PostgreSQL image required by `database.sh`. No running host service is used. +* Supply a genuine llama.cpp gRPC backend and its native library directory. + These are used read-only; backend cwd is the private model directory. + The successful local run used `.sdd-bridge-repro/grpc-server` and + `.sdd-deps/root/usr/lib/x86_64-linux-gnu` from the existing + `mudler-LocalAI/llama-decisions` workspace. Backend SHA-256 was + `ce81165dfa8459b3b450402e54c69fd476c2b1ad17d883a8fa85eea84df5fc78`. + Its source revision is not attested by this smoke; binary identity is logged. + +## What it proves + +The provider binary is copied from the cached image into `.sdd-work`, with a +verified binary digest. Its version is v4.11.0 / source label +`58830f7ac508845a6f4efa32cfca06af422d4d82`, **not** the consumer revision. +A private loopback provider and native backend are started and stopped by the +runner. An actual `/v1/embeddings` request must return 384 finite, nonzero MiniLM +values before testing. Responses and native/provider logs stay in `.sdd-work/logs`. + +The generated test lives in the pinned real `core/services/agentpool` package. +It calls `NewAgentPoolService`, `buildCollectionsConfig`, the production embedded +`collections.NewInProcessBackend`, and `shareCollections`, matching the collection +portion of `startDistributed`. It then exercises the real ForUser upload, list, +entry-content, search and reset methods. No fake engine, HTTP store adapter or +standalone reimplementation substitutes for the consumer. Unrelated schedulers, +skills and NATS are deliberately not started. An alternate modfile replaces +LocalRecall with this worktree; neither source go.mod/go.sum is edited. + +The successful run used PostgreSQL HNSW fallback (vectorscale was unavailable). +It verifies text embedding compatibility, not EmbeddingGemma2 or multimodal +quality. It does not complete Task6: concurrent raw zero-embedding accounting, +full legacy suites, cross-process/restart regressions and the all-runner remain +separate required gates. + +## Evidence + +Initial run failed before inference because the provider flag was incorrectly +spelled `--disable-webui`; provider help identified `--disable-web-ui`. After +that one-line correction, `.sdd-work/logs/localai-real-attempt3.log` reports +`TestLocalRecallRealConsumer` PASS and owned database removal. The earlier +compilation-only evidence is `localai-build-generated-attempt1.log`; it is not +counted as a runtime test. diff --git a/test/raw/localai-smoke.sh b/test/raw/localai-smoke.sh new file mode 100644 index 0000000..2aafa88 --- /dev/null +++ b/test/raw/localai-smoke.sh @@ -0,0 +1,97 @@ +#!/bin/sh +# Prerequisites and provider provenance: see test/raw/LOCALAI-SMOKE.md. +set -eu +ROOT=$(pwd -P) +[ -f test/raw/downstream-lock.json ] || exit 1 +W="$ROOT/.sdd-work" +[ ! -L "$W" ] || exit 1 +mkdir -p "$W/logs" "$W/tmp" "$W/model-smoke" +export TMPDIR="$W/tmp" GOCACHE="$W/cache" GOMODCACHE="$W/mod" GOPATH="$W/gopath" GOENV=off GOTOOLCHAIN=local +export DOCKER_CONFIG="$W/docker" +: "${LOCALAI_SMOKE_BACKEND:?set path to genuine llama.cpp grpc-server}" +: "${LOCALAI_SMOKE_LIBS:?set native library directory}" +export LD_LIBRARY_PATH="$LOCALAI_SMOKE_LIBS${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +python3 - "$ROOT" <<'PY' +import hashlib,json,pathlib,tarfile,sys +r=pathlib.Path(sys.argv[1]); w=r/'.sdd-work' +lock=json.loads((r/'test/raw/downstream-lock.json').read_text()) +assert hashlib.sha256((w/'localai.tar.gz').read_bytes()).hexdigest() == '99e86e821f0094a3176926cfc1508deb88d229b8ab1d292e52a361df08fdd92e', 'unexpected archive digest' +assert hashlib.sha256((w/'model-smoke/minilm.gguf').read_bytes()).hexdigest() == '797b70c4edf85907fe0a49eb85811256f65fa0f7bf52166b147fd16be2be4662', 'unexpected model digest' +with tarfile.open(w/'localai.tar.gz') as tar: + members=tar.getmembers(); prefix=members[0].name.split('/')[0] + assert prefix=='LocalAI-'+lock['revision'], prefix + n=0 + for m in members: + rel=pathlib.PurePosixPath(m.name).relative_to(prefix) + p=w/'localai'/rel + if m.isfile(): + assert p.is_file() and hashlib.sha256(p.read_bytes()).digest()==hashlib.sha256(tar.extractfile(m).read()).digest(), str(rel) + n+=1 + elif m.issym(): assert p.is_symlink() and str(p.readlink())==m.linkname,str(rel) + print('Verified archive source files:',n,'consumer revision:',lock['revision']) +PY +# The archive must have been downloaded from the immutable codeload revision; +# canonical digest recorded independently, never use the enclosing repo's HEAD. +sha256sum "$W/localai.tar.gz" "$W/model-smoke/minilm.gguf" "$LOCALAI_SMOKE_BACKEND" +docker image inspect localai/localai:latest --format 'provider image {{.Id}} {{json .Config.Labels}}' +image=sha256:f1bc435659b9da49180e868fdd0361915171a740cd947bf081b62a896a7ea8d0 +name="lr-embedding-$$" +created= +backend_pid= provider_pid= +cleanup() { + [ -z "$created" ] || docker rm -v "$name" >/dev/null 2>&1 || true + [ -z "$provider_pid" ] || { kill "$provider_pid" 2>/dev/null || true; wait "$provider_pid" 2>/dev/null || true; } + [ -z "$backend_pid" ] || { kill "$backend_pid" 2>/dev/null || true; wait "$backend_pid" 2>/dev/null || true; } +} +trap cleanup EXIT HUP INT TERM +if [ ! -f "$W/model-smoke/local-ai" ]; then + docker create --name "$name" --network none --tmpfs /models --tmpfs /backends --tmpfs /configuration --tmpfs /data "$image" >/dev/null + created=yes + docker cp "$name:/local-ai" "$W/model-smoke/local-ai" + docker rm -v "$name" >/dev/null + created= +fi +echo "3c03e310467418dec59d6c391e395a8e4cdfcee6dd450e66f74eeeefa21a53f6 $W/model-smoke/local-ai" | sha256sum -c - +ports=$(python3 - <<'PY' +import socket +s=[socket.socket(),socket.socket()] +for x in s: x.bind(('127.0.0.1',0)) +print(*(x.getsockname()[1] for x in s)) +PY +) +set -- $ports +backend_port=$1; api_port=$2 +cat > "$W/model-smoke/minilm.yaml" < "$W/logs/localai-native-smoke.log" 2>&1 & +backend_pid=$! +(cd "$W/model-smoke"; exec ./local-ai run --address "127.0.0.1:$api_port" --models-path "$W/model-smoke" --backends-path "$W/model-smoke/backends" --backends-system-path "$W/model-smoke/system" --data-path "$W/model-smoke/data" --localai-config-dir "$W/model-smoke/config" --external-grpc-backends "smoke-llama:127.0.0.1:$backend_port" --disable-web-ui --disable-agents) > "$W/logs/localai-provider-smoke.log" 2>&1 & +provider_pid=$! +export LOCALRECALL_SMOKE_API="http://127.0.0.1:$api_port/v1" +python3 - "$LOCALRECALL_SMOKE_API" "$W/logs/localai-embeddings.json" "$provider_pid" <<'PY' +import urllib.request,time,json,sys,math,os +for i in range(90): + os.kill(int(sys.argv[3]),0) + try: + req=urllib.request.Request(sys.argv[1]+'/embeddings',data=json.dumps({'model':'minilm','input':'The telescope studies distant galaxies.'}).encode(),headers={'Content-Type':'application/json'}) + with urllib.request.urlopen(req,timeout=10) as resp: data=json.load(resp) + v=data['data'][0]['embedding']; assert len(v)==384 and all(math.isfinite(x) for x in v) and any(x!=0 for x in v) + open(sys.argv[2],'w').write(json.dumps(data)); print('Genuine MiniLM /embeddings ready, dimensions',len(v)); break + except Exception as e: + if i==89: raise + time.sleep(1) +PY +cp test/raw/templates/localai_consumer_test.go.txt "$W/localai/core/services/agentpool/localrecall_smoke_test.go" +# Preserve upstream mod/sum; resolve only in private alternate files. +cp "$W/localai/go.mod" "$W/localai-smoke.mod" +cp "$W/localai/go.sum" "$W/localai-smoke.sum" +(cd "$W/localai"; go mod edit -modfile="$W/localai-smoke.mod" -replace="github.com/mudler/localrecall=$ROOT") +echo "consumer: go test -mod=mod -modfile=$W/localai-smoke.mod -count=1 -v ./core/services/agentpool -run ^TestLocalRecallRealConsumer$ (no build tags)" +sh test/raw/database.sh sh -c 'cd "$1/localai"; go test -mod=mod -modfile="$1/localai-smoke.mod" -count=1 -v -timeout=5m ./core/services/agentpool -run "^TestLocalRecallRealConsumer$"' sh "$W" diff --git a/test/raw/templates/localai_consumer_test.go.txt b/test/raw/templates/localai_consumer_test.go.txt new file mode 100644 index 0000000..91efa9d --- /dev/null +++ b/test/raw/templates/localai_consumer_test.go.txt @@ -0,0 +1,69 @@ +package agentpool + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mudler/LocalAGI/webui/collections" + "github.com/mudler/LocalAI/core/config" +) + +// This fixture is copied into the pinned actual consumer package. It uses +// the same collection construction as startDistributed, without starting +// unrelated agent schedulers, messaging or skills services. +func TestLocalRecallRealConsumer(t *testing.T) { + url, db := os.Getenv("LOCALRECALL_SMOKE_API"), os.Getenv("LOCALRECALL_TEST_DATABASE_URL") + if url == "" || db == "" || os.Getenv("LOCALRECALL_TEST_DATABASE_OWNER") == "" { + t.Fatal("owned provider/database required") + } + dir := t.TempDir() + svc, err := NewAgentPoolService(&config.ApplicationConfig{AgentPool: config.AgentPoolConfig{ + VectorEngine: "postgres", DatabaseURL: db, EmbeddingModel: "minilm", MaxChunkingSize: 400, + }}) + if err != nil { + t.Fatal(err) + } + cfg := svc.buildCollectionsConfig(url, "", filepath.Join(dir, "collections"), filepath.Join(dir, "assets")) + inner, state := collections.NewInProcessBackend(cfg) + svc.collectionsBackend = svc.shareCollections(inner, state, cfg) + defer func() { + svc.collectionsBackend.(*sharedCollections).Close() + state.SourceManager.Stop() + for _, kb := range state.Collections { + if kb != nil { + kb.Close() + } + } + }() + const name = "real-consumer" + if err := svc.CreateCollectionForUser("", name); err != nil { + t.Fatal(err) + } + const text = "The observatory telescope studies distant galaxies and bright stars." + entry, err := svc.UploadToCollectionForUser("", name, "astronomy.txt", strings.NewReader(text)) + if err != nil { + t.Fatal(err) + } + entries, err := svc.ListCollectionEntriesForUser("", name) + if err != nil || len(entries) != 1 { + t.Fatalf("entries=%v err=%v", entries, err) + } + content, chunks, err := svc.GetCollectionEntryContentForUser("", name, entry) + if err != nil || !strings.Contains(content, "galaxies") || chunks < 1 { + t.Fatalf("content=%q chunks=%d err=%v", content, chunks, err) + } + results, err := svc.SearchCollectionForUser("", name, "telescope galaxies", 3) + if err != nil || len(results) == 0 || !strings.Contains(results[0].Content, "galaxies") { + t.Fatalf("search=%v err=%v", results, err) + } + if err := svc.ResetCollectionForUser("", name); err != nil { + t.Fatal(err) + } + entries, err = svc.ListCollectionEntriesForUser("", name) + if err != nil || len(entries) != 0 { + t.Fatalf("reset entries=%v err=%v", entries, err) + } + t.Log("actual agentpool construction, upload, entry, real-embedding search and reset passed") +} From 5e2d37c1990549b9f7cf7087d5ee2cf545d0232d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 11 Oct 2026 23:36:21 +0000 Subject: [PATCH 11/14] test(raw): gate owned legacy and consumer fixtures Run legacy regression suites with an owned database and genuine model. Keep raw isolation tests separate from legacy database mutations. Exercise private source ingestion and discovery from fresh processes. Count real consumer embeddings while raw operations share its database. Add a strict all runner with repeated race tests and vet. Document deployment, consistency limits, and evidence boundaries. Validation: sh test/raw/verify.sh all passes with the supplied native backend and library paths. Optional HTTP/2 and additional outage regressions remain outside this bounded validation milestone. --- README.md | 6 + docs/raw-vector-api.md | 110 ++++++++++++++++++ rag/engine/postgres_planning_test.go | 1 + rag/engine/postgres_pool_test.go | 27 +++-- rag/engine/postgres_test.go | 5 +- rag/raw/postgres/process_test.go | 48 ++++++++ rag/sources/router_test.go | 20 +++- test/e2e/raw_legacy_owned_test.go | 108 +++++++++++++++++ test/integration/postgres_integration_test.go | 5 +- test/raw/legacy.sh | 43 +++++++ test/raw/localai-smoke.sh | 11 +- .../templates/localai_consumer_test.go.txt | 57 +++++++++ test/raw/verify.sh | 18 ++- 13 files changed, 443 insertions(+), 16 deletions(-) create mode 100644 docs/raw-vector-api.md create mode 100644 rag/raw/postgres/process_test.go create mode 100644 test/e2e/raw_legacy_owned_test.go create mode 100644 test/raw/legacy.sh diff --git a/README.md b/README.md index dc37385..601ceca 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,12 @@ Currently, LocalRecall is batteries included and supports multiple vector databa It can easily integrate with LocalAI, LocalAGI, and other agent frameworks, offering an intuitive web UI for convenient file management, including support for raw text inputs. +## Caller-supplied vectors + +The PostgreSQL-only `/api/raw/v1` API stores caller-supplied vectors without invoking an embedding model. +It supports vector, BM25, and hybrid retrieval in a namespace separate from legacy collections. +See the [raw vector API guide](docs/raw-vector-api.md) for deployment, request examples, consistency limits, and verification. + ## 📚🆕 Local Stack Family 🆕 LocalAI is now part of a comprehensive suite of AI tools designed to work together: diff --git a/docs/raw-vector-api.md b/docs/raw-vector-api.md new file mode 100644 index 0000000..da209b5 --- /dev/null +++ b/docs/raw-vector-api.md @@ -0,0 +1,110 @@ +# Raw vector API + +The `/api/raw/v1` API stores vectors supplied by your application. It requires PostgreSQL. Legacy collection routes and Go interfaces remain separate. + +LocalRecall does not embed raw records, fetch media, or manage source assets. Your application owns embedding models, chunking, asset storage, and publication. + +## Deployment + +Set `VECTOR_ENGINE=postgres` and `DATABASE_URL`. Install `vector` and `pg_textsearch` through your database administrator. The tested BM25 extension version is 1.3.1. + +The service role needs schema creation, table creation, index creation, and DML privileges for its raw registry and tables. Collection deletion requires ownership privileges. +Raw startup does not repair unknown schemas or corrupt tables. Back up the registry and collection tables together. + +Raw uses one shared pool per service. The default maximum is four connections. `POSTGRES_POOL_MAX_CONNS` changes this maximum; a DSN `pool_max_conns` takes precedence. +Legacy PostgreSQL collections retain their separate pools. Budget approximately `raw pool maximum + loaded legacy collections × legacy pool maximum` per process. +Include administrative connections and replicas in your database budget. + +Raw operations have a 30-second deadline and a five-second database lock timeout. Server shutdown drains HTTP requests, cancels outstanding work, and closes the raw pool. + +## Authentication and readiness + +Raw routes use the existing `API_KEYS` authentication boundary. Keys do not provide collection-level authorization or tenant isolation. +Treat metadata filters as query predicates, not access controls. Use a trusted application gateway when users require different permissions. + +Read `GET /api/raw/v1/capabilities` before selecting a search mode. Implementation support and current readiness are separate fields. +A core database outage produces unavailable responses. Missing BM25 or ANN support disables dependent operations rather than silently changing retrieval semantics. +Non-PostgreSQL raw requests return HTTP 501 before mutation. Legacy backends remain available independently. + +Errors distinguish invalid requests (400), authentication failures (401), missing collections (404), conflicts (409), oversized bodies (413), unsupported operations (501), and unavailable dependencies (503). + +## Create and populate a collection + +Use your deployment address and authentication header in these requests: + +```http +POST /api/raw/v1/collections +Content-Type: application/json + +{"name":"catalog-v1","dimensions":3,"metric":"cosine","embedding_space_id":"example-v1"} +``` + +The response contains a UUID `collection_id`. Raw names occupy a separate namespace from legacy names. +Dimensions, embedding space, metric, and lexical configuration are immutable. Dimensions range from 1 to 2000. Only cosine similarity is supported. +Lexical indexing defaults to enabled. Its configuration captures `BM25_TEXT_CONFIG` at creation, or `english` when unset. + +```http +POST /api/raw/v1/collections/COLLECTION_ID/records/upsert +Content-Type: application/json + +{"embedding_space_id":"example-v1","records":[{"id":"item-1","vector":[1,0,0],"text":"An astronomy telescope","metadata":{"category":"science"}}]} +``` + +Vectors must contain finite values and a nonzero norm. LocalRecall normalizes vectors for storage. +Each upsert replaces the entire record. Omitted text clears text; omitted metadata becomes an empty object. +A batch contains at most 100 unique IDs. The whole batch commits atomically. Request bodies cannot exceed 8 MiB. + +## Retrieve records + +```http +POST /api/raw/v1/collections/COLLECTION_ID/search +Content-Type: application/json + +{"mode":"vector","accuracy":"exact","embedding_space_id":"example-v1","vector":[1,0,0],"top_k":10,"filter":{"metadata_eq":{"category":"science"}}} +``` + +Choose `vector`, `lexical`, or `hybrid` explicitly. Lexical queries supply `text` instead of a vector. Hybrid queries supply both. +Vector-bearing searches default to `approximate`. Use `exact` when you require exhaustive filtered ranking. +Approximate search can return fewer results than requested, especially with selective filters. It does not guarantee recall. + +Hybrid search combines independent ranks with equal-weight reciprocal rank fusion. Each contribution is `0.5 / (60 + rank)`. +Each retrieval branch considers `min(1000, max(100, 10 * top_k))` candidates. Responses expose component ranks and fusion parameters. +Raw lexical retrieval never substitutes native full-text search for BM25. + +Additional routes: + +| Method | Collection-relative path | Purpose | +| --- | --- | --- | +| POST | `/records/get` | Fetch 1–100 IDs; report missing IDs | +| POST | `/records/list` | Traverse records with a keyset cursor | +| POST | `/records/delete` | Delete IDs or a bounded metadata selection | +| POST | `/reset` | Delete all records transactionally | +| DELETE | collection URL | Delete the descriptor and table atomically | + +List pages default to 50 records and allow at most 100. Cursors bind to collection, filters, and projection. +Cursors do not provide snapshots across concurrent writes. Filter deletion rejects more than 1000 matches without partial deletion. + +## Retries and migrations + +Writes use last-committed-write-wins semantics. Retrying an older batch can overwrite a newer record. +After an uncertain commit, read the affected IDs before deciding whether to retry. The API has no revision locks or idempotency receipts. + +Reset is a transactional deletion, not a barrier against concurrent writers. Stop your writers when you require an empty collection after reset. + +For an incompatible embedding space, create a new collection. Populate and validate it before switching your application's active collection reference. +Your application owns that switch. Keep the old collection until in-flight readers finish, then delete it explicitly. +LocalRecall does not activate generations or automatically remove stale collections. + +## Verification and evidence limits + +Run `sh test/raw/verify.sh all` from the worktree root after provisioning the [genuine consumer dependencies](../test/raw/LOCALAI-SMOKE.md). +Missing dependencies, failed suites, and skipped required tests fail this command. Logs and private artifacts stay under `.sdd-work`. +The runner uses disposable databases and owned loopback processes, not existing host services. + +The synthetic retrieval fixture contains 600 three-dimensional vectors with a selective 1-in-17 filter. Logs record EXPLAIN, recall@10, and underfill. +These measurements test query behavior; they do not predict production latency or embedding quality. +The legacy planning fixture uses 2000 eight-dimensional synthetic vectors. Its plan checks vector index ordering. + +The real consumer fixture uses pinned LocalAI source and genuine 384-dimensional MiniLM text embeddings. Provider identity differs from consumer identity. +Its forwarding counter measures actual consumer embedding requests and checks zero additional requests during raw operations. +This evidence does not establish multimodal quality, large-corpus throughput, or production resource limits. diff --git a/rag/engine/postgres_planning_test.go b/rag/engine/postgres_planning_test.go index a01c2cf..2652cc1 100644 --- a/rag/engine/postgres_planning_test.go +++ b/rag/engine/postgres_planning_test.go @@ -51,6 +51,7 @@ var _ = Describe("hybrid search query planning (LocalAI issue #10186)", func() { collectionName: collectionName, tableName: sanitizeTableName(collectionName), embeddingDims: dims, + bm25TextConfig: "english", bm25Weight: 0.5, vectorWeight: 0.5, } diff --git a/rag/engine/postgres_pool_test.go b/rag/engine/postgres_pool_test.go index 1c4514a..482ac67 100644 --- a/rag/engine/postgres_pool_test.go +++ b/rag/engine/postgres_pool_test.go @@ -178,8 +178,8 @@ var _ = Describe("NewPostgresDBCollection pool lifecycle", func() { }, 5*time.Second, 100*time.Millisecond).Should(Equal(0)) }) - // Without the BM25 extension the schema setup fails after the pool - // is open and pinged. That pool must be closed too. + // An invalid lexical configuration fails schema setup after the pool + // is open and pinged, even when BM25 is installed. It("keeps no server connection open after a failed schema setup", func() { client, _, stop := newEmbedder(false) defer stop() @@ -187,12 +187,23 @@ var _ = Describe("NewPostgresDBCollection pool lifecycle", func() { appName := fmt.Sprintf("localrecall_leak_test_%d", time.Now().UnixNano()) url := withApplicationName(databaseURL, appName) - db, err := NewPostgresDBCollection(fmt.Sprintf("leak_test_%d", time.Now().UnixNano()), url, client, "fake") - if err == nil { - _, _ = db.pool.Exec(context.Background(), "DROP TABLE IF EXISTS "+db.tableName+" CASCADE") - db.Close() - Skip("schema setup succeeds on this database; the failure path cannot be exercised") - } + oldConfig, hadConfig := os.LookupEnv("BM25_TEXT_CONFIG") + Expect(os.Setenv("BM25_TEXT_CONFIG", "lr_nonexistent_text_config")).To(Succeed()) + defer func() { + if hadConfig { + _ = os.Setenv("BM25_TEXT_CONFIG", oldConfig) + } else { + _ = os.Unsetenv("BM25_TEXT_CONFIG") + } + }() + name := fmt.Sprintf("leak_test_%d", time.Now().UnixNano()) + adminCleanup, err := pgxpool.New(context.Background(), databaseURL) + Expect(err).NotTo(HaveOccurred()) + defer adminCleanup.Close() + defer func() { + _, _ = adminCleanup.Exec(context.Background(), "DROP TABLE IF EXISTS "+sanitizeTableName(name)+" CASCADE") + }() + _, err = NewPostgresDBCollection(name, url, client, "fake") Expect(err).To(MatchError(ContainSubstring("failed to setup database"))) pools := rec.all() Expect(pools).To(HaveLen(1)) diff --git a/rag/engine/postgres_test.go b/rag/engine/postgres_test.go index 82546ee..f8f8f0f 100644 --- a/rag/engine/postgres_test.go +++ b/rag/engine/postgres_test.go @@ -59,7 +59,10 @@ var _ = Describe("PostgresDB", func() { // Use PostgreSQL from docker-compose // Connection string for docker-compose postgres service - databaseURL = "postgresql://localrecall:localrecall@localhost:5432/localrecall?sslmode=disable" + databaseURL = os.Getenv("LOCALRECALL_TEST_DATABASE_URL") + if databaseURL == "" { + databaseURL = "postgresql://localrecall:localrecall@localhost:5432/localrecall?sslmode=disable" + } // Verify PostgreSQL is accessible ctx := context.Background() diff --git a/rag/raw/postgres/process_test.go b/rag/raw/postgres/process_test.go new file mode 100644 index 0000000..20c6b01 --- /dev/null +++ b/rag/raw/postgres/process_test.go @@ -0,0 +1,48 @@ +package postgres + +import ( + "context" + "os" + osexec "os/exec" + "testing" + "time" + + "github.com/mudler/localrecall/rag/raw" +) + +// Re-exec the test binary, not another Service in the creator process. +func TestRawProcessDiscovery(t *testing.T) { + if id := os.Getenv("LOCALRECALL_RAW_CHILD_ID"); id != "" { + s, ctx := fixture(t) + c, err := s.GetCollection(ctx, id) + if err != nil || c.CollectionID != id { + t.Fatalf("discover: %+v %v", c, err) + } + page, err := s.GetRecords(ctx, id, raw.GetRecordsRequest{IDs: []string{"persisted"}}) + if err != nil || len(page.Records) != 1 { + t.Fatalf("persisted records: %+v %v", page, err) + } + return + } + s, ctx := fixture(t) + c, _, err := s.CreateCollection(ctx, request()) + if err != nil { + t.Fatal(err) + } + defer s.DeleteCollection(ctx, c.CollectionID) + _, err = s.Upsert(ctx, c.CollectionID, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{{ID: "persisted", Vector: []float64{1, 0, 0}}}}) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + childCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + cmd := osexec.CommandContext(childCtx, os.Args[0], "-test.run=^TestRawProcessDiscovery$", "-test.v") + cmd.Env = append(os.Environ(), "LOCALRECALL_RAW_CHILD_ID="+c.CollectionID) + output, err := cmd.CombinedOutput() + cancel() + if err != nil { + t.Fatalf("child %d: %v\n%s", i, err, output) + } + t.Logf("fresh process %d: %s", i, output) + } +} diff --git a/rag/sources/router_test.go b/rag/sources/router_test.go index 49725ac..e1b9b4b 100644 --- a/rag/sources/router_test.go +++ b/rag/sources/router_test.go @@ -4,13 +4,27 @@ import ( . "github.com/mudler/localrecall/rag/sources" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "net/http" + "net/http/httptest" ) var _ = Describe("SourceRouter", func() { + var server *httptest.Server + BeforeEach(func() { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/sitemap.xml" { + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte("")) + return + } + http.NotFound(w, r) + })) + }) + AfterEach(func() { server.Close() }) It("should identify git repository URLs", func() { config := &Config{} // Note: This will fail if git is not available, but tests the routing logic - _, err := SourceRouter("https://example.com/repo.git", config) + _, err := SourceRouter(server.URL+"/repo.git", config) // We expect an error since we're not testing actual git functionality // Just verify it's trying to route to git Expect(err).ToNot(BeNil()) // Git operation will fail, but routing works @@ -19,7 +33,7 @@ var _ = Describe("SourceRouter", func() { It("should identify sitemap URLs", func() { config := &Config{} // This will fail if network is not available, but tests routing - _, err := SourceRouter("https://example.com/sitemap.xml", config) + _, err := SourceRouter(server.URL+"/sitemap.xml", config) // We expect an error since we're not testing actual network functionality Expect(err).ToNot(BeNil()) // Network operation will fail, but routing works }) @@ -27,7 +41,7 @@ var _ = Describe("SourceRouter", func() { It("should default to web page for regular URLs", func() { config := &Config{} // This may succeed or fail depending on network, but tests routing - _, err := SourceRouter("https://example.com/page", config) + _, err := SourceRouter(server.URL+"/page", config) // The function may succeed or fail, but should not panic // We just verify the routing logic works (no panic) _ = err // Accept either success or failure diff --git a/test/e2e/raw_legacy_owned_test.go b/test/e2e/raw_legacy_owned_test.go new file mode 100644 index 0000000..f0c76f0 --- /dev/null +++ b/test/e2e/raw_legacy_owned_test.go @@ -0,0 +1,108 @@ +package e2e_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/mudler/localrecall/pkg/client" +) + +// This release fixture uses the real server, database and embedding provider. +// Source content is private and deterministic; no public website is fetched. +func TestOwnedLegacyLifecycle(t *testing.T) { + base := os.Getenv("LOCALRECALL_ENDPOINT") + if base == "" || os.Getenv("LOCALRECALL_TEST_DATABASE_OWNER") == "" { + t.Fatal("owned legacy runner required") + } + c := client.NewClient(base) + name := fmt.Sprintf("owned-%d", time.Now().UnixNano()) + if err := c.CreateCollection(name); err != nil { + t.Fatal(err) + } + text := "The observatory telescope studies distant galaxies and bright stars." + file := filepath.Join(t.TempDir(), "astronomy.txt") + if err := os.WriteFile(file, []byte(text), 0600); err != nil { + t.Fatal(err) + } + if _, err := c.Store(name, file); err != nil { + t.Fatal(err) + } + entries, err := c.ListEntries(name) + if err != nil || len(entries) != 1 { + t.Fatalf("entries %v %v", entries, err) + } + chunks, err := c.GetEntryContent(name, entries[0]) + if err != nil || len(chunks) == 0 || !strings.Contains(chunks[0].Content, "galaxies") { + t.Fatalf("content %v %v", chunks, err) + } + results, err := c.Search(name, "telescope galaxies", 3) + if err != nil || len(results) == 0 || !strings.Contains(results[0].Content, "galaxies") { + t.Fatalf("search %v %v", results, err) + } + if _, err := c.DeleteEntry(name, entries[0]); err != nil { + t.Fatal(err) + } + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "
Private source astronomy observatory telescope galaxies.
") + })) + defer src.Close() + request := func(method string, body any) []byte { + t.Helper() + data, _ := json.Marshal(body) + req, err := http.NewRequest(method, base+"/api/collections/"+name+"/sources", bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil || resp.StatusCode != 200 { + t.Fatalf("source %s: %d %s %v", method, resp.StatusCode, b, err) + } + return b + } + request("POST", map[string]any{"url": src.URL, "update_interval": 60}) + if !bytes.Contains(request("GET", nil), []byte(src.URL)) { + t.Fatal("source not listed") + } + deadline := time.Now().Add(time.Minute) + for { + entries, err = c.ListEntries(name) + if err == nil && len(entries) == 1 { + break + } + if time.Now().After(deadline) { + t.Fatalf("source never indexed: %v %v", entries, err) + } + time.Sleep(100 * time.Millisecond) + } + chunks, err = c.GetEntryContent(name, entries[0]) + if err != nil || len(chunks) == 0 || !strings.Contains(chunks[0].Content, "Private source") { + t.Fatalf("source content %v %v", chunks, err) + } + request("DELETE", map[string]string{"url": src.URL}) + if bytes.Contains(request("GET", nil), []byte(src.URL)) { + t.Fatal("source not removed") + } + if err = c.Reset(name); err != nil { + t.Fatal(err) + } + entries, err = c.ListEntries(name) + if err != nil || len(entries) != 0 { + t.Fatalf("reset %v %v", entries, err) + } + t.Log("real text upload/search/entry/delete/reset and private HTTP source fetch/list/remove passed") +} diff --git a/test/integration/postgres_integration_test.go b/test/integration/postgres_integration_test.go index 112226c..72f5af6 100644 --- a/test/integration/postgres_integration_test.go +++ b/test/integration/postgres_integration_test.go @@ -58,7 +58,10 @@ var _ = Describe("PostgreSQL Integration", func() { openaiClient = openai.NewClientWithConfig(config) // Use PostgreSQL from docker-compose - databaseURL = "postgresql://localrecall:localrecall@localhost:5432/localrecall?sslmode=disable" + databaseURL = os.Getenv("LOCALRECALL_TEST_DATABASE_URL") + if databaseURL == "" { + databaseURL = "postgresql://localrecall:localrecall@localhost:5432/localrecall?sslmode=disable" + } // Verify PostgreSQL is accessible ctx := context.Background() diff --git a/test/raw/legacy.sh b/test/raw/legacy.sh new file mode 100644 index 0000000..fa6e496 --- /dev/null +++ b/test/raw/legacy.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Never run legacy destructive suites against an externally selected service. +set -eu +if [ "${1:-}" != --owned ]; then + [ "$#" -eq 0 ] || exit 2 + [ -z "${LOCALRECALL_TEST_DATABASE_URL:-}" ] || exit 1 + exec sh test/raw/localai-smoke.sh sh test/raw/database.sh sh test/raw/legacy.sh --owned +fi +: "${LOCALRECALL_TEST_DATABASE_OWNER:?owned database required}" +: "${LOCALAI_ENDPOINT:?ready genuine model required}" +export POSTGRES_TEST_URL="$LOCALRECALL_TEST_DATABASE_URL" +export COLLECTION_DB_PATH="$PWD/.sdd-work/legacy/$LOCALRECALL_TEST_DATABASE_OWNER/collections" FILE_ASSETS="$PWD/.sdd-work/legacy/$LOCALRECALL_TEST_DATABASE_OWNER/assets" +mkdir -p "$COLLECTION_DB_PATH" "$FILE_ASSETS" +log="$PWD/.sdd-work/logs/legacy.log" +if go test -v -mod=readonly -race -count=1 -p=1 -timeout=15m ./rag ./rag/engine ./rag/sources ./pkg/chunk ./pkg/client . ./test/integration >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi +# Ginkgo's summary includes '0 Skipped'; reject nonzero counts, too. +! grep -E -- '--- SKIP:|[1-9][0-9]* Skipped|no tests to run|no test files' "$log" +for suite in TestRag TestEngine TestChunk TestIntegration TestCreateCollectionReusesLoadedCollection; do + grep -q -- "--- PASS: $suite" "$log" +done +# Launch an owned real API on a dynamically allocated loopback port. +go build -mod=readonly -o .sdd-work/legacy/localrecall . +port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])') +export LOCALRECALL_ENDPOINT="http://127.0.0.1:$port" LISTENING_ADDRESS="127.0.0.1:$port" +export DATABASE_URL="$LOCALRECALL_TEST_DATABASE_URL" VECTOR_ENGINE=postgres EMBEDDING_MODEL=minilm OPENAI_BASE_URL="$LOCALAI_ENDPOINT" +.sdd-work/legacy/localrecall >.sdd-work/logs/legacy-server.log 2>&1 & +pid=$! +trap 'kill "$pid" 2>/dev/null || true; wait "$pid" 2>/dev/null || true' EXIT HUP INT TERM +python3 - "$LOCALRECALL_ENDPOINT" "$pid" <<'PY' +import urllib.request,time,sys,os +for i in range(60): + os.kill(int(sys.argv[2]),0) + try: + with urllib.request.urlopen(sys.argv[1]+'/api/collections',timeout=2) as r: assert r.status==200 + break + except Exception: + if i==59: raise + time.sleep(1) +PY +log="$PWD/.sdd-work/logs/legacy-e2e.log" +if go test -v -mod=readonly -race -count=1 -timeout=5m ./test/e2e -run '^TestOwnedLegacyLifecycle$' >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi +grep -q -- '--- PASS: TestOwnedLegacyLifecycle' "$log" +! grep -E -- '--- SKIP:|no tests to run|no test files' "$log" diff --git a/test/raw/localai-smoke.sh b/test/raw/localai-smoke.sh index 2aafa88..edad64f 100644 --- a/test/raw/localai-smoke.sh +++ b/test/raw/localai-smoke.sh @@ -59,8 +59,7 @@ for x in s: x.bind(('127.0.0.1',0)) print(*(x.getsockname()[1] for x in s)) PY ) -set -- $ports -backend_port=$1; api_port=$2 +backend_port=${ports%% *}; api_port=${ports##* } cat > "$W/model-smoke/minilm.yaml" < "$W/model-smoke/legacy.yaml" (cd "$W/model-smoke"; exec "$LOCALAI_SMOKE_BACKEND" --addr "127.0.0.1:$backend_port") > "$W/logs/localai-native-smoke.log" 2>&1 & backend_pid=$! (cd "$W/model-smoke"; exec ./local-ai run --address "127.0.0.1:$api_port" --models-path "$W/model-smoke" --backends-path "$W/model-smoke/backends" --backends-system-path "$W/model-smoke/system" --data-path "$W/model-smoke/data" --localai-config-dir "$W/model-smoke/config" --external-grpc-backends "smoke-llama:127.0.0.1:$backend_port" --disable-web-ui --disable-agents) > "$W/logs/localai-provider-smoke.log" 2>&1 & @@ -88,6 +88,13 @@ for i in range(90): if i==89: raise time.sleep(1) PY +# An optional command runs only after a genuine model response, and remains +# inside this script's owned provider lifetime. No externally supplied endpoint. +if [ "$#" -gt 0 ]; then + export LOCALAI_ENDPOINT="$LOCALRECALL_SMOKE_API" + "$@" + exit $? +fi cp test/raw/templates/localai_consumer_test.go.txt "$W/localai/core/services/agentpool/localrecall_smoke_test.go" # Preserve upstream mod/sum; resolve only in private alternate files. cp "$W/localai/go.mod" "$W/localai-smoke.mod" diff --git a/test/raw/templates/localai_consumer_test.go.txt b/test/raw/templates/localai_consumer_test.go.txt index 91efa9d..d915c99 100644 --- a/test/raw/templates/localai_consumer_test.go.txt +++ b/test/raw/templates/localai_consumer_test.go.txt @@ -1,9 +1,17 @@ package agentpool import ( + "context" + "github.com/mudler/localrecall/rag/raw" + rawpostgres "github.com/mudler/localrecall/rag/raw/postgres" + "net/http" + "net/http/httptest" + "net/http/httputil" + urlpkg "net/url" "os" "path/filepath" "strings" + "sync/atomic" "testing" "github.com/mudler/LocalAGI/webui/collections" @@ -18,6 +26,20 @@ func TestLocalRecallRealConsumer(t *testing.T) { if url == "" || db == "" || os.Getenv("LOCALRECALL_TEST_DATABASE_OWNER") == "" { t.Fatal("owned provider/database required") } + target, err := urlpkg.Parse(url) + if err != nil { + t.Fatal(err) + } + var calls atomic.Int64 + proxy := httputil.NewSingleHostReverseProxy(target) + counting := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/embeddings") { + calls.Add(1) + } + proxy.ServeHTTP(w, r) + })) + defer counting.Close() + url = counting.URL dir := t.TempDir() svc, err := NewAgentPoolService(&config.ApplicationConfig{AgentPool: config.AgentPoolConfig{ VectorEngine: "postgres", DatabaseURL: db, EmbeddingModel: "minilm", MaxChunkingSize: 400, @@ -58,6 +80,41 @@ func TestLocalRecallRealConsumer(t *testing.T) { if err != nil || len(results) == 0 || !strings.Contains(results[0].Content, "galaxies") { t.Fatalf("search=%v err=%v", results, err) } + if calls.Load() == 0 { + t.Fatal("consumer bypassed embedding counter") + } + before := calls.Load() + ctx := context.Background() + store, err := rawpostgres.New(ctx, rawpostgres.Config{DatabaseURL: db}) + if err != nil { + t.Fatal(err) + } + defer store.Close() + rc, _, err := store.CreateCollection(ctx, raw.CreateCollectionRequest{Name: name, Dimensions: 3, Metric: "cosine", EmbeddingSpaceID: "synthetic-v1"}) + if err != nil { + t.Fatal(err) + } + defer store.DeleteCollection(ctx, rc.CollectionID) + if _, err = store.Upsert(ctx, rc.CollectionID, raw.UpsertRequest{EmbeddingSpaceID: "synthetic-v1", Records: []raw.Record{{ID: "raw", Vector: []float64{1, 0, 0}}}}); err != nil { + t.Fatal(err) + } + space, accuracy := "synthetic-v1", "exact" + if _, err = store.Search(ctx, rc.CollectionID, raw.SearchRequest{Mode: "vector", Accuracy: &accuracy, EmbeddingSpaceID: &space, Vector: []float64{1, 0, 0}}); err != nil { + t.Fatal(err) + } + if _, err = store.GetRecords(ctx, rc.CollectionID, raw.GetRecordsRequest{IDs: []string{"raw"}}); err != nil { + t.Fatal(err) + } + if err = store.Reset(ctx, rc.CollectionID); err != nil { + t.Fatal(err) + } + if err = store.DeleteCollection(ctx, rc.CollectionID); err != nil { + t.Fatal(err) + } + if calls.Load() != before { + t.Fatalf("raw embedding traffic: before=%d after=%d", before, calls.Load()) + } + t.Logf("consumer embedding calls=%d; raw lifecycle/search attributed calls=0", before) if err := svc.ResetCollectionForUser("", name); err != nil { t.Fatal(err) } diff --git a/test/raw/verify.sh b/test/raw/verify.sh index 76b0a65..b2a879b 100644 --- a/test/raw/verify.sh +++ b/test/raw/verify.sh @@ -13,6 +13,22 @@ export COLLECTION_DB_PATH="$ROOT/.sdd-work/collections" FILE_ASSETS="$ROOT/.sdd- export GOENV=off GOTOOLCHAIN=local mkdir -p "$TMPDIR" "$GOCACHE" "$GOMODCACHE" "$GOPATH" "$COLLECTION_DB_PATH" "$FILE_ASSETS" "$ROOT/.sdd-work/logs" case "${1:-}" in + all) + sh test/raw/verify.sh api + log="$ROOT/.sdd-work/logs/race10.log" + if sh test/raw/database.sh go test -v -mod=readonly -race -count=10 -timeout=30m ./rag/raw/... ./pkg/client/... >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi + ! grep -E 'SKIP|no tests to run|no test files' "$log" + for suite in TestRawProcessDiscovery TestRawRecordsReplacement TestRawClient; do + [ "$(grep -c -- "^--- PASS: $suite (" "$log")" -eq 10 ] + done + go vet -mod=readonly ./rag/raw/... ./pkg/client/... + sh test/raw/legacy.sh + log="$ROOT/.sdd-work/logs/consumer-all.log" + if sh test/raw/localai-smoke.sh >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi + grep -q -- '--- PASS: TestLocalRecallRealConsumer' "$log" + ! grep -E 'SKIP|no tests to run|no test files' "$log" + git diff --check + ;; api) sh test/raw/verify.sh search log="$ROOT/.sdd-work/logs/api-$(date +%s)-$$.log" @@ -62,5 +78,5 @@ case "${1:-}" in [ "$(grep -c '^ok[[:space:]]' "$log")" -eq 2 ] && ! grep -E '\[no test files\]|\[no tests to run\]' "$log" git diff --check ;; - *) echo 'usage: sh test/raw/verify.sh contracts|lifecycle|records|search|api' >&2; exit 2;; + *) echo 'usage: sh test/raw/verify.sh contracts|lifecycle|records|search|api|all' >&2; exit 2;; esac From 08d20674894e33a0b0e83a7eb94e9ed740e30439 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 11 Oct 2026 23:50:46 +0000 Subject: [PATCH 12/14] test(raw): verify owned outage and consumer overlap Keep the owned fixture alive across PostgreSQL stop/start and preserve BM25 preload settings. Assert bounded failure, retained vectors, recovery, and cancellation with released database sessions. Reject escaping outputs before runner writes. Synchronize real consumer embedding traffic with raw work and record pool and catalog measurements. The full acceptance gate passes, including ten race repetitions. Afterward, stronger vector-retention and readiness assertions pass the focused outage and shutdown tests. Document observed metrics and limits. --- docs/raw-vector-api.md | 29 ++- rag/raw/postgres/operations_test.go | 166 ++++++++++++++++++ test/raw/LOCALAI-SMOKE.md | 12 +- test/raw/confinement.py | 25 +++ test/raw/confinement_test.py | 45 +++++ test/raw/database.sh | 21 ++- test/raw/legacy.sh | 1 + test/raw/localai-smoke.sh | 3 +- .../templates/localai_consumer_test.go.txt | 124 ++++++++++++- test/raw/verify.sh | 4 +- 10 files changed, 418 insertions(+), 12 deletions(-) create mode 100644 rag/raw/postgres/operations_test.go create mode 100644 test/raw/confinement.py create mode 100644 test/raw/confinement_test.py diff --git a/docs/raw-vector-api.md b/docs/raw-vector-api.md index da209b5..d7f2885 100644 --- a/docs/raw-vector-api.md +++ b/docs/raw-vector-api.md @@ -19,7 +19,7 @@ Raw operations have a 30-second deadline and a five-second database lock timeout ## Authentication and readiness -Raw routes use the existing `API_KEYS` authentication boundary. Keys do not provide collection-level authorization or tenant isolation. +Raw routes use the existing `API_KEYS` authentication boundary. **When `API_KEYS` is unset, routes are unauthenticated. Use this mode only on a private, trusted network; do not expose it publicly.** Keys do not provide collection-level authorization or tenant isolation. Treat metadata filters as query predicates, not access controls. Use a trusted application gateway when users require different permissions. Read `GET /api/raw/v1/capabilities` before selecting a search mode. Implementation support and current readiness are separate fields. @@ -108,3 +108,30 @@ The legacy planning fixture uses 2000 eight-dimensional synthetic vectors. Its p The real consumer fixture uses pinned LocalAI source and genuine 384-dimensional MiniLM text embeddings. Provider identity differs from consumer identity. Its forwarding counter measures actual consumer embedding requests and checks zero additional requests during raw operations. This evidence does not establish multimodal quality, large-corpus throughput, or production resource limits. + +### Recorded acceptance sample + +The owned attempt5 full gate passed, including ten race-enabled repetitions of +outage recovery and active-query shutdown. An actual `pg_ctl stop/start` retained +the vectors; unavailable operations returned the raw unavailable error (HTTP 503), +and readiness became false. The supervisor keeps private tmpfs alive, and restart +preserves the required BM25 preload. Shutdown canceled a database-lock waiter and +released all sessions and pool connections. + +The real consumer overlap sample held one embedding HTTP request open for +28.13 ms while three raw collections completed writes and exact searches. +The counter observed exactly one legacy request and zero additional raw requests. +Two legacy collections and one shared raw pool used three sampled database +sessions, excluding the observer connection; the configured pool ceiling was 12. +This is a sampled count, not a continuous peak measurement. Public catalog objects +numbered 8 before raw bootstrap, 30 with the overlap collections, and 18 after +removing them: 12 objects removed. The remainder includes shared raw schema and +the second legacy collection, not leaked per-collection indexes. +The raw lifecycle and overlap portion took 71.04 ms. These are local synthetic +measurements, not throughput or production sizing guarantees. + +The filtered 600-vector fixture returned 10 of 10 results from 36 eligible rows: +recall@10 was 1.00 and underfill was zero. Planner choice can affect this result; +it does not establish universal ANN recall. Full EXPLAIN JSON and timings are in +`.sdd-work/logs/full-attempt5.log`; repeated suites are in `race10.log`, legacy +coverage in `legacy.log`, and downstream evidence in `consumer-all.log`. diff --git a/rag/raw/postgres/operations_test.go b/rag/raw/postgres/operations_test.go new file mode 100644 index 0000000..1b1c317 --- /dev/null +++ b/rag/raw/postgres/operations_test.go @@ -0,0 +1,166 @@ +package postgres + +import ( + "context" + "os" + osexec "os/exec" + "strings" + "testing" + "time" + + "github.com/mudler/localrecall/rag/raw" +) + +// Never control a server selected only by its address. Match both fixture name +// and the random owner label before every administrative command. +func ownedPG(t *testing.T, args ...string) string { + t.Helper() + owner, name := os.Getenv("LOCALRECALL_TEST_DATABASE_OWNER"), os.Getenv("LOCALRECALL_TEST_DATABASE_CONTAINER") + if owner == "" || name != "lr-raw-"+owner { + t.Fatal("owned container identity required") + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + b, e := osexec.CommandContext(ctx, "docker", "inspect", "-f", `{{index .Config.Labels "localrecall.raw.owner"}}`, name).CombinedOutput() + if e != nil || strings.TrimSpace(string(b)) != owner { + t.Fatalf("owner check: %s %v", b, e) + } + b, e = osexec.CommandContext(ctx, "docker", append([]string{"exec", "-u", "postgres", name}, args...)...).CombinedOutput() + if e != nil { + t.Fatalf("owned postgres %v: %s %v", args, b, e) + } + return strings.TrimSpace(string(b)) +} + +func TestRawOwnedOutage(t *testing.T) { + s, ctx := fixture(t) + c, _, e := s.CreateCollection(ctx, request()) + if e != nil { + t.Fatal(e) + } + defer s.DeleteCollection(ctx, c.CollectionID) + put := raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{{ID: "retained", Vector: []float64{1, 0, 0}}}} + if _, e = s.Upsert(ctx, c.CollectionID, put); e != nil { + t.Fatal(e) + } + data := ownedPG(t, "printenv", "PGDATA") + ownedPG(t, "pg_ctl", "-D", data, "-w", "-t", "15", "stop", "-m", "fast") + restarted := false + defer func() { + if !restarted { + ownedPG(t, "pg_ctl", "-D", data, "-l", "/tmp/raw-restart.log", "-w", "-t", "15", "-o", "-c shared_preload_libraries=pg_textsearch", "start") + } + }() + start := time.Now() + limited, cancel := context.WithTimeout(ctx, 2*time.Second) + _, e = s.GetCollection(limited, c.CollectionID) + cancel() + checkCode(t, e, raw.CodeUnavailable) + limited, cancel = context.WithTimeout(ctx, 2*time.Second) + caps, e := s.Capabilities(limited) + cancel() + if e != nil || caps.Ready { + t.Fatalf("outage readiness %+v %v", caps, e) + } + if time.Since(start) > 5*time.Second { + t.Fatal("outage was not bounded") + } + ownedPG(t, "pg_ctl", "-D", data, "-l", "/tmp/raw-restart.log", "-w", "-t", "15", "-o", "-c shared_preload_libraries=pg_textsearch", "start") + restarted = true + caps, e = s.Capabilities(ctx) + if e != nil || !caps.Ready { + t.Fatalf("recovery %+v %v", caps, e) + } + got, e := s.GetRecords(ctx, c.CollectionID, raw.GetRecordsRequest{IDs: []string{"retained"}, IncludeVectors: true}) + if e != nil || len(got.Records) != 1 || len(got.Records[0].Vector) != 3 || got.Records[0].Vector[0] != 1 || got.Records[0].Vector[1] != 0 || got.Records[0].Vector[2] != 0 { + t.Fatalf("retention %+v %v", got, e) + } + put.Records[0].ID = "after" + if _, e = s.Upsert(ctx, c.CollectionID, put); e != nil { + t.Fatal(e) + } + space, accuracy := "space", "exact" + if _, e = s.Search(ctx, c.CollectionID, raw.SearchRequest{Mode: "vector", Accuracy: &accuracy, EmbeddingSpaceID: &space, Vector: []float64{1, 0, 0}}); e != nil { + t.Fatal(e) + } + if e = s.Reset(ctx, c.CollectionID); e != nil { + t.Fatal(e) + } + t.Logf("owned pg_ctl outage/recovery/retained record/CRUD/search elapsed=%s", time.Since(start)) +} + +func TestRawActiveShutdown(t *testing.T) { + admin, ctx := fixture(t) + url := os.Getenv("LOCALRECALL_TEST_DATABASE_URL") + "&application_name=raw_active_shutdown" + s, e := New(ctx, Config{url}) + if e != nil { + t.Fatal(e) + } + defer s.Close() + caps, e := s.Capabilities(ctx) + if e != nil || !caps.Ready { + t.Fatalf("shutdown fixture not ready: %+v %v", caps, e) + } + c, _, e := s.CreateCollection(ctx, request()) + if e != nil { + t.Fatal(e) + } + defer admin.DeleteCollection(ctx, c.CollectionID) + table, _ := tableName(c.CollectionID) + lock, e := admin.pool.Begin(ctx) + if e != nil { + t.Fatal(e) + } + defer lock.Rollback(ctx) + if _, e = lock.Exec(ctx, "LOCK TABLE "+table+" IN ACCESS EXCLUSIVE MODE"); e != nil { + t.Fatal(e) + } + result := make(chan error, 1) + go func() { + _, err := s.GetRecords(ctx, c.CollectionID, raw.GetRecordsRequest{IDs: []string{"held"}}) + result <- err + }() + deadline := time.Now().Add(3 * time.Second) + for { + var n int + e = admin.pool.QueryRow(ctx, `SELECT count(*) FROM pg_stat_activity WHERE application_name='raw_active_shutdown' AND wait_event_type='Lock'`).Scan(&n) + if e != nil { + t.Fatal(e) + } + if n > 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("query never blocked") + } + time.Sleep(10 * time.Millisecond) + } + start := time.Now() + done := make(chan struct{}) + go func() { s.Close(); close(done) }() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("close did not cancel active DB work") + } + checkCode(t, <-result, raw.CodeTimeout) + deadline = time.Now().Add(3 * time.Second) + for { + var n int + e = admin.pool.QueryRow(ctx, `SELECT count(*) FROM pg_stat_activity WHERE application_name='raw_active_shutdown'`).Scan(&n) + if e != nil { + t.Fatal(e) + } + if n == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("leaked sessions=%d", n) + } + time.Sleep(10 * time.Millisecond) + } + if s.pool.Stat().TotalConns() != 0 { + t.Fatal("pool not released") + } + t.Logf("active lock cancellation/drain=%s; sessions baseline=0 final=0 pool=0", time.Since(start)) +} diff --git a/test/raw/LOCALAI-SMOKE.md b/test/raw/LOCALAI-SMOKE.md index 8f1562d..16f1a48 100644 --- a/test/raw/LOCALAI-SMOKE.md +++ b/test/raw/LOCALAI-SMOKE.md @@ -58,9 +58,8 @@ LocalRecall with this worktree; neither source go.mod/go.sum is edited. The successful run used PostgreSQL HNSW fallback (vectorscale was unavailable). It verifies text embedding compatibility, not EmbeddingGemma2 or multimodal -quality. It does not complete Task6: concurrent raw zero-embedding accounting, -full legacy suites, cross-process/restart regressions and the all-runner remain -separate required gates. +quality. The full runner additionally requires concurrent raw zero-embedding accounting, +legacy suites, cross-process/restart regressions and shutdown cleanup. ## Evidence @@ -70,3 +69,10 @@ that one-line correction, `.sdd-work/logs/localai-real-attempt3.log` reports `TestLocalRecallRealConsumer` PASS and owned database removal. The earlier compilation-only evidence is `localai-build-generated-attempt1.log`; it is not counted as a runtime test. + +Attempt5 full acceptance (`.sdd-work/logs/full-attempt5.log`) passed these gates. +The synchronized consumer test held one embedding request open during raw work: +28.13 ms overlap, exactly one legacy call, zero extra raw calls, three raw and two +legacy collections. It sampled three combined database sessions against a ceiling +of 12, excluding its observer. Catalog objects changed 8 → 30 → 18, with 12 +removed on deletion. See `docs/raw-vector-api.md` for measurement limits. diff --git a/test/raw/confinement.py b/test/raw/confinement.py new file mode 100644 index 0000000..35dab25 --- /dev/null +++ b/test/raw/confinement.py @@ -0,0 +1,25 @@ +"""Validate output trees before writes or cleanup; native inputs are read-only.""" +import os +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]).resolve() +work = root / '.sdd-work' +try: + if work.is_symlink(): + raise ValueError(str(work)) + # Check canonical destinations, including dangling links and alternate modfiles. + for name in ('tmp', 'cache', 'mod', 'gopath', 'collections', 'assets', 'logs', + 'docker', 'legacy', 'model-smoke', 'localai', 'localai-smoke.mod', 'localai-smoke.sum', + 'localai.tar.gz'): + top = work / name + paths = [top] + if top.is_dir() and not top.is_symlink(): + for directory, dirs, files in os.walk(top, followlinks=False): + paths.extend(pathlib.Path(directory) / n for n in dirs + files + if (pathlib.Path(directory) / n).is_symlink()) + for path in paths: + if not path.resolve().is_relative_to(work): + raise ValueError(str(path)) +except (ValueError, OSError, RuntimeError) as error: + sys.exit('unsafe output: ' + str(error)) diff --git a/test/raw/confinement_test.py b/test/raw/confinement_test.py new file mode 100644 index 0000000..5f5ff30 --- /dev/null +++ b/test/raw/confinement_test.py @@ -0,0 +1,45 @@ +"""Every public runner must reject escaping outputs before its first write.""" +import os +import pathlib +import shutil +import subprocess +import tempfile +import unittest + + +class Confinement(unittest.TestCase): + def test_runner_entrypoints(self): + source = pathlib.Path('test/raw').resolve() + cases = ('model-smoke/backends', 'localai/core/services/agentpool', + 'localai-smoke.mod', 'logs/nested', 'docker/contexts', + 'cache/nested', 'mod/nested', 'gopath/pkg', 'tmp/nested', + 'collections/nested', 'assets/nested', 'legacy/owner/assets') + for runner in ('verify.sh', 'database.sh', 'localai-smoke.sh', 'legacy.sh'): + for nested in cases: + with self.subTest(runner=runner, nested=nested), tempfile.TemporaryDirectory(dir='.sdd-work/tmp') as tmp: + root = pathlib.Path(tmp).resolve() + shutil.copytree(source, root / 'test/raw') + (root / 'go.mod').write_text('module fixture\n') + (root / 'rag').mkdir() + work = root / '.sdd-work' + work.mkdir() + outside = root / 'outside' + outside.mkdir() + sentinel = outside / 'sentinel' + sentinel.write_text('unchanged') + link = work / nested + link.parent.mkdir(parents=True, exist_ok=True) + link.symlink_to(outside, target_is_directory=True) + before = sorted(str(p.relative_to(root)) for p in root.rglob('*')) + env = dict(os.environ) + env.pop('LOCALRECALL_TEST_DATABASE_URL', None) + result = subprocess.run(['sh', 'test/raw/' + runner], cwd=root, + env=env, capture_output=True, timeout=10) + self.assertNotEqual(result.returncode, 0) + self.assertIn(b'unsafe output', result.stderr) + self.assertEqual(sentinel.read_text(), 'unchanged') + self.assertEqual(before, sorted(str(p.relative_to(root)) for p in root.rglob('*'))) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/raw/database.sh b/test/raw/database.sh index ed0250b..db02a50 100644 --- a/test/raw/database.sh +++ b/test/raw/database.sh @@ -4,6 +4,7 @@ set -eu [ -f test/raw/Dockerfile.postgres ] || { echo 'run from worktree root' >&2; exit 1; } [ -z "${LOCALRECALL_TEST_DATABASE_URL:-}" ] || { echo 'external database targets refused' >&2; exit 1; } ROOT=$(pwd -P) +python3 test/raw/confinement.py "$ROOT" [ ! -L .sdd-work ] && [ ! -L .sdd-work/docker ] || exit 1 mkdir -p .sdd-work/docker .sdd-work/tmp .sdd-work/logs export DOCKER_CONFIG="$ROOT/.sdd-work/docker" TMPDIR="$ROOT/.sdd-work/tmp" @@ -18,8 +19,16 @@ cleanup() { } trap cleanup EXIT HUP INT TERM docker image inspect "$image" >/dev/null -# PG18 declares /var/lib/postgresql as a volume; a private tmpfs avoids persistent volumes. -docker run -d --name "$name" --label "localrecall.raw.owner=$token" --tmpfs /var/lib/postgresql:rw -p 127.0.0.1::5432 -e POSTGRES_PASSWORD="$token" -e POSTGRES_DB=raw_fixture "$image" >/dev/null +# PG18 declares a volume; private tmpfs avoids persistent volumes. A shell PID 1 +# owns the server lifetime but remains alive after pg_ctl stop, preserving tmpfs +# for the explicit recovery test. Never restart/stop an existing container. +docker run -d --name "$name" --label "localrecall.raw.owner=$token" --tmpfs /var/lib/postgresql:rw -p 127.0.0.1::5432 -e POSTGRES_PASSWORD="$token" -e POSTGRES_DB=raw_fixture --entrypoint /bin/sh "$image" -c ' + docker-entrypoint.sh postgres -c shared_preload_libraries=pg_textsearch & + server=$! + trap "kill -TERM $server 2>/dev/null || true; exit 0" TERM INT + wait "$server" || true + while :; do sleep 3600 & wait $!; done +' >/dev/null n=0 until docker exec "$name" pg_isready -U postgres -d raw_fixture >/dev/null 2>&1; do n=$((n+1)); [ "$n" -lt 90 ] || { docker logs "$name"; exit 1; }; sleep 1 @@ -39,5 +48,11 @@ port=$(docker port "$name" 5432/tcp | sed -n 's/^127\.0\.0\.1://p') case "$port" in ''|*[!0-9]*) exit 1;; esac export LOCALRECALL_TEST_DATABASE_URL="postgres://postgres:$token@127.0.0.1:$port/raw_fixture?sslmode=disable" export LOCALRECALL_TEST_DATABASE_OWNER="$token" +export LOCALRECALL_TEST_DATABASE_CONTAINER="$name" printf 'Owned fixture %s at loopback port %s (removed on exit)\n' "$name" "$port" -"$@" +if "$@"; then :; else + status=$? + docker logs "$name" + docker exec "$name" cat /tmp/raw-restart.log 2>/dev/null || true + exit "$status" +fi diff --git a/test/raw/legacy.sh b/test/raw/legacy.sh index fa6e496..2e4d0c5 100644 --- a/test/raw/legacy.sh +++ b/test/raw/legacy.sh @@ -1,6 +1,7 @@ #!/bin/sh # Never run legacy destructive suites against an externally selected service. set -eu +python3 test/raw/confinement.py "$(pwd -P)" if [ "${1:-}" != --owned ]; then [ "$#" -eq 0 ] || exit 2 [ -z "${LOCALRECALL_TEST_DATABASE_URL:-}" ] || exit 1 diff --git a/test/raw/localai-smoke.sh b/test/raw/localai-smoke.sh index edad64f..672bcd3 100644 --- a/test/raw/localai-smoke.sh +++ b/test/raw/localai-smoke.sh @@ -2,6 +2,7 @@ # Prerequisites and provider provenance: see test/raw/LOCALAI-SMOKE.md. set -eu ROOT=$(pwd -P) +python3 test/raw/confinement.py "$ROOT" [ -f test/raw/downstream-lock.json ] || exit 1 W="$ROOT/.sdd-work" [ ! -L "$W" ] || exit 1 @@ -33,8 +34,8 @@ PY # The archive must have been downloaded from the immutable codeload revision; # canonical digest recorded independently, never use the enclosing repo's HEAD. sha256sum "$W/localai.tar.gz" "$W/model-smoke/minilm.gguf" "$LOCALAI_SMOKE_BACKEND" -docker image inspect localai/localai:latest --format 'provider image {{.Id}} {{json .Config.Labels}}' image=sha256:f1bc435659b9da49180e868fdd0361915171a740cd947bf081b62a896a7ea8d0 +docker image inspect "$image" --format 'provider image {{.Id}} {{json .Config.Labels}}' name="lr-embedding-$$" created= backend_pid= provider_pid= diff --git a/test/raw/templates/localai_consumer_test.go.txt b/test/raw/templates/localai_consumer_test.go.txt index d915c99..d2e18ef 100644 --- a/test/raw/templates/localai_consumer_test.go.txt +++ b/test/raw/templates/localai_consumer_test.go.txt @@ -2,6 +2,8 @@ package agentpool import ( "context" + "fmt" + "github.com/jackc/pgx/v5" "github.com/mudler/localrecall/rag/raw" rawpostgres "github.com/mudler/localrecall/rag/raw/postgres" "net/http" @@ -13,6 +15,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "github.com/mudler/LocalAGI/webui/collections" "github.com/mudler/LocalAI/core/config" @@ -31,14 +34,41 @@ func TestLocalRecallRealConsumer(t *testing.T) { t.Fatal(err) } var calls atomic.Int64 + var hold atomic.Bool + entered, release := make(chan struct{}, 1), make(chan struct{}) + defer func() { + select { + case <-release: + default: + close(release) + } + }() proxy := httputil.NewSingleHostReverseProxy(target) counting := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/embeddings") { calls.Add(1) + if hold.Load() { + select { + case entered <- struct{}{}: + default: + } + select { + case <-release: + case <-r.Context().Done(): + return + } + } } proxy.ServeHTTP(w, r) })) - defer counting.Close() + defer func() { + select { + case <-release: + default: + close(release) + } + counting.Close() + }() url = counting.URL dir := t.TempDir() svc, err := NewAgentPoolService(&config.ApplicationConfig{AgentPool: config.AgentPoolConfig{ @@ -84,7 +114,32 @@ func TestLocalRecallRealConsumer(t *testing.T) { t.Fatal("consumer bypassed embedding counter") } before := calls.Load() - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgx.Connect(ctx, db) + if err != nil { + t.Fatal(err) + } + defer admin.Close(ctx) + peak := 0 + sample := func() { + var n int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid()").Scan(&n); err != nil { + t.Fatal(err) + } + if n > peak { + peak = n + } + } + catalog := func() int { + var n int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_class WHERE relnamespace='public'::regnamespace").Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + baselineCatalog := catalog() + start := time.Now() store, err := rawpostgres.New(ctx, rawpostgres.Config{DatabaseURL: db}) if err != nil { t.Fatal(err) @@ -114,7 +169,70 @@ func TestLocalRecallRealConsumer(t *testing.T) { if calls.Load() != before { t.Fatalf("raw embedding traffic: before=%d after=%d", before, calls.Load()) } - t.Logf("consumer embedding calls=%d; raw lifecycle/search attributed calls=0", before) + t.Logf("serial consumer embedding calls=%d; raw lifecycle/search attributed calls=0", before) + // Hold an actual consumer embedding HTTP request while raw database work + // completes. Only the consumer receives this dedicated forwarding endpoint; + // raw Service has no HTTP client or endpoint. Exact delta attributes overlap. + if err := svc.CreateCollectionForUser("", "real-consumer-second"); err != nil { + t.Fatal(err) + } + before = calls.Load() + hold.Store(true) + legacyDone := make(chan error, 1) + go func() { _, e := svc.SearchCollectionForUser("", name, "telescope galaxies", 3); legacyDone <- e }() + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("consumer never entered embedding proxy") + } + overlapStart := time.Now() + rawIDs := []string{} + for i := 0; i < 3; i++ { + rc, _, e := store.CreateCollection(ctx, raw.CreateCollectionRequest{Name: fmt.Sprintf("overlap-%d", i), Dimensions: 3, Metric: "cosine", EmbeddingSpaceID: "synthetic-v1"}) + if e != nil { + t.Fatal(e) + } + rawIDs = append(rawIDs, rc.CollectionID) + if _, e = store.Upsert(ctx, rc.CollectionID, raw.UpsertRequest{EmbeddingSpaceID: "synthetic-v1", Records: []raw.Record{{ID: "raw", Vector: []float64{1, 0, 0}}}}); e != nil { + t.Fatal(e) + } + if _, e = store.Search(ctx, rc.CollectionID, raw.SearchRequest{Mode: "vector", Accuracy: &accuracy, EmbeddingSpaceID: &space, Vector: []float64{1, 0, 0}}); e != nil { + t.Fatal(e) + } + sample() + } + highCatalog := catalog() + if calls.Load() != before+1 { + t.Fatalf("overlap attribution: %d -> %d", before, calls.Load()) + } + for _, id := range rawIDs { + if e := store.DeleteCollection(ctx, id); e != nil { + t.Fatal(e) + } + } + finalCatalog := catalog() + overlapElapsed := time.Since(overlapStart) + close(release) + hold.Store(false) + select { + case e := <-legacyDone: + if e != nil { + t.Fatal(e) + } + case <-time.After(10 * time.Second): + t.Fatal("consumer did not finish") + } + if calls.Load() != before+1 { + t.Fatal("unexpected embedding traffic") + } + // Default four per pool: one raw plus two loaded legacy collections. + if peak < 3 || peak > 12 { + t.Fatalf("combined sampled pool peak=%d budget=12", peak) + } + if highCatalog <= finalCatalog { + t.Fatal("no index/catalog lifecycle observed") + } + t.Logf("operational fixture raw_collections=3 legacy_collections=2 overlap_requests=1 overlap_duration=%s legacy_calls=1 raw_calls=0 combined_sampled_pool_peak=%d configured_budget=12 catalog_before=%d catalog_high=%d catalog_after=%d catalog_removed=%d elapsed=%s", overlapElapsed, peak, baselineCatalog, highCatalog, finalCatalog, highCatalog-finalCatalog, time.Since(start)) if err := svc.ResetCollectionForUser("", name); err != nil { t.Fatal(err) } diff --git a/test/raw/verify.sh b/test/raw/verify.sh index b2a879b..2bdcd69 100644 --- a/test/raw/verify.sh +++ b/test/raw/verify.sh @@ -1,6 +1,7 @@ #!/bin/sh set -eu ROOT=$(pwd -P) +python3 test/raw/confinement.py "$ROOT" [ -f "$ROOT/go.mod" ] && [ -d "$ROOT/rag" ] || { echo 'run from worktree root' >&2; exit 1; } # Reject symlinks before creating any output, including nested cache paths. for path in .sdd-work .sdd-work/tmp .sdd-work/cache .sdd-work/mod .sdd-work/gopath .sdd-work/collections .sdd-work/assets .sdd-work/logs; do @@ -14,11 +15,12 @@ export GOENV=off GOTOOLCHAIN=local mkdir -p "$TMPDIR" "$GOCACHE" "$GOMODCACHE" "$GOPATH" "$COLLECTION_DB_PATH" "$FILE_ASSETS" "$ROOT/.sdd-work/logs" case "${1:-}" in all) + python3 test/raw/confinement_test.py sh test/raw/verify.sh api log="$ROOT/.sdd-work/logs/race10.log" if sh test/raw/database.sh go test -v -mod=readonly -race -count=10 -timeout=30m ./rag/raw/... ./pkg/client/... >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi ! grep -E 'SKIP|no tests to run|no test files' "$log" - for suite in TestRawProcessDiscovery TestRawRecordsReplacement TestRawClient; do + for suite in TestRawProcessDiscovery TestRawRecordsReplacement TestRawOwnedOutage TestRawActiveShutdown TestRawClient; do [ "$(grep -c -- "^--- PASS: $suite (" "$log")" -eq 10 ] done go vet -mod=readonly ./rag/raw/... ./pkg/client/... From 9293c4789fdd55a155c203886417c50e15d1ebae Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 12 Oct 2026 15:20:00 +0000 Subject: [PATCH 13/14] fix(raw): preserve request and error semantics Keep explicitly empty selectors on the wire so invalid deletes stay invalid. Derive response bounds from the contract to accept legal escaped text. Share advertised limits with validation and preserve safe error identity. Regression tests cover selector presence, response bounds, and sanitization. --- pkg/client/raw.go | 61 +++++++-- pkg/client/raw_test.go | 178 +++++++++++++++++++++++-- rag/raw/postgres/service.go | 38 +++++- rag/raw/postgres/service_error_test.go | 68 ++++++++++ rag/raw/store.go | 7 + rag/raw/store_test.go | 14 +- rag/raw/types.go | 70 ++++++++-- rag/raw/validate.go | 18 +-- rag/raw/validate_test.go | 93 +++++++++++++ 9 files changed, 504 insertions(+), 43 deletions(-) create mode 100644 rag/raw/postgres/service_error_test.go diff --git a/pkg/client/raw.go b/pkg/client/raw.go index 32f406a..107a494 100644 --- a/pkg/client/raw.go +++ b/pkg/client/raw.go @@ -69,16 +69,24 @@ func (c *RawClient) do(ctx context.Context, method, path string, q url.Values, b Data json.RawMessage `json:"data"` Error *raw.Error `json:"error"` } - b, err := io.ReadAll(io.LimitReader(resp.Body, (32<<20)+1)) - if err != nil { - return resp.StatusCode, err - } - if len(b) > 32<<20 { - return resp.StatusCode, fmt.Errorf("raw API response exceeds 32 MiB") - } - if err = json.Unmarshal(b, &envelope); err != nil { + // Decode from a bounded stream rather than allocating the theoretical cap. + limit := rawResponseLimit(method, path) + readerLimit := &io.LimitedReader{R: resp.Body, N: limit + 1} + decoder := json.NewDecoder(readerLimit) + decodeErr := decoder.Decode(&envelope) + // Drain the remaining bounded stream, including decoder read-ahead. This + // detects overflow even when a complete JSON value precedes excess bytes. + _, drainErr := io.Copy(rawJSONWhitespace{}, io.MultiReader(decoder.Buffered(), readerLimit)) + if readerLimit.N == 0 { + return resp.StatusCode, fmt.Errorf("raw API response exceeds %d bytes", limit) + } + if drainErr != nil { + return resp.StatusCode, drainErr + } + if decodeErr != nil { return resp.StatusCode, fmt.Errorf("invalid raw API response (HTTP %d)", resp.StatusCode) } + if resp.StatusCode < 200 || resp.StatusCode >= 300 || !envelope.Success { if envelope.Error != nil { e := raw.NewError(envelope.Error.Code) @@ -92,6 +100,43 @@ func (c *RawClient) do(ctx context.Context, method, path string, q url.Values, b } return resp.StatusCode, err } + +// Bounds cover compact server JSON. Strings can expand to six bytes per input +// byte; metadata already has a serialized-byte limit. 32 bytes per vector +// component covers every finite float32 spelling plus its comma. Per-item slack +// covers keys, punctuation, scores/ranks, timestamps and other scalar fields. +const ( + rawItemOverhead int64 = 2048 + rawEnvelopeOverhead int64 = 4096 + 6*raw.MaxCursorBytes + rawRecordBytes int64 = 6*(raw.MaxTextBytes+raw.MaxIDBytes) + raw.MaxMetadataBytes + 32*raw.MaxDimensions + rawItemOverhead + // Disabled lexical indexing retains text_config without catalog resolution; + // its existing validation limit is MaxBodyBytes, not PostgreSQL name length. + rawCollectionBytes int64 = 6*(raw.MaxBodyBytes+raw.MaxNameBytes+raw.MaxSpaceIDBytes) + rawItemOverhead +) + +func rawResponseLimit(method, path string) int64 { + switch { + case path == "/collections" && method == http.MethodGet: + return rawEnvelopeOverhead + raw.MaxListLimit*rawCollectionBytes + case path == "/collections" || (strings.HasPrefix(path, "/collections/") && strings.Count(path, "/") == 2): + return rawEnvelopeOverhead + rawCollectionBytes + default: + return rawEnvelopeOverhead + int64(max(raw.MaxBatch, raw.MaxListLimit, raw.MaxTopK))*rawRecordBytes + } +} + +// Write checks trailing JSON whitespace with constant additional memory. +type rawJSONWhitespace struct{} + +func (rawJSONWhitespace) Write(p []byte) (int, error) { + for _, c := range p { + if c != ' ' && c != '\n' && c != '\r' && c != '\t' { + return 0, fmt.Errorf("invalid trailing raw API response data") + } + } + return len(p), nil +} + func rawCollectionPath(cid string) (string, error) { id, err := uuid.Parse(cid) if err != nil || id.String() != cid { diff --git a/pkg/client/raw_test.go b/pkg/client/raw_test.go index c917d35..695d16a 100644 --- a/pkg/client/raw_test.go +++ b/pkg/client/raw_test.go @@ -1,9 +1,14 @@ package client import ( + "bytes" "context" + "encoding/json" "errors" + "fmt" "github.com/mudler/localrecall/rag/raw" + "io" + "math" "net/http" "net/http/httptest" "strings" @@ -67,15 +72,85 @@ func TestRawClientSafety(t *testing.T) { } } +// spaceReader supplies large padding without allocating the response cap. +type spaceReader struct{} + +func (spaceReader) Read(p []byte) (int, error) { + for i := range p { + p[i] = ' ' + } + return len(p), nil +} + func TestRawClientResponseOverflow(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // The first 32 MiB are valid JSON plus whitespace; truncation hides junk. - prefix := `{"success":true,"data":{}}` - w.Write([]byte(prefix + strings.Repeat(" ", (32<<20)-len(prefix)) + "x")) - })) - defer srv.Close() - if _, err := NewRawClient(srv.URL, "", srv.Client()).Capabilities(context.Background()); err == nil { - t.Fatal("oversized response accepted") + limit := rawResponseLimit(http.MethodGet, "/capabilities") + prefix := `{"success":true,"data":{}}` + for _, extra := range []int64{0, 1} { + body := io.MultiReader(strings.NewReader(prefix), io.LimitReader(spaceReader{}, limit-int64(len(prefix))+extra)) + c := NewRawClient("http://fixture.invalid", "", &http.Client{Transport: rawTransport(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 200, Body: io.NopCloser(body), Header: http.Header{}}, nil + })}) + _, err := c.Capabilities(context.Background()) + if extra == 0 && err != nil { + t.Fatal("exact bound rejected", err) + } + if extra == 1 && (err == nil || !strings.Contains(err.Error(), "exceeds")) { + t.Fatal("overflow not detected", err) + } + } + for _, suffix := range []string{"x", ` {"success":true}`, " \n\t"} { + c := NewRawClient("http://fixture.invalid", "", &http.Client{Transport: rawTransport(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(prefix + suffix)), Header: http.Header{}}, nil + })}) + _, err := c.Capabilities(context.Background()) + if (err == nil) != (suffix == " \n\t") { + t.Fatalf("trailing data %q: %v", suffix, err) + } + } +} + +func TestRawClientResponseBoundAccounting(t *testing.T) { + l := raw.ContractLimits() + text := strings.Repeat("\x01", l.TextBytes) + metadata := map[string]string{} + // Maximal serialized metadata is already covered by its own contract cap. + for i := 0; i < 15; i++ { + metadata[fmt.Sprintf("%02d", i)] = strings.Repeat("v", l.MetadataValueBytes) + } + metadata["last"] = "" + b, err := json.Marshal(metadata) + if err != nil { + t.Fatal(err) + } + metadata["last"] = strings.Repeat("v", l.MetadataBytes-len(b)) + vector := make([]float32, l.Dimensions) + for i := range vector { + vector[i] = -1.2345678e-30 + } + record := raw.CanonicalRecord{ID: strings.Repeat("<", l.IDBytes), Text: &text, Vector: vector, Metadata: metadata} + b, err = json.Marshal(record) + if err != nil { + t.Fatal(err) + } + if int64(len(b)) > rawRecordBytes { + t.Fatal("maximal record exceeds budget", len(b), rawRecordBytes) + } + score := math.MaxFloat64 + rank := int(^uint(0) >> 1) + result := raw.SearchResult{ID: record.ID, Text: &text, Metadata: metadata, Score: score, VectorSimilarity: &score, VectorRank: &rank, LexicalRank: &rank} + b, err = json.Marshal(result) + if err != nil { + t.Fatal(err) + } + if int64(len(b)) > rawRecordBytes { + t.Fatal("scalar/search overhead exceeds budget") + } + // Collection lists have a larger cumulative cap without allocating that cap. + if rawResponseLimit("GET", "/collections") != rawEnvelopeOverhead+int64(l.ListMax)*rawCollectionBytes { + t.Fatal("collection list bound") + } + if rawCollectionBytes < 6*int64(raw.MaxBodyBytes+l.NameBytes+l.SpaceIDBytes)+rawItemOverhead { + t.Fatal("descriptor bound") } } func TestRawClientEncodedPrefix(t *testing.T) { @@ -90,3 +165,90 @@ func TestRawClientEncodedPrefix(t *testing.T) { t.Fatal(err) } } + +// A transport fixture keeps these contract tests independent of a database. +type rawTransport func(*http.Request) (*http.Response, error) + +func (f rawTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestRawClientSelectorPresence(t *testing.T) { + for _, ids := range [][]string{nil, {}, {"x"}} { + for _, vector := range [][]float64{nil, {}, {1}} { + c := NewRawClient("http://fixture.invalid", "", &http.Client{Transport: rawTransport(func(r *http.Request) (*http.Response, error) { + b, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(b, &fields); err != nil { + t.Fatal(err) + } + if strings.HasSuffix(r.URL.Path, "/delete") { + if _, present := fields["ids"]; present != (ids != nil) { + t.Errorf("IDs presence changed: %s", b) + } + var req raw.DeleteRecordsRequest + if err := raw.DecodeRequest(b, &req); err != nil { + t.Fatal(err) + } + if (raw.ValidateDelete(req) == nil) != (ids == nil) { + t.Errorf("delete validity changed: %s", b) + } + } else { + if _, present := fields["vector"]; present != (vector != nil) { + t.Errorf("vector presence changed: %s", b) + } + var req raw.SearchRequest + if err := raw.DecodeRequest(b, &req); err != nil { + t.Fatal(err) + } + _, err := raw.ValidateSearch(raw.Collection{Lexical: raw.LexicalConfig{Enabled: true}}, req) + if (err == nil) != (vector == nil) { + t.Errorf("search validity changed: %s", b) + } + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"success":true,"data":{}}`)), Header: http.Header{}}, nil + })}) + cid := "12345678-1234-1234-1234-123456789abc" + if _, err := c.DeleteRecords(context.Background(), cid, raw.DeleteRecordsRequest{IDs: ids, Filter: &raw.Filter{MetadataEq: map[string]string{"source": "asset"}}}); err != nil { + t.Fatal(err) + } + text := "query" + if _, err := c.Search(context.Background(), cid, raw.SearchRequest{Mode: "lexical", Vector: vector, Text: &text}); err != nil { + t.Fatal(err) + } + } + } +} + +func TestRawClientMaxEscapedRecords(t *testing.T) { + text := strings.Repeat("\x01", raw.MaxTextBytes) + records := make([]raw.CanonicalRecord, raw.MaxBatch) + ids := make([]string, raw.MaxBatch) + for i := range records { + ids[i] = fmt.Sprint(i) + canonical, err := raw.ValidateUpsert(raw.Collection{Dimensions: 1, EmbeddingSpaceID: "s"}, raw.UpsertRequest{EmbeddingSpaceID: "s", Records: []raw.Record{{ID: ids[i], Vector: []float64{1}, Text: &text}}}) + if err != nil { + t.Fatal(err) + } + records[i] = canonical[0] + } + b, err := json.Marshal(struct { + Success bool `json:"success"` + Data raw.GetRecordsResponse `json:"data"` + }{true, raw.GetRecordsResponse{Records: records, MissingIDs: []string{}}}) + if err != nil { + t.Fatal(err) + } + t.Logf("legal response bytes: %d", len(b)) + c := NewRawClient("http://fixture.invalid", "", &http.Client{Transport: rawTransport(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 200, Body: io.NopCloser(bytes.NewReader(b)), Header: http.Header{}}, nil + })}) + got, err := c.GetRecords(context.Background(), "12345678-1234-1234-1234-123456789abc", raw.GetRecordsRequest{IDs: ids, IncludeVectors: true}) + if err != nil { + t.Fatal(err) + } + if len(got.Records) != len(records) || *got.Records[0].Text != text { + t.Fatal("response changed") + } +} diff --git a/rag/raw/postgres/service.go b/rag/raw/postgres/service.go index 3a97b3d..8710db6 100644 --- a/rag/raw/postgres/service.go +++ b/rag/raw/postgres/service.go @@ -3,6 +3,7 @@ package postgres import ( "context" + "encoding/json" "errors" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" @@ -69,19 +70,44 @@ func (s *Service) Close() { s.pool.Close() }) } + +// databaseError retains only allowlisted public information, never the cause. +// In particular, wrapping a raw.Error must not retain an unsafe outer message. +type databaseError struct { + public *raw.Error + canceled bool + deadline bool +} + +func (e *databaseError) Error() string { return e.public.Error() } +func (e *databaseError) Unwrap() error { return e.public } +func (e *databaseError) Is(target error) bool { + return (target == context.Canceled && e.canceled) || + (target == context.DeadlineExceeded && e.deadline) +} +func (e *databaseError) MarshalJSON() ([]byte, error) { return json.Marshal(e.public) } + func dbError(e error) error { if e == nil { return nil } - var r *raw.Error - if errors.As(e, &r) { - return e + safe := &databaseError{ + canceled: errors.Is(e, context.Canceled), + deadline: errors.Is(e, context.DeadlineExceeded), } + code := raw.CodeUnavailable + var r *raw.Error var p *pgconn.PgError - if errors.Is(e, context.DeadlineExceeded) || errors.Is(e, context.Canceled) || (errors.As(e, &p) && (p.Code == "57014" || p.Code == "55P03")) { - return raw.NewError(raw.CodeTimeout) + switch { + case errors.As(e, &r) && r != nil: + code = r.Code + case safe.canceled || safe.deadline: + code = raw.CodeTimeout + case errors.As(e, &p) && p != nil && (p.Code == "57014" || p.Code == "55P03"): + code = raw.CodeTimeout } - return raw.NewError(raw.CodeUnavailable) + safe.public = raw.NewError(code) + return safe } func (s *Service) transaction(ctx context.Context, isolation pgx.TxIsoLevel, fn func(context.Context, pgx.Tx) error) error { ctx, done, e := s.operation(ctx) diff --git a/rag/raw/postgres/service_error_test.go b/rag/raw/postgres/service_error_test.go new file mode 100644 index 0000000..a6c85c7 --- /dev/null +++ b/rag/raw/postgres/service_error_test.go @@ -0,0 +1,68 @@ +package postgres + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/mudler/localrecall/rag/raw" +) + +func TestDBErrorSafeClassification(t *testing.T) { + secret := "SQL password=secret caller-payload" + for _, tc := range []struct { + name string + cause error + code string + canceled, deadline bool + }{ + {"cancel", fmt.Errorf("%s: %w", secret, context.Canceled), raw.CodeTimeout, true, false}, + {"deadline", fmt.Errorf("%s: %w", secret, context.DeadlineExceeded), raw.CodeTimeout, false, true}, + {"both", errors.Join(context.Canceled, context.DeadlineExceeded), raw.CodeTimeout, true, true}, + {"statement", &pgconn.PgError{Code: "57014", Message: secret}, raw.CodeTimeout, false, false}, + {"lock", &pgconn.PgError{Code: "55P03", Detail: secret}, raw.CodeTimeout, false, false}, + {"driver", &pgconn.PgError{Code: "08006", Message: secret}, raw.CodeUnavailable, false, false}, + {"unknown code", &raw.Error{Code: secret, Message: secret}, raw.CodeInternal, false, false}, + {"tainted raw", &raw.Error{Code: raw.CodeValidation, Message: secret}, raw.CodeValidation, false, false}, + {"unknown", errors.New(secret), raw.CodeUnavailable, false, false}, + {"raw wrapped", fmt.Errorf("%s: %w", secret, raw.NewError(raw.CodeValidation)), raw.CodeValidation, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + err := dbError(tc.cause) + var re *raw.Error + if !errors.As(err, &re) || re.Code != tc.code { + t.Fatalf("classification: %v", err) + } + if !errors.Is(err, raw.NewError(tc.code)) { + t.Error("safe classification identity lost") + } + if errors.Is(err, context.Canceled) != tc.canceled || errors.Is(err, context.DeadlineExceeded) != tc.deadline { + t.Error("context identity lost") + } + var driver *pgconn.PgError + if errors.As(err, &driver) { + t.Error("driver exposed") + } + b, e := json.Marshal(err) + if e != nil { + t.Fatal(e) + } + var log bytes.Buffer + slog.New(slog.NewJSONHandler(&log, nil)).Error("operation", "error", err) + for _, output := range []string{err.Error(), fmt.Sprintf("%+v %#v", err, err), string(b), log.String()} { + if strings.Contains(output, secret) { + t.Errorf("unsafe diagnostic: %s", output) + } + } + }) + } + if dbError(nil) != nil { + t.Fatal("nil changed") + } +} diff --git a/rag/raw/store.go b/rag/raw/store.go index 541ff12..4e11333 100644 --- a/rag/raw/store.go +++ b/rag/raw/store.go @@ -44,6 +44,13 @@ type Error struct { } func (e *Error) Error() string { return e.Message } + +// Is compares public classifications, not messages or transport status. +func (e *Error) Is(target error) bool { + other, ok := target.(*Error) + return ok && e != nil && other != nil && e.Code == other.Code +} + func NewError(code string) *Error { e := &Error{Code: code} switch code { diff --git a/rag/raw/store_test.go b/rag/raw/store_test.go index 0f203df..b750f63 100644 --- a/rag/raw/store_test.go +++ b/rag/raw/store_test.go @@ -1,6 +1,9 @@ package raw -import "testing" +import ( + "errors" + "testing" +) func TestRawErrorStatuses(t *testing.T) { for _, tc := range []struct { @@ -25,3 +28,12 @@ func TestRawErrorStatuses(t *testing.T) { t.Fatal(e) } } + +func TestRawErrorClassification(t *testing.T) { + if !errors.Is(NewError(CodeTimeout), NewError(CodeTimeout)) { + t.Fatal("classification lost") + } + if errors.Is(NewError(CodeTimeout), NewError(CodeUnavailable)) { + t.Fatal("different codes match") + } +} diff --git a/rag/raw/types.go b/rag/raw/types.go index 9026bc1..3c1911e 100644 --- a/rag/raw/types.go +++ b/rag/raw/types.go @@ -1,18 +1,32 @@ // Package raw defines embedder-free caller-supplied vector contracts. package raw -import "time" +import ( + "encoding/json" + "time" +) const ( - MaxBodyBytes = 8 << 20 - MaxDimensions = 2000 - MaxBatch = 100 - MaxTextBytes = 64 << 10 - MaxMetadataBytes = 16 << 10 - MaxFilterPredicates = 8 - MaxDeleteMatches = 1000 - MaxCursorBytes = 2048 - SchemaVersion = 1 + MaxNameBytes = 128 + MaxSpaceIDBytes = 256 + MaxIDBytes = 256 + MaxMetadataKeys = 32 + MaxMetadataKeyBytes = 64 + MaxMetadataValueBytes = 1024 + DefaultListLimit = 50 + MaxListLimit = 100 + DefaultTopK = 10 + MaxTopK = 100 + MaxQueryTextBytes = 4096 + MaxBodyBytes = 8 << 20 + MaxDimensions = 2000 + MaxBatch = 100 + MaxTextBytes = 64 << 10 + MaxMetadataBytes = 16 << 10 + MaxFilterPredicates = 8 + MaxDeleteMatches = 1000 + MaxCursorBytes = 2048 + SchemaVersion = 1 ) type LexicalConfig struct { @@ -192,5 +206,39 @@ type Capabilities struct { } func ContractLimits() Limits { - return Limits{MaxBodyBytes, MaxDimensions, 128, 256, 256, MaxTextBytes, 32, 64, 1024, MaxMetadataBytes, MaxBatch, MaxFilterPredicates, MaxDeleteMatches, 50, 100, MaxCursorBytes, 10, 100, 4096} + return Limits{ + BodyBytes: MaxBodyBytes, Dimensions: MaxDimensions, NameBytes: MaxNameBytes, + SpaceIDBytes: MaxSpaceIDBytes, IDBytes: MaxIDBytes, TextBytes: MaxTextBytes, + MetadataKeys: MaxMetadataKeys, MetadataKeyBytes: MaxMetadataKeyBytes, + MetadataValueBytes: MaxMetadataValueBytes, MetadataBytes: MaxMetadataBytes, + Batch: MaxBatch, FilterPredicates: MaxFilterPredicates, DeleteMatches: MaxDeleteMatches, + ListDefault: DefaultListLimit, ListMax: MaxListLimit, CursorBytes: MaxCursorBytes, + TopKDefault: DefaultTopK, TopKMax: MaxTopK, QueryTextBytes: MaxQueryTextBytes, + } +} + +// MarshalJSON preserves the distinction between absent and explicitly empty IDs. +func (r DeleteRecordsRequest) MarshalJSON() ([]byte, error) { + type plain DeleteRecordsRequest + var ids *[]string + if r.IDs != nil { + ids = &r.IDs + } + return json.Marshal(struct { + plain + IDs *[]string `json:"ids,omitempty"` + }{plain: plain(r), IDs: ids}) +} + +// MarshalJSON preserves vector presence for search-mode exclusivity checks. +func (r SearchRequest) MarshalJSON() ([]byte, error) { + type plain SearchRequest + var vector *[]float64 + if r.Vector != nil { + vector = &r.Vector + } + return json.Marshal(struct { + plain + Vector *[]float64 `json:"vector,omitempty"` + }{plain: plain(r), Vector: vector}) } diff --git a/rag/raw/validate.go b/rag/raw/validate.go index e4c50f5..fcad162 100644 --- a/rag/raw/validate.go +++ b/rag/raw/validate.go @@ -16,7 +16,7 @@ func validString(s string, max int) bool { return len(s) <= max && utf8.ValidString(s) && !strings.ContainsRune(s, 0) } func ValidateCreate(r CreateCollectionRequest) error { - if !validString(r.Name, 128) || r.Name == "" || r.Dimensions < 1 || r.Dimensions > MaxDimensions || r.Metric != "cosine" || !validString(r.EmbeddingSpaceID, 256) || r.EmbeddingSpaceID == "" { + if !validString(r.Name, MaxNameBytes) || r.Name == "" || r.Dimensions < 1 || r.Dimensions > MaxDimensions || r.Metric != "cosine" || !validString(r.EmbeddingSpaceID, MaxSpaceIDBytes) || r.EmbeddingSpaceID == "" { return invalid() } if r.Lexical != nil && r.Lexical.TextConfig != nil && (!validString(*r.Lexical.TextConfig, MaxBodyBytes) || *r.Lexical.TextConfig == "") { @@ -77,7 +77,7 @@ func NormalizeVector(v []float64, dimensions int) ([]float32, error) { return out, nil } func ValidateID(id string) error { - if id == "" || !validString(id, 256) { + if id == "" || !validString(id, MaxIDBytes) { return invalid() } for _, r := range id { @@ -105,7 +105,7 @@ func validateMetadata(m map[string]string, maxKeys int) error { return invalid() } for k, v := range m { - if !validString(k, 64) || !validString(v, 1024) { + if !validString(k, MaxMetadataKeyBytes) || !validString(v, MaxMetadataValueBytes) { return invalid() } } @@ -134,7 +134,7 @@ func ValidateUpsert(d Collection, r UpsertRequest) ([]CanonicalRecord, error) { if v.Text != nil && !validString(*v.Text, MaxTextBytes) { return nil, invalid() } - if e := validateMetadata(v.Metadata, 32); e != nil { + if e := validateMetadata(v.Metadata, MaxMetadataKeys); e != nil { return nil, e } vec, e := NormalizeVector(v.Vector, d.Dimensions) @@ -174,9 +174,9 @@ func ValidateDelete(r DeleteRecordsRequest) error { } func ValidateLimit(n *int) (int, error) { if n == nil { - return 50, nil + return DefaultListLimit, nil } - if *n < 1 || *n > 100 { + if *n < 1 || *n > MaxListLimit { return 0, invalid() } return *n, nil @@ -185,11 +185,11 @@ func ValidateSearch(d Collection, r SearchRequest) (SearchRequest, error) { if e := ValidateFilter(r.Filter); e != nil { return r, e } - k := 10 + k := DefaultTopK if r.TopK != nil { k = *r.TopK } - if k < 1 || k > 100 { + if k < 1 || k > MaxTopK { return r, invalid() } r.TopK = &k @@ -207,7 +207,7 @@ func ValidateSearch(d Collection, r SearchRequest) (SearchRequest, error) { return r, invalid() } if r.Mode != "vector" { - if r.Text == nil || !validString(*r.Text, 4096) || strings.TrimSpace(*r.Text) == "" { + if r.Text == nil || !validString(*r.Text, MaxQueryTextBytes) || strings.TrimSpace(*r.Text) == "" { return r, invalid() } if !d.Lexical.Enabled { diff --git a/rag/raw/validate_test.go b/rag/raw/validate_test.go index 8e9c6c0..583a1a5 100644 --- a/rag/raw/validate_test.go +++ b/rag/raw/validate_test.go @@ -2,6 +2,7 @@ package raw import ( "encoding/json" + "fmt" "math" "strings" "testing" @@ -188,3 +189,95 @@ func TestRawError(t *testing.T) { t.Fatal(string(b)) } } + +// Capabilities must describe the same inclusive boundaries as library validation. +func TestRawContractLimitsParity(t *testing.T) { + l := ContractLimits() + create := CreateCollectionRequest{Name: "n", Dimensions: 1, Metric: "cosine", EmbeddingSpaceID: "s"} + for _, tc := range []struct { + name string + max int + check func(int) error + }{ + {"name", l.NameBytes, func(n int) error { r := create; r.Name = strings.Repeat("n", n); return ValidateCreate(r) }}, + {"space", l.SpaceIDBytes, func(n int) error { r := create; r.EmbeddingSpaceID = strings.Repeat("s", n); return ValidateCreate(r) }}, + {"dimensions", l.Dimensions, func(n int) error { r := create; r.Dimensions = n; return ValidateCreate(r) }}, + {"id", l.IDBytes, func(n int) error { return ValidateID(strings.Repeat("i", n)) }}, + {"batch", l.Batch, func(n int) error { + ids := make([]string, n) + for i := range ids { + ids[i] = fmt.Sprint(i) + } + return ValidateIDs(ids) + }}, + {"text", l.TextBytes, func(n int) error { + _, err := ValidateUpsert(descriptor(), UpsertRequest{EmbeddingSpaceID: "s", Records: []Record{{ID: "i", Vector: []float64{1, 0}, Text: ptr(strings.Repeat("t", n))}}}) + return err + }}, + {"metadata keys", l.MetadataKeys, func(n int) error { + m := map[string]string{} + for i := 0; i < n; i++ { + m[fmt.Sprint(i)] = "v" + } + return validateMetadata(m, l.MetadataKeys) + }}, + {"metadata key", l.MetadataKeyBytes, func(n int) error { + return validateMetadata(map[string]string{strings.Repeat("k", n): "v"}, l.MetadataKeys) + }}, + {"metadata value", l.MetadataValueBytes, func(n int) error { + return validateMetadata(map[string]string{"k": strings.Repeat("v", n)}, l.MetadataKeys) + }}, + {"filter", l.FilterPredicates, func(n int) error { + m := map[string]string{} + for i := 0; i < n; i++ { + m[fmt.Sprint(i)] = "v" + } + return ValidateFilter(&Filter{MetadataEq: m}) + }}, + {"list", l.ListMax, func(n int) error { _, err := ValidateLimit(&n); return err }}, + {"top k", l.TopKMax, func(n int) error { + _, err := ValidateSearch(descriptor(), SearchRequest{Mode: "lexical", Text: ptr("q"), TopK: &n}) + return err + }}, + {"query", l.QueryTextBytes, func(n int) error { + _, err := ValidateSearch(descriptor(), SearchRequest{Mode: "lexical", Text: ptr(strings.Repeat("q", n))}) + return err + }}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := tc.check(tc.max); err != nil { + t.Fatalf("advertised maximum rejected: %v", err) + } + if tc.check(tc.max+1) == nil { + t.Fatal("maximum+1 accepted") + } + }) + } + if n, err := ValidateLimit(nil); err != nil || n != l.ListDefault { + t.Fatal("list default parity", n, err) + } + if r, err := ValidateSearch(descriptor(), SearchRequest{Mode: "lexical", Text: ptr("q")}); err != nil || *r.TopK != l.TopKDefault { + t.Fatal("search default parity", r, err) + } + // An exact serialized metadata boundary, with each individual value within its limit. + m := map[string]string{} + for i := 0; i < 15; i++ { + m[fmt.Sprintf("%02d", i)] = strings.Repeat("v", l.MetadataValueBytes) + } + m["last"] = "" + b, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + m["last"] = strings.Repeat("v", l.MetadataBytes-len(b)) + if err := validateMetadata(m, l.MetadataKeys); err != nil { + t.Fatal(err) + } + m["last"] += "v" + if validateMetadata(m, l.MetadataKeys) == nil { + t.Fatal("serialized metadata maximum+1 accepted") + } + if l.BodyBytes != MaxBodyBytes || l.CursorBytes != MaxCursorBytes || l.DeleteMatches != MaxDeleteMatches { + t.Fatal("shared limit parity") + } +} From 45fd633a1c2a3220ea2980eddb1e42da1287e21f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 12 Oct 2026 15:20:00 +0000 Subject: [PATCH 14/14] fix(ci): isolate database fixtures for every suite Run every unit package against a fresh owned database to avoid missing fixtures and catalog contamination. Keep owned E2E coverage explicit. Assert empty PostgreSQL results after reset. Replace mutable source content with a private fixture and verify three completed updates without duplicates. --- .github/workflows/test.yml | 18 ++++---- Makefile | 18 ++++---- test/e2e/e2e_test.go | 8 ++-- test/e2e/raw_legacy_owned_test.go | 2 + test/e2e/source_manager_test.go | 68 ++++++++++++++----------------- test/raw/CI.md | 30 ++++++++++++++ test/raw/database.sh | 1 + test/raw/legacy.sh | 9 ++-- test/raw/unit.sh | 9 ++++ 9 files changed, 103 insertions(+), 60 deletions(-) create mode 100644 test/raw/CI.md create mode 100644 test/raw/unit.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 95cad8f..264a835 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,12 +21,12 @@ jobs: uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: - go-version: '^1.22' + go-version-file: go.mod - name: Install dependencies - run: go mod tidy + run: go mod download - name: Run unit tests run: make test-unit @@ -39,12 +39,12 @@ jobs: uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: - go-version: '^1.22' + go-version-file: go.mod - name: Install dependencies - run: go mod tidy + run: go mod download - name: Run integration tests env: @@ -59,12 +59,12 @@ jobs: uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: - go-version: '^1.22' + go-version-file: go.mod - name: Install dependencies - run: go mod tidy + run: go mod download - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/Makefile b/Makefile index 98a7d2f..5076d3d 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ run: build .PHONY: test test-unit test-integration test-e2e test-all test-coverage .PHONY: start-localai stop-localai wait-localai -.PHONY: start-test-services stop-test-services +.PHONY: start-test-services stop-test-services test-raw-image # Start LocalAI service for unit/integration tests start-localai: @@ -109,9 +109,12 @@ stop-test-services: test: @go test -coverprofile=coverage.txt -covermode=atomic -v ./... -test-unit: start-test-services +test-raw-image: + @docker build -t localrecall-raw-task2-a5975c8 -f test/raw/Dockerfile.postgres . + +test-unit: start-test-services test-raw-image @echo "Running unit tests..." - @LOCALAI_ENDPOINT=http://localhost:8081 go test -v ./rag/... ./pkg/...; \ + @LOCALAI_ENDPOINT=http://localhost:8081 sh test/raw/unit.sh; \ test_exit=$$?; \ $(MAKE) stop-test-services; \ exit $$test_exit @@ -125,9 +128,9 @@ test-integration: start-test-services test-e2e: prepare-e2e run-e2e clean-e2e -test-all: start-test-services +test-all: start-test-services test-raw-image @echo "Running all tests..." - @LOCALAI_ENDPOINT=http://localhost:8081 go test -v ./rag/... ./pkg/...; \ + @LOCALAI_ENDPOINT=http://localhost:8081 sh test/raw/unit.sh; \ unit_exit=$$?; \ INTEGRATION=true LOCALAI_ENDPOINT=http://localhost:8081 go test -v ./test/integration/...; \ integration_exit=$$?; \ @@ -207,5 +210,6 @@ clean-test-services: stop-test-services @echo "Cleaning up test services..." @docker compose rm -f localai postgres || true -run-e2e: - @E2E=true LOCALAI_ENDPOINT=http://localhost:8081 LOCALRECALL_ENDPOINT=http://localhost:8080 go test -v -timeout 30m ./test/e2e/... +run-e2e: test-raw-image + @E2E=true LOCALAI_ENDPOINT=http://localhost:8081 LOCALRECALL_ENDPOINT=http://localhost:8080 go test -v -count=1 -timeout 30m ./test/e2e/... + @LOCALAI_ENDPOINT=http://localhost:8081 EMBEDDING_MODEL=$(EMBEDDING_MODEL) sh test/raw/database.sh sh test/raw/legacy.sh --owned --e2e-only diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index ade88b4..b6b32fa 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -115,9 +115,11 @@ var _ = Describe("API", func() { Expect(err).To(BeNil()) docs, err := localRecall.Search(TestCollection, "spiders", 1) - Expect(err).To(HaveOccurred()) - Expect(docs).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("failed to search collection")) + Expect(err).ToNot(HaveOccurred()) + Expect(docs).To(BeEmpty()) + entries, err := localRecall.ListEntries(TestCollection) + Expect(err).ToNot(HaveOccurred()) + Expect(entries).To(BeEmpty()) }) It("should be able to delete documents", func() { diff --git a/test/e2e/raw_legacy_owned_test.go b/test/e2e/raw_legacy_owned_test.go index f0c76f0..94b1e1a 100644 --- a/test/e2e/raw_legacy_owned_test.go +++ b/test/e2e/raw_legacy_owned_test.go @@ -1,3 +1,5 @@ +//go:build ownedintegration + package e2e_test import ( diff --git a/test/e2e/source_manager_test.go b/test/e2e/source_manager_test.go index 0fb8f88..0783301 100644 --- a/test/e2e/source_manager_test.go +++ b/test/e2e/source_manager_test.go @@ -2,6 +2,8 @@ package e2e_test import ( "fmt" + "net/http" + "net/http/httptest" "os" "path/filepath" "time" @@ -250,47 +252,37 @@ var _ = Describe("SourceManager", func() { }) It("should prevent duplicate content with frequent updates", func() { - // Add a source with a very short update interval - err := sourceManager.AddSource(TestCollection, "https://en.wikipedia.org/wiki/Black-crowned_barwing", 1*time.Second) - Expect(err).To(BeNil()) - - // Start the background service + // Fixed, sub-chunk-size content: public Wikipedia revisions changed the + // old expected count. The real fetch/parser/embed/store path stays intact. + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "
The Black-crowned barwing is a bird of mountain forests. It feeds on insects and fruit.
") + })) + defer fixture.Close() + Expect(sourceManager.AddSource(TestCollection, fixture.URL, time.Second)).To(Succeed()) sourceManager.Start() - // Wait for initial content to be fetched - Eventually(func() []string { - return kb.ListDocuments() - }, 2*time.Minute, 5*time.Second).Should(HaveLen(1)) - - // Wait for initial content to be fetched - Eventually(func() int { - return kb.Count() - }, 2*time.Minute, 5*time.Second).Should(Equal(25)) - - // Let it run for 2 minutes and check for duplicates - Consistently(func() int { - return kb.Count() - }, 3*time.Minute, 5*time.Second).Should(Equal(25)) - - // Verify that search results don't contain duplicates - Consistently(func() bool { - results, err := kb.Search("What is the Black-crowned barwing?", 5) - if err != nil { - fmt.Println("Error searching for content", err) - return false - } - - // Check for duplicate content - seen := make(map[string]bool) - for _, r := range results { - if seen[r.Content] { - fmt.Println("Found a duplicate", r.Content) - return false // Found a duplicate + // StoreOrReplace publishes a fresh UUID document key under the KB lock + // only after replacing chunks. A changed key proves a completed update; + // LastUpdate is set BEFORE fetching and cannot prove successful storage. + previous := "" + for cycle := 0; cycle < 3; cycle++ { + Eventually(func() string { + docs := kb.ListDocuments() + if len(docs) != 1 || docs[0] == previous { + return "" } - seen[r.Content] = true - } - return true - }, 2*time.Minute, 5*time.Second).Should(BeTrue()) + return docs[0] + }, 90*time.Second, TestPollingInterval).ShouldNot(BeEmpty()) + docs := kb.ListDocuments() + Expect(docs).To(HaveLen(1)) + previous = docs[0] + Expect(kb.Count()).To(Equal(1), "chunk count after completed update %d", cycle+1) + results, err := kb.Search("What is the Black-crowned barwing?", 1) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(HaveLen(1)) + Expect(results[0].Content).To(ContainSubstring("Black-crowned barwing")) + } + }) }) diff --git a/test/raw/CI.md b/test/raw/CI.md new file mode 100644 index 0000000..73dcda0 --- /dev/null +++ b/test/raw/CI.md @@ -0,0 +1,30 @@ +# CI fixture selection + +`make test-unit` builds `test/raw/Dockerfile.postgres` and executes all `rag`, +`pkg`, and root tests via `unit.sh`, giving each package a fresh `database.sh` fixture. This includes every raw PostgreSQL +test (outage/recovery, shutdown, process discovery, search and recall), plus raw +HTTP tests. Packages run serially (`-p=1`): outage tests intentionally stop their +owned database and must not interrupt another package. Pristine per-package +databases also keep legacy BM25 objects and registries out of raw bootstrap tests. No database-dependent +raw test is skipped or excluded; absent dependencies fail the job. + +`make run-e2e` runs the conventional E2E suite and then creates a separate owned +database and host API for `TestOwnedLegacyLifecycle` via +`legacy.sh --owned --e2e-only`. That test is classified with `ownedintegration` +because it must never target the conventional/shared server. The CI invocation +uses the already warmed real embedding provider and configured embedding model; +the private HTTP source is reachable by the owned host API. `legacy.sh` enables +the tag explicitly and checks for a PASS (missing tests/skips fail). + +`sh test/raw/verify.sh all` still executes its complete raw, legacy and actual +LocalAI consumer gates; its existing provisioned native-provider prerequisites +are unchanged. `legacy.sh` without `--e2e-only` still runs all legacy suites. + +On shared development hosts, do not run the compose-based Make targets. Use +`database.sh` for a disposable PostgreSQL fixture and the documented private +LocalAI runner for genuine embeddings. CI compose runs are on isolated runners. + +The fixture exports both `LOCALRECALL_TEST_DATABASE_URL` and +`POSTGRES_TEST_URL`, so legacy planning tests never fall back to localhost. +Race invocations in `legacy.sh` explicitly enable CGO, overriding the Makefile +default only for those commands. diff --git a/test/raw/database.sh b/test/raw/database.sh index db02a50..253be2d 100644 --- a/test/raw/database.sh +++ b/test/raw/database.sh @@ -47,6 +47,7 @@ SQL port=$(docker port "$name" 5432/tcp | sed -n 's/^127\.0\.0\.1://p') case "$port" in ''|*[!0-9]*) exit 1;; esac export LOCALRECALL_TEST_DATABASE_URL="postgres://postgres:$token@127.0.0.1:$port/raw_fixture?sslmode=disable" +export POSTGRES_TEST_URL="$LOCALRECALL_TEST_DATABASE_URL" export LOCALRECALL_TEST_DATABASE_OWNER="$token" export LOCALRECALL_TEST_DATABASE_CONTAINER="$name" printf 'Owned fixture %s at loopback port %s (removed on exit)\n' "$name" "$port" diff --git a/test/raw/legacy.sh b/test/raw/legacy.sh index 2e4d0c5..cfcd697 100644 --- a/test/raw/legacy.sh +++ b/test/raw/legacy.sh @@ -12,18 +12,21 @@ fi export POSTGRES_TEST_URL="$LOCALRECALL_TEST_DATABASE_URL" export COLLECTION_DB_PATH="$PWD/.sdd-work/legacy/$LOCALRECALL_TEST_DATABASE_OWNER/collections" FILE_ASSETS="$PWD/.sdd-work/legacy/$LOCALRECALL_TEST_DATABASE_OWNER/assets" mkdir -p "$COLLECTION_DB_PATH" "$FILE_ASSETS" +mkdir -p "$PWD/.sdd-work/logs" +if [ "${2:-}" != --e2e-only ]; then log="$PWD/.sdd-work/logs/legacy.log" -if go test -v -mod=readonly -race -count=1 -p=1 -timeout=15m ./rag ./rag/engine ./rag/sources ./pkg/chunk ./pkg/client . ./test/integration >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi +if CGO_ENABLED=1 go test -v -mod=readonly -race -count=1 -p=1 -timeout=15m ./rag ./rag/engine ./rag/sources ./pkg/chunk ./pkg/client . ./test/integration >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi # Ginkgo's summary includes '0 Skipped'; reject nonzero counts, too. ! grep -E -- '--- SKIP:|[1-9][0-9]* Skipped|no tests to run|no test files' "$log" for suite in TestRag TestEngine TestChunk TestIntegration TestCreateCollectionReusesLoadedCollection; do grep -q -- "--- PASS: $suite" "$log" done +fi # Launch an owned real API on a dynamically allocated loopback port. go build -mod=readonly -o .sdd-work/legacy/localrecall . port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])') export LOCALRECALL_ENDPOINT="http://127.0.0.1:$port" LISTENING_ADDRESS="127.0.0.1:$port" -export DATABASE_URL="$LOCALRECALL_TEST_DATABASE_URL" VECTOR_ENGINE=postgres EMBEDDING_MODEL=minilm OPENAI_BASE_URL="$LOCALAI_ENDPOINT" +export DATABASE_URL="$LOCALRECALL_TEST_DATABASE_URL" VECTOR_ENGINE=postgres EMBEDDING_MODEL="${EMBEDDING_MODEL:-minilm}" OPENAI_BASE_URL="$LOCALAI_ENDPOINT" .sdd-work/legacy/localrecall >.sdd-work/logs/legacy-server.log 2>&1 & pid=$! trap 'kill "$pid" 2>/dev/null || true; wait "$pid" 2>/dev/null || true' EXIT HUP INT TERM @@ -39,6 +42,6 @@ for i in range(60): time.sleep(1) PY log="$PWD/.sdd-work/logs/legacy-e2e.log" -if go test -v -mod=readonly -race -count=1 -timeout=5m ./test/e2e -run '^TestOwnedLegacyLifecycle$' >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi +if CGO_ENABLED=1 go test -tags=ownedintegration -v -mod=readonly -race -count=1 -timeout=5m ./test/e2e -run '^TestOwnedLegacyLifecycle$' >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi grep -q -- '--- PASS: TestOwnedLegacyLifecycle' "$log" ! grep -E -- '--- SKIP:|no tests to run|no test files' "$log" diff --git a/test/raw/unit.sh b/test/raw/unit.sh new file mode 100644 index 0000000..76fee3c --- /dev/null +++ b/test/raw/unit.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Each package owns a pristine database: legacy catalogs must not contaminate +# raw bootstrap/extension tests, and outage tests cannot disrupt other packages. +set -eu +python3 test/raw/confinement.py "$(pwd -P)" +packages=$(go list ./rag/... ./pkg/... .) +for package in $packages; do + sh test/raw/database.sh go test -v -count=1 -p=1 "$package" +done