From 26175f22233126bbe132eba2d83be7faa7ecc4b4 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Mon, 12 Oct 2026 20:19:45 +0000 Subject: [PATCH 01/43] feat: add reusable egress policies for agents and sandboxes Signed-off-by: Eitan Yarmush --- docs/architecture/README.md | 3 + docs/architecture/credential-injection.md | 25 ++- docs/architecture/egress-policies.md | 186 +++++++++++++++++ docs/architecture/sandboxes.md | 6 +- .../typed/api/v1alpha3/api_client.go | 5 + .../typed/api/v1alpha3/egresspolicy.go | 52 +++++ .../api/v1alpha3/fake/fake_api_client.go | 4 + .../api/v1alpha3/fake/fake_egresspolicy.go | 36 ++++ .../typed/api/v1alpha3/generated_expansion.go | 2 + go/api/v1alpha3/agent_types.go | 10 + go/api/v1alpha3/configuration_crd_cel_test.go | 4 +- go/api/v1alpha3/egresspolicy_cel_test.go | 102 +++++++++ go/api/v1alpha3/egresspolicy_types.go | 88 ++++++++ go/api/v1alpha3/sandboxtemplate_types.go | 13 ++ go/api/v1alpha3/zz_generated.deepcopy.go | 131 ++++++++++++ .../internal/controller/apiclient/types.go | 13 ++ go/core/internal/controller/collections.go | 6 +- .../controller/sandbox_collections.go | 11 +- .../controller/sandbox_integration_test.go | 57 +++++ .../internal/controller/sandbox_reconciler.go | 3 +- .../database/runtime_revisions_test.go | 6 +- .../internal/database/sandbox_revisions.go | 13 +- .../database/sandbox_revisions_test.go | 46 +++++ go/core/internal/egress/credentials.go | 50 +++-- go/core/internal/egress/credentials_test.go | 10 +- go/core/internal/egress/destination.go | 78 +++++++ go/core/internal/egress/destination_test.go | 42 ++++ .../internal/service/sandbox/service_test.go | 23 +++ go/core/internal/service/sandbox/workflow.go | 2 +- .../internal/service/session/workflow_test.go | 16 +- go/core/internal/substrate/policy.go | 81 ++------ go/core/internal/substrate/policy_test.go | 28 +++ go/core/internal/substrate/sandbox.go | 14 +- go/core/internal/substrate/sandbox_test.go | 9 +- go/core/internal/translator/collections.go | 1 + go/core/internal/translator/compiler.go | 31 ++- go/core/internal/translator/compiler_test.go | 7 +- go/core/internal/translator/credentials.go | 14 +- .../internal/translator/credentials_test.go | 13 +- go/core/internal/translator/egress.go | 74 +++++++ go/core/internal/translator/egress_test.go | 194 ++++++++++++++++++ go/core/internal/translator/foundry_test.go | 2 +- go/core/test/e2e/egress_test.go | 40 +++- .../templates/api.kagent.dev_agents.yaml | 29 +++ .../api.kagent.dev_egresspolicies.yaml | 141 +++++++++++++ .../api.kagent.dev_sandboxtemplates.yaml | 28 +++ helm/kagent/templates/rbac/getter-role.yaml | 1 + ui/src/api/domain/agents.ts | 1 + ui/src/pages/agents/AgentFormPage.tsx | 1 + 49 files changed, 1609 insertions(+), 143 deletions(-) create mode 100644 docs/architecture/egress-policies.md create mode 100644 go/api/clientset/versioned/typed/api/v1alpha3/egresspolicy.go create mode 100644 go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_egresspolicy.go create mode 100644 go/api/v1alpha3/egresspolicy_cel_test.go create mode 100644 go/api/v1alpha3/egresspolicy_types.go create mode 100644 go/core/internal/database/sandbox_revisions_test.go create mode 100644 go/core/internal/substrate/policy_test.go create mode 100644 go/core/internal/translator/egress.go create mode 100644 go/core/internal/translator/egress_test.go create mode 100644 helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 4502a8a92f..d3f0598df8 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -14,6 +14,7 @@ the agent processes. | `Harness` | Kubernetes (`api.kagent.dev/v1alpha3`) | Runtime implementation, workload, credentials, capacity, and snapshot policy | | `AgentTemplate` | Kubernetes (`api.kagent.dev/v1alpha3`) | Portable agent behavior: model, prompt, tools, skills, and plugins | | `SandboxTemplate` | Kubernetes (`api.kagent.dev/v1alpha3`) | Tools image, environment, and preparation policy for standalone sandboxes | +| `EgressPolicy` | Kubernetes (`api.kagent.dev/v1alpha3`) | Additional HTTP(S) origins and gateway credentials selected by Agents and SandboxTemplates | | prepared revision | PostgreSQL and ate-api | Immutable compiled runtime input and its Substrate ActorTemplate | | `Session` | PostgreSQL, exposed by gRPC | Stable conversation identity and runtime lifecycle | | `Sandbox` | PostgreSQL, exposed by gRPC and MCP | Owner-scoped scratch environment with process/file access and expiration | @@ -95,6 +96,8 @@ cleanup. Session lifecycle workers independently pause/suspend idle Actors. ## Documents +- [Egress policies](egress-policies.md) +- [Runtime credential injection](credential-injection.md) - [Configuration and compilation](configuration-and-compilation.md) - [Runtime and lifecycle](runtime-and-lifecycle.md) - [Client lifecycle retries](../lifecycle-retries.md) diff --git a/docs/architecture/credential-injection.md b/docs/architecture/credential-injection.md index 3e29ad1f5c..aa340ffcb8 100644 --- a/docs/architecture/credential-injection.md +++ b/docs/architecture/credential-injection.md @@ -1,8 +1,8 @@ # Runtime credential injection -Kagent requires Substrate **v0.4.0-alpha1**. The compiler turns ModelConfig API -keys and Secret-backed RemoteMCPServer headers into destination-scoped egress -bindings. Substrate's gateway fetches the referenced Kubernetes Secret and +Kagent requires Substrate **v0.5.0-alpha2**. The compiler turns ModelConfig API +keys, Secret-backed RemoteMCPServer headers, and explicit EgressPolicy +authentication into destination-scoped egress bindings. Substrate's gateway fetches the referenced Kubernetes Secret and replaces the outgoing HTTP header when the request carries a placeholder. SDKs receive an inert placeholder where they require an API key; real credentials never enter compiled environments, runtime configuration, or revision provenance. @@ -58,22 +58,25 @@ overrides of these variables are rejected. | RemoteMCPServer Secret-backed header | Configured header; Secret contains its full value | Provider endpoint overrides determine the allowed HTTP(S) origin. Egress rules -match its scheme, DNS name, and port; credential bindings remain scoped to the -hostname and header. Different credentials for the same hostname and header -are rejected, including conflicts between models, memory embeddings, and MCP -servers. Use distinct DNS names for -origins requiring different credentials. IP-address destinations are unsupported -by Substrate egress policies. +and credential bindings both match its scheme, DNS name, and port. Different +credentials for the same origin and header are rejected, including conflicts +between models, memory embeddings, MCP servers, and explicit policies. Distinct +ports on the same DNS name can use different credentials. IP-address destinations +are unsupported by Substrate egress policies. + +The v1alpha3 revision format stores full origins. It assumes a clean installation; +previously persisted hostname-only credential bindings are rejected rather than +reinterpreted across every port on that host. Harness and SandboxTemplate environment entries accept only literal `value` strings, including empty strings. Configure Secret-backed credentials on -ModelConfig or RemoteMCPServer for gateway injection. +ModelConfig, RemoteMCPServer, or [EgressPolicy](egress-policies.md) for gateway injection. AWS IAM signing keys, Google service-account keys, and OAuth client credentials require mechanisms beyond static header injection and are rejected rather than serialized into runtimes. Caller-token passthrough retains its existing behavior. A passthrough model -cannot share a hostname with static gateway credentials, which would override +cannot share an origin with static gateway credentials, which would override the caller's authentication. ## Deferred API fields diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md new file mode 100644 index 0000000000..d7b646fdda --- /dev/null +++ b/docs/architecture/egress-policies.md @@ -0,0 +1,186 @@ +# Egress policies + +`EgressPolicy` grants additional HTTP(S) access to an Agent or standalone +sandbox. An Agent selects policies through `spec.egressPolicyRefs`; a +SandboxTemplate selects them through the same field. References resolve in the +workload's namespace. Policies are managed through the Kubernetes API. + +AgentTemplate owns reusable behavior, and Harness owns how it runs. Neither +selects egress policies. This lets two Agents reuse a template while granting +different access for their tasks. A shared child template contributes its +inferred model, MCP, and artifact destinations, but cannot select additional +policies. The root and all shared children execute in one Actor and share its +complete grant; there is no per-child network isolation. + +## Grant access for a task + +For example, a dependency-installation policy can grant access to registries: + +```yaml +apiVersion: api.kagent.dev/v1alpha3 +kind: EgressPolicy +metadata: + name: node-dependencies + namespace: team-a +spec: + destinations: + - origin: https://registry.npmjs.org + - origin: https://nodejs.org +--- +apiVersion: api.kagent.dev/v1alpha3 +kind: Agent +metadata: + name: code-reviewer + namespace: team-a +spec: + template: + ref: + name: code-review + harnessRef: + name: codex + egressPolicyRefs: + - name: node-dependencies +``` + +An origin is a scheme, DNS name, and optional port. Omitted ports mean 443 for +HTTPS and 80 for HTTP. Every path on that origin is allowed. URL paths, queries, +fragments, user information, IP addresses, and non-HTTP protocols are rejected. +A wildcard replaces exactly one leftmost label: `https://*.example.com` +allows `api.example.com`, but neither `example.com` nor `a.b.example.com`. +`*` and `*.com` are not accepted. Redirects and downloads from other origins +require their own grants. A registry grant is not a guarantee that every +package's install script or release download can reach its dependencies. + +For a standalone sandbox, select the policy on its template: + +```yaml +apiVersion: api.kagent.dev/v1alpha3 +kind: SandboxTemplate +metadata: + name: node-tools + namespace: team-a +spec: + workload: + image: registry.example.com/tools@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + substrate: + workerPoolRef: + name: default + snapshotPolicy: + location: s3://example/snapshots/ + egressPolicyRefs: + - name: node-dependencies +``` + +Each workload can select up to 16 policies, each containing 1–64 destinations. +The compiler combines them with inferred integration access, removes equivalent +origins, and orders the result deterministically. The final Actor policy is +limited to 256 rules. References are additive; they cannot restrict inferred +model, MCP, artifact, or control-plane access. No references means no additional +access. A standalone sandbox with no references has no allowed egress. + +## Call an API with a credential + +Policies authenticate requests using a header, an optional prefix, and a Secret +in the same namespace. Authentication requires an exact HTTPS origin: + +```yaml +apiVersion: api.kagent.dev/v1alpha3 +kind: EgressPolicy +metadata: + name: ticket-api + namespace: team-a +spec: + destinations: + - origin: https://tickets.example.com + authentication: + header: Authorization + prefix: "Bearer " + secretKeyRef: + name: ticket-auth + key: token + - origin: https://search.example.com:8443 + authentication: + header: X-API-Key + secretKeyRef: + name: search-auth + key: api-key +``` + +The gateway sets the header value to `prefix + secretValue`. Prefixes are literal: +include the trailing space in `"Bearer "`. Omitting the prefix or setting it to +an empty string uses the Secret value directly, as with an API-key header. + +Create these Secrets separately. The compiler validates that each Secret and key +exists, but persists only the reference. Substrate's credential provider fetches +the value and the gateway replaces the request header. Scripts and SDKs must +send that header, using an inert placeholder: + +```sh +curl -H 'Authorization: Bearer kagent-credential-injected' \ + https://tickets.example.com/tickets +curl -H 'X-API-Key: kagent-credential-injected' \ + https://search.example.com:8443/search +``` + +The gateway replaces an existing header; it does not add a missing one. Policy +selection does not automatically configure arbitrary tools or environment +variables. Clients must honor the projected gateway trust bundle. See +[credential injection](credential-injection.md) for provider namespace grants, +CA setup, inferred SDK placeholders, and rotation behavior. + +Bindings use the complete origin and a case-insensitive header name. Different +ports can use different credentials. Identical bindings merge; conflicting +Secret references or prefixes for the same origin/header fail preparation. +Explicit and inferred bindings follow the same rule. Caller-token passthrough +cannot share its origin with static gateway credentials. Routing, framing, +hop-by-hop, proxy-authentication, and Cookie headers cannot carry credentials. +At most 16 credential headers are supported per origin. + +An exact credential-bearing rule wins over a matching wildcard rule. Credentials +are never attached to wildcard rules or implicitly copied to other ports or +schemes. HTTP and HTTPS cannot both use the same hostname pattern and port, +because Substrate cannot distinguish that rule selection. + +## Preparation and lifecycle + +KRT watches selected policies and their Secrets. Missing policies, missing +Secret keys, invalid destinations, and conflicting bindings prevent preparation. +The controller publishes the error on the workload's status and re-evaluates it +when dependencies change. + +Resolved destinations and credential references enter the immutable revision. +Changing a grant or Secret reference changes that revision. Secret values and +UIDs do not; the gateway handles value rotation independently. Policy metadata +changes do not affect the resolved grant. Reference ordering does not affect +revision identity. + +New Sessions and sandboxes pin a prepared revision. Existing instances keep +their original grant through retries and resume; Session checkpoint forks also +retain the source revision. Updating or deleting a policy is not live revocation. +Failed preparation does not overwrite a previous successful revision, and the +existing readiness rules prevent selecting stale inputs for new instances. + +The runtime workflow creates a suspended Actor, installs its compiled `default` +egress policy, and resumes it only after installation succeeds. A retry uses the +same pinned revision. Golden Actor preparation and artifact-startup egress are +unchanged; this API controls runtime Actor access. + +## Authorization and scope + +Policies are namespace-scoped configuration. Administrators can grant separate +RBAC permissions to author EgressPolicy objects; kagent's controller needs only +get/list/watch. Workload authors who can select a policy in that namespace can +use its grant. There is no separate per-policy `use` authorization check. Treat +the namespace as the sharing boundary for selectable policies and credentials. + +The same trust-boundary consideration applies to existing ModelConfig and +RemoteMCPServer Secret references. This resource does not turn the namespace +into a per-workload Secret isolation boundary. Credential injection keeps values +out of compiled inputs; it does not prevent an allowed destination from echoing +credentials back to a client. Do not grant credentials to destinations that +should not receive or disclose them. + +This API intentionally does not expose Substrate rule effects, raw TCP/SSH, +TLS passthrough, OAuth exchange, request signing, cross-namespace references, +or unrestricted public-internet access. Substrate remains the enforcement +backend; kagent exposes task-oriented reusable grants and static authentication. diff --git a/docs/architecture/sandboxes.md b/docs/architecture/sandboxes.md index c2aabd6673..f929128394 100644 --- a/docs/architecture/sandboxes.md +++ b/docs/architecture/sandboxes.md @@ -113,8 +113,10 @@ TTL. Helm supplies these through the controller ConfigMap. Defaults are one CPU, 1 GiB memory, a one-hour TTL, and a 24-hour maximum TTL. Activity does not extend expiration. -Sandboxes have no allowed egress destinations. A follow-up will add destination -configuration to `SandboxTemplate` and pin it with each prepared revision. +Sandboxes have no allowed egress destinations by default. Select same-namespace +[EgressPolicy resources](egress-policies.md) through `SandboxTemplate.spec.egressPolicyRefs` +to grant additional HTTP(S) origins and static gateway credentials. Resolved +grants are pinned with the prepared revision; policy edits affect new sandboxes. ## Lifecycle and recovery diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/api_client.go b/go/api/clientset/versioned/typed/api/v1alpha3/api_client.go index 97de970b4a..901e7cf393 100644 --- a/go/api/clientset/versioned/typed/api/v1alpha3/api_client.go +++ b/go/api/clientset/versioned/typed/api/v1alpha3/api_client.go @@ -14,6 +14,7 @@ type ApiV1alpha3Interface interface { RESTClient() rest.Interface AgentsGetter AgentTemplatesGetter + EgressPoliciesGetter HarnessesGetter ModelConfigsGetter RemoteMCPServersGetter @@ -33,6 +34,10 @@ func (c *ApiV1alpha3Client) AgentTemplates(namespace string) AgentTemplateInterf return newAgentTemplates(c, namespace) } +func (c *ApiV1alpha3Client) EgressPolicies(namespace string) EgressPolicyInterface { + return newEgressPolicies(c, namespace) +} + func (c *ApiV1alpha3Client) Harnesses(namespace string) HarnessInterface { return newHarnesses(c, namespace) } diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/egresspolicy.go b/go/api/clientset/versioned/typed/api/v1alpha3/egresspolicy.go new file mode 100644 index 0000000000..140295069e --- /dev/null +++ b/go/api/clientset/versioned/typed/api/v1alpha3/egresspolicy.go @@ -0,0 +1,52 @@ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha3 + +import ( + context "context" + + scheme "github.com/kagent-dev/kagent/go/api/clientset/versioned/scheme" + apiv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// EgressPoliciesGetter has a method to return a EgressPolicyInterface. +// A group's client should implement this interface. +type EgressPoliciesGetter interface { + EgressPolicies(namespace string) EgressPolicyInterface +} + +// EgressPolicyInterface has methods to work with EgressPolicy resources. +type EgressPolicyInterface interface { + Create(ctx context.Context, egressPolicy *apiv1alpha3.EgressPolicy, opts v1.CreateOptions) (*apiv1alpha3.EgressPolicy, error) + Update(ctx context.Context, egressPolicy *apiv1alpha3.EgressPolicy, opts v1.UpdateOptions) (*apiv1alpha3.EgressPolicy, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*apiv1alpha3.EgressPolicy, error) + List(ctx context.Context, opts v1.ListOptions) (*apiv1alpha3.EgressPolicyList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *apiv1alpha3.EgressPolicy, err error) + EgressPolicyExpansion +} + +// egressPolicies implements EgressPolicyInterface +type egressPolicies struct { + *gentype.ClientWithList[*apiv1alpha3.EgressPolicy, *apiv1alpha3.EgressPolicyList] +} + +// newEgressPolicies returns a EgressPolicies +func newEgressPolicies(c *ApiV1alpha3Client, namespace string) *egressPolicies { + return &egressPolicies{ + gentype.NewClientWithList[*apiv1alpha3.EgressPolicy, *apiv1alpha3.EgressPolicyList]( + "egresspolicies", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *apiv1alpha3.EgressPolicy { return &apiv1alpha3.EgressPolicy{} }, + func() *apiv1alpha3.EgressPolicyList { return &apiv1alpha3.EgressPolicyList{} }, + ), + } +} diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_api_client.go b/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_api_client.go index 19d7ba79b7..931f3f6f1a 100644 --- a/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_api_client.go +++ b/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_api_client.go @@ -20,6 +20,10 @@ func (c *FakeApiV1alpha3) AgentTemplates(namespace string) v1alpha3.AgentTemplat return newFakeAgentTemplates(c, namespace) } +func (c *FakeApiV1alpha3) EgressPolicies(namespace string) v1alpha3.EgressPolicyInterface { + return newFakeEgressPolicies(c, namespace) +} + func (c *FakeApiV1alpha3) Harnesses(namespace string) v1alpha3.HarnessInterface { return newFakeHarnesses(c, namespace) } diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_egresspolicy.go b/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_egresspolicy.go new file mode 100644 index 0000000000..6375df89c7 --- /dev/null +++ b/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_egresspolicy.go @@ -0,0 +1,36 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + apiv1alpha3 "github.com/kagent-dev/kagent/go/api/clientset/versioned/typed/api/v1alpha3" + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + gentype "k8s.io/client-go/gentype" +) + +// fakeEgressPolicies implements EgressPolicyInterface +type fakeEgressPolicies struct { + *gentype.FakeClientWithList[*v1alpha3.EgressPolicy, *v1alpha3.EgressPolicyList] + Fake *FakeApiV1alpha3 +} + +func newFakeEgressPolicies(fake *FakeApiV1alpha3, namespace string) apiv1alpha3.EgressPolicyInterface { + return &fakeEgressPolicies{ + gentype.NewFakeClientWithList[*v1alpha3.EgressPolicy, *v1alpha3.EgressPolicyList]( + fake.Fake, + namespace, + v1alpha3.SchemeGroupVersion.WithResource("egresspolicies"), + v1alpha3.SchemeGroupVersion.WithKind("EgressPolicy"), + func() *v1alpha3.EgressPolicy { return &v1alpha3.EgressPolicy{} }, + func() *v1alpha3.EgressPolicyList { return &v1alpha3.EgressPolicyList{} }, + func(dst, src *v1alpha3.EgressPolicyList) { dst.ListMeta = src.ListMeta }, + func(list *v1alpha3.EgressPolicyList) []*v1alpha3.EgressPolicy { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1alpha3.EgressPolicyList, items []*v1alpha3.EgressPolicy) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/generated_expansion.go b/go/api/clientset/versioned/typed/api/v1alpha3/generated_expansion.go index bc2146b2a4..da4673fc10 100644 --- a/go/api/clientset/versioned/typed/api/v1alpha3/generated_expansion.go +++ b/go/api/clientset/versioned/typed/api/v1alpha3/generated_expansion.go @@ -6,6 +6,8 @@ type AgentExpansion interface{} type AgentTemplateExpansion interface{} +type EgressPolicyExpansion interface{} + type HarnessExpansion interface{} type ModelConfigExpansion interface{} diff --git a/go/api/v1alpha3/agent_types.go b/go/api/v1alpha3/agent_types.go index 67bae21fba..80bee768a3 100644 --- a/go/api/v1alpha3/agent_types.go +++ b/go/api/v1alpha3/agent_types.go @@ -52,6 +52,16 @@ type AgentSpec struct { // +kubebuilder:validation:XValidation:rule="has(self.name) && self.name != ''",message="harnessRef.name must not be empty" // +required HarnessRef corev1.LocalObjectReference `json:"harnessRef"` + + // EgressPolicyRefs grants additional access to the whole runtime, including + // shared subagents. Policies resolve in this Agent's namespace. Integration + // access inferred from models and tools is always included. + // +optional + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:XValidation:rule="self.all(ref, has(ref.name) && ref.name != '')",message="egressPolicyRefs requires non-empty names" + // +listType=map + // +listMapKey=name + EgressPolicyRefs []corev1.LocalObjectReference `json:"egressPolicyRefs,omitempty"` } // AgentTemplateSource selects portable behavior inline or by local reference. diff --git a/go/api/v1alpha3/configuration_crd_cel_test.go b/go/api/v1alpha3/configuration_crd_cel_test.go index 02d17d4b57..8d2b5478b7 100644 --- a/go/api/v1alpha3/configuration_crd_cel_test.go +++ b/go/api/v1alpha3/configuration_crd_cel_test.go @@ -56,7 +56,7 @@ func TestConfigurationCRDValidation(t *testing.T) { names = append(names, resource.Name) } } - require.ElementsMatch(t, []string{"agents", "agenttemplates", "harnesses", "modelconfigs", "modelproviderconfigs", "remotemcpservers", "sandboxtemplates"}, names) + require.ElementsMatch(t, []string{"agents", "agenttemplates", "egresspolicies", "harnesses", "modelconfigs", "modelproviderconfigs", "remotemcpservers", "sandboxtemplates"}, names) _, err = discoveryClient.ServerResourcesForGroupVersion("kagent.dev/v1alpha3") require.True(t, apierrors.IsNotFound(err), "the new CRDs must not publish the legacy group: %v", err) }) @@ -462,6 +462,8 @@ func TestConfigurationCRDValidation(t *testing.T) { }) } + t.Run("egress policies", func(t *testing.T) { testEgressPolicyValidation(t, cl, namespace) }) + t.Run("literal runtime environment", func(t *testing.T) { harness := validHarness(namespace, "env-harness", HarnessSpec{Kagent: &KagentHarness{}}) for _, resource := range []struct { diff --git a/go/api/v1alpha3/egresspolicy_cel_test.go b/go/api/v1alpha3/egresspolicy_cel_test.go new file mode 100644 index 0000000000..8b5a9e54c0 --- /dev/null +++ b/go/api/v1alpha3/egresspolicy_cel_test.go @@ -0,0 +1,102 @@ +package v1alpha3 + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func testEgressPolicyValidation(t *testing.T, cl ctrlclient.Client, namespace string) { + secret := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}, Key: "token"} + bearer := &EgressAuthentication{Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: secret} + for i, tc := range []struct { + name string + destination EgressDestination + reject bool + }{ + {"HTTPS", EgressDestination{Origin: "https://api.example.com"}, false}, + {"HTTP", EgressDestination{Origin: "http://registry.example.com:8080"}, false}, + {"wildcard", EgressDestination{Origin: "https://*.example.com"}, false}, + {"bearer", EgressDestination{Origin: "https://api.example.com:8443", Authentication: bearer}, false}, + {"API key", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "X-API-Key", SecretKeyRef: secret}}, false}, + {"empty prefix", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "X-API-Key", Prefix: new(""), SecretKeyRef: secret}}, false}, + {"custom prefix", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "Authorization", Prefix: new("Token "), SecretKeyRef: secret}}, false}, + {"HTTP auth", EgressDestination{Origin: "http://api.example.com", Authentication: bearer}, true}, + {"wildcard auth", EgressDestination{Origin: "https://*.example.com", Authentication: bearer}, true}, + {"empty auth", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{}}, true}, + {"missing header", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{SecretKeyRef: secret}}, true}, + {"path", EgressDestination{Origin: "https://api.example.com/v1"}, true}, + {"userinfo", EgressDestination{Origin: "https://user@api.example.com"}, true}, + {"query", EgressDestination{Origin: "https://api.example.com?"}, true}, + {"IP", EgressDestination{Origin: "https://192.0.2.1"}, true}, + {"all hosts", EgressDestination{Origin: "https://*"}, true}, + {"top level wildcard", EgressDestination{Origin: "https://*.com"}, true}, + {"interior wildcard", EgressDestination{Origin: "https://api.*.com"}, true}, + {"port zero", EgressDestination{Origin: "https://api.example.com:0"}, true}, + {"port overflow", EgressDestination{Origin: "https://api.example.com:65536"}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + policy := &EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: fmt.Sprintf("egress-%d", i)}, Spec: EgressPolicySpec{Destinations: []EgressDestination{tc.destination}}} + err := cl.Create(t.Context(), policy) + if tc.reject { + require.Error(t, err) + } else { + require.NoError(t, err) + require.NoError(t, cl.Get(t.Context(), ctrlclient.ObjectKeyFromObject(policy), policy)) + require.Equal(t, tc.destination.Authentication, policy.Spec.Destinations[0].Authentication) + } + }) + } + for i, header := range []string{"Host", "cOnNeCtIoN", "Content-Length", "Transfer-Encoding", "Cookie", "Proxy-Authorization", "X-Key\r\nInjected"} { + policy := &EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: fmt.Sprintf("bad-header-%d", i)}, Spec: EgressPolicySpec{Destinations: []EgressDestination{{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: header, SecretKeyRef: secret}}}}} + require.Error(t, cl.Create(t.Context(), policy), header) + } + for i, prefix := range []string{"Bearer\r", "Bearer\n", "\x00", "\x08", "\x0b", "\x1f", "\x7f", strings.Repeat("x", 257)} { + policy := &EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: fmt.Sprintf("bad-prefix-%d", i)}, Spec: EgressPolicySpec{Destinations: []EgressDestination{{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "Authorization", Prefix: new(prefix), SecretKeyRef: secret}}}}} + require.Error(t, cl.Create(t.Context(), policy), "prefix %q", prefix) + } + for i, ref := range []corev1.SecretKeySelector{{Key: "token"}, {LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}}, {LocalObjectReference: secret.LocalObjectReference, Key: "token", Optional: new(true)}} { + policy := &EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: fmt.Sprintf("bad-secret-%d", i)}, Spec: EgressPolicySpec{Destinations: []EgressDestination{{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "Authorization", SecretKeyRef: ref}}}}} + require.Error(t, cl.Create(t.Context(), policy)) + } + for _, kind := range []string{"Agent", "SandboxTemplate", "AgentTemplate", "Harness"} { + t.Run(kind+" attachment", func(t *testing.T) { + var object ctrlclient.Object + switch kind { + case "Agent": + object = &Agent{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "egress-agent"}, Spec: AgentSpec{Template: AgentTemplateSource{Ref: &corev1.LocalObjectReference{Name: "behavior"}}, HarnessRef: corev1.LocalObjectReference{Name: "runner"}}} + case "SandboxTemplate": + object = sandboxTemplateForValidation(namespace, "egress-sandbox", nil) + case "AgentTemplate": + object = validAgentTemplate(namespace, "egress-template", nil) + case "Harness": + object = validHarness(namespace, "egress-harness", HarnessSpec{Kagent: &KagentHarness{}}) + } + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object) + require.NoError(t, err) + resource := &unstructured.Unstructured{Object: data} + resource.SetAPIVersion(GroupVersion.String()) + resource.SetKind(kind) + require.NoError(t, unstructured.SetNestedSlice(data, []any{map[string]any{"name": "tickets"}}, "spec", "egressPolicyRefs")) + err = cl.Create(t.Context(), resource, &ctrlclient.CreateOptions{FieldValidation: metav1.FieldValidationStrict}) + if kind == "AgentTemplate" || kind == "Harness" { + require.ErrorContains(t, err, "unknown field") + return + } + require.NoError(t, err) + require.NoError(t, cl.Get(t.Context(), ctrlclient.ObjectKeyFromObject(resource), resource)) + refs, _, err := unstructured.NestedSlice(resource.Object, "spec", "egressPolicyRefs") + require.NoError(t, err) + require.Equal(t, []any{map[string]any{"name": "tickets"}}, refs) + require.NoError(t, unstructured.SetNestedSlice(resource.Object, []any{map[string]any{"name": ""}}, "spec", "egressPolicyRefs")) + require.Error(t, cl.Update(t.Context(), resource)) + }) + } +} diff --git a/go/api/v1alpha3/egresspolicy_types.go b/go/api/v1alpha3/egresspolicy_types.go new file mode 100644 index 0000000000..d93a1a1e01 --- /dev/null +++ b/go/api/v1alpha3/egresspolicy_types.go @@ -0,0 +1,88 @@ +// Copyright 2026 The kagent Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha3 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// EgressPolicySpec grants additional HTTP(S) access to workloads in its namespace. +type EgressPolicySpec struct { + // Destinations are additional origins the workload may reach. + // +required + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 + // +listType=map + // +listMapKey=origin + Destinations []EgressDestination `json:"destinations"` +} + +// EgressDestination grants access to one HTTP(S) origin, including all its paths. +// +kubebuilder:validation:XValidation:rule="!has(self.authentication) || (self.origin.startsWith('https://') && !self.origin.contains('*'))",message="authentication requires an exact HTTPS origin" +type EgressDestination struct { + // Origin is a scheme, DNS hostname, and optional port. A wildcard replaces + // exactly one leftmost DNS label, for example https://*.example.com. + // +required + // +kubebuilder:validation:MaxLength=270 + // +kubebuilder:validation:Pattern=`^https?://(\*\.([a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?\.)+|([a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?\.)*)[a-z]([-a-z0-9]{0,61}[a-z0-9])?(:([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?$` + Origin string `json:"origin"` + // Authentication replaces a placeholder header at the gateway. Requests + // must send the header; its credential value is never supplied to the runtime. + // +optional + Authentication *EgressAuthentication `json:"authentication,omitempty"` +} + +// EgressAuthentication replaces a request header with a prefix and Secret value. +type EgressAuthentication struct { + // Header is the request header carrying the credential, for example + // Authorization or X-API-Key. + // +required + // +kubebuilder:validation:MaxLength=256 + // +kubebuilder:validation:Pattern=`^[!#$%&'*+.^_|~0-9A-Za-z-]+$` + // +kubebuilder:validation:XValidation:rule="!(self.lowerAscii() in ['host', 'connection', 'content-length', 'transfer-encoding', 'te', 'trailer', 'upgrade', 'keep-alive', 'proxy-authorization', 'proxy-authenticate', 'cookie'])",message="this header cannot carry an egress credential" + Header string `json:"header"` + // Prefix is prepended verbatim to the Secret value, for example "Bearer ". + // Include any separating space. Omitted or empty means no prefix. + // +optional + // +kubebuilder:validation:MaxLength=256 + // +kubebuilder:validation:Pattern=`^[^\x00-\x08\x0a-\x1f\x7f]*$` + Prefix *string `json:"prefix,omitempty"` + // SecretKeyRef selects a required key in the EgressPolicy's namespace. + // +required + // +kubebuilder:validation:XValidation:rule="has(self.name) && self.name != '' && self.key != ''",message="secretKeyRef requires a name and key" + // +kubebuilder:validation:XValidation:rule="!has(self.optional) || !self.optional",message="egress credentials cannot be optional" + SecretKeyRef corev1.SecretKeySelector `json:"secretKeyRef"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=egresspolicies,singular=egresspolicy,scope=Namespaced,categories=kagent + +// EgressPolicy is a reusable grant of additional network access. Workloads in its +// namespace select it explicitly; changing it affects newly prepared revisions. +type EgressPolicy struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + // +required + Spec EgressPolicySpec `json:"spec"` +} + +// +kubebuilder:object:root=true + +// EgressPolicyList contains EgressPolicy resources. +type EgressPolicyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EgressPolicy `json:"items"` +} + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, &EgressPolicy{}, &EgressPolicyList{}) + return nil + }) +} diff --git a/go/api/v1alpha3/sandboxtemplate_types.go b/go/api/v1alpha3/sandboxtemplate_types.go index 9687b688a9..b8b27249c3 100644 --- a/go/api/v1alpha3/sandboxtemplate_types.go +++ b/go/api/v1alpha3/sandboxtemplate_types.go @@ -4,6 +4,7 @@ package v1alpha3 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -37,8 +38,20 @@ type SandboxTemplateSpec struct { // resolved in this template's namespace. // +required Substrate RuntimeSubstratePolicy `json:"substrate"` + + // EgressPolicyRefs grants additional access in this template's namespace. + // Each sandbox retains the access pinned by its prepared revision. + // +optional + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:XValidation:rule="self.all(ref, has(ref.name) && ref.name != '')",message="egressPolicyRefs requires non-empty names" + // +listType=map + // +listMapKey=name + EgressPolicyRefs []corev1.LocalObjectReference `json:"egressPolicyRefs,omitempty"` } +// SandboxTemplateReasonEgressPolicyResolutionFailed identifies an unresolved or invalid egress grant. +const SandboxTemplateReasonEgressPolicyResolutionFailed = "EgressPolicyResolutionFailed" + // SandboxTemplateStatus reports preparation of the current template inputs. type SandboxTemplateStatus struct { // ObservedGeneration is the generation considered by preparation. diff --git a/go/api/v1alpha3/zz_generated.deepcopy.go b/go/api/v1alpha3/zz_generated.deepcopy.go index f21a0d5b50..0179022d34 100644 --- a/go/api/v1alpha3/zz_generated.deepcopy.go +++ b/go/api/v1alpha3/zz_generated.deepcopy.go @@ -91,6 +91,11 @@ func (in *AgentSpec) DeepCopyInto(out *AgentSpec) { *out = *in in.Template.DeepCopyInto(&out.Template) out.HarnessRef = in.HarnessRef + if in.EgressPolicyRefs != nil { + in, out := &in.EgressPolicyRefs, &out.EgressPolicyRefs + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentSpec. @@ -577,6 +582,127 @@ func (in *ConfigMapKeyReference) DeepCopy() *ConfigMapKeyReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressAuthentication) DeepCopyInto(out *EgressAuthentication) { + *out = *in + if in.Prefix != nil { + in, out := &in.Prefix, &out.Prefix + *out = new(string) + **out = **in + } + in.SecretKeyRef.DeepCopyInto(&out.SecretKeyRef) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressAuthentication. +func (in *EgressAuthentication) DeepCopy() *EgressAuthentication { + if in == nil { + return nil + } + out := new(EgressAuthentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressDestination) DeepCopyInto(out *EgressDestination) { + *out = *in + if in.Authentication != nil { + in, out := &in.Authentication, &out.Authentication + *out = new(EgressAuthentication) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressDestination. +func (in *EgressDestination) DeepCopy() *EgressDestination { + if in == nil { + return nil + } + out := new(EgressDestination) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressPolicy) DeepCopyInto(out *EgressPolicy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressPolicy. +func (in *EgressPolicy) DeepCopy() *EgressPolicy { + if in == nil { + return nil + } + out := new(EgressPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressPolicy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressPolicyList) DeepCopyInto(out *EgressPolicyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EgressPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressPolicyList. +func (in *EgressPolicyList) DeepCopy() *EgressPolicyList { + if in == nil { + return nil + } + out := new(EgressPolicyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressPolicyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressPolicySpec) DeepCopyInto(out *EgressPolicySpec) { + *out = *in + if in.Destinations != nil { + in, out := &in.Destinations, &out.Destinations + *out = make([]EgressDestination, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressPolicySpec. +func (in *EgressPolicySpec) DeepCopy() *EgressPolicySpec { + if in == nil { + return nil + } + out := new(EgressPolicySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FoundryConfig) DeepCopyInto(out *FoundryConfig) { *out = *in @@ -1649,6 +1775,11 @@ func (in *SandboxTemplateSpec) DeepCopyInto(out *SandboxTemplateSpec) { copy(*out, *in) } out.Substrate = in.Substrate + if in.EgressPolicyRefs != nil { + in, out := &in.EgressPolicyRefs, &out.EgressPolicyRefs + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxTemplateSpec. diff --git a/go/core/internal/controller/apiclient/types.go b/go/core/internal/controller/apiclient/types.go index c396c10f6d..7e5fdc6910 100644 --- a/go/core/internal/controller/apiclient/types.go +++ b/go/core/internal/controller/apiclient/types.go @@ -23,6 +23,19 @@ func RegisterTypes() { } func registerTypes() { + kubeclient.Register( + kagentv1alpha3.GroupVersion.WithResource("egresspolicies"), + kagentv1alpha3.GroupVersion.WithKind("EgressPolicy"), + func(c kubeclient.ClientGetter, namespace string, options metav1.ListOptions) (runtime.Object, error) { + return c.(Client).Kagent().ApiV1alpha3().EgressPolicies(namespace).List(context.Background(), options) + }, + func(c kubeclient.ClientGetter, namespace string, options metav1.ListOptions) (watch.Interface, error) { + return c.(Client).Kagent().ApiV1alpha3().EgressPolicies(namespace).Watch(context.Background(), options) + }, + func(c kubeclient.ClientGetter, namespace string) kubetypes.WriteAPI[*kagentv1alpha3.EgressPolicy] { + return c.(Client).Kagent().ApiV1alpha3().EgressPolicies(namespace) + }, + ) kubeclient.Register( kagentv1alpha3.GroupVersion.WithResource("agents"), kagentv1alpha3.GroupVersion.WithKind("Agent"), diff --git a/go/core/internal/controller/collections.go b/go/core/internal/controller/collections.go index cdd67b9b2e..068cecbcc3 100644 --- a/go/core/internal/controller/collections.go +++ b/go/core/internal/controller/collections.go @@ -19,6 +19,7 @@ import ( // Collections contains shared Kubernetes inputs and compiled Agent state. type Collections struct { + EgressPolicies krt.Collection[*kagentv1alpha3.EgressPolicy] SandboxTemplates krt.Collection[*kagentv1alpha3.SandboxTemplate] Agents krt.Collection[*kagentv1alpha3.Agent] AgentTemplates krt.Collection[*kagentv1alpha3.AgentTemplate] @@ -63,6 +64,7 @@ func (p AgentRuntimeObservation) Equals(other AgentRuntimeObservation) bool { // NewCollections creates the complete read-only input graph. An empty // watchNamespaces list watches all namespaces. func NewCollections(client kube.Client, watchNamespaces []string, opts krt.OptionsBuilder, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig) Collections { + egressPolicies := typedCollection[*kagentv1alpha3.EgressPolicy](client, watchNamespaces, "EgressPolicies", opts) sandboxTemplates := typedCollection[*kagentv1alpha3.SandboxTemplate](client, watchNamespaces, "SandboxTemplates", opts) agents := typedCollection[*kagentv1alpha3.Agent](client, watchNamespaces, "Agents", opts) agentTemplates := typedCollection[*kagentv1alpha3.AgentTemplate](client, watchNamespaces, "AgentTemplates", opts) @@ -75,13 +77,15 @@ func NewCollections(client kube.Client, watchNamespaces []string, opts krt.Optio agentRuntimeObservations := krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...) modelConfigStatuses, resolvedModelConfigs := newModelConfigReconciliations(modelConfigs, configMaps, secrets, opts) compilerCollections := translator.Collections{ - Harnesses: harnesses, AgentTemplates: agentTemplates, ResolvedModelConfigs: resolvedModelConfigs, RemoteMCPServers: remoteMCPServers, + EgressPolicies: egressPolicies, + Harnesses: harnesses, AgentTemplates: agentTemplates, ResolvedModelConfigs: resolvedModelConfigs, RemoteMCPServers: remoteMCPServers, ConfigMaps: configMaps, Secrets: secrets, WorkerPools: workerPools, } reconciliations := newAgentReconciliations(agents, compilerCollections, agentRuntimeObservations, opts, settings, images) statuses := newAgentStatuses(agents, reconciliations, opts) return Collections{ + EgressPolicies: egressPolicies, SandboxTemplates: sandboxTemplates, Agents: agents, AgentTemplates: agentTemplates, diff --git a/go/core/internal/controller/sandbox_collections.go b/go/core/internal/controller/sandbox_collections.go index 52712a7eea..075b439483 100644 --- a/go/core/internal/controller/sandbox_collections.go +++ b/go/core/internal/controller/sandbox_collections.go @@ -10,7 +10,9 @@ import ( "istio.io/istio/pkg/kube/krt" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" + "github.com/kagent-dev/kagent/go/core/internal/translator" ) // Sandbox preparation uses the same informer graph as Agents. Only observations @@ -21,6 +23,7 @@ type sandboxCollections struct { } type sandboxReconciliation struct { + Egress egress.Access Template *kagentv1alpha3.SandboxTemplate RevisionID string SourceSnapshot json.RawMessage @@ -89,7 +92,13 @@ func newSandboxCollections(inputs Collections, policy substrate.SandboxPolicy, o state.Failure = &ReconciliationFailure{Reason: "WorkerPoolNotFound", Message: "The referenced sandbox WorkerPool does not exist"} } else { var err error - state.DesiredActorTemplate, state.RevisionID, state.SourceSnapshot, err = substrate.SandboxActorTemplate(template, (*pool).Spec.SandboxClass, policy) + state.Egress, err = translator.CompileEgressPolicies(ctx, translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets}, template.Namespace, template.Spec.EgressPolicyRefs) + if err != nil { + state.CompilationError = err.Error() + state.Failure = &ReconciliationFailure{Reason: kagentv1alpha3.SandboxTemplateReasonEgressPolicyResolutionFailed, Message: err.Error()} + return state + } + state.DesiredActorTemplate, state.RevisionID, state.SourceSnapshot, err = substrate.SandboxActorTemplate(template, (*pool).Spec.SandboxClass, policy, state.Egress) if err != nil { state.CompilationError = err.Error() state.Failure = &ReconciliationFailure{Reason: sandboxPreparationFailed, Message: sandboxPreparationFailureMessage} diff --git a/go/core/internal/controller/sandbox_integration_test.go b/go/core/internal/controller/sandbox_integration_test.go index 133fe4b5ee..3e3878c9c8 100644 --- a/go/core/internal/controller/sandbox_integration_test.go +++ b/go/core/internal/controller/sandbox_integration_test.go @@ -42,6 +42,7 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { template := sandboxTestTemplate() template.UID = "" template.Generation = 0 + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "tickets"}} template, err = client.ApiV1alpha3().SandboxTemplates(template.Namespace).Create(t.Context(), template, metav1.CreateOptions{}) require.NoError(t, err) unwatched := template.DeepCopy() @@ -60,6 +61,8 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { options := krt.NewOptionsBuilder(ctx.Done(), "sandbox-integration", nil) pools := krt.NewStaticCollection(nil, []*atev1alpha1.WorkerPool{pool}, options.WithName("WorkerPools")...) runtime := &Runtime{Client: runtimeClient, Options: options, Collections: Collections{ + EgressPolicies: typedCollection[*kagentv1alpha3.EgressPolicy](runtimeClient, []string{"team-a"}, "EgressPolicies", options), + Secrets: typedCollection[*corev1.Secret](runtimeClient, []string{"team-a"}, "Secrets", options), SandboxTemplates: typedCollection[*kagentv1alpha3.SandboxTemplate](runtimeClient, []string{"team-a"}, "SandboxTemplates", options), WorkerPools: pools, }} reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}) @@ -89,6 +92,26 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { } return apimeta.IsStatusConditionTrue(current.Status.Conditions, "Ready") } + waitUnresolved := func(message string) { + t.Helper() + require.Eventually(t, func() bool { + current, err := client.ApiV1alpha3().SandboxTemplates(template.Namespace).Get(t.Context(), template.Name, metav1.GetOptions{}) + if err != nil { + return false + } + condition := apimeta.FindStatusCondition(current.Status.Conditions, "Ready") + return condition != nil && condition.Status == metav1.ConditionFalse && condition.Reason == kagentv1alpha3.SandboxTemplateReasonEgressPolicyResolutionFailed && strings.Contains(condition.Message, message) + }, 10*time.Second, 10*time.Millisecond) + } + waitUnresolved("EgressPolicy") + grant := &kagentv1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "tickets"}, Spec: kagentv1alpha3.EgressPolicySpec{ + Destinations: []kagentv1alpha3.EgressDestination{{Origin: "https://tickets.example.com", Authentication: &kagentv1alpha3.EgressAuthentication{Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "ticket-auth"}, Key: "token"}}}}, + }} + grant, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Create(t.Context(), grant, metav1.CreateOptions{}) + require.NoError(t, err) + waitUnresolved("Secret") + secret, err := kube.Kube().CoreV1().Secrets(template.Namespace).Create(t.Context(), &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "ticket-auth"}, Data: map[string][]byte{"token": []byte("original-token")}}, metav1.CreateOptions{}) + require.NoError(t, err) require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) current, err := client.ApiV1alpha3().SandboxTemplates(template.Namespace).Get(t.Context(), template.Name, metav1.GetOptions{}) require.NoError(t, err) @@ -96,12 +119,46 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { store.mu.Lock() revision := store.revision.Revision require.True(t, store.ready) + require.Equal(t, []string{"https://tickets.example.com:443"}, store.revision.EgressDestinations) + require.Len(t, store.revision.Credentials, 1) + require.NotContains(t, string(store.revision.SourceSnapshot), "original-token") store.mu.Unlock() other, err := client.ApiV1alpha3().SandboxTemplates(unwatched.Namespace).Get(t.Context(), unwatched.Name, metav1.GetOptions{}) require.NoError(t, err) require.Empty(t, other.Finalizers, "namespace filters must apply to the new informer") require.Empty(t, other.Status.Conditions) + // A missing key invalidates preparation; restoring it with a new value + // resolves the same pinned revision without serializing either value. + secret.Data = nil + secret, err = kube.Kube().CoreV1().Secrets(template.Namespace).Update(t.Context(), secret, metav1.UpdateOptions{}) + require.NoError(t, err) + waitUnresolved("missing key") + secret.Data = map[string][]byte{"token": []byte("rotated-token")} + _, err = kube.Kube().CoreV1().Secrets(template.Namespace).Update(t.Context(), secret, metav1.UpdateOptions{}) + require.NoError(t, err) + require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) + store.mu.Lock() + require.Equal(t, revision, store.revision.Revision) + store.mu.Unlock() + grant.Spec.Destinations = append(grant.Spec.Destinations, kagentv1alpha3.EgressDestination{Origin: "https://registry.npmjs.org"}) + grant, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Update(t.Context(), grant, metav1.UpdateOptions{}) + require.NoError(t, err) + require.Eventually(t, func() bool { + store.mu.Lock() + defer store.mu.Unlock() + return store.ready && store.revision.Revision != revision && len(store.revision.EgressDestinations) == 2 + }, 10*time.Second, 10*time.Millisecond) + store.mu.Lock() + revision = store.revision.Revision + store.mu.Unlock() + require.NoError(t, client.ApiV1alpha3().EgressPolicies(template.Namespace).Delete(t.Context(), grant.Name, metav1.DeleteOptions{})) + waitUnresolved("EgressPolicy") + grant.ResourceVersion, grant.UID = "", "" + _, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Create(t.Context(), grant, metav1.CreateOptions{}) + require.NoError(t, err) + require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) + // A dependency update alone must change the persisted revision. changedPool := pool.DeepCopy() changedPool.Spec.SandboxClass = atev1alpha1.SandboxClassMicroVM diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index 884ee3439d..e3586a771a 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -173,7 +173,8 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { RuntimeArtifact: database.RuntimeArtifact{Revision: state.RevisionID, Kind: "sandbox", Namespace: state.Template.Namespace, ActorTemplateAtespace: ref.GetAtespace(), ActorTemplateName: ref.GetName(), ActorTemplateUID: observed.GetMetadata().GetUid()}, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), - SourceSnapshot: state.SourceSnapshot, + SourceSnapshot: state.SourceSnapshot, + EgressDestinations: state.Egress.Destinations, Credentials: state.Egress.Credentials, } if err := s.store.RecordSandboxRevision(ctx, revision, ready); err != nil { return s.observeError(*state, fmt.Errorf("store sandbox revision %s: %w", state.RevisionID, err), true) diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index b623c32060..05245cda28 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -492,7 +492,7 @@ func TestRecordRuntimeRevisionPromotesOnlyCurrentActivePair(t *testing.T) { func TestRuntimeRevisionPersistsCredentialBindings(t *testing.T) { client := NewClient(setupTestDB(t), "public") - revision := RuntimeRevision{Revision: "credential-revision", Namespace: "team", AgentName: "agent", AgentUID: "agent", SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{"api.example.com"}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", Credentials: []egress.Credential{{Hostname: "api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}}} + revision := RuntimeRevision{Revision: "credential-revision", Namespace: "team", AgentName: "agent", AgentUID: "agent", SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{"https://api.example.com:443"}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", Credentials: []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}}} require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, false)) got, err := client.GetRuntimeRevision(t.Context(), revision.Revision) require.NoError(t, err) @@ -508,10 +508,10 @@ func TestRuntimeRevisionRejectsMalformedStoredCredentials(t *testing.T) { revision, err := toRuntimeRevision(runtimeRevisionRow{ Revision: "bad-revision", Credentials: []egress.Credential{{ - Hostname: "*", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token", + Origin: "*", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token", }}, }) require.ErrorContains(t, err, "decode runtime revision bad-revision credentials") - require.ErrorContains(t, err, "exact DNS hostname") + require.ErrorContains(t, err, "exact HTTP(S) DNS origin") require.Nil(t, revision) } diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index 5bee645014..938ca87fca 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/jackc/pgx/v5" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) type SandboxRevision struct { @@ -13,6 +14,8 @@ type SandboxRevision struct { SandboxTemplateName string SandboxTemplateUID string SourceSnapshot json.RawMessage + EgressDestinations []string + Credentials []egress.Credential } // SandboxTemplateDefinition tracks the desired and last successful runtime for a SandboxTemplate UID. @@ -79,7 +82,7 @@ func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevi return err } revision.Kind = runtimeKindSandbox - if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{RuntimeArtifact: revision.RuntimeArtifact, SourceSnapshot: revision.SourceSnapshot}); err != nil { + if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{RuntimeArtifact: revision.RuntimeArtifact, SourceSnapshot: revision.SourceSnapshot, EgressDestinations: revision.EgressDestinations, Credentials: revision.Credentials}); err != nil { return err } if err := execSQL(ctx, tx, ` @@ -100,14 +103,20 @@ func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevi }) } +// GetSandboxRevision reads immutable sandbox inputs, rejecting malformed stored grants. func (c *Client) GetSandboxRevision(ctx context.Context, revision string) (*SandboxRevision, error) { row, err := queryOne(ctx, c.db, ` SELECT r.revision, r.kind, r.namespace, r.actor_template_atespace, r.actor_template_name, r.actor_template_uid, - r.deleted_at, r.source_snapshot, s.sandbox_template_name, s.sandbox_template_uid + r.deleted_at, r.source_snapshot, r.egress_destinations, r.credentials, s.sandbox_template_name, s.sandbox_template_uid FROM runtime_revision r JOIN sandbox_revision s USING (revision) WHERE r.revision = $1 AND r.kind = 'sandbox' `, pgx.RowToStructByName[SandboxRevision], revision) if err != nil { return nil, fmt.Errorf("get sandbox revision: %w", notFoundOr(err)) } + access, err := egress.CanonicalAccess(row.EgressDestinations, row.Credentials) + if err != nil { + return nil, fmt.Errorf("decode sandbox revision %s egress: %w", revision, err) + } + row.EgressDestinations, row.Credentials = access.Destinations, access.Credentials return &row, nil } diff --git a/go/core/internal/database/sandbox_revisions_test.go b/go/core/internal/database/sandbox_revisions_test.go new file mode 100644 index 0000000000..3f53259d07 --- /dev/null +++ b/go/core/internal/database/sandbox_revisions_test.go @@ -0,0 +1,46 @@ +package database + +import ( + "testing" + + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/stretchr/testify/require" +) + +func TestSandboxRevisionPinsEgress(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + input, options := sandboxFixture(t, client, "initial") + revision := SandboxRevision{ + RuntimeArtifact: RuntimeArtifact{Revision: "tickets", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "tickets", ActorTemplateUID: "actor-uid"}, + SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`), + EgressDestinations: []string{"https://tickets.example.com:443"}, + Credentials: []egress.Credential{{Origin: "https://tickets.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/ticket-auth/token"}}, + } + require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision.Revision})) + require.NoError(t, client.RecordSandboxRevision(ctx, revision, true)) + created, _, err := client.CreateSandbox(ctx, input, "create", options) + require.NoError(t, err) + require.Equal(t, revision.Revision, created.PreparedRevision) + saved, err := client.GetSandboxRevision(ctx, created.PreparedRevision) + require.NoError(t, err) + require.Equal(t, revision.EgressDestinations, saved.EgressDestinations) + require.Equal(t, revision.Credentials, saved.Credentials) + // Retries may refresh artifact identity, but cannot rewrite an existing grant. + revision.EgressDestinations, revision.Credentials = nil, nil + require.NoError(t, client.RecordSandboxRevision(ctx, revision, true)) + _, _ = sandboxFixture(t, client, "new-policy") + retried, _, err := client.CreateSandbox(ctx, input, "create", options) + require.NoError(t, err) + require.Equal(t, created.PreparedRevision, retried.PreparedRevision) + pinned, err := client.GetSandboxRevision(ctx, retried.PreparedRevision) + require.NoError(t, err) + require.Equal(t, saved.EgressDestinations, pinned.EgressDestinations) + require.Equal(t, saved.Credentials, pinned.Credentials) + + // Reject malformed durable data instead of silently dropping authentication. + _, err = client.db.Exec(ctx, `UPDATE runtime_revision SET credentials = '[{"hostname":"tickets.example.com","header":"authorization","uri":"ate-secret://k8s.io/default/team-a/ticket-auth/token"}]' WHERE revision = $1`, created.PreparedRevision) + require.NoError(t, err) + _, err = client.GetSandboxRevision(ctx, created.PreparedRevision) + require.ErrorContains(t, err, "decode sandbox revision") +} diff --git a/go/core/internal/egress/credentials.go b/go/core/internal/egress/credentials.go index e0136cdfee..f226807c19 100644 --- a/go/core/internal/egress/credentials.go +++ b/go/core/internal/egress/credentials.go @@ -2,7 +2,6 @@ package egress import ( "fmt" - "net/netip" "net/url" "slices" "strings" @@ -14,28 +13,27 @@ import ( // Credential binds a Secret reference to an exact destination and HTTP header. // Values are fetched by the gateway and never enter a runtime revision. type Credential struct { - Hostname string `json:"hostname"` - Header string `json:"header"` - Prefix string `json:"prefix,omitempty"` - URI string `json:"uri"` + Origin string `json:"origin"` + Header string `json:"header"` + Prefix string `json:"prefix,omitempty"` + URI string `json:"uri"` } -// CanonicalCredentials validates and orders bindings. Substrate matches only -// hostnames, so two credentials for the same host and header cannot coexist. +// CanonicalCredentials validates and orders bindings by complete origin and header. func CanonicalCredentials(bindings []Credential) ([]Credential, error) { result := slices.Clone(bindings) for i := range result { c := &result[i] - c.Hostname = strings.TrimSuffix(strings.ToLower(c.Hostname), ".") - c.Header = strings.ToLower(c.Header) - _, ipErr := netip.ParseAddr(c.Hostname) - if ipErr == nil || len(validation.IsDNS1123Subdomain(c.Hostname)) != 0 { - return nil, fmt.Errorf("credential injection requires an exact DNS hostname, got %q", c.Hostname) + u, err := ParseOrigin(c.Origin) + if err != nil || strings.Contains(c.Origin, "*") { + return nil, fmt.Errorf("credential injection requires an exact HTTP(S) DNS origin, got %q", c.Origin) } - if !httpguts.ValidHeaderFieldName(c.Header) || !httpguts.ValidHeaderFieldValue(c.Prefix) { + c.Origin = u.String() + c.Header = strings.ToLower(c.Header) + if !validCredentialHeader(c.Header) || !httpguts.ValidHeaderFieldValue(c.Prefix) { return nil, fmt.Errorf("invalid credential injection header %q", c.Header) } - u, err := url.Parse(c.URI) + u, err = url.Parse(c.URI) if err != nil || u.Scheme != "ate-secret" || u.Host != "k8s.io" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { return nil, fmt.Errorf("invalid Kubernetes credential URI") } @@ -45,21 +43,33 @@ func CanonicalCredentials(bindings []Credential) ([]Credential, error) { } } slices.SortFunc(result, func(a, b Credential) int { - return strings.Compare(a.Hostname+"\x00"+a.Header, b.Hostname+"\x00"+b.Header) + return strings.Compare(a.Origin+"\x00"+a.Header, b.Origin+"\x00"+b.Header) }) for i := 1; i < len(result); i++ { a, b := result[i-1], result[i] - if a.Hostname == b.Hostname && a.Header == b.Header && a != b { - return nil, fmt.Errorf("conflicting credentials for %s header %s", a.Hostname, a.Header) + if a.Origin == b.Origin && a.Header == b.Header && a != b { + return nil, fmt.Errorf("conflicting credentials for %s header %s", a.Origin, a.Header) } } result = slices.Compact(result) counts := map[string]int{} for _, c := range result { - counts[c.Hostname]++ - if counts[c.Hostname] > 16 { - return nil, fmt.Errorf("credential injection supports at most 16 headers per hostname") + counts[c.Origin]++ + if counts[c.Origin] > 16 { + return nil, fmt.Errorf("credential injection supports at most 16 headers per origin") } } return result, nil } + +// validCredentialHeader excludes routing, framing, hop-by-hop and cookie headers. +func validCredentialHeader(header string) bool { + if len(header) > 256 || !httpguts.ValidHeaderFieldName(header) { + return false + } + switch strings.ToLower(header) { + case "host", "connection", "content-length", "transfer-encoding", "te", "trailer", "upgrade", "keep-alive", "proxy-authorization", "proxy-authenticate", "cookie": + return false + } + return true +} diff --git a/go/core/internal/egress/credentials_test.go b/go/core/internal/egress/credentials_test.go index e24254caaa..453d663065 100644 --- a/go/core/internal/egress/credentials_test.go +++ b/go/core/internal/egress/credentials_test.go @@ -7,17 +7,17 @@ import ( ) func TestCanonicalCredentials(t *testing.T) { - base := Credential{Hostname: "API.Example.com.", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"} + base := Credential{Origin: "https://API.Example.com.", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"} got, err := CanonicalCredentials([]Credential{base, base}) require.NoError(t, err) - require.Equal(t, []Credential{{Hostname: "api.example.com", Header: "authorization", Prefix: "Bearer ", URI: base.URI}}, got) - require.Equal(t, "API.Example.com.", base.Hostname) + require.Equal(t, []Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: base.URI}}, got) + require.Equal(t, "https://API.Example.com.", base.Origin) for _, test := range []struct { name string change func(*Credential) }{ - {"wildcard", func(c *Credential) { c.Hostname = "*.example.com" }}, - {"IP", func(c *Credential) { c.Hostname = "192.0.2.1" }}, + {"wildcard", func(c *Credential) { c.Origin = "https://*.example.com" }}, + {"IP", func(c *Credential) { c.Origin = "https://192.0.2.1" }}, {"header", func(c *Credential) { c.Header = "Authorization\r\nInjected" }}, {"prefix", func(c *Credential) { c.Prefix = "Bearer\n" }}, {"provider", func(c *Credential) { c.URI = "https://example.com/secret" }}, diff --git a/go/core/internal/egress/destination.go b/go/core/internal/egress/destination.go index ade6f203ed..3a44779c2e 100644 --- a/go/core/internal/egress/destination.go +++ b/go/core/internal/egress/destination.go @@ -1,9 +1,15 @@ package egress import ( + "fmt" "net" + "net/netip" "net/url" + "slices" + "strconv" "strings" + + "k8s.io/apimachinery/pkg/util/validation" ) // Origin retains the protocol and destination port needed by Substrate's @@ -24,3 +30,75 @@ func Origin(u *url.URL) string { } return (&url.URL{Scheme: u.Scheme, Host: host}).String() } + +// ParseOrigin validates an HTTP(S) DNS origin and makes its port explicit. +// Wildcards replace one leftmost label and cannot cover an entire top-level domain. +func ParseOrigin(raw string) (*url.URL, error) { + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.Hostname() == "" || u.Path != "" || u.ForceQuery || u.RawQuery != "" || strings.Contains(raw, "#") || strings.HasSuffix(u.Host, ":") { + return nil, fmt.Errorf("invalid egress origin %q: expected scheme, DNS hostname and optional port", raw) + } + host := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") + dns := strings.TrimPrefix(host, "*.") + _, ipErr := netip.ParseAddr(dns) + if ipErr == nil || len(validation.IsDNS1123Subdomain(dns)) != 0 || (strings.HasPrefix(host, "*.") && !strings.Contains(dns, ".")) { + return nil, fmt.Errorf("invalid egress origin %q: expected a DNS name or leftmost-label wildcard", raw) + } + portText := u.Port() + if portText == "" { + portText = "80" + if u.Scheme == "https" { + portText = "443" + } + } + port, err := strconv.ParseUint(portText, 10, 16) + if err != nil || port == 0 { + return nil, fmt.Errorf("invalid egress port in %q", raw) + } + u.Host = net.JoinHostPort(host, strconv.FormatUint(port, 10)) + return u, nil +} + +// Access is the resolved, revision-pinned network grant. It contains references, +// never Secret values. +type Access struct { + Destinations []string `json:"destinations,omitempty"` + Credentials []Credential `json:"credentials,omitempty"` +} + +// CanonicalAccess merges equivalent origins and validates the complete grant +// before a revision can become ready or an Actor can be resumed. +func CanonicalAccess(destinations []string, credentials []Credential) (Access, error) { + result := Access{} + protocols := map[string]string{} + allowed := map[string]bool{} + for _, raw := range destinations { + u, err := ParseOrigin(raw) + if err != nil { + return Access{}, err + } + if scheme, exists := protocols[u.Host]; exists && scheme != u.Scheme { + return Access{}, fmt.Errorf("egress destination %q uses both HTTP and HTTPS on the same port", u.Host) + } + protocols[u.Host] = u.Scheme + allowed[u.String()] = true + } + for origin := range allowed { + result.Destinations = append(result.Destinations, origin) + } + slices.Sort(result.Destinations) + if len(result.Destinations) > 256 { + return Access{}, fmt.Errorf("egress policy exceeds 256 rules") + } + var err error + result.Credentials, err = CanonicalCredentials(credentials) + if err != nil { + return Access{}, err + } + for _, binding := range result.Credentials { + if !allowed[binding.Origin] { + return Access{}, fmt.Errorf("credential destination %q is not explicitly allowed", binding.Origin) + } + } + return result, nil +} diff --git a/go/core/internal/egress/destination_test.go b/go/core/internal/egress/destination_test.go index dc9fa6d812..69da32d12f 100644 --- a/go/core/internal/egress/destination_test.go +++ b/go/core/internal/egress/destination_test.go @@ -1,12 +1,54 @@ package egress import ( + "fmt" "net/url" "testing" "github.com/stretchr/testify/require" ) +func TestCanonicalAccess(t *testing.T) { + credential := Credential{Origin: "https://API.example.com", Header: "Authorization", URI: "ate-secret://k8s.io/default/team/auth/token"} + access, err := CanonicalAccess([]string{"https://*.example.com", "https://api.example.com:0443", "https://API.example.com.", "https://api.example.com:8443", "http://api.example.com"}, []Credential{credential, credential}) + require.NoError(t, err) + require.Equal(t, []string{"http://api.example.com:80", "https://*.example.com:443", "https://api.example.com:443", "https://api.example.com:8443"}, access.Destinations) + require.Len(t, access.Credentials, 1) + require.Equal(t, "https://api.example.com:443", access.Credentials[0].Origin) + for _, raw := range []string{"https://*", "https://*.com", "https://foo.*.com", "https://192.0.2.1", "https://[::1]", "https://user@api.example.com", "https://api.example.com/", "https://api.example.com?", "https://api.example.com#", "https://api.example.com:", "https://api.example.com:0", "https://api.example.com:65536", "ftp://api.example.com"} { + t.Run(raw, func(t *testing.T) { + _, err := ParseOrigin(raw) + require.Error(t, err) + }) + } + for _, allowed := range []string{"https://*.example.com", "http://api.example.com", "https://api.example.com:8443"} { + _, err := CanonicalAccess([]string{allowed}, []Credential{credential}) + require.ErrorContains(t, err, "not explicitly allowed", "credential must require its exact origin") + } + _, err = CanonicalAccess([]string{"http://api.example.com:443", "https://api.example.com"}, nil) + require.ErrorContains(t, err, "both HTTP and HTTPS") + var origins []string + for i := range 257 { + origins = append(origins, fmt.Sprintf("https://api%d.example.com", i)) + } + _, err = CanonicalAccess(origins, nil) + require.ErrorContains(t, err, "256 rules") +} + +func TestCredentialsIsolatePortsAndRejectProtectedHeaders(t *testing.T) { + one := Credential{Origin: "https://api.example.com", Header: "Authorization", URI: "ate-secret://k8s.io/default/team/auth/token"} + two := one + two.Origin, two.URI = "https://api.example.com:8443", "ate-secret://k8s.io/default/team/other/token" + bindings, err := CanonicalCredentials([]Credential{one, two}) + require.NoError(t, err) + require.Len(t, bindings, 2) + for _, header := range []string{"hOsT", "Content-Length", "Connection", "Transfer-Encoding", "Cookie", "Proxy-Authorization"} { + one.Header = header + _, err := CanonicalCredentials([]Credential{one}) + require.Error(t, err) + } +} + func TestOrigin(t *testing.T) { for _, tt := range []struct{ raw, want string }{ {"https://API.Example.com./v1?key=private#fragment", "https://api.example.com:443"}, diff --git a/go/core/internal/service/sandbox/service_test.go b/go/core/internal/service/sandbox/service_test.go index f3de53197d..2ac2fa9411 100644 --- a/go/core/internal/service/sandbox/service_test.go +++ b/go/core/internal/service/sandbox/service_test.go @@ -22,6 +22,7 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/pkg/auth" ) @@ -287,3 +288,25 @@ func TestSandboxConflictingRequestCanRetryAfterActiveAttempt(t *testing.T) { require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, current.Operation) require.Equal(t, []string{"create", "policy", "resume", "suspend", "resume"}, actors.observedCalls()) } + +func TestSandboxInstallsPinnedGrantBeforeResume(t *testing.T) { + service, ctx, actors := serviceFixture(t) + revision := database.SandboxRevision{ + RuntimeArtifact: database.RuntimeArtifact{Revision: "policy-revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "policy-runtime", ActorTemplateUID: "policy-runtime-uid"}, + SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), + EgressDestinations: []string{"https://tickets.example.com:443"}, + Credentials: []egress.Credential{{Origin: "https://tickets.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/ticket-auth/token"}}, + } + store := service.config.Store + require.NoError(t, store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: revision.Revision})) + require.NoError(t, store.RecordSandboxRevision(ctx, revision, true)) + instance, err := service.Create(ctx, createRequest()) + require.NoError(t, err) + require.Equal(t, revision.Revision, instance.PreparedRevision) + expected, err := substrate.ActorEgressPolicy("team-a", revision.EgressDestinations, revision.Credentials) + require.NoError(t, err) + require.True(t, proto.Equal(expected, actors.egressPolicy)) + calls := actors.observedCalls() + require.Less(t, slices.Index(calls, "policy"), slices.Index(calls, "resume")) + require.Contains(t, calls, "policy") +} diff --git a/go/core/internal/service/sandbox/workflow.go b/go/core/internal/service/sandbox/workflow.go index e285492684..6241b08453 100644 --- a/go/core/internal/service/sandbox/workflow.go +++ b/go/core/internal/service/sandbox/workflow.go @@ -43,7 +43,7 @@ func (s *Service) run(ctx context.Context, id string, kind apiv1.RuntimeOperatio TemplateAtespace: revision.ActorTemplateAtespace, TemplateName: revision.ActorTemplateName} var creation *substrate.ActorCreation if kind == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { - policy, err := substrate.ActorEgressPolicy(binding.Atespace, nil, nil) + policy, err := substrate.ActorEgressPolicy(binding.Atespace, revision.EgressDestinations, revision.Credentials) if err != nil { return nil, sandboxError(err) } diff --git a/go/core/internal/service/session/workflow_test.go b/go/core/internal/service/session/workflow_test.go index ff9439c3a4..3c6e9cf563 100644 --- a/go/core/internal/service/session/workflow_test.go +++ b/go/core/internal/service/session/workflow_test.go @@ -306,12 +306,12 @@ func TestActorCreationRetainsEgressPolicyFailure(t *testing.T) { callsBefore := base.policyCalls store.revision.EgressDestinations = []string{"*"} _, err := workflow.Create(t.Context(), session) - require.ErrorContains(t, err, "invalid egress destination") + require.ErrorContains(t, err, "invalid egress origin") require.Zero(t, actors.mutations.Load(), "validate the allowlist before issuing Actor creation") require.Equal(t, callsBefore, base.policyCalls) store.revision.EgressDestinations = []string{"https://api.example.com:443", "https://other.example.com:8443"} - store.revision.Credentials = []egress.Credential{{Hostname: "api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/auth/token"}} + store.revision.Credentials = []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/auth/token"}} base.policyErr = context.DeadlineExceeded _, err = workflow.Create(t.Context(), session) require.ErrorIs(t, err, context.DeadlineExceeded) @@ -349,7 +349,7 @@ func TestActorEgressPolicy(t *testing.T) { policy, err = substrate.ActorEgressPolicy("team-a", nil, nil) require.NoError(t, err) require.Empty(t, policy.Rules, "no destinations must deny all egress") - for _, destination := range []string{"", "*", "api.example.com", "https://api.example.com/path", "http://api.example.com:0", "http://api.example.com:65536", "http://user@api.example.com", "http://api.example.com?key=value", "http://*.example.com"} { + for _, destination := range []string{"", "*", "api.example.com", "https://api.example.com/path", "http://api.example.com:0", "http://api.example.com:65536", "http://user@api.example.com", "http://api.example.com?key=value"} { t.Run(destination, func(t *testing.T) { _, err := substrate.ActorEgressPolicy("team-a", []string{destination}, nil) require.Error(t, err) @@ -357,7 +357,7 @@ func TestActorEgressPolicy(t *testing.T) { } for _, destination := range []string{"http://192.0.2.1", "http://[2001:db8::1]", "http://[::ffff:192.0.2.1]"} { _, err := substrate.ActorEgressPolicy("team-a", []string{destination}, nil) - require.ErrorContains(t, err, "requires a DNS name") + require.ErrorContains(t, err, "expected a DNS name") } _, err = substrate.ActorEgressPolicy("team-a", []string{"http://api.example.com:8443", "https://api.example.com:8443"}, nil) require.ErrorContains(t, err, "both HTTP and HTTPS on the same port") @@ -365,15 +365,15 @@ func TestActorEgressPolicy(t *testing.T) { func TestActorEgressCredentialsRequireAllowedDestination(t *testing.T) { bindings := []egress.Credential{ - {Hostname: "api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}, - {Hostname: "api.example.com", Header: "x-api-key", URI: "ate-secret://k8s.io/default/team/auth/key"}, + {Origin: "https://api.example.com:8443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}, + {Origin: "https://api.example.com:8443", Header: "x-api-key", URI: "ate-secret://k8s.io/default/team/auth/key"}, } _, err := substrate.ActorEgressPolicy("team", []string{"https://other.example.com"}, bindings) - require.ErrorContains(t, err, "is not allowed") + require.ErrorContains(t, err, "is not explicitly allowed") policy, err := substrate.ActorEgressPolicy("team", []string{"http://api.example.com:8080", "https://api.example.com:8443", "https://other.example.com"}, bindings) require.NoError(t, err) require.Len(t, policy.Rules, 3) - require.Len(t, policy.Rules[0].GetHttp().GetEffects().GetReplaceHeaders(), 2, "configured HTTP endpoints also require credential replacement") + require.Empty(t, policy.Rules[0].GetHttp().GetEffects().GetReplaceHeaders(), "credentials must not cross schemes or ports") require.Equal(t, []string{"api.example.com"}, policy.Rules[1].GetHttps().GetHostnames()) require.Equal(t, []int32{8443}, policy.Rules[1].GetHttps().GetPorts().GetNumbers()) require.Len(t, policy.Rules[1].GetHttps().GetEffects().GetReplaceHeaders(), 2) diff --git a/go/core/internal/substrate/policy.go b/go/core/internal/substrate/policy.go index 3fe905fb6e..f373572a00 100644 --- a/go/core/internal/substrate/policy.go +++ b/go/core/internal/substrate/policy.go @@ -1,16 +1,9 @@ package substrate import ( - "fmt" - "net" - "net/netip" - "net/url" - "slices" "strconv" - "strings" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "k8s.io/apimachinery/pkg/util/validation" "github.com/kagent-dev/kagent/go/core/internal/egress" ) @@ -18,71 +11,29 @@ import ( // ActorEgressPolicy compiles HTTP(S) origins into an actor's default allowlist. // Credential bindings are already canonicalized by the store. func ActorEgressPolicy(atespace string, destinations []string, credentials []egress.Credential) (*ateapipb.EgressPolicy, error) { - type originTarget struct { - scheme, hostname string - port int32 - } - origins := map[string]originTarget{} - protocols := map[string]string{} - hosts := map[string]bool{} - for _, destination := range destinations { - u, err := url.Parse(destination) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.Hostname() == "" || u.ForceQuery || u.RawQuery != "" || u.Fragment != "" || u.Path != "" { - return nil, fmt.Errorf("invalid egress destination %q: expected an HTTP(S) origin", destination) - } - host := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") - if _, err := netip.ParseAddr(host); err == nil { - return nil, fmt.Errorf("egress destination %q requires a DNS name: Substrate does not support IP allowlists", destination) - } - if len(validation.IsDNS1123Subdomain(host)) != 0 { - return nil, fmt.Errorf("invalid egress destination %q", destination) - } - origin := egress.Origin(u) - u, err = url.Parse(origin) - if err != nil { - return nil, fmt.Errorf("invalid egress destination %q: %w", destination, err) - } - port, err := strconv.ParseUint(u.Port(), 10, 16) - if err != nil || port == 0 { - return nil, fmt.Errorf("invalid egress port in %q", destination) - } - u.Host = net.JoinHostPort(host, strconv.FormatUint(port, 10)) - if protocol, found := protocols[u.Host]; found && protocol != u.Scheme { - return nil, fmt.Errorf("egress destination %q uses both HTTP and HTTPS on the same port", u.Host) - } - protocols[u.Host] = u.Scheme - hosts[host] = true - origins[u.String()] = originTarget{scheme: u.Scheme, hostname: host, port: int32(port)} + access, err := egress.CanonicalAccess(destinations, credentials) + if err != nil { + return nil, err } effects := map[string]*ateapipb.HttpRuleEffects{} - for _, binding := range credentials { - if !hosts[binding.Hostname] { - return nil, fmt.Errorf("credential destination %q is not allowed", binding.Hostname) - } - if effects[binding.Hostname] == nil { - effects[binding.Hostname] = &ateapipb.HttpRuleEffects{} + for _, binding := range access.Credentials { + if effects[binding.Origin] == nil { + effects[binding.Origin] = &ateapipb.HttpRuleEffects{} } - effects[binding.Hostname].ReplaceHeaders = append(effects[binding.Hostname].ReplaceHeaders, &ateapipb.CredentialHeader{Header: binding.Header, Prefix: binding.Prefix, CredentialUri: binding.URI}) + effects[binding.Origin].ReplaceHeaders = append(effects[binding.Origin].ReplaceHeaders, &ateapipb.CredentialHeader{Header: binding.Header, Prefix: binding.Prefix, CredentialUri: binding.URI}) } - names := make([]string, 0, len(origins)) - for origin := range origins { - names = append(names, origin) - } - slices.Sort(names) policy := &ateapipb.EgressPolicy{Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: "default"}} - for _, origin := range names { - target := origins[origin] - ports := &ateapipb.Ports{Numbers: []int32{target.port}} - // Keep exactly one rule per host and port so an overlapping rule cannot - // bypass credential replacement. - if target.scheme == "https" { - policy.Rules = append(policy.Rules, &ateapipb.EgressRule{Https: &ateapipb.HTTPSRule{Hostnames: []string{target.hostname}, Ports: ports, Effects: effects[target.hostname]}}) + for _, origin := range access.Destinations { + target, _ := egress.ParseOrigin(origin) // validated by CanonicalAccess + port, _ := strconv.ParseInt(target.Port(), 10, 32) + ports := &ateapipb.Ports{Numbers: []int32{int32(port)}} + // One rule per origin: exact matches win over wildcard rules, keeping + // credentials on their exact origin even when a wildcard also grants access. + if target.Scheme == "https" { + policy.Rules = append(policy.Rules, &ateapipb.EgressRule{Https: &ateapipb.HTTPSRule{Hostnames: []string{target.Hostname()}, Ports: ports, Effects: effects[origin]}}) } else { - policy.Rules = append(policy.Rules, &ateapipb.EgressRule{Http: &ateapipb.HTTPRule{Hostnames: []string{target.hostname}, Ports: ports, Effects: effects[target.hostname]}}) + policy.Rules = append(policy.Rules, &ateapipb.EgressRule{Http: &ateapipb.HTTPRule{Hostnames: []string{target.Hostname()}, Ports: ports, Effects: effects[origin]}}) } } - if len(policy.Rules) > 256 { - return nil, fmt.Errorf("egress policy exceeds 256 rules") - } return policy, nil } diff --git a/go/core/internal/substrate/policy_test.go b/go/core/internal/substrate/policy_test.go new file mode 100644 index 0000000000..1102ee8378 --- /dev/null +++ b/go/core/internal/substrate/policy_test.go @@ -0,0 +1,28 @@ +package substrate + +import ( + "testing" + + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/stretchr/testify/require" +) + +func TestActorPolicyKeepsCredentialsOnExactOrigin(t *testing.T) { + policy, err := ActorEgressPolicy("team", []string{"https://*.example.com", "https://api.example.com:443", "https://api.example.com:8443", "http://api.example.com"}, []egress.Credential{ + {Origin: "https://api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}, + {Origin: "https://api.example.com:8443", Header: "x-api-key", URI: "ate-secret://k8s.io/default/team/other/key"}, + }) + require.NoError(t, err) + require.Len(t, policy.Rules, 4) + require.Empty(t, policy.Rules[0].GetHttp().GetEffects().GetReplaceHeaders()) + require.Equal(t, []string{"*.example.com"}, policy.Rules[1].GetHttps().GetHostnames()) + require.Empty(t, policy.Rules[1].GetHttps().GetEffects().GetReplaceHeaders()) + for i, header := range []string{"authorization", "x-api-key"} { + rule := policy.Rules[i+2].GetHttps() + require.Equal(t, []string{"api.example.com"}, rule.GetHostnames()) + require.Len(t, rule.GetEffects().GetReplaceHeaders(), 1) + require.Equal(t, header, rule.GetEffects().GetReplaceHeaders()[0].Header) + } + require.Equal(t, []int32{443}, policy.Rules[2].GetHttps().GetPorts().GetNumbers()) + require.Equal(t, []int32{8443}, policy.Rules[3].GetHttps().GetPorts().GetNumbers()) +} diff --git a/go/core/internal/substrate/sandbox.go b/go/core/internal/substrate/sandbox.go index 5282acf36d..6dd6e05f98 100644 --- a/go/core/internal/substrate/sandbox.go +++ b/go/core/internal/substrate/sandbox.go @@ -5,14 +5,17 @@ import ( "encoding/hex" "encoding/json" "fmt" + "slices" "strings" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/image" ) @@ -32,11 +35,12 @@ type sandboxInputs struct { Spec v1alpha3.SandboxTemplateSpec Class atev1alpha1.SandboxClass Policy SandboxPolicy + Egress egress.Access } // SandboxActorTemplate compiles standalone inputs without an Agent Card or Harness. // The image's entrypoint is replaced by the pinned, platform-managed guest. -func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1.SandboxClass, policy SandboxPolicy) (*ateapipb.ActorTemplate, string, json.RawMessage, error) { +func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1.SandboxClass, policy SandboxPolicy, access egress.Access) (*ateapipb.ActorTemplate, string, json.RawMessage, error) { if !image.IsDigestPinned(policy.GuestImage) || !image.IsDigestPinned(template.Spec.Workload.Image) { return nil, "", nil, fmt.Errorf("sandbox workload and guest images must be pinned by sha256 digest") } @@ -69,7 +73,13 @@ func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1. } // Namespace-scoped placement changes immutable template contents, so it // needs a new revision even when the operator's inputs stay the same. - inputs := sandboxInputs{Kind: "sandbox", Version: 2, Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), Spec: template.Spec, Class: class, Policy: policy} + access, err = egress.CanonicalAccess(access.Destinations, access.Credentials) + if err != nil { + return nil, "", nil, err + } + spec := template.Spec.DeepCopy() + slices.SortFunc(spec.EgressPolicyRefs, func(a, b corev1.LocalObjectReference) int { return strings.Compare(a.Name, b.Name) }) + inputs := sandboxInputs{Kind: "sandbox", Version: 3, Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), Spec: *spec, Class: class, Policy: policy, Egress: access} snapshot, err := json.Marshal(inputs) if err != nil { return nil, "", nil, fmt.Errorf("encode sandbox inputs: %w", err) diff --git a/go/core/internal/substrate/sandbox_test.go b/go/core/internal/substrate/sandbox_test.go index 88edb45c88..e2d74e854d 100644 --- a/go/core/internal/substrate/sandbox_test.go +++ b/go/core/internal/substrate/sandbox_test.go @@ -11,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) func TestSandboxPreparationPinsCompleteInputs(t *testing.T) { @@ -19,7 +20,7 @@ func TestSandboxPreparationPinsCompleteInputs(t *testing.T) { Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "s3://snapshots/"}}, }} policy := SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"} - actor, digest, snapshot, err := SandboxActorTemplate(template, "", policy) + actor, digest, snapshot, err := SandboxActorTemplate(template, "", policy, egress.Access{}) require.NoError(t, err) require.NotEqual(t, "5bf14bc98818a54bf4e59e5aca83de9bde5df22e9efa206b65e1389069a188c3", digest, "namespace-scoped placement must create a fresh immutable ActorTemplate") @@ -32,7 +33,7 @@ func TestSandboxPreparationPinsCompleteInputs(t *testing.T) { trust := actor.Volumes[2].GetSystemInfo().GetDataSources()[0].GetTrustBundle() require.Equal(t, []string{"egress-mitm.ate.dev"}, trust.GetNames()) require.Equal(t, "trust-bundle.pem", trust.GetPath()) - _, same, _, err := SandboxActorTemplate(template, atev1alpha1.SandboxClassGvisor, policy) + _, same, _, err := SandboxActorTemplate(template, atev1alpha1.SandboxClassGvisor, policy, egress.Access{}) require.NoError(t, err) require.Equal(t, digest, same) for _, change := range []struct { @@ -50,13 +51,13 @@ func TestSandboxPreparationPinsCompleteInputs(t *testing.T) { t.Run(change.name, func(t *testing.T) { input, settings := template.DeepCopy(), policy change.mutate(input, &settings) - _, changed, _, err := SandboxActorTemplate(input, change.class, settings) + _, changed, _, err := SandboxActorTemplate(input, change.class, settings, egress.Access{}) require.NoError(t, err) require.NotEqual(t, digest, changed) }) } template.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "SSL_CERT_FILE", Value: "/untrusted"}} - actor, _, _, err = SandboxActorTemplate(template, "", policy) + actor, _, _, err = SandboxActorTemplate(template, "", policy, egress.Access{}) require.ErrorContains(t, err, "reserved") require.Nil(t, actor) } diff --git a/go/core/internal/translator/collections.go b/go/core/internal/translator/collections.go index 02d07639fc..46563043f6 100644 --- a/go/core/internal/translator/collections.go +++ b/go/core/internal/translator/collections.go @@ -11,6 +11,7 @@ import ( // Collections contains every typed input used while compiling a revision. // Production supplies informer-backed collections; tests supply KRT mocks. type Collections struct { + EgressPolicies krt.Collection[*v1alpha3.EgressPolicy] Harnesses krt.Collection[*v1alpha3.Harness] AgentTemplates krt.Collection[*v1alpha3.AgentTemplate] ResolvedModelConfigs krt.Collection[ResolvedModelConfig] diff --git a/go/core/internal/translator/compiler.go b/go/core/internal/translator/compiler.go index 9a1770d9f1..8fb0a6517c 100644 --- a/go/core/internal/translator/compiler.go +++ b/go/core/internal/translator/compiler.go @@ -6,12 +6,15 @@ import ( "fmt" "maps" "slices" + "strings" "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) // Compiler resolves public API objects into a complete, immutable runtime @@ -61,6 +64,7 @@ type resolvedTemplateBinding struct { // HarnessInput contains the Kubernetes inputs needed by a harness compiler. type HarnessInput struct { + Egress egress.Access // AgentName is the runnable identity, independent of inline or referenced inputs. AgentName string Harness *HarnessConfiguration @@ -124,26 +128,32 @@ func (c *Compiler) CompileAgent(ctx context.Context, agent *v1alpha3.Agent) (*Co return nil, fmt.Errorf("resolve Harness %q: not found", harnessKey) } harness := harnessConfiguration(*foundHarness) - result, err := c.compileConfiguration(ctx, agent.Name, harness, template) + result, err := c.compileConfiguration(ctx, agent, harness, template) if err != nil { return nil, err } result.AgentUID = string(agent.UID) + spec := agent.Spec.DeepCopy() + slices.SortFunc(spec.EgressPolicyRefs, func(a, b corev1.LocalObjectReference) int { return strings.Compare(a.Name, b.Name) }) result.Provenance, err = json.Marshal(struct { AgentName string `json:"agentName"` AgentUID string `json:"agentUID"` Spec v1alpha3.AgentSpec `json:"spec"` Inputs json.RawMessage `json:"inputs"` - }{agent.Name, string(agent.UID), agent.Spec, result.Provenance}) + }{agent.Name, string(agent.UID), *spec, result.Provenance}) if err != nil { return nil, fmt.Errorf("encode Agent provenance: %w", err) } return result, nil } -// compileConfiguration compiles resolved configuration for the named Agent. +// compileConfiguration resolves inputs and compiles configuration for the Agent. // The Agent name owns runtime identity; template and Harness names are provenance. -func (c *Compiler) compileConfiguration(ctx context.Context, agentName string, harness *HarnessConfiguration, template *TemplateConfiguration) (*CompileResult, error) { +func (c *Compiler) compileConfiguration(ctx context.Context, agent *v1alpha3.Agent, harness *HarnessConfiguration, template *TemplateConfiguration) (*CompileResult, error) { + access, err := CompileEgressPolicies(c.ctx, c.collections, agent.Namespace, agent.Spec.EgressPolicyRefs) + if err != nil { + return nil, err + } harnessCompiler := c.harnessCompilers[harnessType(harness)] if harnessCompiler == nil { return nil, NewValidationError("Harness runtime is not supported by any compiler") @@ -156,7 +166,8 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agentName string, h if err != nil { return nil, err } - input.AgentName = agentName + input.AgentName = agent.Name + input.Egress = access image, err := harnessImage(harness, c.images) if err != nil { return nil, err @@ -165,12 +176,20 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agentName string, h if err != nil { return nil, err } + // Merge again at the shared boundary so every compiler preserves explicit + // grants. Built-in compilers also consume these bindings when validating + // model passthrough and inferred credentials. + combined, err := egress.CanonicalAccess(append(result.EgressDestinations, access.Destinations...), append(result.Credentials, access.Credentials...)) + if err != nil { + return nil, NewValidationError("%v", err) + } + result.EgressDestinations, result.Credentials = combined.Destinations, combined.Credentials workerKey := types.NamespacedName{Namespace: harness.Namespace, Name: harness.Spec.Substrate.WorkerPoolRef.Name} workerPool := krt.FetchOne(c.ctx, c.collections.WorkerPools, krt.FilterObjectName(workerKey)) if workerPool == nil { return nil, &WorkerPoolNotFoundError{WorkerPool: workerKey} } - result.AgentName = agentName + result.AgentName = agent.Name result.Image = image result.SandboxClass = (*workerPool).Spec.SandboxClass // Shared agents and plugin selections can contribute the same image. diff --git a/go/core/internal/translator/compiler_test.go b/go/core/internal/translator/compiler_test.go index 3b712f52f4..d2f37f47f5 100644 --- a/go/core/internal/translator/compiler_test.go +++ b/go/core/internal/translator/compiler_test.go @@ -227,6 +227,7 @@ func mockCollections(t *testing.T, objects ...any) translator.Collections { t.Helper() mock := krttest.NewMock(t, objects) collections := translator.Collections{ + EgressPolicies: krttest.GetMockCollection[*v1alpha3.EgressPolicy](mock), AgentTemplates: krttest.GetMockCollection[*v1alpha3.AgentTemplate](mock), Harnesses: krttest.GetMockCollection[*v1alpha3.Harness](mock), RemoteMCPServers: krttest.GetMockCollection[*v1alpha3.RemoteMCPServer](mock), @@ -682,7 +683,10 @@ func TestCompileAgentSharedADKConfig(t *testing.T) { runtimeHarness.Spec.BYO = &v1alpha3.BYOHarness{} runtimeHarness.Spec.Workload.Command = []string{"/app"} } - revision, err := compiler(t, modelConfig(), child, runtimeHarness, remoteMCPServer("search", "https://search.example.com/mcp")).CompileAgent(context.Background(), inlineAgent(runtimeHarness, root)) + policy, secret := egressPolicy() + agent := inlineAgent(runtimeHarness, root) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + revision, err := compiler(t, modelConfig(), child, runtimeHarness, policy, secret, remoteMCPServer("search", "https://search.example.com/mcp")).CompileAgent(context.Background(), agent) require.NoError(t, err) var config configv1.ADKConfig require.NoError(t, runtimeconfig.Unmarshal(revision.ConfigJSON, &config)) @@ -692,6 +696,7 @@ func TestCompileAgentSharedADKConfig(t *testing.T) { require.Equal(t, "research carefully", config.Agent.SubAgents[0].Instruction) require.Equal(t, []string{"lookup"}, config.Agent.SubAgents[0].HttpTools[0].Tools) require.Equal(t, "review", config.Agent.SubAgents[0].AgentPlugins.Skills[0].Name) + require.Contains(t, revision.EgressDestinations, "https://tickets.example.com:443") require.Contains(t, revision.EgressDestinations, "https://search.example.com:443") require.NotContains(t, revision.EgressDestinations, "https://ghcr.io:443") require.Equal(t, []translator.SkillImage{translator.SkillImage(child.Spec.Skills[0].Source.OCI)}, revision.SkillImages) diff --git a/go/core/internal/translator/credentials.go b/go/core/internal/translator/credentials.go index 29ac153c2e..e5a80006e1 100644 --- a/go/core/internal/translator/credentials.go +++ b/go/core/internal/translator/credentials.go @@ -21,7 +21,8 @@ const CredentialPlaceholder = "kagent-credential-injected" // placeholders and compiles their destination-scoped gateway bindings. Models // outside the agent tree (such as memory embeddings) are supplied separately. func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, environment []corev1.EnvVar) ([]corev1.EnvVar, []egress.Credential, error) { - var bindings []egress.Credential + bindings := make([]egress.Credential, 0, len(input.Egress.Credentials)) + bindings = append(bindings, input.Egress.Credentials...) boundModels := map[string]bool{} boundMCP := map[string]bool{} bind := func(rawURL, header, prefix, namespace, name, key string) error { @@ -30,7 +31,7 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, return NewValidationError("credential destination must be an absolute HTTP(S) URL without user information or fragment") } uri := "ate-secret://k8s.io/default/" + namespace + "/" + name + "/" + key - bindings = append(bindings, egress.Credential{Hostname: u.Hostname(), Header: header, Prefix: prefix, URI: uri}) + bindings = append(bindings, egress.Credential{Origin: egress.Origin(u), Header: header, Prefix: prefix, URI: uri}) return nil } models := append([]*ResolvedModelConfig(nil), extraModels...) @@ -99,10 +100,13 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, if err != nil { return nil, nil, NewValidationError("invalid passthrough credential destination") } - hostname := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") + origin, err := egress.ParseOrigin(egress.Origin(u)) + if err != nil { + return nil, nil, NewValidationError("invalid passthrough credential destination: %v", err) + } for _, binding := range bindings { - if binding.Hostname == hostname { - return nil, nil, NewValidationError("destination %q cannot combine caller-token passthrough with gateway credentials", hostname) + if binding.Origin == origin.String() { + return nil, nil, NewValidationError("destination %q cannot combine caller-token passthrough with gateway credentials", origin.String()) } } } diff --git a/go/core/internal/translator/credentials_test.go b/go/core/internal/translator/credentials_test.go index ba106e67b3..fb96a7fff7 100644 --- a/go/core/internal/translator/credentials_test.go +++ b/go/core/internal/translator/credentials_test.go @@ -8,6 +8,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) func TestCompileCredentialDestinations(t *testing.T) { @@ -40,7 +41,7 @@ func TestCompileCredentialDestinations(t *testing.T) { require.Nil(t, got[0].ValueFrom) require.NotNil(t, environment[0].ValueFrom, "compilation must not mutate inputs") require.Len(t, bindings, 1) - require.Equal(t, test.host, bindings[0].Hostname) + require.Equal(t, "https://"+test.host+":443", bindings[0].Origin) require.Equal(t, test.header, bindings[0].Header) require.Equal(t, test.prefix, bindings[0].Prefix) require.Equal(t, "ate-secret://k8s.io/default/team/auth/"+test.spec.APIKeySecretKey, bindings[0].URI) @@ -170,3 +171,13 @@ func TestModelCredentialTargetOllamaCloud(t *testing.T) { }) } } + +func TestPassthroughCredentialConflictUsesCanonicalOrigin(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: true, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://API.OPENAI.COM.:0443/v1"}}) + input.Egress.Credentials = []egress.Credential{{Origin: "https://api.openai.com", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token"}} + _, _, err := CompileCredentials(input, nil, nil) + require.ErrorContains(t, err, "cannot combine caller-token passthrough") + input.Root.ResolvedModelConfig.Config.Spec.OpenAI.BaseURL = "https://api.openai.com:8443/v1" + _, _, err = CompileCredentials(input, nil, nil) + require.NoError(t, err, "a different port is a distinct credential scope") +} diff --git a/go/core/internal/translator/egress.go b/go/core/internal/translator/egress.go new file mode 100644 index 0000000000..b72f1db40e --- /dev/null +++ b/go/core/internal/translator/egress.go @@ -0,0 +1,74 @@ +package translator + +import ( + "fmt" + "strings" + + "github.com/kagent-dev/kagent/go/core/internal/egress" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation" +) + +// CompileEgressPolicies resolves same-namespace grants for either an Agent or a +// SandboxTemplate. KRT tracks policy and Secret dependencies; only references +// enter the result, so Secret value rotation does not change a revision. +func CompileEgressPolicies(ctx krt.HandlerContext, collections Collections, namespace string, refs []corev1.LocalObjectReference) (egress.Access, error) { + if len(refs) > 16 { + return egress.Access{}, NewValidationError("at most 16 egressPolicyRefs are supported") + } + var destinations []string + var credentials []egress.Credential + for _, ref := range refs { + if len(validation.IsDNS1123Subdomain(ref.Name)) != 0 { + return egress.Access{}, NewValidationError("invalid EgressPolicy reference %q", ref.Name) + } + key := namespace + "/" + ref.Name + policy := krt.FetchOne(ctx, collections.EgressPolicies, krt.FilterKey(key)) + if policy == nil { + return egress.Access{}, fmt.Errorf("resolve EgressPolicy %q: not found", key) + } + if len((*policy).Spec.Destinations) == 0 || len((*policy).Spec.Destinations) > 64 { + return egress.Access{}, NewValidationError("EgressPolicy %q requires 1 to 64 destinations", key) + } + for _, destination := range (*policy).Spec.Destinations { + origin, err := egress.ParseOrigin(destination.Origin) + if err != nil { + return egress.Access{}, NewValidationError("EgressPolicy %q: %v", key, err) + } + destinations = append(destinations, origin.String()) + auth := destination.Authentication + if auth == nil { + continue + } + if origin.Scheme != "https" || strings.Contains(origin.Host, "*") { + return egress.Access{}, NewValidationError("EgressPolicy %q: authentication requires an exact HTTPS origin", key) + } + secret := auth.SecretKeyRef + binding := egress.Credential{Origin: origin.String(), Header: auth.Header} + if auth.Prefix != nil { + binding.Prefix = *auth.Prefix + } + if secret.Name == "" || secret.Key == "" || (secret.Optional != nil && *secret.Optional) { + return egress.Access{}, NewValidationError("EgressPolicy %q requires a non-optional Secret name and key", key) + } + binding.URI = "ate-secret://k8s.io/default/" + namespace + "/" + secret.Name + "/" + secret.Key + if _, err := egress.CanonicalCredentials([]egress.Credential{binding}); err != nil { + return egress.Access{}, NewValidationError("EgressPolicy %q: %v", key, err) + } + found := krt.FetchOne(ctx, collections.Secrets, krt.FilterKey(namespace+"/"+secret.Name)) + if found == nil { + return egress.Access{}, fmt.Errorf("EgressPolicy %q: Secret %q not found", key, secret.Name) + } + if _, ok := (*found).Data[secret.Key]; !ok { + return egress.Access{}, fmt.Errorf("EgressPolicy %q: Secret %q is missing key %q", key, secret.Name, secret.Key) + } + credentials = append(credentials, binding) + } + } + access, err := egress.CanonicalAccess(destinations, credentials) + if err != nil { + return egress.Access{}, NewValidationError("%v", err) + } + return access, nil +} diff --git a/go/core/internal/translator/egress_test.go b/go/core/internal/translator/egress_test.go new file mode 100644 index 0000000000..848c7e7f6b --- /dev/null +++ b/go/core/internal/translator/egress_test.go @@ -0,0 +1,194 @@ +package translator_test + +import ( + "encoding/json" + "testing" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/stretchr/testify/require" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func egressPolicy() (*v1alpha3.EgressPolicy, *corev1.Secret) { + return &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "tickets"}, Spec: v1alpha3.EgressPolicySpec{ + Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com", Authentication: &v1alpha3.EgressAuthentication{ + Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "ticket-auth"}, Key: "token"}, + }}}, + }}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "ticket-auth"}, Data: map[string][]byte{"token": []byte("private-ticket-token")}} +} + +func TestEgressPolicyCredentialPrefixes(t *testing.T) { + for _, tc := range []struct { + name string + header string + prefix *string + want string + }{ + {name: "bearer token", header: "Authorization", prefix: new("Bearer "), want: "Bearer "}, + {name: "API key", header: "X-API-Key"}, + {name: "empty prefix", header: "X-API-Key", prefix: new("")}, + {name: "custom prefix", header: "Authorization", prefix: new("Token "), want: "Token "}, + {name: "literal prefix", header: "Authorization", prefix: new("Token"), want: "Token"}, + } { + t.Run(tc.name, func(t *testing.T) { + policy, secret := egressPolicy() + auth := policy.Spec.Destinations[0].Authentication + auth.Header, auth.Prefix = tc.header, tc.prefix + access, err := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", []corev1.LocalObjectReference{{Name: policy.Name}}) + require.NoError(t, err) + wantHeader := "authorization" + if tc.header == "X-API-Key" { + wantHeader = "x-api-key" + } + require.Equal(t, []egress.Credential{{Origin: "https://tickets.example.com:443", Header: wantHeader, Prefix: tc.want, URI: "ate-secret://k8s.io/default/test/ticket-auth/token"}}, access.Credentials) + }) + } +} + +func TestAgentEgressPoliciesPinGrantsAcrossHarnesses(t *testing.T) { + for _, kind := range []string{"kagent", "codex", "claude", "byo"} { + t.Run(kind, func(t *testing.T) { + policy, secret := egressPolicy() + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "runner"}, Spec: v1alpha3.HarnessSpec{ + Workload: v1alpha3.HarnessWorkload{Image: "runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + switch kind { + case "kagent": + harness.Spec.Kagent = &v1alpha3.KagentHarness{} + case "codex": + harness.Spec.Codex = &v1alpha3.CodexHarness{} + case "claude": + harness.Spec.Claude = &v1alpha3.ClaudeHarness{} + case "byo": + harness.Spec.BYO = &v1alpha3.BYOHarness{} + } + model := modelConfig() + if kind == "codex" { + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = secret.Name, "token" + model.Spec.OpenAI = &v1alpha3.OpenAIConfig{APIFormat: new(v1alpha3.OpenAIAPIFormatResponses)} + } + if kind == "claude" { + model.Spec.Provider, model.Spec.Model = v1alpha3.ModelProviderAnthropic, "claude-sonnet-4-5" + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = secret.Name, "token" + } + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "helper"}, Spec: v1alpha3.AgentTemplateSpec{ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help"}} + agent := inlineAgent(harness, template) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + compile := func() *translator.CompileResult { + result, err := compiler(t, model, harness, policy, secret).CompileAgent(t.Context(), agent) + require.NoError(t, err) + return result + } + result := compile() + require.Contains(t, result.EgressDestinations, "https://tickets.example.com:443") + require.Greater(t, len(result.EgressDestinations), 1, "explicit policies preserve inferred destinations") + require.Equal(t, "https://tickets.example.com:443", result.Credentials[len(result.Credentials)-1].Origin) + encoded, err := json.Marshal(result) + require.NoError(t, err) + require.NotContains(t, string(encoded), string(secret.Data["token"])) + original, err := result.Digest() + require.NoError(t, err) + secret.Data["token"], secret.UID = []byte("rotated-token"), "replacement" + rotated, err := compile().Digest() + require.NoError(t, err) + require.Equal(t, original, rotated) + policy.Spec.Destinations = append(policy.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://registry.npmjs.org"}) + updated, err := compile().Digest() + require.NoError(t, err) + require.NotEqual(t, original, updated) + require.NotContains(t, result.EgressDestinations, "https://registry.npmjs.org:443", "previous revision remains pinned") + // Explicit policy credentials cannot silently replace inferred model credentials. + policy.Spec.Destinations[0].Origin = "https://api.openai.com" + auth := policy.Spec.Destinations[0].Authentication + if kind == "claude" { + policy.Spec.Destinations[0].Origin = "https://api.anthropic.com" + auth.Header, auth.Prefix = "x-api-key", nil + } + auth.SecretKeyRef.Key = "alternate" + secret.Data["alternate"] = []byte("different-token") + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = secret.Name, "token" + _, err = compiler(t, model, harness, policy, secret).CompileAgent(t.Context(), agent) + require.ErrorContains(t, err, "conflicting credentials") + // Codex and Claude reject caller-token passthrough independently. + if kind == "claude" || kind == "codex" { + return + } + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = "", "" + model.Spec.APIKeyPassthrough = true + _, err = compiler(t, model, harness, policy, secret).CompileAgent(t.Context(), agent) + require.ErrorContains(t, err, "cannot combine caller-token passthrough") + }) + } +} + +func TestEgressPolicyResolutionFailsClosed(t *testing.T) { + for _, name := range []string{"missing policy", "other namespace", "missing secret", "missing key", "optional secret", "HTTP auth", "wildcard auth", "empty auth", "missing header", "protected header", "invalid prefix", "conflicting auth", "conflicting prefix"} { + t.Run(name, func(t *testing.T) { + policy, secret := egressPolicy() + objects := []any{policy, secret} + refs := []corev1.LocalObjectReference{{Name: policy.Name}} + switch name { + case "missing policy": + objects = []any{secret} + case "other namespace": + policy.Namespace = "other" + case "missing secret": + objects = []any{policy} + case "missing key": + secret.Data = nil + case "optional secret": + policy.Spec.Destinations[0].Authentication.SecretKeyRef.Optional = new(true) + case "HTTP auth": + policy.Spec.Destinations[0].Origin = "http://tickets.example.com" + case "wildcard auth": + policy.Spec.Destinations[0].Origin = "https://*.example.com" + case "empty auth": + policy.Spec.Destinations[0].Authentication = &v1alpha3.EgressAuthentication{} + case "missing header": + policy.Spec.Destinations[0].Authentication.Header = "" + case "protected header": + policy.Spec.Destinations[0].Authentication.Header = "Host" + case "invalid prefix": + policy.Spec.Destinations[0].Authentication.Prefix = new("Bearer\r\nInjected: ") + case "conflicting auth", "conflicting prefix": + other := policy.DeepCopy() + other.Name = "other" + if name == "conflicting prefix" { + other.Spec.Destinations[0].Authentication.Prefix = new("Token ") + } else { + other.Spec.Destinations[0].Authentication.SecretKeyRef.Key = "other" + secret.Data["other"] = []byte("different") + } + objects = append(objects, other) + refs = append(refs, corev1.LocalObjectReference{Name: other.Name}) + } + _, err := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, objects...), "test", refs) + require.Error(t, err) + }) + } +} + +func TestEgressPoliciesMergeCredentialsAndOrigins(t *testing.T) { + policy, secret := egressPolicy() + other := policy.DeepCopy() + other.Name = "registries" + other.Spec.Destinations = append(other.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://*.npmjs.org"}, v1alpha3.EgressDestination{ + Origin: "https://tickets.example.com:8443", Authentication: &v1alpha3.EgressAuthentication{Header: "X-API-Key", SecretKeyRef: policy.Spec.Destinations[0].Authentication.SecretKeyRef}, + }) + collections := mockCollections(t, policy, other, secret) + refs := []corev1.LocalObjectReference{{Name: policy.Name}, {Name: other.Name}} + access, err := translator.CompileEgressPolicies(krt.TestingDummyContext{}, collections, "test", refs) + require.NoError(t, err) + require.Len(t, access.Destinations, 3) + require.Len(t, access.Credentials, 2) + require.Equal(t, "x-api-key", access.Credentials[1].Header) + refs[0], refs[1] = refs[1], refs[0] + reversed, err := translator.CompileEgressPolicies(krt.TestingDummyContext{}, collections, "test", refs) + require.NoError(t, err) + require.Equal(t, access, reversed) +} diff --git a/go/core/internal/translator/foundry_test.go b/go/core/internal/translator/foundry_test.go index eeb1491ce3..b2198fc2ed 100644 --- a/go/core/internal/translator/foundry_test.go +++ b/go/core/internal/translator/foundry_test.go @@ -90,7 +90,7 @@ func TestCompileFoundryEndpoint(t *testing.T) { } require.Contains(t, revision.Environment, corev1.EnvVar{Name: "FOUNDRY_ENDPOINT", Value: endpoint}) require.Len(t, revision.Credentials, 1) - require.Equal(t, host, revision.Credentials[0].Hostname) + require.Equal(t, endpoint+":443", revision.Credentials[0].Origin) require.Equal(t, "api-key", revision.Credentials[0].Header) require.Equal(t, "ate-secret://k8s.io/default/test/foundry-auth/token", revision.Credentials[0].URI) require.Contains(t, revision.EgressDestinations, "https://"+host+":443") diff --git a/go/core/test/e2e/egress_test.go b/go/core/test/e2e/egress_test.go index 0088ccea35..eabb81fd0e 100644 --- a/go/core/test/e2e/egress_test.go +++ b/go/core/test/e2e/egress_test.go @@ -6,16 +6,21 @@ import ( "net/http" "net/http/httptest" "net/url" + "strings" "sync/atomic" "testing" + "time" a2atype "github.com/a2aproject/a2a-go/v2/a2a" + "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ) -func TestSessionEgressDeniesUnconfiguredDestination(t *testing.T) { +func TestSessionEgressPolicyPinsAdditionalDestinations(t *testing.T) { t.Parallel() forEachHarness(t, func(t *testing.T, harness testHarness) { t.Parallel() @@ -28,7 +33,7 @@ func TestSessionEgressDeniesUnconfiguredDestination(t *testing.T) { denied, err := url.Parse(reachableModelURL(t, origin.URL)) require.NoError(t, err) // Give the destination a distinct hostname even when host mocks use - // host.docker.internal. The gateway allowlist is keyed by hostname. + // host.docker.internal. Keep the redirect visibly distinct from the model origin. kube := interactionKubeClient(t) alias := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{GenerateName: "denied-origin-", Namespace: "kagent"}, @@ -71,5 +76,36 @@ func TestSessionEgressDeniesUnconfiguredDestination(t *testing.T) { } require.Equal(t, a2atype.TaskStateFailed, task.Status.State, "redirect task text = %q", taskText(task)) require.False(t, reachedDenied.Load(), "redirect reached an unconfigured destination") + + grant := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "redirect-access-", Namespace: "kagent"}, Spec: v1alpha3.EgressPolicySpec{ + Destinations: []v1alpha3.EgressDestination{{Origin: denied.Scheme + "://" + denied.Host}}, + }} + require.NoError(t, kube.Create(t.Context(), grant)) + t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), grant)) }) + agent := &v1alpha3.Agent{} + name := strings.TrimPrefix(fixture.tenant, "kagent/") + key := ctrlclient.ObjectKey{Namespace: "kagent", Name: name} + require.NoError(t, kube.Get(t.Context(), key, agent)) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: grant.Name}} + require.NoError(t, kube.Update(t.Context(), agent)) + require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, agent); err != nil { + return false, err + } + for _, condition := range agent.Status.Conditions { + if condition.Type == v1alpha3.AgentConditionReady && condition.ObservedGeneration == agent.Generation { + return condition.Status == metav1.ConditionTrue, nil + } + } + return false, nil + })) + allowed := newInteractionFixtureForTemplate(t, harness, target, name) + _, _, task = allowed.send(t, "What is 2+2?") + require.Equal(t, a2atype.TaskStateCompleted, task.Status.State, "policy-granted redirect text = %q", taskText(task)) + require.True(t, reachedDenied.Load()) + reachedDenied.Store(false) + _, _, task = fixture.send(t, "What is 2+2?") + require.Equal(t, a2atype.TaskStateFailed, task.Status.State) + require.False(t, reachedDenied.Load(), "existing session must retain its earlier grant") }) } diff --git a/helm/kagent-crds/templates/api.kagent.dev_agents.yaml b/helm/kagent-crds/templates/api.kagent.dev_agents.yaml index bdc7bbab3e..91ac641434 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_agents.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_agents.yaml @@ -52,6 +52,35 @@ spec: are local to the Agent's namespace, including references nested in inline template specs. properties: + egressPolicyRefs: + description: |- + EgressPolicyRefs grants additional access to the whole runtime, including + shared subagents. Policies resolve in this Agent's namespace. Integration + access inferred from models and tools is always included. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: egressPolicyRefs requires non-empty names + rule: self.all(ref, has(ref.name) && ref.name != '') harnessRef: description: |- HarnessRef selects the platform-managed runtime configuration that executes diff --git a/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml b/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml new file mode 100644 index 0000000000..427a74fbc1 --- /dev/null +++ b/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml @@ -0,0 +1,141 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: egresspolicies.api.kagent.dev +spec: + group: api.kagent.dev + names: + categories: + - kagent + kind: EgressPolicy + listKind: EgressPolicyList + plural: egresspolicies + singular: egresspolicy + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: |- + EgressPolicy is a reusable grant of additional network access. Workloads in its + namespace select it explicitly; changing it affects newly prepared revisions. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EgressPolicySpec grants additional HTTP(S) access to workloads + in its namespace. + properties: + destinations: + description: Destinations are additional origins the workload may + reach. + items: + description: EgressDestination grants access to one HTTP(S) origin, + including all its paths. + properties: + authentication: + description: |- + Authentication replaces a placeholder header at the gateway. Requests + must send the header; its credential value is never supplied to the runtime. + properties: + header: + description: |- + Header is the request header carrying the credential, for example + Authorization or X-API-Key. + maxLength: 256 + pattern: ^[!#$%&'*+.^_|~0-9A-Za-z-]+$ + type: string + x-kubernetes-validations: + - message: this header cannot carry an egress credential + rule: '!(self.lowerAscii() in [''host'', ''connection'', + ''content-length'', ''transfer-encoding'', ''te'', ''trailer'', + ''upgrade'', ''keep-alive'', ''proxy-authorization'', + ''proxy-authenticate'', ''cookie''])' + prefix: + description: |- + Prefix is prepended verbatim to the Secret value, for example "Bearer ". + Include any separating space. Omitted or empty means no prefix. + maxLength: 256 + pattern: ^[^\x00-\x08\x0a-\x1f\x7f]*$ + type: string + secretKeyRef: + description: SecretKeyRef selects a required key in the + EgressPolicy's namespace. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: secretKeyRef requires a name and key + rule: has(self.name) && self.name != '' && self.key != + '' + - message: egress credentials cannot be optional + rule: '!has(self.optional) || !self.optional' + required: + - header + - secretKeyRef + type: object + origin: + description: |- + Origin is a scheme, DNS hostname, and optional port. A wildcard replaces + exactly one leftmost DNS label, for example https://*.example.com. + maxLength: 270 + pattern: ^https?://(\*\.([a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?\.)+|([a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?\.)*)[a-z]([-a-z0-9]{0,61}[a-z0-9])?(:([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?$ + type: string + required: + - origin + type: object + x-kubernetes-validations: + - message: authentication requires an exact HTTPS origin + rule: '!has(self.authentication) || (self.origin.startsWith(''https://'') + && !self.origin.contains(''*''))' + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - origin + x-kubernetes-list-type: map + required: + - destinations + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml b/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml index 4749ca0147..5ee69f5757 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml @@ -51,6 +51,34 @@ spec: description: SandboxTemplateSpec defines standalone sandbox environment configuration. properties: + egressPolicyRefs: + description: |- + EgressPolicyRefs grants additional access in this template's namespace. + Each sandbox retains the access pinned by its prepared revision. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: egressPolicyRefs requires non-empty names + rule: self.all(ref, has(ref.name) && ref.name != '') env: description: |- Env supplies runtime environment defaults. diff --git a/helm/kagent/templates/rbac/getter-role.yaml b/helm/kagent/templates/rbac/getter-role.yaml index c136b629ec..007b3fbab0 100644 --- a/helm/kagent/templates/rbac/getter-role.yaml +++ b/helm/kagent/templates/rbac/getter-role.yaml @@ -4,6 +4,7 @@ resources: - harnesses - agents + - egresspolicies - sandboxtemplates - agenttemplates - modelconfigs diff --git a/ui/src/api/domain/agents.ts b/ui/src/api/domain/agents.ts index 2ee45c74a6..e818b80df4 100644 --- a/ui/src/api/domain/agents.ts +++ b/ui/src/api/domain/agents.ts @@ -8,6 +8,7 @@ export type AgentTemplateSource = export interface AgentSpec { template: AgentTemplateSource; harnessRef: { name: string }; + egressPolicyRefs?: { name: string }[]; } /** One condition the controller recorded for an Agent. */ export interface AgentCondition { diff --git a/ui/src/pages/agents/AgentFormPage.tsx b/ui/src/pages/agents/AgentFormPage.tsx index 17eb636b01..44bc160331 100644 --- a/ui/src/pages/agents/AgentFormPage.tsx +++ b/ui/src/pages/agents/AgentFormPage.tsx @@ -117,6 +117,7 @@ function AgentForm({ agent }: { agent?: Agent }) { setSaving(true); setFailure(undefined); const agentSpec = { + ...(spec?.egressPolicyRefs ? { egressPolicyRefs: spec.egressPolicyRefs } : {}), template: templateSource === "reference" ? { ref: { name: templateRef! } } : { inline: specFromDraft(inlineTemplate, spec?.template.inline) }, From 7928853157eb5e0f0d54af1e4b9cda04f19660bf Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Tue, 13 Oct 2026 00:57:42 +0000 Subject: [PATCH 02/43] test(helm): verify read-only egress policy RBAC Signed-off-by: Eitan Yarmush --- helm/kagent/tests/rbac_test.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/helm/kagent/tests/rbac_test.yaml b/helm/kagent/tests/rbac_test.yaml index ef6d6debb2..4e5a82d805 100644 --- a/helm/kagent/tests/rbac_test.yaml +++ b/helm/kagent/tests/rbac_test.yaml @@ -140,9 +140,18 @@ tests: - equal: path: rules[0].apiGroups value: [api.kagent.dev] + - equal: + path: rules[0].resources + value: [harnesses, agents, egresspolicies, sandboxtemplates, agenttemplates, modelconfigs, modelproviderconfigs, remotemcpservers] + template: rbac/getter-role.yaml + - equal: + path: rules[0].verbs + value: [get, list, watch] + template: rbac/getter-role.yaml - equal: path: rules[0].resources value: [harnesses, agents, sandboxtemplates, agenttemplates, modelconfigs, modelproviderconfigs, remotemcpservers] + template: rbac/writer-role.yaml - it: should grant namespaced agent access in the new group set: @@ -154,9 +163,18 @@ tests: - equal: path: rules[0].apiGroups value: [api.kagent.dev] + - equal: + path: rules[0].resources + value: [harnesses, agents, egresspolicies, sandboxtemplates, agenttemplates, modelconfigs, modelproviderconfigs, remotemcpservers] + template: rbac/getter-role.yaml + - equal: + path: rules[0].verbs + value: [get, list, watch] + template: rbac/getter-role.yaml - equal: path: rules[0].resources value: [harnesses, agents, sandboxtemplates, agenttemplates, modelconfigs, modelproviderconfigs, remotemcpservers] + template: rbac/writer-role.yaml - it: should have correct labels on all resources templates: From 7902870ee75082245039fe55f48eb54c78257f20 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Tue, 13 Oct 2026 18:29:38 +0000 Subject: [PATCH 03/43] fix: validate MCP credential headers and test tool egress Signed-off-by: Eitan Yarmush --- docs/architecture/egress-policies.md | 5 + go/api/v1alpha3/common_types.go | 1 + go/api/v1alpha3/configuration_crd_cel_test.go | 1 + go/api/v1alpha3/remotemcpserver_cel_test.go | 90 +++++++++++++++++ go/api/v1alpha3/remotemcpserver_types.go | 6 ++ go/core/test/e2e/egress_test.go | 98 ++++++++++++------- .../api.kagent.dev_egresspolicies.yaml | 2 +- .../api.kagent.dev_remotemcpservers.yaml | 16 +++ 8 files changed, 182 insertions(+), 37 deletions(-) create mode 100644 go/api/v1alpha3/remotemcpserver_cel_test.go diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index d7b646fdda..e699519349 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -136,6 +136,11 @@ cannot share its origin with static gateway credentials. Routing, framing, hop-by-hop, proxy-authentication, and Cookie headers cannot carry credentials. At most 16 credential headers are supported per origin. +`RemoteMCPServer.spec.headersFrom` applies the same credential-header rules to +Secret-backed entries during Kubernetes admission. Literal and ConfigMap-backed +headers do not use gateway credential injection. A server may configure up to +64 headers, with names of at most 256 characters. + An exact credential-bearing rule wins over a matching wildcard rule. Credentials are never attached to wildcard rules or implicitly copied to other ports or schemes. HTTP and HTTPS cannot both use the same hostname pattern and port, diff --git a/go/api/v1alpha3/common_types.go b/go/api/v1alpha3/common_types.go index 481189eb0f..80a19120f0 100644 --- a/go/api/v1alpha3/common_types.go +++ b/go/api/v1alpha3/common_types.go @@ -157,6 +157,7 @@ func (s *ValueSource) Resolve(ctx context.Context, client client.Client, namespa // +kubebuilder:validation:ExactlyOneOf=value;valueFrom type ValueRef struct { // +required + // +kubebuilder:validation:MaxLength=256 Name string `json:"name"` // +optional Value string `json:"value,omitempty"` diff --git a/go/api/v1alpha3/configuration_crd_cel_test.go b/go/api/v1alpha3/configuration_crd_cel_test.go index 8d2b5478b7..ab236cd4b7 100644 --- a/go/api/v1alpha3/configuration_crd_cel_test.go +++ b/go/api/v1alpha3/configuration_crd_cel_test.go @@ -463,6 +463,7 @@ func TestConfigurationCRDValidation(t *testing.T) { } t.Run("egress policies", func(t *testing.T) { testEgressPolicyValidation(t, cl, namespace) }) + t.Run("MCP credential headers", func(t *testing.T) { testMCPCredentialHeaderValidation(t, cl, namespace) }) t.Run("literal runtime environment", func(t *testing.T) { harness := validHarness(namespace, "env-harness", HarnessSpec{Kagent: &KagentHarness{}}) diff --git a/go/api/v1alpha3/remotemcpserver_cel_test.go b/go/api/v1alpha3/remotemcpserver_cel_test.go new file mode 100644 index 0000000000..44a0020bfd --- /dev/null +++ b/go/api/v1alpha3/remotemcpserver_cel_test.go @@ -0,0 +1,90 @@ +package v1alpha3 + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func testMCPCredentialHeaderValidation(t *testing.T, cl ctrlclient.Client, namespace string) { + t.Helper() + for _, tc := range []struct { + name string + header string + reject string + }{ + {name: "authorization", header: "Authorization"}, + {name: "api-key", header: "X-API-Key"}, + {name: "token-characters", header: "X-!#$%&'*+-.^_`|~0123456789"}, + {name: "max-length", header: strings.Repeat("x", 256)}, + {name: "empty", reject: "valid HTTP header name"}, + {name: "too-long", header: strings.Repeat("x", 257), reject: "may not be more than 256"}, + {name: "whitespace", header: "X Key", reject: "valid HTTP header name"}, + {name: "newline", header: "X-Key\r\nInjected", reject: "valid HTTP header name"}, + {name: "colon", header: "X:Key", reject: "valid HTTP header name"}, + {name: "backslash", header: "X\\Key", reject: "valid HTTP header name"}, + {name: "unicode", header: "X-名", reject: "valid HTTP header name"}, + {name: "host", header: "Host", reject: "cannot carry an egress credential"}, + {name: "connection", header: "cOnNeCtIoN", reject: "cannot carry an egress credential"}, + {name: "content-length", header: "Content-Length", reject: "cannot carry an egress credential"}, + {name: "transfer-encoding", header: "Transfer-Encoding", reject: "cannot carry an egress credential"}, + {name: "te", header: "TE", reject: "cannot carry an egress credential"}, + {name: "trailer", header: "Trailer", reject: "cannot carry an egress credential"}, + {name: "upgrade", header: "Upgrade", reject: "cannot carry an egress credential"}, + {name: "keep-alive", header: "Keep-Alive", reject: "cannot carry an egress credential"}, + {name: "proxy-authorization", header: "Proxy-Authorization", reject: "cannot carry an egress credential"}, + {name: "proxy-authenticate", header: "Proxy-Authenticate", reject: "cannot carry an egress credential"}, + {name: "cookie", header: "Cookie", reject: "cannot carry an egress credential"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := &RemoteMCPServer{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "mcp-header-" + tc.name}, + Spec: RemoteMCPServerSpec{ + Description: "MCP credential header validation", + URL: "https://mcp.example.com/mcp", + HeadersFrom: []ValueRef{{Name: tc.header, ValueFrom: &ValueSource{Type: SecretValueSource, Name: "auth", Key: "token"}}}, + }, + } + err := cl.Create(t.Context(), server) + if tc.reject != "" { + require.ErrorContains(t, err, tc.reject) + require.ErrorContains(t, err, "spec.headersFrom[0].name") + return + } + require.NoError(t, err) + require.NoError(t, cl.Get(t.Context(), ctrlclient.ObjectKeyFromObject(server), server)) + require.Equal(t, tc.header, server.Spec.HeadersFrom[0].Name) + server.Spec.HeadersFrom[0].Name = "Cookie" + require.ErrorContains(t, cl.Update(t.Context(), server), "cannot carry an egress credential") + }) + } + for _, source := range []ValueRef{ + {Name: "Cookie", Value: "session=example"}, + {Name: "Cookie", ValueFrom: &ValueSource{Type: ConfigMapValueSource, Name: "headers", Key: "cookie"}}, + } { + server := &RemoteMCPServer{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, GenerateName: "mcp-ordinary-header-"}, + Spec: RemoteMCPServerSpec{Description: "Ordinary headers are not gateway credentials", URL: "https://mcp.example.com/mcp", HeadersFrom: []ValueRef{source}}, + } + require.NoError(t, cl.Create(t.Context(), server)) + server.Spec.HeadersFrom[0] = ValueRef{Name: "Cookie", ValueFrom: &ValueSource{Type: SecretValueSource, Name: "auth", Key: "token"}} + require.ErrorContains(t, cl.Update(t.Context(), server), "cannot carry an egress credential") + } + t.Run("header count", func(t *testing.T) { + headers := make([]ValueRef, 64) + for i := range headers { + headers[i] = ValueRef{Name: fmt.Sprintf("X-Header-%d", i), Value: "example"} + } + server := &RemoteMCPServer{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "mcp-header-count"}, + Spec: RemoteMCPServerSpec{Description: "Header count validation", URL: "https://mcp.example.com/mcp", HeadersFrom: headers}, + } + require.NoError(t, cl.Create(t.Context(), server)) + server.Spec.HeadersFrom = append(server.Spec.HeadersFrom, ValueRef{Name: "X-Extra", Value: "example"}) + require.ErrorContains(t, cl.Update(t.Context(), server), "must have at most 64 items") + }) +} diff --git a/go/api/v1alpha3/remotemcpserver_types.go b/go/api/v1alpha3/remotemcpserver_types.go index 41a4c5c885..4fb2b9ce06 100644 --- a/go/api/v1alpha3/remotemcpserver_types.go +++ b/go/api/v1alpha3/remotemcpserver_types.go @@ -48,7 +48,13 @@ type RemoteMCPServerSpec struct { // +kubebuilder:validation:MinLength=1 // +required URL string `json:"url"` + // HeadersFrom supplies request headers. Secret-backed headers are injected by + // the egress gateway and cannot use routing, framing, hop-by-hop, + // proxy-authentication, or Cookie headers. At most 64 headers may be supplied. // +optional + // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:items:XValidation:rule="!has(self.valueFrom) || self.valueFrom.type != 'Secret' || self.name.matches('^[!#$%&\\\\x27*+.^_`|~0-9A-Za-z-]+$')",message="Secret-backed headers require a valid HTTP header name",fieldPath=".name" + // +kubebuilder:validation:items:XValidation:rule="!has(self.valueFrom) || self.valueFrom.type != 'Secret' || !(self.name.lowerAscii() in ['host', 'connection', 'content-length', 'transfer-encoding', 'te', 'trailer', 'upgrade', 'keep-alive', 'proxy-authorization', 'proxy-authenticate', 'cookie'])",message="this header cannot carry an egress credential",fieldPath=".name" HeadersFrom []ValueRef `json:"headersFrom,omitempty"` // +optional // +kubebuilder:default="30s" diff --git a/go/core/test/e2e/egress_test.go b/go/core/test/e2e/egress_test.go index eabb81fd0e..fd62f45d91 100644 --- a/go/core/test/e2e/egress_test.go +++ b/go/core/test/e2e/egress_test.go @@ -1,12 +1,15 @@ package e2e_test import ( + "bytes" "context" + "encoding/json" + "io" "net" "net/http" "net/http/httptest" + "net/http/httputil" "net/url" - "strings" "sync/atomic" "testing" "time" @@ -25,15 +28,11 @@ func TestSessionEgressPolicyPinsAdditionalDestinations(t *testing.T) { forEachHarness(t, func(t *testing.T, harness testHarness) { t.Parallel() target := interactionTarget(t) - var reachedDenied atomic.Bool - origin := startModelRecorder(t, startMockLLMServer(t, interactionMocks, "mocks/invoke_agent.json"), func([]byte) error { - reachedDenied.Store(true) - return nil - }) - denied, err := url.Parse(reachableModelURL(t, origin.URL)) + mcpURL, mcpServer := startMCPMock(t) + denied, err := url.Parse(mcpURL) require.NoError(t, err) // Give the destination a distinct hostname even when host mocks use - // host.docker.internal. Keep the redirect visibly distinct from the model origin. + // host.docker.internal. It must not share the model or configured MCP origin. kube := interactionKubeClient(t) alias := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{GenerateName: "denied-origin-", Namespace: "kagent"}, @@ -43,23 +42,51 @@ func TestSessionEgressPolicyPinsAdditionalDestinations(t *testing.T) { t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), alias)) }) denied.Host = net.JoinHostPort(alias.Name+".kagent.svc.cluster.local", denied.Port()) - // First prove the destination is reachable and speaks this harness's - // model protocol when explicitly configured. - control := newInteractionFixture(t, harness, target, denied.String()) - _, _, task := control.send(t, "What is 2+2?") + toolCalls := func() int { + calls := 0 + for _, request := range mcpServer.Requests() { + var message struct { + Method string `json:"method"` + } + if json.Unmarshal(request.Body, &message) == nil && message.Method == "tools/call" { + calls++ + } + } + return calls + } + const prompt = "Add 3 and 5 using the configured MCP server." + // Prove the destination is reachable when explicitly configured. + controlName, _ := createMCPInteractionTemplate(t, harness, denied.String(), false) + control := newInteractionFixtureForTemplate(t, harness, target, controlName) + _, _, task := control.send(t, prompt) require.Equal(t, a2atype.TaskStateCompleted, task.Status.State) - require.Contains(t, taskText(task), "The answer is 4.") - require.True(t, reachedDenied.Load()) - reachedDenied.Store(false) + require.Contains(t, taskText(task), "result is 8") + controlCalls := toolCalls() + require.Positive(t, controlCalls) - var reachedAllowed atomic.Bool + upstream := *denied + upstream.Host, upstream.Path = net.JoinHostPort("127.0.0.1", denied.Port()), "" + proxy := httputil.NewSingleHostReverseProxy(&upstream) + var redirects atomic.Int64 server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - reachedAllowed.Store(true) - // The model host is allowed, but its redirect must not expand the - // Actor's policy to another reachable, unconfigured destination. - redirect := *denied - redirect.Path, redirect.RawQuery = r.URL.Path, r.URL.RawQuery - http.Redirect(w, r, redirect.String(), http.StatusTemporaryRedirect) + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + _ = r.Body.Close() + var message struct { + Method string `json:"method"` + } + if json.Unmarshal(body, &message) == nil && message.Method == "tools/call" { + redirects.Add(1) + http.Redirect(w, r, denied.String(), http.StatusTemporaryRedirect) + return + } + // Keep initialization and discovery on the configured endpoint so + // only the runtime's tool call exercises the additional destination. + r.Body = io.NopCloser(bytes.NewReader(body)) + proxy.ServeHTTP(w, r) })) _ = server.Listener.Close() listener, err := net.Listen("tcp", "0.0.0.0:0") @@ -69,13 +96,11 @@ func TestSessionEgressPolicyPinsAdditionalDestinations(t *testing.T) { server.Listener = listener server.Start() t.Cleanup(server.Close) - fixture := newInteractionFixture(t, harness, target, reachableModelURL(t, server.URL)) - _, _, task = fixture.send(t, "What is 2+2?") - if !reachedAllowed.Load() { - t.Fatal("the configured model destination was not reached") - } - require.Equal(t, a2atype.TaskStateFailed, task.Status.State, "redirect task text = %q", taskText(task)) - require.False(t, reachedDenied.Load(), "redirect reached an unconfigured destination") + name, _ := createMCPInteractionTemplate(t, harness, reachableServerURL(t, server.URL, denied.Path), false) + fixture := newInteractionFixtureForTemplate(t, harness, target, name) + fixture.send(t, prompt) + require.Positive(t, redirects.Load(), "the runtime must attempt the MCP tool call") + require.Equal(t, controlCalls, toolCalls(), "redirect reached an unconfigured destination") grant := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "redirect-access-", Namespace: "kagent"}, Spec: v1alpha3.EgressPolicySpec{ Destinations: []v1alpha3.EgressDestination{{Origin: denied.Scheme + "://" + denied.Host}}, @@ -83,7 +108,6 @@ func TestSessionEgressPolicyPinsAdditionalDestinations(t *testing.T) { require.NoError(t, kube.Create(t.Context(), grant)) t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), grant)) }) agent := &v1alpha3.Agent{} - name := strings.TrimPrefix(fixture.tenant, "kagent/") key := ctrlclient.ObjectKey{Namespace: "kagent", Name: name} require.NoError(t, kube.Get(t.Context(), key, agent)) agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: grant.Name}} @@ -100,12 +124,14 @@ func TestSessionEgressPolicyPinsAdditionalDestinations(t *testing.T) { return false, nil })) allowed := newInteractionFixtureForTemplate(t, harness, target, name) - _, _, task = allowed.send(t, "What is 2+2?") + _, _, task = allowed.send(t, prompt) require.Equal(t, a2atype.TaskStateCompleted, task.Status.State, "policy-granted redirect text = %q", taskText(task)) - require.True(t, reachedDenied.Load()) - reachedDenied.Store(false) - _, _, task = fixture.send(t, "What is 2+2?") - require.Equal(t, a2atype.TaskStateFailed, task.Status.State) - require.False(t, reachedDenied.Load(), "existing session must retain its earlier grant") + require.Contains(t, taskText(task), "result is 8") + allowedCalls := toolCalls() + require.Greater(t, allowedCalls, controlCalls, "the new session must reach the policy-granted destination") + before := redirects.Load() + fixture.send(t, prompt) + require.Greater(t, redirects.Load(), before, "the existing session must retry the tool call") + require.Equal(t, allowedCalls, toolCalls(), "existing session must retain its earlier grant") }) } diff --git a/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml b/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml index 427a74fbc1..ba4d64bdbb 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.22.0 name: egresspolicies.api.kagent.dev spec: group: api.kagent.dev diff --git a/helm/kagent-crds/templates/api.kagent.dev_remotemcpservers.yaml b/helm/kagent-crds/templates/api.kagent.dev_remotemcpservers.yaml index 0ade50f33d..5f78ca86b9 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_remotemcpservers.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_remotemcpservers.yaml @@ -128,10 +128,15 @@ spec: description: type: string headersFrom: + description: |- + HeadersFrom supplies request headers. Secret-backed headers are injected by + the egress gateway and cannot use routing, framing, hop-by-hop, + proxy-authentication, or Cookie headers. At most 64 headers may be supplied. items: description: ValueRef represents a configuration value properties: name: + maxLength: 256 type: string value: type: string @@ -161,9 +166,20 @@ spec: - name type: object x-kubernetes-validations: + - fieldPath: .name + message: Secret-backed headers require a valid HTTP header name + rule: '!has(self.valueFrom) || self.valueFrom.type != ''Secret'' + || self.name.matches(''^[!#$%&\\x27*+.^_`|~0-9A-Za-z-]+$'')' + - fieldPath: .name + message: this header cannot carry an egress credential + rule: '!has(self.valueFrom) || self.valueFrom.type != ''Secret'' + || !(self.name.lowerAscii() in [''host'', ''connection'', ''content-length'', + ''transfer-encoding'', ''te'', ''trailer'', ''upgrade'', ''keep-alive'', + ''proxy-authorization'', ''proxy-authenticate'', ''cookie''])' - message: exactly one of the fields in [value valueFrom] must be set rule: (has(self.value)?1:0)+(has(self.valueFrom)?1:0) == 1 + maxItems: 64 type: array protocol: default: STREAMABLE_HTTP From 94891d02f835a9138293df7888e351e694c55531 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Tue, 13 Oct 2026 21:34:12 +0000 Subject: [PATCH 04/43] style: align egress imports with main Signed-off-by: Eitan Yarmush --- go/core/internal/database/sandbox_revisions.go | 1 + go/core/internal/database/sandbox_revisions_test.go | 3 ++- go/core/internal/substrate/policy_test.go | 3 ++- go/core/internal/translator/egress.go | 3 ++- go/core/internal/translator/egress_test.go | 7 ++++--- go/core/test/e2e/egress_test.go | 3 ++- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index 938ca87fca..c25ee5695f 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/jackc/pgx/v5" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) diff --git a/go/core/internal/database/sandbox_revisions_test.go b/go/core/internal/database/sandbox_revisions_test.go index 3f53259d07..e774467e46 100644 --- a/go/core/internal/database/sandbox_revisions_test.go +++ b/go/core/internal/database/sandbox_revisions_test.go @@ -3,8 +3,9 @@ package database import ( "testing" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/stretchr/testify/require" + + "github.com/kagent-dev/kagent/go/core/internal/egress" ) func TestSandboxRevisionPinsEgress(t *testing.T) { diff --git a/go/core/internal/substrate/policy_test.go b/go/core/internal/substrate/policy_test.go index 1102ee8378..525efb3b89 100644 --- a/go/core/internal/substrate/policy_test.go +++ b/go/core/internal/substrate/policy_test.go @@ -3,8 +3,9 @@ package substrate import ( "testing" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/stretchr/testify/require" + + "github.com/kagent-dev/kagent/go/core/internal/egress" ) func TestActorPolicyKeepsCredentialsOnExactOrigin(t *testing.T) { diff --git a/go/core/internal/translator/egress.go b/go/core/internal/translator/egress.go index b72f1db40e..848f9abac6 100644 --- a/go/core/internal/translator/egress.go +++ b/go/core/internal/translator/egress.go @@ -4,10 +4,11 @@ import ( "fmt" "strings" - "github.com/kagent-dev/kagent/go/core/internal/egress" "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/validation" + + "github.com/kagent-dev/kagent/go/core/internal/egress" ) // CompileEgressPolicies resolves same-namespace grants for either an Agent or a diff --git a/go/core/internal/translator/egress_test.go b/go/core/internal/translator/egress_test.go index 848c7e7f6b..021ba8b7a5 100644 --- a/go/core/internal/translator/egress_test.go +++ b/go/core/internal/translator/egress_test.go @@ -4,13 +4,14 @@ import ( "encoding/json" "testing" - "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" - "github.com/kagent-dev/kagent/go/core/internal/translator" "github.com/stretchr/testify/require" "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/translator" ) func egressPolicy() (*v1alpha3.EgressPolicy, *corev1.Secret) { diff --git a/go/core/test/e2e/egress_test.go b/go/core/test/e2e/egress_test.go index fd62f45d91..73bb00952f 100644 --- a/go/core/test/e2e/egress_test.go +++ b/go/core/test/e2e/egress_test.go @@ -15,12 +15,13 @@ import ( "time" a2atype "github.com/a2aproject/a2a-go/v2/a2a" - "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" ) func TestSessionEgressPolicyPinsAdditionalDestinations(t *testing.T) { From 5f2417641a53e306b790de534a7b6d3f6381741c Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 14 Oct 2026 21:34:59 +0000 Subject: [PATCH 05/43] test: adapt egress fixture to optional harness image Signed-off-by: Eitan Yarmush --- go/core/internal/translator/egress_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/core/internal/translator/egress_test.go b/go/core/internal/translator/egress_test.go index 021ba8b7a5..5139d9d38d 100644 --- a/go/core/internal/translator/egress_test.go +++ b/go/core/internal/translator/egress_test.go @@ -55,7 +55,7 @@ func TestAgentEgressPoliciesPinGrantsAcrossHarnesses(t *testing.T) { t.Run(kind, func(t *testing.T) { policy, secret := egressPolicy() harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "runner"}, Spec: v1alpha3.HarnessSpec{ - Workload: v1alpha3.HarnessWorkload{Image: "runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + Workload: v1alpha3.HarnessWorkload{Image: new("runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, }} switch kind { From f98a4677e20c5c657eb6cb52ac9525623f3b27af Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 15 Oct 2026 16:21:36 +0000 Subject: [PATCH 06/43] fix: require current prepared revisions for new instances Signed-off-by: Eitan Yarmush --- .../configuration-and-compilation.md | 10 +- docs/architecture/egress-policies.md | 16 ++- docs/architecture/runtime-and-lifecycle.md | 7 +- docs/architecture/sandboxes.md | 7 +- .../internal/controller/preparation_test.go | 5 +- .../internal/controller/reconciler_test.go | 18 ++- .../database/runtime_creation_test.go | 136 ++++++++++++++++++ .../database/runtime_revisions_test.go | 35 ++++- go/core/internal/database/sandboxes.go | 11 +- go/core/internal/database/sessions.go | 29 ++-- go/core/test/e2e/egress_test.go | 62 +++++++- 11 files changed, 293 insertions(+), 43 deletions(-) create mode 100644 go/core/internal/database/runtime_creation_test.go diff --git a/docs/architecture/configuration-and-compilation.md b/docs/architecture/configuration-and-compilation.md index 5340355368..ea08a5a7ff 100644 --- a/docs/architecture/configuration-and-compilation.md +++ b/docs/architecture/configuration-and-compilation.md @@ -172,9 +172,10 @@ revision. AgentTemplate is shared context with no runtime status. Child template references are resolved under the root Agent's Harness; they do not require separate Agents or matching Harness references. -A failed compile or apply leaves the previous successful revision available. -Sessions pin a prepared revision, so later template edits do not mutate a -running session. +A failed compile or apply retains the previous successful revision. Fresh +Sessions require the desired and latest successful revisions to match, so +pending or failed preparation blocks creation after reconciliation records the +change. Existing Sessions and creation retries keep their pinned revision. Harness compilers only translate inputs. The controller and Substrate adapter own application and readiness. The central entry points are @@ -391,7 +392,8 @@ no synthetic Kubernetes object is created for an inline template. Create a session with `kagent agent session create --agent assistant -n kagent`. The gRPC create request and ScheduledRun target one `agent` resource reference. -The controller selects that Agent's latest successful revision. Deleting an Agent +The store selects that Agent's current desired revision only after it has been +successfully prepared. Deleting an Agent retires its definition; sessions and checkpoints retain their pinned revisions. Recreating the same name creates a new identity and cannot inherit the old Agent's last successful revision. diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index e699519349..a814681697 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -159,11 +159,17 @@ UIDs do not; the gateway handles value rotation independently. Policy metadata changes do not affect the resolved grant. Reference ordering does not affect revision identity. -New Sessions and sandboxes pin a prepared revision. Existing instances keep -their original grant through retries and resume; Session checkpoint forks also -retain the source revision. Updating or deleting a policy is not live revocation. -Failed preparation does not overwrite a previous successful revision, and the -existing readiness rules prevent selecting stale inputs for new instances. +New Sessions and sandboxes pin the current desired revision only when it matches +the latest successful revision. The store checks this atomically with creation. +Unresolved policies and pending or failed preparation block fresh creation, +including during ordinary configuration updates. The previous successful +revision remains retained but cannot be selected for fresh instances while the +current desired revision is unprepared. + +This gate takes effect after the controller reconciles the changed inputs into +the database; updating or deleting a policy is not instantaneous revocation. +Existing instances keep their original grant through creation retries and +resume; Session checkpoint forks also retain the source revision. The runtime workflow creates a suspended Actor, installs its compiled `default` egress policy, and resumes it only after installation succeeds. A retry uses the diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 53fb8a1090..4be954eca3 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -6,7 +6,12 @@ Kubernetes resource. ## Creation and state -Creation selects the latest successful revision for the Agent, creates a deterministic Actor initially suspended, and marks the session +Fresh creation requires the Agent's current desired revision to match its latest +successful revision. Once reconciliation records unresolved inputs or a pending +update, creation fails until the current revision is prepared. Creation retries +retain the existing session's pinned revision. + +The control plane creates a deterministic Actor initially suspended and marks the session ready after Substrate accepts it. Readiness of the image was already established while preparing the ate-api ActorTemplate; Session creation does not resume an Actor merely to probe `/readyz`. diff --git a/docs/architecture/sandboxes.md b/docs/architecture/sandboxes.md index f929128394..72d8f4e859 100644 --- a/docs/architecture/sandboxes.md +++ b/docs/architecture/sandboxes.md @@ -97,8 +97,11 @@ finalizer, including when deletion starts while the controller is offline. The controller advances the latest-successful revision only after the golden snapshot is ready and reports preparation through the template's `Ready` condition and observed generation. Failed preparation preserves the previous -successful revision. Existing Sandboxes retain their revision when a template -changes. Template deletion retires preparation; retained instances continue to +successful revision. Fresh creation requires the desired revision to match that +successful revision; unresolved inputs and pending or failed preparation block +creation after reconciliation records the change. Existing Sandboxes and +creation retries retain their revision when a template changes. +Template deletion retires preparation; retained instances continue to pin the runtime artifacts they need. Sandbox execution is enabled by default. Lifecycle calls run on the serving API diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index 46a07ba93c..c2a14615b0 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -106,12 +106,11 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { require.NoError(t, err) require.Nil(t, retained, "last-successful A must remain protected without any sessions") - session, _, err := store.CreateSession(ctx, &apiv1.Session{ + _, _, err = store.CreateSession(ctx, &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}, }, "last-good-session") - require.NoError(t, err) - require.Equal(t, initial.Target.RevisionID.String(), session.GetPreparedRevision()) + require.ErrorIs(t, err, database.ErrNotFound, "unresolved inputs must prevent fresh sessions from selecting the retained revision") collector, _ := newTestRuntimeRevisionGC(t, store, templates) require.NoError(t, collector.collect(ctx, abandoned.Revision)) diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index 8548e6adc1..8163c3f0d8 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -183,11 +183,6 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { states.UpdateObject(state) require.NoError(t, reconciler.reconcileAgent(ctx, state.ResourceName())) - // Compile failures preserve the current UID's last successful runtime. - state.Target = nil - states.UpdateObject(state) - require.NoError(t, reconciler.reconcileAgent(ctx, state.ResourceName())) - require.Empty(t, reconciler.collections.AgentRuntimeObservations.List(), "invalid preparation must release its observation before GC") request := &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}, @@ -195,6 +190,19 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { session, _, err := store.CreateSession(ctx, request, "session") require.NoError(t, err) require.Equal(t, id.String(), session.GetPreparedRevision()) + + // Compile failures preserve pinned runtimes but block fresh sessions. + state.Target = nil + states.UpdateObject(state) + require.NoError(t, reconciler.reconcileAgent(ctx, state.ResourceName())) + require.Empty(t, reconciler.collections.AgentRuntimeObservations.List(), "invalid preparation must release its observation before GC") + retried, created, err := store.CreateSession(ctx, request, "session") + require.NoError(t, err) + require.False(t, created) + require.Equal(t, session.GetId(), retried.GetId()) + require.Equal(t, id.String(), retried.GetPreparedRevision()) + _, _, err = store.CreateSession(ctx, request, "unprepared-session") + require.ErrorIs(t, err, database.ErrNotFound) operation, err := store.BeginSessionOperation(ctx, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE) require.NoError(t, err) executor := uuid.New() diff --git a/go/core/internal/database/runtime_creation_test.go b/go/core/internal/database/runtime_creation_test.go new file mode 100644 index 0000000000..e7d6ee82b3 --- /dev/null +++ b/go/core/internal/database/runtime_creation_test.go @@ -0,0 +1,136 @@ +package database + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func TestRuntimeCreationRequiresCurrentPreparedRevision(t *testing.T) { + for _, kind := range []string{"session", "sandbox"} { + for _, invalidateFirst := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/invalidate_first=%t", kind, invalidateFirst), func(t *testing.T) { + pool := setupTestDB(t) + client := NewClient(pool, "public") + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + var create func(*Client, string) (string, bool, error) + var desire func(*Client, string) error + var prepare func(string) + var createQuery, updateQuery, unresolved string + if kind == "session" { + prepare = func(revision string) { + sessionFixture(t, client, ctx, "team-a", revision, "assistant", "kagent") + } + create = func(c *Client, requestID string) (string, bool, error) { + session, created, err := c.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), requestID) + return session.GetPreparedRevision(), created, err + } + desire = func(c *Client, revision string) error { + return c.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", DesiredRevision: revision}) + } + createQuery, updateQuery = "FROM agent_definition p", "INSERT INTO agent_definition" + } else { + input, options := sandboxFixture(t, client, "revision-1") + prepare = func(revision string) { + _, _ = sandboxFixture(t, client, revision) + } + create = func(c *Client, requestID string) (string, bool, error) { + request := proto.CloneOf(input) + request.Id = uuid.NewString() + instance, created, err := c.CreateSandbox(ctx, request, requestID, options) + return instance.GetPreparedRevision(), created, err + } + desire = func(c *Client, revision string) error { + return c.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) + } + createQuery, updateQuery = "FROM sandbox_template_definition", "INSERT INTO sandbox_template_definition" + unresolved = "pending:uid:2" + } + prepare("revision-1") + revision, created, err := create(client, "original") + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "revision-1", revision) + + // Pause the first transaction with the definition row locked. The + // other operation must wait and observe the committed desired state. + query, waitingQuery := createQuery, updateQuery + if invalidateFirst { + query, waitingQuery = updateQuery, createQuery + } + barrier := &runtimeReferenceBarrier{query: query, afterQuery: true, reached: make(chan struct{}), resume: make(chan struct{})} + var resume sync.Once + defer resume.Do(func() { close(barrier.resume) }) + config := pool.Config() + config.ConnConfig.Tracer = barrier + firstPool, err := pgxpool.NewWithConfig(ctx, config) + require.NoError(t, err) + t.Cleanup(firstPool.Close) + first := NewClient(firstPool, "public") + creation, invalidation := make(chan error, 1), make(chan error, 1) + createConcurrent := func(c *Client) { + _, _, err := create(c, "concurrent") + creation <- err + } + invalidate := func(c *Client) { invalidation <- desire(c, unresolved) } + if invalidateFirst { + go invalidate(first) + } else { + go createConcurrent(first) + } + select { + case <-barrier.reached: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + if invalidateFirst { + go createConcurrent(client) + } else { + go invalidate(client) + } + require.Eventually(t, func() bool { + var waiting bool + err := pool.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE datname = current_database() AND query LIKE $1 AND cardinality(pg_blocking_pids(pid)) > 0)", "%"+waitingQuery+"%").Scan(&waiting) + return err == nil && waiting + }, 5*time.Second, 10*time.Millisecond) + resume.Do(func() { close(barrier.resume) }) + require.NoError(t, <-invalidation) + if invalidateFirst { + require.ErrorIs(t, <-creation, ErrNotFound) + } else { + require.NoError(t, <-creation) + } + + // Both unresolved inputs and an ordinary pending update reject fresh + // requests while retries keep their original revision. + for _, desired := range []string{unresolved, "revision-2"} { + require.NoError(t, desire(client, desired)) + _, created, err = create(client, "fresh") + require.ErrorIs(t, err, ErrNotFound) + require.False(t, created) + revision, created, err = create(client, "original") + require.NoError(t, err) + require.False(t, created) + require.Equal(t, "revision-1", revision) + } + prepare("revision-2") + revision, created, err = create(client, "fresh") + require.NoError(t, err) + require.True(t, created, "rejected requests must not reserve an idempotency key") + require.Equal(t, "revision-2", revision) + revision, created, err = create(client, "original") + require.NoError(t, err) + require.False(t, created) + require.Equal(t, "revision-1", revision) + }) + } + } +} diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index 05245cda28..5c2b627fd6 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -161,14 +161,21 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { _, _, err = client.CreateSession(ctx, request, "session") require.ErrorIs(t, err, ErrNotFound, "a replacement must not select the previous template UID") require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, true)) + session, _, err := client.CreateSession(ctx, request, "session") + require.NoError(t, err) + require.Equal(t, "new", session.GetPreparedRevision()) // Reconciliation of the same identity must preserve its last success while // a new desired revision is still preparing. pair.DesiredRevision = "pending" require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) - session, _, err := client.CreateSession(ctx, request, "session") + retried, created, err := client.CreateSession(ctx, request, "session") require.NoError(t, err) - require.Equal(t, "new", session.GetPreparedRevision()) + require.False(t, created) + require.Equal(t, session.GetId(), retried.GetId()) + require.Equal(t, "new", retried.GetPreparedRevision()) + _, _, err = client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "pending") + require.ErrorIs(t, err, ErrNotFound) revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) require.Len(t, revisions, 1) @@ -215,7 +222,7 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") - query := "FROM runtime_revision WHERE revision = $1 FOR UPDATE" + query := "FROM agent_definition p" if source != "session" { query = "FOR UPDATE OF r" require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) @@ -252,6 +259,17 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) t.Fatal(ctx.Err()) } if source == "session" { + if referenceFirst { + // Creation holds the definition lock until its reference commits, + // so retirement must follow creation before GC can claim it. + resume.Do(func() { close(barrier.resume) }) + require.NoError(t, <-created) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "revision") + require.NoError(t, err) + require.Nil(t, claimed) + return + } require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) } type claimResult struct { @@ -281,7 +299,11 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) require.NoError(t, result.err) require.NotNil(t, result.revision) resume.Do(func() { close(barrier.resume) }) - require.ErrorIs(t, <-created, ErrObjectDeleting) + if source == "session" { + require.ErrorIs(t, <-created, ErrNotFound) + } else { + require.ErrorIs(t, <-created, ErrObjectDeleting) + } } }) } @@ -295,7 +317,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") pair := AgentDefinition{ Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", - DesiredRevision: "pending", + DesiredRevision: "revision", } require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) // A skipped finalization must leave last-good intact for reactivation. @@ -487,6 +509,9 @@ func TestRecordRuntimeRevisionPromotesOnlyCurrentActivePair(t *testing.T) { assertAvailableRevision("") // Reviving the pair must retain the last success, not a report made while retired. require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) + assertAvailableRevision("") + pair.DesiredRevision = "second" + require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) assertAvailableRevision("second") } diff --git a/go/core/internal/database/sandboxes.go b/go/core/internal/database/sandboxes.go index d758492026..4cb97fec1d 100644 --- a/go/core/internal/database/sandboxes.go +++ b/go/core/internal/database/sandboxes.go @@ -63,8 +63,9 @@ type SandboxCreateOptions struct { TTL time.Duration } -// CreateSandbox atomically reserves a sandbox and its prepared revision. A -// duplicate creator/requestID returns the original resource when inputs match. +// CreateSandbox atomically reserves a sandbox and its current desired revision, +// or returns ErrNotFound if it is not prepared. A duplicate creator/requestID +// returns the original resource when inputs match. // The boolean reports whether this call created the sandbox. func (c *Client) CreateSandbox(ctx context.Context, request *apiv1.Sandbox, requestID string, options SandboxCreateOptions) (*apiv1.Sandbox, bool, error) { existing, err := readSandboxRequest(ctx, c.db, request.GetCreator(), requestID) @@ -113,7 +114,8 @@ func readSandboxRequest(ctx context.Context, db dbExecutor, userID, requestID st } // insertSandbox reserves a new sandbox in CREATING/CREATE and pins the active -// template's latest successful revision. A duplicate creator/requestID returns +// template's current desired revision only when it is prepared. A missing prepared +// target returns ErrNotFound. A duplicate creator/requestID returns // pgx.ErrNoRows so the caller can roll back and read the winning transaction. func insertSandbox(ctx context.Context, tx pgx.Tx, request *apiv1.Sandbox, requestID string, options SandboxCreateOptions) (sandboxRow, error) { if options.TTL < time.Second || options.TTL > 24*time.Hour { @@ -124,7 +126,8 @@ func insertSandbox(ctx context.Context, tx pgx.Tx, request *apiv1.Sandbox, reque err := tx.QueryRow(ctx, ` SELECT latest_successful_revision, clock_timestamp() FROM sandbox_template_definition WHERE namespace = $1 AND sandbox_template_name = $2 AND sandbox_template_uid = $3 - AND retired_at IS NULL AND latest_successful_revision IS NOT NULL FOR UPDATE + AND retired_at IS NULL AND desired_revision = latest_successful_revision + FOR UPDATE `, request.GetSandboxTemplate().GetNamespace(), request.GetSandboxTemplate().GetName(), options.TemplateUID).Scan(&revision, &createdAt) if err != nil { return sandboxRow{}, notFoundOr(err) diff --git a/go/core/internal/database/sessions.go b/go/core/internal/database/sessions.go index dd0e4c496d..eebb7287ce 100644 --- a/go/core/internal/database/sessions.go +++ b/go/core/internal/database/sessions.go @@ -53,11 +53,11 @@ func sameSessionRequest(session, request *apiv1.Session) bool { } // CreateSession atomically reserves a session, its conversation history, and the -// latest successful runtime revision. A repeated creator/requestID returns the existing -// session when the Agent matches, or ErrIdempotencyConflict otherwise. A -// deleted session returns ErrFailedPrecondition and keeps its request ID reserved. The -// boolean reports whether this call created the session; runtime provisioning belongs to -// the caller. +// current desired revision, or returns ErrNotFound if it is not prepared. A repeated +// creator/requestID returns the existing session when the Agent matches, or +// ErrIdempotencyConflict otherwise. A deleted session returns ErrFailedPrecondition +// and keeps its request ID reserved. The boolean reports whether this call created +// the session; runtime provisioning belongs to the caller. func (c *Client) CreateSession(ctx context.Context, request *apiv1.Session, requestID string) (*apiv1.Session, bool, error) { existing, err := readSessionRequest(ctx, c.db, request.GetCreator(), requestID) if err == nil { @@ -101,22 +101,25 @@ func (c *Client) CreateSession(ctx context.Context, request *apiv1.Session, requ } // insertSession reserves a new conversation in CREATING/CREATE and pins the active -// Agent's latest successful revision. Callers must supply a transaction so -// the history and session commit together. A missing prepared target returns ErrNotFound; -// a duplicate creator/requestID returns pgx.ErrNoRows for the caller to resolve. +// Agent's current desired revision only when it is prepared. It locks the definition +// before the revision to serialize creation with reconciliation. Callers must supply +// a transaction so the history and session commit together. A missing prepared target +// returns ErrNotFound; a duplicate creator/requestID returns pgx.ErrNoRows for the caller +// to resolve. func insertSession(ctx context.Context, db pgx.Tx, request *apiv1.Session, requestID string) (sessionRow, error) { type preparedRevision struct { Revision string DBTime time.Time } revision, err := queryOne(ctx, db, ` - SELECT r.revision, clock_timestamp() AS db_time + SELECT p.latest_successful_revision AS revision, clock_timestamp() AS db_time FROM agent_definition p - JOIN agent_runtime_revision r ON r.revision = p.latest_successful_revision - WHERE p.namespace = $1 AND p.agent_name = $2 AND p.retired_at IS NULL - `, pgx.RowToStructByName[preparedRevision], request.GetAgent().GetNamespace(), request.GetAgent().GetName()) + WHERE p.namespace = $1 AND p.agent_name = $2 AND p.retired_at IS NULL + AND p.desired_revision = p.latest_successful_revision + FOR UPDATE OF p + `, pgx.RowToStructByName[preparedRevision], request.GetAgent().GetNamespace(), request.GetAgent().GetName()) if err != nil { - return sessionRow{}, fmt.Errorf("get latest successful runtime revision: %w", notFoundOr(err)) + return sessionRow{}, fmt.Errorf("get current prepared runtime revision: %w", notFoundOr(err)) } if _, err := getAvailableRuntimeRevisionForUpdate(ctx, db, revision.Revision); err != nil { return sessionRow{}, err diff --git a/go/core/test/e2e/egress_test.go b/go/core/test/e2e/egress_test.go index 73bb00952f..b0383d826f 100644 --- a/go/core/test/e2e/egress_test.go +++ b/go/core/test/e2e/egress_test.go @@ -15,12 +15,17 @@ import ( "time" a2atype "github.com/a2aproject/a2a-go/v2/a2a" + "github.com/google/uuid" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" ) @@ -107,7 +112,7 @@ func TestSessionEgressPolicyPinsAdditionalDestinations(t *testing.T) { Destinations: []v1alpha3.EgressDestination{{Origin: denied.Scheme + "://" + denied.Host}}, }} require.NoError(t, kube.Create(t.Context(), grant)) - t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), grant)) }) + t.Cleanup(func() { require.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), grant))) }) agent := &v1alpha3.Agent{} key := ctrlclient.ObjectKey{Namespace: "kagent", Name: name} require.NoError(t, kube.Get(t.Context(), key, agent)) @@ -134,5 +139,60 @@ func TestSessionEgressPolicyPinsAdditionalDestinations(t *testing.T) { fixture.send(t, prompt) require.Greater(t, redirects.Load(), before, "the existing session must retry the tool call") require.Equal(t, allowedCalls, toolCalls(), "existing session must retain its earlier grant") + + require.NoError(t, kube.Delete(t.Context(), grant)) + require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, agent); err != nil { + return false, err + } + condition := apimeta.FindStatusCondition(agent.Status.Conditions, v1alpha3.AgentConditionReady) + return condition != nil && condition.ObservedGeneration == agent.Generation && condition.Status == metav1.ConditionFalse, nil + })) + // Check creation directly, without the fixture's preparation retry loop. + _, err = allowed.sessions.CreateSession(allowed.ctx, &apiv1.CreateSessionRequest{ + Agent: &apiv1.ResourceReference{Namespace: key.Namespace, Name: key.Name}, RequestId: uuid.NewString(), + }) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + _, _, task = allowed.send(t, prompt) + require.Equal(t, a2atype.TaskStateCompleted, task.Status.State) + require.Greater(t, toolCalls(), allowedCalls, "policy deletion must preserve the existing session's pinned grant") }) } + +func TestSandboxEgressPolicyDeletionBlocksCreation(t *testing.T) { + t.Parallel() + f := newSandboxFixture(t) + kube := interactionKubeClient(t) + grant := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "sandbox-access-", Namespace: f.template.Namespace}, Spec: v1alpha3.EgressPolicySpec{ + Destinations: []v1alpha3.EgressDestination{{Origin: "https://example.com"}}, + }} + require.NoError(t, kube.Create(f.ctx, grant)) + t.Cleanup(func() { require.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), grant))) }) + template := &v1alpha3.SandboxTemplate{} + key := ctrlclient.ObjectKey{Namespace: f.template.Namespace, Name: f.template.Name} + require.NoError(t, kube.Get(f.ctx, key, template)) + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: grant.Name}} + require.NoError(t, kube.Update(f.ctx, template)) + // Observe the grant's generation before creating the instance to avoid + // selecting the template's earlier revision during reconciliation. + waitReady := func(want metav1.ConditionStatus) { + t.Helper() + require.NoError(t, wait.PollUntilContextTimeout(f.ctx, time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, template); err != nil { + return false, err + } + condition := apimeta.FindStatusCondition(template.Status.Conditions, "Ready") + return condition != nil && condition.ObservedGeneration == template.Generation && condition.Status == want, nil + })) + } + waitReady(metav1.ConditionTrue) + first := f.create(t, 5*time.Minute) + require.NoError(t, kube.Delete(f.ctx, grant)) + waitReady(metav1.ConditionFalse) + _, err := f.client.CreateSandbox(f.ctx, &apiv1.CreateSandboxRequest{SandboxTemplate: f.template, RequestId: uuid.NewString()}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + current, err := f.client.GetSandbox(f.ctx, &apiv1.GetSandboxRequest{SandboxId: first.Id}) + require.NoError(t, err) + require.Equal(t, first.PreparedRevision, current.Sandbox.PreparedRevision) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, current.Sandbox.State) +} From 4a53b01ea8ea0fb182ef585c6a8eff5c557a395e Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 15 Oct 2026 18:38:22 +0000 Subject: [PATCH 07/43] test: align runtime lifecycle with revision readiness Signed-off-by: Eitan Yarmush --- go/core/test/e2e/runtime_revision_test.go | 35 +++++++++-------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/go/core/test/e2e/runtime_revision_test.go b/go/core/test/e2e/runtime_revision_test.go index 888ad5ca2b..9965507caf 100644 --- a/go/core/test/e2e/runtime_revision_test.go +++ b/go/core/test/e2e/runtime_revision_test.go @@ -68,11 +68,12 @@ func TestRuntimeRevisionLifecycle(t *testing.T) { } // No FailedPrecondition retry: Ready must already be backed by a persisted // successful revision when the first create request arrives. - created, err := sessions.CreateSession(ctx, request(templateName)) + createRequest := request(templateName) + created, err := sessions.CreateSession(ctx, createRequest) require.NoError(t, err) source := created.GetSession() t.Cleanup(func() { deleteSession(source.GetId()) }) - sourceTaskID := send(source.GetId()) + send(source.GetId()) // Observe the actual runtime through the public inventory API before deleting // references, so an empty response cannot falsely prove cleanup later. @@ -110,12 +111,13 @@ func TestRuntimeRevisionLifecycle(t *testing.T) { } return false, nil })) - fallback, err := sessions.CreateSession(ctx, request(templateName)) - require.NoError(t, err, "an invalid edit must preserve the current UID's last-good runtime") - t.Cleanup(func() { deleteSession(fallback.GetSession().GetId()) }) - require.Equal(t, source.GetPreparedRevision(), fallback.GetSession().GetPreparedRevision()) - send(fallback.GetSession().GetId()) - deleteSession(fallback.GetSession().GetId()) + _, err = sessions.CreateSession(ctx, request(templateName)) + require.Equal(t, codes.FailedPrecondition, status.Code(err), "an invalid edit must block fresh sessions") + retried, err := sessions.CreateSession(ctx, createRequest) + require.NoError(t, err, "creation retries must retain the existing session's pinned runtime") + require.Equal(t, source.GetId(), retried.GetSession().GetId()) + require.Equal(t, source.GetPreparedRevision(), retried.GetSession().GetPreparedRevision()) + sourceTaskID := send(source.GetId()) checkpointResponse := createCheckpoint(t, ctx, checkpoints, &apiv1.CreateCheckpointRequest{SessionId: source.GetId(), RequestId: uuid.NewString(), ExpectedHeadTaskId: sourceTaskID}) checkpointID := checkpointResponse.GetCheckpoint().GetId() @@ -132,19 +134,10 @@ func TestRuntimeRevisionLifecycle(t *testing.T) { agent := &v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: template.Name}} require.NoError(t, kube.Get(ctx, ctrlclient.ObjectKeyFromObject(agent), agent)) require.NoError(t, kube.Delete(ctx, agent)) - // Wait for the controller to retire the pair, using the same public create - // path as a client. Dispose of any session created before retirement wins. - require.NoError(t, wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - response, err := sessions.CreateSession(ctx, request(templateName)) - if status.Code(err) == codes.FailedPrecondition { - return true, nil - } - if err != nil { - return false, err - } - deleteSession(response.GetSession().GetId()) - return false, nil - })) + // Deletion keeps fresh creation blocked. Eventual runtime collection below + // verifies retirement after the final checkpoint reference is released. + _, err = sessions.CreateSession(ctx, request(templateName)) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) forked, err := checkpoints.ForkSession(ctx, &apiv1.ForkSessionRequest{CheckpointId: checkpointID, RequestId: uuid.NewString()}) require.NoError(t, err, "a checkpoint must retain runnable inputs after its source and template are deleted") forkID := forked.GetSession().GetId() From 5b39d41dc89b3b15242f4a8ffe1e5aafb2790141 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 15 Oct 2026 20:43:30 +0000 Subject: [PATCH 08/43] test: exercise Codex egress through shell requests Signed-off-by: Eitan Yarmush --- go/core/test/e2e/egress_test.go | 160 ++++++++++++++++++++++---------- 1 file changed, 109 insertions(+), 51 deletions(-) diff --git a/go/core/test/e2e/egress_test.go b/go/core/test/e2e/egress_test.go index b0383d826f..a2ecfd3543 100644 --- a/go/core/test/e2e/egress_test.go +++ b/go/core/test/e2e/egress_test.go @@ -16,6 +16,7 @@ import ( a2atype "github.com/a2aproject/a2a-go/v2/a2a" "github.com/google/uuid" + "github.com/kagent-dev/mockllm" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -33,6 +34,10 @@ func TestSessionEgressPolicyPinsAdditionalDestinations(t *testing.T) { t.Parallel() forEachHarness(t, func(t *testing.T, harness testHarness) { t.Parallel() + if harness.name == codexE2EHarness { + testCodexSessionEgressPolicy(t, harness) + return + } target := interactionTarget(t) mcpURL, mcpServer := startMCPMock(t) denied, err := url.Parse(mcpURL) @@ -103,62 +108,115 @@ func TestSessionEgressPolicyPinsAdditionalDestinations(t *testing.T) { server.Start() t.Cleanup(server.Close) name, _ := createMCPInteractionTemplate(t, harness, reachableServerURL(t, server.URL, denied.Path), false) - fixture := newInteractionFixtureForTemplate(t, harness, target, name) - fixture.send(t, prompt) - require.Positive(t, redirects.Load(), "the runtime must attempt the MCP tool call") - require.Equal(t, controlCalls, toolCalls(), "redirect reached an unconfigured destination") - - grant := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "redirect-access-", Namespace: "kagent"}, Spec: v1alpha3.EgressPolicySpec{ - Destinations: []v1alpha3.EgressDestination{{Origin: denied.Scheme + "://" + denied.Host}}, - }} - require.NoError(t, kube.Create(t.Context(), grant)) - t.Cleanup(func() { require.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), grant))) }) - agent := &v1alpha3.Agent{} - key := ctrlclient.ObjectKey{Namespace: "kagent", Name: name} - require.NoError(t, kube.Get(t.Context(), key, agent)) - agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: grant.Name}} - require.NoError(t, kube.Update(t.Context(), agent)) - require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { - if err := kube.Get(ctx, key, agent); err != nil { - return false, err - } - for _, condition := range agent.Status.Conditions { - if condition.Type == v1alpha3.AgentConditionReady && condition.ObservedGeneration == agent.Generation { - return condition.Status == metav1.ConditionTrue, nil - } - } - return false, nil - })) - allowed := newInteractionFixtureForTemplate(t, harness, target, name) - _, _, task = allowed.send(t, prompt) - require.Equal(t, a2atype.TaskStateCompleted, task.Status.State, "policy-granted redirect text = %q", taskText(task)) - require.Contains(t, taskText(task), "result is 8") - allowedCalls := toolCalls() - require.Greater(t, allowedCalls, controlCalls, "the new session must reach the policy-granted destination") - before := redirects.Load() - fixture.send(t, prompt) - require.Greater(t, redirects.Load(), before, "the existing session must retry the tool call") - require.Equal(t, allowedCalls, toolCalls(), "existing session must retain its earlier grant") - - require.NoError(t, kube.Delete(t.Context(), grant)) - require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - if err := kube.Get(ctx, key, agent); err != nil { - return false, err + testSessionEgressPolicyPinsDestination(t, harness, name, denied.Scheme+"://"+denied.Host, func(fixture *interactionFixture, allowed bool) { + beforeRedirects, beforeCalls := redirects.Load(), toolCalls() + _, _, task := fixture.send(t, prompt) + require.Greater(t, redirects.Load(), beforeRedirects, "the runtime must attempt the MCP tool call") + if allowed { + require.Equal(t, a2atype.TaskStateCompleted, task.Status.State, "policy-granted redirect text = %q", taskText(task)) + require.Contains(t, taskText(task), "result is 8") + require.Greater(t, toolCalls(), beforeCalls, "the session must reach the policy-granted destination") + } else { + require.Equal(t, beforeCalls, toolCalls(), "redirect reached an unconfigured destination") } - condition := apimeta.FindStatusCondition(agent.Status.Conditions, v1alpha3.AgentConditionReady) - return condition != nil && condition.ObservedGeneration == agent.Generation && condition.Status == metav1.ConditionFalse, nil - })) - // Check creation directly, without the fixture's preparation retry loop. - _, err = allowed.sessions.CreateSession(allowed.ctx, &apiv1.CreateSessionRequest{ - Agent: &apiv1.ResourceReference{Namespace: key.Namespace, Name: key.Name}, RequestId: uuid.NewString(), }) - require.Equal(t, codes.FailedPrecondition, status.Code(err)) - _, _, task = allowed.send(t, prompt) - require.Equal(t, a2atype.TaskStateCompleted, task.Status.State) - require.Greater(t, toolCalls(), allowedCalls, "policy deletion must preserve the existing session's pinned grant") }) } +func testCodexSessionEgressPolicy(t *testing.T, harness testHarness) { + t.Helper() + interactionTarget(t) + // Codex's MCP client rejects cross-origin redirects before the gateway can + // authorize them. Exercise additional destinations through its shell tool. + var requests atomic.Int64 + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + _, _ = io.WriteString(w, "egress destination reached") + })) + _ = server.Listener.Close() + listener, err := net.Listen("tcp", "0.0.0.0:0") + require.NoError(t, err) + server.Listener = listener + server.Start() + t.Cleanup(server.Close) + destination, err := url.Parse(reachableServerURL(t, server.URL, "/")) + require.NoError(t, err) + // Keep the destination distinct from the model origin on Docker Desktop too. + kube := interactionKubeClient(t) + alias := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "shell-origin-", Namespace: "kagent"}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeExternalName, ExternalName: destination.Hostname()}, + } + require.NoError(t, kube.Create(t.Context(), alias)) + t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), alias)) }) + destination.Host = net.JoinHostPort(alias.Name+".kagent.svc.cluster.local", destination.Port()) + + raw, err := codexInteractionMocks.ReadFile("mocks/invoke_codex_builtin_tools.json") + require.NoError(t, err) + // Always emit the fixture marker so the mock model finishes after denied + // requests too. The destination's request count proves whether egress worked. + command := "wget -T 10 -qO- " + destination.String() + "; printf CODEX_TOOL_FILE" + raw = bytes.ReplaceAll(raw, []byte("printf CODEX_TOOL_FILE"), []byte(command)) + var cfg mockllm.Config + require.NoError(t, json.Unmarshal(raw, &cfg)) + modelURL := reachableModelURL(t, startMockLLMConfig(t, cfg)) + name := createInteractionTemplate(t, harness, modelURL) + testSessionEgressPolicyPinsDestination(t, harness, name, destination.Scheme+"://"+destination.Host, func(fixture *interactionFixture, allowed bool) { + before := requests.Load() + streamed := sendStreaming(t, fixture, "Run the requested shell command.") + require.Equal(t, a2atype.TaskStateCompleted, streamed.state, "egress probe text = %q, failure = %q", streamed.text, streamed.failureText) + assertToolEvents(t, streamed.toolEvents, "command_execution") + if allowed { + require.Greater(t, requests.Load(), before, "the session must reach the policy-granted destination") + } else { + require.Equal(t, before, requests.Load(), "shell reached an unconfigured destination") + } + }) +} + +func testSessionEgressPolicyPinsDestination(t *testing.T, harness testHarness, name, origin string, probe func(*interactionFixture, bool)) { + t.Helper() + target := interactionTarget(t) + kube := interactionKubeClient(t) + fixture := newInteractionFixtureForTemplate(t, harness, target, name) + probe(fixture, false) + grant := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "additional-access-", Namespace: "kagent"}, Spec: v1alpha3.EgressPolicySpec{ + Destinations: []v1alpha3.EgressDestination{{Origin: origin}}, + }} + require.NoError(t, kube.Create(t.Context(), grant)) + t.Cleanup(func() { require.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), grant))) }) + agent := &v1alpha3.Agent{} + key := ctrlclient.ObjectKey{Namespace: "kagent", Name: name} + require.NoError(t, kube.Get(t.Context(), key, agent)) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: grant.Name}} + require.NoError(t, kube.Update(t.Context(), agent)) + require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, agent); err != nil { + return false, err + } + condition := apimeta.FindStatusCondition(agent.Status.Conditions, v1alpha3.AgentConditionReady) + return condition != nil && condition.ObservedGeneration == agent.Generation && condition.Status == metav1.ConditionTrue, nil + })) + allowed := newInteractionFixtureForTemplate(t, harness, target, name) + probe(allowed, true) + probe(fixture, false) // Existing sessions retain their earlier grants. + + require.NoError(t, kube.Delete(t.Context(), grant)) + require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, agent); err != nil { + return false, err + } + condition := apimeta.FindStatusCondition(agent.Status.Conditions, v1alpha3.AgentConditionReady) + return condition != nil && condition.ObservedGeneration == agent.Generation && condition.Status == metav1.ConditionFalse, nil + })) + // Check creation directly, without the fixture's preparation retry loop. + _, err := allowed.sessions.CreateSession(allowed.ctx, &apiv1.CreateSessionRequest{ + Agent: &apiv1.ResourceReference{Namespace: key.Namespace, Name: key.Name}, RequestId: uuid.NewString(), + }) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + probe(allowed, true) // Deletion preserves the existing session's pinned grant. +} + func TestSandboxEgressPolicyDeletionBlocksCreation(t *testing.T) { t.Parallel() f := newSandboxFixture(t) From 855514a33a3b3117b9013696f244ee2b3e9dbe76 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 15 Oct 2026 22:16:34 +0000 Subject: [PATCH 09/43] fix: reconcile egress policy changes on live actors Signed-off-by: Eitan Yarmush --- docs/architecture/egress-policies.md | 55 +++-- go/api/v1alpha3/agent_types.go | 3 +- go/api/v1alpha3/egresspolicy_types.go | 2 +- go/api/v1alpha3/sandboxtemplate_types.go | 2 +- go/core/internal/controller/egress.go | 189 ++++++++++++++++++ go/core/internal/controller/egress_test.go | 181 +++++++++++++++++ go/core/internal/controller/reconciler.go | 1 + .../controller/sandbox_collections.go | 2 +- .../controller/sandbox_integration_test.go | 16 +- .../internal/controller/sandbox_reconciler.go | 3 +- go/core/internal/database/models.go | 1 + go/core/internal/database/runtime_egress.go | 74 +++++++ .../internal/database/runtime_egress_test.go | 73 +++++++ .../internal/database/runtime_revisions.go | 18 +- go/core/internal/egress/live.go | 28 +++ .../internal/service/sandbox/guest_test.go | 2 +- .../internal/service/sandbox/service_test.go | 17 +- .../service/sandbox/transports_test.go | 2 +- go/core/internal/service/sandbox/workflow.go | 8 +- go/core/internal/service/session/workflow.go | 8 +- .../internal/service/session/workflow_test.go | 32 +-- go/core/internal/substrate/config.go | 7 +- go/core/internal/substrate/egress.go | 51 +++-- go/core/internal/substrate/egress_test.go | 92 +++++++-- go/core/internal/substrate/lifecycle.go | 24 ++- go/core/internal/substrate/lifecycle_test.go | 18 +- go/core/internal/substrate/sandbox.go | 14 +- go/core/internal/substrate/sandbox_test.go | 13 +- go/core/internal/translator/byo/compiler.go | 4 +- .../internal/translator/claude/compiler.go | 8 +- go/core/internal/translator/codex/compiler.go | 4 +- go/core/internal/translator/compiler.go | 13 +- go/core/internal/translator/compiler_test.go | 2 +- go/core/internal/translator/credentials.go | 34 ++-- .../internal/translator/credentials_test.go | 22 +- go/core/internal/translator/egress_test.go | 17 +- .../internal/translator/kagent/compiler.go | 4 +- go/core/internal/translator/revision.go | 5 +- go/core/pkg/app/app.go | 5 + .../pkg/migrations/core/000001_initial.sql | 1 + .../templates/api.kagent.dev_agents.yaml | 3 +- .../api.kagent.dev_egresspolicies.yaml | 2 +- .../api.kagent.dev_sandboxtemplates.yaml | 2 +- 43 files changed, 849 insertions(+), 213 deletions(-) create mode 100644 go/core/internal/controller/egress.go create mode 100644 go/core/internal/controller/egress_test.go create mode 100644 go/core/internal/database/runtime_egress.go create mode 100644 go/core/internal/database/runtime_egress_test.go create mode 100644 go/core/internal/egress/live.go diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index a814681697..3650230bea 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -153,28 +153,39 @@ Secret keys, invalid destinations, and conflicting bindings prevent preparation. The controller publishes the error on the workload's status and re-evaluates it when dependencies change. -Resolved destinations and credential references enter the immutable revision. -Changing a grant or Secret reference changes that revision. Secret values and -UIDs do not; the gateway handles value rotation independently. Policy metadata -changes do not affect the resolved grant. Reference ordering does not affect -revision identity. - -New Sessions and sandboxes pin the current desired revision only when it matches -the latest successful revision. The store checks this atomically with creation. -Unresolved policies and pending or failed preparation block fresh creation, -including during ordinary configuration updates. The previous successful -revision remains retained but cannot be selected for fresh instances while the -current desired revision is unprepared. - -This gate takes effect after the controller reconciles the changed inputs into -the database; updating or deleting a policy is not instantaneous revocation. -Existing instances keep their original grant through creation retries and -resume; Session checkpoint forks also retain the source revision. - -The runtime workflow creates a suspended Actor, installs its compiled `default` -egress policy, and resumes it only after installation succeeds. A retry uses the -same pinned revision. Golden Actor preparation and artifact-startup egress are -unchanged; this API controls runtime Actor access. +Runtime revisions retain the integration access inferred from models and tools, +including credential references and caller-token passthrough constraints. +Explicit policy grants and policy references do not enter revision identity. +Changing a grant or its Secret reference does not prepare a new runtime image. +Secret values remain outside compiled inputs; the gateway handles value rotation. + +The controller resolves current grants from the owning Agent or SandboxTemplate +and reconciles each existing Actor's Substrate policy. Kubernetes events trigger +updates, with a periodic sweep to recover creation races and controller restarts. +This runs independently of agent invocations and runtime preparation. Running and +suspended Actors receive updates without replacement. Missing or invalid policies +and Secrets revoke explicit grants; pinned integration access remains available. +A grant that conflicts with a pinned credential or caller-token origin is also +withheld. Deleting and recreating an Agent or SandboxTemplate does not transfer +its grants to instances owned by the old Kubernetes UID. + +New Sessions and sandboxes pin the current desired runtime revision only when it +matches the latest successful revision. The store checks this atomically with +creation. Unresolved policies and pending or failed preparation block fresh +creation after reconciliation. Existing instances and checkpoint forks retain +their runtime configuration, but resolve current authorization on creation, +creation retries, and resume. A fork cannot restore an old policy from a snapshot. + +The runtime workflow creates a suspended Actor and installs its current `default` +egress policy before it can run. Policy updates use Substrate's UID and version +preconditions; conflicting writes re-read both backend state and current grants. +No database transaction is held across the Substrate calls. + +Policy convergence is asynchronous: a Kubernetes write is not a synchronous +revocation barrier. Once the gateway observes the changed policy, subsequent +requests use it. An update does not cancel a request already in flight. Golden +Actor preparation and artifact-startup egress are separate from this API's runtime +Actor access. ## Authorization and scope diff --git a/go/api/v1alpha3/agent_types.go b/go/api/v1alpha3/agent_types.go index 80bee768a3..61fb786bf0 100644 --- a/go/api/v1alpha3/agent_types.go +++ b/go/api/v1alpha3/agent_types.go @@ -55,7 +55,8 @@ type AgentSpec struct { // EgressPolicyRefs grants additional access to the whole runtime, including // shared subagents. Policies resolve in this Agent's namespace. Integration - // access inferred from models and tools is always included. + // access inferred from models and tools is always included. Changes update + // existing Sessions without a new invocation or runtime restart. // +optional // +kubebuilder:validation:MaxItems=16 // +kubebuilder:validation:XValidation:rule="self.all(ref, has(ref.name) && ref.name != '')",message="egressPolicyRefs requires non-empty names" diff --git a/go/api/v1alpha3/egresspolicy_types.go b/go/api/v1alpha3/egresspolicy_types.go index d93a1a1e01..2cd00bf4a4 100644 --- a/go/api/v1alpha3/egresspolicy_types.go +++ b/go/api/v1alpha3/egresspolicy_types.go @@ -62,7 +62,7 @@ type EgressAuthentication struct { // +kubebuilder:resource:path=egresspolicies,singular=egresspolicy,scope=Namespaced,categories=kagent // EgressPolicy is a reusable grant of additional network access. Workloads in its -// namespace select it explicitly; changing it affects newly prepared revisions. +// namespace select it explicitly; changes update their existing runtime Actors. type EgressPolicy struct { metav1.TypeMeta `json:",inline"` // +optional diff --git a/go/api/v1alpha3/sandboxtemplate_types.go b/go/api/v1alpha3/sandboxtemplate_types.go index b8b27249c3..1c1796715c 100644 --- a/go/api/v1alpha3/sandboxtemplate_types.go +++ b/go/api/v1alpha3/sandboxtemplate_types.go @@ -40,7 +40,7 @@ type SandboxTemplateSpec struct { Substrate RuntimeSubstratePolicy `json:"substrate"` // EgressPolicyRefs grants additional access in this template's namespace. - // Each sandbox retains the access pinned by its prepared revision. + // Changes update the access of existing sandboxes without restarting them. // +optional // +kubebuilder:validation:MaxItems=16 // +kubebuilder:validation:XValidation:rule="self.all(ref, has(ref.name) && ref.name != '')",message="egressPolicyRefs requires non-empty names" diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go new file mode 100644 index 0000000000..299c6650e8 --- /dev/null +++ b/go/core/internal/controller/egress.go @@ -0,0 +1,189 @@ +package controller + +import ( + "context" + "fmt" + "slices" + "time" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "istio.io/istio/pkg/kube/krt" + "sigs.k8s.io/controller-runtime/pkg/manager" + + kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/substrate" + "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/kagent-dev/kagent/go/pkg/logging" +) + +type runtimeEgressStore interface { + GetRuntimeEgress(context.Context, string) (*database.RuntimeEgress, error) + ListRuntimeEgressTargets(context.Context, string, int) ([]database.RuntimeEgressTarget, error) +} + +type policyGrants struct { + Key string + UID string + Access egress.Access +} + +func (p policyGrants) ResourceName() string { return p.Key } + +// EgressResolver resolves mutable policy grants independently of runtime +// compilation. Broken templates and pools must not prevent revocation. +type EgressResolver struct { + grants krt.Collection[policyGrants] + store runtimeEgressStore + namespaces []string +} + +var _ egress.Resolver = (*EgressResolver)(nil) + +func NewEgressResolver(runtime *Runtime, store runtimeEgressStore, namespaces []string) *EgressResolver { + inputs := runtime.Collections + collections := translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets} + agents := krt.NewCollection(inputs.Agents, func(ctx krt.HandlerContext, agent *kagentv1alpha3.Agent) *policyGrants { + grants := &policyGrants{Key: "agent/" + agent.Namespace + "/" + agent.Name, UID: string(agent.UID)} + if agent.DeletionTimestamp.IsZero() { + // CompileEgressPolicies returns empty access on every failure. Runtime + // compilation reports resolution errors through the source's status. + access, err := translator.CompileEgressPolicies(ctx, collections, agent.Namespace, agent.Spec.EgressPolicyRefs) + if err == nil { + grants.Access = access + } + } + return grants + }, runtime.Options.WithName("AgentEgressGrants")...) + sandboxes := krt.NewCollection(inputs.SandboxTemplates, func(ctx krt.HandlerContext, template *kagentv1alpha3.SandboxTemplate) *policyGrants { + grants := &policyGrants{Key: "sandbox/" + template.Namespace + "/" + template.Name, UID: string(template.UID)} + if template.DeletionTimestamp.IsZero() { + access, err := translator.CompileEgressPolicies(ctx, collections, template.Namespace, template.Spec.EgressPolicyRefs) + if err == nil { + grants.Access = access + } + } + return grants + }, runtime.Options.WithName("SandboxEgressGrants")...) + return &EgressResolver{store: store, namespaces: slices.Clone(namespaces), grants: krt.JoinCollection([]krt.Collection[policyGrants]{agents, sandboxes}, runtime.Options.WithName("RuntimeEgressGrants")...)} +} + +func (e *EgressResolver) watches(namespace string) bool { + return len(e.namespaces) == 0 || slices.Contains(e.namespaces, namespace) +} + +func (e *EgressResolver) ResolveEgress(ctx context.Context, revision string) (egress.Access, error) { + if !e.grants.WaitUntilSynced(ctx.Done()) { + return egress.Access{}, fmt.Errorf("egress policy cache has not synchronized: %w", ctx.Err()) + } + pinned, err := e.store.GetRuntimeEgress(ctx, revision) + if err != nil { + return egress.Access{}, err + } + if !e.watches(pinned.Namespace) { + return egress.Access{}, fmt.Errorf("runtime namespace %q is not watched", pinned.Namespace) + } + base := egress.Access{Destinations: pinned.EgressDestinations, Credentials: pinned.Credentials} + grants := e.grants.GetKey(pinned.Kind + "/" + pinned.Namespace + "/" + pinned.SourceName) + // Deleted and recreated sources cannot pass their grants to old sessions. + if grants == nil || grants.UID != pinned.SourceUID { + return base, nil + } + access, err := egress.MergeAccess(base, grants.Access, pinned.PassthroughOrigins) + if err != nil { + // A current grant may conflict with an old session's pinned integrations. + // Revoke its explicit access instead of preserving a stale policy. + logging.FromContext(ctx).DebugContext(ctx, "live egress grants conflict with pinned runtime", "revision", revision, "error", err) + return base, nil + } + return access, nil +} + +type actorEgressClient interface { + GetActor(context.Context, string, string) (*ateapipb.Actor, error) + EnsureActorEgressPolicy(context.Context, string, string, string) error +} + +// EgressReconciler refreshes every allocated actor, including suspended actors. +// Events apply changes promptly; polling recovers actor-creation races, missed +// events and controller restarts. No model turn or runtime restart is required. +type EgressReconciler struct { + resolver *EgressResolver + actors actorEgressClient +} + +var _ manager.LeaderElectionRunnable = (*EgressReconciler)(nil) + +func NewEgressReconciler(resolver *EgressResolver, actors actorEgressClient) *EgressReconciler { + return &EgressReconciler{resolver: resolver, actors: actors} +} + +func (e *EgressReconciler) NeedLeaderElection() bool { return true } + +func (e *EgressReconciler) Start(ctx context.Context) error { + wake := make(chan struct{}, 1) + handler := e.resolver.grants.RegisterBatch(func([]krt.Event[policyGrants]) { + select { + case wake <- struct{}{}: + default: + } + }, true) + defer handler.UnregisterHandler() + if !handler.WaitUntilSynced(ctx.Done()) { + return ctx.Err() + } + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + if err := e.reconcile(ctx); err != nil && ctx.Err() == nil { + logging.FromContext(ctx).ErrorContext(ctx, "failed to reconcile live egress", "error", err) + } + select { + case <-ctx.Done(): + return nil + case <-wake: + case <-ticker.C: + } + } +} + +func (e *EgressReconciler) reconcile(ctx context.Context) error { + const pageSize = 100 + after := "" + for { + targets, err := e.resolver.store.ListRuntimeEgressTargets(ctx, after, pageSize) + if err != nil { + return err + } + for _, target := range targets { + if !e.resolver.watches(target.Atespace) { + continue + } + if err := e.reconcileActor(ctx, target); err != nil && status.Code(err) != codes.NotFound { + logging.FromContext(ctx).ErrorContext(ctx, "failed to update actor egress", "instance_id", target.ID, "error", err) + } + if err := ctx.Err(); err != nil { + return err + } + } + if len(targets) < pageSize { + return nil + } + after = targets[len(targets)-1].ID + } +} + +func (e *EgressReconciler) reconcileActor(ctx context.Context, target database.RuntimeEgressTarget) error { + binding := substrate.ActorBinding{Atespace: target.Atespace, Name: substrate.ActorName(target.ID), TemplateAtespace: target.Atespace, TemplateName: target.TemplateName} + actor, err := e.actors.GetActor(ctx, binding.Atespace, binding.Name) + if err != nil { + return err + } + if !binding.Matches(actor) || (target.ActorUID != "" && target.ActorUID != actor.GetMetadata().GetUid()) { + return fmt.Errorf("actor identity or template changed") + } + return e.actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, target.Revision) +} diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go new file mode 100644 index 0000000000..6044066e9b --- /dev/null +++ b/go/core/internal/controller/egress_test.go @@ -0,0 +1,181 @@ +package controller + +import ( + "context" + "fmt" + "reflect" + "testing" + "time" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/stretchr/testify/require" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +type egressTestStore struct { + pinned database.RuntimeEgress + targets []database.RuntimeEgressTarget +} + +var _ runtimeEgressStore = (*egressTestStore)(nil) + +func (e *egressTestStore) GetRuntimeEgress(context.Context, string) (*database.RuntimeEgress, error) { + return &e.pinned, nil +} +func (e *egressTestStore) ListRuntimeEgressTargets(_ context.Context, after string, limit int) ([]database.RuntimeEgressTarget, error) { + if after != "" { + return nil, nil + } + return e.targets, nil +} + +func TestLiveEgressResolvesCurrentSource(t *testing.T) { + for _, kind := range []string{"agent", "sandbox"} { + t.Run(kind, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) + agents := krt.NewStaticCollection[*v1alpha3.Agent](nil, nil, opts.WithName("Agents")...) + sandboxes := krt.NewStaticCollection[*v1alpha3.SandboxTemplate](nil, nil, opts.WithName("Sandboxes")...) + policies := krt.NewStaticCollection[*v1alpha3.EgressPolicy](nil, nil, opts.WithName("Policies")...) + secrets := krt.NewStaticCollection[*corev1.Secret](nil, nil, opts.WithName("Secrets")...) + setSource := func(uid string, refs []corev1.LocalObjectReference) { + metadata := metav1.ObjectMeta{Namespace: "team", Name: "source"} + metadata.SetUID(types.UID(uid)) + if kind == "agent" { + agents.UpdateObject(&v1alpha3.Agent{ObjectMeta: metadata, Spec: v1alpha3.AgentSpec{EgressPolicyRefs: refs}}) + } else { + sandboxes.UpdateObject(&v1alpha3.SandboxTemplate{ObjectMeta: metadata, Spec: v1alpha3.SandboxTemplateSpec{EgressPolicyRefs: refs}}) + } + } + refs := []corev1.LocalObjectReference{{Name: "access"}} + setSource("original", refs) + base := egress.Access{Destinations: []string{"https://model.example.com:443"}} + store := &egressTestStore{pinned: database.RuntimeEgress{Kind: kind, Namespace: "team", SourceName: "source", SourceUID: "original", EgressDestinations: base.Destinations}} + resolver := NewEgressResolver(&Runtime{Options: opts, Collections: Collections{Agents: agents, SandboxTemplates: sandboxes, EgressPolicies: policies, Secrets: secrets}}, store, nil) + expect := func(want egress.Access) { + t.Helper() + require.Eventually(t, func() bool { + got, err := resolver.ResolveEgress(ctx, "pinned") + return err == nil && reflect.DeepEqual(want, got) + }, time.Second, 10*time.Millisecond) + } + expect(base) // Missing reference; unrelated runtime configuration is also invalid. + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} + policies.UpdateObject(policy.DeepCopy()) + allowed := egress.Access{Destinations: []string{"https://model.example.com:443", "https://tickets.example.com:443"}} + expect(allowed) + setSource("original", nil) + expect(base) + setSource("original", refs) + expect(allowed) + policies.DeleteObject("team/access") + expect(base) + policies.UpdateObject(policy.DeepCopy()) + expect(allowed) + setSource("replacement", refs) + expect(base) + setSource("original", refs) + expect(allowed) + policy.Spec.Destinations[0].Authentication = &v1alpha3.EgressAuthentication{Header: "authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}, Key: "token"}} + policies.UpdateObject(policy.DeepCopy()) + expect(base) + secrets.UpdateObject(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "auth"}, Data: map[string][]byte{"token": []byte("secret")}}) + allowed.Credentials = []egress.Credential{{Origin: "https://tickets.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token"}} + expect(allowed) + secrets.DeleteObject("team/auth") + expect(base) + }) + } +} + +func TestLiveEgressRejectsConflictsWithPinnedRuntime(t *testing.T) { + for _, passthrough := range []bool{false, true} { + t.Run(fmt.Sprint("passthrough=", passthrough), func(t *testing.T) { + base := egress.Access{Destinations: []string{"https://model.example.com:443"}} + credential := egress.Credential{Origin: base.Destinations[0], Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token"} + pinned := database.RuntimeEgress{Kind: "agent", Namespace: "team", SourceName: "source", SourceUID: "uid", EgressDestinations: base.Destinations} + if passthrough { + pinned.PassthroughOrigins = base.Destinations + } else { + base.Credentials = []egress.Credential{credential} + pinned.Credentials = base.Credentials + } + credential.URI = "ate-secret://k8s.io/default/team/other/token" + opts := krt.NewOptionsBuilder(t.Context().Done(), "test", nil) + grants := krt.NewStaticCollection(nil, []policyGrants{{Key: "agent/team/source", UID: "uid", Access: egress.Access{Destinations: base.Destinations, Credentials: []egress.Credential{credential}}}}, opts.WithName("Grants")...) + resolver := &EgressResolver{grants: grants, store: &egressTestStore{pinned: pinned}} + got, err := resolver.ResolveEgress(t.Context(), "old-revision") + require.NoError(t, err) + require.Equal(t, base, got) + }) + } +} + +type observingEgressActors struct { + resolver *EgressResolver + applied chan egress.Access +} + +var _ actorEgressClient = (*observingEgressActors)(nil) + +func (a *observingEgressActors) EnsureActorEgressPolicy(ctx context.Context, _, _, revision string) error { + access, err := a.resolver.ResolveEgress(ctx, revision) + if err != nil { + return err + } + select { + case a.applied <- access: + case <-ctx.Done(): + return ctx.Err() + } + return nil +} + +func TestLiveEgressReconcilesWithoutInvocationAndAfterRestart(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) + grant := policyGrants{Key: "agent/team/source", UID: "uid", Access: egress.Access{Destinations: []string{"https://tickets.example.com:443"}}} + grants := krt.NewStaticCollection(nil, []policyGrants{grant}, opts.WithName("Grants")...) + store := &egressTestStore{pinned: database.RuntimeEgress{Kind: "agent", Namespace: "team", SourceName: "source", SourceUID: "uid"}, targets: []database.RuntimeEgressTarget{{ID: "instance", Revision: "old-revision", Atespace: "team", TemplateName: "runtime"}}} + resolver := &EgressResolver{grants: grants, store: store} + actors := &observingEgressActors{resolver: resolver, applied: make(chan egress.Access, 10)} + expect := func(want []string) { + t.Helper() + for { + select { + case access := <-actors.applied: + if reflect.DeepEqual(want, access.Destinations) { + return + } + case <-ctx.Done(): + t.Fatal("policy did not converge") + } + } + } + for i := range 2 { + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- NewEgressReconciler(resolver, actors).Start(runCtx) }() + expect(grant.Access.Destinations) + if i == 0 { + grant.Access = egress.Access{} + grants.UpdateObject(grant) + expect(nil) + } + stop() + require.NoError(t, <-done) + } +} + +func (a *observingEgressActors) GetActor(_ context.Context, space, name string) (*ateapipb.Actor, error) { + return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: "uid"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: "runtime"}}, nil +} diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index df2c2eda5a..e7191dbf86 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -370,6 +370,7 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { SourceSnapshot: target.Revision.Provenance, AgentCard: target.Revision.AgentCard, EgressDestinations: target.Revision.EgressDestinations, Credentials: target.Revision.Credentials, + PassthroughOrigins: target.Revision.PassthroughOrigins, ActorTemplateAtespace: observed.GetMetadata().GetAtespace(), ActorTemplateName: observed.GetMetadata().GetName(), ActorTemplateUID: observed.GetMetadata().GetUid(), } ready := observed.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() != nil diff --git a/go/core/internal/controller/sandbox_collections.go b/go/core/internal/controller/sandbox_collections.go index 075b439483..b115994329 100644 --- a/go/core/internal/controller/sandbox_collections.go +++ b/go/core/internal/controller/sandbox_collections.go @@ -98,7 +98,7 @@ func newSandboxCollections(inputs Collections, policy substrate.SandboxPolicy, o state.Failure = &ReconciliationFailure{Reason: kagentv1alpha3.SandboxTemplateReasonEgressPolicyResolutionFailed, Message: err.Error()} return state } - state.DesiredActorTemplate, state.RevisionID, state.SourceSnapshot, err = substrate.SandboxActorTemplate(template, (*pool).Spec.SandboxClass, policy, state.Egress) + state.DesiredActorTemplate, state.RevisionID, state.SourceSnapshot, err = substrate.SandboxActorTemplate(template, (*pool).Spec.SandboxClass, policy) if err != nil { state.CompilationError = err.Error() state.Failure = &ReconciliationFailure{Reason: sandboxPreparationFailed, Message: sandboxPreparationFailureMessage} diff --git a/go/core/internal/controller/sandbox_integration_test.go b/go/core/internal/controller/sandbox_integration_test.go index 3e3878c9c8..91d1eb3816 100644 --- a/go/core/internal/controller/sandbox_integration_test.go +++ b/go/core/internal/controller/sandbox_integration_test.go @@ -119,8 +119,8 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { store.mu.Lock() revision := store.revision.Revision require.True(t, store.ready) - require.Equal(t, []string{"https://tickets.example.com:443"}, store.revision.EgressDestinations) - require.Len(t, store.revision.Credentials, 1) + require.Empty(t, store.revision.EgressDestinations) + require.Empty(t, store.revision.Credentials) require.NotContains(t, string(store.revision.SourceSnapshot), "original-token") store.mu.Unlock() other, err := client.ApiV1alpha3().SandboxTemplates(unwatched.Namespace).Get(t.Context(), unwatched.Name, metav1.GetOptions{}) @@ -144,14 +144,6 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { grant.Spec.Destinations = append(grant.Spec.Destinations, kagentv1alpha3.EgressDestination{Origin: "https://registry.npmjs.org"}) grant, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Update(t.Context(), grant, metav1.UpdateOptions{}) require.NoError(t, err) - require.Eventually(t, func() bool { - store.mu.Lock() - defer store.mu.Unlock() - return store.ready && store.revision.Revision != revision && len(store.revision.EgressDestinations) == 2 - }, 10*time.Second, 10*time.Millisecond) - store.mu.Lock() - revision = store.revision.Revision - store.mu.Unlock() require.NoError(t, client.ApiV1alpha3().EgressPolicies(template.Namespace).Delete(t.Context(), grant.Name, metav1.DeleteOptions{})) waitUnresolved("EgressPolicy") grant.ResourceVersion, grant.UID = "", "" @@ -159,6 +151,10 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { require.NoError(t, err) require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) + store.mu.Lock() + require.Equal(t, revision, store.revision.Revision, "policy changes do not reprepare the runtime") + store.mu.Unlock() + // A dependency update alone must change the persisted revision. changedPool := pool.DeepCopy() changedPool.Spec.SandboxClass = atev1alpha1.SandboxClassMicroVM diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index e3586a771a..884ee3439d 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -173,8 +173,7 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { RuntimeArtifact: database.RuntimeArtifact{Revision: state.RevisionID, Kind: "sandbox", Namespace: state.Template.Namespace, ActorTemplateAtespace: ref.GetAtespace(), ActorTemplateName: ref.GetName(), ActorTemplateUID: observed.GetMetadata().GetUid()}, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), - SourceSnapshot: state.SourceSnapshot, - EgressDestinations: state.Egress.Destinations, Credentials: state.Egress.Credentials, + SourceSnapshot: state.SourceSnapshot, } if err := s.store.RecordSandboxRevision(ctx, revision, ready); err != nil { return s.observeError(*state, fmt.Errorf("store sandbox revision %s: %w", state.RevisionID, err), true) diff --git a/go/core/internal/database/models.go b/go/core/internal/database/models.go index 3f66aea9e6..463a8062af 100644 --- a/go/core/internal/database/models.go +++ b/go/core/internal/database/models.go @@ -67,6 +67,7 @@ type RuntimeRevision struct { AgentCard *a2apb.AgentCard Credentials []egress.Credential EgressDestinations []string + PassthroughOrigins []string ActorTemplateAtespace string ActorTemplateName string ActorTemplateUID string diff --git a/go/core/internal/database/runtime_egress.go b/go/core/internal/database/runtime_egress.go new file mode 100644 index 0000000000..9398b9e2f4 --- /dev/null +++ b/go/core/internal/database/runtime_egress.go @@ -0,0 +1,74 @@ +package database + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5" + + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +// RuntimeEgress retains integration access and the source identity of a pinned +// revision. Explicit policy grants are resolved from the current Kubernetes source. +type RuntimeEgress struct { + Kind string + Namespace string + SourceName string + SourceUID string + EgressDestinations []string + Credentials []egress.Credential + PassthroughOrigins []string +} + +// GetRuntimeEgress reads immutable access and source identity for either runtime +// kind. Missing revisions return ErrNotFound; malformed grants return an error. +func (c *Client) GetRuntimeEgress(ctx context.Context, revision string) (*RuntimeEgress, error) { + row, err := queryOne(ctx, c.db, ` + SELECT r.kind, r.namespace, r.egress_destinations, r.credentials, r.passthrough_origins, + COALESCE(a.agent_name, s.sandbox_template_name) AS source_name, + COALESCE(a.agent_uid, s.sandbox_template_uid) AS source_uid + FROM runtime_revision r + LEFT JOIN agent_revision a USING (revision) + LEFT JOIN sandbox_revision s USING (revision) + WHERE r.revision = $1 + `, pgx.RowToStructByName[RuntimeEgress], revision) + if err != nil { + return nil, fmt.Errorf("get runtime egress %s: %w", revision, notFoundOr(err)) + } + access, err := egress.MergeAccess(egress.Access{Destinations: row.EgressDestinations, Credentials: row.Credentials}, egress.Access{}, row.PassthroughOrigins) + if err != nil { + return nil, fmt.Errorf("decode runtime egress %s: %w", revision, err) + } + row.EgressDestinations, row.Credentials = access.Destinations, access.Credentials + return &row, nil +} + +// RuntimeEgressTarget identifies an allocated runtime, including suspended or +// partially created actors that still need live policy reconciliation. +type RuntimeEgressTarget struct { + ID string + Revision string + Atespace string + TemplateName string + ActorUID string +} + +// ListRuntimeEgressTargets lists a bounded page in ID order. Deleting instances +// and unissued creations are excluded; a concurrent creation is picked up by the next reconciliation. +func (c *Client) ListRuntimeEgressTargets(ctx context.Context, afterID string, limit int) ([]RuntimeEgressTarget, error) { + rows, err := queryMany(ctx, c.db, ` + SELECT i.id::text AS id, i.prepared_revision AS revision, r.actor_template_atespace AS atespace, + r.actor_template_name AS template_name, COALESCE(s.actor_uid, '') AS actor_uid + FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision + LEFT JOIN session s ON s.id = i.id + WHERE i.id > COALESCE(NULLIF($1, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) + AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' + ORDER BY i.id LIMIT $2 + `, pgx.RowToStructByName[RuntimeEgressTarget], afterID, limit) + if err != nil { + return nil, fmt.Errorf("list runtime egress targets: %w", err) + } + return rows, nil +} diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go new file mode 100644 index 0000000000..5e031a823b --- /dev/null +++ b/go/core/internal/database/runtime_egress_test.go @@ -0,0 +1,73 @@ +package database + +import ( + "testing" + + a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" +) + +func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", PassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} + require.NoError(t, client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", DesiredRevision: "agent"})) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, true)) + instance, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "request") + require.NoError(t, err) + op, err := client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) + require.NoError(t, err) + executor := uuid.New() + claimed, err := client.ClaimSessionOperation(ctx, instance.Id, op.ID, executor) + require.NoError(t, err) + require.True(t, claimed) + _, err = client.FinishSessionOperation(ctx, instance.Id, op.ID, executor, "actor.example.com", "actor-uid", "") + require.NoError(t, err) + op, err = client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND) + require.NoError(t, err) + executor = uuid.New() + claimed, err = client.ClaimSessionOperation(ctx, instance.Id, op.ID, executor) + require.NoError(t, err) + require.True(t, claimed) + _, err = client.FinishSessionOperation(ctx, instance.Id, op.ID, executor, "", "", "") + require.NoError(t, err) + // A recreated Agent must not transfer authorization to an old pinned session. + require.NoError(t, client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "new-uid", DesiredRevision: "pending"})) + pinned, err := client.GetRuntimeEgress(ctx, instance.PreparedRevision) + require.NoError(t, err) + require.Equal(t, "old-uid", pinned.SourceUID) + require.Equal(t, "agent", pinned.Kind) + require.Equal(t, revision.PassthroughOrigins, pinned.PassthroughOrigins) + sandbox, options := sandboxFixture(t, client, "sandbox") + created, _, err := client.CreateSandbox(ctx, sandbox, "sandbox", options) + require.NoError(t, err) + sandboxOp, err := client.BeginSandboxOperation(ctx, created.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) + require.NoError(t, err) + claimed, err = client.ClaimSandboxOperation(ctx, created.Id, sandboxOp.ID, uuid.New()) + require.NoError(t, err) + require.True(t, claimed) + pinned, err = client.GetRuntimeEgress(ctx, created.PreparedRevision) + require.NoError(t, err) + require.Equal(t, "sandbox", pinned.Kind) + require.Equal(t, "scratch", pinned.SourceName) + require.Equal(t, "uid", pinned.SourceUID) + first, err := client.ListRuntimeEgressTargets(ctx, "", 1) + require.NoError(t, err) + require.Len(t, first, 1) + second, err := client.ListRuntimeEgressTargets(ctx, first[0].ID, 1) + require.NoError(t, err) + require.Len(t, second, 1) + require.ElementsMatch(t, []string{instance.Id, created.Id}, []string{first[0].ID, second[0].ID}) + last, err := client.ListRuntimeEgressTargets(ctx, second[0].ID, 1) + require.NoError(t, err) + require.Empty(t, last) + _, err = client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE) + require.NoError(t, err) + remaining, err := client.ListRuntimeEgressTargets(ctx, "", 10) + require.NoError(t, err) + require.Len(t, remaining, 1) + require.Equal(t, created.Id, remaining[0].ID) +} diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index be54f34260..0fff04e33b 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -97,7 +97,7 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{ RuntimeArtifact: RuntimeArtifact{Revision: revision.Revision, Kind: runtimeKindAgent, Namespace: revision.Namespace, ActorTemplateAtespace: revision.ActorTemplateAtespace, ActorTemplateName: revision.ActorTemplateName, ActorTemplateUID: revision.ActorTemplateUID}, - SourceSnapshot: revision.SourceSnapshot, EgressDestinations: revision.EgressDestinations, Credentials: revision.Credentials, + SourceSnapshot: revision.SourceSnapshot, EgressDestinations: revision.EgressDestinations, Credentials: revision.Credentials, PassthroughOrigins: revision.PassthroughOrigins, }); err != nil { return err } @@ -128,7 +128,7 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi func (c *Client) GetRuntimeRevision(ctx context.Context, revision string) (*RuntimeRevision, error) { row, err := queryOne(ctx, c.db, ` SELECT revision, namespace, agent_name, agent_uid, - source_snapshot, egress_destinations, credentials, actor_template_atespace, actor_template_name, actor_template_uid, + source_snapshot, egress_destinations, passthrough_origins, credentials, actor_template_atespace, actor_template_name, actor_template_uid, agent_card, deleted_at FROM agent_runtime_revision WHERE revision = $1 `, pgx.RowToStructByName[runtimeRevisionRow], revision) if err != nil { @@ -153,6 +153,7 @@ func toRuntimeRevision(row runtimeRevisionRow) (*RuntimeRevision, error) { AgentName: row.AgentName, AgentUID: row.AgentUID, SourceSnapshot: row.SourceSnapshot, AgentCard: card, EgressDestinations: row.EgressDestinations, + PassthroughOrigins: row.PassthroughOrigins, Credentials: credentials, ActorTemplateAtespace: row.ActorTemplateAtespace, ActorTemplateName: row.ActorTemplateName, ActorTemplateUID: row.ActorTemplateUID, @@ -204,7 +205,7 @@ func getAvailableRuntimeRevisionForUpdate(ctx context.Context, tx pgx.Tx, revisi } return queryOne(ctx, tx, ` SELECT revision, namespace, agent_name, agent_uid, - source_snapshot, egress_destinations, credentials, actor_template_atespace, actor_template_name, actor_template_uid, + source_snapshot, egress_destinations, passthrough_origins, credentials, actor_template_atespace, actor_template_name, actor_template_uid, agent_card, deleted_at FROM agent_runtime_revision WHERE revision = $1 `, pgx.RowToStructByName[runtimeRevisionRow], revision) } @@ -229,6 +230,7 @@ type runtimeRevisionRecord struct { RuntimeArtifact SourceSnapshot json.RawMessage EgressDestinations []string + PassthroughOrigins []string Credentials []egress.Credential } @@ -242,14 +244,17 @@ func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevis if revision.EgressDestinations == nil { revision.EgressDestinations = []string{} } + if revision.PassthroughOrigins == nil { + revision.PassthroughOrigins = []string{} + } result, err := tx.Exec(ctx, ` INSERT INTO runtime_revision (revision, kind, namespace, source_snapshot, egress_destinations, - actor_template_atespace, actor_template_name, actor_template_uid, credentials) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + actor_template_atespace, actor_template_name, actor_template_uid, credentials, passthrough_origins) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (revision) DO UPDATE SET actor_template_uid = EXCLUDED.actor_template_uid, updated_at = NOW() WHERE runtime_revision.deleted_at IS NULL AND runtime_revision.kind = EXCLUDED.kind `, revision.Revision, revision.Kind, revision.Namespace, revision.SourceSnapshot, revision.EgressDestinations, - revision.ActorTemplateAtespace, revision.ActorTemplateName, revision.ActorTemplateUID, revision.Credentials) + revision.ActorTemplateAtespace, revision.ActorTemplateName, revision.ActorTemplateUID, revision.Credentials, revision.PassthroughOrigins) if err != nil { return fmt.Errorf("record runtime revision %s: %w", revision.Revision, err) } @@ -343,6 +348,7 @@ type runtimeRevisionRow struct { AgentUID string SourceSnapshot []byte EgressDestinations []string + PassthroughOrigins []string Credentials []egress.Credential ActorTemplateAtespace string ActorTemplateName string diff --git a/go/core/internal/egress/live.go b/go/core/internal/egress/live.go new file mode 100644 index 0000000000..45ac3cc7af --- /dev/null +++ b/go/core/internal/egress/live.go @@ -0,0 +1,28 @@ +package egress + +import ( + "context" + "fmt" + "slices" +) + +// Resolver combines a pinned revision's integration access with current policy +// grants. Missing or invalid grants must fail closed without retaining old access. +type Resolver interface { + ResolveEgress(context.Context, string) (Access, error) +} + +// MergeAccess validates live grants against the pinned runtime's credentials and +// caller-token origins. Inputs are never modified. +func MergeAccess(base, grants Access, passthroughOrigins []string) (Access, error) { + access, err := CanonicalAccess(append(slices.Clone(base.Destinations), grants.Destinations...), append(slices.Clone(base.Credentials), grants.Credentials...)) + if err != nil { + return Access{}, err + } + for _, binding := range access.Credentials { + if slices.Contains(passthroughOrigins, binding.Origin) { + return Access{}, fmt.Errorf("destination %q cannot combine caller-token passthrough with gateway credentials", binding.Origin) + } + } + return access, nil +} diff --git a/go/core/internal/service/sandbox/guest_test.go b/go/core/internal/service/sandbox/guest_test.go index c4c2bd2daf..2d14eb1820 100644 --- a/go/core/internal/service/sandbox/guest_test.go +++ b/go/core/internal/service/sandbox/guest_test.go @@ -82,7 +82,7 @@ func (a *testActors) CreateActor(_ context.Context, space, name, templateSpace, Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}} return proto.CloneOf(a.actor), a.mutationErr } -func (a *testActors) EnsureActorEgressPolicy(context.Context, string, string, *ateapipb.EgressPolicy) error { +func (a *testActors) EnsureActorEgressPolicy(context.Context, string, string, string) error { a.mu.Lock() defer a.mu.Unlock() a.calls = append(a.calls, "policy") diff --git a/go/core/internal/service/sandbox/service_test.go b/go/core/internal/service/sandbox/service_test.go index 2ac2fa9411..23440bbc1a 100644 --- a/go/core/internal/service/sandbox/service_test.go +++ b/go/core/internal/service/sandbox/service_test.go @@ -46,7 +46,7 @@ type testActors struct { substrate.LifecycleClient mu sync.Mutex actor *ateapipb.Actor - egressPolicy *ateapipb.EgressPolicy + egressRevision string calls []string readErr error mutationErr error @@ -74,11 +74,11 @@ func (a *testActors) CreateActor(_ context.Context, space, name, templateSpace, Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}} return proto.CloneOf(a.actor), a.mutationErr } -func (a *testActors) EnsureActorEgressPolicy(_ context.Context, _, _ string, policy *ateapipb.EgressPolicy) error { +func (a *testActors) EnsureActorEgressPolicy(_ context.Context, _, _ string, revision string) error { a.mu.Lock() defer a.mu.Unlock() a.calls = append(a.calls, "policy") - a.egressPolicy = proto.CloneOf(policy) + a.egressRevision = revision return a.mutationErr } func (a *testActors) ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) { @@ -216,8 +216,7 @@ func TestSandboxLifecycleClientRetriesAfterRestart(t *testing.T) { } } require.Equal(t, 1, count, "retry must preserve the original Actor and files") - require.NotNil(t, actors.egressPolicy) - require.Empty(t, actors.egressPolicy.Rules) + require.NotEmpty(t, actors.egressRevision) }) } } @@ -286,10 +285,10 @@ func TestSandboxConflictingRequestCanRetryAfterActiveAttempt(t *testing.T) { require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, current.State) require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, current.Operation) - require.Equal(t, []string{"create", "policy", "resume", "suspend", "resume"}, actors.observedCalls()) + require.Equal(t, []string{"create", "policy", "resume", "suspend", "policy", "resume"}, actors.observedCalls()) } -func TestSandboxInstallsPinnedGrantBeforeResume(t *testing.T) { +func TestSandboxResolvesCurrentGrantsBeforeResume(t *testing.T) { service, ctx, actors := serviceFixture(t) revision := database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "policy-revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "policy-runtime", ActorTemplateUID: "policy-runtime-uid"}, @@ -303,9 +302,7 @@ func TestSandboxInstallsPinnedGrantBeforeResume(t *testing.T) { instance, err := service.Create(ctx, createRequest()) require.NoError(t, err) require.Equal(t, revision.Revision, instance.PreparedRevision) - expected, err := substrate.ActorEgressPolicy("team-a", revision.EgressDestinations, revision.Credentials) - require.NoError(t, err) - require.True(t, proto.Equal(expected, actors.egressPolicy)) + require.Equal(t, revision.Revision, actors.egressRevision) calls := actors.observedCalls() require.Less(t, slices.Index(calls, "policy"), slices.Index(calls, "resume")) require.Contains(t, calls, "policy") diff --git a/go/core/internal/service/sandbox/transports_test.go b/go/core/internal/service/sandbox/transports_test.go index eca00727d0..0874a3f9f0 100644 --- a/go/core/internal/service/sandbox/transports_test.go +++ b/go/core/internal/service/sandbox/transports_test.go @@ -250,5 +250,5 @@ func TestSandboxTransports(t *testing.T) { require.NoError(t, err) _, err = client.DeleteSandbox(ctx, &apiv1.DeleteSandboxRequest{SandboxId: id}) require.NoError(t, err) - require.Equal(t, []string{"create", "policy", "resume", "suspend", "suspend", "resume", "suspend", "delete"}, actors.observedCalls()) + require.Equal(t, []string{"create", "policy", "resume", "suspend", "suspend", "policy", "resume", "suspend", "delete"}, actors.observedCalls()) } diff --git a/go/core/internal/service/sandbox/workflow.go b/go/core/internal/service/sandbox/workflow.go index 6241b08453..fc1df12f15 100644 --- a/go/core/internal/service/sandbox/workflow.go +++ b/go/core/internal/service/sandbox/workflow.go @@ -39,15 +39,11 @@ func (s *Service) run(ctx context.Context, id string, kind apiv1.RuntimeOperatio if err != nil { return nil, sandboxError(err) } - binding := substrate.ActorBinding{Atespace: revision.ActorTemplateAtespace, Name: substrate.ActorName(id), + binding := substrate.ActorBinding{Revision: revision.Revision, Atespace: revision.ActorTemplateAtespace, Name: substrate.ActorName(id), TemplateAtespace: revision.ActorTemplateAtespace, TemplateName: revision.ActorTemplateName} var creation *substrate.ActorCreation if kind == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { - policy, err := substrate.ActorEgressPolicy(binding.Atespace, revision.EgressDestinations, revision.Credentials) - if err != nil { - return nil, sandboxError(err) - } - creation = &substrate.ActorCreation{EgressPolicy: policy, Resume: true} + creation = &substrate.ActorCreation{Resume: true} } prepare := substrate.PrepareActorTransition if op.ExecutorID != uuid.Nil { diff --git a/go/core/internal/service/session/workflow.go b/go/core/internal/service/session/workflow.go index 63a0517ead..805e073bb4 100644 --- a/go/core/internal/service/session/workflow.go +++ b/go/core/internal/service/session/workflow.go @@ -158,15 +158,11 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session tagName = "checkpoint-" + operation.SourceCheckpointID.String() } - binding := substrate.ActorBinding{Atespace: revision.ActorTemplateAtespace, Name: substrate.ActorName(session.Id), + binding := substrate.ActorBinding{Revision: revision.Revision, Atespace: revision.ActorTemplateAtespace, Name: substrate.ActorName(session.Id), TemplateAtespace: revision.ActorTemplateAtespace, TemplateName: revision.ActorTemplateName} var creation *substrate.ActorCreation if kind == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { - policy, err := substrate.ActorEgressPolicy(binding.Atespace, revision.EgressDestinations, revision.Credentials) - if err != nil { - return w.failPreparation(ctx, operation, err) - } - creation = &substrate.ActorCreation{EgressPolicy: policy} + creation = &substrate.ActorCreation{} if snapshot != nil { creation.Snapshot = &substrate.ActorSnapshot{Tag: &ateapipb.ObjectRef{Atespace: snapshot.Atespace, Name: tagName}, URI: snapshot.URI, ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA} } diff --git a/go/core/internal/service/session/workflow_test.go b/go/core/internal/service/session/workflow_test.go index 3c6e9cf563..b503fb5386 100644 --- a/go/core/internal/service/session/workflow_test.go +++ b/go/core/internal/service/session/workflow_test.go @@ -165,12 +165,12 @@ func (s *lifecycleTestStore) GetRuntimeRevision(context.Context, string) (*datab } type lifecycleTestActors struct { - mu sync.Mutex - actors map[string]*ateapipb.Actor - policyErr error - policy *ateapipb.EgressPolicy - policyActor string - policyCalls int + mu sync.Mutex + actors map[string]*ateapipb.Actor + policyErr error + policyRevision string + policyActor string + policyCalls int } func actorKey(atespace, name string) string { return atespace + "/" + name } @@ -284,11 +284,11 @@ func lifecycleForkFixture(t *testing.T, store *lifecycleTestStore, actors *lifec return fork, checkpoint.Id } -func (a *lifecycleTestActors) EnsureActorEgressPolicy(_ context.Context, atespace, name string, policy *ateapipb.EgressPolicy) error { +func (a *lifecycleTestActors) EnsureActorEgressPolicy(_ context.Context, atespace, name string, revision string) error { a.mu.Lock() defer a.mu.Unlock() a.policyActor = actorKey(atespace, name) - a.policy = proto.CloneOf(policy) + a.policyRevision = revision a.policyCalls++ return a.policyErr } @@ -304,27 +304,15 @@ func TestActorCreationRetainsEgressPolicyFailure(t *testing.T) { actors := &retryTestActors{lifecycleTestActors: base} workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second) callsBefore := base.policyCalls - store.revision.EgressDestinations = []string{"*"} - _, err := workflow.Create(t.Context(), session) - require.ErrorContains(t, err, "invalid egress origin") - require.Zero(t, actors.mutations.Load(), "validate the allowlist before issuing Actor creation") - require.Equal(t, callsBefore, base.policyCalls) - - store.revision.EgressDestinations = []string{"https://api.example.com:443", "https://other.example.com:8443"} - store.revision.Credentials = []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/auth/token"}} base.policyErr = context.DeadlineExceeded - _, err = workflow.Create(t.Context(), session) + _, err := workflow.Create(t.Context(), session) require.ErrorIs(t, err, context.DeadlineExceeded) current, err := store.GetSessionByID(t.Context(), session.Id) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_CREATING, current.State) require.Empty(t, current.A2AAuthority) require.Equal(t, actorKey("team-a", substrate.ActorName(session.Id)), base.policyActor) - require.Equal(t, &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "default"}, base.policy.Metadata) - require.Len(t, base.policy.Rules, 2) - require.Equal(t, &ateapipb.CredentialHeader{Header: "authorization", Prefix: "Bearer ", CredentialUri: "ate-secret://k8s.io/default/team-a/auth/token"}, base.policy.Rules[0].GetHttps().GetEffects().GetReplaceHeaders()[0]) - require.Equal(t, []string{"api.example.com"}, base.policy.Rules[0].GetHttps().GetHostnames()) - require.Equal(t, []string{"other.example.com"}, base.policy.Rules[1].GetHttps().GetHostnames()) + require.Equal(t, session.PreparedRevision, base.policyRevision) // Retry completes policy setup for the existing Actor before readiness. base.policyErr = nil diff --git a/go/core/internal/substrate/config.go b/go/core/internal/substrate/config.go index 55939730fb..3e78a7ff2a 100644 --- a/go/core/internal/substrate/config.go +++ b/go/core/internal/substrate/config.go @@ -1,9 +1,14 @@ package substrate -import "time" +import ( + "time" + + "github.com/kagent-dev/kagent/go/core/internal/egress" +) // Config holds connection settings for Agent Substrate ate-api. type Config struct { + EgressResolver egress.Resolver // AteAPIEndpoint is a gRPC target (e.g. dns:///api.ate-system.svc:443). AteAPIEndpoint string // CAFile contains the CA certificates used to verify ate-api. diff --git a/go/core/internal/substrate/egress.go b/go/core/internal/substrate/egress.go index dd7d0be647..f720e83897 100644 --- a/go/core/internal/substrate/egress.go +++ b/go/core/internal/substrate/egress.go @@ -10,25 +10,42 @@ import ( "google.golang.org/protobuf/proto" ) -// EnsureActorEgressPolicy can be retried after a lost create response. A prepared -// revision is immutable, so an existing policy must match; never accept or -// overwrite a different allowlist. Substrate deletes the policy with its Actor. -func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name string, policy *ateapipb.EgressPolicy) error { +// EnsureActorEgressPolicy installs current authorization without restarting the +// actor. Read the backend version BEFORE resolving grants: an older in-flight +// writer then cannot overwrite a newer policy. A conflict retries resolution, +// never the stale policy. Substrate deletes the policy with its Actor. +func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name, revision string) error { ctx, cancel := c.callCtx(ctx) defer cancel() - actor := actorRef(atespace, name) - _, err := c.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{ - Actor: actor, EgressPolicy: policy, - }) - if status.Code(err) != codes.AlreadyExists { - return err - } - existing, err := c.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) - if err != nil { - return err + if c.cfg.EgressResolver == nil { + return fmt.Errorf("actor egress resolver is required") } - if !proto.Equal(&ateapipb.EgressPolicy{Rules: existing.GetRules()}, &ateapipb.EgressPolicy{Rules: policy.Rules}) { - return fmt.Errorf("existing Actor egress policy does not match the prepared revision") + actor := actorRef(atespace, name) + for range 5 { + existing, err := c.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) + if err != nil && status.Code(err) != codes.NotFound { + return err + } + access, err := c.cfg.EgressResolver.ResolveEgress(ctx, revision) + if err != nil { + return fmt.Errorf("resolve actor egress: %w", err) + } + policy, err := ActorEgressPolicy(atespace, access.Destinations, access.Credentials) + if err != nil { + return err + } + if existing == nil { + _, err = c.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{Actor: actor, EgressPolicy: policy}) + } else { + if proto.Equal(&ateapipb.EgressPolicy{Rules: existing.GetRules()}, &ateapipb.EgressPolicy{Rules: policy.GetRules()}) { + return nil + } + policy.Metadata = proto.CloneOf(existing.GetMetadata()) + _, err = c.UpdateActorEgressPolicy(ctx, &ateapipb.UpdateActorEgressPolicyRequest{Actor: actor, EgressPolicy: policy}) + } + if status.Code(err) != codes.AlreadyExists && status.Code(err) != codes.Aborted { + return err + } } - return nil + return status.Error(codes.Aborted, "actor egress policy changed during reconciliation") } diff --git a/go/core/internal/substrate/egress_test.go b/go/core/internal/substrate/egress_test.go index 79f4e2484b..7082a78668 100644 --- a/go/core/internal/substrate/egress_test.go +++ b/go/core/internal/substrate/egress_test.go @@ -10,28 +10,65 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + + "github.com/kagent-dev/kagent/go/core/internal/egress" ) -func TestEnsureActorEgressPolicyRetriesLostResponse(t *testing.T) { +type egressResolverFunc func(context.Context, string) (egress.Access, error) + +func (f egressResolverFunc) ResolveEgress(ctx context.Context, revision string) (egress.Access, error) { + return f(ctx, revision) +} + +var _ egress.Resolver = egressResolverFunc(nil) + +func TestEnsureActorEgressPolicyUpdatesCurrentGrants(t *testing.T) { fake := &egressPolicyFake{createErr: context.DeadlineExceeded} - client := &Client{ControlClient: fake} - policy := &ateapipb.EgressPolicy{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "default"}, - Rules: []*ateapipb.EgressRule{ - {Http: &ateapipb.HTTPRule{Hostnames: []string{"api.example.com"}}}, - {Https: &ateapipb.HTTPSRule{Hostnames: []string{"api.example.com"}}}, - }, - } - require.ErrorIs(t, client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", policy), context.DeadlineExceeded) + access := egress.Access{Destinations: []string{"https://api.example.com:443"}} + client := &Client{ControlClient: fake, cfg: Config{EgressResolver: egressResolverFunc(func(_ context.Context, revision string) (egress.Access, error) { + require.Equal(t, "pinned-revision", revision) + return access, nil + })}} + reconcile := func() error { return client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "pinned-revision") } + require.ErrorIs(t, reconcile(), context.DeadlineExceeded) require.NotNil(t, fake.policy, "the server committed before the response was lost") fake.createErr = nil - require.NoError(t, client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", policy)) - require.NoError(t, client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", policy)) + require.NoError(t, reconcile()) + require.NoError(t, reconcile()) require.Equal(t, 1, fake.created) + require.Zero(t, fake.updated) require.Equal(t, actorRef("team-a", "actor"), fake.actor) - require.ErrorContains(t, client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", &ateapipb.EgressPolicy{}), "does not match") + + access = egress.Access{} + require.NoError(t, reconcile()) + require.Empty(t, fake.policy.Rules, "removing the grant revokes the existing actor") + require.Equal(t, 1, fake.updated) + access = egress.Access{Destinations: []string{"https://api.example.com:443"}, Credentials: []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team-a/auth/token", Prefix: "Bearer "}}} + require.NoError(t, reconcile()) + require.Equal(t, "Bearer ", fake.policy.Rules[0].GetHttps().GetEffects().GetReplaceHeaders()[0].Prefix) + access.Credentials[0].URI = "ate-secret://k8s.io/default/team-a/rotated/token" + require.NoError(t, reconcile()) + require.Equal(t, access.Credentials[0].URI, fake.policy.Rules[0].GetHttps().GetEffects().GetReplaceHeaders()[0].CredentialUri) fake.getErr = status.Error(codes.Unavailable, "unavailable") - require.Equal(t, codes.Unavailable, status.Code(client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", policy))) + require.Equal(t, codes.Unavailable, status.Code(reconcile())) +} + +func TestEnsureActorEgressPolicyReResolvesAfterConflict(t *testing.T) { + fake := &egressPolicyFake{} + access := egress.Access{} + client := &Client{ControlClient: fake, cfg: Config{EgressResolver: egressResolverFunc(func(context.Context, string) (egress.Access, error) { return access, nil })}} + reconcile := func() error { return client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "revision") } + require.NoError(t, reconcile()) + access.Destinations = []string{"https://revoked.example.com:443"} + fake.beforeUpdate = func() { + // Another reconciliation committed revocation after this attempt read the + // backend version. Retrying must resolve current grants, not replay the allow. + access = egress.Access{} + fake.policy.Metadata.Version++ + } + require.NoError(t, reconcile()) + require.Empty(t, fake.policy.Rules) + require.Zero(t, fake.updated) } type egressPolicyFake struct { @@ -39,7 +76,8 @@ type egressPolicyFake struct { policy *ateapipb.EgressPolicy actor *ateapipb.ObjectRef createErr, getErr error - created int + created, updated int + beforeUpdate func() } func (f *egressPolicyFake) CreateActorEgressPolicy(_ context.Context, req *ateapipb.CreateActorEgressPolicyRequest, _ ...grpc.CallOption) (*ateapipb.EgressPolicy, error) { @@ -51,10 +89,30 @@ func (f *egressPolicyFake) CreateActorEgressPolicy(_ context.Context, req *ateap f.policy.Metadata.Uid = "policy-uid" f.policy.Metadata.Version = 1 f.created++ - return f.policy, f.createErr + return proto.CloneOf(f.policy), f.createErr } func (f *egressPolicyFake) GetActorEgressPolicy(_ context.Context, req *ateapipb.GetActorEgressPolicyRequest, _ ...grpc.CallOption) (*ateapipb.EgressPolicy, error) { f.actor = req.Actor - return f.policy, f.getErr + if f.getErr != nil { + return nil, f.getErr + } + if f.policy == nil { + return nil, status.Error(codes.NotFound, "missing") + } + return proto.CloneOf(f.policy), nil +} + +func (f *egressPolicyFake) UpdateActorEgressPolicy(_ context.Context, req *ateapipb.UpdateActorEgressPolicyRequest, _ ...grpc.CallOption) (*ateapipb.EgressPolicy, error) { + if f.beforeUpdate != nil { + f.beforeUpdate() + f.beforeUpdate = nil + } + if req.EgressPolicy.Metadata.Version != f.policy.Metadata.Version || req.EgressPolicy.Metadata.Uid != f.policy.Metadata.Uid { + return nil, status.Error(codes.Aborted, "version conflict") + } + f.policy = proto.CloneOf(req.EgressPolicy) + f.policy.Metadata.Version++ + f.updated++ + return proto.CloneOf(f.policy), nil } diff --git a/go/core/internal/substrate/lifecycle.go b/go/core/internal/substrate/lifecycle.go index 169647eded..f42c40a7d9 100644 --- a/go/core/internal/substrate/lifecycle.go +++ b/go/core/internal/substrate/lifecycle.go @@ -16,7 +16,7 @@ type LifecycleClient interface { GetActor(context.Context, string, string) (*ateapipb.Actor, error) CreateActor(context.Context, string, string, string, string) (*ateapipb.Actor, error) CreateActorFromTag(context.Context, string, string, string, string, string, string) (*ateapipb.Actor, error) - EnsureActorEgressPolicy(context.Context, string, string, *ateapipb.EgressPolicy) error + EnsureActorEgressPolicy(context.Context, string, string, string) error ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) SuspendActor(context.Context, string, string) (*ateapipb.Actor, error) DeleteActor(context.Context, string, string) error @@ -25,6 +25,7 @@ type LifecycleClient interface { var _ LifecycleClient = (*Client)(nil) type ActorBinding struct { + Revision string Atespace string Name string TemplateAtespace string @@ -40,9 +41,8 @@ type ActorSnapshot struct { // ActorCreation describes runtime startup, independent of the owning API. // Agent creation stays suspended; a standalone guest starts serving immediately. type ActorCreation struct { - EgressPolicy *ateapipb.EgressPolicy - Snapshot *ActorSnapshot - Resume bool + Snapshot *ActorSnapshot + Resume bool } type ActorTransition struct { @@ -75,8 +75,8 @@ func prepareActorTransition(ctx context.Context, actors LifecycleClient, binding return nil, fmt.Errorf("invalid actor lifecycle operation %s", operation) } if operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { - if creation == nil || creation.EgressPolicy == nil { - return nil, fmt.Errorf("actor creation requires an egress policy") + if creation == nil || binding.Revision == "" { + return nil, fmt.Errorf("actor creation requires a prepared revision") } if snapshot := creation.Snapshot; snapshot != nil && (snapshot.Tag.GetAtespace() == "" || snapshot.Tag.GetName() == "" || snapshot.URI == "" || snapshot.ContentScope != ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA) { return nil, fmt.Errorf("actor restoration requires a retained DATA snapshot") @@ -96,7 +96,7 @@ func prepareActorTransition(ctx context.Context, actors LifecycleClient, binding if operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE && !reconcile { return nil, fmt.Errorf("refuse to adopt existing Actor %s/%s while creation is still pending", binding.Atespace, binding.Name) } - if !binding.matches(actor) { + if !binding.Matches(actor) { return nil, fmt.Errorf("actor %s/%s identity or template changed", binding.Atespace, binding.Name) } if operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME || operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND { @@ -144,7 +144,7 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio } } if err == nil { - err = actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, creation.EgressPolicy) + err = actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, binding.Revision) } if err == nil && creation.Resume && actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_RUNNING { actor, err = actors.ResumeActor(ctx, binding.Atespace, binding.Name) @@ -153,6 +153,9 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio } } case apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME: + if err := actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, binding.Revision); err != nil { + return err + } actor, err = actors.ResumeActor(ctx, binding.Atespace, binding.Name) if err == nil { err = transition.checkResult(actor, ateapipb.ActorState_ACTOR_STATE_RUNNING) @@ -192,13 +195,14 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio return nil } -func (b ActorBinding) matches(actor *ateapipb.Actor) bool { +// Matches checks that the actor belongs to the expected runtime and template. +func (b ActorBinding) Matches(actor *ateapipb.Actor) bool { return actor.GetMetadata().GetName() == b.Name && actor.GetMetadata().GetAtespace() == b.Atespace && actor.GetMetadata().GetUid() != "" && actor.GetActorTemplate().GetAtespace() == b.TemplateAtespace && actor.GetActorTemplate().GetName() == b.TemplateName } func (t *ActorTransition) checkResult(actor *ateapipb.Actor, state ateapipb.ActorState) error { - if !t.binding.matches(actor) || actor.GetStatus().GetState() != state || (t.actor != nil && actor.GetMetadata().GetUid() != t.actor.GetMetadata().GetUid()) { + if !t.binding.Matches(actor) || actor.GetStatus().GetState() != state || (t.actor != nil && actor.GetMetadata().GetUid() != t.actor.GetMetadata().GetUid()) { return fmt.Errorf("actor returned unexpected identity or state") } return nil diff --git a/go/core/internal/substrate/lifecycle_test.go b/go/core/internal/substrate/lifecycle_test.go index 6abfa64346..a4e1301b25 100644 --- a/go/core/internal/substrate/lifecycle_test.go +++ b/go/core/internal/substrate/lifecycle_test.go @@ -41,7 +41,7 @@ func (a *transitionActors) CreateActor(_ context.Context, space, name, templateS return proto.CloneOf(a.actor), nil } -func (a *transitionActors) EnsureActorEgressPolicy(context.Context, string, string, *ateapipb.EgressPolicy) error { +func (a *transitionActors) EnsureActorEgressPolicy(context.Context, string, string, string) error { a.calls = append(a.calls, "policy") return a.policyErr } @@ -88,9 +88,9 @@ func TestActorCreationStartupPolicy(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { actors := &transitionActors{policyErr: test.policyErr, resumeUID: test.resumeUID} - binding := ActorBinding{Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} + binding := ActorBinding{Revision: "revision", Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} transition, err := PrepareActorTransition(t.Context(), actors, binding, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, - &ActorCreation{EgressPolicy: &ateapipb.EgressPolicy{}, Resume: test.resume}) + &ActorCreation{Resume: test.resume}) require.NoError(t, err) require.Empty(t, actors.calls, "preparation must not mutate the backend") err = ApplyActorTransition(t.Context(), actors, transition) @@ -118,8 +118,8 @@ func TestActorRetryContinuesExistingCreation(t *testing.T) { Status: &ateapipb.ActorStatus{State: state}, }} transition, err := PrepareActorRetry(t.Context(), actors, - ActorBinding{Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"}, - apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, &ActorCreation{EgressPolicy: &ateapipb.EgressPolicy{}, Resume: true}) + ActorBinding{Revision: "revision", Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"}, + apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, &ActorCreation{Resume: true}) require.NoError(t, err) require.NoError(t, ApplyActorTransition(t.Context(), actors, transition)) require.Equal(t, "original", actors.actor.Metadata.Uid) @@ -152,8 +152,8 @@ func TestActorTransitionAdmission(t *testing.T) { if test.mutate != nil { test.mutate(actors.actor) } - binding := ActorBinding{Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} - _, err = PrepareActorTransition(t.Context(), actors, binding, test.operation, &ActorCreation{EgressPolicy: &ateapipb.EgressPolicy{}}) + binding := ActorBinding{Revision: "revision", Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} + _, err = PrepareActorTransition(t.Context(), actors, binding, test.operation, &ActorCreation{}) require.Error(t, err) require.Empty(t, actors.calls) }) @@ -166,7 +166,7 @@ func TestActorDeleteStopsRunningCompute(t *testing.T) { require.NoError(t, err) actors.actor.Status.State = ateapipb.ActorState_ACTOR_STATE_RUNNING actors.calls = nil - binding := ActorBinding{Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} + binding := ActorBinding{Revision: "revision", Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} transition, err := PrepareActorTransition(t.Context(), actors, binding, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE, nil) require.NoError(t, err) require.NoError(t, ApplyActorTransition(t.Context(), actors, transition)) @@ -191,7 +191,7 @@ func TestActorSuspendRequiresDataSnapshot(t *testing.T) { actors := &transitionActors{suspendSnapshot: test.snapshot} _, err := actors.CreateActor(t.Context(), "team", "instance", "team", "revision") require.NoError(t, err) - transition, err := PrepareActorTransition(t.Context(), actors, ActorBinding{Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"}, apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND, nil) + transition, err := PrepareActorTransition(t.Context(), actors, ActorBinding{Revision: "revision", Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"}, apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND, nil) require.NoError(t, err) require.ErrorContains(t, ApplyActorTransition(t.Context(), actors, transition), "external DATA snapshot") }) diff --git a/go/core/internal/substrate/sandbox.go b/go/core/internal/substrate/sandbox.go index 6dd6e05f98..ebb2497d61 100644 --- a/go/core/internal/substrate/sandbox.go +++ b/go/core/internal/substrate/sandbox.go @@ -5,17 +5,14 @@ import ( "encoding/hex" "encoding/json" "fmt" - "slices" "strings" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/image" ) @@ -35,12 +32,11 @@ type sandboxInputs struct { Spec v1alpha3.SandboxTemplateSpec Class atev1alpha1.SandboxClass Policy SandboxPolicy - Egress egress.Access } // SandboxActorTemplate compiles standalone inputs without an Agent Card or Harness. // The image's entrypoint is replaced by the pinned, platform-managed guest. -func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1.SandboxClass, policy SandboxPolicy, access egress.Access) (*ateapipb.ActorTemplate, string, json.RawMessage, error) { +func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1.SandboxClass, policy SandboxPolicy) (*ateapipb.ActorTemplate, string, json.RawMessage, error) { if !image.IsDigestPinned(policy.GuestImage) || !image.IsDigestPinned(template.Spec.Workload.Image) { return nil, "", nil, fmt.Errorf("sandbox workload and guest images must be pinned by sha256 digest") } @@ -73,13 +69,9 @@ func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1. } // Namespace-scoped placement changes immutable template contents, so it // needs a new revision even when the operator's inputs stay the same. - access, err = egress.CanonicalAccess(access.Destinations, access.Credentials) - if err != nil { - return nil, "", nil, err - } spec := template.Spec.DeepCopy() - slices.SortFunc(spec.EgressPolicyRefs, func(a, b corev1.LocalObjectReference) int { return strings.Compare(a.Name, b.Name) }) - inputs := sandboxInputs{Kind: "sandbox", Version: 3, Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), Spec: *spec, Class: class, Policy: policy, Egress: access} + spec.EgressPolicyRefs = nil + inputs := sandboxInputs{Kind: "sandbox", Version: 4, Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), Spec: *spec, Class: class, Policy: policy} snapshot, err := json.Marshal(inputs) if err != nil { return nil, "", nil, fmt.Errorf("encode sandbox inputs: %w", err) diff --git a/go/core/internal/substrate/sandbox_test.go b/go/core/internal/substrate/sandbox_test.go index e2d74e854d..0502a59cb6 100644 --- a/go/core/internal/substrate/sandbox_test.go +++ b/go/core/internal/substrate/sandbox_test.go @@ -11,7 +11,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" ) func TestSandboxPreparationPinsCompleteInputs(t *testing.T) { @@ -20,7 +19,7 @@ func TestSandboxPreparationPinsCompleteInputs(t *testing.T) { Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "s3://snapshots/"}}, }} policy := SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"} - actor, digest, snapshot, err := SandboxActorTemplate(template, "", policy, egress.Access{}) + actor, digest, snapshot, err := SandboxActorTemplate(template, "", policy) require.NoError(t, err) require.NotEqual(t, "5bf14bc98818a54bf4e59e5aca83de9bde5df22e9efa206b65e1389069a188c3", digest, "namespace-scoped placement must create a fresh immutable ActorTemplate") @@ -33,9 +32,13 @@ func TestSandboxPreparationPinsCompleteInputs(t *testing.T) { trust := actor.Volumes[2].GetSystemInfo().GetDataSources()[0].GetTrustBundle() require.Equal(t, []string{"egress-mitm.ate.dev"}, trust.GetNames()) require.Equal(t, "trust-bundle.pem", trust.GetPath()) - _, same, _, err := SandboxActorTemplate(template, atev1alpha1.SandboxClassGvisor, policy, egress.Access{}) + _, same, _, err := SandboxActorTemplate(template, atev1alpha1.SandboxClassGvisor, policy) require.NoError(t, err) require.Equal(t, digest, same) + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "live"}} + _, unchanged, _, err := SandboxActorTemplate(template, "", policy) + require.NoError(t, err) + require.Equal(t, digest, unchanged, "policy references are live authorization") for _, change := range []struct { name string mutate func(*v1alpha3.SandboxTemplate, *SandboxPolicy) @@ -51,13 +54,13 @@ func TestSandboxPreparationPinsCompleteInputs(t *testing.T) { t.Run(change.name, func(t *testing.T) { input, settings := template.DeepCopy(), policy change.mutate(input, &settings) - _, changed, _, err := SandboxActorTemplate(input, change.class, settings, egress.Access{}) + _, changed, _, err := SandboxActorTemplate(input, change.class, settings) require.NoError(t, err) require.NotEqual(t, digest, changed) }) } template.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "SSL_CERT_FILE", Value: "/untrusted"}} - actor, _, _, err = SandboxActorTemplate(template, "", policy, egress.Access{}) + actor, _, _, err = SandboxActorTemplate(template, "", policy) require.ErrorContains(t, err, "reserved") require.Nil(t, actor) } diff --git a/go/core/internal/translator/byo/compiler.go b/go/core/internal/translator/byo/compiler.go index c8b173811f..a4a24485bd 100644 --- a/go/core/internal/translator/byo/compiler.go +++ b/go/core/internal/translator/byo/compiler.go @@ -70,7 +70,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build revision provenance: %w", err) } - environment, credentials, err := translator.CompileCredentials(input, compiled.Models, environment) + environment, credentials, passthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment) if err != nil { return nil, err } @@ -84,7 +84,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) Environment: environment, ConfigJSON: configJSON, AgentCard: card, SkillImages: compiled.SkillImages, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - Credentials: credentials, Provenance: provenance, EgressDestinations: slices.Compact(compiled.Egress), + PassthroughOrigins: passthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: slices.Compact(compiled.Egress), }}, nil } diff --git a/go/core/internal/translator/claude/compiler.go b/go/core/internal/translator/claude/compiler.go index 59714932a7..2bb2df6375 100644 --- a/go/core/internal/translator/claude/compiler.go +++ b/go/core/internal/translator/claude/compiler.go @@ -110,7 +110,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build Claude revision provenance: %w", err) } - environment, credentials, err := translator.CompileCredentials(input, nil, environment) + environment, credentials, passthroughOrigins, err := translator.CompileCredentials(input, nil, environment) if err != nil { return nil, err } @@ -127,9 +127,9 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) SkillImages: skills.Images, Environment: environment, ConfigJSON: configJSON, AgentCard: card, - WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, - SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - Credentials: credentials, Provenance: provenance, EgressDestinations: egress, + WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, + SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, + PassthroughOrigins: passthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: egress, }, Warnings: mcp.warnings, }, nil diff --git a/go/core/internal/translator/codex/compiler.go b/go/core/internal/translator/codex/compiler.go index 9323d220fe..22328086c8 100644 --- a/go/core/internal/translator/codex/compiler.go +++ b/go/core/internal/translator/codex/compiler.go @@ -130,7 +130,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build Codex revision provenance: %w", err) } - environment, credentials, err := translator.CompileCredentials(input, nil, environment) + environment, credentials, passthroughOrigins, err := translator.CompileCredentials(input, nil, environment) if err != nil { return nil, err } @@ -146,7 +146,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) SkillImages: skills.Images, Environment: environment, ConfigJSON: configJSON, AgentCard: card, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - Credentials: credentials, Provenance: provenance, EgressDestinations: egress, + PassthroughOrigins: passthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: egress, }, Warnings: mcp.warnings, }, nil diff --git a/go/core/internal/translator/compiler.go b/go/core/internal/translator/compiler.go index 8fb0a6517c..ad42d6ca90 100644 --- a/go/core/internal/translator/compiler.go +++ b/go/core/internal/translator/compiler.go @@ -5,11 +5,8 @@ import ( "encoding/json" "fmt" "maps" - "slices" - "strings" "istio.io/istio/pkg/kube/krt" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" @@ -134,7 +131,7 @@ func (c *Compiler) CompileAgent(ctx context.Context, agent *v1alpha3.Agent) (*Co } result.AgentUID = string(agent.UID) spec := agent.Spec.DeepCopy() - slices.SortFunc(spec.EgressPolicyRefs, func(a, b corev1.LocalObjectReference) int { return strings.Compare(a.Name, b.Name) }) + spec.EgressPolicyRefs = nil result.Provenance, err = json.Marshal(struct { AgentName string `json:"agentName"` AgentUID string `json:"agentUID"` @@ -176,14 +173,10 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agent *v1alpha3.Age if err != nil { return nil, err } - // Merge again at the shared boundary so every compiler preserves explicit - // grants. Built-in compilers also consume these bindings when validating - // model passthrough and inferred credentials. - combined, err := egress.CanonicalAccess(append(result.EgressDestinations, access.Destinations...), append(result.Credentials, access.Credentials...)) - if err != nil { + // Explicit grants are live authorization, not immutable runtime inputs. + if _, err := egress.MergeAccess(egress.Access{Destinations: result.EgressDestinations, Credentials: result.Credentials}, access, result.PassthroughOrigins); err != nil { return nil, NewValidationError("%v", err) } - result.EgressDestinations, result.Credentials = combined.Destinations, combined.Credentials workerKey := types.NamespacedName{Namespace: harness.Namespace, Name: harness.Spec.Substrate.WorkerPoolRef.Name} workerPool := krt.FetchOne(c.ctx, c.collections.WorkerPools, krt.FilterObjectName(workerKey)) if workerPool == nil { diff --git a/go/core/internal/translator/compiler_test.go b/go/core/internal/translator/compiler_test.go index d2f37f47f5..3115182abf 100644 --- a/go/core/internal/translator/compiler_test.go +++ b/go/core/internal/translator/compiler_test.go @@ -696,7 +696,7 @@ func TestCompileAgentSharedADKConfig(t *testing.T) { require.Equal(t, "research carefully", config.Agent.SubAgents[0].Instruction) require.Equal(t, []string{"lookup"}, config.Agent.SubAgents[0].HttpTools[0].Tools) require.Equal(t, "review", config.Agent.SubAgents[0].AgentPlugins.Skills[0].Name) - require.Contains(t, revision.EgressDestinations, "https://tickets.example.com:443") + require.NotContains(t, revision.EgressDestinations, "https://tickets.example.com:443") require.Contains(t, revision.EgressDestinations, "https://search.example.com:443") require.NotContains(t, revision.EgressDestinations, "https://ghcr.io:443") require.Equal(t, []translator.SkillImage{translator.SkillImage(child.Spec.Skills[0].Source.OCI)}, revision.SkillImages) diff --git a/go/core/internal/translator/credentials.go b/go/core/internal/translator/credentials.go index e5a80006e1..a5437343c8 100644 --- a/go/core/internal/translator/credentials.go +++ b/go/core/internal/translator/credentials.go @@ -20,9 +20,9 @@ const CredentialPlaceholder = "kagent-credential-injected" // CompileCredentials replaces secret-backed environment values with inert SDK // placeholders and compiles their destination-scoped gateway bindings. Models // outside the agent tree (such as memory embeddings) are supplied separately. -func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, environment []corev1.EnvVar) ([]corev1.EnvVar, []egress.Credential, error) { - bindings := make([]egress.Credential, 0, len(input.Egress.Credentials)) - bindings = append(bindings, input.Egress.Credentials...) +func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, environment []corev1.EnvVar) ([]corev1.EnvVar, []egress.Credential, []string, error) { + var bindings []egress.Credential + var passthroughOrigins []string boundModels := map[string]bool{} boundMCP := map[string]bool{} bind := func(rawURL, header, prefix, namespace, name, key string) error { @@ -58,7 +58,7 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, return nil } if err := visit(input.Root); err != nil { - return nil, nil, err + return nil, nil, nil, err } for _, resolved := range models { model := resolved.Config @@ -83,13 +83,13 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, } } if err := bind(endpoint, header, prefix, model.Namespace, model.Spec.APIKeySecret, key); err != nil { - return nil, nil, err + return nil, nil, nil, err } boundModels[name+"\x00"+model.Spec.APIKeySecret+"\x00"+key] = true } bindings, err := egress.CanonicalCredentials(bindings) if err != nil { - return nil, nil, NewValidationError("%v", err) + return nil, nil, nil, NewValidationError("%v", err) } for _, resolved := range models { if !resolved.Config.Spec.APIKeyPassthrough { @@ -98,18 +98,22 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, _, endpoint, _, _ := modelCredentialTarget(resolved) u, err := url.Parse(endpoint) if err != nil { - return nil, nil, NewValidationError("invalid passthrough credential destination") + return nil, nil, nil, NewValidationError("invalid passthrough credential destination") } origin, err := egress.ParseOrigin(egress.Origin(u)) if err != nil { - return nil, nil, NewValidationError("invalid passthrough credential destination: %v", err) - } - for _, binding := range bindings { - if binding.Origin == origin.String() { - return nil, nil, NewValidationError("destination %q cannot combine caller-token passthrough with gateway credentials", origin.String()) - } + return nil, nil, nil, NewValidationError("invalid passthrough credential destination: %v", err) } + passthroughOrigins = append(passthroughOrigins, origin.String()) + } + var destinations []string + for _, binding := range bindings { + destinations = append(destinations, binding.Origin) } + if _, err := egress.MergeAccess(egress.Access{Destinations: destinations, Credentials: bindings}, input.Egress, passthroughOrigins); err != nil { + return nil, nil, nil, NewValidationError("%v", err) + } + result := append([]corev1.EnvVar(nil), environment...) for i, variable := range result { if variable.ValueFrom == nil { @@ -118,11 +122,11 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, ref := variable.ValueFrom.SecretKeyRef isMCP := strings.HasPrefix(variable.Name, "KAGENT_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CODEX_MCP_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CLAUDE_MCP_CREDENTIAL_") if ref == nil || (!boundModels[variable.Name+"\x00"+ref.Name+"\x00"+ref.Key] && (!isMCP || !boundMCP[ref.Name+"\x00"+ref.Key])) { - return nil, nil, NewValidationError("environment credential %q cannot use gateway header injection; local signing and arbitrary secret environment variables are unsupported", variable.Name) + return nil, nil, nil, NewValidationError("environment credential %q cannot use gateway header injection; local signing and arbitrary secret environment variables are unsupported", variable.Name) } result[i].Value, result[i].ValueFrom = CredentialPlaceholder, nil } - return result, bindings, nil + return result, bindings, passthroughOrigins, nil } func modelCredentialTarget(resolved *ResolvedModelConfig) (name, endpoint, header, prefix string) { diff --git a/go/core/internal/translator/credentials_test.go b/go/core/internal/translator/credentials_test.go index fb96a7fff7..1cf5d36ca6 100644 --- a/go/core/internal/translator/credentials_test.go +++ b/go/core/internal/translator/credentials_test.go @@ -35,7 +35,7 @@ func TestCompileCredentialDestinations(t *testing.T) { } input := credentialInput(test.spec) environment := []corev1.EnvVar{credentialEnv(test.env, "auth", test.spec.APIKeySecretKey)} - got, bindings, err := CompileCredentials(input, nil, environment) + got, bindings, _, err := CompileCredentials(input, nil, environment) require.NoError(t, err) require.Equal(t, CredentialPlaceholder, got[0].Value) require.Nil(t, got[0].ValueFrom) @@ -55,30 +55,33 @@ func TestCompileCredentialsRejectsConflictingSharedModels(t *testing.T) { spec.APIKeySecret = "child" input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(spec).Root}} // The ADK environment is deduplicated, but every model still needs its own binding. - _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) + environment, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) require.ErrorContains(t, err, "conflicting credentials") + require.Nil(t, environment) input.Root.Shared[0].Agent.ResolvedModelConfig.Config.Spec.OpenAI = &v1alpha3.OpenAIConfig{BaseURL: "https://child.example.com/v1"} - _, bindings, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) + _, bindings, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) require.NoError(t, err) require.Len(t, bindings, 2) } func TestCompileCredentialsRejectsLocalSecrets(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{}) - _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("AWS_SECRET_ACCESS_KEY", "auth", "token")}) + environment, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("AWS_SECRET_ACCESS_KEY", "auth", "token")}) require.ErrorContains(t, err, "cannot use gateway header injection") + require.Nil(t, environment) } func TestCompileCredentialsPreservesPassthrough(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: true}) - _, bindings, err := CompileCredentials(input, nil, nil) + _, bindings, _, err := CompileCredentials(input, nil, nil) require.NoError(t, err) require.Empty(t, bindings) input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(v1alpha3.ModelConfigSpec{ Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "auth", APIKeySecretKey: "token", }).Root}} - _, _, err = CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}) + environment, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}) require.ErrorContains(t, err, "cannot combine caller-token passthrough") + require.Nil(t, environment) } func credentialInput(spec v1alpha3.ModelConfigSpec) *HarnessInput { @@ -174,10 +177,13 @@ func TestModelCredentialTargetOllamaCloud(t *testing.T) { func TestPassthroughCredentialConflictUsesCanonicalOrigin(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: true, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://API.OPENAI.COM.:0443/v1"}}) + input.Egress.Destinations = []string{"https://api.openai.com"} input.Egress.Credentials = []egress.Credential{{Origin: "https://api.openai.com", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token"}} - _, _, err := CompileCredentials(input, nil, nil) + environment, _, _, err := CompileCredentials(input, nil, nil) require.ErrorContains(t, err, "cannot combine caller-token passthrough") + require.Nil(t, environment) input.Root.ResolvedModelConfig.Config.Spec.OpenAI.BaseURL = "https://api.openai.com:8443/v1" - _, _, err = CompileCredentials(input, nil, nil) + environment, _, _, err = CompileCredentials(input, nil, nil) + require.Empty(t, environment) require.NoError(t, err, "a different port is a distinct credential scope") } diff --git a/go/core/internal/translator/egress_test.go b/go/core/internal/translator/egress_test.go index 5139d9d38d..e8408c02d6 100644 --- a/go/core/internal/translator/egress_test.go +++ b/go/core/internal/translator/egress_test.go @@ -50,7 +50,7 @@ func TestEgressPolicyCredentialPrefixes(t *testing.T) { } } -func TestAgentEgressPoliciesPinGrantsAcrossHarnesses(t *testing.T) { +func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { for _, kind := range []string{"kagent", "codex", "claude", "byo"} { t.Run(kind, func(t *testing.T) { policy, secret := egressPolicy() @@ -86,9 +86,11 @@ func TestAgentEgressPoliciesPinGrantsAcrossHarnesses(t *testing.T) { return result } result := compile() - require.Contains(t, result.EgressDestinations, "https://tickets.example.com:443") - require.Greater(t, len(result.EgressDestinations), 1, "explicit policies preserve inferred destinations") - require.Equal(t, "https://tickets.example.com:443", result.Credentials[len(result.Credentials)-1].Origin) + require.NotContains(t, result.EgressDestinations, "https://tickets.example.com:443") + require.NotEmpty(t, result.EgressDestinations, "preserve inferred integration access") + for _, binding := range result.Credentials { + require.NotEqual(t, "https://tickets.example.com:443", binding.Origin) + } encoded, err := json.Marshal(result) require.NoError(t, err) require.NotContains(t, string(encoded), string(secret.Data["token"])) @@ -101,7 +103,12 @@ func TestAgentEgressPoliciesPinGrantsAcrossHarnesses(t *testing.T) { policy.Spec.Destinations = append(policy.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://registry.npmjs.org"}) updated, err := compile().Digest() require.NoError(t, err) - require.NotEqual(t, original, updated) + require.Equal(t, original, updated) + agent.Spec.EgressPolicyRefs = nil + unreferenced, err := compile().Digest() + require.NoError(t, err) + require.Equal(t, original, unreferenced) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} require.NotContains(t, result.EgressDestinations, "https://registry.npmjs.org:443", "previous revision remains pinned") // Explicit policy credentials cannot silently replace inferred model credentials. policy.Spec.Destinations[0].Origin = "https://api.openai.com" diff --git a/go/core/internal/translator/kagent/compiler.go b/go/core/internal/translator/kagent/compiler.go index 3bcb0b7401..aaba414f4a 100644 --- a/go/core/internal/translator/kagent/compiler.go +++ b/go/core/internal/translator/kagent/compiler.go @@ -61,7 +61,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build revision provenance: %w", err) } - environment, credentials, err := translator.CompileCredentials(input, compiled.Models, environment) + environment, credentials, passthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment) if err != nil { return nil, err } @@ -75,6 +75,6 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) Environment: environment, ConfigJSON: configJSON, AgentCard: card, SkillImages: compiled.SkillImages, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - Credentials: credentials, Provenance: provenance, EgressDestinations: compiled.Egress, + PassthroughOrigins: passthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: compiled.Egress, }}, nil } diff --git a/go/core/internal/translator/revision.go b/go/core/internal/translator/revision.go index 4772cb3bf9..134c197cfb 100644 --- a/go/core/internal/translator/revision.go +++ b/go/core/internal/translator/revision.go @@ -66,6 +66,8 @@ type Revision struct { Provenance json.RawMessage // Credentials contains references resolved by the egress gateway. Credentials []egress.Credential + // PassthroughOrigins cannot receive gateway credentials, including live grants. + PassthroughOrigins []string // EgressDestinations is the HTTP(S) origin allowlist required by this revision. EgressDestinations []string } @@ -105,6 +107,7 @@ func (r *Revision) Digest() (RevisionID, error) { SnapshotLocation string `json:"snapshotLocation"` Provenance json.RawMessage `json:"provenance"` Credentials []egress.Credential `json:"credentials,omitempty"` + PassthroughOrigins []string `json:"passthroughOrigins,omitempty"` EgressDestinations []string `json:"egressDestinations"` SandboxClass atev1alpha1.SandboxClass `json:"sandboxClass"` @@ -116,7 +119,7 @@ func (r *Revision) Digest() (RevisionID, error) { Image: r.Image, Command: r.Command, Args: r.Args, Environment: r.Environment, ConfigJSON: r.ConfigJSON, SkillImages: r.SkillImages, WorkerPoolName: r.WorkerPoolName, SnapshotLocation: r.SnapshotLocation, Provenance: r.Provenance, - Credentials: r.Credentials, EgressDestinations: r.EgressDestinations, + PassthroughOrigins: r.PassthroughOrigins, Credentials: r.Credentials, EgressDestinations: r.EgressDestinations, SandboxClass: sandboxClass, WorkerPoolNamespace: r.Namespace, }) if err != nil { diff --git a/go/core/pkg/app/app.go b/go/core/pkg/app/app.go index 13bfd5fe13..08ac011ad9 100644 --- a/go/core/pkg/app/app.go +++ b/go/core/pkg/app/app.go @@ -276,7 +276,9 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er if err != nil { return err } + egressResolver := controller.NewEgressResolver(runtime, store, watchNamespaces) actors, err := substrate.Dial(ctx, substrate.Config{ + EgressResolver: egressResolver, AteAPIEndpoint: cfg.GetSubstrate().GetApiEndpoint(), CAFile: cfg.GetSubstrate().GetCaFile(), ClientCertFile: cfg.GetSubstrate().GetClientCertFile(), @@ -286,6 +288,9 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er return err } defer actors.Close() + if err := manager.Add(controller.NewEgressReconciler(egressResolver, actors)); err != nil { + return fmt.Errorf("add egress reconciler: %w", err) + } reconciler, err := controller.NewReconciler(kubeConfig, runtime.Collections, store, actors) if err != nil { return err diff --git a/go/core/pkg/migrations/core/000001_initial.sql b/go/core/pkg/migrations/core/000001_initial.sql index d04ebba522..86d74923f8 100644 --- a/go/core/pkg/migrations/core/000001_initial.sql +++ b/go/core/pkg/migrations/core/000001_initial.sql @@ -33,6 +33,7 @@ CREATE TABLE runtime_revision ( namespace TEXT NOT NULL, source_snapshot JSONB NOT NULL, egress_destinations TEXT[] NOT NULL DEFAULT '{}', + passthrough_origins TEXT[] NOT NULL DEFAULT '{}', credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(credentials) = 'array'), actor_template_atespace TEXT CONSTRAINT runtime_revision_actor_template_namespace_not_null NOT NULL, actor_template_name TEXT NOT NULL, diff --git a/helm/kagent-crds/templates/api.kagent.dev_agents.yaml b/helm/kagent-crds/templates/api.kagent.dev_agents.yaml index 91ac641434..09a81343e1 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_agents.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_agents.yaml @@ -56,7 +56,8 @@ spec: description: |- EgressPolicyRefs grants additional access to the whole runtime, including shared subagents. Policies resolve in this Agent's namespace. Integration - access inferred from models and tools is always included. + access inferred from models and tools is always included. Changes update + existing Sessions without a new invocation or runtime restart. items: description: |- LocalObjectReference contains enough information to let you locate the diff --git a/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml b/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml index ba4d64bdbb..6e2e978cc0 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml @@ -21,7 +21,7 @@ spec: openAPIV3Schema: description: |- EgressPolicy is a reusable grant of additional network access. Workloads in its - namespace select it explicitly; changing it affects newly prepared revisions. + namespace select it explicitly; changes update their existing runtime Actors. properties: apiVersion: description: |- diff --git a/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml b/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml index 5ee69f5757..b49df3f122 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml @@ -54,7 +54,7 @@ spec: egressPolicyRefs: description: |- EgressPolicyRefs grants additional access in this template's namespace. - Each sandbox retains the access pinned by its prepared revision. + Changes update the access of existing sandboxes without restarting them. items: description: |- LocalObjectReference contains enough information to let you locate the From 57adcf86ea3ee0e0254bb4bba4fbd728c46abec9 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 13:30:01 +0000 Subject: [PATCH 10/43] fix: add live egress storage through a new migration Signed-off-by: Eitan Yarmush --- go/core/internal/translator/compiler.go | 1 + go/core/pkg/migrations/core/000001_initial.sql | 1 - .../core/000002_runtime_passthrough_origins.sql | 17 +++++++++++++++++ go/core/pkg/migrations/runner_test.go | 6 ++++++ 4 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 go/core/pkg/migrations/core/000002_runtime_passthrough_origins.sql diff --git a/go/core/internal/translator/compiler.go b/go/core/internal/translator/compiler.go index ad42d6ca90..700007e932 100644 --- a/go/core/internal/translator/compiler.go +++ b/go/core/internal/translator/compiler.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "maps" + "slices" "istio.io/istio/pkg/kube/krt" "k8s.io/apimachinery/pkg/types" diff --git a/go/core/pkg/migrations/core/000001_initial.sql b/go/core/pkg/migrations/core/000001_initial.sql index 86d74923f8..d04ebba522 100644 --- a/go/core/pkg/migrations/core/000001_initial.sql +++ b/go/core/pkg/migrations/core/000001_initial.sql @@ -33,7 +33,6 @@ CREATE TABLE runtime_revision ( namespace TEXT NOT NULL, source_snapshot JSONB NOT NULL, egress_destinations TEXT[] NOT NULL DEFAULT '{}', - passthrough_origins TEXT[] NOT NULL DEFAULT '{}', credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(credentials) = 'array'), actor_template_atespace TEXT CONSTRAINT runtime_revision_actor_template_namespace_not_null NOT NULL, actor_template_name TEXT NOT NULL, diff --git a/go/core/pkg/migrations/core/000002_runtime_passthrough_origins.sql b/go/core/pkg/migrations/core/000002_runtime_passthrough_origins.sql new file mode 100644 index 0000000000..8e6e529642 --- /dev/null +++ b/go/core/pkg/migrations/core/000002_runtime_passthrough_origins.sql @@ -0,0 +1,17 @@ +-- +goose Up + +ALTER TABLE runtime_revision ADD COLUMN passthrough_origins TEXT[] NOT NULL DEFAULT '{}'; + +-- PostgreSQL expands r.* when the view is created. +DROP VIEW agent_runtime_revision; +CREATE VIEW agent_runtime_revision AS + SELECT r.*, a.agent_name, a.agent_uid, a.agent_card + FROM runtime_revision r JOIN agent_revision a USING (revision) WHERE r.kind = 'agent'; + +-- +goose Down + +DROP VIEW agent_runtime_revision; +ALTER TABLE runtime_revision DROP COLUMN passthrough_origins; +CREATE VIEW agent_runtime_revision AS + SELECT r.*, a.agent_name, a.agent_uid, a.agent_card + FROM runtime_revision r JOIN agent_revision a USING (revision) WHERE r.kind = 'agent'; diff --git a/go/core/pkg/migrations/runner_test.go b/go/core/pkg/migrations/runner_test.go index 6348ccea19..efcdf0610e 100644 --- a/go/core/pkg/migrations/runner_test.go +++ b/go/core/pkg/migrations/runner_test.go @@ -319,10 +319,16 @@ func TestBuiltinMigrationsRoundTrip(t *testing.T) { } for _, source := range sources { expected := []int64{0, 1} + if source.TrackingTable == "schema_migrations" { + expected = append(expected, 2) + } if versions := testVersions(t, dsn, source.TrackingTable); !slices.Equal(versions, expected) { t.Fatalf("%s versions = %v, want %v", source.Name, versions, expected) } } + if !testColumnExists(t, dsn, "runtime_revision", "passthrough_origins") || !testColumnExists(t, dsn, "agent_runtime_revision", "passthrough_origins") { + t.Fatal("runtime and agent revisions must expose passthrough origins") + } if !testColumnExists(t, dsn, "session", "deletion_reason") { t.Fatal("session deletion reason is missing") } From ddf5d7f19ca9d7d1d4790299e0f10fb2b2ef6b05 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 13:30:01 +0000 Subject: [PATCH 11/43] test: exercise live egress through a shared skill on every harness Signed-off-by: Eitan Yarmush --- go/core/test/e2e/README.md | 9 + go/core/test/e2e/egress_test.go | 564 ++++++++++++------ go/core/test/e2e/harness_test.go | 11 + go/core/test/e2e/interaction_test.go | 2 +- .../test/e2e/mocks/invoke_egress_skill.json | 409 +++++++++++++ go/core/test/e2e/skills_test.go | 10 +- 6 files changed, 823 insertions(+), 182 deletions(-) create mode 100644 go/core/test/e2e/mocks/invoke_egress_skill.json diff --git a/go/core/test/e2e/README.md b/go/core/test/e2e/README.md index f59a3e04e4..451bf31af4 100644 --- a/go/core/test/e2e/README.md +++ b/go/core/test/e2e/README.md @@ -238,6 +238,15 @@ selection quality. The registry must accept pushes from the test runner and be reachable by Substrate workers through their existing registry rewrite. Private-registry credentials are not required. +`TestSessionEgressPolicyUpdatesLiveActors` loads the same OCI skill and runs +the same HTTP probe script through each harness's shell tool. During a single +active invocation, it adds a policy reference and revokes the destination, +checking actual script output and destination requests without replacing the +actor. It also checks that deleting a policy while suspended revokes access +after resume and in a checkpoint fork. Runtime configuration stays on the same +prepared revision. `TestSandboxEgressPolicyUpdatesLiveActor` exercises live +grant, revocation, and resume through the standalone sandbox process API. + `TestSessionHTTPInteraction` discovers an Agent's Agent Card, invokes the advertised JSON-RPC interface, streams a second turn, and checks task persistence across HTTP and gRPC. `TestSessionHTTPResubscribeAndCancel` diff --git a/go/core/test/e2e/egress_test.go b/go/core/test/e2e/egress_test.go index a2ecfd3543..cffa29e3f9 100644 --- a/go/core/test/e2e/egress_test.go +++ b/go/core/test/e2e/egress_test.go @@ -4,17 +4,24 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "net" "net/http" "net/http/httptest" "net/http/httputil" "net/url" + "strings" + "sync" "sync/atomic" "testing" "time" a2atype "github.com/a2aproject/a2a-go/v2/a2a" + a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" + "github.com/a2aproject/a2a-go/v2/a2apb/v1/pbconv" + guestpb "github.com/agent-substrate/env/proto/ateenv/v1alpha" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" "github.com/kagent-dev/mockllm" "github.com/stretchr/testify/require" @@ -28,110 +35,259 @@ import ( apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/substrate" ) -func TestSessionEgressPolicyPinsAdditionalDestinations(t *testing.T) { +// Every harness executes this same skill through its normal shell tool. The +// child keeps probing while the model waits for the test to change policy; this +// avoids each shell tool's foreground timeout without requiring another turn. +const egressProbeScript = `#!/bin/sh +control=$1 +target=$2 +run=$3 +for i in $(seq 1 900); do + phase=$(wget -T 5 -qO- "$control/phase") || exit 1 + if [ "$phase" = stop ]; then + echo "EGRESS_PROBE_FINISHED_$run" + wget -T 5 -qO- "$control/stopped" >/dev/null + exit + fi + if wget -T 3 -qO- "$target?phase=$phase"; then + result=ALLOWED + else + result=DENIED + fi + echo "EGRESS_RESULT_${phase}_$result" + wget -T 5 -qO- "$control/result?phase=$phase&result=$result" >/dev/null || exit 1 + sleep 0.2 +done +exit 1 +` + +func TestSessionEgressPolicyUpdatesLiveActors(t *testing.T) { t.Parallel() + target := interactionTarget(t) + image := publishSkillImage(t) forEachHarness(t, func(t *testing.T, harness testHarness) { t.Parallel() - if harness.name == codexE2EHarness { - testCodexSessionEgressPolicy(t, harness) - return - } - target := interactionTarget(t) - mcpURL, mcpServer := startMCPMock(t) - denied, err := url.Parse(mcpURL) - require.NoError(t, err) - // Give the destination a distinct hostname even when host mocks use - // host.docker.internal. It must not share the model or configured MCP origin. kube := interactionKubeClient(t) - alias := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{GenerateName: "denied-origin-", Namespace: "kagent"}, - Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeExternalName, ExternalName: denied.Hostname()}, + destination, requests := startEgressDestination(t) + modelURL, model, runs := startEgressSkillMock(t, harness.skillsRoot(), destination) + config := harness.createModel(t, kube, modelURL, nil) + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{GenerateName: "live-egress-", Namespace: "kagent", Labels: harness.labels()}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: config.Name}, SystemPrompt: "Load the egress skill and run its probe.", + Skills: []v1alpha3.AgentTemplateSkill{{Name: "egress", Source: v1alpha3.ArtifactSource{OCI: image, Path: "egress"}}}, + }} + createAndWaitInteractionTemplate(t, harness, kube, template) + fixture := newInteractionFixtureForTemplate(t, harness, target, template.Name) + actor, err := findSubstrateActor(fixture.ctx, fixture.system, "kagent", substrate.ActorName(fixture.sessionID)) + require.NoError(t, err) + actorUID := actor.GetMetadata().GetUid() + agent := &v1alpha3.Agent{} + key := ctrlclient.ObjectKey{Namespace: "kagent", Name: template.Name} + require.NoError(t, kube.Get(t.Context(), key, agent)) + revision := agent.Status.LatestSuccessfulRevision + grant := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "live-access-", Namespace: "kagent"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: destination}}}} + require.NoError(t, kube.Create(t.Context(), grant)) + t.Cleanup(func() { require.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), grant))) }) + setGrant := func(origin string) { + t.Helper() + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKeyFromObject(grant), grant)) + grant.Spec.Destinations[0].Origin = origin + require.NoError(t, kube.Update(t.Context(), grant)) } - require.NoError(t, kube.Create(t.Context(), alias)) - t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), alias)) }) - denied.Host = net.JoinHostPort(alias.Name+".kagent.svc.cluster.local", denied.Port()) - toolCalls := func() int { - calls := 0 - for _, request := range mcpServer.Requests() { - var message struct { - Method string `json:"method"` - } - if json.Unmarshal(request.Body, &message) == nil && message.Method == "tools/call" { - calls++ - } - } - return calls - } - const prompt = "Add 3 and 5 using the configured MCP server." - // Prove the destination is reachable when explicitly configured. - controlName, _ := createMCPInteractionTemplate(t, harness, denied.String(), false) - control := newInteractionFixtureForTemplate(t, harness, target, controlName) - _, _, task := control.send(t, prompt) - require.Equal(t, a2atype.TaskStateCompleted, task.Status.State) - require.Contains(t, taskText(task), "result is 8") - controlCalls := toolCalls() - require.Positive(t, controlCalls) + run := runs("live") + finish := runEgressSkill(t, fixture, model, run) + run.expect(t, "initial", "DENIED") + require.Zero(t, requests("live_initial"), "ungranted requests reached the destination") + require.NoError(t, kube.Get(t.Context(), key, agent)) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: grant.Name}} + require.NoError(t, kube.Update(t.Context(), agent)) + run.setPhase("allowed") + run.expect(t, "allowed", "ALLOWED") + require.Positive(t, requests("live_allowed")) + setGrant("https://unused.example.com") + run.setPhase("revoked") + run.expect(t, "revoked", "DENIED") + revokedCount := requests("live_revoked") + require.Never(t, func() bool { return requests("live_revoked") != revokedCount }, time.Second, 100*time.Millisecond) + actor, err = findSubstrateActor(fixture.ctx, fixture.system, "kagent", substrate.ActorName(fixture.sessionID)) + require.NoError(t, err) + require.Equal(t, actorUID, actor.GetMetadata().GetUid(), "live updates must not replace the actor") + require.Equal(t, ateapipb.ActorState_ACTOR_STATE_RUNNING, actor.GetStatus().GetState()) + finish() + assertActorSuspended(t, fixture) - upstream := *denied - upstream.Host, upstream.Path = net.JoinHostPort("127.0.0.1", denied.Port()), "" - proxy := httputil.NewSingleHostReverseProxy(&upstream) - var redirects atomic.Int64 - server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - _ = r.Body.Close() - var message struct { - Method string `json:"method"` - } - if json.Unmarshal(body, &message) == nil && message.Method == "tools/call" { - redirects.Add(1) - http.Redirect(w, r, denied.String(), http.StatusTemporaryRedirect) - return + // Save a checkpoint while the source is granted access, then revoke it while + // suspended. Neither resume nor a DATA fork may resurrect that old grant. + setGrant(destination) + run = runs("checkpoint") + finish = runEgressSkill(t, fixture, model, run) + run.expect(t, "initial", "ALLOWED") + task := finish() + assertActorSuspended(t, fixture) + checkpoint := createCheckpoint(t, fixture.ctx, fixture.checkpoints, &apiv1.CreateCheckpointRequest{SessionId: fixture.sessionID, RequestId: uuid.NewString(), ExpectedHeadTaskId: string(task.ID)}).GetCheckpoint() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(fixture.ctx), time.Minute) + defer cancel() + _, err := fixture.checkpoints.DeleteCheckpoint(ctx, &apiv1.DeleteCheckpointRequest{CheckpointId: checkpoint.Id}) + require.NoError(t, err) + }) + require.NoError(t, kube.Delete(t.Context(), grant)) + require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, agent); err != nil { + return false, err } - // Keep initialization and discovery on the configured endpoint so - // only the runtime's tool call exercises the additional destination. - r.Body = io.NopCloser(bytes.NewReader(body)) - proxy.ServeHTTP(w, r) + return apimeta.IsStatusConditionFalse(agent.Status.Conditions, v1alpha3.AgentConditionReady), nil })) - _ = server.Listener.Close() - listener, err := net.Listen("tcp", "0.0.0.0:0") - if err != nil { - t.Fatal(err) - } - server.Listener = listener - server.Start() - t.Cleanup(server.Close) - name, _ := createMCPInteractionTemplate(t, harness, reachableServerURL(t, server.URL, denied.Path), false) - testSessionEgressPolicyPinsDestination(t, harness, name, denied.Scheme+"://"+denied.Host, func(fixture *interactionFixture, allowed bool) { - beforeRedirects, beforeCalls := redirects.Load(), toolCalls() - _, _, task := fixture.send(t, prompt) - require.Greater(t, redirects.Load(), beforeRedirects, "the runtime must attempt the MCP tool call") - if allowed { - require.Equal(t, a2atype.TaskStateCompleted, task.Status.State, "policy-granted redirect text = %q", taskText(task)) - require.Contains(t, taskText(task), "result is 8") - require.Greater(t, toolCalls(), beforeCalls, "the session must reach the policy-granted destination") - } else { - require.Equal(t, beforeCalls, toolCalls(), "redirect reached an unconfigured destination") - } + require.Equal(t, revision, agent.Status.LatestSuccessfulRevision, "policy edits must not replace the runtime revision") + _, err = fixture.sessions.CreateSession(fixture.ctx, &apiv1.CreateSessionRequest{Agent: &apiv1.ResourceReference{Namespace: key.Namespace, Name: key.Name}, RequestId: uuid.NewString()}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + run = runs("resumed") + finish = runEgressSkill(t, fixture, model, run) + run.expect(t, "initial", "DENIED") + require.Zero(t, requests("resumed_initial")) + finish() + assertActorSuspended(t, fixture) + actor, err = findSubstrateActor(fixture.ctx, fixture.system, "kagent", substrate.ActorName(fixture.sessionID)) + require.NoError(t, err) + require.Equal(t, actorUID, actor.GetMetadata().GetUid()) + forked, err := fixture.checkpoints.ForkSession(fixture.ctx, &apiv1.ForkSessionRequest{CheckpointId: checkpoint.Id, RequestId: uuid.NewString()}) + require.NoError(t, err) + fork := *fixture + fork.sessionID = forked.Session.Id + fork.contextID = forked.Session.ContextId + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(fork.ctx), time.Minute) + defer cancel() + require.NoError(t, deleteIdleSession(ctx, fork.sessions, fork.sessionID)) }) + run = runs("fork") + finish = runEgressSkill(t, &fork, model, run) + run.expect(t, "initial", "DENIED") + require.Zero(t, requests("fork_initial")) + finish() }) } -func testCodexSessionEgressPolicy(t *testing.T, harness testHarness) { +type egressProbeRun struct { + name string + proxy *httputil.ReverseProxy + mu sync.Mutex + phase string + lastPhase, lastResult string + consecutive int + stopped chan struct{} + release chan struct{} + stopOnce, releaseOnce sync.Once +} + +func (e *egressProbeRun) setPhase(phase string) { + e.mu.Lock() + defer e.mu.Unlock() + e.phase = e.name + "_" + phase +} +func (e *egressProbeRun) expect(t *testing.T, phase, result string) { t.Helper() - interactionTarget(t) - // Codex's MCP client rejects cross-origin redirects before the gateway can - // authorize them. Exercise additional destinations through its shell tool. - var requests atomic.Int64 + require.Eventually(t, func() bool { + e.mu.Lock() + defer e.mu.Unlock() + return e.lastPhase == e.name+"_"+phase && e.lastResult == result && e.consecutive >= 3 + }, 90*time.Second, 100*time.Millisecond, "%s %s did not become %s", e.name, phase, result) +} +func (e *egressProbeRun) finish(ctx context.Context) error { + e.mu.Lock() + e.phase = "stop" + e.mu.Unlock() + select { + case <-e.stopped: + e.releaseOnce.Do(func() { close(e.release) }) + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func runEgressSkill(t *testing.T, fixture *interactionFixture, model *modelRecorder, run *egressProbeRun) func() *a2atype.Task { + t.Helper() + start := len(model.Requests("", "")) + _, request := newMessageRequest(t, "Run egress skill "+run.name+".") + request.Message.ContextId = fixture.sessionID + request.Tenant = fixture.tenant + type response struct { + message *a2apb.SendMessageResponse + err error + } + done := make(chan response, 1) + ctx, cancel := context.WithCancel(fixture.ctx) + t.Cleanup(func() { cancel(); run.releaseOnce.Do(func() { close(run.release) }) }) + go func() { r, err := sendMessageWithRetry(ctx, fixture.client, request); done <- response{r, err} }() + return func() *a2atype.Task { + t.Helper() + stopCtx, stop := context.WithTimeout(ctx, 15*time.Second) + defer stop() + require.NoError(t, run.finish(stopCtx)) + var result response + select { + case result = <-done: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + require.NoError(t, result.err) + parsed, err := pbconv.FromProtoSendMessageResponse(result.message) + require.NoError(t, err) + task, ok := parsed.(*a2atype.Task) + require.True(t, ok) + require.Equal(t, a2atype.TaskStateCompleted, task.Status.State, taskText(task)) + require.Contains(t, taskText(task), "EGRESS_"+run.name+"_DONE") + // Check actual tool output in fresh model inputs, not the model's fixed text + // or conversation history copied into a resumed session or checkpoint fork. + var outputs []byte + for _, request := range model.Requests("", "")[start:] { + last, err := lastEgressModelInput(request.Body) + require.NoError(t, err) + outputs = append(outputs, last...) + } + for _, marker := range []string{"SKILL_BODY_EGRESS", "EGRESS_PROBE_STARTED_" + run.name, "EGRESS_PROBE_FINISHED_" + run.name} { + require.Contains(t, string(outputs), marker) + } + if run.name == "live" { + for _, marker := range []string{"EGRESS_RESULT_live_initial_DENIED", "EGRESS_RESULT_live_allowed_ALLOWED", "EGRESS_RESULT_live_revoked_DENIED"} { + require.Contains(t, string(outputs), marker) + } + } + return task + } +} + +func lastEgressModelInput(raw []byte) (json.RawMessage, error) { + if len(raw) == 0 { + return nil, nil + } + var body struct { + Messages []json.RawMessage `json:"messages"` + Input []json.RawMessage `json:"input"` + } + if err := json.Unmarshal(raw, &body); err != nil { + return nil, err + } + messages := append(body.Messages, body.Input...) + if len(messages) == 0 { + return nil, nil + } + return messages[len(messages)-1], nil +} + +func startEgressDestination(t *testing.T) (string, func(string) int) { + t.Helper() + var mu sync.Mutex + requests := map[string]int{} server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - _, _ = io.WriteString(w, "egress destination reached") + mu.Lock() + requests[r.URL.Query().Get("phase")]++ + mu.Unlock() + _, _ = io.WriteString(w, "EGRESS_DESTINATION_REACHED\n") })) _ = server.Listener.Close() listener, err := net.Listen("tcp", "0.0.0.0:0") @@ -139,91 +295,148 @@ func testCodexSessionEgressPolicy(t *testing.T, harness testHarness) { server.Listener = listener server.Start() t.Cleanup(server.Close) - destination, err := url.Parse(reachableServerURL(t, server.URL, "/")) + destination, err := url.Parse(reachableServerURL(t, server.URL, "")) require.NoError(t, err) - // Keep the destination distinct from the model origin on Docker Desktop too. + // Keep a distinct origin even on Docker Desktop, where host mocks otherwise + // share host.docker.internal with the implicitly allowed model endpoint. kube := interactionKubeClient(t) - alias := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{GenerateName: "shell-origin-", Namespace: "kagent"}, - Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeExternalName, ExternalName: destination.Hostname()}, - } + alias := &corev1.Service{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-origin-", Namespace: "kagent"}, Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeExternalName, ExternalName: destination.Hostname()}} require.NoError(t, kube.Create(t.Context(), alias)) t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), alias)) }) destination.Host = net.JoinHostPort(alias.Name+".kagent.svc.cluster.local", destination.Port()) - - raw, err := codexInteractionMocks.ReadFile("mocks/invoke_codex_builtin_tools.json") - require.NoError(t, err) - // Always emit the fixture marker so the mock model finishes after denied - // requests too. The destination's request count proves whether egress worked. - command := "wget -T 10 -qO- " + destination.String() + "; printf CODEX_TOOL_FILE" - raw = bytes.ReplaceAll(raw, []byte("printf CODEX_TOOL_FILE"), []byte(command)) - var cfg mockllm.Config - require.NoError(t, json.Unmarshal(raw, &cfg)) - modelURL := reachableModelURL(t, startMockLLMConfig(t, cfg)) - name := createInteractionTemplate(t, harness, modelURL) - testSessionEgressPolicyPinsDestination(t, harness, name, destination.Scheme+"://"+destination.Host, func(fixture *interactionFixture, allowed bool) { - before := requests.Load() - streamed := sendStreaming(t, fixture, "Run the requested shell command.") - require.Equal(t, a2atype.TaskStateCompleted, streamed.state, "egress probe text = %q, failure = %q", streamed.text, streamed.failureText) - assertToolEvents(t, streamed.toolEvents, "command_execution") - if allowed { - require.Greater(t, requests.Load(), before, "the session must reach the policy-granted destination") - } else { - require.Equal(t, before, requests.Load(), "shell reached an unconfigured destination") - } - }) + return destination.String(), func(phase string) int { mu.Lock(); defer mu.Unlock(); return requests[phase] } } -func testSessionEgressPolicyPinsDestination(t *testing.T, harness testHarness, name, origin string, probe func(*interactionFixture, bool)) { +func startEgressSkillMock(t *testing.T, root, destination string) (string, *modelRecorder, func(string) *egressProbeRun) { t.Helper() - target := interactionTarget(t) - kube := interactionKubeClient(t) - fixture := newInteractionFixtureForTemplate(t, harness, target, name) - probe(fixture, false) - grant := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "additional-access-", Namespace: "kagent"}, Spec: v1alpha3.EgressPolicySpec{ - Destinations: []v1alpha3.EgressDestination{{Origin: origin}}, - }} - require.NoError(t, kube.Create(t.Context(), grant)) - t.Cleanup(func() { require.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), grant))) }) - agent := &v1alpha3.Agent{} - key := ctrlclient.ObjectKey{Namespace: "kagent", Name: name} - require.NoError(t, kube.Get(t.Context(), key, agent)) - agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: grant.Name}} - require.NoError(t, kube.Update(t.Context(), agent)) - require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { - if err := kube.Get(ctx, key, agent); err != nil { - return false, err + var active atomic.Pointer[egressProbeRun] + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + run := active.Load() + if run == nil { + http.Error(w, "no active probe", http.StatusServiceUnavailable) + return } - condition := apimeta.FindStatusCondition(agent.Status.Conditions, v1alpha3.AgentConditionReady) - return condition != nil && condition.ObservedGeneration == agent.Generation && condition.Status == metav1.ConditionTrue, nil - })) - allowed := newInteractionFixtureForTemplate(t, harness, target, name) - probe(allowed, true) - probe(fixture, false) // Existing sessions retain their earlier grants. - - require.NoError(t, kube.Delete(t.Context(), grant)) - require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - if err := kube.Get(ctx, key, agent); err != nil { - return false, err + if strings.HasPrefix(r.URL.Path, "/egress/") { + if !strings.HasPrefix(r.URL.Path, "/egress/"+run.name+"/") { + http.Error(w, "old probe", http.StatusGone) + return + } + switch { + case strings.HasSuffix(r.URL.Path, "/phase"): + run.mu.Lock() + phase := run.phase + run.mu.Unlock() + _, _ = io.WriteString(w, phase) + case strings.HasSuffix(r.URL.Path, "/result"): + phase, result := r.URL.Query().Get("phase"), r.URL.Query().Get("result") + run.mu.Lock() + if phase == run.lastPhase && result == run.lastResult { + run.consecutive++ + } else { + run.lastPhase, run.lastResult, run.consecutive = phase, result, 1 + } + run.mu.Unlock() + case strings.HasSuffix(r.URL.Path, "/stopped"): + run.stopOnce.Do(func() { close(run.stopped) }) + default: + http.NotFound(w, r) + } + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), 400) + return + } + _ = r.Body.Close() + last, err := lastEgressModelInput(body) + if err != nil { + http.Error(w, err.Error(), 400) + return + } + if bytes.Contains(last, []byte("EGRESS_PROBE_STARTED_"+run.name)) { + select { + case <-run.release: + case <-r.Context().Done(): + return + } } - condition := apimeta.FindStatusCondition(agent.Status.Conditions, v1alpha3.AgentConditionReady) - return condition != nil && condition.ObservedGeneration == agent.Generation && condition.Status == metav1.ConditionFalse, nil + r.Body = io.NopCloser(bytes.NewReader(body)) + run.proxy.ServeHTTP(w, r) })) - // Check creation directly, without the fixture's preparation retry loop. - _, err := allowed.sessions.CreateSession(allowed.ctx, &apiv1.CreateSessionRequest{ - Agent: &apiv1.ResourceReference{Namespace: key.Namespace, Name: key.Name}, RequestId: uuid.NewString(), - }) - require.Equal(t, codes.FailedPrecondition, status.Code(err)) - probe(allowed, true) // Deletion preserves the existing session's pinned grant. + t.Cleanup(server.Close) + recorder := startModelRecorder(t, server.URL, nil) + modelURL := reachableModelURL(t, recorder.URL) + origin := strings.TrimSuffix(modelURL, "/v1") + raw, err := interactionMocks.ReadFile("mocks/invoke_egress_skill.json") + require.NoError(t, err) + runs := map[string]*egressProbeRun{} + for _, phase := range []string{"live", "checkpoint", "resumed", "fork"} { + logPath := "/data/egress-" + phase + ".log" + start := fmt.Sprintf("nohup sh %s/egress/probe.sh '%s/egress/%s' '%s' %s >%s 2>&1 Date: Fri, 16 Oct 2026 13:45:52 +0000 Subject: [PATCH 12/43] test: keep live egress probes in the foreground Signed-off-by: Eitan Yarmush --- go/core/test/e2e/egress_test.go | 85 ++++++------- .../test/e2e/mocks/invoke_egress_skill.json | 113 +----------------- 2 files changed, 40 insertions(+), 158 deletions(-) diff --git a/go/core/test/e2e/egress_test.go b/go/core/test/e2e/egress_test.go index cffa29e3f9..3df9f3f9b2 100644 --- a/go/core/test/e2e/egress_test.go +++ b/go/core/test/e2e/egress_test.go @@ -1,7 +1,6 @@ package e2e_test import ( - "bytes" "context" "encoding/json" "fmt" @@ -38,13 +37,13 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/substrate" ) -// Every harness executes this same skill through its normal shell tool. The -// child keeps probing while the model waits for the test to change policy; this -// avoids each shell tool's foreground timeout without requiring another turn. +// Every harness executes this same foreground skill through its normal shell +// tool. The test changes policy while the invocation is waiting for the probe. const egressProbeScript = `#!/bin/sh control=$1 target=$2 run=$3 +echo "EGRESS_PROBE_STARTED_$run" for i in $(seq 1 900); do phase=$(wget -T 5 -qO- "$control/phase") || exit 1 if [ "$phase" = stop ]; then @@ -52,7 +51,7 @@ for i in $(seq 1 900); do wget -T 5 -qO- "$control/stopped" >/dev/null exit fi - if wget -T 3 -qO- "$target?phase=$phase"; then + if wget -T 3 -qO- "$target/?phase=$phase"; then result=ALLOWED else result=DENIED @@ -179,8 +178,7 @@ type egressProbeRun struct { lastPhase, lastResult string consecutive int stopped chan struct{} - release chan struct{} - stopOnce, releaseOnce sync.Once + stopOnce sync.Once } func (e *egressProbeRun) setPhase(phase string) { @@ -202,7 +200,6 @@ func (e *egressProbeRun) finish(ctx context.Context) error { e.mu.Unlock() select { case <-e.stopped: - e.releaseOnce.Do(func() { close(e.release) }) return nil case <-ctx.Done(): return ctx.Err() @@ -221,7 +218,17 @@ func runEgressSkill(t *testing.T, fixture *interactionFixture, model *modelRecor } done := make(chan response, 1) ctx, cancel := context.WithCancel(fixture.ctx) - t.Cleanup(func() { cancel(); run.releaseOnce.Do(func() { close(run.release) }) }) + t.Cleanup(func() { + cancel() + if t.Failed() { + for _, request := range model.Requests("", "")[start:] { + last, err := lastEgressModelInput(request.Body) + if err == nil && len(last) != 0 { + t.Logf("egress %s model input: %s", run.name, last) + } + } + } + }) go func() { r, err := sendMessageWithRetry(ctx, fixture.client, request); done <- response{r, err} }() return func() *a2atype.Task { t.Helper() @@ -343,25 +350,6 @@ func startEgressSkillMock(t *testing.T, root, destination string) (string, *mode } return } - body, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), 400) - return - } - _ = r.Body.Close() - last, err := lastEgressModelInput(body) - if err != nil { - http.Error(w, err.Error(), 400) - return - } - if bytes.Contains(last, []byte("EGRESS_PROBE_STARTED_"+run.name)) { - select { - case <-run.release: - case <-r.Context().Done(): - return - } - } - r.Body = io.NopCloser(bytes.NewReader(body)) run.proxy.ServeHTTP(w, r) })) t.Cleanup(server.Close) @@ -372,32 +360,29 @@ func startEgressSkillMock(t *testing.T, root, destination string) (string, *mode require.NoError(t, err) runs := map[string]*egressProbeRun{} for _, phase := range []string{"live", "checkpoint", "resumed", "fork"} { - logPath := "/data/egress-" + phase + ".log" - start := fmt.Sprintf("nohup sh %s/egress/probe.sh '%s/egress/%s' '%s' %s >%s 2>&1 Date: Fri, 16 Oct 2026 15:06:28 +0000 Subject: [PATCH 13/43] fix: target live egress updates and keep prepared runtimes available Signed-off-by: Eitan Yarmush --- .../configuration-and-compilation.md | 13 +- docs/architecture/credential-injection.md | 10 +- docs/architecture/egress-policies.md | 25 ++- docs/architecture/runtime-and-lifecycle.md | 9 +- docs/architecture/sandboxes.md | 10 +- go/core/internal/controller/egress.go | 174 ++++++++++------ go/core/internal/controller/egress_test.go | 173 ++++++++++++++-- .../internal/controller/preparation_test.go | 5 +- .../internal/controller/reconciler_test.go | 4 +- .../controller/sandbox_collections.go | 5 +- .../controller/sandbox_integration_test.go | 2 - .../database/runtime_creation_test.go | 21 +- go/core/internal/database/runtime_egress.go | 20 +- .../internal/database/runtime_egress_test.go | 27 ++- .../database/runtime_revisions_test.go | 8 +- .../internal/database/sandbox_revisions.go | 15 +- .../database/sandbox_revisions_test.go | 47 ----- go/core/internal/database/sandboxes.go | 8 +- go/core/internal/database/sessions.go | 8 +- .../internal/service/sandbox/service_test.go | 3 - go/core/internal/substrate/egress.go | 7 +- go/core/internal/translator/egress.go | 58 +++++- go/core/pkg/app/app.go | 2 +- .../000002_runtime_passthrough_origins.sql | 10 + go/core/test/e2e/README.md | 11 +- go/core/test/e2e/egress_credentials_test.go | 189 ++++++++++++++++++ go/core/test/e2e/egress_test.go | 27 ++- 27 files changed, 666 insertions(+), 225 deletions(-) delete mode 100644 go/core/internal/database/sandbox_revisions_test.go create mode 100644 go/core/test/e2e/egress_credentials_test.go diff --git a/docs/architecture/configuration-and-compilation.md b/docs/architecture/configuration-and-compilation.md index ea08a5a7ff..b05b685714 100644 --- a/docs/architecture/configuration-and-compilation.md +++ b/docs/architecture/configuration-and-compilation.md @@ -172,10 +172,11 @@ revision. AgentTemplate is shared context with no runtime status. Child template references are resolved under the root Agent's Harness; they do not require separate Agents or matching Harness references. -A failed compile or apply retains the previous successful revision. Fresh -Sessions require the desired and latest successful revisions to match, so -pending or failed preparation blocks creation after reconciliation records the -change. Existing Sessions and creation retries keep their pinned revision. +A failed compile or apply retains the previous successful revision, which +remains available for fresh Sessions during pending or failed edits. Initial +creation requires a successful revision. Existing Sessions and creation retries +keep their pinned revision. Explicit egress grants use current policies +independently of runtime preparation. Harness compilers only translate inputs. The controller and Substrate adapter own application and readiness. The central entry points are @@ -392,8 +393,8 @@ no synthetic Kubernetes object is created for an inline template. Create a session with `kagent agent session create --agent assistant -n kagent`. The gRPC create request and ScheduledRun target one `agent` resource reference. -The store selects that Agent's current desired revision only after it has been -successfully prepared. Deleting an Agent +The store selects that Agent's latest successful revision, including while an +edit is pending or failed. Deleting an Agent retires its definition; sessions and checkpoints retain their pinned revisions. Recreating the same name creates a new identity and cannot inherit the old Agent's last successful revision. diff --git a/docs/architecture/credential-injection.md b/docs/architecture/credential-injection.md index aa340ffcb8..0df74d9e50 100644 --- a/docs/architecture/credential-injection.md +++ b/docs/architecture/credential-injection.md @@ -7,10 +7,12 @@ replaces the outgoing HTTP header when the request carries a placeholder. SDKs receive an inert placeholder where they require an API key; real credentials never enter compiled environments, runtime configuration, or revision provenance. -Bindings are persisted with the prepared revision and installed before a -session becomes ready, including retries and checkpoint forks. Secret names, -keys, destinations, and headers affect revision identity. Secret values and -Secret UIDs do not. Rotation is handled by the gateway; its credential cache can +Inferred integration bindings are persisted with the prepared revision; their +Secret names, keys, destinations, and headers affect revision identity. Explicit +EgressPolicy bindings are resolved from current policies and update live actors +without changing that revision. Both are installed before execution, including +creation retries, resume, and checkpoint forks. Secret values and Secret UIDs +do not affect revision identity. Rotation is handled by the gateway; its credential cache can take up to five minutes to refresh, without recompiling or restarting an agent. ## Installation diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index 3650230bea..4c21d13898 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -161,24 +161,31 @@ Secret values remain outside compiled inputs; the gateway handles value rotation The controller resolves current grants from the owning Agent or SandboxTemplate and reconciles each existing Actor's Substrate policy. Kubernetes events trigger -updates, with a periodic sweep to recover creation races and controller restarts. +updates only for the affected Agent or SandboxTemplate. A five-minute sweep +recovers creation races, missed events, and controller restarts. Failed actor +updates retry with exponential backoff independently of successful updates. This runs independently of agent invocations and runtime preparation. Running and suspended Actors receive updates without replacement. Missing or invalid policies and Secrets revoke explicit grants; pinned integration access remains available. A grant that conflicts with a pinned credential or caller-token origin is also -withheld. Deleting and recreating an Agent or SandboxTemplate does not transfer +withheld, with rate-limited warnings identifying the affected revision. The +same applies when the merged grants exceed the rule limit. Deleting and recreating an Agent or SandboxTemplate does not transfer its grants to instances owned by the old Kubernetes UID. -New Sessions and sandboxes pin the current desired runtime revision only when it -matches the latest successful revision. The store checks this atomically with -creation. Unresolved policies and pending or failed preparation block fresh -creation after reconciliation. Existing instances and checkpoint forks retain -their runtime configuration, but resolve current authorization on creation, -creation retries, and resume. A fork cannot restore an old policy from a snapshot. +New Sessions and sandboxes atomically pin the latest successful runtime revision, +including while a configuration edit is unresolved, preparing, or failed. Initial +creation still requires a successful revision. Existing instances and checkpoint +forks retain their runtime configuration. Creation, creation retries, and resume +resolve current authorization independently of that revision; a fork cannot +restore an old policy from a snapshot. The runtime workflow creates a suspended Actor and installs its current `default` egress policy before it can run. Policy updates use Substrate's UID and version -preconditions; conflicting writes re-read both backend state and current grants. +preconditions. After reading the backend version, each attempt reads the source, +policies, and Secrets directly from the Kubernetes API server, bypassing informer +caches on serving replicas. Conflicts repeat both reads. These reads do not form +an atomic snapshot across Kubernetes resources; a concurrent edit can require +another reconciliation. Transient API failures return errors and retry. No database transaction is held across the Substrate calls. Policy convergence is asynchronous: a Kubernetes write is not a synchronous diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 4be954eca3..27efbf01f2 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -6,10 +6,11 @@ Kubernetes resource. ## Creation and state -Fresh creation requires the Agent's current desired revision to match its latest -successful revision. Once reconciliation records unresolved inputs or a pending -update, creation fails until the current revision is prepared. Creation retries -retain the existing session's pinned revision. +Fresh creation pins the Agent's latest successful runtime revision. Pending or +failed edits leave that revision available; an Agent with no successful revision +cannot create sessions. Creation retries retain the existing session's pinned +revision. Explicit egress grants are resolved from current Kubernetes policies +independently of the runtime revision before execution. The control plane creates a deterministic Actor initially suspended and marks the session ready after Substrate accepts it. Readiness of the image was already established diff --git a/docs/architecture/sandboxes.md b/docs/architecture/sandboxes.md index 72d8f4e859..bc4da363aa 100644 --- a/docs/architecture/sandboxes.md +++ b/docs/architecture/sandboxes.md @@ -97,9 +97,9 @@ finalizer, including when deletion starts while the controller is offline. The controller advances the latest-successful revision only after the golden snapshot is ready and reports preparation through the template's `Ready` condition and observed generation. Failed preparation preserves the previous -successful revision. Fresh creation requires the desired revision to match that -successful revision; unresolved inputs and pending or failed preparation block -creation after reconciliation records the change. Existing Sandboxes and +successful revision, which remains available for fresh creation while edits +are unresolved, pending, or failed. Initial creation requires a successful +revision. Existing Sandboxes and creation retries retain their revision when a template changes. Template deletion retires preparation; retained instances continue to pin the runtime artifacts they need. @@ -119,7 +119,9 @@ expiration. Sandboxes have no allowed egress destinations by default. Select same-namespace [EgressPolicy resources](egress-policies.md) through `SandboxTemplate.spec.egressPolicyRefs` to grant additional HTTP(S) origins and static gateway credentials. Resolved -grants are pinned with the prepared revision; policy edits affect new sandboxes. +grants are resolved from the current template independently of its prepared +revision. Policy edits update running and suspended sandboxes without replacing +their Actor. Creation and resume apply current grants before execution. ## Lifecycle and recovery diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go index 299c6650e8..56b392a58a 100644 --- a/go/core/internal/controller/egress.go +++ b/go/core/internal/controller/egress.go @@ -2,14 +2,21 @@ package controller import ( "context" + "errors" "fmt" "slices" + "sync" "time" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "golang.org/x/time/rate" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "istio.io/istio/pkg/kube/controllers" "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" @@ -22,16 +29,18 @@ import ( type runtimeEgressStore interface { GetRuntimeEgress(context.Context, string) (*database.RuntimeEgress, error) - ListRuntimeEgressTargets(context.Context, string, int) ([]database.RuntimeEgressTarget, error) + ListRuntimeEgressTargets(context.Context, database.RuntimeEgressSource, string, int) ([]database.RuntimeEgressTarget, error) } type policyGrants struct { - Key string + Source database.RuntimeEgressSource UID string Access egress.Access } -func (p policyGrants) ResourceName() string { return p.Key } +func (p policyGrants) ResourceName() string { + return p.Source.Kind + "/" + p.Source.Namespace + "/" + p.Source.Name +} // EgressResolver resolves mutable policy grants independently of runtime // compilation. Broken templates and pools must not prevent revocation. @@ -39,15 +48,17 @@ type EgressResolver struct { grants krt.Collection[policyGrants] store runtimeEgressStore namespaces []string + reader ctrlclient.Reader + warnings *rate.Limiter } var _ egress.Resolver = (*EgressResolver)(nil) -func NewEgressResolver(runtime *Runtime, store runtimeEgressStore, namespaces []string) *EgressResolver { +func NewEgressResolver(runtime *Runtime, store runtimeEgressStore, reader ctrlclient.Reader, namespaces []string) *EgressResolver { inputs := runtime.Collections collections := translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets} agents := krt.NewCollection(inputs.Agents, func(ctx krt.HandlerContext, agent *kagentv1alpha3.Agent) *policyGrants { - grants := &policyGrants{Key: "agent/" + agent.Namespace + "/" + agent.Name, UID: string(agent.UID)} + grants := &policyGrants{Source: database.RuntimeEgressSource{Kind: "agent", Namespace: agent.Namespace, Name: agent.Name}, UID: string(agent.UID)} if agent.DeletionTimestamp.IsZero() { // CompileEgressPolicies returns empty access on every failure. Runtime // compilation reports resolution errors through the source's status. @@ -59,7 +70,7 @@ func NewEgressResolver(runtime *Runtime, store runtimeEgressStore, namespaces [] return grants }, runtime.Options.WithName("AgentEgressGrants")...) sandboxes := krt.NewCollection(inputs.SandboxTemplates, func(ctx krt.HandlerContext, template *kagentv1alpha3.SandboxTemplate) *policyGrants { - grants := &policyGrants{Key: "sandbox/" + template.Namespace + "/" + template.Name, UID: string(template.UID)} + grants := &policyGrants{Source: database.RuntimeEgressSource{Kind: "sandbox", Namespace: template.Namespace, Name: template.Name}, UID: string(template.UID)} if template.DeletionTimestamp.IsZero() { access, err := translator.CompileEgressPolicies(ctx, collections, template.Namespace, template.Spec.EgressPolicyRefs) if err == nil { @@ -68,7 +79,7 @@ func NewEgressResolver(runtime *Runtime, store runtimeEgressStore, namespaces [] } return grants }, runtime.Options.WithName("SandboxEgressGrants")...) - return &EgressResolver{store: store, namespaces: slices.Clone(namespaces), grants: krt.JoinCollection([]krt.Collection[policyGrants]{agents, sandboxes}, runtime.Options.WithName("RuntimeEgressGrants")...)} + return &EgressResolver{reader: reader, warnings: rate.NewLimiter(rate.Every(30*time.Second), 5), store: store, namespaces: slices.Clone(namespaces), grants: krt.JoinCollection([]krt.Collection[policyGrants]{agents, sandboxes}, runtime.Options.WithName("RuntimeEgressGrants")...)} } func (e *EgressResolver) watches(namespace string) bool { @@ -76,9 +87,6 @@ func (e *EgressResolver) watches(namespace string) bool { } func (e *EgressResolver) ResolveEgress(ctx context.Context, revision string) (egress.Access, error) { - if !e.grants.WaitUntilSynced(ctx.Done()) { - return egress.Access{}, fmt.Errorf("egress policy cache has not synchronized: %w", ctx.Err()) - } pinned, err := e.store.GetRuntimeEgress(ctx, revision) if err != nil { return egress.Access{}, err @@ -87,34 +95,78 @@ func (e *EgressResolver) ResolveEgress(ctx context.Context, revision string) (eg return egress.Access{}, fmt.Errorf("runtime namespace %q is not watched", pinned.Namespace) } base := egress.Access{Destinations: pinned.EgressDestinations, Credentials: pinned.Credentials} - grants := e.grants.GetKey(pinned.Kind + "/" + pinned.Namespace + "/" + pinned.SourceName) - // Deleted and recreated sources cannot pass their grants to old sessions. - if grants == nil || grants.UID != pinned.SourceUID { - return base, nil + grants, err := e.currentGrants(ctx, pinned) + if err != nil { + return egress.Access{}, err } - access, err := egress.MergeAccess(base, grants.Access, pinned.PassthroughOrigins) + access, err := egress.MergeAccess(base, grants, pinned.PassthroughOrigins) if err != nil { // A current grant may conflict with an old session's pinned integrations. // Revoke its explicit access instead of preserving a stale policy. - logging.FromContext(ctx).DebugContext(ctx, "live egress grants conflict with pinned runtime", "revision", revision, "error", err) + if e.warnings.Allow() { + logging.FromContext(ctx).WarnContext(ctx, "explicit egress grants withheld", "revision", revision, "namespace", pinned.Namespace, "source", pinned.SourceName, "error", err) + } return base, nil } return access, nil } +// Read through the API server, never an informer cache: create and resume may +// run on a replica whose watches have not observed a revocation yet. +func (e *EgressResolver) currentGrants(ctx context.Context, pinned *database.RuntimeEgress) (egress.Access, error) { + var source ctrlclient.Object + var refs []corev1.LocalObjectReference + key := ctrlclient.ObjectKey{Namespace: pinned.Namespace, Name: pinned.SourceName} + switch pinned.Kind { + case "agent": + source = &kagentv1alpha3.Agent{} + case "sandbox": + source = &kagentv1alpha3.SandboxTemplate{} + default: + return egress.Access{}, fmt.Errorf("unknown runtime kind %q", pinned.Kind) + } + if err := e.reader.Get(ctx, key, source); err != nil { + if apierrors.IsNotFound(err) { + return egress.Access{}, nil + } + return egress.Access{}, fmt.Errorf("read egress source %s: %w", key, err) + } + if string(source.GetUID()) != pinned.SourceUID || !source.GetDeletionTimestamp().IsZero() { + return egress.Access{}, nil + } + switch object := source.(type) { + case *kagentv1alpha3.Agent: + refs = object.Spec.EgressPolicyRefs + case *kagentv1alpha3.SandboxTemplate: + refs = object.Spec.EgressPolicyRefs + } + access, err := translator.ResolveEgressPolicies(ctx, e.reader, pinned.Namespace, refs) + var invalid *translator.ValidationError + if apierrors.IsNotFound(err) || errors.As(err, &invalid) { + return egress.Access{}, nil + } + return access, err +} + type actorEgressClient interface { GetActor(context.Context, string, string) (*ateapipb.Actor, error) EnsureActorEgressPolicy(context.Context, string, string, string) error } -// EgressReconciler refreshes every allocated actor, including suspended actors. -// Events apply changes promptly; polling recovers actor-creation races, missed -// events and controller restarts. No model turn or runtime restart is required. +// EgressReconciler queues only instances of changed sources. A slow sweep +// recovers missed events and actor-creation races, including suspended actors. +// Pages and actors have independent retry queues so failures cannot stall grants +// for another source or cause successful actors to be retried with a failed one. type EgressReconciler struct { resolver *EgressResolver actors actorEgressClient } +type egressPage struct { + Source database.RuntimeEgressSource + After string +} + var _ manager.LeaderElectionRunnable = (*EgressReconciler)(nil) func NewEgressReconciler(resolver *EgressResolver, actors actorEgressClient) *EgressReconciler { @@ -124,58 +176,68 @@ func NewEgressReconciler(resolver *EgressResolver, actors actorEgressClient) *Eg func (e *EgressReconciler) NeedLeaderElection() bool { return true } func (e *EgressReconciler) Start(ctx context.Context) error { - wake := make(chan struct{}, 1) - handler := e.resolver.grants.RegisterBatch(func([]krt.Event[policyGrants]) { - select { - case wake <- struct{}{}: - default: + reconcile := func(item any) error { + err := e.reconcileActor(ctx, item.(database.RuntimeEgressTarget)) + if status.Code(err) == codes.NotFound { + return nil } - }, true) + return err + } + actors := newReconciliationQueue("live-egress-actors", reconcile) + // Recovery has its own worker so a large sweep cannot delay event-driven revocations. + recovery := newReconciliationQueue("live-egress-recovery", reconcile) + var pages controllers.Queue + pages = newReconciliationQueue("live-egress-sources", func(item any) error { + page := item.(egressPage) + const pageSize = 100 + targets, err := e.resolver.store.ListRuntimeEgressTargets(ctx, page.Source, page.After, pageSize) + if err != nil { + return err + } + for _, target := range targets { + if e.resolver.watches(target.Atespace) { + if page.Source.Kind == "" { + recovery.Add(target) + } else { + actors.Add(target) + } + } + } + if len(targets) == pageSize { + pages.Add(egressPage{Source: page.Source, After: targets[len(targets)-1].ID}) + } + return nil + }) + defer actors.ShutDownEarly() + defer recovery.ShutDownEarly() + defer pages.ShutDownEarly() + handler := e.resolver.grants.RegisterBatch(func(events []krt.Event[policyGrants]) { + for _, event := range events { + pages.Add(egressPage{Source: event.Latest().Source}) + } + }, false) defer handler.UnregisterHandler() if !handler.WaitUntilSynced(ctx.Done()) { return ctx.Err() } - ticker := time.NewTicker(2 * time.Second) + var workers sync.WaitGroup + workers.Go(func() { actors.Run(ctx.Done()) }) + workers.Go(func() { recovery.Run(ctx.Done()) }) + workers.Go(func() { pages.Run(ctx.Done()) }) + defer workers.Wait() + pages.Add(egressPage{}) + ticker := time.NewTicker(5 * time.Minute) defer ticker.Stop() for { - if err := e.reconcile(ctx); err != nil && ctx.Err() == nil { - logging.FromContext(ctx).ErrorContext(ctx, "failed to reconcile live egress", "error", err) - } select { case <-ctx.Done(): return nil - case <-wake: case <-ticker.C: + pages.Add(egressPage{}) } } } -func (e *EgressReconciler) reconcile(ctx context.Context) error { - const pageSize = 100 - after := "" - for { - targets, err := e.resolver.store.ListRuntimeEgressTargets(ctx, after, pageSize) - if err != nil { - return err - } - for _, target := range targets { - if !e.resolver.watches(target.Atespace) { - continue - } - if err := e.reconcileActor(ctx, target); err != nil && status.Code(err) != codes.NotFound { - logging.FromContext(ctx).ErrorContext(ctx, "failed to update actor egress", "instance_id", target.ID, "error", err) - } - if err := ctx.Err(); err != nil { - return err - } - } - if len(targets) < pageSize { - return nil - } - after = targets[len(targets)-1].ID - } -} - func (e *EgressReconciler) reconcileActor(ctx context.Context, target database.RuntimeEgressTarget) error { binding := substrate.ActorBinding{Atespace: target.Atespace, Name: substrate.ActorName(target.ID), TemplateAtespace: target.Atespace, TemplateName: target.TemplateName} actor, err := e.actors.GetActor(ctx, binding.Atespace, binding.Name) diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go index 6044066e9b..6f0f1fc340 100644 --- a/go/core/internal/controller/egress_test.go +++ b/go/core/internal/controller/egress_test.go @@ -1,27 +1,40 @@ package controller import ( + "bytes" "context" + "errors" "fmt" + "log/slog" "reflect" + "strings" + "sync" + "sync/atomic" "testing" "time" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/stretchr/testify/require" + "golang.org/x/time/rate" "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/pkg/logging" ) type egressTestStore struct { pinned database.RuntimeEgress targets []database.RuntimeEgressTarget + listed chan database.RuntimeEgressSource } var _ runtimeEgressStore = (*egressTestStore)(nil) @@ -29,8 +42,11 @@ var _ runtimeEgressStore = (*egressTestStore)(nil) func (e *egressTestStore) GetRuntimeEgress(context.Context, string) (*database.RuntimeEgress, error) { return &e.pinned, nil } -func (e *egressTestStore) ListRuntimeEgressTargets(_ context.Context, after string, limit int) ([]database.RuntimeEgressTarget, error) { - if after != "" { +func (e *egressTestStore) ListRuntimeEgressTargets(_ context.Context, source database.RuntimeEgressSource, after string, limit int) ([]database.RuntimeEgressTarget, error) { + if e.listed != nil { + e.listed <- source + } + if after != "" || (source.Kind != "" && source.Name != e.pinned.SourceName) { return nil, nil } return e.targets, nil @@ -46,20 +62,27 @@ func TestLiveEgressResolvesCurrentSource(t *testing.T) { sandboxes := krt.NewStaticCollection[*v1alpha3.SandboxTemplate](nil, nil, opts.WithName("Sandboxes")...) policies := krt.NewStaticCollection[*v1alpha3.EgressPolicy](nil, nil, opts.WithName("Policies")...) secrets := krt.NewStaticCollection[*corev1.Secret](nil, nil, opts.WithName("Secrets")...) + reader := egressTestReader(t) setSource := func(uid string, refs []corev1.LocalObjectReference) { metadata := metav1.ObjectMeta{Namespace: "team", Name: "source"} metadata.SetUID(types.UID(uid)) if kind == "agent" { - agents.UpdateObject(&v1alpha3.Agent{ObjectMeta: metadata, Spec: v1alpha3.AgentSpec{EgressPolicyRefs: refs}}) + source := &v1alpha3.Agent{ObjectMeta: metadata, Spec: v1alpha3.AgentSpec{EgressPolicyRefs: refs}} + require.NoError(t, ctrlclient.IgnoreNotFound(reader.Delete(ctx, source))) + require.NoError(t, reader.Create(ctx, source)) + agents.UpdateObject(source.DeepCopy()) } else { - sandboxes.UpdateObject(&v1alpha3.SandboxTemplate{ObjectMeta: metadata, Spec: v1alpha3.SandboxTemplateSpec{EgressPolicyRefs: refs}}) + source := &v1alpha3.SandboxTemplate{ObjectMeta: metadata, Spec: v1alpha3.SandboxTemplateSpec{EgressPolicyRefs: refs}} + require.NoError(t, ctrlclient.IgnoreNotFound(reader.Delete(ctx, source))) + require.NoError(t, reader.Create(ctx, source)) + sandboxes.UpdateObject(source.DeepCopy()) } } refs := []corev1.LocalObjectReference{{Name: "access"}} setSource("original", refs) base := egress.Access{Destinations: []string{"https://model.example.com:443"}} store := &egressTestStore{pinned: database.RuntimeEgress{Kind: kind, Namespace: "team", SourceName: "source", SourceUID: "original", EgressDestinations: base.Destinations}} - resolver := NewEgressResolver(&Runtime{Options: opts, Collections: Collections{Agents: agents, SandboxTemplates: sandboxes, EgressPolicies: policies, Secrets: secrets}}, store, nil) + resolver := NewEgressResolver(&Runtime{Options: opts, Collections: Collections{Agents: agents, SandboxTemplates: sandboxes, EgressPolicies: policies, Secrets: secrets}}, store, reader, nil) expect := func(want egress.Access) { t.Helper() require.Eventually(t, func() bool { @@ -69,28 +92,30 @@ func TestLiveEgressResolvesCurrentSource(t *testing.T) { } expect(base) // Missing reference; unrelated runtime configuration is also invalid. policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} - policies.UpdateObject(policy.DeepCopy()) + require.NoError(t, reader.Create(ctx, policy.DeepCopy())) allowed := egress.Access{Destinations: []string{"https://model.example.com:443", "https://tickets.example.com:443"}} expect(allowed) setSource("original", nil) expect(base) setSource("original", refs) expect(allowed) - policies.DeleteObject("team/access") + require.NoError(t, reader.Delete(ctx, policy)) + policy.ResourceVersion = "" expect(base) - policies.UpdateObject(policy.DeepCopy()) + require.NoError(t, reader.Create(ctx, policy)) expect(allowed) setSource("replacement", refs) expect(base) setSource("original", refs) expect(allowed) policy.Spec.Destinations[0].Authentication = &v1alpha3.EgressAuthentication{Header: "authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}, Key: "token"}} - policies.UpdateObject(policy.DeepCopy()) + require.NoError(t, reader.Update(ctx, policy)) expect(base) - secrets.UpdateObject(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "auth"}, Data: map[string][]byte{"token": []byte("secret")}}) + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "auth"}, Data: map[string][]byte{"token": []byte("secret")}} + require.NoError(t, reader.Create(ctx, secret)) allowed.Credentials = []egress.Credential{{Origin: "https://tickets.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token"}} expect(allowed) - secrets.DeleteObject("team/auth") + require.NoError(t, reader.Delete(ctx, secret)) expect(base) }) } @@ -108,10 +133,8 @@ func TestLiveEgressRejectsConflictsWithPinnedRuntime(t *testing.T) { base.Credentials = []egress.Credential{credential} pinned.Credentials = base.Credentials } - credential.URI = "ate-secret://k8s.io/default/team/other/token" - opts := krt.NewOptionsBuilder(t.Context().Done(), "test", nil) - grants := krt.NewStaticCollection(nil, []policyGrants{{Key: "agent/team/source", UID: "uid", Access: egress.Access{Destinations: base.Destinations, Credentials: []egress.Credential{credential}}}}, opts.WithName("Grants")...) - resolver := &EgressResolver{grants: grants, store: &egressTestStore{pinned: pinned}} + reader := egressTestReader(t, egressTestAgent(), egressTestPolicy(base.Destinations[0], &v1alpha3.EgressAuthentication{Header: "authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "other"}, Key: "token"}}), &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "other"}, Data: map[string][]byte{"token": []byte("secret")}}) + resolver := &EgressResolver{reader: reader, warnings: rate.NewLimiter(rate.Every(time.Minute), 1), store: &egressTestStore{pinned: pinned}} got, err := resolver.ResolveEgress(t.Context(), "old-revision") require.NoError(t, err) require.Equal(t, base, got) @@ -143,10 +166,11 @@ func TestLiveEgressReconcilesWithoutInvocationAndAfterRestart(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) - grant := policyGrants{Key: "agent/team/source", UID: "uid", Access: egress.Access{Destinations: []string{"https://tickets.example.com:443"}}} + grant := policyGrants{Source: database.RuntimeEgressSource{Kind: "agent", Namespace: "team", Name: "source"}, UID: "uid", Access: egress.Access{Destinations: []string{"https://tickets.example.com:443"}}} grants := krt.NewStaticCollection(nil, []policyGrants{grant}, opts.WithName("Grants")...) store := &egressTestStore{pinned: database.RuntimeEgress{Kind: "agent", Namespace: "team", SourceName: "source", SourceUID: "uid"}, targets: []database.RuntimeEgressTarget{{ID: "instance", Revision: "old-revision", Atespace: "team", TemplateName: "runtime"}}} - resolver := &EgressResolver{grants: grants, store: store} + reader := egressTestReader(t, egressTestAgent(), egressTestPolicy("https://tickets.example.com", nil)) + resolver := &EgressResolver{grants: grants, store: store, reader: reader} actors := &observingEgressActors{resolver: resolver, applied: make(chan egress.Access, 10)} expect := func(want []string) { t.Helper() @@ -167,6 +191,7 @@ func TestLiveEgressReconcilesWithoutInvocationAndAfterRestart(t *testing.T) { go func() { done <- NewEgressReconciler(resolver, actors).Start(runCtx) }() expect(grant.Access.Destinations) if i == 0 { + require.NoError(t, reader.Delete(ctx, egressTestPolicy("https://tickets.example.com", nil))) grant.Access = egress.Access{} grants.UpdateObject(grant) expect(nil) @@ -179,3 +204,117 @@ func TestLiveEgressReconcilesWithoutInvocationAndAfterRestart(t *testing.T) { func (a *observingEgressActors) GetActor(_ context.Context, space, name string) (*ateapipb.Actor, error) { return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: "uid"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: "runtime"}}, nil } + +func egressTestReader(t *testing.T, objects ...ctrlclient.Object) ctrlclient.WithWatch { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, v1alpha3.AddToScheme(scheme)) + require.NoError(t, corev1.AddToScheme(scheme)) + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() +} + +func egressTestAgent() *v1alpha3.Agent { + return &v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "source", UID: "uid"}, Spec: v1alpha3.AgentSpec{EgressPolicyRefs: []corev1.LocalObjectReference{{Name: "access"}}}} +} + +func egressTestPolicy(origin string, auth *v1alpha3.EgressAuthentication) *v1alpha3.EgressPolicy { + return &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: origin, Authentication: auth}}}} +} + +func TestLiveEgressIgnoresLaggingCache(t *testing.T) { + ctx := t.Context() + policy := egressTestPolicy("https://tickets.example.com", nil) + reader := egressTestReader(t, egressTestAgent(), policy) + opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) + stale := policyGrants{Source: database.RuntimeEgressSource{Kind: "agent", Namespace: "team", Name: "source"}, UID: "uid", Access: egress.Access{Destinations: []string{"https://tickets.example.com:443"}}} + grants := krt.NewStaticCollection(nil, []policyGrants{stale}, opts.WithName("StaleGrants")...) + resolver := &EgressResolver{reader: reader, grants: grants, store: &egressTestStore{pinned: database.RuntimeEgress{Kind: "agent", Namespace: "team", SourceName: "source", SourceUID: "uid"}}} + got, err := resolver.ResolveEgress(ctx, "revision") + require.NoError(t, err) + require.Equal(t, stale.Access, got) + require.NoError(t, reader.Delete(ctx, policy)) + got, err = resolver.ResolveEgress(ctx, "revision") + require.NoError(t, err) + require.Empty(t, got, "a replica with stale grants must not undo revocation") + require.Equal(t, stale.Access, grants.GetKey(stale.ResourceName()).Access) + + unavailable := errors.New("API unavailable") + resolver.reader = interceptor.NewClient(reader, interceptor.Funcs{Get: func(context.Context, ctrlclient.WithWatch, ctrlclient.ObjectKey, ctrlclient.Object, ...ctrlclient.GetOption) error { + return unavailable + }}) + _, err = resolver.ResolveEgress(ctx, "revision") + require.ErrorIs(t, err, unavailable, "transport failures must retry instead of becoming an empty grant") +} + +type retryEgressActors struct { + observingEgressActors + failures atomic.Int32 + mu sync.Mutex + attempts map[string][]time.Time +} + +func (a *retryEgressActors) EnsureActorEgressPolicy(_ context.Context, _, name, _ string) error { + a.mu.Lock() + defer a.mu.Unlock() + a.attempts[name] = append(a.attempts[name], time.Now()) + if name == "session-failing" && a.failures.Add(1) <= 2 { + return errors.New("backend unavailable") + } + return nil +} + +func TestLiveEgressTargetsChangesAndRetriesFailedActors(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) + grant := policyGrants{Source: database.RuntimeEgressSource{Kind: "agent", Namespace: "team", Name: "source"}, UID: "uid"} + grants := krt.NewStaticCollection(nil, []policyGrants{grant}, opts.WithName("Grants")...) + store := &egressTestStore{pinned: database.RuntimeEgress{SourceName: "source"}, listed: make(chan database.RuntimeEgressSource, 10), targets: []database.RuntimeEgressTarget{ + {ID: "failing", Atespace: "team", TemplateName: "runtime"}, {ID: "healthy", Atespace: "team", TemplateName: "runtime"}, + }} + actors := &retryEgressActors{attempts: map[string][]time.Time{}} + done := make(chan error, 1) + go func() { done <- NewEgressReconciler(&EgressResolver{store: store, grants: grants}, actors).Start(ctx) }() + defer func() { cancel(); require.NoError(t, <-done) }() + require.Equal(t, database.RuntimeEgressSource{}, <-store.listed, "startup recovers all actors") + require.Eventually(t, func() bool { + actors.mu.Lock() + defer actors.mu.Unlock() + return len(actors.attempts["session-failing"]) == 3 + }, 5*time.Second, 10*time.Millisecond) + actors.mu.Lock() + attempts := actors.attempts["session-failing"] + healthy := len(actors.attempts["session-healthy"]) + actors.mu.Unlock() + require.Len(t, attempts, 3) + require.GreaterOrEqual(t, attempts[1].Sub(attempts[0]), time.Second) + require.GreaterOrEqual(t, attempts[2].Sub(attempts[1]), 2*time.Second) + require.Equal(t, 1, healthy, "a failure must not retry successful actors") + grant.Source.Name = "unrelated" + grants.UpdateObject(grant) + require.Equal(t, grant.Source, <-store.listed, "events must select only their own source") + grants.DeleteObject(grant.ResourceName()) + require.Equal(t, grant.Source, <-store.listed, "source deletion must also revoke its old actors") +} + +func TestLiveEgressWarnsWhenMergedGrantsExceedLimit(t *testing.T) { + base := egress.Access{} + for i := range 256 { + base.Destinations = append(base.Destinations, fmt.Sprintf("https://host-%03d.example.com:443", i)) + } + var logs bytes.Buffer + ctx := logging.IntoContext(t.Context(), slog.New(slog.NewJSONHandler(&logs, nil))) + resolver := &EgressResolver{ + reader: egressTestReader(t, egressTestAgent(), egressTestPolicy("https://extra.example.com", nil)), + warnings: rate.NewLimiter(rate.Every(time.Hour), 1), + store: &egressTestStore{pinned: database.RuntimeEgress{Kind: "agent", Namespace: "team", SourceName: "source", SourceUID: "uid", EgressDestinations: base.Destinations}}, + } + for range 3 { + got, err := resolver.ResolveEgress(ctx, "full-revision") + require.NoError(t, err) + require.Equal(t, base, got) + } + require.Equal(t, 1, strings.Count(logs.String(), "explicit egress grants withheld")) + require.Contains(t, logs.String(), `"level":"WARN"`) + require.Contains(t, logs.String(), `"revision":"full-revision"`) +} diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index c2a14615b0..0bdefc2ef3 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -106,11 +106,12 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { require.NoError(t, err) require.Nil(t, retained, "last-successful A must remain protected without any sessions") - _, _, err = store.CreateSession(ctx, &apiv1.Session{ + fresh, _, err := store.CreateSession(ctx, &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}, }, "last-good-session") - require.ErrorIs(t, err, database.ErrNotFound, "unresolved inputs must prevent fresh sessions from selecting the retained revision") + require.NoError(t, err, "unresolved edits must leave the last successful revision available") + require.Equal(t, initial.Target.RevisionID.String(), fresh.PreparedRevision) collector, _ := newTestRuntimeRevisionGC(t, store, templates) require.NoError(t, collector.collect(ctx, abandoned.Revision)) diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index 8163c3f0d8..6c873e7b18 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -191,7 +191,7 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { require.NoError(t, err) require.Equal(t, id.String(), session.GetPreparedRevision()) - // Compile failures preserve pinned runtimes but block fresh sessions. + // Compile failures preserve the last successful runtime and pinned retries. state.Target = nil states.UpdateObject(state) require.NoError(t, reconciler.reconcileAgent(ctx, state.ResourceName())) @@ -201,8 +201,6 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { require.False(t, created) require.Equal(t, session.GetId(), retried.GetId()) require.Equal(t, id.String(), retried.GetPreparedRevision()) - _, _, err = store.CreateSession(ctx, request, "unprepared-session") - require.ErrorIs(t, err, database.ErrNotFound) operation, err := store.BeginSessionOperation(ctx, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE) require.NoError(t, err) executor := uuid.New() diff --git a/go/core/internal/controller/sandbox_collections.go b/go/core/internal/controller/sandbox_collections.go index b115994329..25a21be69a 100644 --- a/go/core/internal/controller/sandbox_collections.go +++ b/go/core/internal/controller/sandbox_collections.go @@ -10,7 +10,6 @@ import ( "istio.io/istio/pkg/kube/krt" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -23,7 +22,6 @@ type sandboxCollections struct { } type sandboxReconciliation struct { - Egress egress.Access Template *kagentv1alpha3.SandboxTemplate RevisionID string SourceSnapshot json.RawMessage @@ -91,8 +89,7 @@ func newSandboxCollections(inputs Collections, policy substrate.SandboxPolicy, o if pool == nil { state.Failure = &ReconciliationFailure{Reason: "WorkerPoolNotFound", Message: "The referenced sandbox WorkerPool does not exist"} } else { - var err error - state.Egress, err = translator.CompileEgressPolicies(ctx, translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets}, template.Namespace, template.Spec.EgressPolicyRefs) + _, err := translator.CompileEgressPolicies(ctx, translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets}, template.Namespace, template.Spec.EgressPolicyRefs) if err != nil { state.CompilationError = err.Error() state.Failure = &ReconciliationFailure{Reason: kagentv1alpha3.SandboxTemplateReasonEgressPolicyResolutionFailed, Message: err.Error()} diff --git a/go/core/internal/controller/sandbox_integration_test.go b/go/core/internal/controller/sandbox_integration_test.go index 91d1eb3816..c2a16ce1d4 100644 --- a/go/core/internal/controller/sandbox_integration_test.go +++ b/go/core/internal/controller/sandbox_integration_test.go @@ -119,8 +119,6 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { store.mu.Lock() revision := store.revision.Revision require.True(t, store.ready) - require.Empty(t, store.revision.EgressDestinations) - require.Empty(t, store.revision.Credentials) require.NotContains(t, string(store.revision.SourceSnapshot), "original-token") store.mu.Unlock() other, err := client.ApiV1alpha3().SandboxTemplates(unwatched.Namespace).Get(t.Context(), unwatched.Name, metav1.GetOptions{}) diff --git a/go/core/internal/database/runtime_creation_test.go b/go/core/internal/database/runtime_creation_test.go index e7d6ee82b3..e4d716196a 100644 --- a/go/core/internal/database/runtime_creation_test.go +++ b/go/core/internal/database/runtime_creation_test.go @@ -13,7 +13,7 @@ import ( "google.golang.org/protobuf/proto" ) -func TestRuntimeCreationRequiresCurrentPreparedRevision(t *testing.T) { +func TestRuntimeCreationUsesLatestSuccessfulRevision(t *testing.T) { for _, kind := range []string{"session", "sandbox"} { for _, invalidateFirst := range []bool{false, true} { t.Run(fmt.Sprintf("%s/invalidate_first=%t", kind, invalidateFirst), func(t *testing.T) { @@ -103,19 +103,16 @@ func TestRuntimeCreationRequiresCurrentPreparedRevision(t *testing.T) { }, 5*time.Second, 10*time.Millisecond) resume.Do(func() { close(barrier.resume) }) require.NoError(t, <-invalidation) - if invalidateFirst { - require.ErrorIs(t, <-creation, ErrNotFound) - } else { - require.NoError(t, <-creation) - } + require.NoError(t, <-creation) - // Both unresolved inputs and an ordinary pending update reject fresh - // requests while retries keep their original revision. + // Unresolved and pending edits keep the last successful runtime available. + // Retries retain their original revision even after a newer one is ready. for _, desired := range []string{unresolved, "revision-2"} { require.NoError(t, desire(client, desired)) - _, created, err = create(client, "fresh") - require.ErrorIs(t, err, ErrNotFound) - require.False(t, created) + revision, created, err = create(client, "fresh-"+desired) + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "revision-1", revision) revision, created, err = create(client, "original") require.NoError(t, err) require.False(t, created) @@ -124,7 +121,7 @@ func TestRuntimeCreationRequiresCurrentPreparedRevision(t *testing.T) { prepare("revision-2") revision, created, err = create(client, "fresh") require.NoError(t, err) - require.True(t, created, "rejected requests must not reserve an idempotency key") + require.True(t, created) require.Equal(t, "revision-2", revision) revision, created, err = create(client, "original") require.NoError(t, err) diff --git a/go/core/internal/database/runtime_egress.go b/go/core/internal/database/runtime_egress.go index 9398b9e2f4..ea5d1872d6 100644 --- a/go/core/internal/database/runtime_egress.go +++ b/go/core/internal/database/runtime_egress.go @@ -44,6 +44,15 @@ func (c *Client) GetRuntimeEgress(ctx context.Context, revision string) (*Runtim return &row, nil } +// RuntimeEgressSource selects all revisions of an Agent or SandboxTemplate, +// including former UIDs whose grants must be revoked after source recreation. +// The zero value selects every source for recovery sweeps. +type RuntimeEgressSource struct { + Kind string + Namespace string + Name string +} + // RuntimeEgressTarget identifies an allocated runtime, including suspended or // partially created actors that still need live policy reconciliation. type RuntimeEgressTarget struct { @@ -54,19 +63,24 @@ type RuntimeEgressTarget struct { ActorUID string } -// ListRuntimeEgressTargets lists a bounded page in ID order. Deleting instances +// ListRuntimeEgressTargets lists a bounded page for a source in ID order. Deleting instances // and unissued creations are excluded; a concurrent creation is picked up by the next reconciliation. -func (c *Client) ListRuntimeEgressTargets(ctx context.Context, afterID string, limit int) ([]RuntimeEgressTarget, error) { +func (c *Client) ListRuntimeEgressTargets(ctx context.Context, source RuntimeEgressSource, afterID string, limit int) ([]RuntimeEgressTarget, error) { rows, err := queryMany(ctx, c.db, ` SELECT i.id::text AS id, i.prepared_revision AS revision, r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, COALESCE(s.actor_uid, '') AS actor_uid FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision LEFT JOIN session s ON s.id = i.id WHERE i.id > COALESCE(NULLIF($1, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + AND ($3 = '' OR (r.namespace = $4 AND r.revision IN ( + SELECT revision FROM agent_revision WHERE $3 = 'agent' AND agent_name = $5 + UNION ALL + SELECT revision FROM sandbox_revision WHERE $3 = 'sandbox' AND sandbox_template_name = $5 + ))) AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' ORDER BY i.id LIMIT $2 - `, pgx.RowToStructByName[RuntimeEgressTarget], afterID, limit) + `, pgx.RowToStructByName[RuntimeEgressTarget], afterID, limit, source.Kind, source.Namespace, source.Name) if err != nil { return nil, fmt.Errorf("list runtime egress targets: %w", err) } diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index 5e031a823b..1c4cf1357f 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -54,19 +54,38 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { require.Equal(t, "sandbox", pinned.Kind) require.Equal(t, "scratch", pinned.SourceName) require.Equal(t, "uid", pinned.SourceUID) - first, err := client.ListRuntimeEgressTargets(ctx, "", 1) + for _, test := range []struct { + source RuntimeEgressSource + want string + }{ + {RuntimeEgressSource{Kind: "agent", Namespace: "team-a", Name: "assistant"}, instance.Id}, + {RuntimeEgressSource{Kind: "sandbox", Namespace: "team-a", Name: "scratch"}, created.Id}, + {RuntimeEgressSource{Kind: "agent", Namespace: "other", Name: "assistant"}, ""}, + {RuntimeEgressSource{Kind: "agent", Namespace: "team-a", Name: "unrelated"}, ""}, + } { + targets, err := client.ListRuntimeEgressTargets(ctx, test.source, "", 10) + require.NoError(t, err) + if test.want == "" { + require.Empty(t, targets) + } else { + require.Len(t, targets, 1) + require.Equal(t, test.want, targets[0].ID) + } + } + + first, err := client.ListRuntimeEgressTargets(ctx, RuntimeEgressSource{}, "", 1) require.NoError(t, err) require.Len(t, first, 1) - second, err := client.ListRuntimeEgressTargets(ctx, first[0].ID, 1) + second, err := client.ListRuntimeEgressTargets(ctx, RuntimeEgressSource{}, first[0].ID, 1) require.NoError(t, err) require.Len(t, second, 1) require.ElementsMatch(t, []string{instance.Id, created.Id}, []string{first[0].ID, second[0].ID}) - last, err := client.ListRuntimeEgressTargets(ctx, second[0].ID, 1) + last, err := client.ListRuntimeEgressTargets(ctx, RuntimeEgressSource{}, second[0].ID, 1) require.NoError(t, err) require.Empty(t, last) _, err = client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE) require.NoError(t, err) - remaining, err := client.ListRuntimeEgressTargets(ctx, "", 10) + remaining, err := client.ListRuntimeEgressTargets(ctx, RuntimeEgressSource{}, "", 10) require.NoError(t, err) require.Len(t, remaining, 1) require.Equal(t, created.Id, remaining[0].ID) diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index 5c2b627fd6..e45359b62a 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -174,8 +174,10 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { require.False(t, created) require.Equal(t, session.GetId(), retried.GetId()) require.Equal(t, "new", retried.GetPreparedRevision()) - _, _, err = client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "pending") - require.ErrorIs(t, err, ErrNotFound) + fresh, created, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "pending") + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "new", fresh.PreparedRevision) revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) require.Len(t, revisions, 1) @@ -509,7 +511,7 @@ func TestRecordRuntimeRevisionPromotesOnlyCurrentActivePair(t *testing.T) { assertAvailableRevision("") // Reviving the pair must retain the last success, not a report made while retired. require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) - assertAvailableRevision("") + assertAvailableRevision("second") pair.DesiredRevision = "second" require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) assertAvailableRevision("second") diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index c25ee5695f..ef3a535275 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -6,8 +6,6 @@ import ( "fmt" "github.com/jackc/pgx/v5" - - "github.com/kagent-dev/kagent/go/core/internal/egress" ) type SandboxRevision struct { @@ -15,8 +13,6 @@ type SandboxRevision struct { SandboxTemplateName string SandboxTemplateUID string SourceSnapshot json.RawMessage - EgressDestinations []string - Credentials []egress.Credential } // SandboxTemplateDefinition tracks the desired and last successful runtime for a SandboxTemplate UID. @@ -83,7 +79,7 @@ func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevi return err } revision.Kind = runtimeKindSandbox - if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{RuntimeArtifact: revision.RuntimeArtifact, SourceSnapshot: revision.SourceSnapshot, EgressDestinations: revision.EgressDestinations, Credentials: revision.Credentials}); err != nil { + if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{RuntimeArtifact: revision.RuntimeArtifact, SourceSnapshot: revision.SourceSnapshot}); err != nil { return err } if err := execSQL(ctx, tx, ` @@ -104,20 +100,15 @@ func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevi }) } -// GetSandboxRevision reads immutable sandbox inputs, rejecting malformed stored grants. +// GetSandboxRevision reads immutable sandbox inputs; missing revisions return ErrNotFound. func (c *Client) GetSandboxRevision(ctx context.Context, revision string) (*SandboxRevision, error) { row, err := queryOne(ctx, c.db, ` SELECT r.revision, r.kind, r.namespace, r.actor_template_atespace, r.actor_template_name, r.actor_template_uid, - r.deleted_at, r.source_snapshot, r.egress_destinations, r.credentials, s.sandbox_template_name, s.sandbox_template_uid + r.deleted_at, r.source_snapshot, s.sandbox_template_name, s.sandbox_template_uid FROM runtime_revision r JOIN sandbox_revision s USING (revision) WHERE r.revision = $1 AND r.kind = 'sandbox' `, pgx.RowToStructByName[SandboxRevision], revision) if err != nil { return nil, fmt.Errorf("get sandbox revision: %w", notFoundOr(err)) } - access, err := egress.CanonicalAccess(row.EgressDestinations, row.Credentials) - if err != nil { - return nil, fmt.Errorf("decode sandbox revision %s egress: %w", revision, err) - } - row.EgressDestinations, row.Credentials = access.Destinations, access.Credentials return &row, nil } diff --git a/go/core/internal/database/sandbox_revisions_test.go b/go/core/internal/database/sandbox_revisions_test.go deleted file mode 100644 index e774467e46..0000000000 --- a/go/core/internal/database/sandbox_revisions_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package database - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/kagent-dev/kagent/go/core/internal/egress" -) - -func TestSandboxRevisionPinsEgress(t *testing.T) { - client := NewClient(setupTestDB(t), "public") - ctx := t.Context() - input, options := sandboxFixture(t, client, "initial") - revision := SandboxRevision{ - RuntimeArtifact: RuntimeArtifact{Revision: "tickets", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "tickets", ActorTemplateUID: "actor-uid"}, - SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`), - EgressDestinations: []string{"https://tickets.example.com:443"}, - Credentials: []egress.Credential{{Origin: "https://tickets.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/ticket-auth/token"}}, - } - require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision.Revision})) - require.NoError(t, client.RecordSandboxRevision(ctx, revision, true)) - created, _, err := client.CreateSandbox(ctx, input, "create", options) - require.NoError(t, err) - require.Equal(t, revision.Revision, created.PreparedRevision) - saved, err := client.GetSandboxRevision(ctx, created.PreparedRevision) - require.NoError(t, err) - require.Equal(t, revision.EgressDestinations, saved.EgressDestinations) - require.Equal(t, revision.Credentials, saved.Credentials) - // Retries may refresh artifact identity, but cannot rewrite an existing grant. - revision.EgressDestinations, revision.Credentials = nil, nil - require.NoError(t, client.RecordSandboxRevision(ctx, revision, true)) - _, _ = sandboxFixture(t, client, "new-policy") - retried, _, err := client.CreateSandbox(ctx, input, "create", options) - require.NoError(t, err) - require.Equal(t, created.PreparedRevision, retried.PreparedRevision) - pinned, err := client.GetSandboxRevision(ctx, retried.PreparedRevision) - require.NoError(t, err) - require.Equal(t, saved.EgressDestinations, pinned.EgressDestinations) - require.Equal(t, saved.Credentials, pinned.Credentials) - - // Reject malformed durable data instead of silently dropping authentication. - _, err = client.db.Exec(ctx, `UPDATE runtime_revision SET credentials = '[{"hostname":"tickets.example.com","header":"authorization","uri":"ate-secret://k8s.io/default/team-a/ticket-auth/token"}]' WHERE revision = $1`, created.PreparedRevision) - require.NoError(t, err) - _, err = client.GetSandboxRevision(ctx, created.PreparedRevision) - require.ErrorContains(t, err, "decode sandbox revision") -} diff --git a/go/core/internal/database/sandboxes.go b/go/core/internal/database/sandboxes.go index 4cb97fec1d..412e21cc40 100644 --- a/go/core/internal/database/sandboxes.go +++ b/go/core/internal/database/sandboxes.go @@ -63,8 +63,8 @@ type SandboxCreateOptions struct { TTL time.Duration } -// CreateSandbox atomically reserves a sandbox and its current desired revision, -// or returns ErrNotFound if it is not prepared. A duplicate creator/requestID +// CreateSandbox atomically reserves a sandbox and its latest successful revision, +// or returns ErrNotFound if none is available. A duplicate creator/requestID // returns the original resource when inputs match. // The boolean reports whether this call created the sandbox. func (c *Client) CreateSandbox(ctx context.Context, request *apiv1.Sandbox, requestID string, options SandboxCreateOptions) (*apiv1.Sandbox, bool, error) { @@ -114,7 +114,7 @@ func readSandboxRequest(ctx context.Context, db dbExecutor, userID, requestID st } // insertSandbox reserves a new sandbox in CREATING/CREATE and pins the active -// template's current desired revision only when it is prepared. A missing prepared +// template's latest successful revision. A missing prepared // target returns ErrNotFound. A duplicate creator/requestID returns // pgx.ErrNoRows so the caller can roll back and read the winning transaction. func insertSandbox(ctx context.Context, tx pgx.Tx, request *apiv1.Sandbox, requestID string, options SandboxCreateOptions) (sandboxRow, error) { @@ -126,7 +126,7 @@ func insertSandbox(ctx context.Context, tx pgx.Tx, request *apiv1.Sandbox, reque err := tx.QueryRow(ctx, ` SELECT latest_successful_revision, clock_timestamp() FROM sandbox_template_definition WHERE namespace = $1 AND sandbox_template_name = $2 AND sandbox_template_uid = $3 - AND retired_at IS NULL AND desired_revision = latest_successful_revision + AND retired_at IS NULL AND latest_successful_revision IS NOT NULL FOR UPDATE `, request.GetSandboxTemplate().GetNamespace(), request.GetSandboxTemplate().GetName(), options.TemplateUID).Scan(&revision, &createdAt) if err != nil { diff --git a/go/core/internal/database/sessions.go b/go/core/internal/database/sessions.go index eebb7287ce..918f51801a 100644 --- a/go/core/internal/database/sessions.go +++ b/go/core/internal/database/sessions.go @@ -53,7 +53,7 @@ func sameSessionRequest(session, request *apiv1.Session) bool { } // CreateSession atomically reserves a session, its conversation history, and the -// current desired revision, or returns ErrNotFound if it is not prepared. A repeated +// latest successful revision, or returns ErrNotFound if none is available. A repeated // creator/requestID returns the existing session when the Agent matches, or // ErrIdempotencyConflict otherwise. A deleted session returns ErrFailedPrecondition // and keeps its request ID reserved. The boolean reports whether this call created @@ -101,7 +101,7 @@ func (c *Client) CreateSession(ctx context.Context, request *apiv1.Session, requ } // insertSession reserves a new conversation in CREATING/CREATE and pins the active -// Agent's current desired revision only when it is prepared. It locks the definition +// Agent's latest successful revision. It locks the definition // before the revision to serialize creation with reconciliation. Callers must supply // a transaction so the history and session commit together. A missing prepared target // returns ErrNotFound; a duplicate creator/requestID returns pgx.ErrNoRows for the caller @@ -115,11 +115,11 @@ func insertSession(ctx context.Context, db pgx.Tx, request *apiv1.Session, reque SELECT p.latest_successful_revision AS revision, clock_timestamp() AS db_time FROM agent_definition p WHERE p.namespace = $1 AND p.agent_name = $2 AND p.retired_at IS NULL - AND p.desired_revision = p.latest_successful_revision + AND p.latest_successful_revision IS NOT NULL FOR UPDATE OF p `, pgx.RowToStructByName[preparedRevision], request.GetAgent().GetNamespace(), request.GetAgent().GetName()) if err != nil { - return sessionRow{}, fmt.Errorf("get current prepared runtime revision: %w", notFoundOr(err)) + return sessionRow{}, fmt.Errorf("get latest prepared runtime revision: %w", notFoundOr(err)) } if _, err := getAvailableRuntimeRevisionForUpdate(ctx, db, revision.Revision); err != nil { return sessionRow{}, err diff --git a/go/core/internal/service/sandbox/service_test.go b/go/core/internal/service/sandbox/service_test.go index 23440bbc1a..19aebfda3b 100644 --- a/go/core/internal/service/sandbox/service_test.go +++ b/go/core/internal/service/sandbox/service_test.go @@ -22,7 +22,6 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/pkg/auth" ) @@ -293,8 +292,6 @@ func TestSandboxResolvesCurrentGrantsBeforeResume(t *testing.T) { revision := database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "policy-revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "policy-runtime", ActorTemplateUID: "policy-runtime-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - EgressDestinations: []string{"https://tickets.example.com:443"}, - Credentials: []egress.Credential{{Origin: "https://tickets.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/ticket-auth/token"}}, } store := service.config.Store require.NoError(t, store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: revision.Revision})) diff --git a/go/core/internal/substrate/egress.go b/go/core/internal/substrate/egress.go index f720e83897..e3a3f895af 100644 --- a/go/core/internal/substrate/egress.go +++ b/go/core/internal/substrate/egress.go @@ -11,9 +11,10 @@ import ( ) // EnsureActorEgressPolicy installs current authorization without restarting the -// actor. Read the backend version BEFORE resolving grants: an older in-flight -// writer then cannot overwrite a newer policy. A conflict retries resolution, -// never the stale policy. Substrate deletes the policy with its Actor. +// actor. Read the backend version before resolving grants through authoritative +// Kubernetes reads. The version precondition orders competing policy writes; it +// cannot make cached Kubernetes state current. Conflicts retry both reads. +// Substrate deletes the policy with its Actor. func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name, revision string) error { ctx, cancel := c.callCtx(ctx) defer cancel() diff --git a/go/core/internal/translator/egress.go b/go/core/internal/translator/egress.go index 848f9abac6..01cfd0942a 100644 --- a/go/core/internal/translator/egress.go +++ b/go/core/internal/translator/egress.go @@ -1,13 +1,18 @@ package translator import ( + "context" "fmt" "strings" "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/egress" ) @@ -15,6 +20,36 @@ import ( // SandboxTemplate. KRT tracks policy and Secret dependencies; only references // enter the result, so Secret value rotation does not change a revision. func CompileEgressPolicies(ctx krt.HandlerContext, collections Collections, namespace string, refs []corev1.LocalObjectReference) (egress.Access, error) { + return compileEgressPolicies(namespace, refs, func(name string) (*v1alpha3.EgressPolicy, error) { + policy := krt.FetchOne(ctx, collections.EgressPolicies, krt.FilterKey(namespace+"/"+name)) + if policy == nil { + return nil, apierrors.NewNotFound(schema.GroupResource{Group: "api.kagent.dev", Resource: "egresspolicies"}, name) + } + return *policy, nil + }, func(name string) (*corev1.Secret, error) { + secret := krt.FetchOne(ctx, collections.Secrets, krt.FilterKey(namespace+"/"+name)) + if secret == nil { + return nil, apierrors.NewNotFound(corev1.Resource("secrets"), name) + } + return *secret, nil + }) +} + +// ResolveEgressPolicies applies the same validation as preparation to direct +// Kubernetes reads. Live authorization must supply an uncached reader. +func ResolveEgressPolicies(ctx context.Context, reader ctrlclient.Reader, namespace string, refs []corev1.LocalObjectReference) (egress.Access, error) { + return compileEgressPolicies(namespace, refs, func(name string) (*v1alpha3.EgressPolicy, error) { + policy := &v1alpha3.EgressPolicy{} + err := reader.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: name}, policy) + return policy, err + }, func(name string) (*corev1.Secret, error) { + secret := &corev1.Secret{} + err := reader.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: name}, secret) + return secret, err + }) +} + +func compileEgressPolicies(namespace string, refs []corev1.LocalObjectReference, getPolicy func(string) (*v1alpha3.EgressPolicy, error), getSecret func(string) (*corev1.Secret, error)) (egress.Access, error) { if len(refs) > 16 { return egress.Access{}, NewValidationError("at most 16 egressPolicyRefs are supported") } @@ -25,14 +60,17 @@ func CompileEgressPolicies(ctx krt.HandlerContext, collections Collections, name return egress.Access{}, NewValidationError("invalid EgressPolicy reference %q", ref.Name) } key := namespace + "/" + ref.Name - policy := krt.FetchOne(ctx, collections.EgressPolicies, krt.FilterKey(key)) - if policy == nil { - return egress.Access{}, fmt.Errorf("resolve EgressPolicy %q: not found", key) + policy, err := getPolicy(ref.Name) + if err != nil { + return egress.Access{}, fmt.Errorf("resolve EgressPolicy %q: %w", key, err) } - if len((*policy).Spec.Destinations) == 0 || len((*policy).Spec.Destinations) > 64 { + if !policy.DeletionTimestamp.IsZero() { + return egress.Access{}, NewValidationError("EgressPolicy %q is being deleted", key) + } + if len(policy.Spec.Destinations) == 0 || len(policy.Spec.Destinations) > 64 { return egress.Access{}, NewValidationError("EgressPolicy %q requires 1 to 64 destinations", key) } - for _, destination := range (*policy).Spec.Destinations { + for _, destination := range policy.Spec.Destinations { origin, err := egress.ParseOrigin(destination.Origin) if err != nil { return egress.Access{}, NewValidationError("EgressPolicy %q: %v", key, err) @@ -57,12 +95,12 @@ func CompileEgressPolicies(ctx krt.HandlerContext, collections Collections, name if _, err := egress.CanonicalCredentials([]egress.Credential{binding}); err != nil { return egress.Access{}, NewValidationError("EgressPolicy %q: %v", key, err) } - found := krt.FetchOne(ctx, collections.Secrets, krt.FilterKey(namespace+"/"+secret.Name)) - if found == nil { - return egress.Access{}, fmt.Errorf("EgressPolicy %q: Secret %q not found", key, secret.Name) + found, err := getSecret(secret.Name) + if err != nil { + return egress.Access{}, fmt.Errorf("EgressPolicy %q: Secret %q: %w", key, secret.Name, err) } - if _, ok := (*found).Data[secret.Key]; !ok { - return egress.Access{}, fmt.Errorf("EgressPolicy %q: Secret %q is missing key %q", key, secret.Name, secret.Key) + if _, ok := found.Data[secret.Key]; !ok { + return egress.Access{}, NewValidationError("EgressPolicy %q: Secret %q is missing key %q", key, secret.Name, secret.Key) } credentials = append(credentials, binding) } diff --git a/go/core/pkg/app/app.go b/go/core/pkg/app/app.go index 08ac011ad9..864c43188c 100644 --- a/go/core/pkg/app/app.go +++ b/go/core/pkg/app/app.go @@ -276,7 +276,7 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er if err != nil { return err } - egressResolver := controller.NewEgressResolver(runtime, store, watchNamespaces) + egressResolver := controller.NewEgressResolver(runtime, store, manager.GetAPIReader(), watchNamespaces) actors, err := substrate.Dial(ctx, substrate.Config{ EgressResolver: egressResolver, AteAPIEndpoint: cfg.GetSubstrate().GetApiEndpoint(), diff --git a/go/core/pkg/migrations/core/000002_runtime_passthrough_origins.sql b/go/core/pkg/migrations/core/000002_runtime_passthrough_origins.sql index 8e6e529642..1280ecd57c 100644 --- a/go/core/pkg/migrations/core/000002_runtime_passthrough_origins.sql +++ b/go/core/pkg/migrations/core/000002_runtime_passthrough_origins.sql @@ -8,8 +8,18 @@ CREATE VIEW agent_runtime_revision AS SELECT r.*, a.agent_name, a.agent_uid, a.agent_card FROM runtime_revision r JOIN agent_revision a USING (revision) WHERE r.kind = 'agent'; +-- Target changed sources without scanning all live instances. +CREATE INDEX agent_revision_source_idx ON agent_revision (agent_name, revision); +CREATE INDEX sandbox_revision_source_idx ON sandbox_revision (sandbox_template_name, revision); +CREATE INDEX runtime_instance_revision_idx ON runtime_instance (prepared_revision, id) + WHERE state <> 'RUNTIME_STATE_DELETED' AND operation <> 'RUNTIME_OPERATION_DELETE'; + -- +goose Down +DROP INDEX runtime_instance_revision_idx; +DROP INDEX sandbox_revision_source_idx; +DROP INDEX agent_revision_source_idx; + DROP VIEW agent_runtime_revision; ALTER TABLE runtime_revision DROP COLUMN passthrough_origins; CREATE VIEW agent_runtime_revision AS diff --git a/go/core/test/e2e/README.md b/go/core/test/e2e/README.md index 451bf31af4..cca55676bf 100644 --- a/go/core/test/e2e/README.md +++ b/go/core/test/e2e/README.md @@ -244,9 +244,18 @@ active invocation, it adds a policy reference and revokes the destination, checking actual script output and destination requests without replacing the actor. It also checks that deleting a policy while suspended revokes access after resume and in a checkpoint fork. Runtime configuration stays on the same -prepared revision. `TestSandboxEgressPolicyUpdatesLiveActor` exercises live +prepared revision. New sessions continue using that revision with current grants +even when a policy cannot resolve. `TestSandboxEgressPolicyUpdatesLiveActor` exercises live grant, revocation, and resume through the standalone sandbox process API. +`TestEgressPolicyCredentialDelivery` runs the same skill on every harness against +an HTTPS server that rejects requests unless both Authorization (with a Bearer +prefix) and X-API-Key (without a prefix) match the Secret. The script has only +inert placeholders. This test runs sequentially because it temporarily installs +a test origin CA in the Substrate gateway, rolls it out, and restores the gateway +configuration afterwards. Actor identity, policy checks, and credential injection +use the normal Substrate path. + `TestSessionHTTPInteraction` discovers an Agent's Agent Card, invokes the advertised JSON-RPC interface, streams a second turn, and checks task persistence across HTTP and gRPC. `TestSessionHTTPResubscribeAndCancel` diff --git a/go/core/test/e2e/egress_credentials_test.go b/go/core/test/e2e/egress_credentials_test.go new file mode 100644 index 0000000000..a0f87b7b1d --- /dev/null +++ b/go/core/test/e2e/egress_credentials_test.go @@ -0,0 +1,189 @@ +package e2e_test + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" +) + +// Keep the parent sequential: the fixture installs a temporary HTTPS root in +// the gateway and restores its configuration before other parallel E2Es run. +func TestEgressPolicyCredentialDelivery(t *testing.T) { + target := interactionTarget(t) + kube := interactionKubeClient(t) + const token = "explicit-policy-test-token" + destination := startAuthenticatedEgressDestination(t, kube, token) + image := publishSkillImage(t) + forEachHarness(t, func(t *testing.T, harness testHarness) { + t.Parallel() + modelURL, model, runs := startEgressSkillMock(t, harness.skillsRoot(), destination) + config := harness.createModel(t, kube, modelURL, nil) + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-credentials-", Namespace: "kagent", Labels: harness.labels()}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: config.Name}, SystemPrompt: "Load the egress skill and run its probe.", + Skills: []v1alpha3.AgentTemplateSkill{{Name: "egress", Source: v1alpha3.ArtifactSource{OCI: image, Path: "egress"}}}, + }} + createAndWaitInteractionTemplate(t, harness, kube, template) + fixture := newInteractionFixtureForTemplate(t, harness, target, template.Name) + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-header-", Namespace: "kagent"}, StringData: map[string]string{"token": token}} + require.NoError(t, kube.Create(t.Context(), secret)) + t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), secret)) }) + ref := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secret.Name}, Key: "token"} + var refs []corev1.LocalObjectReference + for _, authentication := range []*v1alpha3.EgressAuthentication{ + {Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: ref}, + {Header: "X-API-Key", SecretKeyRef: ref}, + } { + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-header-", Namespace: "kagent"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: destination, Authentication: authentication}}}} + require.NoError(t, kube.Create(t.Context(), policy)) + t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), policy)) }) + refs = append(refs, corev1.LocalObjectReference{Name: policy.Name}) + } + run := runs("credentials") + finish := runEgressSkill(t, fixture, model, run) + run.expect(t, "initial", "DENIED") + agent := &v1alpha3.Agent{} + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKey{Namespace: "kagent", Name: template.Name}, agent)) + agent.Spec.EgressPolicyRefs = refs + require.NoError(t, kube.Update(t.Context(), agent)) + run.setPhase("authenticated") + run.expect(t, "authenticated", "ALLOWED") + // The HTTPS server returns success only when both placeholders have been + // replaced from the Secret, with and without a prefix respectively. + finish() + }) +} + +func startAuthenticatedEgressDestination(t *testing.T, kube ctrlclient.Client, token string) string { + t.Helper() + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer "+token || r.Header.Get("X-API-Key") != token { + http.Error(w, "credential injection required", http.StatusForbidden) + return + } + _, _ = io.WriteString(w, "EGRESS_DESTINATION_AUTHENTICATED\n") + })) + require.NoError(t, server.Listener.Close()) + listener, err := net.Listen("tcp", "0.0.0.0:0") + require.NoError(t, err) + server.Listener = listener + origin := reachableServerURL(t, "https://"+listener.Addr().String(), "") + parsed, err := url.Parse(origin) + require.NoError(t, err) + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + ca := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "kagent E2E CA"}, + NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) + require.NoError(t, err) + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + certificate := &x509.Certificate{ + SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "kagent E2E origin"}, DNSNames: []string{parsed.Hostname()}, IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + NotBefore: ca.NotBefore, NotAfter: ca.NotAfter, + BasicConstraintsValid: true, KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, certificate, ca, &key.PublicKey, caKey) + require.NoError(t, err) + server.TLS = &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{{Certificate: [][]byte{der, caDER}, PrivateKey: key}}} + server.StartTLS() + t.Cleanup(server.Close) + // Prove a successful fetch cannot be produced by a missing/unchanged header. + for _, value := range []string{"", "kagent-credential-injected"} { + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://"+net.JoinHostPort("127.0.0.1", parsed.Port()), nil) + require.NoError(t, err) + request.Header.Set("Authorization", "Bearer "+value) + request.Header.Set("X-API-Key", value) + response, err := server.Client().Do(request) + require.NoError(t, err) + require.NoError(t, response.Body.Close()) + require.Equal(t, http.StatusForbidden, response.StatusCode) + } + trustEgressTestOrigin(t, kube, string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}))) + return origin +} + +// Configure only the gateway's destination TLS trust. Actor identity, policy +// lookup, credential provider, and guest MITM trust use the normal installation. +func trustEgressTestOrigin(t *testing.T, kube ctrlclient.Client, certificate string) { + t.Helper() + require.NoError(t, appsv1.AddToScheme(kube.Scheme())) + deployments := &appsv1.DeploymentList{} + require.NoError(t, kube.List(t.Context(), deployments, ctrlclient.InNamespace("ate-system"), ctrlclient.MatchingLabels{"app": "atenet-egress"})) + require.Len(t, deployments.Items, 1) + deployment := &deployments.Items[0] + var configName string + for _, volume := range deployment.Spec.Template.Spec.Volumes { + if volume.Name == "config" && volume.ConfigMap != nil { + configName = volume.ConfigMap.Name + } + } + require.NotEmpty(t, configName) + config := &corev1.ConfigMap{} + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKey{Namespace: deployment.Namespace, Name: configName}, config)) + original := config.DeepCopy() + const rootFile = "egress-test-root.pem" + require.NotContains(t, config.Data, rootFile) + require.Equal(t, 1, strings.Count(config.Data["config.yaml"], "backendTLS: {}")) + config.Data["config.yaml"] = strings.Replace(config.Data["config.yaml"], "backendTLS: {}", "backendTLS:\n root: /etc/agentgateway/"+rootFile, 1) + config.Data[rootFile] = certificate + rollout := func(ctx context.Context) { + t.Helper() + current := &appsv1.Deployment{} + deploymentKey := ctrlclient.ObjectKeyFromObject(deployment) + require.NoError(t, kube.Get(ctx, deploymentKey, current)) + before := current.DeepCopy() + if current.Spec.Template.Annotations == nil { + current.Spec.Template.Annotations = map[string]string{} + } + current.Spec.Template.Annotations["kagent.dev/egress-test-rollout"] = uuid.NewString() + require.NoError(t, kube.Patch(ctx, current, ctrlclient.MergeFrom(before))) + require.NoError(t, wait.PollUntilContextTimeout(ctx, time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + observed := &appsv1.Deployment{} + if err := kube.Get(ctx, deploymentKey, observed); err != nil { + return false, err + } + return observed.Status.ObservedGeneration >= current.Generation && observed.Status.UpdatedReplicas == *observed.Spec.Replicas && observed.Status.AvailableReplicas == *observed.Spec.Replicas && observed.Status.Replicas == *observed.Spec.Replicas, nil + })) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + current := &corev1.ConfigMap{} + require.NoError(t, kube.Get(ctx, ctrlclient.ObjectKeyFromObject(original), current)) + before := current.DeepCopy() + current.Data["config.yaml"] = original.Data["config.yaml"] + delete(current.Data, rootFile) + require.NoError(t, kube.Patch(ctx, current, ctrlclient.MergeFrom(before))) + rollout(ctx) + }) + require.NoError(t, kube.Patch(t.Context(), config, ctrlclient.MergeFrom(original))) + rollout(t.Context()) +} diff --git a/go/core/test/e2e/egress_test.go b/go/core/test/e2e/egress_test.go index 3df9f3f9b2..112714b15a 100644 --- a/go/core/test/e2e/egress_test.go +++ b/go/core/test/e2e/egress_test.go @@ -24,8 +24,6 @@ import ( "github.com/google/uuid" "github.com/kagent-dev/mockllm" "github.com/stretchr/testify/require" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -51,7 +49,7 @@ for i in $(seq 1 900); do wget -T 5 -qO- "$control/stopped" >/dev/null exit fi - if wget -T 3 -qO- "$target/?phase=$phase"; then + if wget -T 3 -qO- --header 'Authorization: Bearer kagent-credential-injected' --header 'X-API-Key: kagent-credential-injected' "$target/?phase=$phase"; then result=ALLOWED else result=DENIED @@ -141,8 +139,19 @@ func TestSessionEgressPolicyUpdatesLiveActors(t *testing.T) { return apimeta.IsStatusConditionFalse(agent.Status.Conditions, v1alpha3.AgentConditionReady), nil })) require.Equal(t, revision, agent.Status.LatestSuccessfulRevision, "policy edits must not replace the runtime revision") - _, err = fixture.sessions.CreateSession(fixture.ctx, &apiv1.CreateSessionRequest{Agent: &apiv1.ResourceReference{Namespace: key.Namespace, Name: key.Name}, RequestId: uuid.NewString()}) - require.Equal(t, codes.FailedPrecondition, status.Code(err)) + created, err := fixture.sessions.CreateSession(fixture.ctx, &apiv1.CreateSessionRequest{Agent: &apiv1.ResourceReference{Namespace: key.Namespace, Name: key.Name}, RequestId: uuid.NewString()}) + require.NoError(t, err, "unresolved policy must not block the last successful runtime") + require.Equal(t, revision, created.Session.PreparedRevision) + fresh := *fixture + fresh.sessionID, fresh.contextID = created.Session.Id, created.Session.ContextId + t.Cleanup(func() { + require.NoError(t, deleteIdleSession(context.WithoutCancel(fresh.ctx), fresh.sessions, fresh.sessionID)) + }) + run = runs("fresh") + finish = runEgressSkill(t, &fresh, model, run) + run.expect(t, "initial", "DENIED") + require.Zero(t, requests("fresh_initial")) + finish() run = runs("resumed") finish = runEgressSkill(t, fixture, model, run) run.expect(t, "initial", "DENIED") @@ -439,8 +448,12 @@ func TestSandboxEgressPolicyUpdatesLiveActor(t *testing.T) { } return apimeta.IsStatusConditionFalse(template.Status.Conditions, "Ready"), nil })) - _, err = f.client.CreateSandbox(f.ctx, &apiv1.CreateSandboxRequest{SandboxTemplate: f.template, RequestId: uuid.NewString()}) - require.Equal(t, codes.FailedPrecondition, status.Code(err)) + original := instance + instance = f.create(t, 5*time.Minute) + require.Equal(t, original.PreparedRevision, instance.PreparedRevision) + require.Contains(t, probe("fresh"), "DENIED") + require.Zero(t, requests("fresh")) + instance = original _, err = f.client.SuspendSandbox(f.ctx, &apiv1.SuspendSandboxRequest{SandboxId: instance.Id}) require.NoError(t, err) _, err = f.client.ResumeSandbox(f.ctx, &apiv1.ResumeSandboxRequest{SandboxId: instance.Id}) From a12f66442dc31acf9baf64b5458b0a6b18f28940 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 15:17:04 +0000 Subject: [PATCH 14/43] refactor: clarify token passthrough origin naming Signed-off-by: Eitan Yarmush --- go/core/internal/controller/egress.go | 2 +- go/core/internal/controller/egress_test.go | 2 +- go/core/internal/controller/reconciler.go | 8 +-- go/core/internal/database/models.go | 24 ++++---- go/core/internal/database/runtime_egress.go | 18 +++--- .../internal/database/runtime_egress_test.go | 4 +- .../internal/database/runtime_revisions.go | 56 +++++++++---------- go/core/internal/egress/live.go | 4 +- go/core/internal/translator/byo/compiler.go | 4 +- .../internal/translator/claude/compiler.go | 8 +-- go/core/internal/translator/codex/compiler.go | 4 +- go/core/internal/translator/compiler.go | 2 +- go/core/internal/translator/credentials.go | 8 +-- .../internal/translator/kagent/compiler.go | 4 +- go/core/internal/translator/revision.go | 38 ++++++------- ...002_runtime_token_passthrough_origins.sql} | 4 +- go/core/pkg/migrations/runner_test.go | 2 +- 17 files changed, 96 insertions(+), 96 deletions(-) rename go/core/pkg/migrations/core/{000002_runtime_passthrough_origins.sql => 000002_runtime_token_passthrough_origins.sql} (86%) diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go index 56b392a58a..80f22a0755 100644 --- a/go/core/internal/controller/egress.go +++ b/go/core/internal/controller/egress.go @@ -99,7 +99,7 @@ func (e *EgressResolver) ResolveEgress(ctx context.Context, revision string) (eg if err != nil { return egress.Access{}, err } - access, err := egress.MergeAccess(base, grants, pinned.PassthroughOrigins) + access, err := egress.MergeAccess(base, grants, pinned.TokenPassthroughOrigins) if err != nil { // A current grant may conflict with an old session's pinned integrations. // Revoke its explicit access instead of preserving a stale policy. diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go index 6f0f1fc340..706ba49127 100644 --- a/go/core/internal/controller/egress_test.go +++ b/go/core/internal/controller/egress_test.go @@ -128,7 +128,7 @@ func TestLiveEgressRejectsConflictsWithPinnedRuntime(t *testing.T) { credential := egress.Credential{Origin: base.Destinations[0], Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token"} pinned := database.RuntimeEgress{Kind: "agent", Namespace: "team", SourceName: "source", SourceUID: "uid", EgressDestinations: base.Destinations} if passthrough { - pinned.PassthroughOrigins = base.Destinations + pinned.TokenPassthroughOrigins = base.Destinations } else { base.Credentials = []egress.Credential{credential} pinned.Credentials = base.Credentials diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index e7191dbf86..1b72eeaddc 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -368,10 +368,10 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { Revision: target.RevisionID.String(), Namespace: definition.Namespace, AgentName: definition.AgentName, AgentUID: definition.AgentUID, SourceSnapshot: target.Revision.Provenance, AgentCard: target.Revision.AgentCard, - EgressDestinations: target.Revision.EgressDestinations, - Credentials: target.Revision.Credentials, - PassthroughOrigins: target.Revision.PassthroughOrigins, - ActorTemplateAtespace: observed.GetMetadata().GetAtespace(), ActorTemplateName: observed.GetMetadata().GetName(), ActorTemplateUID: observed.GetMetadata().GetUid(), + EgressDestinations: target.Revision.EgressDestinations, + Credentials: target.Revision.Credentials, + TokenPassthroughOrigins: target.Revision.TokenPassthroughOrigins, + ActorTemplateAtespace: observed.GetMetadata().GetAtespace(), ActorTemplateName: observed.GetMetadata().GetName(), ActorTemplateUID: observed.GetMetadata().GetUid(), } ready := observed.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() != nil if err := r.store.RecordRuntimeRevision(ctx, revision, ready); err != nil { diff --git a/go/core/internal/database/models.go b/go/core/internal/database/models.go index 463a8062af..f7d9a5a453 100644 --- a/go/core/internal/database/models.go +++ b/go/core/internal/database/models.go @@ -59,18 +59,18 @@ type AgentDefinition struct { } type RuntimeRevision struct { - Revision string - Namespace string - AgentName string - AgentUID string - SourceSnapshot json.RawMessage - AgentCard *a2apb.AgentCard - Credentials []egress.Credential - EgressDestinations []string - PassthroughOrigins []string - ActorTemplateAtespace string - ActorTemplateName string - ActorTemplateUID string + Revision string + Namespace string + AgentName string + AgentUID string + SourceSnapshot json.RawMessage + AgentCard *a2apb.AgentCard + Credentials []egress.Credential + EgressDestinations []string + TokenPassthroughOrigins []string + ActorTemplateAtespace string + ActorTemplateName string + ActorTemplateUID string } // SessionQuery narrows a page of sessions to an optional Agent. diff --git a/go/core/internal/database/runtime_egress.go b/go/core/internal/database/runtime_egress.go index ea5d1872d6..4b12302398 100644 --- a/go/core/internal/database/runtime_egress.go +++ b/go/core/internal/database/runtime_egress.go @@ -12,20 +12,20 @@ import ( // RuntimeEgress retains integration access and the source identity of a pinned // revision. Explicit policy grants are resolved from the current Kubernetes source. type RuntimeEgress struct { - Kind string - Namespace string - SourceName string - SourceUID string - EgressDestinations []string - Credentials []egress.Credential - PassthroughOrigins []string + Kind string + Namespace string + SourceName string + SourceUID string + EgressDestinations []string + Credentials []egress.Credential + TokenPassthroughOrigins []string } // GetRuntimeEgress reads immutable access and source identity for either runtime // kind. Missing revisions return ErrNotFound; malformed grants return an error. func (c *Client) GetRuntimeEgress(ctx context.Context, revision string) (*RuntimeEgress, error) { row, err := queryOne(ctx, c.db, ` - SELECT r.kind, r.namespace, r.egress_destinations, r.credentials, r.passthrough_origins, + SELECT r.kind, r.namespace, r.egress_destinations, r.credentials, r.token_passthrough_origins, COALESCE(a.agent_name, s.sandbox_template_name) AS source_name, COALESCE(a.agent_uid, s.sandbox_template_uid) AS source_uid FROM runtime_revision r @@ -36,7 +36,7 @@ func (c *Client) GetRuntimeEgress(ctx context.Context, revision string) (*Runtim if err != nil { return nil, fmt.Errorf("get runtime egress %s: %w", revision, notFoundOr(err)) } - access, err := egress.MergeAccess(egress.Access{Destinations: row.EgressDestinations, Credentials: row.Credentials}, egress.Access{}, row.PassthroughOrigins) + access, err := egress.MergeAccess(egress.Access{Destinations: row.EgressDestinations, Credentials: row.Credentials}, egress.Access{}, row.TokenPassthroughOrigins) if err != nil { return nil, fmt.Errorf("decode runtime egress %s: %w", revision, err) } diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index 1c4cf1357f..d220c398d7 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -13,7 +13,7 @@ import ( func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() - revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", PassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} + revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", TokenPassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} require.NoError(t, client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", DesiredRevision: "agent"})) require.NoError(t, client.RecordRuntimeRevision(ctx, revision, true)) instance, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "request") @@ -40,7 +40,7 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { require.NoError(t, err) require.Equal(t, "old-uid", pinned.SourceUID) require.Equal(t, "agent", pinned.Kind) - require.Equal(t, revision.PassthroughOrigins, pinned.PassthroughOrigins) + require.Equal(t, revision.TokenPassthroughOrigins, pinned.TokenPassthroughOrigins) sandbox, options := sandboxFixture(t, client, "sandbox") created, _, err := client.CreateSandbox(ctx, sandbox, "sandbox", options) require.NoError(t, err) diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index 0fff04e33b..dbd9245dab 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -97,7 +97,7 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{ RuntimeArtifact: RuntimeArtifact{Revision: revision.Revision, Kind: runtimeKindAgent, Namespace: revision.Namespace, ActorTemplateAtespace: revision.ActorTemplateAtespace, ActorTemplateName: revision.ActorTemplateName, ActorTemplateUID: revision.ActorTemplateUID}, - SourceSnapshot: revision.SourceSnapshot, EgressDestinations: revision.EgressDestinations, Credentials: revision.Credentials, PassthroughOrigins: revision.PassthroughOrigins, + SourceSnapshot: revision.SourceSnapshot, EgressDestinations: revision.EgressDestinations, Credentials: revision.Credentials, TokenPassthroughOrigins: revision.TokenPassthroughOrigins, }); err != nil { return err } @@ -128,7 +128,7 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi func (c *Client) GetRuntimeRevision(ctx context.Context, revision string) (*RuntimeRevision, error) { row, err := queryOne(ctx, c.db, ` SELECT revision, namespace, agent_name, agent_uid, - source_snapshot, egress_destinations, passthrough_origins, credentials, actor_template_atespace, actor_template_name, actor_template_uid, + source_snapshot, egress_destinations, token_passthrough_origins, credentials, actor_template_atespace, actor_template_name, actor_template_uid, agent_card, deleted_at FROM agent_runtime_revision WHERE revision = $1 `, pgx.RowToStructByName[runtimeRevisionRow], revision) if err != nil { @@ -152,10 +152,10 @@ func toRuntimeRevision(row runtimeRevisionRow) (*RuntimeRevision, error) { Revision: row.Revision, Namespace: row.Namespace, AgentName: row.AgentName, AgentUID: row.AgentUID, SourceSnapshot: row.SourceSnapshot, AgentCard: card, - EgressDestinations: row.EgressDestinations, - PassthroughOrigins: row.PassthroughOrigins, - Credentials: credentials, - ActorTemplateAtespace: row.ActorTemplateAtespace, ActorTemplateName: row.ActorTemplateName, + EgressDestinations: row.EgressDestinations, + TokenPassthroughOrigins: row.TokenPassthroughOrigins, + Credentials: credentials, + ActorTemplateAtespace: row.ActorTemplateAtespace, ActorTemplateName: row.ActorTemplateName, ActorTemplateUID: row.ActorTemplateUID, }, nil } @@ -205,7 +205,7 @@ func getAvailableRuntimeRevisionForUpdate(ctx context.Context, tx pgx.Tx, revisi } return queryOne(ctx, tx, ` SELECT revision, namespace, agent_name, agent_uid, - source_snapshot, egress_destinations, passthrough_origins, credentials, actor_template_atespace, actor_template_name, actor_template_uid, + source_snapshot, egress_destinations, token_passthrough_origins, credentials, actor_template_atespace, actor_template_name, actor_template_uid, agent_card, deleted_at FROM agent_runtime_revision WHERE revision = $1 `, pgx.RowToStructByName[runtimeRevisionRow], revision) } @@ -228,10 +228,10 @@ func lockRuntimeRevisionForReference(ctx context.Context, tx pgx.Tx, revision, k type runtimeRevisionRecord struct { RuntimeArtifact - SourceSnapshot json.RawMessage - EgressDestinations []string - PassthroughOrigins []string - Credentials []egress.Credential + SourceSnapshot json.RawMessage + EgressDestinations []string + TokenPassthroughOrigins []string + Credentials []egress.Credential } // recordRuntimeRevision writes immutable common inputs with mutable artifact @@ -244,17 +244,17 @@ func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevis if revision.EgressDestinations == nil { revision.EgressDestinations = []string{} } - if revision.PassthroughOrigins == nil { - revision.PassthroughOrigins = []string{} + if revision.TokenPassthroughOrigins == nil { + revision.TokenPassthroughOrigins = []string{} } result, err := tx.Exec(ctx, ` INSERT INTO runtime_revision (revision, kind, namespace, source_snapshot, egress_destinations, - actor_template_atespace, actor_template_name, actor_template_uid, credentials, passthrough_origins) + actor_template_atespace, actor_template_name, actor_template_uid, credentials, token_passthrough_origins) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (revision) DO UPDATE SET actor_template_uid = EXCLUDED.actor_template_uid, updated_at = NOW() WHERE runtime_revision.deleted_at IS NULL AND runtime_revision.kind = EXCLUDED.kind `, revision.Revision, revision.Kind, revision.Namespace, revision.SourceSnapshot, revision.EgressDestinations, - revision.ActorTemplateAtespace, revision.ActorTemplateName, revision.ActorTemplateUID, revision.Credentials, revision.PassthroughOrigins) + revision.ActorTemplateAtespace, revision.ActorTemplateName, revision.ActorTemplateUID, revision.Credentials, revision.TokenPassthroughOrigins) if err != nil { return fmt.Errorf("record runtime revision %s: %w", revision.Revision, err) } @@ -342,17 +342,17 @@ func (c *Client) DeleteRuntimeRevision(ctx context.Context, revision, actorTempl } type runtimeRevisionRow struct { - Revision string - Namespace string - AgentName string - AgentUID string - SourceSnapshot []byte - EgressDestinations []string - PassthroughOrigins []string - Credentials []egress.Credential - ActorTemplateAtespace string - ActorTemplateName string - ActorTemplateUID string - AgentCard []byte - DeletedAt *time.Time + Revision string + Namespace string + AgentName string + AgentUID string + SourceSnapshot []byte + EgressDestinations []string + TokenPassthroughOrigins []string + Credentials []egress.Credential + ActorTemplateAtespace string + ActorTemplateName string + ActorTemplateUID string + AgentCard []byte + DeletedAt *time.Time } diff --git a/go/core/internal/egress/live.go b/go/core/internal/egress/live.go index 45ac3cc7af..e9b752f266 100644 --- a/go/core/internal/egress/live.go +++ b/go/core/internal/egress/live.go @@ -14,13 +14,13 @@ type Resolver interface { // MergeAccess validates live grants against the pinned runtime's credentials and // caller-token origins. Inputs are never modified. -func MergeAccess(base, grants Access, passthroughOrigins []string) (Access, error) { +func MergeAccess(base, grants Access, tokenPassthroughOrigins []string) (Access, error) { access, err := CanonicalAccess(append(slices.Clone(base.Destinations), grants.Destinations...), append(slices.Clone(base.Credentials), grants.Credentials...)) if err != nil { return Access{}, err } for _, binding := range access.Credentials { - if slices.Contains(passthroughOrigins, binding.Origin) { + if slices.Contains(tokenPassthroughOrigins, binding.Origin) { return Access{}, fmt.Errorf("destination %q cannot combine caller-token passthrough with gateway credentials", binding.Origin) } } diff --git a/go/core/internal/translator/byo/compiler.go b/go/core/internal/translator/byo/compiler.go index a4a24485bd..53f90f7fb9 100644 --- a/go/core/internal/translator/byo/compiler.go +++ b/go/core/internal/translator/byo/compiler.go @@ -70,7 +70,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build revision provenance: %w", err) } - environment, credentials, passthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment) + environment, credentials, tokenPassthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment) if err != nil { return nil, err } @@ -84,7 +84,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) Environment: environment, ConfigJSON: configJSON, AgentCard: card, SkillImages: compiled.SkillImages, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - PassthroughOrigins: passthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: slices.Compact(compiled.Egress), + TokenPassthroughOrigins: tokenPassthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: slices.Compact(compiled.Egress), }}, nil } diff --git a/go/core/internal/translator/claude/compiler.go b/go/core/internal/translator/claude/compiler.go index 2bb2df6375..d5bb297486 100644 --- a/go/core/internal/translator/claude/compiler.go +++ b/go/core/internal/translator/claude/compiler.go @@ -110,7 +110,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build Claude revision provenance: %w", err) } - environment, credentials, passthroughOrigins, err := translator.CompileCredentials(input, nil, environment) + environment, credentials, tokenPassthroughOrigins, err := translator.CompileCredentials(input, nil, environment) if err != nil { return nil, err } @@ -127,9 +127,9 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) SkillImages: skills.Images, Environment: environment, ConfigJSON: configJSON, AgentCard: card, - WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, - SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - PassthroughOrigins: passthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: egress, + WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, + SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, + TokenPassthroughOrigins: tokenPassthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: egress, }, Warnings: mcp.warnings, }, nil diff --git a/go/core/internal/translator/codex/compiler.go b/go/core/internal/translator/codex/compiler.go index 22328086c8..4300916d8f 100644 --- a/go/core/internal/translator/codex/compiler.go +++ b/go/core/internal/translator/codex/compiler.go @@ -130,7 +130,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build Codex revision provenance: %w", err) } - environment, credentials, passthroughOrigins, err := translator.CompileCredentials(input, nil, environment) + environment, credentials, tokenPassthroughOrigins, err := translator.CompileCredentials(input, nil, environment) if err != nil { return nil, err } @@ -146,7 +146,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) SkillImages: skills.Images, Environment: environment, ConfigJSON: configJSON, AgentCard: card, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - PassthroughOrigins: passthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: egress, + TokenPassthroughOrigins: tokenPassthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: egress, }, Warnings: mcp.warnings, }, nil diff --git a/go/core/internal/translator/compiler.go b/go/core/internal/translator/compiler.go index 700007e932..8020582cc6 100644 --- a/go/core/internal/translator/compiler.go +++ b/go/core/internal/translator/compiler.go @@ -175,7 +175,7 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agent *v1alpha3.Age return nil, err } // Explicit grants are live authorization, not immutable runtime inputs. - if _, err := egress.MergeAccess(egress.Access{Destinations: result.EgressDestinations, Credentials: result.Credentials}, access, result.PassthroughOrigins); err != nil { + if _, err := egress.MergeAccess(egress.Access{Destinations: result.EgressDestinations, Credentials: result.Credentials}, access, result.TokenPassthroughOrigins); err != nil { return nil, NewValidationError("%v", err) } workerKey := types.NamespacedName{Namespace: harness.Namespace, Name: harness.Spec.Substrate.WorkerPoolRef.Name} diff --git a/go/core/internal/translator/credentials.go b/go/core/internal/translator/credentials.go index a5437343c8..f10a9d4a3b 100644 --- a/go/core/internal/translator/credentials.go +++ b/go/core/internal/translator/credentials.go @@ -22,7 +22,7 @@ const CredentialPlaceholder = "kagent-credential-injected" // outside the agent tree (such as memory embeddings) are supplied separately. func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, environment []corev1.EnvVar) ([]corev1.EnvVar, []egress.Credential, []string, error) { var bindings []egress.Credential - var passthroughOrigins []string + var tokenPassthroughOrigins []string boundModels := map[string]bool{} boundMCP := map[string]bool{} bind := func(rawURL, header, prefix, namespace, name, key string) error { @@ -104,13 +104,13 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, if err != nil { return nil, nil, nil, NewValidationError("invalid passthrough credential destination: %v", err) } - passthroughOrigins = append(passthroughOrigins, origin.String()) + tokenPassthroughOrigins = append(tokenPassthroughOrigins, origin.String()) } var destinations []string for _, binding := range bindings { destinations = append(destinations, binding.Origin) } - if _, err := egress.MergeAccess(egress.Access{Destinations: destinations, Credentials: bindings}, input.Egress, passthroughOrigins); err != nil { + if _, err := egress.MergeAccess(egress.Access{Destinations: destinations, Credentials: bindings}, input.Egress, tokenPassthroughOrigins); err != nil { return nil, nil, nil, NewValidationError("%v", err) } @@ -126,7 +126,7 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, } result[i].Value, result[i].ValueFrom = CredentialPlaceholder, nil } - return result, bindings, passthroughOrigins, nil + return result, bindings, tokenPassthroughOrigins, nil } func modelCredentialTarget(resolved *ResolvedModelConfig) (name, endpoint, header, prefix string) { diff --git a/go/core/internal/translator/kagent/compiler.go b/go/core/internal/translator/kagent/compiler.go index aaba414f4a..8fd46c88df 100644 --- a/go/core/internal/translator/kagent/compiler.go +++ b/go/core/internal/translator/kagent/compiler.go @@ -61,7 +61,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build revision provenance: %w", err) } - environment, credentials, passthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment) + environment, credentials, tokenPassthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment) if err != nil { return nil, err } @@ -75,6 +75,6 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) Environment: environment, ConfigJSON: configJSON, AgentCard: card, SkillImages: compiled.SkillImages, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - PassthroughOrigins: passthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: compiled.Egress, + TokenPassthroughOrigins: tokenPassthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: compiled.Egress, }}, nil } diff --git a/go/core/internal/translator/revision.go b/go/core/internal/translator/revision.go index 134c197cfb..da68522ef6 100644 --- a/go/core/internal/translator/revision.go +++ b/go/core/internal/translator/revision.go @@ -66,8 +66,8 @@ type Revision struct { Provenance json.RawMessage // Credentials contains references resolved by the egress gateway. Credentials []egress.Credential - // PassthroughOrigins cannot receive gateway credentials, including live grants. - PassthroughOrigins []string + // TokenPassthroughOrigins cannot receive gateway credentials, including live grants. + TokenPassthroughOrigins []string // EgressDestinations is the HTTP(S) origin allowlist required by this revision. EgressDestinations []string } @@ -94,22 +94,22 @@ func (r *Revision) Digest() (RevisionID, error) { return RevisionID{}, fmt.Errorf("unsupported sandbox class %q", sandboxClass) } raw, err := json.Marshal(struct { - AgentName string `json:"agentName"` - AgentUID string `json:"agentUID"` - Namespace string `json:"namespace"` - Image string `json:"image"` - Command []string `json:"command,omitempty"` - Args []string `json:"args,omitempty"` - Environment []corev1.EnvVar `json:"environment"` - ConfigJSON json.RawMessage `json:"config"` - SkillImages []SkillImage `json:"skillImages,omitempty"` - WorkerPoolName string `json:"workerPoolName"` - SnapshotLocation string `json:"snapshotLocation"` - Provenance json.RawMessage `json:"provenance"` - Credentials []egress.Credential `json:"credentials,omitempty"` - PassthroughOrigins []string `json:"passthroughOrigins,omitempty"` - EgressDestinations []string `json:"egressDestinations"` - SandboxClass atev1alpha1.SandboxClass `json:"sandboxClass"` + AgentName string `json:"agentName"` + AgentUID string `json:"agentUID"` + Namespace string `json:"namespace"` + Image string `json:"image"` + Command []string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Environment []corev1.EnvVar `json:"environment"` + ConfigJSON json.RawMessage `json:"config"` + SkillImages []SkillImage `json:"skillImages,omitempty"` + WorkerPoolName string `json:"workerPoolName"` + SnapshotLocation string `json:"snapshotLocation"` + Provenance json.RawMessage `json:"provenance"` + Credentials []egress.Credential `json:"credentials,omitempty"` + TokenPassthroughOrigins []string `json:"tokenPassthroughOrigins,omitempty"` + EgressDestinations []string `json:"egressDestinations"` + SandboxClass atev1alpha1.SandboxClass `json:"sandboxClass"` // Placement now matches the pool's namespace as well as its name. // Hash it explicitly so existing immutable templates do not conflict after upgrades. @@ -119,7 +119,7 @@ func (r *Revision) Digest() (RevisionID, error) { Image: r.Image, Command: r.Command, Args: r.Args, Environment: r.Environment, ConfigJSON: r.ConfigJSON, SkillImages: r.SkillImages, WorkerPoolName: r.WorkerPoolName, SnapshotLocation: r.SnapshotLocation, Provenance: r.Provenance, - PassthroughOrigins: r.PassthroughOrigins, Credentials: r.Credentials, EgressDestinations: r.EgressDestinations, + TokenPassthroughOrigins: r.TokenPassthroughOrigins, Credentials: r.Credentials, EgressDestinations: r.EgressDestinations, SandboxClass: sandboxClass, WorkerPoolNamespace: r.Namespace, }) if err != nil { diff --git a/go/core/pkg/migrations/core/000002_runtime_passthrough_origins.sql b/go/core/pkg/migrations/core/000002_runtime_token_passthrough_origins.sql similarity index 86% rename from go/core/pkg/migrations/core/000002_runtime_passthrough_origins.sql rename to go/core/pkg/migrations/core/000002_runtime_token_passthrough_origins.sql index 1280ecd57c..1ebf273838 100644 --- a/go/core/pkg/migrations/core/000002_runtime_passthrough_origins.sql +++ b/go/core/pkg/migrations/core/000002_runtime_token_passthrough_origins.sql @@ -1,6 +1,6 @@ -- +goose Up -ALTER TABLE runtime_revision ADD COLUMN passthrough_origins TEXT[] NOT NULL DEFAULT '{}'; +ALTER TABLE runtime_revision ADD COLUMN token_passthrough_origins TEXT[] NOT NULL DEFAULT '{}'; -- PostgreSQL expands r.* when the view is created. DROP VIEW agent_runtime_revision; @@ -21,7 +21,7 @@ DROP INDEX sandbox_revision_source_idx; DROP INDEX agent_revision_source_idx; DROP VIEW agent_runtime_revision; -ALTER TABLE runtime_revision DROP COLUMN passthrough_origins; +ALTER TABLE runtime_revision DROP COLUMN token_passthrough_origins; CREATE VIEW agent_runtime_revision AS SELECT r.*, a.agent_name, a.agent_uid, a.agent_card FROM runtime_revision r JOIN agent_revision a USING (revision) WHERE r.kind = 'agent'; diff --git a/go/core/pkg/migrations/runner_test.go b/go/core/pkg/migrations/runner_test.go index efcdf0610e..8fe69c8029 100644 --- a/go/core/pkg/migrations/runner_test.go +++ b/go/core/pkg/migrations/runner_test.go @@ -326,7 +326,7 @@ func TestBuiltinMigrationsRoundTrip(t *testing.T) { t.Fatalf("%s versions = %v, want %v", source.Name, versions, expected) } } - if !testColumnExists(t, dsn, "runtime_revision", "passthrough_origins") || !testColumnExists(t, dsn, "agent_runtime_revision", "passthrough_origins") { + if !testColumnExists(t, dsn, "runtime_revision", "token_passthrough_origins") || !testColumnExists(t, dsn, "agent_runtime_revision", "token_passthrough_origins") { t.Fatal("runtime and agent revisions must expose passthrough origins") } if !testColumnExists(t, dsn, "session", "deletion_reason") { From 47c38d1e4716be0334fa0baed219424d244aad42 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 15:45:44 +0000 Subject: [PATCH 15/43] refactor: persist controller-translated egress policy per revision Signed-off-by: Eitan Yarmush --- docs/architecture/egress-policies.md | 60 +-- docs/architecture/runtime-and-lifecycle.md | 5 +- docs/architecture/sandboxes.md | 9 +- .../internal/controller/collections_test.go | 3 +- go/core/internal/controller/egress.go | 285 ++++++----- go/core/internal/controller/egress_test.go | 466 ++++++++++-------- .../internal/controller/preparation_test.go | 40 +- go/core/internal/controller/reconciler.go | 17 +- .../internal/controller/reconciler_test.go | 7 +- .../controller/sandbox_integration_test.go | 2 +- .../internal/controller/sandbox_reconciler.go | 15 +- .../controller/sandbox_reconciler_test.go | 21 + go/core/internal/database/runtime_egress.go | 125 ++++- .../internal/database/runtime_egress_test.go | 81 ++- go/core/internal/egress/destination.go | 2 +- go/core/internal/egress/live.go | 8 +- go/core/internal/substrate/config.go | 2 +- go/core/internal/substrate/egress.go | 36 +- go/core/internal/substrate/egress_test.go | 41 +- go/core/internal/translator/egress.go | 16 - go/core/pkg/app/app.go | 10 +- ...0002_runtime_token_passthrough_origins.sql | 9 + go/core/pkg/migrations/runner_test.go | 3 + 23 files changed, 787 insertions(+), 476 deletions(-) diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index 4c21d13898..1931e03b2a 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -159,34 +159,38 @@ Explicit policy grants and policy references do not enter revision identity. Changing a grant or its Secret reference does not prepare a new runtime image. Secret values remain outside compiled inputs; the gateway handles value rotation. -The controller resolves current grants from the owning Agent or SandboxTemplate -and reconciles each existing Actor's Substrate policy. Kubernetes events trigger -updates only for the affected Agent or SandboxTemplate. A five-minute sweep -recovers creation races, missed events, and controller restarts. Failed actor -updates retry with exponential backoff independently of successful updates. -This runs independently of agent invocations and runtime preparation. Running and -suspended Actors receive updates without replacement. Missing or invalid policies -and Secrets revoke explicit grants; pinned integration access remains available. -A grant that conflicts with a pinned credential or caller-token origin is also -withheld, with rate-limited warnings identifying the affected revision. The -same applies when the merged grants exceed the rule limit. Deleting and recreating an Agent or SandboxTemplate does not transfer -its grants to instances owned by the old Kubernetes UID. - -New Sessions and sandboxes atomically pin the latest successful runtime revision, -including while a configuration edit is unresolved, preparing, or failed. Initial -creation still requires a successful revision. Existing instances and checkpoint -forks retain their runtime configuration. Creation, creation retries, and resume -resolve current authorization independently of that revision; a fork cannot -restore an old policy from a snapshot. - -The runtime workflow creates a suspended Actor and installs its current `default` -egress policy before it can run. Policy updates use Substrate's UID and version -preconditions. After reading the backend version, each attempt reads the source, -policies, and Secrets directly from the Kubernetes API server, bypassing informer -caches on serving replicas. Conflicts repeat both reads. These reads do not form -an atomic snapshot across Kubernetes resources; a concurrent edit can require -another reconciliation. Transient API failures return errors and retry. -No database transaction is held across the Substrate calls. +The elected controller compiles current grants from cached, dependency-tracked +Kubernetes inputs and persists the intended policy for each retained revision. +This mutable output is stored separately from immutable revision inputs. Initial +preparation publishes the policy before promoting a new revision to ready. +Each revision merges its pinned integration access with the current grants of its +owning Agent or SandboxTemplate, even when that source's next runtime revision +cannot be prepared. + +Policy and Secret events update only revisions of the affected source and queue +their existing Actors. A five-minute sweep repairs missed events and controller +restarts. Actor updates retry with exponential backoff independently of successful +updates. Running and suspended Actors receive updates without replacement or a +new invocation. Missing or invalid policies and Secrets revoke explicit grants; +pinned integration access remains available. Grants that conflict with pinned +credentials or caller-token origins, or exceed the merged rule limit, are withheld +with rate-limited warnings. Recreating an Agent or SandboxTemplate does not +transfer its grants to revisions belonging to the previous Kubernetes UID. + +New Sessions and sandboxes pin the latest successful runtime revision, including +while a configuration edit is unresolved, preparing, or failed. Initial creation +still requires a successful revision. Existing instances and checkpoint forks +retain their runtime configuration. + +Create, creation retries, and resume load and apply the controller's persisted +intended policy. They do not read or translate Kubernetes policy. The workflow +creates a suspended Actor and installs its `default` egress policy before it can +run; a fork cannot restore an old policy from a snapshot. Applications read the +Substrate policy version before loading intended policy, then write using its UID +and version preconditions. Conflicts reload both. After applying, they recheck +intended policy to catch updates whose actor discovery raced with creation. +Missing or malformed persisted policy blocks execution and is retried. No database +transaction or lock is held across Substrate calls. Policy convergence is asynchronous: a Kubernetes write is not a synchronous revocation barrier. Once the gateway observes the changed policy, subsequent diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 27efbf01f2..f6b10efe95 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -9,8 +9,9 @@ Kubernetes resource. Fresh creation pins the Agent's latest successful runtime revision. Pending or failed edits leave that revision available; an Agent with no successful revision cannot create sessions. Creation retries retain the existing session's pinned -revision. Explicit egress grants are resolved from current Kubernetes policies -independently of the runtime revision before execution. +revision. Explicit egress grants are applied from the controller's persisted +intended policy before execution. Lifecycle +operations do not read or translate Kubernetes policy. The control plane creates a deterministic Actor initially suspended and marks the session ready after Substrate accepts it. Readiness of the image was already established diff --git a/docs/architecture/sandboxes.md b/docs/architecture/sandboxes.md index bc4da363aa..98353aba1b 100644 --- a/docs/architecture/sandboxes.md +++ b/docs/architecture/sandboxes.md @@ -118,10 +118,11 @@ expiration. Sandboxes have no allowed egress destinations by default. Select same-namespace [EgressPolicy resources](egress-policies.md) through `SandboxTemplate.spec.egressPolicyRefs` -to grant additional HTTP(S) origins and static gateway credentials. Resolved -grants are resolved from the current template independently of its prepared -revision. Policy edits update running and suspended sandboxes without replacing -their Actor. Creation and resume apply current grants before execution. +to grant additional HTTP(S) origins and static gateway credentials. The controller +translates current grants and persists an intended policy per prepared revision. +Policy edits update running and suspended sandboxes without replacing their Actor. +Creation and resume apply that persisted policy before execution, without +translating Kubernetes inputs. ## Lifecycle and recovery diff --git a/go/core/internal/controller/collections_test.go b/go/core/internal/controller/collections_test.go index 8e8ab3a99d..09c74fd5a6 100644 --- a/go/core/internal/controller/collections_test.go +++ b/go/core/internal/controller/collections_test.go @@ -1,6 +1,7 @@ package controller import ( + "context" "strings" "testing" "time" @@ -102,7 +103,7 @@ func TestReconciliationCollectionsCompileAndObserveRevision(t *testing.T) { observed.Metadata.Uid = "actor-template-uid" observed.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}}} store := &fakeRuntimeRevisionStore{} - reconciler := &Reconciler{ + reconciler := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: collections, templates: &fakeActorTemplates{template: observed}, store: store, } require.NoError(t, reconciler.reconcileAgent(t.Context(), state.ResourceName())) diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go index 80f22a0755..5d371757b1 100644 --- a/go/core/internal/controller/egress.go +++ b/go/core/internal/controller/egress.go @@ -3,7 +3,6 @@ package controller import ( "context" "errors" - "fmt" "slices" "sync" "time" @@ -14,9 +13,6 @@ import ( "google.golang.org/grpc/status" "istio.io/istio/pkg/kube/controllers" "istio.io/istio/pkg/kube/krt" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" @@ -29,7 +25,16 @@ import ( type runtimeEgressStore interface { GetRuntimeEgress(context.Context, string) (*database.RuntimeEgress, error) - ListRuntimeEgressTargets(context.Context, database.RuntimeEgressSource, string, int) ([]database.RuntimeEgressTarget, error) + SetRuntimeEgressPolicy(context.Context, string, egress.Access) error + ListRuntimeEgressRevisions(context.Context, string, int) ([]string, error) + ListRuntimeEgressSourceRevisions(context.Context, database.RuntimeEgressSource, string, int) ([]string, error) + ListRuntimeEgressTargets(context.Context, string, string, int) ([]database.RuntimeEgressTarget, error) +} + +// revisionEgressReconciler publishes the initial policy before preparation can +// promote a revision. Later egress-only translations use the same writer. +type revisionEgressReconciler interface { + ReconcileRevision(context.Context, string) error } type policyGrants struct { @@ -39,26 +44,14 @@ type policyGrants struct { } func (p policyGrants) ResourceName() string { - return p.Source.Kind + "/" + p.Source.Namespace + "/" + p.Source.Name -} - -// EgressResolver resolves mutable policy grants independently of runtime -// compilation. Broken templates and pools must not prevent revocation. -type EgressResolver struct { - grants krt.Collection[policyGrants] - store runtimeEgressStore - namespaces []string - reader ctrlclient.Reader - warnings *rate.Limiter + return string(p.Source.Kind) + "/" + p.Source.Namespace + "/" + p.Source.Name } -var _ egress.Resolver = (*EgressResolver)(nil) - -func NewEgressResolver(runtime *Runtime, store runtimeEgressStore, reader ctrlclient.Reader, namespaces []string) *EgressResolver { +func newEgressGrants(runtime *Runtime) krt.Collection[policyGrants] { inputs := runtime.Collections collections := translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets} agents := krt.NewCollection(inputs.Agents, func(ctx krt.HandlerContext, agent *kagentv1alpha3.Agent) *policyGrants { - grants := &policyGrants{Source: database.RuntimeEgressSource{Kind: "agent", Namespace: agent.Namespace, Name: agent.Name}, UID: string(agent.UID)} + grants := &policyGrants{Source: database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: agent.Namespace, Name: agent.Name}, UID: string(agent.UID)} if agent.DeletionTimestamp.IsZero() { // CompileEgressPolicies returns empty access on every failure. Runtime // compilation reports resolution errors through the source's status. @@ -70,7 +63,7 @@ func NewEgressResolver(runtime *Runtime, store runtimeEgressStore, reader ctrlcl return grants }, runtime.Options.WithName("AgentEgressGrants")...) sandboxes := krt.NewCollection(inputs.SandboxTemplates, func(ctx krt.HandlerContext, template *kagentv1alpha3.SandboxTemplate) *policyGrants { - grants := &policyGrants{Source: database.RuntimeEgressSource{Kind: "sandbox", Namespace: template.Namespace, Name: template.Name}, UID: string(template.UID)} + grants := &policyGrants{Source: database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: template.Namespace, Name: template.Name}, UID: string(template.UID)} if template.DeletionTimestamp.IsZero() { access, err := translator.CompileEgressPolicies(ctx, collections, template.Namespace, template.Spec.EgressPolicyRefs) if err == nil { @@ -79,73 +72,7 @@ func NewEgressResolver(runtime *Runtime, store runtimeEgressStore, reader ctrlcl } return grants }, runtime.Options.WithName("SandboxEgressGrants")...) - return &EgressResolver{reader: reader, warnings: rate.NewLimiter(rate.Every(30*time.Second), 5), store: store, namespaces: slices.Clone(namespaces), grants: krt.JoinCollection([]krt.Collection[policyGrants]{agents, sandboxes}, runtime.Options.WithName("RuntimeEgressGrants")...)} -} - -func (e *EgressResolver) watches(namespace string) bool { - return len(e.namespaces) == 0 || slices.Contains(e.namespaces, namespace) -} - -func (e *EgressResolver) ResolveEgress(ctx context.Context, revision string) (egress.Access, error) { - pinned, err := e.store.GetRuntimeEgress(ctx, revision) - if err != nil { - return egress.Access{}, err - } - if !e.watches(pinned.Namespace) { - return egress.Access{}, fmt.Errorf("runtime namespace %q is not watched", pinned.Namespace) - } - base := egress.Access{Destinations: pinned.EgressDestinations, Credentials: pinned.Credentials} - grants, err := e.currentGrants(ctx, pinned) - if err != nil { - return egress.Access{}, err - } - access, err := egress.MergeAccess(base, grants, pinned.TokenPassthroughOrigins) - if err != nil { - // A current grant may conflict with an old session's pinned integrations. - // Revoke its explicit access instead of preserving a stale policy. - if e.warnings.Allow() { - logging.FromContext(ctx).WarnContext(ctx, "explicit egress grants withheld", "revision", revision, "namespace", pinned.Namespace, "source", pinned.SourceName, "error", err) - } - return base, nil - } - return access, nil -} - -// Read through the API server, never an informer cache: create and resume may -// run on a replica whose watches have not observed a revocation yet. -func (e *EgressResolver) currentGrants(ctx context.Context, pinned *database.RuntimeEgress) (egress.Access, error) { - var source ctrlclient.Object - var refs []corev1.LocalObjectReference - key := ctrlclient.ObjectKey{Namespace: pinned.Namespace, Name: pinned.SourceName} - switch pinned.Kind { - case "agent": - source = &kagentv1alpha3.Agent{} - case "sandbox": - source = &kagentv1alpha3.SandboxTemplate{} - default: - return egress.Access{}, fmt.Errorf("unknown runtime kind %q", pinned.Kind) - } - if err := e.reader.Get(ctx, key, source); err != nil { - if apierrors.IsNotFound(err) { - return egress.Access{}, nil - } - return egress.Access{}, fmt.Errorf("read egress source %s: %w", key, err) - } - if string(source.GetUID()) != pinned.SourceUID || !source.GetDeletionTimestamp().IsZero() { - return egress.Access{}, nil - } - switch object := source.(type) { - case *kagentv1alpha3.Agent: - refs = object.Spec.EgressPolicyRefs - case *kagentv1alpha3.SandboxTemplate: - refs = object.Spec.EgressPolicyRefs - } - access, err := translator.ResolveEgressPolicies(ctx, e.reader, pinned.Namespace, refs) - var invalid *translator.ValidationError - if apierrors.IsNotFound(err) || errors.As(err, &invalid) { - return egress.Access{}, nil - } - return access, err + return krt.JoinCollection([]krt.Collection[policyGrants]{agents, sandboxes}, runtime.Options.WithName("RuntimeEgressGrants")...) } type actorEgressClient interface { @@ -153,29 +80,41 @@ type actorEgressClient interface { EnsureActorEgressPolicy(context.Context, string, string, string) error } -// EgressReconciler queues only instances of changed sources. A slow sweep -// recovers missed events and actor-creation races, including suspended actors. -// Pages and actors have independent retry queues so failures cannot stall grants -// for another source or cause successful actors to be retried with a failed one. +// EgressReconciler publishes KRT's current grants per retained revision. Actors +// and API replicas apply this persisted output without translating Kubernetes. +// Recovery has separate workers so a full sweep cannot delay changed sources. type EgressReconciler struct { - resolver *EgressResolver - actors actorEgressClient + grants krt.Collection[policyGrants] + store runtimeEgressStore + actors actorEgressClient + namespaces []string + warnings *rate.Limiter + // Initial preparation and source/recovery queues share a publisher. Serialize + // cache lookup through commit so an older translation cannot commit last. + publishMu sync.Mutex + sources controllers.Queue + pages controllers.Queue + updates controllers.Queue + recovery controllers.Queue + sweeps controllers.Queue } -type egressPage struct { +type egressSourcePage struct { Source database.RuntimeEgressSource After string } -var _ manager.LeaderElectionRunnable = (*EgressReconciler)(nil) - -func NewEgressReconciler(resolver *EgressResolver, actors actorEgressClient) *EgressReconciler { - return &EgressReconciler{resolver: resolver, actors: actors} +type egressActorPage struct { + Revision string + After string + Recovery bool } -func (e *EgressReconciler) NeedLeaderElection() bool { return true } +var _ manager.LeaderElectionRunnable = (*EgressReconciler)(nil) -func (e *EgressReconciler) Start(ctx context.Context) error { +func NewEgressReconciler(ctx context.Context, runtime *Runtime, store runtimeEgressStore, actors actorEgressClient, namespaces []string) *EgressReconciler { + e := &EgressReconciler{grants: newEgressGrants(runtime), store: store, actors: actors, + namespaces: slices.Clone(namespaces), warnings: rate.NewLimiter(rate.Every(30*time.Second), 5)} reconcile := func(item any) error { err := e.reconcileActor(ctx, item.(database.RuntimeEgressTarget)) if status.Code(err) == codes.NotFound { @@ -183,49 +122,137 @@ func (e *EgressReconciler) Start(ctx context.Context) error { } return err } - actors := newReconciliationQueue("live-egress-actors", reconcile) - // Recovery has its own worker so a large sweep cannot delay event-driven revocations. - recovery := newReconciliationQueue("live-egress-recovery", reconcile) - var pages controllers.Queue - pages = newReconciliationQueue("live-egress-sources", func(item any) error { - page := item.(egressPage) - const pageSize = 100 - targets, err := e.resolver.store.ListRuntimeEgressTargets(ctx, page.Source, page.After, pageSize) + e.updates = newReconciliationQueue("live-egress-actors", reconcile) + e.recovery = newReconciliationQueue("live-egress-recovery", reconcile) + const pageSize = 100 + e.pages = newReconciliationQueue("live-egress-revision-actors", func(item any) error { + page := item.(egressActorPage) + targets, err := e.store.ListRuntimeEgressTargets(ctx, page.Revision, page.After, pageSize) if err != nil { return err } + queue := e.updates + if page.Recovery { + queue = e.recovery + } for _, target := range targets { - if e.resolver.watches(target.Atespace) { - if page.Source.Kind == "" { - recovery.Add(target) - } else { - actors.Add(target) - } - } + queue.Add(target) } if len(targets) == pageSize { - pages.Add(egressPage{Source: page.Source, After: targets[len(targets)-1].ID}) + page.After = targets[len(targets)-1].ID + e.pages.Add(page) + } + return nil + }) + e.sources = newReconciliationQueue("live-egress-sources", func(item any) error { + page := item.(egressSourcePage) + revisions, err := e.store.ListRuntimeEgressSourceRevisions(ctx, page.Source, page.After, pageSize) + if err != nil { + return err + } + for _, revision := range revisions { + if err := e.ReconcileRevision(ctx, revision); err != nil && !errors.Is(err, database.ErrNotFound) { + return err + } + } + if len(revisions) == pageSize { + page.After = revisions[len(revisions)-1] + e.sources.Add(page) + } + return nil + }) + e.sweeps = newReconciliationQueue("live-egress-sweeps", func(item any) error { + revisions, err := e.store.ListRuntimeEgressRevisions(ctx, item.(string), pageSize) + if err != nil { + return err + } + for _, revision := range revisions { + watched, err := e.publishRevision(ctx, revision) + if errors.Is(err, database.ErrNotFound) { + continue + } + if err != nil { + return err + } + if watched { + e.pages.Add(egressActorPage{Revision: revision, Recovery: true}) + } + } + if len(revisions) == pageSize { + e.sweeps.Add(revisions[len(revisions)-1]) } return nil }) - defer actors.ShutDownEarly() - defer recovery.ShutDownEarly() - defer pages.ShutDownEarly() - handler := e.resolver.grants.RegisterBatch(func(events []krt.Event[policyGrants]) { + return e +} + +// ReconcileRevision publishes current intended access before making a new +// revision ready, and schedules existing actors independently of preparation. +func (e *EgressReconciler) ReconcileRevision(ctx context.Context, revision string) error { + if !e.grants.WaitUntilSynced(ctx.Done()) { + return ctx.Err() + } + watched, err := e.publishRevision(ctx, revision) + if err != nil { + return err + } + if watched { + e.pages.Add(egressActorPage{Revision: revision}) + } + return nil +} + +func (e *EgressReconciler) publishRevision(ctx context.Context, revision string) (bool, error) { + e.publishMu.Lock() + defer e.publishMu.Unlock() + pinned, err := e.store.GetRuntimeEgress(ctx, revision) + if err != nil { + return false, err + } + if len(e.namespaces) > 0 && !slices.Contains(e.namespaces, pinned.Namespace) { + return false, nil + } + base := egress.Access{Destinations: pinned.EgressDestinations, Credentials: pinned.Credentials} + var grants egress.Access + key := policyGrants{Source: database.RuntimeEgressSource{Kind: pinned.Kind, Namespace: pinned.Namespace, Name: pinned.SourceName}} + if current := e.grants.GetKey(key.ResourceName()); current != nil && current.UID == pinned.SourceUID { + grants = current.Access + } + access, err := egress.MergeAccess(base, grants, pinned.TokenPassthroughOrigins) + if err != nil { + if e.warnings.Allow() { + logging.FromContext(ctx).WarnContext(ctx, "explicit egress grants withheld", "revision", revision, "namespace", pinned.Namespace, "source", pinned.SourceName, "error", err) + } + access = base + } + if err := e.store.SetRuntimeEgressPolicy(ctx, revision, access); err != nil { + return false, err + } + return true, nil +} + +func (e *EgressReconciler) NeedLeaderElection() bool { return true } + +func (e *EgressReconciler) Start(ctx context.Context) error { + handler := e.grants.RegisterBatch(func(events []krt.Event[policyGrants]) { for _, event := range events { - pages.Add(egressPage{Source: event.Latest().Source}) + e.sources.Add(egressSourcePage{Source: event.Latest().Source}) } }, false) defer handler.UnregisterHandler() + queues := []controllers.Queue{e.sources, e.pages, e.updates, e.recovery, e.sweeps} if !handler.WaitUntilSynced(ctx.Done()) { + for _, queue := range queues { + queue.ShutDownEarly() + } return ctx.Err() } var workers sync.WaitGroup - workers.Go(func() { actors.Run(ctx.Done()) }) - workers.Go(func() { recovery.Run(ctx.Done()) }) - workers.Go(func() { pages.Run(ctx.Done()) }) + for _, queue := range queues { + workers.Go(func() { queue.Run(ctx.Done()) }) + } defer workers.Wait() - pages.Add(egressPage{}) + e.sweeps.Add("") ticker := time.NewTicker(5 * time.Minute) defer ticker.Stop() for { @@ -233,7 +260,7 @@ func (e *EgressReconciler) Start(ctx context.Context) error { case <-ctx.Done(): return nil case <-ticker.C: - pages.Add(egressPage{}) + e.sweeps.Add("") } } } @@ -245,7 +272,9 @@ func (e *EgressReconciler) reconcileActor(ctx context.Context, target database.R return err } if !binding.Matches(actor) || (target.ActorUID != "" && target.ActorUID != actor.GetMetadata().GetUid()) { - return fmt.Errorf("actor identity or template changed") + // This queued identity no longer owns the actor. Recovery discovers its + // current binding; retrying this stale target cannot repair it. + return nil } return e.actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, target.Revision) } diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go index 706ba49127..18dc2f9cbe 100644 --- a/go/core/internal/controller/egress_test.go +++ b/go/core/internal/controller/egress_test.go @@ -7,9 +7,9 @@ import ( "fmt" "log/slog" "reflect" + "slices" "strings" "sync" - "sync/atomic" "testing" "time" @@ -19,302 +19,344 @@ import ( "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/pkg/logging" ) +type revisionEgressFunc func(context.Context, string) error + +func (f revisionEgressFunc) ReconcileRevision(ctx context.Context, revision string) error { + return f(ctx, revision) +} + type egressTestStore struct { - pinned database.RuntimeEgress - targets []database.RuntimeEgressTarget - listed chan database.RuntimeEgressSource + mu sync.Mutex + pinned map[string]database.RuntimeEgress + intended map[string]egress.Access + targets map[string][]database.RuntimeEgressTarget + listed chan database.RuntimeEgressSource + publishErr error } var _ runtimeEgressStore = (*egressTestStore)(nil) -func (e *egressTestStore) GetRuntimeEgress(context.Context, string) (*database.RuntimeEgress, error) { - return &e.pinned, nil +func (e *egressTestStore) GetRuntimeEgress(_ context.Context, revision string) (*database.RuntimeEgress, error) { + e.mu.Lock() + defer e.mu.Unlock() + row, ok := e.pinned[revision] + if !ok { + return nil, database.ErrNotFound + } + return &row, nil +} +func (e *egressTestStore) SetRuntimeEgressPolicy(_ context.Context, revision string, access egress.Access) error { + e.mu.Lock() + defer e.mu.Unlock() + if e.publishErr != nil { + return e.publishErr + } + if e.intended == nil { + e.intended = map[string]egress.Access{} + } + e.intended[revision] = access + return nil +} +func (e *egressTestStore) GetRuntimeEgressPolicy(_ context.Context, revision string) (egress.Access, error) { + e.mu.Lock() + defer e.mu.Unlock() + policy, ok := e.intended[revision] + if !ok { + return egress.Access{}, database.ErrNotFound + } + return policy, nil +} +func (e *egressTestStore) ListRuntimeEgressRevisions(_ context.Context, after string, limit int) ([]string, error) { + e.mu.Lock() + defer e.mu.Unlock() + var revisions []string + for key := range e.pinned { + if key > after { + revisions = append(revisions, key) + } + } + slices.Sort(revisions) + return revisions[:min(limit, len(revisions))], nil } -func (e *egressTestStore) ListRuntimeEgressTargets(_ context.Context, source database.RuntimeEgressSource, after string, limit int) ([]database.RuntimeEgressTarget, error) { +func (e *egressTestStore) ListRuntimeEgressSourceRevisions(_ context.Context, source database.RuntimeEgressSource, after string, limit int) ([]string, error) { + e.mu.Lock() + defer e.mu.Unlock() if e.listed != nil { e.listed <- source } - if after != "" || (source.Kind != "" && source.Name != e.pinned.SourceName) { - return nil, nil + var revisions []string + for key, pinned := range e.pinned { + if pinned.Kind == source.Kind && pinned.Namespace == source.Namespace && pinned.SourceName == source.Name && key > after { + revisions = append(revisions, key) + } + } + slices.Sort(revisions) + return revisions[:min(limit, len(revisions))], nil +} +func (e *egressTestStore) ListRuntimeEgressTargets(_ context.Context, revision, after string, limit int) ([]database.RuntimeEgressTarget, error) { + e.mu.Lock() + defer e.mu.Unlock() + var targets []database.RuntimeEgressTarget + for _, target := range e.targets[revision] { + if target.ID > after { + targets = append(targets, target) + } } - return e.targets, nil + return targets[:min(limit, len(targets))], nil } -func TestLiveEgressResolvesCurrentSource(t *testing.T) { - for _, kind := range []string{"agent", "sandbox"} { - t.Run(kind, func(t *testing.T) { +type egressTestInputs struct { + runtime *Runtime + agents krt.StaticCollection[*v1alpha3.Agent] + sandboxes krt.StaticCollection[*v1alpha3.SandboxTemplate] + policies krt.StaticCollection[*v1alpha3.EgressPolicy] + secrets krt.StaticCollection[*corev1.Secret] +} + +func newEgressTestInputs(ctx context.Context) egressTestInputs { + opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) + inputs := egressTestInputs{ + agents: krt.NewStaticCollection[*v1alpha3.Agent](nil, nil, opts.WithName("Agents")...), + sandboxes: krt.NewStaticCollection[*v1alpha3.SandboxTemplate](nil, nil, opts.WithName("Sandboxes")...), + policies: krt.NewStaticCollection[*v1alpha3.EgressPolicy](nil, nil, opts.WithName("Policies")...), + secrets: krt.NewStaticCollection[*corev1.Secret](nil, nil, opts.WithName("Secrets")...), + } + inputs.runtime = &Runtime{Options: opts, Collections: Collections{Agents: inputs.agents, SandboxTemplates: inputs.sandboxes, EgressPolicies: inputs.policies, Secrets: inputs.secrets}} + return inputs +} +func (i egressTestInputs) source(kind database.RuntimeEgressSourceKind, uid string, refs []corev1.LocalObjectReference) { + metadata := metav1.ObjectMeta{Namespace: "team", Name: "source", UID: types.UID(uid)} + if kind == database.RuntimeEgressAgent { + i.agents.UpdateObject(&v1alpha3.Agent{ObjectMeta: metadata, Spec: v1alpha3.AgentSpec{EgressPolicyRefs: refs}}) + } else { + i.sandboxes.UpdateObject(&v1alpha3.SandboxTemplate{ObjectMeta: metadata, Spec: v1alpha3.SandboxTemplateSpec{EgressPolicyRefs: refs}}) + } +} +func egressTestPolicy(origin string, auth *v1alpha3.EgressAuthentication) *v1alpha3.EgressPolicy { + return &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: origin, Authentication: auth}}}} +} + +func TestEgressTranslationUsesCachedDependencies(t *testing.T) { + for _, kind := range []database.RuntimeEgressSourceKind{database.RuntimeEgressAgent, database.RuntimeEgressSandbox} { + t.Run(string(kind), func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() - opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) - agents := krt.NewStaticCollection[*v1alpha3.Agent](nil, nil, opts.WithName("Agents")...) - sandboxes := krt.NewStaticCollection[*v1alpha3.SandboxTemplate](nil, nil, opts.WithName("Sandboxes")...) - policies := krt.NewStaticCollection[*v1alpha3.EgressPolicy](nil, nil, opts.WithName("Policies")...) - secrets := krt.NewStaticCollection[*corev1.Secret](nil, nil, opts.WithName("Secrets")...) - reader := egressTestReader(t) - setSource := func(uid string, refs []corev1.LocalObjectReference) { - metadata := metav1.ObjectMeta{Namespace: "team", Name: "source"} - metadata.SetUID(types.UID(uid)) - if kind == "agent" { - source := &v1alpha3.Agent{ObjectMeta: metadata, Spec: v1alpha3.AgentSpec{EgressPolicyRefs: refs}} - require.NoError(t, ctrlclient.IgnoreNotFound(reader.Delete(ctx, source))) - require.NoError(t, reader.Create(ctx, source)) - agents.UpdateObject(source.DeepCopy()) - } else { - source := &v1alpha3.SandboxTemplate{ObjectMeta: metadata, Spec: v1alpha3.SandboxTemplateSpec{EgressPolicyRefs: refs}} - require.NoError(t, ctrlclient.IgnoreNotFound(reader.Delete(ctx, source))) - require.NoError(t, reader.Create(ctx, source)) - sandboxes.UpdateObject(source.DeepCopy()) - } - } + inputs := newEgressTestInputs(ctx) refs := []corev1.LocalObjectReference{{Name: "access"}} - setSource("original", refs) + inputs.source(kind, "original", refs) base := egress.Access{Destinations: []string{"https://model.example.com:443"}} - store := &egressTestStore{pinned: database.RuntimeEgress{Kind: kind, Namespace: "team", SourceName: "source", SourceUID: "original", EgressDestinations: base.Destinations}} - resolver := NewEgressResolver(&Runtime{Options: opts, Collections: Collections{Agents: agents, SandboxTemplates: sandboxes, EgressPolicies: policies, Secrets: secrets}}, store, reader, nil) + store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"pinned": {Kind: kind, Namespace: "team", SourceName: "source", SourceUID: "original", EgressDestinations: base.Destinations}}} + controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) expect := func(want egress.Access) { t.Helper() require.Eventually(t, func() bool { - got, err := resolver.ResolveEgress(ctx, "pinned") + if err := controller.ReconcileRevision(ctx, "pinned"); err != nil { + return false + } + got, err := store.GetRuntimeEgressPolicy(ctx, "pinned") return err == nil && reflect.DeepEqual(want, got) }, time.Second, 10*time.Millisecond) } - expect(base) // Missing reference; unrelated runtime configuration is also invalid. - policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} - require.NoError(t, reader.Create(ctx, policy.DeepCopy())) + expect(base) // Missing refs and broken unrelated runtime inputs cannot preserve stale grants. + policy := egressTestPolicy("https://tickets.example.com", nil) + inputs.policies.UpdateObject(policy.DeepCopy()) allowed := egress.Access{Destinations: []string{"https://model.example.com:443", "https://tickets.example.com:443"}} expect(allowed) - setSource("original", nil) + inputs.source(kind, "original", nil) expect(base) - setSource("original", refs) + inputs.source(kind, "original", refs) expect(allowed) - require.NoError(t, reader.Delete(ctx, policy)) - policy.ResourceVersion = "" + inputs.policies.DeleteObject("team/access") expect(base) - require.NoError(t, reader.Create(ctx, policy)) + inputs.policies.UpdateObject(policy.DeepCopy()) expect(allowed) - setSource("replacement", refs) + inputs.source(kind, "replacement", refs) expect(base) - setSource("original", refs) + inputs.source(kind, "original", refs) expect(allowed) policy.Spec.Destinations[0].Authentication = &v1alpha3.EgressAuthentication{Header: "authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}, Key: "token"}} - require.NoError(t, reader.Update(ctx, policy)) + inputs.policies.UpdateObject(policy.DeepCopy()) expect(base) - secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "auth"}, Data: map[string][]byte{"token": []byte("secret")}} - require.NoError(t, reader.Create(ctx, secret)) + inputs.secrets.UpdateObject(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "auth"}, Data: map[string][]byte{"token": []byte("secret")}}) allowed.Credentials = []egress.Credential{{Origin: "https://tickets.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token"}} expect(allowed) - require.NoError(t, reader.Delete(ctx, secret)) + inputs.secrets.DeleteObject("team/auth") expect(base) }) } } -func TestLiveEgressRejectsConflictsWithPinnedRuntime(t *testing.T) { - for _, passthrough := range []bool{false, true} { - t.Run(fmt.Sprint("passthrough=", passthrough), func(t *testing.T) { - base := egress.Access{Destinations: []string{"https://model.example.com:443"}} - credential := egress.Credential{Origin: base.Destinations[0], Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token"} - pinned := database.RuntimeEgress{Kind: "agent", Namespace: "team", SourceName: "source", SourceUID: "uid", EgressDestinations: base.Destinations} - if passthrough { - pinned.TokenPassthroughOrigins = base.Destinations - } else { - base.Credentials = []egress.Credential{credential} - pinned.Credentials = base.Credentials - } - reader := egressTestReader(t, egressTestAgent(), egressTestPolicy(base.Destinations[0], &v1alpha3.EgressAuthentication{Header: "authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "other"}, Key: "token"}}), &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "other"}, Data: map[string][]byte{"token": []byte("secret")}}) - resolver := &EgressResolver{reader: reader, warnings: rate.NewLimiter(rate.Every(time.Minute), 1), store: &egressTestStore{pinned: pinned}} - got, err := resolver.ResolveEgress(t.Context(), "old-revision") - require.NoError(t, err) - require.Equal(t, base, got) - }) - } -} - type observingEgressActors struct { - resolver *EgressResolver - applied chan egress.Access + store *egressTestStore + mu sync.Mutex + applied map[string]egress.Access + attempts map[string][]time.Time + failing string + failures int } -var _ actorEgressClient = (*observingEgressActors)(nil) - -func (a *observingEgressActors) EnsureActorEgressPolicy(ctx context.Context, _, _, revision string) error { - access, err := a.resolver.ResolveEgress(ctx, revision) +func (a *observingEgressActors) EnsureActorEgressPolicy(ctx context.Context, _, name, revision string) error { + a.mu.Lock() + defer a.mu.Unlock() + if a.attempts == nil { + a.attempts = map[string][]time.Time{} + } + a.attempts[name] = append(a.attempts[name], time.Now()) + if name == a.failing && len(a.attempts[name]) <= a.failures { + return errors.New("backend unavailable") + } + access, err := a.store.GetRuntimeEgressPolicy(ctx, revision) if err != nil { return err } - select { - case a.applied <- access: - case <-ctx.Done(): - return ctx.Err() + if a.applied == nil { + a.applied = map[string]egress.Access{} } + a.applied[name] = access return nil } +func (a *observingEgressActors) GetActor(_ context.Context, space, name string) (*ateapipb.Actor, error) { + return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: "uid"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: "runtime"}}, nil +} -func TestLiveEgressReconcilesWithoutInvocationAndAfterRestart(t *testing.T) { +func TestEgressUpdatesRetainedRevisionsAndRecoversAfterRestart(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() - opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) - grant := policyGrants{Source: database.RuntimeEgressSource{Kind: "agent", Namespace: "team", Name: "source"}, UID: "uid", Access: egress.Access{Destinations: []string{"https://tickets.example.com:443"}}} - grants := krt.NewStaticCollection(nil, []policyGrants{grant}, opts.WithName("Grants")...) - store := &egressTestStore{pinned: database.RuntimeEgress{Kind: "agent", Namespace: "team", SourceName: "source", SourceUID: "uid"}, targets: []database.RuntimeEgressTarget{{ID: "instance", Revision: "old-revision", Atespace: "team", TemplateName: "runtime"}}} - reader := egressTestReader(t, egressTestAgent(), egressTestPolicy("https://tickets.example.com", nil)) - resolver := &EgressResolver{grants: grants, store: store, reader: reader} - actors := &observingEgressActors{resolver: resolver, applied: make(chan egress.Access, 10)} - expect := func(want []string) { + inputs := newEgressTestInputs(ctx) + inputs.source(database.RuntimeEgressAgent, "uid", []corev1.LocalObjectReference{{Name: "access"}}) + inputs.policies.UpdateObject(egressTestPolicy("https://tickets.example.com", nil)) + pinned := database.RuntimeEgress{Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"} + old := pinned + old.EgressDestinations = []string{"https://old-model.example.com:443"} + store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"new": pinned, "old": old}, targets: map[string][]database.RuntimeEgressTarget{ + "new": {{ID: "new", Revision: "new", Atespace: "team", TemplateName: "runtime"}}, + "old": {{ID: "old", Revision: "old", Atespace: "team", TemplateName: "runtime"}}, + }} + actors := &observingEgressActors{store: store} + expect := func(granted bool) { t.Helper() - for { - select { - case access := <-actors.applied: - if reflect.DeepEqual(want, access.Destinations) { - return - } - case <-ctx.Done(): - t.Fatal("policy did not converge") - } - } + require.Eventually(t, func() bool { + actors.mu.Lock() + defer actors.mu.Unlock() + current, ok := actors.applied["session-new"] + previous, oldOK := actors.applied["session-old"] + return ok && oldOK && slices.Contains(current.Destinations, "https://tickets.example.com:443") == granted && + slices.Contains(previous.Destinations, "https://tickets.example.com:443") == granted && slices.Contains(previous.Destinations, "https://old-model.example.com:443") + }, 3*time.Second, 10*time.Millisecond) } for i := range 2 { runCtx, stop := context.WithCancel(ctx) + controller := NewEgressReconciler(runCtx, inputs.runtime, store, actors, nil) done := make(chan error, 1) - go func() { done <- NewEgressReconciler(resolver, actors).Start(runCtx) }() - expect(grant.Access.Destinations) + go func() { done <- controller.Start(runCtx) }() if i == 0 { - require.NoError(t, reader.Delete(ctx, egressTestPolicy("https://tickets.example.com", nil))) - grant.Access = egress.Access{} - grants.UpdateObject(grant) - expect(nil) + expect(true) + inputs.policies.DeleteObject("team/access") + expect(false) + } else { + expect(false) } stop() require.NoError(t, <-done) + // Startup repairs existing actors without a new Kubernetes event/invocation. + actors.mu.Lock() + actors.applied = map[string]egress.Access{} + actors.mu.Unlock() } } -func (a *observingEgressActors) GetActor(_ context.Context, space, name string) (*ateapipb.Actor, error) { - return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: "uid"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: "runtime"}}, nil -} - -func egressTestReader(t *testing.T, objects ...ctrlclient.Object) ctrlclient.WithWatch { - t.Helper() - scheme := runtime.NewScheme() - require.NoError(t, v1alpha3.AddToScheme(scheme)) - require.NoError(t, corev1.AddToScheme(scheme)) - return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() -} - -func egressTestAgent() *v1alpha3.Agent { - return &v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "source", UID: "uid"}, Spec: v1alpha3.AgentSpec{EgressPolicyRefs: []corev1.LocalObjectReference{{Name: "access"}}}} -} - -func egressTestPolicy(origin string, auth *v1alpha3.EgressAuthentication) *v1alpha3.EgressPolicy { - return &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: origin, Authentication: auth}}}} -} - -func TestLiveEgressIgnoresLaggingCache(t *testing.T) { - ctx := t.Context() - policy := egressTestPolicy("https://tickets.example.com", nil) - reader := egressTestReader(t, egressTestAgent(), policy) - opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) - stale := policyGrants{Source: database.RuntimeEgressSource{Kind: "agent", Namespace: "team", Name: "source"}, UID: "uid", Access: egress.Access{Destinations: []string{"https://tickets.example.com:443"}}} - grants := krt.NewStaticCollection(nil, []policyGrants{stale}, opts.WithName("StaleGrants")...) - resolver := &EgressResolver{reader: reader, grants: grants, store: &egressTestStore{pinned: database.RuntimeEgress{Kind: "agent", Namespace: "team", SourceName: "source", SourceUID: "uid"}}} - got, err := resolver.ResolveEgress(ctx, "revision") - require.NoError(t, err) - require.Equal(t, stale.Access, got) - require.NoError(t, reader.Delete(ctx, policy)) - got, err = resolver.ResolveEgress(ctx, "revision") - require.NoError(t, err) - require.Empty(t, got, "a replica with stale grants must not undo revocation") - require.Equal(t, stale.Access, grants.GetKey(stale.ResourceName()).Access) - - unavailable := errors.New("API unavailable") - resolver.reader = interceptor.NewClient(reader, interceptor.Funcs{Get: func(context.Context, ctrlclient.WithWatch, ctrlclient.ObjectKey, ctrlclient.Object, ...ctrlclient.GetOption) error { - return unavailable - }}) - _, err = resolver.ResolveEgress(ctx, "revision") - require.ErrorIs(t, err, unavailable, "transport failures must retry instead of becoming an empty grant") -} - -type retryEgressActors struct { - observingEgressActors - failures atomic.Int32 - mu sync.Mutex - attempts map[string][]time.Time -} - -func (a *retryEgressActors) EnsureActorEgressPolicy(_ context.Context, _, name, _ string) error { - a.mu.Lock() - defer a.mu.Unlock() - a.attempts[name] = append(a.attempts[name], time.Now()) - if name == "session-failing" && a.failures.Add(1) <= 2 { - return errors.New("backend unavailable") - } - return nil -} - -func TestLiveEgressTargetsChangesAndRetriesFailedActors(t *testing.T) { +func TestEgressTargetsSourceChangesAndBacksOffActorFailures(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() - opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) - grant := policyGrants{Source: database.RuntimeEgressSource{Kind: "agent", Namespace: "team", Name: "source"}, UID: "uid"} - grants := krt.NewStaticCollection(nil, []policyGrants{grant}, opts.WithName("Grants")...) - store := &egressTestStore{pinned: database.RuntimeEgress{SourceName: "source"}, listed: make(chan database.RuntimeEgressSource, 10), targets: []database.RuntimeEgressTarget{ - {ID: "failing", Atespace: "team", TemplateName: "runtime"}, {ID: "healthy", Atespace: "team", TemplateName: "runtime"}, - }} - actors := &retryEgressActors{attempts: map[string][]time.Time{}} + inputs := newEgressTestInputs(ctx) + inputs.source(database.RuntimeEgressAgent, "uid", nil) + store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"revision": {Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"}}, + listed: make(chan database.RuntimeEgressSource, 20), targets: map[string][]database.RuntimeEgressTarget{"revision": { + {ID: "failing", Revision: "revision", Atespace: "team", TemplateName: "runtime"}, + {ID: "healthy", Revision: "revision", Atespace: "team", TemplateName: "runtime"}, + }}} + actors := &observingEgressActors{store: store, failing: "session-failing", failures: 2} + controller := NewEgressReconciler(ctx, inputs.runtime, store, actors, nil) + // Exercise recovery backoff independently of initial informer delivery. + require.True(t, controller.grants.WaitUntilSynced(ctx.Done())) done := make(chan error, 1) - go func() { done <- NewEgressReconciler(&EgressResolver{store: store, grants: grants}, actors).Start(ctx) }() + go func() { done <- controller.Start(ctx) }() defer func() { cancel(); require.NoError(t, <-done) }() - require.Equal(t, database.RuntimeEgressSource{}, <-store.listed, "startup recovers all actors") require.Eventually(t, func() bool { actors.mu.Lock() defer actors.mu.Unlock() return len(actors.attempts["session-failing"]) == 3 }, 5*time.Second, 10*time.Millisecond) actors.mu.Lock() - attempts := actors.attempts["session-failing"] + attempts := slices.Clone(actors.attempts["session-failing"]) healthy := len(actors.attempts["session-healthy"]) actors.mu.Unlock() - require.Len(t, attempts, 3) require.GreaterOrEqual(t, attempts[1].Sub(attempts[0]), time.Second) require.GreaterOrEqual(t, attempts[2].Sub(attempts[1]), 2*time.Second) - require.Equal(t, 1, healthy, "a failure must not retry successful actors") - grant.Source.Name = "unrelated" - grants.UpdateObject(grant) - require.Equal(t, grant.Source, <-store.listed, "events must select only their own source") - grants.DeleteObject(grant.ResourceName()) - require.Equal(t, grant.Source, <-store.listed, "source deletion must also revoke its old actors") -} - -func TestLiveEgressWarnsWhenMergedGrantsExceedLimit(t *testing.T) { - base := egress.Access{} - for i := range 256 { - base.Destinations = append(base.Destinations, fmt.Sprintf("https://host-%03d.example.com:443", i)) + require.Equal(t, 1, healthy) + inputs.agents.UpdateObject(&v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "unrelated", UID: "other"}}) + source := database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team", Name: "unrelated"} + select { + case got := <-store.listed: + require.Equal(t, source, got) + case <-ctx.Done(): + t.Fatal("source change was not queued") } - var logs bytes.Buffer - ctx := logging.IntoContext(t.Context(), slog.New(slog.NewJSONHandler(&logs, nil))) - resolver := &EgressResolver{ - reader: egressTestReader(t, egressTestAgent(), egressTestPolicy("https://extra.example.com", nil)), - warnings: rate.NewLimiter(rate.Every(time.Hour), 1), - store: &egressTestStore{pinned: database.RuntimeEgress{Kind: "agent", Namespace: "team", SourceName: "source", SourceUID: "uid", EgressDestinations: base.Destinations}}, + inputs.agents.DeleteObject("team/unrelated") + select { + case got := <-store.listed: + require.Equal(t, source, got) + case <-ctx.Done(): + t.Fatal("source deletion was not queued") } - for range 3 { - got, err := resolver.ResolveEgress(ctx, "full-revision") - require.NoError(t, err) - require.Equal(t, base, got) +} + +func TestEgressWithholdsConflictingGrantsAndWarns(t *testing.T) { + for _, reason := range []string{"credential", "passthrough", "rule limit"} { + t.Run(reason, func(t *testing.T) { + var logs bytes.Buffer + ctx := logging.IntoContext(t.Context(), slog.New(slog.NewJSONHandler(&logs, nil))) + inputs := newEgressTestInputs(ctx) + inputs.source(database.RuntimeEgressAgent, "uid", []corev1.LocalObjectReference{{Name: "access"}}) + base := egress.Access{Destinations: []string{"https://model.example.com:443"}} + pinned := database.RuntimeEgress{Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"} + authentication := &v1alpha3.EgressAuthentication{Header: "authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "other"}, Key: "token"}} + switch reason { + case "credential": + base.Credentials = []egress.Credential{{Origin: base.Destinations[0], Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token"}} + case "passthrough": + pinned.TokenPassthroughOrigins = base.Destinations + case "rule limit": + base.Destinations = nil + for i := range 256 { + base.Destinations = append(base.Destinations, fmt.Sprintf("https://host-%03d.example.com:443", i)) + } + authentication = nil + } + pinned.EgressDestinations, pinned.Credentials = base.Destinations, base.Credentials + inputs.policies.UpdateObject(egressTestPolicy("https://model.example.com", authentication)) + inputs.secrets.UpdateObject(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "other"}, Data: map[string][]byte{"token": []byte("private-value")}}) + store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"revision": pinned}} + controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) + controller.warnings = rate.NewLimiter(rate.Every(time.Hour), 1) + for range 3 { + require.NoError(t, controller.ReconcileRevision(ctx, "revision")) + got, err := store.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + require.Equal(t, base, got) + } + require.Equal(t, 1, strings.Count(logs.String(), "explicit egress grants withheld")) + require.Contains(t, logs.String(), `"level":"WARN"`) + require.Contains(t, logs.String(), `"revision":"revision"`) + require.NotContains(t, logs.String(), "private-value") + }) } - require.Equal(t, 1, strings.Count(logs.String(), "explicit egress grants withheld")) - require.Contains(t, logs.String(), `"level":"WARN"`) - require.Contains(t, logs.String(), `"revision":"full-revision"`) } diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index 0bdefc2ef3..e9611c4ff1 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -54,7 +54,7 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { templates.template.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}, }} - reconciler := &Reconciler{collections: collections, templates: templates, store: store} + reconciler := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: collections, templates: templates, store: store} require.NoError(t, reconciler.reconcileAgent(ctx, initial.ResourceName())) updatedHarness := harnesses.List()[0].DeepCopy() @@ -155,7 +155,7 @@ func TestPreparationErrorsPublishStatusAndRecover(t *testing.T) { collections, _ := newPreparationTestCollections(t, "microvm") initial := collections.Reconciliations.List()[0] statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() - reconciler := &Reconciler{collections: collections, templates: &test.templates, store: &test.store, status: statusClient} + reconciler := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: collections, templates: &test.templates, store: &test.store, status: statusClient} err := reconciler.reconcileAgent(t.Context(), initial.ResourceName()) require.Error(t, err) require.Equal(t, test.code, status.Code(err)) @@ -236,7 +236,7 @@ func TestPreparationRevisionIsolation(t *testing.T) { collections, harnesses := newPreparationTestCollections(t, "microvm") initial := collections.Reconciliations.List()[0] templates := &fakeActorTemplates{createErr: status.Error(codes.FailedPrecondition, `SandboxConfig "microvm" not found`)} - reconciler := &Reconciler{collections: collections, templates: templates, store: &fakeRuntimeRevisionStore{}} + reconciler := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: collections, templates: templates, store: &fakeRuntimeRevisionStore{}} require.Error(t, reconciler.reconcileAgent(t.Context(), initial.ResourceName())) waitFor(t, func() bool { return collections.Reconciliations.GetKey(initial.ResourceName()).PreparationFailure != nil @@ -278,7 +278,7 @@ func TestPreparationQueueAndPollerBackoff(t *testing.T) { initial := collections.Reconciliations.List()[0] statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() backend := &preparationTestBackend{templates: &test.templates, store: &test.store} - r := newReconciler(collections, backend, backend, statusClient) + r := newReconciler(collections, backend, backend, statusClient, revisionEgressFunc(func(context.Context, string) error { return nil })) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -405,7 +405,7 @@ func TestPreparationDesiredChangesBypassBackoff(t *testing.T) { initial := collections.Reconciliations.List()[0] templates := &fakeActorTemplates{createErr: status.Error(codes.FailedPrecondition, "missing config")} store := &fakeRuntimeRevisionStore{} - r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3()) + r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3(), revisionEgressFunc(func(context.Context, string) error { return nil })) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -459,7 +459,7 @@ func TestPreparationTerminalFailures(t *testing.T) { } store := &fakeRuntimeRevisionStore{} statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() - r := newReconciler(collections, templates, store, statusClient) + r := newReconciler(collections, templates, store, statusClient, revisionEgressFunc(func(context.Context, string) error { return nil })) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -518,7 +518,7 @@ func TestInvalidActorTemplateHasNoCompiledTarget(t *testing.T) { require.Nil(t, state.PreparationFailure) store := &fakeRuntimeRevisionStore{} templates := &fakeActorTemplates{getErr: errors.New("Substrate must not be called for an invalid target")} - r := &Reconciler{collections: collections, templates: templates, store: store} + r := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: collections, templates: templates, store: store} require.NoError(t, r.reconcileAgent(t.Context(), initial.ResourceName())) require.Empty(t, store.pair.DesiredRevision) require.Nil(t, templates.template) @@ -572,3 +572,29 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections }) return collections, harnesses } + +func TestInitialRevisionWaitsForEgressPublication(t *testing.T) { + collections, _ := newPreparationTestCollections(t, "gvisor") + initial := collections.Reconciliations.List()[0] + template := proto.CloneOf(initial.Target.ActorTemplate) + template.Metadata.Uid = "prepared" + template.Status = sandboxGoldenStatus() + store := &fakeRuntimeRevisionStore{} + publishErr := errors.New("policy persistence unavailable") + r := &Reconciler{collections: collections, templates: &fakeActorTemplates{template: template}, store: store, + egress: revisionEgressFunc(func(_ context.Context, revision string) error { + require.NotNil(t, store.revision) + require.Equal(t, store.revision.Revision, revision) + require.False(t, store.markedSuccessful, "initial policy must precede readiness") + return publishErr + }), + } + require.ErrorIs(t, r.reconcileAgent(t.Context(), initial.ResourceName()), publishErr) + require.False(t, store.markedSuccessful) + waitFor(t, func() bool { + return collections.Reconciliations.GetKey(initial.ResourceName()).PreparationFailure != nil + }) + publishErr = nil + require.NoError(t, r.reconcileAgent(t.Context(), initial.ResourceName())) + require.True(t, store.markedSuccessful) +} diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index 1b72eeaddc..a9f159a8b6 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -182,6 +182,7 @@ type Reconciler struct { collections Collections templates actorTemplateClient store runtimeRevisionStore + egress revisionEgressReconciler status kagentclient.ApiV1alpha3Interface agents controllers.Queue @@ -194,12 +195,12 @@ type Reconciler struct { // NewReconciler creates the Kubernetes and database write boundary. Run starts // its queues after the registered KRT handlers have received initial state. -func NewReconciler(config *rest.Config, collections Collections, store runtimeRevisionStore, templates actorTemplateClient) (*Reconciler, error) { +func NewReconciler(config *rest.Config, collections Collections, store runtimeRevisionStore, templates actorTemplateClient, egress revisionEgressReconciler) (*Reconciler, error) { statusClient, err := kagentclient.NewForConfig(config) if err != nil { return nil, fmt.Errorf("create kagent status client: %w", err) } - return newReconciler(collections, templates, store, statusClient), nil + return newReconciler(collections, templates, store, statusClient, egress), nil } func newReconciler( @@ -207,11 +208,13 @@ func newReconciler( templates actorTemplateClient, store runtimeRevisionStore, status kagentclient.ApiV1alpha3Interface, + egress revisionEgressReconciler, ) *Reconciler { r := &Reconciler{ collections: collections, templates: templates, store: store, + egress: egress, status: status, } r.agents = newReconciliationQueue("agents", func(item any) error { @@ -374,9 +377,17 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { ActorTemplateAtespace: observed.GetMetadata().GetAtespace(), ActorTemplateName: observed.GetMetadata().GetName(), ActorTemplateUID: observed.GetMetadata().GetUid(), } ready := observed.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() != nil - if err := r.store.RecordRuntimeRevision(ctx, revision, ready); err != nil { + if err := r.store.RecordRuntimeRevision(ctx, revision, false); err != nil { return r.observePreparationError(*state, fmt.Errorf("store runtime revision %s: %w", target.RevisionID, err)) } + if err := r.egress.ReconcileRevision(ctx, revision.Revision); err != nil { + return r.observePreparationError(*state, fmt.Errorf("publish runtime egress %s: %w", target.RevisionID, err)) + } + if ready { + if err := r.store.RecordRuntimeRevision(ctx, revision, true); err != nil { + return r.observePreparationError(*state, fmt.Errorf("promote runtime revision %s: %w", target.RevisionID, err)) + } + } // This observation drives Kubernetes Ready status on a separate queue. // Publish it only after session creation can select the persisted revision. r.observePreparation(*state, observed, nil) diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index 6c873e7b18..e6ddfdb851 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -55,7 +55,7 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { store := &fakeRuntimeRevisionStore{} templates := &fakeActorTemplates{} statusClient := kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3() - reconciler := &Reconciler{ + reconciler := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: Collections{ Agents: krttest.GetMockCollection[*kagentv1alpha3.Agent](mock), AgentRuntimeObservations: krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...), @@ -109,7 +109,7 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { } // A fresh reconciler must clean up observations without remembering earlier calls. - reconciler = &Reconciler{collections: reconciler.collections, templates: templates, store: store, status: statusClient} + reconciler = &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: reconciler.collections, templates: templates, store: store, status: statusClient} nextTarget := *state.Target state.Target = &nextTarget state.Target.ActorTemplate = proto.CloneOf(state.Target.ActorTemplate) @@ -157,7 +157,7 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) states := krt.NewStaticCollection[AgentReconciliation](nil, nil, opts.WithName("Reconciliations")...) templates := &fakeActorTemplates{} - reconciler := &Reconciler{ + reconciler := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: Collections{ AgentRuntimeObservations: krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...), Reconciliations: states, @@ -391,6 +391,7 @@ func TestReconcilerUpdatesModelConfigStatusOnSecretHashChange(t *testing.T) { &fakeActorTemplates{}, &fakeRuntimeRevisionStore{}, statusClient, + revisionEgressFunc(func(context.Context, string) error { return nil }), ) go reconciler.Run(stop) diff --git a/go/core/internal/controller/sandbox_integration_test.go b/go/core/internal/controller/sandbox_integration_test.go index c2a16ce1d4..aa3d5f8638 100644 --- a/go/core/internal/controller/sandbox_integration_test.go +++ b/go/core/internal/controller/sandbox_integration_test.go @@ -65,7 +65,7 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { Secrets: typedCollection[*corev1.Secret](runtimeClient, []string{"team-a"}, "Secrets", options), SandboxTemplates: typedCollection[*kagentv1alpha3.SandboxTemplate](runtimeClient, []string{"team-a"}, "SandboxTemplates", options), WorkerPools: pools, }} - reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}) + reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, revisionEgressFunc(func(context.Context, string) error { return nil })) require.NoError(t, err) results := make(chan error, 2) go func() { results <- runtime.Start(ctx) }() diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index 884ee3439d..4fc6a061dc 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -41,18 +41,19 @@ type sandboxRevisionStore interface { type SandboxReconciler struct { collections sandboxCollections store sandboxRevisionStore + egress revisionEgressReconciler actors actorTemplateClient client kagentclient.ApiV1alpha3Interface } var _ manager.LeaderElectionRunnable = (*SandboxReconciler)(nil) -func NewSandboxReconciler(config *rest.Config, runtime *Runtime, store sandboxRevisionStore, actors actorTemplateClient, policy substrate.SandboxPolicy) (*SandboxReconciler, error) { +func NewSandboxReconciler(config *rest.Config, runtime *Runtime, store sandboxRevisionStore, actors actorTemplateClient, policy substrate.SandboxPolicy, egress revisionEgressReconciler) (*SandboxReconciler, error) { client, err := kagentclient.NewForConfig(config) if err != nil { return nil, fmt.Errorf("create SandboxTemplate client: %w", err) } - return &SandboxReconciler{collections: newSandboxCollections(runtime.Collections, policy, runtime.Options), store: store, actors: actors, client: client}, nil + return &SandboxReconciler{collections: newSandboxCollections(runtime.Collections, policy, runtime.Options), store: store, actors: actors, client: client, egress: egress}, nil } func (s *SandboxReconciler) NeedLeaderElection() bool { return true } @@ -175,9 +176,17 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), SourceSnapshot: state.SourceSnapshot, } - if err := s.store.RecordSandboxRevision(ctx, revision, ready); err != nil { + if err := s.store.RecordSandboxRevision(ctx, revision, false); err != nil { return s.observeError(*state, fmt.Errorf("store sandbox revision %s: %w", state.RevisionID, err), true) } + if err := s.egress.ReconcileRevision(ctx, revision.Revision); err != nil { + return s.observeError(*state, fmt.Errorf("publish sandbox egress %s: %w", state.RevisionID, err), true) + } + if ready { + if err := s.store.RecordSandboxRevision(ctx, revision, true); err != nil { + return s.observeError(*state, fmt.Errorf("promote sandbox revision %s: %w", state.RevisionID, err), true) + } + } // Kubernetes readiness can only follow a revision that callers can select // from the store. Failed persistence must never publish a ready observation. s.collections.observations.ConditionalUpdateObject(sandboxRuntimeObservation{Key: key, RevisionID: state.RevisionID, Template: observed}) diff --git a/go/core/internal/controller/sandbox_reconciler_test.go b/go/core/internal/controller/sandbox_reconciler_test.go index eee798f795..b2f421cb8b 100644 --- a/go/core/internal/controller/sandbox_reconciler_test.go +++ b/go/core/internal/controller/sandbox_reconciler_test.go @@ -126,6 +126,7 @@ func newSandboxTestReconciler(t *testing.T, guestImage string) (*SandboxReconcil actors := &sandboxTestActors{store: store, templates: map[string]*ateapipb.ActorTemplate{}} reconciler := &SandboxReconciler{ collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools}, substrate.SandboxPolicy{GuestImage: guestImage, CPU: "1", Memory: "1Gi"}, opts), + egress: revisionEgressFunc(func(context.Context, string) error { return nil }), store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3(), } waitFor(t, func() bool { return reconciler.collections.states.GetKey("team-a/scratch") != nil }) @@ -295,3 +296,23 @@ func TestSandboxPendingStatusRecoversWithoutGraphEvent(t *testing.T) { return err == nil && apimeta.IsStatusConditionTrue(current.Status.Conditions, "Ready") }, 5*time.Second, 10*time.Millisecond) } + +func TestInitialSandboxRevisionWaitsForEgressPublication(t *testing.T) { + s, templates, _ := newSandboxTestReconciler(t, "guest@sha256:"+strings.Repeat("b", 64)) + store := s.store.(*sandboxTestStore) + s.actors.(*sandboxTestActors).autoReady = true + require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) + syncSandboxTemplate(t, s, templates) + publishErr := errors.New("policy persistence unavailable") + s.egress = revisionEgressFunc(func(_ context.Context, revision string) error { + require.Equal(t, store.revision.Revision, revision) + require.False(t, store.ready, "initial policy must precede readiness") + return publishErr + }) + require.ErrorIs(t, s.reconcile(t.Context(), "team-a/scratch"), publishErr) + require.False(t, store.ready) + waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch").Failure != nil }) + publishErr = nil + require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) + require.True(t, store.ready) +} diff --git a/go/core/internal/database/runtime_egress.go b/go/core/internal/database/runtime_egress.go index 4b12302398..2c238cdf3f 100644 --- a/go/core/internal/database/runtime_egress.go +++ b/go/core/internal/database/runtime_egress.go @@ -12,7 +12,7 @@ import ( // RuntimeEgress retains integration access and the source identity of a pinned // revision. Explicit policy grants are resolved from the current Kubernetes source. type RuntimeEgress struct { - Kind string + Kind RuntimeEgressSourceKind Namespace string SourceName string SourceUID string @@ -31,7 +31,7 @@ func (c *Client) GetRuntimeEgress(ctx context.Context, revision string) (*Runtim FROM runtime_revision r LEFT JOIN agent_revision a USING (revision) LEFT JOIN sandbox_revision s USING (revision) - WHERE r.revision = $1 + WHERE r.revision = $1 AND r.deleted_at IS NULL `, pgx.RowToStructByName[RuntimeEgress], revision) if err != nil { return nil, fmt.Errorf("get runtime egress %s: %w", revision, notFoundOr(err)) @@ -44,15 +44,101 @@ func (c *Client) GetRuntimeEgress(ctx context.Context, revision string) (*Runtim return &row, nil } +// RuntimeEgressSourceKind identifies the Kubernetes owner of a runtime revision. +type RuntimeEgressSourceKind string + +const ( + RuntimeEgressAgent RuntimeEgressSourceKind = runtimeKindAgent + RuntimeEgressSandbox RuntimeEgressSourceKind = runtimeKindSandbox +) + // RuntimeEgressSource selects all revisions of an Agent or SandboxTemplate, // including former UIDs whose grants must be revoked after source recreation. -// The zero value selects every source for recovery sweeps. type RuntimeEgressSource struct { - Kind string + Kind RuntimeEgressSourceKind Namespace string Name string } +// ListRuntimeEgressRevisions lists retained revisions in key order for recovery, +// including revisions without actors. Deleting revisions are excluded. +func (c *Client) ListRuntimeEgressRevisions(ctx context.Context, after string, limit int) ([]string, error) { + return queryMany(ctx, c.db, ` + SELECT revision FROM runtime_revision + WHERE revision > $1 AND deleted_at IS NULL ORDER BY revision LIMIT $2 + `, pgx.RowTo[string], after, limit) +} + +// ListRuntimeEgressSourceRevisions lists a source's retained revisions across +// Kubernetes UIDs. Empty and unsupported kinds are errors, never sweep selectors. +func (c *Client) ListRuntimeEgressSourceRevisions(ctx context.Context, source RuntimeEgressSource, after string, limit int) ([]string, error) { + if source.Namespace == "" || source.Name == "" { + return nil, fmt.Errorf("runtime egress source requires namespace and name") + } + switch source.Kind { + case RuntimeEgressAgent: + return queryMany(ctx, c.db, ` + SELECT r.revision FROM runtime_revision r JOIN agent_revision a USING (revision) + WHERE r.namespace = $1 AND a.agent_name = $2 AND r.revision > $3 + AND r.deleted_at IS NULL ORDER BY r.revision LIMIT $4 + `, pgx.RowTo[string], source.Namespace, source.Name, after, limit) + case RuntimeEgressSandbox: + return queryMany(ctx, c.db, ` + SELECT r.revision FROM runtime_revision r JOIN sandbox_revision s USING (revision) + WHERE r.namespace = $1 AND s.sandbox_template_name = $2 AND r.revision > $3 + AND r.deleted_at IS NULL ORDER BY r.revision LIMIT $4 + `, pgx.RowTo[string], source.Namespace, source.Name, after, limit) + default: + return nil, fmt.Errorf("unknown runtime egress source kind %q", source.Kind) + } +} + +// SetRuntimeEgressPolicy publishes controller output for a retained revision. +// It does not mutate the revision's pinned inputs or readiness. Missing/deleting +// revisions return ErrNotFound; malformed policy input is rejected before writing. +func (c *Client) SetRuntimeEgressPolicy(ctx context.Context, revision string, access egress.Access) error { + canonical, err := egress.CanonicalAccess(access.Destinations, access.Credentials) + if err != nil { + return fmt.Errorf("validate runtime egress policy %s: %w", revision, err) + } + if canonical.Destinations == nil { + canonical.Destinations = []string{} + } + if canonical.Credentials == nil { + canonical.Credentials = []egress.Credential{} + } + result, err := c.db.Exec(ctx, ` + INSERT INTO runtime_egress_policy (revision, destinations, credentials) + SELECT revision, $2, $3 FROM runtime_revision WHERE revision = $1 AND deleted_at IS NULL + ON CONFLICT (revision) DO UPDATE SET destinations = EXCLUDED.destinations, credentials = EXCLUDED.credentials + `, revision, canonical.Destinations, canonical.Credentials) + if err != nil { + return fmt.Errorf("store runtime egress policy %s: %w", revision, err) + } + if result.RowsAffected() == 0 { + return ErrNotFound + } + return nil +} + +// GetRuntimeEgressPolicy reads the latest controller output, independent of any +// replica's Kubernetes cache. Missing and malformed policies fail closed. +func (c *Client) GetRuntimeEgressPolicy(ctx context.Context, revision string) (egress.Access, error) { + access, err := queryOne(ctx, c.db, ` + SELECT destinations, credentials FROM runtime_egress_policy WHERE revision = $1 + `, pgx.RowToStructByName[egress.Access], revision) + if err != nil { + return egress.Access{}, fmt.Errorf("get runtime egress policy %s: %w", revision, notFoundOr(err)) + } + canonical, err := egress.CanonicalAccess(access.Destinations, access.Credentials) + if err != nil { + return egress.Access{}, fmt.Errorf("decode runtime egress policy %s: %w", revision, err) + } + return canonical, nil +} + +var _ egress.PolicyStore = (*Client)(nil) + // RuntimeEgressTarget identifies an allocated runtime, including suspended or // partially created actors that still need live policy reconciliation. type RuntimeEgressTarget struct { @@ -63,24 +149,21 @@ type RuntimeEgressTarget struct { ActorUID string } -// ListRuntimeEgressTargets lists a bounded page for a source in ID order. Deleting instances -// and unissued creations are excluded; a concurrent creation is picked up by the next reconciliation. -func (c *Client) ListRuntimeEgressTargets(ctx context.Context, source RuntimeEgressSource, afterID string, limit int) ([]RuntimeEgressTarget, error) { +// ListRuntimeEgressTargets lists a revision's allocated instances in ID order. +// Deleting instances and unissued creations are excluded. Concurrent creation +// reads the persisted policy itself and rechecks it after applying. +func (c *Client) ListRuntimeEgressTargets(ctx context.Context, revision, afterID string, limit int) ([]RuntimeEgressTarget, error) { rows, err := queryMany(ctx, c.db, ` - SELECT i.id::text AS id, i.prepared_revision AS revision, r.actor_template_atespace AS atespace, - r.actor_template_name AS template_name, COALESCE(s.actor_uid, '') AS actor_uid - FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision - LEFT JOIN session s ON s.id = i.id - WHERE i.id > COALESCE(NULLIF($1, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) - AND ($3 = '' OR (r.namespace = $4 AND r.revision IN ( - SELECT revision FROM agent_revision WHERE $3 = 'agent' AND agent_name = $5 - UNION ALL - SELECT revision FROM sandbox_revision WHERE $3 = 'sandbox' AND sandbox_template_name = $5 - ))) - AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) - AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' - ORDER BY i.id LIMIT $2 - `, pgx.RowToStructByName[RuntimeEgressTarget], afterID, limit, source.Kind, source.Namespace, source.Name) + SELECT i.id::text AS id, i.prepared_revision AS revision, r.actor_template_atespace AS atespace, + r.actor_template_name AS template_name, COALESCE(s.actor_uid, '') AS actor_uid + FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision + LEFT JOIN session s ON s.id = i.id + WHERE i.prepared_revision = $1 + AND i.id > COALESCE(NULLIF($2, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) + AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' + ORDER BY i.id LIMIT $3 + `, pgx.RowToStructByName[RuntimeEgressTarget], revision, afterID, limit) if err != nil { return nil, fmt.Errorf("list runtime egress targets: %w", err) } diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index d220c398d7..e56630a372 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { @@ -39,7 +40,7 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { pinned, err := client.GetRuntimeEgress(ctx, instance.PreparedRevision) require.NoError(t, err) require.Equal(t, "old-uid", pinned.SourceUID) - require.Equal(t, "agent", pinned.Kind) + require.Equal(t, RuntimeEgressAgent, pinned.Kind) require.Equal(t, revision.TokenPassthroughOrigins, pinned.TokenPassthroughOrigins) sandbox, options := sandboxFixture(t, client, "sandbox") created, _, err := client.CreateSandbox(ctx, sandbox, "sandbox", options) @@ -51,42 +52,86 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { require.True(t, claimed) pinned, err = client.GetRuntimeEgress(ctx, created.PreparedRevision) require.NoError(t, err) - require.Equal(t, "sandbox", pinned.Kind) + require.Equal(t, RuntimeEgressSandbox, pinned.Kind) require.Equal(t, "scratch", pinned.SourceName) require.Equal(t, "uid", pinned.SourceUID) for _, test := range []struct { source RuntimeEgressSource want string }{ - {RuntimeEgressSource{Kind: "agent", Namespace: "team-a", Name: "assistant"}, instance.Id}, - {RuntimeEgressSource{Kind: "sandbox", Namespace: "team-a", Name: "scratch"}, created.Id}, - {RuntimeEgressSource{Kind: "agent", Namespace: "other", Name: "assistant"}, ""}, - {RuntimeEgressSource{Kind: "agent", Namespace: "team-a", Name: "unrelated"}, ""}, + {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, instance.PreparedRevision}, + {RuntimeEgressSource{Kind: RuntimeEgressSandbox, Namespace: "team-a", Name: "scratch"}, created.PreparedRevision}, + {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "other", Name: "assistant"}, ""}, + {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "unrelated"}, ""}, } { - targets, err := client.ListRuntimeEgressTargets(ctx, test.source, "", 10) + revisions, err := client.ListRuntimeEgressSourceRevisions(ctx, test.source, "", 10) require.NoError(t, err) if test.want == "" { - require.Empty(t, targets) + require.Empty(t, revisions) } else { - require.Len(t, targets, 1) - require.Equal(t, test.want, targets[0].ID) + require.Equal(t, []string{test.want}, revisions) } } - - first, err := client.ListRuntimeEgressTargets(ctx, RuntimeEgressSource{}, "", 1) + for _, kind := range []RuntimeEgressSourceKind{"", "typo"} { + _, err := client.ListRuntimeEgressSourceRevisions(ctx, RuntimeEgressSource{Kind: kind, Namespace: "team-a", Name: "assistant"}, "", 10) + require.Error(t, err) + } + first, err := client.ListRuntimeEgressRevisions(ctx, "", 1) require.NoError(t, err) require.Len(t, first, 1) - second, err := client.ListRuntimeEgressTargets(ctx, RuntimeEgressSource{}, first[0].ID, 1) + second, err := client.ListRuntimeEgressRevisions(ctx, first[0], 1) require.NoError(t, err) require.Len(t, second, 1) - require.ElementsMatch(t, []string{instance.Id, created.Id}, []string{first[0].ID, second[0].ID}) - last, err := client.ListRuntimeEgressTargets(ctx, RuntimeEgressSource{}, second[0].ID, 1) + require.ElementsMatch(t, []string{instance.PreparedRevision, created.PreparedRevision}, []string{first[0], second[0]}) + last, err := client.ListRuntimeEgressRevisions(ctx, second[0], 1) require.NoError(t, err) require.Empty(t, last) + for _, test := range []struct{ revision, id string }{{instance.PreparedRevision, instance.Id}, {created.PreparedRevision, created.Id}} { + targets, err := client.ListRuntimeEgressTargets(ctx, test.revision, "", 1) + require.NoError(t, err) + require.Len(t, targets, 1) + require.Equal(t, test.id, targets[0].ID) + targets, err = client.ListRuntimeEgressTargets(ctx, test.revision, test.id, 1) + require.NoError(t, err) + require.Empty(t, targets) + } _, err = client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE) require.NoError(t, err) - remaining, err := client.ListRuntimeEgressTargets(ctx, RuntimeEgressSource{}, "", 10) + remaining, err := client.ListRuntimeEgressTargets(ctx, instance.PreparedRevision, "", 10) + require.NoError(t, err) + require.Empty(t, remaining) +} + +func TestRuntimeEgressPolicyIsMutableControllerOutput(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + revision := RuntimeRevision{Revision: "policy-revision", Namespace: "team-a", AgentName: "assistant", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", EgressDestinations: []string{"https://model.example.com:443"}} + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, false)) + _, err := client.GetRuntimeEgressPolicy(ctx, revision.Revision) + require.ErrorIs(t, err, ErrNotFound, "absence must not become an empty allowlist") + allowed := egress.Access{Destinations: []string{"https://model.example.com:443", "https://tools.example.com:443"}, Credentials: []egress.Credential{{Origin: "https://tools.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/auth/token"}}} + require.NoError(t, client.SetRuntimeEgressPolicy(ctx, revision.Revision, allowed)) + // A different API replica reads the same intended policy without any K8s cache. + replica := NewClient(client.db, "public") + got, err := replica.GetRuntimeEgressPolicy(ctx, revision.Revision) + require.NoError(t, err) + require.Equal(t, allowed, got) + require.NoError(t, client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{Destinations: revision.EgressDestinations})) + // Preparation observations must not restore an earlier policy. + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, true)) + got, err = replica.GetRuntimeEgressPolicy(ctx, revision.Revision) + require.NoError(t, err) + require.Equal(t, revision.EgressDestinations, got.Destinations) + require.Empty(t, got.Credentials) + pinned, err := client.GetRuntimeEgress(ctx, revision.Revision) + require.NoError(t, err) + require.Equal(t, revision.EgressDestinations, pinned.EgressDestinations) + require.Empty(t, pinned.Credentials) + require.ErrorIs(t, client.SetRuntimeEgressPolicy(ctx, "missing", allowed), ErrNotFound) + require.Error(t, client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{Destinations: []string{"invalid"}})) + // Durable malformed data must fail rather than granting a partial policy. + _, err = client.db.Exec(ctx, `UPDATE runtime_egress_policy SET credentials = '[{"hostname":"legacy","header":"authorization"}]' WHERE revision = $1`, revision.Revision) require.NoError(t, err) - require.Len(t, remaining, 1) - require.Equal(t, created.Id, remaining[0].ID) + _, err = replica.GetRuntimeEgressPolicy(ctx, revision.Revision) + require.Error(t, err) } diff --git a/go/core/internal/egress/destination.go b/go/core/internal/egress/destination.go index 3a44779c2e..398a3022dd 100644 --- a/go/core/internal/egress/destination.go +++ b/go/core/internal/egress/destination.go @@ -59,7 +59,7 @@ func ParseOrigin(raw string) (*url.URL, error) { return u, nil } -// Access is the resolved, revision-pinned network grant. It contains references, +// Access is a compiled network grant. It contains references, // never Secret values. type Access struct { Destinations []string `json:"destinations,omitempty"` diff --git a/go/core/internal/egress/live.go b/go/core/internal/egress/live.go index e9b752f266..066c89e72f 100644 --- a/go/core/internal/egress/live.go +++ b/go/core/internal/egress/live.go @@ -6,10 +6,10 @@ import ( "slices" ) -// Resolver combines a pinned revision's integration access with current policy -// grants. Missing or invalid grants must fail closed without retaining old access. -type Resolver interface { - ResolveEgress(context.Context, string) (Access, error) +// PolicyStore reads the controller's persisted intended policy for a revision. +// A missing or malformed policy is an error, never permission to retain old access. +type PolicyStore interface { + GetRuntimeEgressPolicy(context.Context, string) (Access, error) } // MergeAccess validates live grants against the pinned runtime's credentials and diff --git a/go/core/internal/substrate/config.go b/go/core/internal/substrate/config.go index 3e78a7ff2a..8699c200c3 100644 --- a/go/core/internal/substrate/config.go +++ b/go/core/internal/substrate/config.go @@ -8,7 +8,7 @@ import ( // Config holds connection settings for Agent Substrate ate-api. type Config struct { - EgressResolver egress.Resolver + EgressPolicies egress.PolicyStore // AteAPIEndpoint is a gRPC target (e.g. dns:///api.ate-system.svc:443). AteAPIEndpoint string // CAFile contains the CA certificates used to verify ate-api. diff --git a/go/core/internal/substrate/egress.go b/go/core/internal/substrate/egress.go index e3a3f895af..27417b0d35 100644 --- a/go/core/internal/substrate/egress.go +++ b/go/core/internal/substrate/egress.go @@ -3,6 +3,7 @@ package substrate import ( "context" "fmt" + "slices" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -11,15 +12,15 @@ import ( ) // EnsureActorEgressPolicy installs current authorization without restarting the -// actor. Read the backend version before resolving grants through authoritative -// Kubernetes reads. The version precondition orders competing policy writes; it -// cannot make cached Kubernetes state current. Conflicts retry both reads. +// actor. Read the backend version before the controller's persisted intended +// policy so concurrent applications cannot overwrite a newer policy with an old +// one. Conflicts reload both; lifecycle paths never translate Kubernetes. // Substrate deletes the policy with its Actor. func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name, revision string) error { ctx, cancel := c.callCtx(ctx) defer cancel() - if c.cfg.EgressResolver == nil { - return fmt.Errorf("actor egress resolver is required") + if c.cfg.EgressPolicies == nil { + return fmt.Errorf("actor egress policy store is required") } actor := actorRef(atespace, name) for range 5 { @@ -27,9 +28,9 @@ func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name, re if err != nil && status.Code(err) != codes.NotFound { return err } - access, err := c.cfg.EgressResolver.ResolveEgress(ctx, revision) + access, err := c.cfg.EgressPolicies.GetRuntimeEgressPolicy(ctx, revision) if err != nil { - return fmt.Errorf("resolve actor egress: %w", err) + return fmt.Errorf("load intended actor egress: %w", err) } policy, err := ActorEgressPolicy(atespace, access.Destinations, access.Credentials) if err != nil { @@ -38,15 +39,26 @@ func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name, re if existing == nil { _, err = c.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{Actor: actor, EgressPolicy: policy}) } else { - if proto.Equal(&ateapipb.EgressPolicy{Rules: existing.GetRules()}, &ateapipb.EgressPolicy{Rules: policy.GetRules()}) { - return nil + if !proto.Equal(&ateapipb.EgressPolicy{Rules: existing.GetRules()}, &ateapipb.EgressPolicy{Rules: policy.GetRules()}) { + policy.Metadata = proto.CloneOf(existing.GetMetadata()) + _, err = c.UpdateActorEgressPolicy(ctx, &ateapipb.UpdateActorEgressPolicyRequest{Actor: actor, EgressPolicy: policy}) } - policy.Metadata = proto.CloneOf(existing.GetMetadata()) - _, err = c.UpdateActorEgressPolicy(ctx, &ateapipb.UpdateActorEgressPolicyRequest{Actor: actor, EgressPolicy: policy}) } - if status.Code(err) != codes.AlreadyExists && status.Code(err) != codes.Aborted { + if status.Code(err) == codes.AlreadyExists || status.Code(err) == codes.Aborted { + continue + } + if err != nil { return err } + // A source update may have discovered this instance before its Actor + // existed. Recheck after applying so creation cannot miss that update. + latest, err := c.cfg.EgressPolicies.GetRuntimeEgressPolicy(ctx, revision) + if err != nil { + return fmt.Errorf("reload intended actor egress: %w", err) + } + if slices.Equal(access.Destinations, latest.Destinations) && slices.Equal(access.Credentials, latest.Credentials) { + return nil + } } return status.Error(codes.Aborted, "actor egress policy changed during reconciliation") } diff --git a/go/core/internal/substrate/egress_test.go b/go/core/internal/substrate/egress_test.go index 7082a78668..2b917691a8 100644 --- a/go/core/internal/substrate/egress_test.go +++ b/go/core/internal/substrate/egress_test.go @@ -2,6 +2,7 @@ package substrate import ( "context" + "errors" "testing" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -14,18 +15,18 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/egress" ) -type egressResolverFunc func(context.Context, string) (egress.Access, error) +type egressPolicyStoreFunc func(context.Context, string) (egress.Access, error) -func (f egressResolverFunc) ResolveEgress(ctx context.Context, revision string) (egress.Access, error) { +func (f egressPolicyStoreFunc) GetRuntimeEgressPolicy(ctx context.Context, revision string) (egress.Access, error) { return f(ctx, revision) } -var _ egress.Resolver = egressResolverFunc(nil) +var _ egress.PolicyStore = egressPolicyStoreFunc(nil) func TestEnsureActorEgressPolicyUpdatesCurrentGrants(t *testing.T) { fake := &egressPolicyFake{createErr: context.DeadlineExceeded} access := egress.Access{Destinations: []string{"https://api.example.com:443"}} - client := &Client{ControlClient: fake, cfg: Config{EgressResolver: egressResolverFunc(func(_ context.Context, revision string) (egress.Access, error) { + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(_ context.Context, revision string) (egress.Access, error) { require.Equal(t, "pinned-revision", revision) return access, nil })}} @@ -53,10 +54,10 @@ func TestEnsureActorEgressPolicyUpdatesCurrentGrants(t *testing.T) { require.Equal(t, codes.Unavailable, status.Code(reconcile())) } -func TestEnsureActorEgressPolicyReResolvesAfterConflict(t *testing.T) { +func TestEnsureActorEgressPolicyReloadsAfterConflict(t *testing.T) { fake := &egressPolicyFake{} access := egress.Access{} - client := &Client{ControlClient: fake, cfg: Config{EgressResolver: egressResolverFunc(func(context.Context, string) (egress.Access, error) { return access, nil })}} + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Access, error) { return access, nil })}} reconcile := func() error { return client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "revision") } require.NoError(t, reconcile()) access.Destinations = []string{"https://revoked.example.com:443"} @@ -78,6 +79,7 @@ type egressPolicyFake struct { createErr, getErr error created, updated int beforeUpdate func() + afterCreate func() } func (f *egressPolicyFake) CreateActorEgressPolicy(_ context.Context, req *ateapipb.CreateActorEgressPolicyRequest, _ ...grpc.CallOption) (*ateapipb.EgressPolicy, error) { @@ -89,6 +91,9 @@ func (f *egressPolicyFake) CreateActorEgressPolicy(_ context.Context, req *ateap f.policy.Metadata.Uid = "policy-uid" f.policy.Metadata.Version = 1 f.created++ + if f.afterCreate != nil { + f.afterCreate() + } return proto.CloneOf(f.policy), f.createErr } @@ -116,3 +121,27 @@ func (f *egressPolicyFake) UpdateActorEgressPolicy(_ context.Context, req *ateap f.updated++ return proto.CloneOf(f.policy), nil } + +func TestEnsureActorEgressPolicyCatchesUpdateDuringCreation(t *testing.T) { + fake := &egressPolicyFake{} + access := egress.Access{Destinations: []string{"https://revoked.example.com:443"}} + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Access, error) { return access, nil })}} + fake.afterCreate = func() { + // The controller published revocation, but its actor discovery ran before + // creation. There is no concurrent Substrate write to cause a CAS conflict. + access = egress.Access{} + } + require.NoError(t, client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision")) + require.Empty(t, fake.policy.Rules) + require.Equal(t, 1, fake.created) + require.Equal(t, 1, fake.updated) +} + +func TestEnsureActorEgressPolicyRequiresPersistedPolicy(t *testing.T) { + fake := &egressPolicyFake{} + unavailable := errors.New("intended policy unavailable") + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Access, error) { return egress.Access{}, unavailable })}} + require.ErrorIs(t, client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision"), unavailable) + require.Zero(t, fake.created) + require.Zero(t, fake.updated) +} diff --git a/go/core/internal/translator/egress.go b/go/core/internal/translator/egress.go index 01cfd0942a..a0b3e447d4 100644 --- a/go/core/internal/translator/egress.go +++ b/go/core/internal/translator/egress.go @@ -1,7 +1,6 @@ package translator import ( - "context" "fmt" "strings" @@ -10,7 +9,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation" - ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/egress" @@ -35,20 +33,6 @@ func CompileEgressPolicies(ctx krt.HandlerContext, collections Collections, name }) } -// ResolveEgressPolicies applies the same validation as preparation to direct -// Kubernetes reads. Live authorization must supply an uncached reader. -func ResolveEgressPolicies(ctx context.Context, reader ctrlclient.Reader, namespace string, refs []corev1.LocalObjectReference) (egress.Access, error) { - return compileEgressPolicies(namespace, refs, func(name string) (*v1alpha3.EgressPolicy, error) { - policy := &v1alpha3.EgressPolicy{} - err := reader.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: name}, policy) - return policy, err - }, func(name string) (*corev1.Secret, error) { - secret := &corev1.Secret{} - err := reader.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: name}, secret) - return secret, err - }) -} - func compileEgressPolicies(namespace string, refs []corev1.LocalObjectReference, getPolicy func(string) (*v1alpha3.EgressPolicy, error), getSecret func(string) (*corev1.Secret, error)) (egress.Access, error) { if len(refs) > 16 { return egress.Access{}, NewValidationError("at most 16 egressPolicyRefs are supported") diff --git a/go/core/pkg/app/app.go b/go/core/pkg/app/app.go index 864c43188c..21ca283250 100644 --- a/go/core/pkg/app/app.go +++ b/go/core/pkg/app/app.go @@ -276,9 +276,8 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er if err != nil { return err } - egressResolver := controller.NewEgressResolver(runtime, store, manager.GetAPIReader(), watchNamespaces) actors, err := substrate.Dial(ctx, substrate.Config{ - EgressResolver: egressResolver, + EgressPolicies: store, AteAPIEndpoint: cfg.GetSubstrate().GetApiEndpoint(), CAFile: cfg.GetSubstrate().GetCaFile(), ClientCertFile: cfg.GetSubstrate().GetClientCertFile(), @@ -288,10 +287,11 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er return err } defer actors.Close() - if err := manager.Add(controller.NewEgressReconciler(egressResolver, actors)); err != nil { + egressReconciler := controller.NewEgressReconciler(ctx, runtime, store, actors, watchNamespaces) + if err := manager.Add(egressReconciler); err != nil { return fmt.Errorf("add egress reconciler: %w", err) } - reconciler, err := controller.NewReconciler(kubeConfig, runtime.Collections, store, actors) + reconciler, err := controller.NewReconciler(kubeConfig, runtime.Collections, store, actors, egressReconciler) if err != nil { return err } @@ -379,7 +379,7 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er CPU: cfg.GetSandboxes().GetCpu(), Memory: cfg.GetSandboxes().GetMemory(), } - preparation, err := controller.NewSandboxReconciler(kubeConfig, runtime, store, actors, policy) + preparation, err := controller.NewSandboxReconciler(kubeConfig, runtime, store, actors, policy, egressReconciler) if err != nil { return err } diff --git a/go/core/pkg/migrations/core/000002_runtime_token_passthrough_origins.sql b/go/core/pkg/migrations/core/000002_runtime_token_passthrough_origins.sql index 1ebf273838..aa435c45f0 100644 --- a/go/core/pkg/migrations/core/000002_runtime_token_passthrough_origins.sql +++ b/go/core/pkg/migrations/core/000002_runtime_token_passthrough_origins.sql @@ -8,6 +8,13 @@ CREATE VIEW agent_runtime_revision AS SELECT r.*, a.agent_name, a.agent_uid, a.agent_card FROM runtime_revision r JOIN agent_revision a USING (revision) WHERE r.kind = 'agent'; +-- Mutable controller output lives outside immutable runtime revision inputs. +CREATE TABLE runtime_egress_policy ( + revision TEXT PRIMARY KEY REFERENCES runtime_revision (revision) ON DELETE CASCADE, + destinations TEXT[] NOT NULL, + credentials JSONB NOT NULL CHECK (jsonb_typeof(credentials) = 'array') +); + -- Target changed sources without scanning all live instances. CREATE INDEX agent_revision_source_idx ON agent_revision (agent_name, revision); CREATE INDEX sandbox_revision_source_idx ON sandbox_revision (sandbox_template_name, revision); @@ -16,6 +23,8 @@ CREATE INDEX runtime_instance_revision_idx ON runtime_instance (prepared_revisio -- +goose Down +DROP TABLE runtime_egress_policy; + DROP INDEX runtime_instance_revision_idx; DROP INDEX sandbox_revision_source_idx; DROP INDEX agent_revision_source_idx; diff --git a/go/core/pkg/migrations/runner_test.go b/go/core/pkg/migrations/runner_test.go index 8fe69c8029..8fb5a4454b 100644 --- a/go/core/pkg/migrations/runner_test.go +++ b/go/core/pkg/migrations/runner_test.go @@ -329,6 +329,9 @@ func TestBuiltinMigrationsRoundTrip(t *testing.T) { if !testColumnExists(t, dsn, "runtime_revision", "token_passthrough_origins") || !testColumnExists(t, dsn, "agent_runtime_revision", "token_passthrough_origins") { t.Fatal("runtime and agent revisions must expose passthrough origins") } + if !testTableExists(t, dsn, "runtime_egress_policy") { + t.Fatal("mutable runtime egress policies are missing") + } if !testColumnExists(t, dsn, "session", "deletion_reason") { t.Fatal("session deletion reason is missing") } From ed6a09de6b9ca6be2b0d4ff28e957049aa5792bf Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 16:14:52 +0000 Subject: [PATCH 16/43] fix: finish live egress review follow-ups Signed-off-by: Eitan Yarmush --- go/core/internal/controller/egress.go | 12 +- go/core/internal/controller/egress_test.go | 14 +- .../database/runtime_creation_test.go | 156 +++++---------- go/core/internal/database/sessions.go | 4 +- go/core/internal/substrate/sandbox.go | 5 +- go/core/test/e2e/a2a_http_test.go | 2 - go/core/test/e2e/egress_credentials_test.go | 180 +++++++++++++++--- go/core/test/e2e/interaction_test.go | 13 +- go/core/test/e2e/main_test.go | 12 ++ go/core/test/e2e/session_expiration_test.go | 1 - 10 files changed, 245 insertions(+), 154 deletions(-) diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go index 5d371757b1..e12788b084 100644 --- a/go/core/internal/controller/egress.go +++ b/go/core/internal/controller/egress.go @@ -8,11 +8,11 @@ import ( "time" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "golang.org/x/time/rate" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "istio.io/istio/pkg/kube/controllers" "istio.io/istio/pkg/kube/krt" + "k8s.io/apimachinery/pkg/util/cache" "sigs.k8s.io/controller-runtime/pkg/manager" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" @@ -88,7 +88,7 @@ type EgressReconciler struct { store runtimeEgressStore actors actorEgressClient namespaces []string - warnings *rate.Limiter + warnings *cache.LRUExpireCache // Initial preparation and source/recovery queues share a publisher. Serialize // cache lookup through commit so an older translation cannot commit last. publishMu sync.Mutex @@ -114,7 +114,7 @@ var _ manager.LeaderElectionRunnable = (*EgressReconciler)(nil) func NewEgressReconciler(ctx context.Context, runtime *Runtime, store runtimeEgressStore, actors actorEgressClient, namespaces []string) *EgressReconciler { e := &EgressReconciler{grants: newEgressGrants(runtime), store: store, actors: actors, - namespaces: slices.Clone(namespaces), warnings: rate.NewLimiter(rate.Every(30*time.Second), 5)} + namespaces: slices.Clone(namespaces), warnings: cache.NewLRUExpireCache(1024)} reconcile := func(item any) error { err := e.reconcileActor(ctx, item.(database.RuntimeEgressTarget)) if status.Code(err) == codes.NotFound { @@ -220,10 +220,14 @@ func (e *EgressReconciler) publishRevision(ctx context.Context, revision string) } access, err := egress.MergeAccess(base, grants, pinned.TokenPassthroughOrigins) if err != nil { - if e.warnings.Allow() { + // Bound warning state while letting every revision report its own failures. + if _, warned := e.warnings.Get(revision); !warned { logging.FromContext(ctx).WarnContext(ctx, "explicit egress grants withheld", "revision", revision, "namespace", pinned.Namespace, "source", pinned.SourceName, "error", err) + e.warnings.Add(revision, struct{}{}, 30*time.Second) } access = base + } else { + e.warnings.Remove(revision) } if err := e.store.SetRuntimeEgressPolicy(ctx, revision, access); err != nil { return false, err diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go index 18dc2f9cbe..2c59955a6e 100644 --- a/go/core/internal/controller/egress_test.go +++ b/go/core/internal/controller/egress_test.go @@ -15,11 +15,12 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/stretchr/testify/require" - "golang.org/x/time/rate" "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/cache" + clocktesting "k8s.io/utils/clock/testing" "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" @@ -344,9 +345,10 @@ func TestEgressWithholdsConflictingGrantsAndWarns(t *testing.T) { pinned.EgressDestinations, pinned.Credentials = base.Destinations, base.Credentials inputs.policies.UpdateObject(egressTestPolicy("https://model.example.com", authentication)) inputs.secrets.UpdateObject(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "other"}, Data: map[string][]byte{"token": []byte("private-value")}}) - store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"revision": pinned}} + store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"revision": pinned, "other": pinned}} controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) - controller.warnings = rate.NewLimiter(rate.Every(time.Hour), 1) + clock := clocktesting.NewFakeClock(time.Now()) + controller.warnings = cache.NewLRUExpireCacheWithClock(1024, clock) for range 3 { require.NoError(t, controller.ReconcileRevision(ctx, "revision")) got, err := store.GetRuntimeEgressPolicy(ctx, "revision") @@ -354,6 +356,12 @@ func TestEgressWithholdsConflictingGrantsAndWarns(t *testing.T) { require.Equal(t, base, got) } require.Equal(t, 1, strings.Count(logs.String(), "explicit egress grants withheld")) + require.NoError(t, controller.ReconcileRevision(ctx, "other")) + require.Contains(t, logs.String(), `"revision":"other"`, "a noisy revision must not suppress another revision's warning") + require.Equal(t, 2, strings.Count(logs.String(), "explicit egress grants withheld")) + clock.Step(31 * time.Second) + require.NoError(t, controller.ReconcileRevision(ctx, "revision")) + require.Equal(t, 3, strings.Count(logs.String(), "explicit egress grants withheld")) require.Contains(t, logs.String(), `"level":"WARN"`) require.Contains(t, logs.String(), `"revision":"revision"`) require.NotContains(t, logs.String(), "private-value") diff --git a/go/core/internal/database/runtime_creation_test.go b/go/core/internal/database/runtime_creation_test.go index e4d716196a..b7d7624d49 100644 --- a/go/core/internal/database/runtime_creation_test.go +++ b/go/core/internal/database/runtime_creation_test.go @@ -2,132 +2,80 @@ package database import ( "context" - "fmt" - "sync" "testing" "time" "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgxpool" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" ) func TestRuntimeCreationUsesLatestSuccessfulRevision(t *testing.T) { for _, kind := range []string{"session", "sandbox"} { - for _, invalidateFirst := range []bool{false, true} { - t.Run(fmt.Sprintf("%s/invalidate_first=%t", kind, invalidateFirst), func(t *testing.T) { - pool := setupTestDB(t) - client := NewClient(pool, "public") - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - var create func(*Client, string) (string, bool, error) - var desire func(*Client, string) error - var prepare func(string) - var createQuery, updateQuery, unresolved string - if kind == "session" { - prepare = func(revision string) { - sessionFixture(t, client, ctx, "team-a", revision, "assistant", "kagent") - } - create = func(c *Client, requestID string) (string, bool, error) { - session, created, err := c.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), requestID) - return session.GetPreparedRevision(), created, err - } - desire = func(c *Client, revision string) error { - return c.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", DesiredRevision: revision}) - } - createQuery, updateQuery = "FROM agent_definition p", "INSERT INTO agent_definition" - } else { - input, options := sandboxFixture(t, client, "revision-1") - prepare = func(revision string) { - _, _ = sandboxFixture(t, client, revision) - } - create = func(c *Client, requestID string) (string, bool, error) { - request := proto.CloneOf(input) - request.Id = uuid.NewString() - instance, created, err := c.CreateSandbox(ctx, request, requestID, options) - return instance.GetPreparedRevision(), created, err - } - desire = func(c *Client, revision string) error { - return c.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) - } - createQuery, updateQuery = "FROM sandbox_template_definition", "INSERT INTO sandbox_template_definition" - unresolved = "pending:uid:2" + t.Run(kind, func(t *testing.T) { + pool := setupTestDB(t) + client := NewClient(pool, "public") + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + var create func(*Client, string) (string, bool, error) + var desire func(*Client, string) error + var prepare func(string) + var unresolved string + if kind == "session" { + prepare = func(revision string) { + sessionFixture(t, client, ctx, "team-a", revision, "assistant", "kagent") } - prepare("revision-1") - revision, created, err := create(client, "original") - require.NoError(t, err) - require.True(t, created) - require.Equal(t, "revision-1", revision) - - // Pause the first transaction with the definition row locked. The - // other operation must wait and observe the committed desired state. - query, waitingQuery := createQuery, updateQuery - if invalidateFirst { - query, waitingQuery = updateQuery, createQuery + create = func(c *Client, requestID string) (string, bool, error) { + session, created, err := c.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), requestID) + return session.GetPreparedRevision(), created, err } - barrier := &runtimeReferenceBarrier{query: query, afterQuery: true, reached: make(chan struct{}), resume: make(chan struct{})} - var resume sync.Once - defer resume.Do(func() { close(barrier.resume) }) - config := pool.Config() - config.ConnConfig.Tracer = barrier - firstPool, err := pgxpool.NewWithConfig(ctx, config) - require.NoError(t, err) - t.Cleanup(firstPool.Close) - first := NewClient(firstPool, "public") - creation, invalidation := make(chan error, 1), make(chan error, 1) - createConcurrent := func(c *Client) { - _, _, err := create(c, "concurrent") - creation <- err + desire = func(c *Client, revision string) error { + return c.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", DesiredRevision: revision}) } - invalidate := func(c *Client) { invalidation <- desire(c, unresolved) } - if invalidateFirst { - go invalidate(first) - } else { - go createConcurrent(first) + } else { + input, options := sandboxFixture(t, client, "revision-1") + prepare = func(revision string) { + _, _ = sandboxFixture(t, client, revision) } - select { - case <-barrier.reached: - case <-ctx.Done(): - t.Fatal(ctx.Err()) + create = func(c *Client, requestID string) (string, bool, error) { + request := proto.CloneOf(input) + request.Id = uuid.NewString() + instance, created, err := c.CreateSandbox(ctx, request, requestID, options) + return instance.GetPreparedRevision(), created, err } - if invalidateFirst { - go createConcurrent(client) - } else { - go invalidate(client) + desire = func(c *Client, revision string) error { + return c.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) } - require.Eventually(t, func() bool { - var waiting bool - err := pool.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE datname = current_database() AND query LIKE $1 AND cardinality(pg_blocking_pids(pid)) > 0)", "%"+waitingQuery+"%").Scan(&waiting) - return err == nil && waiting - }, 5*time.Second, 10*time.Millisecond) - resume.Do(func() { close(barrier.resume) }) - require.NoError(t, <-invalidation) - require.NoError(t, <-creation) + unresolved = "pending:uid:2" + } + prepare("revision-1") + revision, created, err := create(client, "original") + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "revision-1", revision) - // Unresolved and pending edits keep the last successful runtime available. - // Retries retain their original revision even after a newer one is ready. - for _, desired := range []string{unresolved, "revision-2"} { - require.NoError(t, desire(client, desired)) - revision, created, err = create(client, "fresh-"+desired) - require.NoError(t, err) - require.True(t, created) - require.Equal(t, "revision-1", revision) - revision, created, err = create(client, "original") - require.NoError(t, err) - require.False(t, created) - require.Equal(t, "revision-1", revision) - } - prepare("revision-2") - revision, created, err = create(client, "fresh") + // Unresolved and pending edits keep the last successful runtime available. + // Retries retain their original revision even after a newer one is ready. + for _, desired := range []string{unresolved, "revision-2"} { + require.NoError(t, desire(client, desired)) + revision, created, err = create(client, "fresh-"+desired) require.NoError(t, err) require.True(t, created) - require.Equal(t, "revision-2", revision) + require.Equal(t, "revision-1", revision) revision, created, err = create(client, "original") require.NoError(t, err) require.False(t, created) require.Equal(t, "revision-1", revision) - }) - } + } + prepare("revision-2") + revision, created, err = create(client, "fresh") + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "revision-2", revision) + revision, created, err = create(client, "original") + require.NoError(t, err) + require.False(t, created) + require.Equal(t, "revision-1", revision) + }) } } diff --git a/go/core/internal/database/sessions.go b/go/core/internal/database/sessions.go index 918f51801a..096f5ac661 100644 --- a/go/core/internal/database/sessions.go +++ b/go/core/internal/database/sessions.go @@ -101,8 +101,7 @@ func (c *Client) CreateSession(ctx context.Context, request *apiv1.Session, requ } // insertSession reserves a new conversation in CREATING/CREATE and pins the active -// Agent's latest successful revision. It locks the definition -// before the revision to serialize creation with reconciliation. Callers must supply +// Agent's latest successful revision, locking the revision against cleanup. Callers must supply // a transaction so the history and session commit together. A missing prepared target // returns ErrNotFound; a duplicate creator/requestID returns pgx.ErrNoRows for the caller // to resolve. @@ -116,7 +115,6 @@ func insertSession(ctx context.Context, db pgx.Tx, request *apiv1.Session, reque FROM agent_definition p WHERE p.namespace = $1 AND p.agent_name = $2 AND p.retired_at IS NULL AND p.latest_successful_revision IS NOT NULL - FOR UPDATE OF p `, pgx.RowToStructByName[preparedRevision], request.GetAgent().GetNamespace(), request.GetAgent().GetName()) if err != nil { return sessionRow{}, fmt.Errorf("get latest prepared runtime revision: %w", notFoundOr(err)) diff --git a/go/core/internal/substrate/sandbox.go b/go/core/internal/substrate/sandbox.go index ebb2497d61..f73eb6d835 100644 --- a/go/core/internal/substrate/sandbox.go +++ b/go/core/internal/substrate/sandbox.go @@ -67,11 +67,10 @@ func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1. if len(environment) > 32 { return nil, "", nil, fmt.Errorf("sandbox exceeds Substrate's 32 environment variables") } - // Namespace-scoped placement changes immutable template contents, so it - // needs a new revision even when the operator's inputs stay the same. spec := template.Spec.DeepCopy() spec.EgressPolicyRefs = nil - inputs := sandboxInputs{Kind: "sandbox", Version: 4, Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), Spec: *spec, Class: class, Policy: policy} + // Version 2 accounts for namespace-scoped placement in immutable templates. + inputs := sandboxInputs{Kind: "sandbox", Version: 2, Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), Spec: *spec, Class: class, Policy: policy} snapshot, err := json.Marshal(inputs) if err != nil { return nil, "", nil, fmt.Errorf("encode sandbox inputs: %w", err) diff --git a/go/core/test/e2e/a2a_http_test.go b/go/core/test/e2e/a2a_http_test.go index fb386fdc48..eeae1f04f6 100644 --- a/go/core/test/e2e/a2a_http_test.go +++ b/go/core/test/e2e/a2a_http_test.go @@ -246,8 +246,6 @@ func TestSessionHTTPPushFromNonleader(t *testing.T) { func preparePushLeaderHandoff(t *testing.T) (string, func()) { t.Helper() kube := interactionKubeClient(t) - require.NoError(t, appsv1.AddToScheme(kube.Scheme())) - require.NoError(t, coordinationv1.AddToScheme(kube.Scheme())) deployments := &appsv1.DeploymentList{} require.NoError(t, kube.List(t.Context(), deployments, ctrlclient.InNamespace("kagent"), ctrlclient.MatchingLabels{"app.kubernetes.io/component": "controller"})) require.Len(t, deployments.Items, 1) diff --git a/go/core/test/e2e/egress_credentials_test.go b/go/core/test/e2e/egress_credentials_test.go index a0f87b7b1d..a7c77ffca5 100644 --- a/go/core/test/e2e/egress_credentials_test.go +++ b/go/core/test/e2e/egress_credentials_test.go @@ -9,23 +9,27 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "fmt" "io" + "maps" "math/big" "net" "net/http" "net/http/httptest" "net/url" - "strings" "testing" "time" "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" "github.com/kagent-dev/kagent/go/api/v1alpha3" ) @@ -50,7 +54,7 @@ func TestEgressPolicyCredentialDelivery(t *testing.T) { fixture := newInteractionFixtureForTemplate(t, harness, target, template.Name) secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-header-", Namespace: "kagent"}, StringData: map[string]string{"token": token}} require.NoError(t, kube.Create(t.Context(), secret)) - t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), secret)) }) + t.Cleanup(func() { assert.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), secret))) }) ref := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secret.Name}, Key: "token"} var refs []corev1.LocalObjectReference for _, authentication := range []*v1alpha3.EgressAuthentication{ @@ -59,10 +63,10 @@ func TestEgressPolicyCredentialDelivery(t *testing.T) { } { policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-header-", Namespace: "kagent"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: destination, Authentication: authentication}}}} require.NoError(t, kube.Create(t.Context(), policy)) - t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), policy)) }) + t.Cleanup(func() { assert.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), policy))) }) refs = append(refs, corev1.LocalObjectReference{Name: policy.Name}) } - run := runs("credentials") + run := runs("live") finish := runEgressSkill(t, fixture, model, run) run.expect(t, "initial", "DENIED") agent := &v1alpha3.Agent{} @@ -134,7 +138,6 @@ func startAuthenticatedEgressDestination(t *testing.T, kube ctrlclient.Client, t // lookup, credential provider, and guest MITM trust use the normal installation. func trustEgressTestOrigin(t *testing.T, kube ctrlclient.Client, certificate string) { t.Helper() - require.NoError(t, appsv1.AddToScheme(kube.Scheme())) deployments := &appsv1.DeploymentList{} require.NoError(t, kube.List(t.Context(), deployments, ctrlclient.InNamespace("ate-system"), ctrlclient.MatchingLabels{"app": "atenet-egress"})) require.Len(t, deployments.Items, 1) @@ -149,41 +152,174 @@ func trustEgressTestOrigin(t *testing.T, kube ctrlclient.Client, certificate str config := &corev1.ConfigMap{} require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKey{Namespace: deployment.Namespace, Name: configName}, config)) original := config.DeepCopy() - const rootFile = "egress-test-root.pem" - require.NotContains(t, config.Data, rootFile) - require.Equal(t, 1, strings.Count(config.Data["config.yaml"], "backendTLS: {}")) - config.Data["config.yaml"] = strings.Replace(config.Data["config.yaml"], "backendTLS: {}", "backendTLS:\n root: /etc/agentgateway/"+rootFile, 1) - config.Data[rootFile] = certificate - rollout := func(ctx context.Context) { - t.Helper() + patched, restored, err := egressGatewayTrustConfig(config.Data, certificate) + require.NoError(t, err) + config.Data = patched + rollout := func(ctx context.Context) error { current := &appsv1.Deployment{} deploymentKey := ctrlclient.ObjectKeyFromObject(deployment) - require.NoError(t, kube.Get(ctx, deploymentKey, current)) + if err := kube.Get(ctx, deploymentKey, current); err != nil { + return err + } before := current.DeepCopy() if current.Spec.Template.Annotations == nil { current.Spec.Template.Annotations = map[string]string{} } current.Spec.Template.Annotations["kagent.dev/egress-test-rollout"] = uuid.NewString() - require.NoError(t, kube.Patch(ctx, current, ctrlclient.MergeFrom(before))) - require.NoError(t, wait.PollUntilContextTimeout(ctx, time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Patch(ctx, current, ctrlclient.MergeFrom(before)); err != nil { + return err + } + return wait.PollUntilContextTimeout(ctx, time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { observed := &appsv1.Deployment{} if err := kube.Get(ctx, deploymentKey, observed); err != nil { return false, err } return observed.Status.ObservedGeneration >= current.Generation && observed.Status.UpdatedReplicas == *observed.Spec.Replicas && observed.Status.AvailableReplicas == *observed.Spec.Replicas && observed.Status.Replicas == *observed.Spec.Replicas, nil - })) + }) } t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() current := &corev1.ConfigMap{} - require.NoError(t, kube.Get(ctx, ctrlclient.ObjectKeyFromObject(original), current)) + if !assert.NoError(t, kube.Get(ctx, ctrlclient.ObjectKeyFromObject(original), current), "get gateway config for cleanup") { + return + } before := current.DeepCopy() - current.Data["config.yaml"] = original.Data["config.yaml"] - delete(current.Data, rootFile) - require.NoError(t, kube.Patch(ctx, current, ctrlclient.MergeFrom(before))) - rollout(ctx) + current.Data["config.yaml"] = restored["config.yaml"] + delete(current.Data, egressTestRootFile) + if !assert.NoError(t, kube.Patch(ctx, current, ctrlclient.MergeFrom(before)), "restore gateway config") { + return + } + assert.NoError(t, rollout(ctx), "roll out restored gateway config") }) require.NoError(t, kube.Patch(t.Context(), config, ctrlclient.MergeFrom(original))) - rollout(t.Context()) + require.NoError(t, rollout(t.Context())) +} + +const egressTestRootFile = "egress-test-root.pem" + +// Patch only dynamic HTTPS destinations, preserving the gateway's other policies. +// The restore configuration also removes test trust left by an interrupted run. +func egressGatewayTrustConfig(data map[string]string, certificate string) (patched, restored map[string]string, err error) { + var config map[string]any + if err := yaml.Unmarshal([]byte(data["config.yaml"]), &config); err != nil { + return nil, nil, fmt.Errorf("decode gateway config: %w", err) + } + var backends []map[string]any + binds, _ := config["binds"].([]any) + for _, value := range binds { + bind, _ := value.(map[string]any) + if bind["mode"] != "internal" { + continue + } + listeners, _ := bind["listeners"].([]any) + for _, value := range listeners { + listener, _ := value.(map[string]any) + if listener["protocol"] != "HTTPS" { + continue + } + routes, _ := listener["routes"].([]any) + for _, value := range routes { + route, _ := value.(map[string]any) + items, _ := route["backends"].([]any) + for _, value := range items { + backend, _ := value.(map[string]any) + if _, dynamic := backend["dynamic"]; dynamic { + backends = append(backends, backend) + } + } + } + } + } + if len(backends) != 1 { + return nil, nil, fmt.Errorf("expected one dynamic HTTPS backend, found %d", len(backends)) + } + backend := backends[0] + path := []string{"policies", "backendTLS", "root"} + root, _, err := unstructured.NestedString(backend, path...) + if err != nil { + return nil, nil, fmt.Errorf("read gateway TLS root: %w", err) + } + restored = maps.Clone(data) + delete(restored, egressTestRootFile) + const testRoot = "/etc/agentgateway/" + egressTestRootFile + if root == testRoot { + unstructured.RemoveNestedField(backend, path...) + cleaned, err := yaml.Marshal(config) + if err != nil { + return nil, nil, fmt.Errorf("encode restored gateway config: %w", err) + } + restored["config.yaml"] = string(cleaned) + } + if err := unstructured.SetNestedField(backend, testRoot, path...); err != nil { + return nil, nil, fmt.Errorf("set gateway TLS root: %w", err) + } + updated, err := yaml.Marshal(config) + if err != nil { + return nil, nil, fmt.Errorf("encode gateway config: %w", err) + } + patched = maps.Clone(restored) + patched["config.yaml"] = string(updated) + patched[egressTestRootFile] = certificate + return patched, restored, nil +} + +func TestEgressGatewayTrustConfig(t *testing.T) { + const gateway = `frontendPolicies: + substrateEgressActorResolution: + policies: + backendTLS: {root: /identity-ca.pem} +binds: +- mode: internal + listeners: + - protocol: HTTP + routes: + - backends: + - dynamic: {} + - protocol: HTTPS + tls: {mode: dynamicCa, cert: /mitm.pem} + routes: + - policies: + substrateEgress: + policies: + backendTLS: {root: /control-plane-ca.pem} + backends: + - dynamic: {} + policies: + backendTLS: %s +` + for _, tc := range []struct { + name, tls, restoredTLS string + }{ + {"clean", "{}", "{}"}, + {"interrupted", "{root: /etc/agentgateway/egress-test-root.pem}", "{}"}, + {"existing trust", "{root: /custom-ca.pem, hostname: example.com}", "{root: /custom-ca.pem, hostname: example.com}"}, + {"block format", "\n root: /etc/agentgateway/egress-test-root.pem", "{}"}, + } { + t.Run(tc.name, func(t *testing.T) { + input := map[string]string{"config.yaml": fmt.Sprintf(gateway, tc.tls), egressTestRootFile: "leftover CA", "other.pem": "keep"} + before := maps.Clone(input) + patched, restored, err := egressGatewayTrustConfig(input, "new CA") + require.NoError(t, err) + require.Equal(t, before, input, "patch must not mutate the saved config") + require.Equal(t, "new CA", patched[egressTestRootFile]) + require.Equal(t, "keep", patched["other.pem"]) + require.NotContains(t, restored, egressTestRootFile) + var want, got map[string]any + require.NoError(t, yaml.Unmarshal([]byte(fmt.Sprintf(gateway, tc.restoredTLS)), &want)) + require.NoError(t, yaml.Unmarshal([]byte(restored["config.yaml"]), &got)) + require.Equal(t, want, got, "cleanup must preserve unrelated TLS policy") + repeated, _, err := egressGatewayTrustConfig(patched, "replacement CA") + require.NoError(t, err) + require.Equal(t, patched["config.yaml"], repeated["config.yaml"], "reapplying trust must be idempotent") + require.Equal(t, "replacement CA", repeated[egressTestRootFile]) + require.Contains(t, patched["config.yaml"], "root: /etc/agentgateway/egress-test-root.pem") + require.Contains(t, patched["config.yaml"], "root: /identity-ca.pem") + require.Contains(t, patched["config.yaml"], "root: /control-plane-ca.pem") + }) + } + for _, input := range []string{"[invalid", "binds: []"} { + _, _, err := egressGatewayTrustConfig(map[string]string{"config.yaml": input}, "CA") + require.Error(t, err) + } } diff --git a/go/core/test/e2e/interaction_test.go b/go/core/test/e2e/interaction_test.go index b8bb59f81e..e0169dd08d 100644 --- a/go/core/test/e2e/interaction_test.go +++ b/go/core/test/e2e/interaction_test.go @@ -37,7 +37,6 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -1243,17 +1242,7 @@ func interactionKubeClient(t *testing.T) ctrlclient.Client { if err != nil { t.Fatalf("load Kubernetes config: %v", err) } - clientScheme := k8sruntime.NewScheme() - if err := corev1.AddToScheme(clientScheme); err != nil { - t.Fatalf("register Kubernetes core API: %v", err) - } - if err := discoveryv1.AddToScheme(clientScheme); err != nil { - t.Fatalf("register discovery API: %v", err) - } - if err := v1alpha3.AddToScheme(clientScheme); err != nil { - t.Fatalf("register kagent API: %v", err) - } - kube, err := ctrlclient.New(cfg, ctrlclient.Options{Scheme: clientScheme}) + kube, err := ctrlclient.New(cfg, ctrlclient.Options{Scheme: suiteKubeScheme}) if err != nil { t.Fatalf("create Kubernetes client: %v", err) } diff --git a/go/core/test/e2e/main_test.go b/go/core/test/e2e/main_test.go index 252d6bfa74..945816a0a0 100644 --- a/go/core/test/e2e/main_test.go +++ b/go/core/test/e2e/main_test.go @@ -8,16 +8,28 @@ import ( "os" "testing" + appsv1 "k8s.io/api/apps/v1" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" + "github.com/kagent-dev/kagent/go/api/v1alpha3" kagentenv "github.com/kagent-dev/kagent/go/core/pkg/env" "github.com/kagent-dev/kagent/go/pkg/logging" ) var suiteTraceReceiver *otlpTraceReceiver +var suiteKubeScheme = runtime.NewScheme() func TestMain(m *testing.M) { ctrl.SetLogger(logging.AsLogr(slog.Default())) + schemeBuilder := runtime.SchemeBuilder{corev1.AddToScheme, discoveryv1.AddToScheme, appsv1.AddToScheme, coordinationv1.AddToScheme, v1alpha3.AddToScheme} + if err := schemeBuilder.AddToScheme(suiteKubeScheme); err != nil { + fmt.Fprintf(os.Stderr, "register Kubernetes APIs: %v\n", err) + os.Exit(1) + } address := kagentenv.E2EOTLPListenAddress.Get() if address == "" { os.Exit(m.Run()) diff --git a/go/core/test/e2e/session_expiration_test.go b/go/core/test/e2e/session_expiration_test.go index a045e2b468..9ab841c561 100644 --- a/go/core/test/e2e/session_expiration_test.go +++ b/go/core/test/e2e/session_expiration_test.go @@ -92,7 +92,6 @@ func TestSessionIdleExpiration(t *testing.T) { func setSessionExpirationPolicy(t *testing.T, target, ttl, pollInterval string) { t.Helper() kube := interactionKubeClient(t) - require.NoError(t, appsv1.AddToScheme(kube.Scheme())) deployments := &appsv1.DeploymentList{} require.NoError(t, kube.List(t.Context(), deployments, ctrlclient.InNamespace("kagent"), ctrlclient.MatchingLabels{"app.kubernetes.io/component": "controller"})) require.Len(t, deployments.Items, 1) From 36c5ef6c7eef6baf127514898ce032cd43ee5c6d Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 16:36:48 +0000 Subject: [PATCH 17/43] fix: skip unchanged egress updates and separate revision promotion Signed-off-by: Eitan Yarmush --- docs/architecture/egress-policies.md | 8 ++- .../internal/controller/collections_test.go | 3 +- go/core/internal/controller/egress.go | 46 ++++++------ go/core/internal/controller/egress_test.go | 71 ++++++++++++++++--- .../internal/controller/preparation_test.go | 32 ++++++--- go/core/internal/controller/reconciler.go | 7 +- .../internal/controller/reconciler_test.go | 25 +++++-- .../controller/sandbox_integration_test.go | 2 +- .../internal/controller/sandbox_reconciler.go | 7 +- .../controller/sandbox_reconciler_test.go | 26 ++++++- .../internal/database/client_session_test.go | 9 ++- go/core/internal/database/protobuf_test.go | 4 +- go/core/internal/database/runtime_egress.go | 30 ++++---- .../internal/database/runtime_egress_test.go | 41 +++++++++-- .../database/runtime_lifecycle_test.go | 4 +- .../internal/database/runtime_revisions.go | 53 +++++++------- .../database/runtime_revisions_test.go | 49 +++++++------ .../internal/database/sandbox_revisions.go | 37 +++++++--- go/core/internal/database/sandboxes_test.go | 44 +++++++++++- .../internal/grpcserver/scheduledrun_test.go | 3 +- .../internal/grpcserver/task_store_test.go | 3 +- .../internal/service/sandbox/guest_test.go | 3 +- .../internal/service/sandbox/service_test.go | 6 +- .../internal/service/session/workflow_test.go | 3 +- 24 files changed, 364 insertions(+), 152 deletions(-) diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index 1931e03b2a..ca6d0efee3 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -163,13 +163,17 @@ The elected controller compiles current grants from cached, dependency-tracked Kubernetes inputs and persists the intended policy for each retained revision. This mutable output is stored separately from immutable revision inputs. Initial preparation publishes the policy before promoting a new revision to ready. +The grants cache can briefly lag initial compilation, so a new workload may +become ready with only pinned integration access until its grants event arrives. Each revision merges its pinned integration access with the current grants of its owning Agent or SandboxTemplate, even when that source's next runtime revision cannot be prepared. Policy and Secret events update only revisions of the affected source and queue -their existing Actors. A five-minute sweep repairs missed events and controller -restarts. Actor updates retry with exponential backoff independently of successful +their existing Actors when the intended policy changes. A five-minute sweep +repairs missed translations without rewriting unchanged policies or checking +their Actors. Controller startup checks all Actors to recover interrupted updates. +Actor updates retry with exponential backoff independently of successful updates. Running and suspended Actors receive updates without replacement or a new invocation. Missing or invalid policies and Secrets revoke explicit grants; pinned integration access remains available. Grants that conflict with pinned diff --git a/go/core/internal/controller/collections_test.go b/go/core/internal/controller/collections_test.go index 09c74fd5a6..46b7030f24 100644 --- a/go/core/internal/controller/collections_test.go +++ b/go/core/internal/controller/collections_test.go @@ -1,7 +1,6 @@ package controller import ( - "context" "strings" "testing" "time" @@ -103,7 +102,7 @@ func TestReconciliationCollectionsCompileAndObserveRevision(t *testing.T) { observed.Metadata.Uid = "actor-template-uid" observed.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}}} store := &fakeRuntimeRevisionStore{} - reconciler := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), + reconciler := &Reconciler{egress: noopRevisionEgress, collections: collections, templates: &fakeActorTemplates{template: observed}, store: store, } require.NoError(t, reconciler.reconcileAgent(t.Context(), state.ResourceName())) diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go index e12788b084..92cdcf3abd 100644 --- a/go/core/internal/controller/egress.go +++ b/go/core/internal/controller/egress.go @@ -25,7 +25,7 @@ import ( type runtimeEgressStore interface { GetRuntimeEgress(context.Context, string) (*database.RuntimeEgress, error) - SetRuntimeEgressPolicy(context.Context, string, egress.Access) error + SetRuntimeEgressPolicy(context.Context, string, egress.Access) (bool, error) ListRuntimeEgressRevisions(context.Context, string, int) ([]string, error) ListRuntimeEgressSourceRevisions(context.Context, database.RuntimeEgressSource, string, int) ([]string, error) ListRuntimeEgressTargets(context.Context, string, string, int) ([]database.RuntimeEgressTarget, error) @@ -110,6 +110,11 @@ type egressActorPage struct { Recovery bool } +type egressSweepPage struct { + After string + RecoverActors bool +} + var _ manager.LeaderElectionRunnable = (*EgressReconciler)(nil) func NewEgressReconciler(ctx context.Context, runtime *Runtime, store runtimeEgressStore, actors actorEgressClient, namespaces []string) *EgressReconciler { @@ -162,24 +167,23 @@ func NewEgressReconciler(ctx context.Context, runtime *Runtime, store runtimeEgr return nil }) e.sweeps = newReconciliationQueue("live-egress-sweeps", func(item any) error { - revisions, err := e.store.ListRuntimeEgressRevisions(ctx, item.(string), pageSize) + page := item.(egressSweepPage) + revisions, err := e.store.ListRuntimeEgressRevisions(ctx, page.After, pageSize) if err != nil { return err } for _, revision := range revisions { - watched, err := e.publishRevision(ctx, revision) + err := e.publishRevision(ctx, revision, page.RecoverActors) if errors.Is(err, database.ErrNotFound) { continue } if err != nil { return err } - if watched { - e.pages.Add(egressActorPage{Revision: revision, Recovery: true}) - } } if len(revisions) == pageSize { - e.sweeps.Add(revisions[len(revisions)-1]) + page.After = revisions[len(revisions)-1] + e.sweeps.Add(page) } return nil }) @@ -192,25 +196,18 @@ func (e *EgressReconciler) ReconcileRevision(ctx context.Context, revision strin if !e.grants.WaitUntilSynced(ctx.Done()) { return ctx.Err() } - watched, err := e.publishRevision(ctx, revision) - if err != nil { - return err - } - if watched { - e.pages.Add(egressActorPage{Revision: revision}) - } - return nil + return e.publishRevision(ctx, revision, false) } -func (e *EgressReconciler) publishRevision(ctx context.Context, revision string) (bool, error) { +func (e *EgressReconciler) publishRevision(ctx context.Context, revision string, recoverActors bool) error { e.publishMu.Lock() defer e.publishMu.Unlock() pinned, err := e.store.GetRuntimeEgress(ctx, revision) if err != nil { - return false, err + return err } if len(e.namespaces) > 0 && !slices.Contains(e.namespaces, pinned.Namespace) { - return false, nil + return nil } base := egress.Access{Destinations: pinned.EgressDestinations, Credentials: pinned.Credentials} var grants egress.Access @@ -229,10 +226,13 @@ func (e *EgressReconciler) publishRevision(ctx context.Context, revision string) } else { e.warnings.Remove(revision) } - if err := e.store.SetRuntimeEgressPolicy(ctx, revision, access); err != nil { - return false, err + changed, err := e.store.SetRuntimeEgressPolicy(ctx, revision, access) + if changed || recoverActors || err != nil { + // A failed response may hide a committed write. Queue actor application + // even then: a publication retry could observe an unchanged policy. + e.pages.Add(egressActorPage{Revision: revision, Recovery: recoverActors}) } - return true, nil + return err } func (e *EgressReconciler) NeedLeaderElection() bool { return true } @@ -256,7 +256,7 @@ func (e *EgressReconciler) Start(ctx context.Context) error { workers.Go(func() { queue.Run(ctx.Done()) }) } defer workers.Wait() - e.sweeps.Add("") + e.sweeps.Add(egressSweepPage{RecoverActors: true}) ticker := time.NewTicker(5 * time.Minute) defer ticker.Stop() for { @@ -264,7 +264,7 @@ func (e *EgressReconciler) Start(ctx context.Context) error { case <-ctx.Done(): return nil case <-ticker.C: - e.sweeps.Add("") + e.sweeps.Add(egressSweepPage{}) } } } diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go index 2c59955a6e..b03e5614f7 100644 --- a/go/core/internal/controller/egress_test.go +++ b/go/core/internal/controller/egress_test.go @@ -11,10 +11,12 @@ import ( "strings" "sync" "testing" + "testing/synctest" "time" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/stretchr/testify/require" + "istio.io/istio/pkg/kube/controllers" "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,17 +32,20 @@ import ( type revisionEgressFunc func(context.Context, string) error +var noopRevisionEgress = revisionEgressFunc(func(context.Context, string) error { return nil }) + func (f revisionEgressFunc) ReconcileRevision(ctx context.Context, revision string) error { return f(ctx, revision) } type egressTestStore struct { - mu sync.Mutex - pinned map[string]database.RuntimeEgress - intended map[string]egress.Access - targets map[string][]database.RuntimeEgressTarget - listed chan database.RuntimeEgressSource - publishErr error + mu sync.Mutex + pinned map[string]database.RuntimeEgress + intended map[string]egress.Access + targets map[string][]database.RuntimeEgressTarget + listed chan database.RuntimeEgressSource + publishErr error + losePublishResponse bool } var _ runtimeEgressStore = (*egressTestStore)(nil) @@ -54,17 +59,24 @@ func (e *egressTestStore) GetRuntimeEgress(_ context.Context, revision string) ( } return &row, nil } -func (e *egressTestStore) SetRuntimeEgressPolicy(_ context.Context, revision string, access egress.Access) error { +func (e *egressTestStore) SetRuntimeEgressPolicy(_ context.Context, revision string, access egress.Access) (bool, error) { e.mu.Lock() defer e.mu.Unlock() if e.publishErr != nil { - return e.publishErr + return false, e.publishErr } if e.intended == nil { e.intended = map[string]egress.Access{} } + if current, exists := e.intended[revision]; exists && reflect.DeepEqual(current, access) { + return false, nil + } e.intended[revision] = access - return nil + if e.losePublishResponse { + e.losePublishResponse = false + return false, errors.New("lost publication response") + } + return true, nil } func (e *egressTestStore) GetRuntimeEgressPolicy(_ context.Context, revision string) (egress.Access, error) { e.mu.Lock() @@ -274,6 +286,47 @@ func TestEgressUpdatesRetainedRevisionsAndRecoversAfterRestart(t *testing.T) { } } +func TestEgressSkipsUnchangedPoliciesAndRecoversLostWriteResponse(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + inputs := newEgressTestInputs(ctx) + inputs.source(database.RuntimeEgressAgent, "uid", []corev1.LocalObjectReference{{Name: "access"}}) + inputs.policies.UpdateObject(egressTestPolicy("https://tickets.example.com", nil)) + store := &egressTestStore{ + pinned: map[string]database.RuntimeEgress{"revision": {Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"}}, + targets: map[string][]database.RuntimeEgressTarget{"revision": {{ID: "actor", Revision: "revision", Atespace: "team", TemplateName: "runtime"}}}, + losePublishResponse: true, + } + actors := &observingEgressActors{store: store} + controller := NewEgressReconciler(ctx, inputs.runtime, store, actors, nil) + defer controller.sources.ShutDownEarly() + var workers sync.WaitGroup + for _, queue := range []controllers.Queue{controller.pages, controller.updates, controller.recovery, controller.sweeps} { + workers.Go(func() { queue.Run(ctx.Done()) }) + } + defer func() { cancel(); workers.Wait() }() + require.ErrorContains(t, controller.ReconcileRevision(ctx, "revision"), "lost publication response") + synctest.Wait() + require.Len(t, actors.attempts["session-actor"], 1, "an uncertain write must still apply the persisted policy") + require.Contains(t, actors.applied["session-actor"].Destinations, "https://tickets.example.com:443") + for range 3 { + require.NoError(t, controller.ReconcileRevision(ctx, "revision")) + controller.sweeps.Add(egressSweepPage{}) + synctest.Wait() + } + require.Len(t, actors.attempts["session-actor"], 1, "unchanged preparation and sweeps must not touch actors") + inputs.policies.DeleteObject("team/access") + synctest.Wait() + controller.sweeps.Add(egressSweepPage{}) + synctest.Wait() + require.Len(t, actors.attempts["session-actor"], 2, "a sweep must repair a missed policy event") + require.Empty(t, actors.applied["session-actor"].Destinations) + controller.sweeps.Add(egressSweepPage{RecoverActors: true}) + synctest.Wait() + require.Len(t, actors.attempts["session-actor"], 3, "startup must also check actors with unchanged policies") + }) +} + func TestEgressTargetsSourceChangesAndBacksOffActorFailures(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index e9611c4ff1..cedadd56cf 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -54,7 +54,7 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { templates.template.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}, }} - reconciler := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: collections, templates: templates, store: store} + reconciler := &Reconciler{egress: noopRevisionEgress, collections: collections, templates: templates, store: store} require.NoError(t, reconciler.reconcileAgent(ctx, initial.ResourceName())) updatedHarness := harnesses.List()[0].DeepCopy() @@ -97,7 +97,8 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { // A delayed success for the abandoned preparation must not replace A. abandoned, err := store.GetRuntimeRevision(ctx, preparing.Target.RevisionID.String()) require.NoError(t, err) - require.NoError(t, store.RecordRuntimeRevision(ctx, *abandoned, true)) + require.NoError(t, store.RecordRuntimeRevision(ctx, *abandoned)) + require.NoError(t, store.PromoteRuntimeRevision(ctx, abandoned.Revision)) unreferenced, err = store.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) require.Len(t, unreferenced, 1) @@ -155,7 +156,7 @@ func TestPreparationErrorsPublishStatusAndRecover(t *testing.T) { collections, _ := newPreparationTestCollections(t, "microvm") initial := collections.Reconciliations.List()[0] statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() - reconciler := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: collections, templates: &test.templates, store: &test.store, status: statusClient} + reconciler := &Reconciler{egress: noopRevisionEgress, collections: collections, templates: &test.templates, store: &test.store, status: statusClient} err := reconciler.reconcileAgent(t.Context(), initial.ResourceName()) require.Error(t, err) require.Equal(t, test.code, status.Code(err)) @@ -236,7 +237,7 @@ func TestPreparationRevisionIsolation(t *testing.T) { collections, harnesses := newPreparationTestCollections(t, "microvm") initial := collections.Reconciliations.List()[0] templates := &fakeActorTemplates{createErr: status.Error(codes.FailedPrecondition, `SandboxConfig "microvm" not found`)} - reconciler := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: collections, templates: templates, store: &fakeRuntimeRevisionStore{}} + reconciler := &Reconciler{egress: noopRevisionEgress, collections: collections, templates: templates, store: &fakeRuntimeRevisionStore{}} require.Error(t, reconciler.reconcileAgent(t.Context(), initial.ResourceName())) waitFor(t, func() bool { return collections.Reconciliations.GetKey(initial.ResourceName()).PreparationFailure != nil @@ -278,7 +279,7 @@ func TestPreparationQueueAndPollerBackoff(t *testing.T) { initial := collections.Reconciliations.List()[0] statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() backend := &preparationTestBackend{templates: &test.templates, store: &test.store} - r := newReconciler(collections, backend, backend, statusClient, revisionEgressFunc(func(context.Context, string) error { return nil })) + r := newReconciler(collections, backend, backend, statusClient, noopRevisionEgress) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -387,10 +388,16 @@ func (p *preparationTestBackend) UpsertAgentDefinition(ctx context.Context, defi return p.store.UpsertAgentDefinition(ctx, definition) } -func (p *preparationTestBackend) RecordRuntimeRevision(ctx context.Context, revision database.RuntimeRevision, ready bool) error { +func (p *preparationTestBackend) RecordRuntimeRevision(ctx context.Context, revision database.RuntimeRevision) error { p.mu.Lock() defer p.mu.Unlock() - return p.store.RecordRuntimeRevision(ctx, revision, ready) + return p.store.RecordRuntimeRevision(ctx, revision) +} + +func (p *preparationTestBackend) PromoteRuntimeRevision(ctx context.Context, revision string) error { + p.mu.Lock() + defer p.mu.Unlock() + return p.store.PromoteRuntimeRevision(ctx, revision) } func (p *preparationTestBackend) RetireAgentIdentities(ctx context.Context, namespace, name string, except *database.AgentDefinition) error { @@ -405,7 +412,7 @@ func TestPreparationDesiredChangesBypassBackoff(t *testing.T) { initial := collections.Reconciliations.List()[0] templates := &fakeActorTemplates{createErr: status.Error(codes.FailedPrecondition, "missing config")} store := &fakeRuntimeRevisionStore{} - r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3(), revisionEgressFunc(func(context.Context, string) error { return nil })) + r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3(), noopRevisionEgress) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -459,7 +466,7 @@ func TestPreparationTerminalFailures(t *testing.T) { } store := &fakeRuntimeRevisionStore{} statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() - r := newReconciler(collections, templates, store, statusClient, revisionEgressFunc(func(context.Context, string) error { return nil })) + r := newReconciler(collections, templates, store, statusClient, noopRevisionEgress) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -518,7 +525,7 @@ func TestInvalidActorTemplateHasNoCompiledTarget(t *testing.T) { require.Nil(t, state.PreparationFailure) store := &fakeRuntimeRevisionStore{} templates := &fakeActorTemplates{getErr: errors.New("Substrate must not be called for an invalid target")} - r := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: collections, templates: templates, store: store} + r := &Reconciler{egress: noopRevisionEgress, collections: collections, templates: templates, store: store} require.NoError(t, r.reconcileAgent(t.Context(), initial.ResourceName())) require.Empty(t, store.pair.DesiredRevision) require.Nil(t, templates.template) @@ -595,6 +602,11 @@ func TestInitialRevisionWaitsForEgressPublication(t *testing.T) { return collections.Reconciliations.GetKey(initial.ResourceName()).PreparationFailure != nil }) publishErr = nil + store.promoteErr = errors.New("promotion unavailable") + require.ErrorIs(t, r.reconcileAgent(t.Context(), initial.ResourceName()), store.promoteErr) + require.False(t, store.markedSuccessful) + store.promoteErr = nil require.NoError(t, r.reconcileAgent(t.Context(), initial.ResourceName())) require.True(t, store.markedSuccessful) + require.Equal(t, 3, store.recordCalls, "each attempt must record inputs only once") } diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index a9f159a8b6..3ffa625518 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -166,7 +166,8 @@ func newAgentReconciliations( // or a Session or checkpoint references them. type runtimeRevisionStore interface { UpsertAgentDefinition(context.Context, database.AgentDefinition) error - RecordRuntimeRevision(context.Context, database.RuntimeRevision, bool) error + RecordRuntimeRevision(context.Context, database.RuntimeRevision) error + PromoteRuntimeRevision(context.Context, string) error RetireAgentIdentities(ctx context.Context, namespace, name string, except *database.AgentDefinition) error } @@ -377,14 +378,14 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { ActorTemplateAtespace: observed.GetMetadata().GetAtespace(), ActorTemplateName: observed.GetMetadata().GetName(), ActorTemplateUID: observed.GetMetadata().GetUid(), } ready := observed.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() != nil - if err := r.store.RecordRuntimeRevision(ctx, revision, false); err != nil { + if err := r.store.RecordRuntimeRevision(ctx, revision); err != nil { return r.observePreparationError(*state, fmt.Errorf("store runtime revision %s: %w", target.RevisionID, err)) } if err := r.egress.ReconcileRevision(ctx, revision.Revision); err != nil { return r.observePreparationError(*state, fmt.Errorf("publish runtime egress %s: %w", target.RevisionID, err)) } if ready { - if err := r.store.RecordRuntimeRevision(ctx, revision, true); err != nil { + if err := r.store.PromoteRuntimeRevision(ctx, revision.Revision); err != nil { return r.observePreparationError(*state, fmt.Errorf("promote runtime revision %s: %w", target.RevisionID, err)) } } diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index e6ddfdb851..5b49417dc7 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -55,7 +55,7 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { store := &fakeRuntimeRevisionStore{} templates := &fakeActorTemplates{} statusClient := kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3() - reconciler := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), + reconciler := &Reconciler{egress: noopRevisionEgress, collections: Collections{ Agents: krttest.GetMockCollection[*kagentv1alpha3.Agent](mock), AgentRuntimeObservations: krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...), @@ -109,7 +109,7 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { } // A fresh reconciler must clean up observations without remembering earlier calls. - reconciler = &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), collections: reconciler.collections, templates: templates, store: store, status: statusClient} + reconciler = &Reconciler{egress: noopRevisionEgress, collections: reconciler.collections, templates: templates, store: store, status: statusClient} nextTarget := *state.Target state.Target = &nextTarget state.Target.ActorTemplate = proto.CloneOf(state.Target.ActorTemplate) @@ -157,7 +157,7 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) states := krt.NewStaticCollection[AgentReconciliation](nil, nil, opts.WithName("Reconciliations")...) templates := &fakeActorTemplates{} - reconciler := &Reconciler{egress: revisionEgressFunc(func(context.Context, string) error { return nil }), + reconciler := &Reconciler{egress: noopRevisionEgress, collections: Collections{ AgentRuntimeObservations: krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...), Reconciliations: states, @@ -324,6 +324,8 @@ type fakeRuntimeRevisionStore struct { markedSuccessful bool retired string revisionErr error + promoteErr error + recordCalls int pairErr error pairCalls int } @@ -334,12 +336,23 @@ func (s *fakeRuntimeRevisionStore) UpsertAgentDefinition(_ context.Context, pair return s.pairErr } -func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision, ready bool) error { +func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision) error { + s.recordCalls++ if s.revisionErr != nil { return s.revisionErr } s.revision = &revision - s.markedSuccessful = ready + return nil +} + +func (s *fakeRuntimeRevisionStore) PromoteRuntimeRevision(_ context.Context, revision string) error { + if s.promoteErr != nil { + return s.promoteErr + } + if s.revision == nil || s.revision.Revision != revision { + return database.ErrNotFound + } + s.markedSuccessful = true return nil } @@ -391,7 +404,7 @@ func TestReconcilerUpdatesModelConfigStatusOnSecretHashChange(t *testing.T) { &fakeActorTemplates{}, &fakeRuntimeRevisionStore{}, statusClient, - revisionEgressFunc(func(context.Context, string) error { return nil }), + noopRevisionEgress, ) go reconciler.Run(stop) diff --git a/go/core/internal/controller/sandbox_integration_test.go b/go/core/internal/controller/sandbox_integration_test.go index aa3d5f8638..b4e9a6bed1 100644 --- a/go/core/internal/controller/sandbox_integration_test.go +++ b/go/core/internal/controller/sandbox_integration_test.go @@ -65,7 +65,7 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { Secrets: typedCollection[*corev1.Secret](runtimeClient, []string{"team-a"}, "Secrets", options), SandboxTemplates: typedCollection[*kagentv1alpha3.SandboxTemplate](runtimeClient, []string{"team-a"}, "SandboxTemplates", options), WorkerPools: pools, }} - reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, revisionEgressFunc(func(context.Context, string) error { return nil })) + reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, noopRevisionEgress) require.NoError(t, err) results := make(chan error, 2) go func() { results <- runtime.Start(ctx) }() diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index 4fc6a061dc..91f29f18e7 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -33,7 +33,8 @@ const ( type sandboxRevisionStore interface { UpsertSandboxTemplateDefinition(context.Context, database.SandboxTemplateDefinition) error RetireSandboxTemplateIdentities(context.Context, string, string) error - RecordSandboxRevision(context.Context, database.SandboxRevision, bool) error + RecordSandboxRevision(context.Context, database.SandboxRevision) error + PromoteSandboxRevision(context.Context, string) error } // SandboxReconciler owns the queued side effects of SandboxTemplate preparation. @@ -176,14 +177,14 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), SourceSnapshot: state.SourceSnapshot, } - if err := s.store.RecordSandboxRevision(ctx, revision, false); err != nil { + if err := s.store.RecordSandboxRevision(ctx, revision); err != nil { return s.observeError(*state, fmt.Errorf("store sandbox revision %s: %w", state.RevisionID, err), true) } if err := s.egress.ReconcileRevision(ctx, revision.Revision); err != nil { return s.observeError(*state, fmt.Errorf("publish sandbox egress %s: %w", state.RevisionID, err), true) } if ready { - if err := s.store.RecordSandboxRevision(ctx, revision, true); err != nil { + if err := s.store.PromoteSandboxRevision(ctx, revision.Revision); err != nil { return s.observeError(*state, fmt.Errorf("promote sandbox revision %s: %w", state.RevisionID, err), true) } } diff --git a/go/core/internal/controller/sandbox_reconciler_test.go b/go/core/internal/controller/sandbox_reconciler_test.go index b2f421cb8b..ab239ef9ba 100644 --- a/go/core/internal/controller/sandbox_reconciler_test.go +++ b/go/core/internal/controller/sandbox_reconciler_test.go @@ -38,6 +38,8 @@ type sandboxTestStore struct { retireAttempts int retireErr error recordErr error + promoteErr error + recordCalls int } func (s *sandboxTestStore) UpsertSandboxTemplateDefinition(_ context.Context, value database.SandboxTemplateDefinition) error { @@ -46,13 +48,26 @@ func (s *sandboxTestStore) UpsertSandboxTemplateDefinition(_ context.Context, va s.desired = value return nil } -func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision, ready bool) error { +func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision) error { s.mu.Lock() defer s.mu.Unlock() + s.recordCalls++ if s.recordErr != nil { return s.recordErr } - s.revision, s.ready = value, ready + s.revision = value + return nil +} +func (s *sandboxTestStore) PromoteSandboxRevision(_ context.Context, revision string) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.promoteErr != nil { + return s.promoteErr + } + if s.revision.Revision != revision { + return database.ErrNotFound + } + s.ready = true return nil } func (s *sandboxTestStore) RetireSandboxTemplateIdentities(context.Context, string, string) error { @@ -126,7 +141,7 @@ func newSandboxTestReconciler(t *testing.T, guestImage string) (*SandboxReconcil actors := &sandboxTestActors{store: store, templates: map[string]*ateapipb.ActorTemplate{}} reconciler := &SandboxReconciler{ collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools}, substrate.SandboxPolicy{GuestImage: guestImage, CPU: "1", Memory: "1Gi"}, opts), - egress: revisionEgressFunc(func(context.Context, string) error { return nil }), + egress: noopRevisionEgress, store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3(), } waitFor(t, func() bool { return reconciler.collections.states.GetKey("team-a/scratch") != nil }) @@ -313,6 +328,11 @@ func TestInitialSandboxRevisionWaitsForEgressPublication(t *testing.T) { require.False(t, store.ready) waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch").Failure != nil }) publishErr = nil + store.promoteErr = errors.New("promotion unavailable") + require.ErrorIs(t, s.reconcile(t.Context(), "team-a/scratch"), store.promoteErr) + require.False(t, store.ready) + store.promoteErr = nil require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) require.True(t, store.ready) + require.Equal(t, 3, store.recordCalls, "each attempt must record inputs only once") } diff --git a/go/core/internal/database/client_session_test.go b/go/core/internal/database/client_session_test.go index 99a49bde77..d432d1cc0e 100644 --- a/go/core/internal/database/client_session_test.go +++ b/go/core/internal/database/client_session_test.go @@ -377,9 +377,10 @@ func TestForkSessionCopiesBoundedHistory(t *testing.T) { if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision, true); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision); err != nil { t.Fatal(err) } + require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) source, _, err := client.CreateSession(ctx, &apiv1.Session{ Id: sourceID, Creator: "alice", Name: "Namespace explanation", @@ -536,9 +537,10 @@ func TestSessionCreateAndTransitions(t *testing.T) { if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision, true); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision); err != nil { t.Fatal(err) } + require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) storedRevision, err := client.GetRuntimeRevision(ctx, revision.Revision) if err != nil { @@ -638,9 +640,10 @@ func sessionFixture(t *testing.T, client *Client, ctx context.Context, namespace if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision, true); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision); err != nil { t.Fatal(err) } + require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) } func newSessionRequest(id, template, harness, name string) *apiv1.Session { diff --git a/go/core/internal/database/protobuf_test.go b/go/core/internal/database/protobuf_test.go index 6e962a5771..86b4005309 100644 --- a/go/core/internal/database/protobuf_test.go +++ b/go/core/internal/database/protobuf_test.go @@ -286,11 +286,11 @@ func TestShareAndAgentCardProtobufPersistence(t *testing.T) { require.NoError(t, err) addUnknown(card) revision := RuntimeRevision{Revision: "revision", Namespace: "team-a", AgentName: "assistant", AgentUID: "template", SourceSnapshot: []byte(`{}`), AgentCard: card, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "template"} - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, false)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) // Reconciliation can update runtime identity, but the pinned card is immutable. revision.AgentCard = &a2apb.AgentCard{Name: "replacement"} revision.ActorTemplateUID = "new-uid" - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, false)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) stored, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) require.True(t, proto.Equal(card, stored.AgentCard)) diff --git a/go/core/internal/database/runtime_egress.go b/go/core/internal/database/runtime_egress.go index 2c238cdf3f..864c8b7c16 100644 --- a/go/core/internal/database/runtime_egress.go +++ b/go/core/internal/database/runtime_egress.go @@ -94,12 +94,13 @@ func (c *Client) ListRuntimeEgressSourceRevisions(ctx context.Context, source Ru } // SetRuntimeEgressPolicy publishes controller output for a retained revision. +// It writes and returns true only when the canonical policy changes. // It does not mutate the revision's pinned inputs or readiness. Missing/deleting // revisions return ErrNotFound; malformed policy input is rejected before writing. -func (c *Client) SetRuntimeEgressPolicy(ctx context.Context, revision string, access egress.Access) error { +func (c *Client) SetRuntimeEgressPolicy(ctx context.Context, revision string, access egress.Access) (bool, error) { canonical, err := egress.CanonicalAccess(access.Destinations, access.Credentials) if err != nil { - return fmt.Errorf("validate runtime egress policy %s: %w", revision, err) + return false, fmt.Errorf("validate runtime egress policy %s: %w", revision, err) } if canonical.Destinations == nil { canonical.Destinations = []string{} @@ -107,18 +108,23 @@ func (c *Client) SetRuntimeEgressPolicy(ctx context.Context, revision string, ac if canonical.Credentials == nil { canonical.Credentials = []egress.Credential{} } - result, err := c.db.Exec(ctx, ` - INSERT INTO runtime_egress_policy (revision, destinations, credentials) - SELECT revision, $2, $3 FROM runtime_revision WHERE revision = $1 AND deleted_at IS NULL - ON CONFLICT (revision) DO UPDATE SET destinations = EXCLUDED.destinations, credentials = EXCLUDED.credentials - `, revision, canonical.Destinations, canonical.Credentials) + changed, err := queryOne(ctx, c.db, ` + WITH retained AS ( + SELECT revision FROM runtime_revision WHERE revision = $1 AND deleted_at IS NULL + ), changed AS ( + INSERT INTO runtime_egress_policy (revision, destinations, credentials) + SELECT revision, $2, $3 FROM retained + ON CONFLICT (revision) DO UPDATE SET destinations = EXCLUDED.destinations, credentials = EXCLUDED.credentials + WHERE (runtime_egress_policy.destinations, runtime_egress_policy.credentials) + IS DISTINCT FROM (EXCLUDED.destinations, EXCLUDED.credentials) + RETURNING revision + ) + SELECT EXISTS (SELECT 1 FROM changed) FROM retained + `, pgx.RowTo[bool], revision, canonical.Destinations, canonical.Credentials) if err != nil { - return fmt.Errorf("store runtime egress policy %s: %w", revision, err) + return false, fmt.Errorf("store runtime egress policy %s: %w", revision, notFoundOr(err)) } - if result.RowsAffected() == 0 { - return ErrNotFound - } - return nil + return changed, nil } // GetRuntimeEgressPolicy reads the latest controller output, independent of any diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index e56630a372..2dd064ea43 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -16,7 +16,8 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { ctx := t.Context() revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", TokenPassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} require.NoError(t, client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", DesiredRevision: "agent"})) - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) + require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) instance, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "request") require.NoError(t, err) op, err := client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) @@ -106,19 +107,38 @@ func TestRuntimeEgressPolicyIsMutableControllerOutput(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() revision := RuntimeRevision{Revision: "policy-revision", Namespace: "team-a", AgentName: "assistant", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", EgressDestinations: []string{"https://model.example.com:443"}} - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, false)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) _, err := client.GetRuntimeEgressPolicy(ctx, revision.Revision) require.ErrorIs(t, err, ErrNotFound, "absence must not become an empty allowlist") allowed := egress.Access{Destinations: []string{"https://model.example.com:443", "https://tools.example.com:443"}, Credentials: []egress.Credential{{Origin: "https://tools.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/auth/token"}}} - require.NoError(t, client.SetRuntimeEgressPolicy(ctx, revision.Revision, allowed)) + changed, err := client.SetRuntimeEgressPolicy(ctx, revision.Revision, allowed) + require.NoError(t, err) + require.True(t, changed) + var before string + require.NoError(t, client.db.QueryRow(ctx, `SELECT xmin::text FROM runtime_egress_policy WHERE revision = $1`, revision.Revision).Scan(&before)) + equivalent := egress.Access{Destinations: []string{"https://tools.example.com", "https://model.example.com", "https://tools.example.com:443"}, Credentials: allowed.Credentials} + changed, err = client.SetRuntimeEgressPolicy(ctx, revision.Revision, equivalent) + require.NoError(t, err) + require.False(t, changed, "equivalent canonical policies must not trigger actor updates") + var after string + require.NoError(t, client.db.QueryRow(ctx, `SELECT xmin::text FROM runtime_egress_policy WHERE revision = $1`, revision.Revision).Scan(&after)) + require.Equal(t, before, after, "unchanged policies must not rewrite the row") // A different API replica reads the same intended policy without any K8s cache. replica := NewClient(client.db, "public") got, err := replica.GetRuntimeEgressPolicy(ctx, revision.Revision) require.NoError(t, err) require.Equal(t, allowed, got) - require.NoError(t, client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{Destinations: revision.EgressDestinations})) + rotated := egress.Access{Destinations: allowed.Destinations, Credentials: []egress.Credential{allowed.Credentials[0]}} + rotated.Credentials[0].URI = "ate-secret://k8s.io/default/team-a/rotated/token" + changed, err = client.SetRuntimeEgressPolicy(ctx, revision.Revision, rotated) + require.NoError(t, err) + require.True(t, changed, "a credential-only change must update actors") + changed, err = client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{Destinations: revision.EgressDestinations}) + require.NoError(t, err) + require.True(t, changed) // Preparation observations must not restore an earlier policy. - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) + require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) got, err = replica.GetRuntimeEgressPolicy(ctx, revision.Revision) require.NoError(t, err) require.Equal(t, revision.EgressDestinations, got.Destinations) @@ -127,11 +147,18 @@ func TestRuntimeEgressPolicyIsMutableControllerOutput(t *testing.T) { require.NoError(t, err) require.Equal(t, revision.EgressDestinations, pinned.EgressDestinations) require.Empty(t, pinned.Credentials) - require.ErrorIs(t, client.SetRuntimeEgressPolicy(ctx, "missing", allowed), ErrNotFound) - require.Error(t, client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{Destinations: []string{"invalid"}})) + changed, err = client.SetRuntimeEgressPolicy(ctx, "missing", allowed) + require.ErrorIs(t, err, ErrNotFound) + require.False(t, changed) + _, err = client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{Destinations: []string{"invalid"}}) + require.Error(t, err) // Durable malformed data must fail rather than granting a partial policy. _, err = client.db.Exec(ctx, `UPDATE runtime_egress_policy SET credentials = '[{"hostname":"legacy","header":"authorization"}]' WHERE revision = $1`, revision.Revision) require.NoError(t, err) _, err = replica.GetRuntimeEgressPolicy(ctx, revision.Revision) require.Error(t, err) + _, err = client.BeginRuntimeRevisionDeletion(ctx, revision.Revision) + require.NoError(t, err) + _, err = client.SetRuntimeEgressPolicy(ctx, revision.Revision, allowed) + require.ErrorIs(t, err, ErrNotFound, "deleting revisions must remain distinguishable from unchanged policies") } diff --git a/go/core/internal/database/runtime_lifecycle_test.go b/go/core/internal/database/runtime_lifecycle_test.go index d6dcddc980..81afb6ba2b 100644 --- a/go/core/internal/database/runtime_lifecycle_test.go +++ b/go/core/internal/database/runtime_lifecycle_test.go @@ -69,7 +69,9 @@ func TestAgentWritesRejectSandboxRevisions(t *testing.T) { revision, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) revision.Revision, revision.ActorTemplateUID = "sandbox-revision", "replacement-uid" - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision, true), ErrConflict) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision), ErrConflict) + require.ErrorIs(t, client.PromoteRuntimeRevision(ctx, "sandbox-revision"), ErrConflict) + require.ErrorIs(t, client.PromoteSandboxRevision(ctx, "revision"), ErrConflict) var uid string require.NoError(t, pool.QueryRow(ctx, "SELECT actor_template_uid FROM runtime_revision WHERE revision = 'sandbox-revision'").Scan(&uid)) require.Equal(t, "original-uid", uid) diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index dbd9245dab..49505b376b 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -67,11 +67,10 @@ func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefi }) } -// RecordRuntimeRevision stores a prepared revision and, when ready, atomically -// promotes it for an active definition that still desires it. Stale reports leave the -// definition unchanged. Existing revisions retain immutable inputs; only their actor -// UID and update time are refreshed. Deleting revisions return ErrObjectDeleting. -func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevision, ready bool) error { +// RecordRuntimeRevision stores revision inputs without making them available for +// new Sessions. Existing revisions retain immutable inputs; only their actor UID +// and update time are refreshed. Deleting revisions return ErrObjectDeleting. +func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevision) error { if revision.AgentCard == nil { return fmt.Errorf("runtime revision %s has no Agent Card", revision.Revision) } @@ -80,20 +79,6 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi return fmt.Errorf("encode runtime revision Agent Card: %w", err) } return c.withTx(ctx, func(tx pgx.Tx) error { - if ready { - // Match GC finalization's lock order: definition before revision. Use the - // stored identity when present, since revision inputs are immutable. - _, err := queryOne(ctx, tx, ` - SELECT 1 FROM agent_definition p - LEFT JOIN agent_runtime_revision r ON r.revision = $1 - WHERE p.namespace = COALESCE(r.namespace, $2) - AND p.agent_uid = COALESCE(r.agent_uid, $3) - FOR UPDATE OF p - `, pgx.RowTo[int], revision.Revision, revision.Namespace, revision.AgentUID) - if err != nil && !errors.Is(err, pgx.ErrNoRows) { - return err - } - } if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{ RuntimeArtifact: RuntimeArtifact{Revision: revision.Revision, Kind: runtimeKindAgent, Namespace: revision.Namespace, ActorTemplateAtespace: revision.ActorTemplateAtespace, ActorTemplateName: revision.ActorTemplateName, ActorTemplateUID: revision.ActorTemplateUID}, @@ -107,8 +92,27 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi `, revision.Revision, revision.AgentName, revision.AgentUID, card); err != nil { return err } - if !ready { - return nil + return nil + }) +} + +// PromoteRuntimeRevision makes a recorded, prepared revision available only to +// the active Agent identity that still desires it. Callers must publish its +// intended egress policy first. Missing/deleting revisions return ErrNotFound / +// ErrObjectDeleting; stale reports leave the definition unchanged. +func (c *Client) PromoteRuntimeRevision(ctx context.Context, revision string) error { + return c.withTx(ctx, func(tx pgx.Tx) error { + // Match GC finalization's lock order: definition before revision. + _, err := queryOne(ctx, tx, ` + SELECT 1 FROM agent_definition p + JOIN agent_runtime_revision r ON p.namespace = r.namespace AND p.agent_uid = r.agent_uid + WHERE r.revision = $1 FOR UPDATE OF p + `, pgx.RowTo[int], revision) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return err + } + if err := lockRuntimeRevisionForReference(ctx, tx, revision, runtimeKindAgent); err != nil { + return err } return execSQL(ctx, tx, ` UPDATE agent_definition p @@ -119,7 +123,8 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi AND p.agent_uid = r.agent_uid AND p.desired_revision = r.revision AND p.retired_at IS NULL - `, revision.Revision) + AND p.latest_successful_revision IS DISTINCT FROM r.revision + `, revision) }) } @@ -235,8 +240,8 @@ type runtimeRevisionRecord struct { } // recordRuntimeRevision writes immutable common inputs with mutable artifact -// status. The caller writes its typed extension and promotion pointer in this -// transaction, holding preparation locks before taking the revision lock. +// status. The caller writes its typed extension in the same transaction. +// Promotion is a separate transition after intended egress policy is published. func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevisionRecord) error { if revision.Credentials == nil { revision.Credentials = []egress.Credential{} diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index e45359b62a..9ec73c074f 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -151,7 +151,7 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { revision.Revision = "new" revision.AgentUID = "replacement-uid" revision.ActorTemplateName = "new-actor-template" - require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, false)) + require.NoError(t, client.RecordRuntimeRevision(ctx, *revision)) pair := AgentDefinition{ Namespace: "team-a", AgentName: "assistant", AgentUID: revision.AgentUID, DesiredRevision: "new", @@ -160,7 +160,7 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { request := newSessionRequest(uuid.NewString(), "assistant", "kagent", "") _, _, err = client.CreateSession(ctx, request, "session") require.ErrorIs(t, err, ErrNotFound, "a replacement must not select the previous template UID") - require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, true)) + require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) session, _, err := client.CreateSession(ctx, request, "session") require.NoError(t, err) require.Equal(t, "new", session.GetPreparedRevision()) @@ -341,8 +341,8 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) pair.AgentUID = "replacement-uid" pair.DesiredRevision = "revision" require.ErrorIs(t, client.UpsertAgentDefinition(ctx, pair), ErrObjectDeleting) - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, false), ErrObjectDeleting) - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, true), ErrObjectDeleting) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original), ErrObjectDeleting) + require.ErrorIs(t, client.PromoteRuntimeRevision(ctx, original.Revision), ErrObjectDeleting) // A new client rediscovers and retries the committed claim after a crash. restarted := NewClient(pool, "public") revisions, err := restarted.ListUnreferencedRuntimeRevisions(ctx) @@ -358,7 +358,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.ErrorIs(t, err, ErrNotFound) // The same digest can be prepared again once cleanup has completed. original.ActorTemplateUID = "recreated-actor-uid" - require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, false)) + require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original)) newClaim, err := restarted.BeginRuntimeRevisionDeletion(ctx, "revision") require.NoError(t, err) require.NotNil(t, newClaim) @@ -366,12 +366,12 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) _, err = restarted.GetRuntimeRevision(ctx, "revision") require.NoError(t, err, "a delayed collector must not finalize the new runtime") require.NoError(t, restarted.DeleteRuntimeRevision(ctx, "revision", "recreated-actor-uid")) - require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, false)) + require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original)) require.NoError(t, restarted.UpsertAgentDefinition(ctx, pair)) } func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { - for _, operation := range []string{"reactivate", "record"} { + for _, operation := range []string{"reactivate", "promote"} { for _, finalizeFirst := range []bool{false, true} { t.Run(fmt.Sprintf("%s/finalize_first=%t", operation, finalizeFirst), func(t *testing.T) { pool := setupTestDB(t) @@ -380,8 +380,6 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { defer cancel() sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) - original, err := client.GetRuntimeRevision(ctx, "revision") - require.NoError(t, err) claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "revision") require.NoError(t, err) require.NotNil(t, claimed) @@ -389,14 +387,14 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { barrier := &runtimeReferenceBarrier{ query: "FOR UPDATE OF r", reached: make(chan struct{}), resume: make(chan struct{}), } - if operation == "record" { - barrier.query = "INSERT INTO runtime_revision" + if operation == "promote" { + barrier.query = "FROM runtime_revision WHERE revision = $1 FOR UPDATE" } waitingQuery := "ORDER BY namespace, agent_uid" if finalizeFirst { barrier.query, barrier.afterQuery = "FROM runtime_revision WHERE revision = $1 FOR UPDATE", true waitingQuery = "INSERT INTO agent_definition" - if operation == "record" { + if operation == "promote" { waitingQuery = "FOR UPDATE OF p" } } @@ -417,8 +415,8 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { } created, deleted := make(chan error, 1), make(chan error, 1) create := func() { - if operation == "record" { - created <- creating.RecordRuntimeRevision(ctx, *original, true) + if operation == "promote" { + created <- creating.PromoteRuntimeRevision(ctx, "revision") return } created <- creating.UpsertAgentDefinition(ctx, pair) @@ -445,7 +443,9 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { return err == nil && waiting }, 5*time.Second, 10*time.Millisecond) resume.Do(func() { close(barrier.resume) }) - if finalizeFirst { + if finalizeFirst && operation == "promote" { + require.ErrorIs(t, <-created, ErrNotFound) + } else if finalizeFirst { require.NoError(t, <-created) } else { require.ErrorIs(t, <-created, ErrObjectDeleting) @@ -457,7 +457,7 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { } } -func TestRecordRuntimeRevisionPromotesOnlyCurrentActivePair(t *testing.T) { +func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { pool := setupTestDB(t) c := NewClient(pool, "public") ctx := t.Context() @@ -486,28 +486,31 @@ func TestRecordRuntimeRevisionPromotesOnlyCurrentActivePair(t *testing.T) { require.Equal(t, want, session.PreparedRevision) } require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, false)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) assertAvailableRevision("") stored, err := c.GetRuntimeRevision(ctx, revision.Revision) require.NoError(t, err) require.Equal(t, "original", stored.AgentCard.Name) - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, true)) + require.ErrorIs(t, c.PromoteRuntimeRevision(ctx, "missing"), ErrNotFound) + require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("first") pair.DesiredRevision = "second" require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) revision.Revision, revision.ActorTemplateName = "second", "second" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) + require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("second") // A delayed report for the old revision must not replace the newer ready revision. revision.Revision, revision.ActorTemplateName = "first", "first" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("second") pair.DesiredRevision = "third" require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) require.NoError(t, c.RetireAgentIdentities(ctx, pair.Namespace, pair.AgentName, nil)) revision.Revision, revision.ActorTemplateName = "third", "third" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) + require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("") // Reviving the pair must retain the last success, not a report made while retired. require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) @@ -520,12 +523,12 @@ func TestRecordRuntimeRevisionPromotesOnlyCurrentActivePair(t *testing.T) { func TestRuntimeRevisionPersistsCredentialBindings(t *testing.T) { client := NewClient(setupTestDB(t), "public") revision := RuntimeRevision{Revision: "credential-revision", Namespace: "team", AgentName: "agent", AgentUID: "agent", SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{"https://api.example.com:443"}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", Credentials: []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}}} - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, false)) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision)) got, err := client.GetRuntimeRevision(t.Context(), revision.Revision) require.NoError(t, err) require.Equal(t, revision.Credentials, got.Credentials) revision.Credentials[0].URI = "ate-secret://k8s.io/default/team/other/token" - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, false)) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision)) unchanged, err := client.GetRuntimeRevision(t.Context(), revision.Revision) require.NoError(t, err) require.Equal(t, got.Credentials, unchanged.Credentials, "revision credentials are immutable") diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index ef3a535275..961b796acf 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -69,15 +69,10 @@ func (c *Client) RetireSandboxTemplateIdentities(ctx context.Context, namespace, `, namespace, name) } -// RecordSandboxRevision promotes only the active identity still desiring this -// revision. A late preparation response cannot replace a newer successful input. -func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevision, ready bool) error { +// RecordSandboxRevision stores immutable sandbox inputs and refreshes artifact +// identity without promoting the revision. Deleting revisions return ErrObjectDeleting. +func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevision) error { return c.withTx(ctx, func(tx pgx.Tx) error { - if _, err := queryMany(ctx, tx, ` - SELECT 1 FROM sandbox_template_definition WHERE namespace = $1 AND sandbox_template_uid = $2 FOR UPDATE - `, pgx.RowTo[int], revision.Namespace, revision.SandboxTemplateUID); err != nil { - return err - } revision.Kind = runtimeKindSandbox if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{RuntimeArtifact: revision.RuntimeArtifact, SourceSnapshot: revision.SourceSnapshot}); err != nil { return err @@ -88,15 +83,35 @@ func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevi `, revision.Revision, revision.SandboxTemplateName, revision.SandboxTemplateUID); err != nil { return err } - if !ready { - return nil + return nil + }) +} + +// PromoteSandboxRevision makes a recorded, prepared revision available only to +// the active template identity that still desires it. Callers must publish its +// intended egress policy first. Missing/deleting revisions return ErrNotFound / +// ErrObjectDeleting; stale reports leave the definition unchanged. +func (c *Client) PromoteSandboxRevision(ctx context.Context, revision string) error { + return c.withTx(ctx, func(tx pgx.Tx) error { + // Match GC finalization's lock order: definition before revision. + if _, err := queryMany(ctx, tx, ` + SELECT 1 FROM sandbox_template_definition p + JOIN runtime_revision r ON p.namespace = r.namespace + JOIN sandbox_revision s ON r.revision = s.revision AND p.sandbox_template_uid = s.sandbox_template_uid + WHERE r.revision = $1 FOR UPDATE OF p + `, pgx.RowTo[int], revision); err != nil { + return err + } + if err := lockRuntimeRevisionForReference(ctx, tx, revision, runtimeKindSandbox); err != nil { + return err } return execSQL(ctx, tx, ` UPDATE sandbox_template_definition p SET latest_successful_revision = r.revision, updated_at = NOW() FROM runtime_revision r JOIN sandbox_revision s USING (revision) WHERE r.revision = $1 AND p.namespace = r.namespace AND p.sandbox_template_uid = s.sandbox_template_uid AND p.desired_revision = r.revision AND p.retired_at IS NULL - `, revision.Revision) + AND p.latest_successful_revision IS DISTINCT FROM r.revision + `, revision) }) } diff --git a/go/core/internal/database/sandboxes_test.go b/go/core/internal/database/sandboxes_test.go index 766118b448..083872bfe7 100644 --- a/go/core/internal/database/sandboxes_test.go +++ b/go/core/internal/database/sandboxes_test.go @@ -18,12 +18,54 @@ func sandboxFixture(t *testing.T, client *Client, revision string) (*apiv1.Sandb ctx := t.Context() require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision})) require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: revision, Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: revision, ActorTemplateUID: "actor-uid"}, - SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, true)) + SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)})) + require.NoError(t, client.PromoteSandboxRevision(ctx, revision)) hash := sha256.Sum256([]byte("request")) return &apiv1.Sandbox{Id: uuid.NewString(), Creator: "alice", SandboxTemplate: &apiv1.ResourceReference{Namespace: "team-a", Name: "scratch"}, Name: "experiment"}, SandboxCreateOptions{RequestHash: hash[:], TemplateUID: "uid", TTL: time.Hour} } +func TestPromoteSandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + request, options := sandboxFixture(t, client, "first") + definition := SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: "second"} + require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, definition)) + revision, err := client.GetSandboxRevision(ctx, "first") + require.NoError(t, err) + revision.Revision, revision.ActorTemplateName = "second", "second" + require.NoError(t, client.RecordSandboxRevision(ctx, *revision)) + assertAvailable := func(want string) { + t.Helper() + input := proto.CloneOf(request) + input.Id = uuid.NewString() + instance, _, err := client.CreateSandbox(ctx, input, uuid.NewString(), options) + if want == "" { + require.ErrorIs(t, err, ErrNotFound) + return + } + require.NoError(t, err) + require.Equal(t, want, instance.PreparedRevision) + } + assertAvailable("first") + require.ErrorIs(t, client.PromoteSandboxRevision(ctx, "missing"), ErrNotFound) + require.NoError(t, client.PromoteSandboxRevision(ctx, "second")) + assertAvailable("second") + require.NoError(t, client.PromoteSandboxRevision(ctx, "first")) + assertAvailable("second") + require.NoError(t, client.RetireSandboxTemplateIdentities(ctx, "team-a", "scratch")) + require.NoError(t, client.PromoteSandboxRevision(ctx, "first")) + assertAvailable("") + require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, definition)) + assertAvailable("second") + revision.Revision, revision.ActorTemplateName = "unused", "unused" + require.NoError(t, client.RecordSandboxRevision(ctx, *revision)) + claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "unused") + require.NoError(t, err) + require.NotNil(t, claimed) + require.ErrorIs(t, client.PromoteSandboxRevision(ctx, "unused"), ErrObjectDeleting) +} + func finishSandbox(t *testing.T, client *Client, id string, kind apiv1.RuntimeOperation) *apiv1.Sandbox { t.Helper() op, err := client.BeginSandboxOperation(t.Context(), id, kind) diff --git a/go/core/internal/grpcserver/scheduledrun_test.go b/go/core/internal/grpcserver/scheduledrun_test.go index 4fc98efd31..6838598e8c 100644 --- a/go/core/internal/grpcserver/scheduledrun_test.go +++ b/go/core/internal/grpcserver/scheduledrun_test.go @@ -185,7 +185,8 @@ func scheduledRunTestServer(t *testing.T) (*database.Client, apiv1.ScheduledRunS Revision: pair.DesiredRevision, Namespace: pair.Namespace, AgentName: pair.AgentName, AgentUID: pair.AgentUID, SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: "runtime-uid", - }, true)) + })) + require.NoError(t, store.PromoteRuntimeRevision(t.Context(), pair.DesiredRevision)) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) agent := &v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "report"}, Spec: v1alpha3.AgentSpec{ diff --git a/go/core/internal/grpcserver/task_store_test.go b/go/core/internal/grpcserver/task_store_test.go index 5f1367897e..50b1732a05 100644 --- a/go/core/internal/grpcserver/task_store_test.go +++ b/go/core/internal/grpcserver/task_store_test.go @@ -641,7 +641,8 @@ func createTaskStoreSession(t *testing.T, store *database.Client) *apiv1.Session Namespace: revision.Namespace, AgentName: revision.AgentName, AgentUID: revision.AgentUID, DesiredRevision: revision.Revision, })) - require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision, true)) + require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision)) + require.NoError(t, store.PromoteRuntimeRevision(t.Context(), revision.Revision)) session, _, err := store.CreateSession(t.Context(), &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", diff --git a/go/core/internal/service/sandbox/guest_test.go b/go/core/internal/service/sandbox/guest_test.go index 2d14eb1820..1bfd364431 100644 --- a/go/core/internal/service/sandbox/guest_test.go +++ b/go/core/internal/service/sandbox/guest_test.go @@ -133,7 +133,8 @@ func guestFixture(t *testing.T) (*sandboxservice.Service, context.Context, *test require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - }, true)) + })) + require.NoError(t, store.PromoteSandboxRevision(ctx, "revision")) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&v1alpha3.SandboxTemplate{ diff --git a/go/core/internal/service/sandbox/service_test.go b/go/core/internal/service/sandbox/service_test.go index 19aebfda3b..19a0f9cd01 100644 --- a/go/core/internal/service/sandbox/service_test.go +++ b/go/core/internal/service/sandbox/service_test.go @@ -138,7 +138,8 @@ func serviceFixture(t *testing.T) (*Service, context.Context, *testActors) { require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - }, true)) + })) + require.NoError(t, store.PromoteSandboxRevision(ctx, "revision")) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&v1alpha3.SandboxTemplate{ @@ -295,7 +296,8 @@ func TestSandboxResolvesCurrentGrantsBeforeResume(t *testing.T) { } store := service.config.Store require.NoError(t, store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: revision.Revision})) - require.NoError(t, store.RecordSandboxRevision(ctx, revision, true)) + require.NoError(t, store.RecordSandboxRevision(ctx, revision)) + require.NoError(t, store.PromoteSandboxRevision(ctx, revision.Revision)) instance, err := service.Create(ctx, createRequest()) require.NoError(t, err) require.Equal(t, revision.Revision, instance.PreparedRevision) diff --git a/go/core/internal/service/session/workflow_test.go b/go/core/internal/service/session/workflow_test.go index b503fb5386..0550d15b9d 100644 --- a/go/core/internal/service/session/workflow_test.go +++ b/go/core/internal/service/session/workflow_test.go @@ -149,7 +149,8 @@ func lifecycleFixture(t *testing.T) (*lifecycleTestStore, *apiv1.Session) { ActorTemplateAtespace: "team-a", ActorTemplateName: "assistant-kagent-revision", ActorTemplateUID: "actor-template-uid", } require.NoError(t, client.UpsertAgentDefinition(t.Context(), database.AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision})) - require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision, true)) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision)) + require.NoError(t, client.PromoteRuntimeRevision(t.Context(), revision.Revision)) session, _, err := client.CreateSession(t.Context(), &apiv1.Session{Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}}, uuid.NewString()) require.NoError(t, err) return &lifecycleTestStore{Client: client, revision: revision}, session From c0220a82a7eb9a528533d3e7a9154ce4e75800ae Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 16:36:54 +0000 Subject: [PATCH 18/43] test: make egress probe runs and expectations explicit Signed-off-by: Eitan Yarmush --- go/core/test/e2e/egress_credentials_test.go | 6 ++-- go/core/test/e2e/egress_test.go | 35 ++++++++++++--------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/go/core/test/e2e/egress_credentials_test.go b/go/core/test/e2e/egress_credentials_test.go index a7c77ffca5..d435e1ab51 100644 --- a/go/core/test/e2e/egress_credentials_test.go +++ b/go/core/test/e2e/egress_credentials_test.go @@ -44,7 +44,7 @@ func TestEgressPolicyCredentialDelivery(t *testing.T) { image := publishSkillImage(t) forEachHarness(t, func(t *testing.T, harness testHarness) { t.Parallel() - modelURL, model, runs := startEgressSkillMock(t, harness.skillsRoot(), destination) + modelURL, model, runs := startEgressSkillMock(t, harness.skillsRoot(), destination, "credentials") config := harness.createModel(t, kube, modelURL, nil) template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-credentials-", Namespace: "kagent", Labels: harness.labels()}, Spec: v1alpha3.AgentTemplateSpec{ ModelConfig: &corev1.LocalObjectReference{Name: config.Name}, SystemPrompt: "Load the egress skill and run its probe.", @@ -66,8 +66,8 @@ func TestEgressPolicyCredentialDelivery(t *testing.T) { t.Cleanup(func() { assert.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), policy))) }) refs = append(refs, corev1.LocalObjectReference{Name: policy.Name}) } - run := runs("live") - finish := runEgressSkill(t, fixture, model, run) + run := runs("credentials") + finish := runEgressSkill(t, fixture, model, run, "initial_DENIED", "authenticated_ALLOWED") run.expect(t, "initial", "DENIED") agent := &v1alpha3.Agent{} require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKey{Namespace: "kagent", Name: template.Name}, agent)) diff --git a/go/core/test/e2e/egress_test.go b/go/core/test/e2e/egress_test.go index 112714b15a..37272619e1 100644 --- a/go/core/test/e2e/egress_test.go +++ b/go/core/test/e2e/egress_test.go @@ -69,7 +69,7 @@ func TestSessionEgressPolicyUpdatesLiveActors(t *testing.T) { t.Parallel() kube := interactionKubeClient(t) destination, requests := startEgressDestination(t) - modelURL, model, runs := startEgressSkillMock(t, harness.skillsRoot(), destination) + modelURL, model, runs := startEgressSkillMock(t, harness.skillsRoot(), destination, "live", "checkpoint", "fresh", "resumed", "fork") config := harness.createModel(t, kube, modelURL, nil) template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{GenerateName: "live-egress-", Namespace: "kagent", Labels: harness.labels()}, Spec: v1alpha3.AgentTemplateSpec{ ModelConfig: &corev1.LocalObjectReference{Name: config.Name}, SystemPrompt: "Load the egress skill and run its probe.", @@ -95,7 +95,7 @@ func TestSessionEgressPolicyUpdatesLiveActors(t *testing.T) { } run := runs("live") - finish := runEgressSkill(t, fixture, model, run) + finish := runEgressSkill(t, fixture, model, run, "initial_DENIED", "allowed_ALLOWED", "revoked_DENIED") run.expect(t, "initial", "DENIED") require.Zero(t, requests("live_initial"), "ungranted requests reached the destination") require.NoError(t, kube.Get(t.Context(), key, agent)) @@ -120,7 +120,7 @@ func TestSessionEgressPolicyUpdatesLiveActors(t *testing.T) { // suspended. Neither resume nor a DATA fork may resurrect that old grant. setGrant(destination) run = runs("checkpoint") - finish = runEgressSkill(t, fixture, model, run) + finish = runEgressSkill(t, fixture, model, run, "initial_ALLOWED") run.expect(t, "initial", "ALLOWED") task := finish() assertActorSuspended(t, fixture) @@ -148,12 +148,12 @@ func TestSessionEgressPolicyUpdatesLiveActors(t *testing.T) { require.NoError(t, deleteIdleSession(context.WithoutCancel(fresh.ctx), fresh.sessions, fresh.sessionID)) }) run = runs("fresh") - finish = runEgressSkill(t, &fresh, model, run) + finish = runEgressSkill(t, &fresh, model, run, "initial_DENIED") run.expect(t, "initial", "DENIED") require.Zero(t, requests("fresh_initial")) finish() run = runs("resumed") - finish = runEgressSkill(t, fixture, model, run) + finish = runEgressSkill(t, fixture, model, run, "initial_DENIED") run.expect(t, "initial", "DENIED") require.Zero(t, requests("resumed_initial")) finish() @@ -172,7 +172,7 @@ func TestSessionEgressPolicyUpdatesLiveActors(t *testing.T) { require.NoError(t, deleteIdleSession(ctx, fork.sessions, fork.sessionID)) }) run = runs("fork") - finish = runEgressSkill(t, &fork, model, run) + finish = runEgressSkill(t, &fork, model, run, "initial_DENIED") run.expect(t, "initial", "DENIED") require.Zero(t, requests("fork_initial")) finish() @@ -215,7 +215,7 @@ func (e *egressProbeRun) finish(ctx context.Context) error { } } -func runEgressSkill(t *testing.T, fixture *interactionFixture, model *modelRecorder, run *egressProbeRun) func() *a2atype.Task { +func runEgressSkill(t *testing.T, fixture *interactionFixture, model *modelRecorder, run *egressProbeRun, expectedResults ...string) func() *a2atype.Task { t.Helper() start := len(model.Requests("", "")) _, request := newMessageRequest(t, "Run egress skill "+run.name+".") @@ -268,10 +268,8 @@ func runEgressSkill(t *testing.T, fixture *interactionFixture, model *modelRecor for _, marker := range []string{"SKILL_BODY_EGRESS", "EGRESS_PROBE_STARTED_" + run.name, "EGRESS_PROBE_FINISHED_" + run.name} { require.Contains(t, string(outputs), marker) } - if run.name == "live" { - for _, marker := range []string{"EGRESS_RESULT_live_initial_DENIED", "EGRESS_RESULT_live_allowed_ALLOWED", "EGRESS_RESULT_live_revoked_DENIED"} { - require.Contains(t, string(outputs), marker) - } + for _, result := range expectedResults { + require.Contains(t, string(outputs), "EGRESS_RESULT_"+run.name+"_"+result) } return task } @@ -323,8 +321,9 @@ func startEgressDestination(t *testing.T) (string, func(string) int) { return destination.String(), func(phase string) int { mu.Lock(); defer mu.Unlock(); return requests[phase] } } -func startEgressSkillMock(t *testing.T, root, destination string) (string, *modelRecorder, func(string) *egressProbeRun) { +func startEgressSkillMock(t *testing.T, root, destination string, runNames ...string) (string, *modelRecorder, func(string) *egressProbeRun) { t.Helper() + require.NotEmpty(t, runNames) var active atomic.Pointer[egressProbeRun] server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { run := active.Load() @@ -368,7 +367,7 @@ func startEgressSkillMock(t *testing.T, root, destination string) (string, *mode raw, err := interactionMocks.ReadFile("mocks/invoke_egress_skill.json") require.NoError(t, err) runs := map[string]*egressProbeRun{} - for _, phase := range []string{"live", "checkpoint", "resumed", "fork"} { + for _, phase := range runNames { command := fmt.Sprintf("sh %s/egress/probe.sh '%s/egress/%s' '%s' %s", root, origin, phase, destination, phase) replacements := []string{"PHASE", phase, "ROOT", root} args, err := json.Marshal(struct { @@ -395,8 +394,14 @@ func startEgressSkillMock(t *testing.T, root, destination string) (string, *mode run.setPhase("initial") runs[phase] = run } - active.Store(runs["live"]) - return modelURL, recorder, func(phase string) *egressProbeRun { run := runs[phase]; active.Store(run); return run } + active.Store(runs[runNames[0]]) + return modelURL, recorder, func(name string) *egressProbeRun { + t.Helper() + run := runs[name] + require.NotNil(t, run, "unregistered egress probe %q", name) + active.Store(run) + return run + } } func TestSandboxEgressPolicyUpdatesLiveActor(t *testing.T) { From 88c88b919a7fc15355902b14a5916f70aeea7087 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 17:10:41 +0000 Subject: [PATCH 19/43] fix: serialize egress publication per revision Signed-off-by: Eitan Yarmush --- go/core/internal/controller/egress.go | 56 ++++++++++++--- go/core/internal/controller/egress_test.go | 82 +++++++++++++++++++++- 2 files changed, 126 insertions(+), 12 deletions(-) diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go index 92cdcf3abd..86bd8df27a 100644 --- a/go/core/internal/controller/egress.go +++ b/go/core/internal/controller/egress.go @@ -89,14 +89,15 @@ type EgressReconciler struct { actors actorEgressClient namespaces []string warnings *cache.LRUExpireCache - // Initial preparation and source/recovery queues share a publisher. Serialize - // cache lookup through commit so an older translation cannot commit last. - publishMu sync.Mutex - sources controllers.Queue - pages controllers.Queue - updates controllers.Queue - recovery controllers.Queue - sweeps controllers.Queue + // Only active publications retain a lock. The mutex protects this map; + // cache lookup and database writes are serialized per revision. + publishMu sync.Mutex + publishing map[string]chan struct{} + sources controllers.Queue + pages controllers.Queue + updates controllers.Queue + recovery controllers.Queue + sweeps controllers.Queue } type egressSourcePage struct { @@ -119,7 +120,7 @@ var _ manager.LeaderElectionRunnable = (*EgressReconciler)(nil) func NewEgressReconciler(ctx context.Context, runtime *Runtime, store runtimeEgressStore, actors actorEgressClient, namespaces []string) *EgressReconciler { e := &EgressReconciler{grants: newEgressGrants(runtime), store: store, actors: actors, - namespaces: slices.Clone(namespaces), warnings: cache.NewLRUExpireCache(1024)} + namespaces: slices.Clone(namespaces), warnings: cache.NewLRUExpireCache(1024), publishing: make(map[string]chan struct{})} reconcile := func(item any) error { err := e.reconcileActor(ctx, item.(database.RuntimeEgressTarget)) if status.Code(err) == codes.NotFound { @@ -200,8 +201,11 @@ func (e *EgressReconciler) ReconcileRevision(ctx context.Context, revision strin } func (e *EgressReconciler) publishRevision(ctx context.Context, revision string, recoverActors bool) error { - e.publishMu.Lock() - defer e.publishMu.Unlock() + unlock, err := e.lockPublication(ctx, revision) + if err != nil { + return err + } + defer unlock() pinned, err := e.store.GetRuntimeEgress(ctx, revision) if err != nil { return err @@ -235,6 +239,36 @@ func (e *EgressReconciler) publishRevision(ctx context.Context, revision string, return err } +// Initial preparation and source/recovery queues share a publisher. Hold the +// revision lock from cache lookup through commit so an older translation cannot +// commit last. Waiters retry after release; unrelated revisions never wait on I/O. +func (e *EgressReconciler) lockPublication(ctx context.Context, revision string) (func(), error) { + for { + if err := ctx.Err(); err != nil { + return nil, err + } + e.publishMu.Lock() + done, busy := e.publishing[revision] + if !busy { + done = make(chan struct{}) + e.publishing[revision] = done + e.publishMu.Unlock() + return func() { + e.publishMu.Lock() + defer e.publishMu.Unlock() + delete(e.publishing, revision) + close(done) + }, nil + } + e.publishMu.Unlock() + select { + case <-done: + case <-ctx.Done(): + return nil, ctx.Err() + } + } +} + func (e *EgressReconciler) NeedLeaderElection() bool { return true } func (e *EgressReconciler) Start(ctx context.Context) error { diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go index b03e5614f7..6d5c25db24 100644 --- a/go/core/internal/controller/egress_test.go +++ b/go/core/internal/controller/egress_test.go @@ -46,6 +46,7 @@ type egressTestStore struct { listed chan database.RuntimeEgressSource publishErr error losePublishResponse bool + beforePublish func(context.Context, string, egress.Access) error } var _ runtimeEgressStore = (*egressTestStore)(nil) @@ -59,7 +60,12 @@ func (e *egressTestStore) GetRuntimeEgress(_ context.Context, revision string) ( } return &row, nil } -func (e *egressTestStore) SetRuntimeEgressPolicy(_ context.Context, revision string, access egress.Access) (bool, error) { +func (e *egressTestStore) SetRuntimeEgressPolicy(ctx context.Context, revision string, access egress.Access) (bool, error) { + if e.beforePublish != nil { + if err := e.beforePublish(ctx, revision, access); err != nil { + return false, err + } + } e.mu.Lock() defer e.mu.Unlock() if e.publishErr != nil { @@ -327,6 +333,80 @@ func TestEgressSkipsUnchangedPoliciesAndRecoversLostWriteResponse(t *testing.T) }) } +func TestEgressPublicationSerializesOnlyTheSameRevision(t *testing.T) { + for _, failFirst := range []bool{false, true} { + t.Run(fmt.Sprintf("fail_first=%t", failFirst), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + inputs := newEgressTestInputs(ctx) + inputs.source(database.RuntimeEgressAgent, "uid", []corev1.LocalObjectReference{{Name: "access"}}) + inputs.policies.UpdateObject(egressTestPolicy("https://tickets.example.com", nil)) + pinned := database.RuntimeEgress{Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"} + entered, release := make(chan struct{}), make(chan struct{}) + firstErr := errors.New("publication failed") + store := &egressTestStore{ + pinned: map[string]database.RuntimeEgress{"revision": pinned, "other": pinned}, + beforePublish: func(ctx context.Context, revision string, access egress.Access) error { + if revision != "revision" || len(access.Destinations) == 0 { + return nil + } + close(entered) + select { + case <-release: + case <-ctx.Done(): + return ctx.Err() + } + if failFirst { + return firstErr + } + return nil + }, + } + controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) + defer func() { + for _, queue := range []controllers.Queue{controller.sources, controller.pages, controller.updates, controller.recovery, controller.sweeps} { + queue.ShutDownEarly() + } + }() + first, second, other := make(chan error, 1), make(chan error, 1), make(chan error, 1) + go func() { first <- controller.ReconcileRevision(ctx, "revision") }() + <-entered + inputs.policies.DeleteObject("team/access") + synctest.Wait() + go func() { second <- controller.ReconcileRevision(ctx, "revision") }() + go func() { other <- controller.ReconcileRevision(ctx, "other") }() + synctest.Wait() + require.Len(t, other, 1, "another revision must publish while the first database write is blocked") + require.NoError(t, <-other) + require.Empty(t, second, "the same revision must wait through commit") + + waitingCtx, stopWaiting := context.WithCancel(ctx) + canceled := make(chan error, 1) + go func() { canceled <- controller.ReconcileRevision(waitingCtx, "revision") }() + synctest.Wait() + stopWaiting() + synctest.Wait() + require.Len(t, canceled, 1, "a canceled waiter must leave without waiting for the writer") + require.ErrorIs(t, <-canceled, context.Canceled) + close(release) + if failFirst { + require.ErrorIs(t, <-first, firstErr) + } else { + require.NoError(t, <-first) + } + require.NoError(t, <-second) + for _, revision := range []string{"revision", "other"} { + access, err := store.GetRuntimeEgressPolicy(ctx, revision) + require.NoError(t, err) + require.Empty(t, access.Destinations, "publication must use grants read after acquiring the revision lock") + } + require.Empty(t, controller.publishing, "completed publications must release lock state") + }) + }) + } +} + func TestEgressTargetsSourceChangesAndBacksOffActorFailures(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() From 0c445aa3cb93aab4953272a94840451ca6127a1c Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 17:10:41 +0000 Subject: [PATCH 20/43] test: verify sessions use the last successful revision after invalid edits Signed-off-by: Eitan Yarmush --- go/core/test/e2e/runtime_revision_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/go/core/test/e2e/runtime_revision_test.go b/go/core/test/e2e/runtime_revision_test.go index 9965507caf..36efc987a7 100644 --- a/go/core/test/e2e/runtime_revision_test.go +++ b/go/core/test/e2e/runtime_revision_test.go @@ -111,8 +111,14 @@ func TestRuntimeRevisionLifecycle(t *testing.T) { } return false, nil })) - _, err = sessions.CreateSession(ctx, request(templateName)) - require.Equal(t, codes.FailedPrecondition, status.Code(err), "an invalid edit must block fresh sessions") + fallback, err := sessions.CreateSession(ctx, request(templateName)) + require.NoError(t, err, "an invalid edit must preserve the last successful runtime for fresh sessions") + fallbackID := fallback.GetSession().GetId() + t.Cleanup(func() { deleteSession(fallbackID) }) + require.NotEqual(t, source.GetId(), fallbackID) + require.Equal(t, source.GetPreparedRevision(), fallback.GetSession().GetPreparedRevision()) + send(fallbackID) + deleteSession(fallbackID) retried, err := sessions.CreateSession(ctx, createRequest) require.NoError(t, err, "creation retries must retain the existing session's pinned runtime") require.Equal(t, source.GetId(), retried.GetSession().GetId()) @@ -134,7 +140,7 @@ func TestRuntimeRevisionLifecycle(t *testing.T) { agent := &v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: template.Name}} require.NoError(t, kube.Get(ctx, ctrlclient.ObjectKeyFromObject(agent), agent)) require.NoError(t, kube.Delete(ctx, agent)) - // Deletion keeps fresh creation blocked. Eventual runtime collection below + // Deletion blocks fresh creation. Eventual runtime collection below // verifies retirement after the final checkpoint reference is released. _, err = sessions.CreateSession(ctx, request(templateName)) require.Equal(t, codes.FailedPrecondition, status.Code(err)) From 43712a577636342c404ebd9ac03af71ec80332cd Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 17:10:42 +0000 Subject: [PATCH 21/43] refactor: fold egress schema into the initial migration Signed-off-by: Eitan Yarmush --- .../pkg/migrations/core/000001_initial.sql | 13 +++++++ ...0002_runtime_token_passthrough_origins.sql | 36 ------------------- go/core/pkg/migrations/runner_test.go | 3 -- 3 files changed, 13 insertions(+), 39 deletions(-) delete mode 100644 go/core/pkg/migrations/core/000002_runtime_token_passthrough_origins.sql diff --git a/go/core/pkg/migrations/core/000001_initial.sql b/go/core/pkg/migrations/core/000001_initial.sql index d04ebba522..71bc68c1b7 100644 --- a/go/core/pkg/migrations/core/000001_initial.sql +++ b/go/core/pkg/migrations/core/000001_initial.sql @@ -34,6 +34,7 @@ CREATE TABLE runtime_revision ( source_snapshot JSONB NOT NULL, egress_destinations TEXT[] NOT NULL DEFAULT '{}', credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(credentials) = 'array'), + token_passthrough_origins TEXT[] NOT NULL DEFAULT '{}', actor_template_atespace TEXT CONSTRAINT runtime_revision_actor_template_namespace_not_null NOT NULL, actor_template_name TEXT NOT NULL, actor_template_uid TEXT NOT NULL DEFAULT '', @@ -45,6 +46,13 @@ CREATE TABLE runtime_revision ( UNIQUE (actor_template_atespace, actor_template_name) ); +-- Mutable controller output lives outside immutable runtime revision inputs. +CREATE TABLE runtime_egress_policy ( + revision TEXT PRIMARY KEY REFERENCES runtime_revision (revision) ON DELETE CASCADE, + destinations TEXT[] NOT NULL, + credentials JSONB NOT NULL CHECK (jsonb_typeof(credentials) = 'array') +); + CREATE TABLE agent_revision ( revision TEXT PRIMARY KEY, agent_name TEXT NOT NULL, @@ -52,6 +60,7 @@ CREATE TABLE agent_revision ( agent_card BYTEA NOT NULL, FOREIGN KEY (revision) REFERENCES runtime_revision(revision) ON DELETE CASCADE ); +CREATE INDEX agent_revision_source_idx ON agent_revision (agent_name, revision); CREATE TABLE sandbox_revision ( revision TEXT PRIMARY KEY, @@ -59,6 +68,7 @@ CREATE TABLE sandbox_revision ( sandbox_template_uid TEXT NOT NULL, FOREIGN KEY (revision) REFERENCES runtime_revision(revision) ON DELETE CASCADE ); +CREATE INDEX sandbox_revision_source_idx ON sandbox_revision (sandbox_template_name, revision); CREATE TABLE agent_definition ( namespace TEXT NOT NULL, @@ -135,6 +145,8 @@ CREATE TABLE runtime_instance ( ); CREATE INDEX runtime_instance_owner_idx ON runtime_instance (kind, user_id, id) WHERE state <> 'RUNTIME_STATE_DELETED'; +CREATE INDEX runtime_instance_revision_idx ON runtime_instance (prepared_revision, id) + WHERE state <> 'RUNTIME_STATE_DELETED' AND operation <> 'RUNTIME_OPERATION_DELETE'; CREATE TABLE session ( id UUID PRIMARY KEY, @@ -390,6 +402,7 @@ DROP TABLE sandbox_template_definition; DROP TABLE agent_definition; DROP TABLE sandbox_revision; DROP TABLE agent_revision; +DROP TABLE runtime_egress_policy; DROP TABLE runtime_revision; DROP TABLE toolserver; DROP TABLE tool; diff --git a/go/core/pkg/migrations/core/000002_runtime_token_passthrough_origins.sql b/go/core/pkg/migrations/core/000002_runtime_token_passthrough_origins.sql deleted file mode 100644 index aa435c45f0..0000000000 --- a/go/core/pkg/migrations/core/000002_runtime_token_passthrough_origins.sql +++ /dev/null @@ -1,36 +0,0 @@ --- +goose Up - -ALTER TABLE runtime_revision ADD COLUMN token_passthrough_origins TEXT[] NOT NULL DEFAULT '{}'; - --- PostgreSQL expands r.* when the view is created. -DROP VIEW agent_runtime_revision; -CREATE VIEW agent_runtime_revision AS - SELECT r.*, a.agent_name, a.agent_uid, a.agent_card - FROM runtime_revision r JOIN agent_revision a USING (revision) WHERE r.kind = 'agent'; - --- Mutable controller output lives outside immutable runtime revision inputs. -CREATE TABLE runtime_egress_policy ( - revision TEXT PRIMARY KEY REFERENCES runtime_revision (revision) ON DELETE CASCADE, - destinations TEXT[] NOT NULL, - credentials JSONB NOT NULL CHECK (jsonb_typeof(credentials) = 'array') -); - --- Target changed sources without scanning all live instances. -CREATE INDEX agent_revision_source_idx ON agent_revision (agent_name, revision); -CREATE INDEX sandbox_revision_source_idx ON sandbox_revision (sandbox_template_name, revision); -CREATE INDEX runtime_instance_revision_idx ON runtime_instance (prepared_revision, id) - WHERE state <> 'RUNTIME_STATE_DELETED' AND operation <> 'RUNTIME_OPERATION_DELETE'; - --- +goose Down - -DROP TABLE runtime_egress_policy; - -DROP INDEX runtime_instance_revision_idx; -DROP INDEX sandbox_revision_source_idx; -DROP INDEX agent_revision_source_idx; - -DROP VIEW agent_runtime_revision; -ALTER TABLE runtime_revision DROP COLUMN token_passthrough_origins; -CREATE VIEW agent_runtime_revision AS - SELECT r.*, a.agent_name, a.agent_uid, a.agent_card - FROM runtime_revision r JOIN agent_revision a USING (revision) WHERE r.kind = 'agent'; diff --git a/go/core/pkg/migrations/runner_test.go b/go/core/pkg/migrations/runner_test.go index 8fb5a4454b..5394bf9ba2 100644 --- a/go/core/pkg/migrations/runner_test.go +++ b/go/core/pkg/migrations/runner_test.go @@ -319,9 +319,6 @@ func TestBuiltinMigrationsRoundTrip(t *testing.T) { } for _, source := range sources { expected := []int64{0, 1} - if source.TrackingTable == "schema_migrations" { - expected = append(expected, 2) - } if versions := testVersions(t, dsn, source.TrackingTable); !slices.Equal(versions, expected) { t.Fatalf("%s versions = %v, want %v", source.Name, versions, expected) } From 7bc82ca49af4df3a627dd41f3cf6a7f389d7ffb3 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 17:52:08 +0000 Subject: [PATCH 22/43] test: wait for agent retirement before asserting session rejection Signed-off-by: Eitan Yarmush --- go/core/test/e2e/runtime_revision_test.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/go/core/test/e2e/runtime_revision_test.go b/go/core/test/e2e/runtime_revision_test.go index 36efc987a7..268d0fc860 100644 --- a/go/core/test/e2e/runtime_revision_test.go +++ b/go/core/test/e2e/runtime_revision_test.go @@ -140,10 +140,21 @@ func TestRuntimeRevisionLifecycle(t *testing.T) { agent := &v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: template.Name}} require.NoError(t, kube.Get(ctx, ctrlclient.ObjectKeyFromObject(agent), agent)) require.NoError(t, kube.Delete(ctx, agent)) - // Deletion blocks fresh creation. Eventual runtime collection below - // verifies retirement after the final checkpoint reference is released. - _, err = sessions.CreateSession(ctx, request(templateName)) - require.Equal(t, codes.FailedPrecondition, status.Code(err)) + // Kubernetes deletion reaches the database through the controller. Wait for + // retirement, releasing any sessions created before that event is observed + // so they cannot keep the runtime alive during the GC assertion below. + require.NoError(t, wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + created, err := sessions.CreateSession(ctx, request(templateName)) + if status.Code(err) == codes.FailedPrecondition { + return true, nil + } + if err != nil { + return false, err + } + id := created.GetSession().GetId() + t.Cleanup(func() { deleteSession(id) }) + return false, deleteIdleSession(ctx, sessions, id) + }), "Agent deletion must eventually block fresh sessions") forked, err := checkpoints.ForkSession(ctx, &apiv1.ForkSessionRequest{CheckpointId: checkpointID, RequestId: uuid.NewString()}) require.NoError(t, err, "a checkpoint must retain runnable inputs after its source and template are deleted") forkID := forked.GetSession().GetId() From d16abd9beb86baf3d9a0e6281fe69838e036f225 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 18:33:00 +0000 Subject: [PATCH 23/43] refactor: queue egress policy publication by revision Use a revision-keyed workqueue to serialize publication and replay changes received during an in-flight write. Keep startup actor recovery separate, and require a persisted policy before the store promotes a revision. Signed-off-by: Eitan Yarmush --- docs/architecture/egress-policies.md | 9 +- .../internal/controller/collections_test.go | 2 +- go/core/internal/controller/egress.go | 145 +++++++++--------- go/core/internal/controller/egress_test.go | 141 +++++++++++------ .../internal/controller/preparation_test.go | 85 ++++++---- go/core/internal/controller/reconciler.go | 30 ++-- .../internal/controller/reconciler_test.go | 12 +- .../controller/sandbox_integration_test.go | 2 +- .../internal/controller/sandbox_reconciler.go | 18 +-- .../controller/sandbox_reconciler_test.go | 16 +- .../internal/database/client_session_test.go | 7 + go/core/internal/database/errors.go | 4 + .../internal/database/runtime_egress_test.go | 2 + .../internal/database/runtime_revisions.go | 42 +++-- .../database/runtime_revisions_test.go | 12 ++ .../internal/database/sandbox_revisions.go | 34 ++-- go/core/internal/database/sandboxes_test.go | 7 + .../internal/grpcserver/scheduledrun_test.go | 3 + .../internal/grpcserver/task_store_test.go | 3 + .../internal/service/sandbox/guest_test.go | 3 + .../internal/service/sandbox/service_test.go | 5 + .../internal/service/session/workflow_test.go | 2 + go/core/pkg/app/app.go | 4 +- 23 files changed, 368 insertions(+), 220 deletions(-) diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index ca6d0efee3..34f941ca2a 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -161,8 +161,13 @@ Secret values remain outside compiled inputs; the gateway handles value rotation The elected controller compiles current grants from cached, dependency-tracked Kubernetes inputs and persists the intended policy for each retained revision. -This mutable output is stored separately from immutable revision inputs. Initial -preparation publishes the policy before promoting a new revision to ready. +This mutable output is stored separately from immutable revision inputs. Preparation, +source events, and sweeps enqueue revision IDs into one publication queue. It +serializes writes for each revision and reruns with the latest grants when an +event arrives during publication; unrelated revisions can publish concurrently. +The store prevents promotion until an initial policy has been published. +Preparation retries with backoff while publication is pending, so initial +readiness may wait an extra queue hop and retry interval. The grants cache can briefly lag initial compilation, so a new workload may become ready with only pinned integration access until its grants event arrives. Each revision merges its pinned integration access with the current grants of its diff --git a/go/core/internal/controller/collections_test.go b/go/core/internal/controller/collections_test.go index 46b7030f24..78c2bf3274 100644 --- a/go/core/internal/controller/collections_test.go +++ b/go/core/internal/controller/collections_test.go @@ -102,7 +102,7 @@ func TestReconciliationCollectionsCompileAndObserveRevision(t *testing.T) { observed.Metadata.Uid = "actor-template-uid" observed.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}}} store := &fakeRuntimeRevisionStore{} - reconciler := &Reconciler{egress: noopRevisionEgress, + reconciler := &Reconciler{enqueueEgress: noopEnqueueEgress, collections: collections, templates: &fakeActorTemplates{template: observed}, store: store, } require.NoError(t, reconciler.reconcileAgent(t.Context(), state.ResourceName())) diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go index 86bd8df27a..be63820881 100644 --- a/go/core/internal/controller/egress.go +++ b/go/core/internal/controller/egress.go @@ -13,6 +13,7 @@ import ( "istio.io/istio/pkg/kube/controllers" "istio.io/istio/pkg/kube/krt" "k8s.io/apimachinery/pkg/util/cache" + "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/manager" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" @@ -31,12 +32,6 @@ type runtimeEgressStore interface { ListRuntimeEgressTargets(context.Context, string, string, int) ([]database.RuntimeEgressTarget, error) } -// revisionEgressReconciler publishes the initial policy before preparation can -// promote a revision. Later egress-only translations use the same writer. -type revisionEgressReconciler interface { - ReconcileRevision(context.Context, string) error -} - type policyGrants struct { Source database.RuntimeEgressSource UID string @@ -82,22 +77,19 @@ type actorEgressClient interface { // EgressReconciler publishes KRT's current grants per retained revision. Actors // and API replicas apply this persisted output without translating Kubernetes. -// Recovery has separate workers so a full sweep cannot delay changed sources. +// Startup actor recovery has separate workers from live actor updates. type EgressReconciler struct { - grants krt.Collection[policyGrants] - store runtimeEgressStore - actors actorEgressClient - namespaces []string - warnings *cache.LRUExpireCache - // Only active publications retain a lock. The mutex protects this map; - // cache lookup and database writes are serialized per revision. - publishMu sync.Mutex - publishing map[string]chan struct{} - sources controllers.Queue - pages controllers.Queue - updates controllers.Queue - recovery controllers.Queue - sweeps controllers.Queue + grants krt.Collection[policyGrants] + store runtimeEgressStore + actors actorEgressClient + namespaces []string + warnings *cache.LRUExpireCache + publications workqueue.TypedRateLimitingInterface[string] + sources controllers.Queue + pages controllers.Queue + updates controllers.Queue + recovery controllers.Queue + sweeps controllers.Queue } type egressSourcePage struct { @@ -120,7 +112,11 @@ var _ manager.LeaderElectionRunnable = (*EgressReconciler)(nil) func NewEgressReconciler(ctx context.Context, runtime *Runtime, store runtimeEgressStore, actors actorEgressClient, namespaces []string) *EgressReconciler { e := &EgressReconciler{grants: newEgressGrants(runtime), store: store, actors: actors, - namespaces: slices.Clone(namespaces), warnings: cache.NewLRUExpireCache(1024), publishing: make(map[string]chan struct{})} + namespaces: slices.Clone(namespaces), warnings: cache.NewLRUExpireCache(1024), + publications: workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.NewTypedItemExponentialFailureRateLimiter[string](time.Second, 30*time.Second), + workqueue.TypedRateLimitingQueueConfig[string]{Name: "live-egress-publications"}), + } reconcile := func(item any) error { err := e.reconcileActor(ctx, item.(database.RuntimeEgressTarget)) if status.Code(err) == codes.NotFound { @@ -133,6 +129,20 @@ func NewEgressReconciler(ctx context.Context, runtime *Runtime, store runtimeEgr const pageSize = 100 e.pages = newReconciliationQueue("live-egress-revision-actors", func(item any) error { page := item.(egressActorPage) + // Startup recovery lists all revisions independently of publication. + // Check watch scope before expanding its first actor page. + if page.Recovery && page.After == "" && len(e.namespaces) > 0 { + pinned, err := e.store.GetRuntimeEgress(ctx, page.Revision) + if errors.Is(err, database.ErrNotFound) { + return nil + } + if err != nil { + return err + } + if !slices.Contains(e.namespaces, pinned.Namespace) { + return nil + } + } targets, err := e.store.ListRuntimeEgressTargets(ctx, page.Revision, page.After, pageSize) if err != nil { return err @@ -157,9 +167,7 @@ func NewEgressReconciler(ctx context.Context, runtime *Runtime, store runtimeEgr return err } for _, revision := range revisions { - if err := e.ReconcileRevision(ctx, revision); err != nil && !errors.Is(err, database.ErrNotFound) { - return err - } + e.EnqueueRevision(revision) } if len(revisions) == pageSize { page.After = revisions[len(revisions)-1] @@ -174,12 +182,9 @@ func NewEgressReconciler(ctx context.Context, runtime *Runtime, store runtimeEgr return err } for _, revision := range revisions { - err := e.publishRevision(ctx, revision, page.RecoverActors) - if errors.Is(err, database.ErrNotFound) { - continue - } - if err != nil { - return err + e.EnqueueRevision(revision) + if page.RecoverActors { + e.pages.Add(egressActorPage{Revision: revision, Recovery: true}) } } if len(revisions) == pageSize { @@ -191,21 +196,15 @@ func NewEgressReconciler(ctx context.Context, runtime *Runtime, store runtimeEgr return e } -// ReconcileRevision publishes current intended access before making a new -// revision ready, and schedules existing actors independently of preparation. -func (e *EgressReconciler) ReconcileRevision(ctx context.Context, revision string) error { - if !e.grants.WaitUntilSynced(ctx.Done()) { - return ctx.Err() - } - return e.publishRevision(ctx, revision, false) +// EnqueueRevision schedules publication from the latest cached grants. The +// revision alone is the key: re-adding it during publication guarantees a rerun. +func (e *EgressReconciler) EnqueueRevision(revision string) { + e.publications.Add(revision) } -func (e *EgressReconciler) publishRevision(ctx context.Context, revision string, recoverActors bool) error { - unlock, err := e.lockPublication(ctx, revision) - if err != nil { - return err - } - defer unlock() +// publishRevision is called only by publication workers; the queue serializes +// cache lookup through persistence for each revision. +func (e *EgressReconciler) publishRevision(ctx context.Context, revision string) error { pinned, err := e.store.GetRuntimeEgress(ctx, revision) if err != nil { return err @@ -231,42 +230,40 @@ func (e *EgressReconciler) publishRevision(ctx context.Context, revision string, e.warnings.Remove(revision) } changed, err := e.store.SetRuntimeEgressPolicy(ctx, revision, access) - if changed || recoverActors || err != nil { + if changed || err != nil { // A failed response may hide a committed write. Queue actor application // even then: a publication retry could observe an unchanged policy. - e.pages.Add(egressActorPage{Revision: revision, Recovery: recoverActors}) + e.pages.Add(egressActorPage{Revision: revision}) } return err } -// Initial preparation and source/recovery queues share a publisher. Hold the -// revision lock from cache lookup through commit so an older translation cannot -// commit last. Waiters retry after release; unrelated revisions never wait on I/O. -func (e *EgressReconciler) lockPublication(ctx context.Context, revision string) (func(), error) { - for { - if err := ctx.Err(); err != nil { - return nil, err - } - e.publishMu.Lock() - done, busy := e.publishing[revision] - if !busy { - done = make(chan struct{}) - e.publishing[revision] = done - e.publishMu.Unlock() - return func() { - e.publishMu.Lock() - defer e.publishMu.Unlock() - delete(e.publishing, revision) - close(done) - }, nil - } - e.publishMu.Unlock() - select { - case <-done: - case <-ctx.Done(): - return nil, ctx.Err() - } +func (e *EgressReconciler) runPublications(ctx context.Context) { + var workers sync.WaitGroup + for range 4 { + workers.Go(func() { + for e.processPublication(ctx) { + } + }) + } + <-ctx.Done() + e.publications.ShutDown() + workers.Wait() +} + +func (e *EgressReconciler) processPublication(ctx context.Context) bool { + revision, shutdown := e.publications.Get() + if shutdown { + return false + } + defer e.publications.Done(revision) + if err := e.publishRevision(ctx, revision); err != nil && !errors.Is(err, database.ErrNotFound) && ctx.Err() == nil { + logging.FromContext(ctx).ErrorContext(ctx, "publish runtime egress policy", "revision", revision, "error", err) + e.publications.AddRateLimited(revision) + } else { + e.publications.Forget(revision) } + return true } func (e *EgressReconciler) NeedLeaderElection() bool { return true } @@ -280,12 +277,14 @@ func (e *EgressReconciler) Start(ctx context.Context) error { defer handler.UnregisterHandler() queues := []controllers.Queue{e.sources, e.pages, e.updates, e.recovery, e.sweeps} if !handler.WaitUntilSynced(ctx.Done()) { + e.publications.ShutDown() for _, queue := range queues { queue.ShutDownEarly() } return ctx.Err() } var workers sync.WaitGroup + workers.Go(func() { e.runPublications(ctx) }) for _, queue := range queues { workers.Go(func() { queue.Run(ctx.Done()) }) } diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go index 6d5c25db24..956e8c1e6b 100644 --- a/go/core/internal/controller/egress_test.go +++ b/go/core/internal/controller/egress_test.go @@ -30,13 +30,7 @@ import ( "github.com/kagent-dev/kagent/go/pkg/logging" ) -type revisionEgressFunc func(context.Context, string) error - -var noopRevisionEgress = revisionEgressFunc(func(context.Context, string) error { return nil }) - -func (f revisionEgressFunc) ReconcileRevision(ctx context.Context, revision string) error { - return f(ctx, revision) -} +func noopEnqueueEgress(string) {} type egressTestStore struct { mu sync.Mutex @@ -174,12 +168,12 @@ func TestEgressTranslationUsesCachedDependencies(t *testing.T) { base := egress.Access{Destinations: []string{"https://model.example.com:443"}} store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"pinned": {Kind: kind, Namespace: "team", SourceName: "source", SourceUID: "original", EgressDestinations: base.Destinations}}} controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) + done := make(chan error, 1) + go func() { done <- controller.Start(ctx) }() + defer func() { cancel(); require.NoError(t, <-done) }() expect := func(want egress.Access) { t.Helper() require.Eventually(t, func() bool { - if err := controller.ReconcileRevision(ctx, "pinned"); err != nil { - return false - } got, err := store.GetRuntimeEgressPolicy(ctx, "pinned") return err == nil && reflect.DeepEqual(want, got) }, time.Second, 10*time.Millisecond) @@ -310,13 +304,15 @@ func TestEgressSkipsUnchangedPoliciesAndRecoversLostWriteResponse(t *testing.T) for _, queue := range []controllers.Queue{controller.pages, controller.updates, controller.recovery, controller.sweeps} { workers.Go(func() { queue.Run(ctx.Done()) }) } + workers.Go(func() { controller.runPublications(ctx) }) defer func() { cancel(); workers.Wait() }() - require.ErrorContains(t, controller.ReconcileRevision(ctx, "revision"), "lost publication response") + require.True(t, controller.grants.WaitUntilSynced(ctx.Done())) + controller.EnqueueRevision("revision") synctest.Wait() require.Len(t, actors.attempts["session-actor"], 1, "an uncertain write must still apply the persisted policy") require.Contains(t, actors.applied["session-actor"].Destinations, "https://tickets.example.com:443") for range 3 { - require.NoError(t, controller.ReconcileRevision(ctx, "revision")) + controller.EnqueueRevision("revision") controller.sweeps.Add(egressSweepPage{}) synctest.Wait() } @@ -333,18 +329,48 @@ func TestEgressSkipsUnchangedPoliciesAndRecoversLostWriteResponse(t *testing.T) }) } -func TestEgressPublicationSerializesOnlyTheSameRevision(t *testing.T) { +func TestEgressStartupRecoveryRespectsWatchNamespaces(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + inputs := newEgressTestInputs(ctx) + store := &egressTestStore{ + pinned: map[string]database.RuntimeEgress{ + "watched": {Kind: database.RuntimeEgressAgent, Namespace: "team"}, + "unwatched": {Kind: database.RuntimeEgressAgent, Namespace: "other"}, + }, + intended: map[string]egress.Access{ + "watched": {}, + "unwatched": {Destinations: []string{"https://tickets.example.com:443"}}, + }, + targets: map[string][]database.RuntimeEgressTarget{ + "watched": {{ID: "watched", Revision: "watched", Atespace: "team", TemplateName: "runtime"}}, + "unwatched": {{ID: "unwatched", Revision: "unwatched", Atespace: "other", TemplateName: "runtime"}}, + }, + } + actors := &observingEgressActors{store: store} + controller := NewEgressReconciler(ctx, inputs.runtime, store, actors, []string{"team"}) + var workers sync.WaitGroup + workers.Go(func() { require.NoError(t, controller.Start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + require.Contains(t, actors.applied, "session-watched") + require.NotContains(t, actors.attempts, "session-unwatched") + unwatched, err := store.GetRuntimeEgressPolicy(ctx, "unwatched") + require.NoError(t, err) + require.Equal(t, []string{"https://tickets.example.com:443"}, unwatched.Destinations) + }) +} + +func TestEgressPublicationRequeuesRevocationDuringWrite(t *testing.T) { for _, failFirst := range []bool{false, true} { t.Run(fmt.Sprintf("fail_first=%t", failFirst), func(t *testing.T) { synctest.Test(t, func(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) - defer cancel() inputs := newEgressTestInputs(ctx) inputs.source(database.RuntimeEgressAgent, "uid", []corev1.LocalObjectReference{{Name: "access"}}) inputs.policies.UpdateObject(egressTestPolicy("https://tickets.example.com", nil)) pinned := database.RuntimeEgress{Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"} entered, release := make(chan struct{}), make(chan struct{}) - firstErr := errors.New("publication failed") store := &egressTestStore{ pinned: map[string]database.RuntimeEgress{"revision": pinned, "other": pinned}, beforePublish: func(ctx context.Context, revision string, access egress.Access) error { @@ -358,62 +384,78 @@ func TestEgressPublicationSerializesOnlyTheSameRevision(t *testing.T) { return ctx.Err() } if failFirst { - return firstErr + return errors.New("publication failed") } return nil }, } controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) + var workers sync.WaitGroup + workers.Go(func() { controller.runPublications(ctx) }) defer func() { + cancel() + workers.Wait() for _, queue := range []controllers.Queue{controller.sources, controller.pages, controller.updates, controller.recovery, controller.sweeps} { queue.ShutDownEarly() } }() - first, second, other := make(chan error, 1), make(chan error, 1), make(chan error, 1) - go func() { first <- controller.ReconcileRevision(ctx, "revision") }() + require.True(t, controller.grants.WaitUntilSynced(ctx.Done())) + controller.EnqueueRevision("revision") <-entered inputs.policies.DeleteObject("team/access") synctest.Wait() - go func() { second <- controller.ReconcileRevision(ctx, "revision") }() - go func() { other <- controller.ReconcileRevision(ctx, "other") }() - synctest.Wait() - require.Len(t, other, 1, "another revision must publish while the first database write is blocked") - require.NoError(t, <-other) - require.Empty(t, second, "the same revision must wait through commit") - - waitingCtx, stopWaiting := context.WithCancel(ctx) - canceled := make(chan error, 1) - go func() { canceled <- controller.ReconcileRevision(waitingCtx, "revision") }() + controller.EnqueueRevision("revision") + controller.EnqueueRevision("other") synctest.Wait() - stopWaiting() - synctest.Wait() - require.Len(t, canceled, 1, "a canceled waiter must leave without waiting for the writer") - require.ErrorIs(t, <-canceled, context.Canceled) + _, err := store.GetRuntimeEgressPolicy(ctx, "other") + require.NoError(t, err, "another revision must publish while the first database write is blocked") + _, err = store.GetRuntimeEgressPolicy(ctx, "revision") + require.ErrorIs(t, err, database.ErrNotFound, "the same revision must wait through commit") close(release) - if failFirst { - require.ErrorIs(t, <-first, firstErr) - } else { - require.NoError(t, <-first) - } - require.NoError(t, <-second) + synctest.Wait() for _, revision := range []string{"revision", "other"} { access, err := store.GetRuntimeEgressPolicy(ctx, revision) require.NoError(t, err) - require.Empty(t, access.Destinations, "publication must use grants read after acquiring the revision lock") + require.Empty(t, access.Destinations, "a dirty revision must rerun with the revoked grants") } - require.Empty(t, controller.publishing, "completed publications must release lock state") }) }) } } +func TestEgressPublicationRetriesWithoutAnotherEvent(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + inputs := newEgressTestInputs(ctx) + store := &egressTestStore{ + pinned: map[string]database.RuntimeEgress{"revision": {Kind: database.RuntimeEgressAgent, Namespace: "team"}}, + publishErr: errors.New("database unavailable"), + } + controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) + var workers sync.WaitGroup + workers.Go(func() { require.NoError(t, controller.Start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + _, err := store.GetRuntimeEgressPolicy(ctx, "revision") + require.ErrorIs(t, err, database.ErrNotFound) + store.mu.Lock() + store.publishErr = nil + store.mu.Unlock() + time.Sleep(time.Second) + synctest.Wait() + _, err = store.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + }) +} + func TestEgressTargetsSourceChangesAndBacksOffActorFailures(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() inputs := newEgressTestInputs(ctx) inputs.source(database.RuntimeEgressAgent, "uid", nil) store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"revision": {Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"}}, - listed: make(chan database.RuntimeEgressSource, 20), targets: map[string][]database.RuntimeEgressTarget{"revision": { + intended: map[string]egress.Access{"revision": {}}, + listed: make(chan database.RuntimeEgressSource, 20), targets: map[string][]database.RuntimeEgressTarget{"revision": { {ID: "failing", Revision: "revision", Atespace: "team", TemplateName: "runtime"}, {ID: "healthy", Revision: "revision", Atespace: "team", TemplateName: "runtime"}, }}} @@ -480,20 +522,31 @@ func TestEgressWithholdsConflictingGrantsAndWarns(t *testing.T) { inputs.secrets.UpdateObject(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "other"}, Data: map[string][]byte{"token": []byte("private-value")}}) store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"revision": pinned, "other": pinned}} controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) + defer controller.publications.ShutDown() + defer func() { + for _, queue := range []controllers.Queue{controller.sources, controller.pages, controller.updates, controller.recovery, controller.sweeps} { + queue.ShutDownEarly() + } + }() + require.True(t, controller.grants.WaitUntilSynced(ctx.Done())) + publish := func(revision string) { + controller.EnqueueRevision(revision) + require.True(t, controller.processPublication(ctx)) + } clock := clocktesting.NewFakeClock(time.Now()) controller.warnings = cache.NewLRUExpireCacheWithClock(1024, clock) for range 3 { - require.NoError(t, controller.ReconcileRevision(ctx, "revision")) + publish("revision") got, err := store.GetRuntimeEgressPolicy(ctx, "revision") require.NoError(t, err) require.Equal(t, base, got) } require.Equal(t, 1, strings.Count(logs.String(), "explicit egress grants withheld")) - require.NoError(t, controller.ReconcileRevision(ctx, "other")) + publish("other") require.Contains(t, logs.String(), `"revision":"other"`, "a noisy revision must not suppress another revision's warning") require.Equal(t, 2, strings.Count(logs.String(), "explicit egress grants withheld")) clock.Step(31 * time.Second) - require.NoError(t, controller.ReconcileRevision(ctx, "revision")) + publish("revision") require.Equal(t, 3, strings.Count(logs.String(), "explicit egress grants withheld")) require.Contains(t, logs.String(), `"level":"WARN"`) require.Contains(t, logs.String(), `"revision":"revision"`) diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index cedadd56cf..b4868070bf 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -27,6 +27,7 @@ import ( kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -54,7 +55,10 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { templates.template.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}, }} - reconciler := &Reconciler{egress: noopRevisionEgress, collections: collections, templates: templates, store: store} + reconciler := &Reconciler{enqueueEgress: func(revision string) { + _, err := store.SetRuntimeEgressPolicy(ctx, revision, egress.Access{}) + require.NoError(t, err) + }, collections: collections, templates: templates, store: store} require.NoError(t, reconciler.reconcileAgent(ctx, initial.ResourceName())) updatedHarness := harnesses.List()[0].DeepCopy() @@ -156,7 +160,7 @@ func TestPreparationErrorsPublishStatusAndRecover(t *testing.T) { collections, _ := newPreparationTestCollections(t, "microvm") initial := collections.Reconciliations.List()[0] statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() - reconciler := &Reconciler{egress: noopRevisionEgress, collections: collections, templates: &test.templates, store: &test.store, status: statusClient} + reconciler := &Reconciler{enqueueEgress: noopEnqueueEgress, collections: collections, templates: &test.templates, store: &test.store, status: statusClient} err := reconciler.reconcileAgent(t.Context(), initial.ResourceName()) require.Error(t, err) require.Equal(t, test.code, status.Code(err)) @@ -237,7 +241,7 @@ func TestPreparationRevisionIsolation(t *testing.T) { collections, harnesses := newPreparationTestCollections(t, "microvm") initial := collections.Reconciliations.List()[0] templates := &fakeActorTemplates{createErr: status.Error(codes.FailedPrecondition, `SandboxConfig "microvm" not found`)} - reconciler := &Reconciler{egress: noopRevisionEgress, collections: collections, templates: templates, store: &fakeRuntimeRevisionStore{}} + reconciler := &Reconciler{enqueueEgress: noopEnqueueEgress, collections: collections, templates: templates, store: &fakeRuntimeRevisionStore{}} require.Error(t, reconciler.reconcileAgent(t.Context(), initial.ResourceName())) waitFor(t, func() bool { return collections.Reconciliations.GetKey(initial.ResourceName()).PreparationFailure != nil @@ -279,7 +283,7 @@ func TestPreparationQueueAndPollerBackoff(t *testing.T) { initial := collections.Reconciliations.List()[0] statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() backend := &preparationTestBackend{templates: &test.templates, store: &test.store} - r := newReconciler(collections, backend, backend, statusClient, noopRevisionEgress) + r := newReconciler(collections, backend, backend, statusClient, noopEnqueueEgress) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -412,7 +416,7 @@ func TestPreparationDesiredChangesBypassBackoff(t *testing.T) { initial := collections.Reconciliations.List()[0] templates := &fakeActorTemplates{createErr: status.Error(codes.FailedPrecondition, "missing config")} store := &fakeRuntimeRevisionStore{} - r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3(), noopRevisionEgress) + r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3(), noopEnqueueEgress) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -466,7 +470,7 @@ func TestPreparationTerminalFailures(t *testing.T) { } store := &fakeRuntimeRevisionStore{} statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() - r := newReconciler(collections, templates, store, statusClient, noopRevisionEgress) + r := newReconciler(collections, templates, store, statusClient, noopEnqueueEgress) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -525,7 +529,7 @@ func TestInvalidActorTemplateHasNoCompiledTarget(t *testing.T) { require.Nil(t, state.PreparationFailure) store := &fakeRuntimeRevisionStore{} templates := &fakeActorTemplates{getErr: errors.New("Substrate must not be called for an invalid target")} - r := &Reconciler{egress: noopRevisionEgress, collections: collections, templates: templates, store: store} + r := &Reconciler{enqueueEgress: noopEnqueueEgress, collections: collections, templates: templates, store: store} require.NoError(t, r.reconcileAgent(t.Context(), initial.ResourceName())) require.Empty(t, store.pair.DesiredRevision) require.Nil(t, templates.template) @@ -581,32 +585,45 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections } func TestInitialRevisionWaitsForEgressPublication(t *testing.T) { - collections, _ := newPreparationTestCollections(t, "gvisor") - initial := collections.Reconciliations.List()[0] - template := proto.CloneOf(initial.Target.ActorTemplate) - template.Metadata.Uid = "prepared" - template.Status = sandboxGoldenStatus() - store := &fakeRuntimeRevisionStore{} - publishErr := errors.New("policy persistence unavailable") - r := &Reconciler{collections: collections, templates: &fakeActorTemplates{template: template}, store: store, - egress: revisionEgressFunc(func(_ context.Context, revision string) error { - require.NotNil(t, store.revision) - require.Equal(t, store.revision.Revision, revision) - require.False(t, store.markedSuccessful, "initial policy must precede readiness") - return publishErr - }), - } - require.ErrorIs(t, r.reconcileAgent(t.Context(), initial.ResourceName()), publishErr) - require.False(t, store.markedSuccessful) - waitFor(t, func() bool { - return collections.Reconciliations.GetKey(initial.ResourceName()).PreparationFailure != nil + synctest.Test(t, func(t *testing.T) { + collections, _ := newPreparationTestCollections(t, "gvisor") + initial := collections.Reconciliations.List()[0] + template := proto.CloneOf(initial.Target.ActorTemplate) + template.Metadata.Uid = "prepared" + template.Status = sandboxGoldenStatus() + store := &fakeRuntimeRevisionStore{promoteErr: database.ErrEgressPolicyNotPublished} + backend := &preparationTestBackend{templates: &fakeActorTemplates{template: template}, store: store} + queued := 0 + r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3(), func(revision string) { + backend.inspect(func() { + require.NotNil(t, store.revision) + require.Equal(t, store.revision.Revision, revision) + queued++ + }) + }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var workers sync.WaitGroup + workers.Go(func() { r.Run(ctx.Done()) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + backend.inspect(func() { + require.Equal(t, 1, queued) + require.False(t, store.markedSuccessful, "enqueueing publication must not make the revision ready") + }) + require.NotNil(t, collections.Reconciliations.GetKey(initial.ResourceName()).PreparationFailure) + time.Sleep(500 * time.Millisecond) + synctest.Wait() + backend.inspect(func() { + require.Equal(t, 1, store.recordCalls, "pending publication must retry with backoff") + store.promoteErr = nil // Publication completes independently of preparation. + }) + time.Sleep(500 * time.Millisecond) + synctest.Wait() + backend.inspect(func() { + require.True(t, store.markedSuccessful, "preparation must recover without another Kubernetes event") + require.Equal(t, 2, store.recordCalls, "each attempt must record inputs only once") + }) + require.Nil(t, collections.Reconciliations.GetKey(initial.ResourceName()).PreparationFailure) }) - publishErr = nil - store.promoteErr = errors.New("promotion unavailable") - require.ErrorIs(t, r.reconcileAgent(t.Context(), initial.ResourceName()), store.promoteErr) - require.False(t, store.markedSuccessful) - store.promoteErr = nil - require.NoError(t, r.reconcileAgent(t.Context(), initial.ResourceName())) - require.True(t, store.markedSuccessful) - require.Equal(t, 3, store.recordCalls, "each attempt must record inputs only once") } diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index 3ffa625518..ab92ae8c12 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -180,11 +180,11 @@ type actorTemplateClient interface { // Reconciler is the side-effect boundary for the pure KRT graph. Collection // handlers enqueue stable keys; retries always read the latest derived state. type Reconciler struct { - collections Collections - templates actorTemplateClient - store runtimeRevisionStore - egress revisionEgressReconciler - status kagentclient.ApiV1alpha3Interface + collections Collections + templates actorTemplateClient + store runtimeRevisionStore + enqueueEgress func(string) + status kagentclient.ApiV1alpha3Interface agents controllers.Queue agentStatuses controllers.Queue @@ -196,12 +196,12 @@ type Reconciler struct { // NewReconciler creates the Kubernetes and database write boundary. Run starts // its queues after the registered KRT handlers have received initial state. -func NewReconciler(config *rest.Config, collections Collections, store runtimeRevisionStore, templates actorTemplateClient, egress revisionEgressReconciler) (*Reconciler, error) { +func NewReconciler(config *rest.Config, collections Collections, store runtimeRevisionStore, templates actorTemplateClient, enqueueEgress func(string)) (*Reconciler, error) { statusClient, err := kagentclient.NewForConfig(config) if err != nil { return nil, fmt.Errorf("create kagent status client: %w", err) } - return newReconciler(collections, templates, store, statusClient, egress), nil + return newReconciler(collections, templates, store, statusClient, enqueueEgress), nil } func newReconciler( @@ -209,14 +209,14 @@ func newReconciler( templates actorTemplateClient, store runtimeRevisionStore, status kagentclient.ApiV1alpha3Interface, - egress revisionEgressReconciler, + enqueueEgress func(string), ) *Reconciler { r := &Reconciler{ - collections: collections, - templates: templates, - store: store, - egress: egress, - status: status, + collections: collections, + templates: templates, + store: store, + enqueueEgress: enqueueEgress, + status: status, } r.agents = newReconciliationQueue("agents", func(item any) error { return r.reconcileAgent(context.Background(), item.(string)) @@ -381,9 +381,7 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { if err := r.store.RecordRuntimeRevision(ctx, revision); err != nil { return r.observePreparationError(*state, fmt.Errorf("store runtime revision %s: %w", target.RevisionID, err)) } - if err := r.egress.ReconcileRevision(ctx, revision.Revision); err != nil { - return r.observePreparationError(*state, fmt.Errorf("publish runtime egress %s: %w", target.RevisionID, err)) - } + r.enqueueEgress(revision.Revision) if ready { if err := r.store.PromoteRuntimeRevision(ctx, revision.Revision); err != nil { return r.observePreparationError(*state, fmt.Errorf("promote runtime revision %s: %w", target.RevisionID, err)) diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index 5b49417dc7..1be613fdc1 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -26,6 +26,7 @@ import ( kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -55,7 +56,7 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { store := &fakeRuntimeRevisionStore{} templates := &fakeActorTemplates{} statusClient := kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3() - reconciler := &Reconciler{egress: noopRevisionEgress, + reconciler := &Reconciler{enqueueEgress: noopEnqueueEgress, collections: Collections{ Agents: krttest.GetMockCollection[*kagentv1alpha3.Agent](mock), AgentRuntimeObservations: krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...), @@ -109,7 +110,7 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { } // A fresh reconciler must clean up observations without remembering earlier calls. - reconciler = &Reconciler{egress: noopRevisionEgress, collections: reconciler.collections, templates: templates, store: store, status: statusClient} + reconciler = &Reconciler{enqueueEgress: noopEnqueueEgress, collections: reconciler.collections, templates: templates, store: store, status: statusClient} nextTarget := *state.Target state.Target = &nextTarget state.Target.ActorTemplate = proto.CloneOf(state.Target.ActorTemplate) @@ -157,7 +158,7 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) states := krt.NewStaticCollection[AgentReconciliation](nil, nil, opts.WithName("Reconciliations")...) templates := &fakeActorTemplates{} - reconciler := &Reconciler{egress: noopRevisionEgress, + reconciler := &Reconciler{enqueueEgress: noopEnqueueEgress, collections: Collections{ AgentRuntimeObservations: krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...), Reconciliations: states, @@ -181,6 +182,9 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { Target: &compiledTarget{Revision: *revision, RevisionID: id, ActorTemplate: desired}, } states.UpdateObject(state) + require.ErrorIs(t, reconciler.reconcileAgent(ctx, state.ResourceName()), database.ErrEgressPolicyNotPublished) + _, err = store.SetRuntimeEgressPolicy(ctx, id.String(), egress.Access{}) + require.NoError(t, err) require.NoError(t, reconciler.reconcileAgent(ctx, state.ResourceName())) request := &apiv1.Session{ @@ -404,7 +408,7 @@ func TestReconcilerUpdatesModelConfigStatusOnSecretHashChange(t *testing.T) { &fakeActorTemplates{}, &fakeRuntimeRevisionStore{}, statusClient, - noopRevisionEgress, + noopEnqueueEgress, ) go reconciler.Run(stop) diff --git a/go/core/internal/controller/sandbox_integration_test.go b/go/core/internal/controller/sandbox_integration_test.go index b4e9a6bed1..4e086ec507 100644 --- a/go/core/internal/controller/sandbox_integration_test.go +++ b/go/core/internal/controller/sandbox_integration_test.go @@ -65,7 +65,7 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { Secrets: typedCollection[*corev1.Secret](runtimeClient, []string{"team-a"}, "Secrets", options), SandboxTemplates: typedCollection[*kagentv1alpha3.SandboxTemplate](runtimeClient, []string{"team-a"}, "SandboxTemplates", options), WorkerPools: pools, }} - reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, noopRevisionEgress) + reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, noopEnqueueEgress) require.NoError(t, err) results := make(chan error, 2) go func() { results <- runtime.Start(ctx) }() diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index 91f29f18e7..996609c82b 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -40,21 +40,21 @@ type sandboxRevisionStore interface { // SandboxReconciler owns the queued side effects of SandboxTemplate preparation. // Sandbox instance lifecycle remains owned by the PostgreSQL-backed worker. type SandboxReconciler struct { - collections sandboxCollections - store sandboxRevisionStore - egress revisionEgressReconciler - actors actorTemplateClient - client kagentclient.ApiV1alpha3Interface + collections sandboxCollections + store sandboxRevisionStore + enqueueEgress func(string) + actors actorTemplateClient + client kagentclient.ApiV1alpha3Interface } var _ manager.LeaderElectionRunnable = (*SandboxReconciler)(nil) -func NewSandboxReconciler(config *rest.Config, runtime *Runtime, store sandboxRevisionStore, actors actorTemplateClient, policy substrate.SandboxPolicy, egress revisionEgressReconciler) (*SandboxReconciler, error) { +func NewSandboxReconciler(config *rest.Config, runtime *Runtime, store sandboxRevisionStore, actors actorTemplateClient, policy substrate.SandboxPolicy, enqueueEgress func(string)) (*SandboxReconciler, error) { client, err := kagentclient.NewForConfig(config) if err != nil { return nil, fmt.Errorf("create SandboxTemplate client: %w", err) } - return &SandboxReconciler{collections: newSandboxCollections(runtime.Collections, policy, runtime.Options), store: store, actors: actors, client: client, egress: egress}, nil + return &SandboxReconciler{collections: newSandboxCollections(runtime.Collections, policy, runtime.Options), store: store, actors: actors, client: client, enqueueEgress: enqueueEgress}, nil } func (s *SandboxReconciler) NeedLeaderElection() bool { return true } @@ -180,9 +180,7 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { if err := s.store.RecordSandboxRevision(ctx, revision); err != nil { return s.observeError(*state, fmt.Errorf("store sandbox revision %s: %w", state.RevisionID, err), true) } - if err := s.egress.ReconcileRevision(ctx, revision.Revision); err != nil { - return s.observeError(*state, fmt.Errorf("publish sandbox egress %s: %w", state.RevisionID, err), true) - } + s.enqueueEgress(revision.Revision) if ready { if err := s.store.PromoteSandboxRevision(ctx, revision.Revision); err != nil { return s.observeError(*state, fmt.Errorf("promote sandbox revision %s: %w", state.RevisionID, err), true) diff --git a/go/core/internal/controller/sandbox_reconciler_test.go b/go/core/internal/controller/sandbox_reconciler_test.go index ab239ef9ba..6c70a14063 100644 --- a/go/core/internal/controller/sandbox_reconciler_test.go +++ b/go/core/internal/controller/sandbox_reconciler_test.go @@ -140,9 +140,9 @@ func newSandboxTestReconciler(t *testing.T, guestImage string) (*SandboxReconcil store := &sandboxTestStore{} actors := &sandboxTestActors{store: store, templates: map[string]*ateapipb.ActorTemplate{}} reconciler := &SandboxReconciler{ - collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools}, substrate.SandboxPolicy{GuestImage: guestImage, CPU: "1", Memory: "1Gi"}, opts), - egress: noopRevisionEgress, - store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3(), + collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools}, substrate.SandboxPolicy{GuestImage: guestImage, CPU: "1", Memory: "1Gi"}, opts), + enqueueEgress: noopEnqueueEgress, + store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3(), } waitFor(t, func() bool { return reconciler.collections.states.GetKey("team-a/scratch") != nil }) return reconciler, templates, pools @@ -318,16 +318,14 @@ func TestInitialSandboxRevisionWaitsForEgressPublication(t *testing.T) { s.actors.(*sandboxTestActors).autoReady = true require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) syncSandboxTemplate(t, s, templates) - publishErr := errors.New("policy persistence unavailable") - s.egress = revisionEgressFunc(func(_ context.Context, revision string) error { + store.promoteErr = database.ErrEgressPolicyNotPublished + s.enqueueEgress = func(revision string) { require.Equal(t, store.revision.Revision, revision) require.False(t, store.ready, "initial policy must precede readiness") - return publishErr - }) - require.ErrorIs(t, s.reconcile(t.Context(), "team-a/scratch"), publishErr) + } + require.ErrorIs(t, s.reconcile(t.Context(), "team-a/scratch"), database.ErrEgressPolicyNotPublished) require.False(t, store.ready) waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch").Failure != nil }) - publishErr = nil store.promoteErr = errors.New("promotion unavailable") require.ErrorIs(t, s.reconcile(t.Context(), "team-a/scratch"), store.promoteErr) require.False(t, store.ready) diff --git a/go/core/internal/database/client_session_test.go b/go/core/internal/database/client_session_test.go index d432d1cc0e..163f23b4b9 100644 --- a/go/core/internal/database/client_session_test.go +++ b/go/core/internal/database/client_session_test.go @@ -15,6 +15,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) func TestMalformedDatabaseIDsReturnErrors(t *testing.T) { @@ -380,6 +381,8 @@ func TestForkSessionCopiesBoundedHistory(t *testing.T) { if err := client.RecordRuntimeRevision(ctx, revision); err != nil { t.Fatal(err) } + _, policyErr := client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) + require.NoError(t, policyErr) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) source, _, err := client.CreateSession(ctx, &apiv1.Session{ @@ -540,6 +543,8 @@ func TestSessionCreateAndTransitions(t *testing.T) { if err := client.RecordRuntimeRevision(ctx, revision); err != nil { t.Fatal(err) } + _, policyErr := client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) + require.NoError(t, policyErr) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) storedRevision, err := client.GetRuntimeRevision(ctx, revision.Revision) @@ -643,6 +648,8 @@ func sessionFixture(t *testing.T, client *Client, ctx context.Context, namespace if err := client.RecordRuntimeRevision(ctx, revision); err != nil { t.Fatal(err) } + _, policyErr := client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) + require.NoError(t, policyErr) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) } diff --git a/go/core/internal/database/errors.go b/go/core/internal/database/errors.go index 3184e03884..bb08026ef9 100644 --- a/go/core/internal/database/errors.go +++ b/go/core/internal/database/errors.go @@ -21,3 +21,7 @@ var ( // ErrObjectDeleting reports that deletion has started. The object is unavailable // to new callers, so it also matches ErrNotFound. var ErrObjectDeleting = fmt.Errorf("object is being deleted: %w", ErrNotFound) + +// ErrEgressPolicyNotPublished asks preparation to retry promotion after the +// controller has published the revision's initial intended policy. +var ErrEgressPolicyNotPublished = fmt.Errorf("egress policy is not published: %w", ErrFailedPrecondition) diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index 2dd064ea43..705afac869 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -17,6 +17,8 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", TokenPassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} require.NoError(t, client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", DesiredRevision: "agent"})) require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) + _, policyErr := client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) + require.NoError(t, policyErr) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) instance, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "request") require.NoError(t, err) diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index 49505b376b..823b75a5fb 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -97,9 +97,10 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi } // PromoteRuntimeRevision makes a recorded, prepared revision available only to -// the active Agent identity that still desires it. Callers must publish its -// intended egress policy first. Missing/deleting revisions return ErrNotFound / -// ErrObjectDeleting; stale reports leave the definition unchanged. +// the active Agent identity that still desires it. Without a published intended +// policy it returns ErrEgressPolicyNotPublished for retry. Missing/deleting +// revisions return ErrNotFound / ErrObjectDeleting; stale reports leave the +// definition unchanged. func (c *Client) PromoteRuntimeRevision(ctx context.Context, revision string) error { return c.withTx(ctx, func(tx pgx.Tx) error { // Match GC finalization's lock order: definition before revision. @@ -114,17 +115,30 @@ func (c *Client) PromoteRuntimeRevision(ctx context.Context, revision string) er if err := lockRuntimeRevisionForReference(ctx, tx, revision, runtimeKindAgent); err != nil { return err } - return execSQL(ctx, tx, ` - UPDATE agent_definition p - SET latest_successful_revision = r.revision, updated_at = NOW() - FROM agent_runtime_revision r - WHERE r.revision = $1 - AND p.namespace = r.namespace - AND p.agent_uid = r.agent_uid - AND p.desired_revision = r.revision - AND p.retired_at IS NULL - AND p.latest_successful_revision IS DISTINCT FROM r.revision - `, revision) + published, err := queryOne(ctx, tx, ` + WITH published AS ( + SELECT revision FROM runtime_egress_policy WHERE revision = $1 + ), promoted AS ( + UPDATE agent_definition p + SET latest_successful_revision = r.revision, updated_at = NOW() + FROM agent_runtime_revision r + WHERE r.revision = $1 + AND p.namespace = r.namespace + AND p.agent_uid = r.agent_uid + AND p.desired_revision = r.revision + AND p.retired_at IS NULL + AND p.latest_successful_revision IS DISTINCT FROM r.revision + AND EXISTS (SELECT 1 FROM published) + ) + SELECT EXISTS (SELECT 1 FROM published) + `, pgx.RowTo[bool], revision) + if err != nil { + return err + } + if !published { + return ErrEgressPolicyNotPublished + } + return nil }) } diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index 9ec73c074f..0e9d69c56c 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -160,6 +160,8 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { request := newSessionRequest(uuid.NewString(), "assistant", "kagent", "") _, _, err = client.CreateSession(ctx, request, "session") require.ErrorIs(t, err, ErrNotFound, "a replacement must not select the previous template UID") + _, policyErr := client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) + require.NoError(t, policyErr) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) session, _, err := client.CreateSession(ctx, request, "session") require.NoError(t, err) @@ -493,12 +495,20 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { require.Equal(t, "original", stored.AgentCard.Name) require.ErrorIs(t, c.PromoteRuntimeRevision(ctx, "missing"), ErrNotFound) + require.ErrorIs(t, c.PromoteRuntimeRevision(ctx, revision.Revision), ErrEgressPolicyNotPublished) + assertAvailableRevision("") + _, err = c.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) + require.NoError(t, err) require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("first") pair.DesiredRevision = "second" require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) revision.Revision, revision.ActorTemplateName = "second", "second" require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) + require.ErrorIs(t, c.PromoteRuntimeRevision(ctx, revision.Revision), ErrEgressPolicyNotPublished) + assertAvailableRevision("first") + _, err = c.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) + require.NoError(t, err) require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("second") // A delayed report for the old revision must not replace the newer ready revision. @@ -510,6 +520,8 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { require.NoError(t, c.RetireAgentIdentities(ctx, pair.Namespace, pair.AgentName, nil)) revision.Revision, revision.ActorTemplateName = "third", "third" require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) + _, err = c.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) + require.NoError(t, err) require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("") // Reviving the pair must retain the last success, not a report made while retired. diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index 961b796acf..9e1d16138c 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -88,9 +88,10 @@ func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevi } // PromoteSandboxRevision makes a recorded, prepared revision available only to -// the active template identity that still desires it. Callers must publish its -// intended egress policy first. Missing/deleting revisions return ErrNotFound / -// ErrObjectDeleting; stale reports leave the definition unchanged. +// the active template identity that still desires it. Without a published intended +// policy it returns ErrEgressPolicyNotPublished for retry. Missing/deleting +// revisions return ErrNotFound / ErrObjectDeleting; stale reports leave the +// definition unchanged. func (c *Client) PromoteSandboxRevision(ctx context.Context, revision string) error { return c.withTx(ctx, func(tx pgx.Tx) error { // Match GC finalization's lock order: definition before revision. @@ -105,13 +106,26 @@ func (c *Client) PromoteSandboxRevision(ctx context.Context, revision string) er if err := lockRuntimeRevisionForReference(ctx, tx, revision, runtimeKindSandbox); err != nil { return err } - return execSQL(ctx, tx, ` - UPDATE sandbox_template_definition p SET latest_successful_revision = r.revision, updated_at = NOW() - FROM runtime_revision r JOIN sandbox_revision s USING (revision) - WHERE r.revision = $1 AND p.namespace = r.namespace AND p.sandbox_template_uid = s.sandbox_template_uid - AND p.desired_revision = r.revision AND p.retired_at IS NULL - AND p.latest_successful_revision IS DISTINCT FROM r.revision - `, revision) + published, err := queryOne(ctx, tx, ` + WITH published AS ( + SELECT revision FROM runtime_egress_policy WHERE revision = $1 + ), promoted AS ( + UPDATE sandbox_template_definition p SET latest_successful_revision = r.revision, updated_at = NOW() + FROM runtime_revision r JOIN sandbox_revision s USING (revision) + WHERE r.revision = $1 AND p.namespace = r.namespace AND p.sandbox_template_uid = s.sandbox_template_uid + AND p.desired_revision = r.revision AND p.retired_at IS NULL + AND p.latest_successful_revision IS DISTINCT FROM r.revision + AND EXISTS (SELECT 1 FROM published) + ) + SELECT EXISTS (SELECT 1 FROM published) + `, pgx.RowTo[bool], revision) + if err != nil { + return err + } + if !published { + return ErrEgressPolicyNotPublished + } + return nil }) } diff --git a/go/core/internal/database/sandboxes_test.go b/go/core/internal/database/sandboxes_test.go index 083872bfe7..d771df10e0 100644 --- a/go/core/internal/database/sandboxes_test.go +++ b/go/core/internal/database/sandboxes_test.go @@ -11,6 +11,7 @@ import ( "google.golang.org/protobuf/proto" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) func sandboxFixture(t *testing.T, client *Client, revision string) (*apiv1.Sandbox, SandboxCreateOptions) { @@ -19,6 +20,8 @@ func sandboxFixture(t *testing.T, client *Client, revision string) (*apiv1.Sandb require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision})) require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: revision, Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: revision, ActorTemplateUID: "actor-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)})) + _, policyErr := client.SetRuntimeEgressPolicy(ctx, revision, egress.Access{}) + require.NoError(t, policyErr) require.NoError(t, client.PromoteSandboxRevision(ctx, revision)) hash := sha256.Sum256([]byte("request")) return &apiv1.Sandbox{Id: uuid.NewString(), Creator: "alice", SandboxTemplate: &apiv1.ResourceReference{Namespace: "team-a", Name: "scratch"}, Name: "experiment"}, @@ -49,6 +52,10 @@ func TestPromoteSandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { } assertAvailable("first") require.ErrorIs(t, client.PromoteSandboxRevision(ctx, "missing"), ErrNotFound) + require.ErrorIs(t, client.PromoteSandboxRevision(ctx, "second"), ErrEgressPolicyNotPublished) + assertAvailable("first") + _, err = client.SetRuntimeEgressPolicy(ctx, "second", egress.Access{}) + require.NoError(t, err) require.NoError(t, client.PromoteSandboxRevision(ctx, "second")) assertAvailable("second") require.NoError(t, client.PromoteSandboxRevision(ctx, "first")) diff --git a/go/core/internal/grpcserver/scheduledrun_test.go b/go/core/internal/grpcserver/scheduledrun_test.go index 6838598e8c..1eecccc5ed 100644 --- a/go/core/internal/grpcserver/scheduledrun_test.go +++ b/go/core/internal/grpcserver/scheduledrun_test.go @@ -26,6 +26,7 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" authimpl "github.com/kagent-dev/kagent/go/core/internal/httpserver/auth" "github.com/kagent-dev/kagent/go/core/internal/service/scheduledrun" sessionsvc "github.com/kagent-dev/kagent/go/core/internal/service/session" @@ -186,6 +187,8 @@ func scheduledRunTestServer(t *testing.T) (*database.Client, apiv1.ScheduledRunS SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: "runtime-uid", })) + _, policyErr := store.SetRuntimeEgressPolicy(t.Context(), pair.DesiredRevision, egress.Access{}) + require.NoError(t, policyErr) require.NoError(t, store.PromoteRuntimeRevision(t.Context(), pair.DesiredRevision)) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) diff --git a/go/core/internal/grpcserver/task_store_test.go b/go/core/internal/grpcserver/task_store_test.go index 50b1732a05..bbf5b1000e 100644 --- a/go/core/internal/grpcserver/task_store_test.go +++ b/go/core/internal/grpcserver/task_store_test.go @@ -43,6 +43,7 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/a2agateway" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" authimpl "github.com/kagent-dev/kagent/go/core/internal/httpserver/auth" sessionsvc "github.com/kagent-dev/kagent/go/core/internal/service/session" "github.com/kagent-dev/kagent/go/core/internal/service/taskstore" @@ -642,6 +643,8 @@ func createTaskStoreSession(t *testing.T, store *database.Client) *apiv1.Session DesiredRevision: revision.Revision, })) require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision)) + _, policyErr := store.SetRuntimeEgressPolicy(t.Context(), revision.Revision, egress.Access{}) + require.NoError(t, policyErr) require.NoError(t, store.PromoteRuntimeRevision(t.Context(), revision.Revision)) session, _, err := store.CreateSession(t.Context(), &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", diff --git a/go/core/internal/service/sandbox/guest_test.go b/go/core/internal/service/sandbox/guest_test.go index 1bfd364431..0b69885cee 100644 --- a/go/core/internal/service/sandbox/guest_test.go +++ b/go/core/internal/service/sandbox/guest_test.go @@ -28,6 +28,7 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" sandboxservice "github.com/kagent-dev/kagent/go/core/internal/service/sandbox" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/pkg/auth" @@ -134,6 +135,8 @@ func guestFixture(t *testing.T) (*sandboxservice.Service, context.Context, *test RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), })) + _, policyErr := store.SetRuntimeEgressPolicy(ctx, "revision", egress.Access{}) + require.NoError(t, policyErr) require.NoError(t, store.PromoteSandboxRevision(ctx, "revision")) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) diff --git a/go/core/internal/service/sandbox/service_test.go b/go/core/internal/service/sandbox/service_test.go index 19a0f9cd01..7a09fecd15 100644 --- a/go/core/internal/service/sandbox/service_test.go +++ b/go/core/internal/service/sandbox/service_test.go @@ -22,6 +22,7 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/pkg/auth" ) @@ -139,6 +140,8 @@ func serviceFixture(t *testing.T) (*Service, context.Context, *testActors) { RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), })) + _, policyErr := store.SetRuntimeEgressPolicy(ctx, "revision", egress.Access{}) + require.NoError(t, policyErr) require.NoError(t, store.PromoteSandboxRevision(ctx, "revision")) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) @@ -297,6 +300,8 @@ func TestSandboxResolvesCurrentGrantsBeforeResume(t *testing.T) { store := service.config.Store require.NoError(t, store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: revision.Revision})) require.NoError(t, store.RecordSandboxRevision(ctx, revision)) + _, policyErr := store.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) + require.NoError(t, policyErr) require.NoError(t, store.PromoteSandboxRevision(ctx, revision.Revision)) instance, err := service.Create(ctx, createRequest()) require.NoError(t, err) diff --git a/go/core/internal/service/session/workflow_test.go b/go/core/internal/service/session/workflow_test.go index 0550d15b9d..0ab947325f 100644 --- a/go/core/internal/service/session/workflow_test.go +++ b/go/core/internal/service/session/workflow_test.go @@ -150,6 +150,8 @@ func lifecycleFixture(t *testing.T) (*lifecycleTestStore, *apiv1.Session) { } require.NoError(t, client.UpsertAgentDefinition(t.Context(), database.AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision})) require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision)) + _, policyErr := client.SetRuntimeEgressPolicy(t.Context(), revision.Revision, egress.Access{}) + require.NoError(t, policyErr) require.NoError(t, client.PromoteRuntimeRevision(t.Context(), revision.Revision)) session, _, err := client.CreateSession(t.Context(), &apiv1.Session{Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}}, uuid.NewString()) require.NoError(t, err) diff --git a/go/core/pkg/app/app.go b/go/core/pkg/app/app.go index 21ca283250..836d70d215 100644 --- a/go/core/pkg/app/app.go +++ b/go/core/pkg/app/app.go @@ -291,7 +291,7 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er if err := manager.Add(egressReconciler); err != nil { return fmt.Errorf("add egress reconciler: %w", err) } - reconciler, err := controller.NewReconciler(kubeConfig, runtime.Collections, store, actors, egressReconciler) + reconciler, err := controller.NewReconciler(kubeConfig, runtime.Collections, store, actors, egressReconciler.EnqueueRevision) if err != nil { return err } @@ -379,7 +379,7 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er CPU: cfg.GetSandboxes().GetCpu(), Memory: cfg.GetSandboxes().GetMemory(), } - preparation, err := controller.NewSandboxReconciler(kubeConfig, runtime, store, actors, policy, egressReconciler) + preparation, err := controller.NewSandboxReconciler(kubeConfig, runtime, store, actors, policy, egressReconciler.EnqueueRevision) if err != nil { return err } From ca099500d2bf28407e4eef885aa6da4415532dfa Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 18:48:20 +0000 Subject: [PATCH 24/43] refactor: record revisions and initial egress policies atomically Signed-off-by: Eitan Yarmush --- docs/architecture/egress-policies.md | 21 +++-- go/core/internal/controller/egress_test.go | 32 +++++++ go/core/internal/controller/equality_test.go | 3 + .../internal/controller/preparation_test.go | 88 ++++++++----------- go/core/internal/controller/reconciler.go | 16 +++- .../internal/controller/reconciler_test.go | 7 +- .../controller/sandbox_collections.go | 5 +- .../internal/controller/sandbox_reconciler.go | 9 +- .../controller/sandbox_reconciler_test.go | 60 ++++++++----- .../internal/database/client_session_test.go | 12 +-- go/core/internal/database/errors.go | 4 - go/core/internal/database/protobuf_test.go | 5 +- .../internal/database/runtime_egress_test.go | 77 ++++++++++++++-- .../database/runtime_lifecycle_test.go | 4 +- .../internal/database/runtime_revisions.go | 82 ++++++++--------- .../database/runtime_revisions_test.go | 29 ++---- .../internal/database/sandbox_revisions.go | 45 ++++------ go/core/internal/database/sandboxes_test.go | 12 +-- .../internal/grpcserver/scheduledrun_test.go | 4 +- .../internal/grpcserver/task_store_test.go | 4 +- .../internal/service/sandbox/guest_test.go | 4 +- .../internal/service/sandbox/service_test.go | 8 +- .../internal/service/session/workflow_test.go | 4 +- go/core/internal/translator/compiler.go | 1 + go/core/internal/translator/egress_test.go | 6 +- go/core/internal/translator/revision.go | 7 +- 26 files changed, 317 insertions(+), 232 deletions(-) diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index 34f941ca2a..bfc2e40850 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -161,15 +161,18 @@ Secret values remain outside compiled inputs; the gateway handles value rotation The elected controller compiles current grants from cached, dependency-tracked Kubernetes inputs and persists the intended policy for each retained revision. -This mutable output is stored separately from immutable revision inputs. Preparation, -source events, and sweeps enqueue revision IDs into one publication queue. It -serializes writes for each revision and reruns with the latest grants when an -event arrives during publication; unrelated revisions can publish concurrently. -The store prevents promotion until an initial policy has been published. -Preparation retries with backoff while publication is pending, so initial -readiness may wait an extra queue hop and retry interval. -The grants cache can briefly lag initial compilation, so a new workload may -become ready with only pinned integration access until its grants event arrives. +This mutable output is stored separately from immutable revision inputs. +Preparation carries the explicit grants from the same KRT compilation as the +revision, merges them with pinned access, and records the revision and initial +policy in one transaction. Repeated preparation preserves the existing policy; +only the egress reconciler updates it. Promotion remains a separate step once +the runtime is prepared and does not wait for the publication queue. + +After recording commits, preparation enqueues the revision to catch grants that +changed while source-event discovery could not yet see the new revision. Source +events and sweeps use the same publication queue. It serializes writes for each +revision and reruns with the latest grants when an event arrives during +publication; unrelated revisions can publish concurrently. Each revision merges its pinned integration access with the current grants of its owning Agent or SandboxTemplate, even when that source's next runtime revision cannot be prepared. diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go index 956e8c1e6b..d954324ba3 100644 --- a/go/core/internal/controller/egress_test.go +++ b/go/core/internal/controller/egress_test.go @@ -448,6 +448,38 @@ func TestEgressPublicationRetriesWithoutAnotherEvent(t *testing.T) { }) } +func TestEgressPostCommitEnqueueRecoversEventBeforeRevisionExists(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + inputs := newEgressTestInputs(ctx) + inputs.source(database.RuntimeEgressAgent, "uid", []corev1.LocalObjectReference{{Name: "access"}}) + inputs.policies.UpdateObject(egressTestPolicy("https://tickets.example.com", nil)) + store := &egressTestStore{listed: make(chan database.RuntimeEgressSource, 1)} + controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) + require.True(t, controller.grants.WaitUntilSynced(ctx.Done())) + var workers sync.WaitGroup + workers.Go(func() { require.NoError(t, controller.Start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + // Compilation saw the grant, but its revision has not committed yet. + initial := controller.grants.GetKey("agent/team/source").Access + require.NotEmpty(t, initial.Destinations) + inputs.policies.DeleteObject("team/access") + synctest.Wait() + require.Len(t, store.listed, 1, "revocation discovery must run before the revision exists") + require.Equal(t, database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team", Name: "source"}, <-store.listed) + store.mu.Lock() + store.pinned = map[string]database.RuntimeEgress{"revision": {Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"}} + store.intended = map[string]egress.Access{"revision": initial} + store.mu.Unlock() + controller.EnqueueRevision("revision") + synctest.Wait() + policy, err := store.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + require.Empty(t, policy.Destinations, "post-commit publication must apply the missed revocation") + }) +} + func TestEgressTargetsSourceChangesAndBacksOffActorFailures(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() diff --git a/go/core/internal/controller/equality_test.go b/go/core/internal/controller/equality_test.go index 6de9fe439f..373b37a3af 100644 --- a/go/core/internal/controller/equality_test.go +++ b/go/core/internal/controller/equality_test.go @@ -55,6 +55,9 @@ func TestAgentReconciliationEquality(t *testing.T) { proto.Size(r.ObservedActorTemplate) }, equal: true}, {name: "template source", change: func(r *AgentReconciliation) { r.Agent.Generation++ }}, + {name: "live grants", change: func(r *AgentReconciliation) { + r.Target.Egress.Destinations = []string{"https://tickets.example.com:443"} + }}, {name: "revision identity", change: func(r *AgentReconciliation) { r.Target.RevisionID[0]++ }}, {name: "revision inputs", change: func(r *AgentReconciliation) { r.Target.Revision.SandboxClass = "microvm" }}, {name: "agent card", change: func(r *AgentReconciliation) { r.Target.Revision.AgentCard.Name = "changed" }}, diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index b4868070bf..63adb535eb 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -55,10 +55,7 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { templates.template.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}, }} - reconciler := &Reconciler{enqueueEgress: func(revision string) { - _, err := store.SetRuntimeEgressPolicy(ctx, revision, egress.Access{}) - require.NoError(t, err) - }, collections: collections, templates: templates, store: store} + reconciler := &Reconciler{enqueueEgress: noopEnqueueEgress, collections: collections, templates: templates, store: store} require.NoError(t, reconciler.reconcileAgent(ctx, initial.ResourceName())) updatedHarness := harnesses.List()[0].DeepCopy() @@ -101,7 +98,7 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { // A delayed success for the abandoned preparation must not replace A. abandoned, err := store.GetRuntimeRevision(ctx, preparing.Target.RevisionID.String()) require.NoError(t, err) - require.NoError(t, store.RecordRuntimeRevision(ctx, *abandoned)) + require.NoError(t, store.RecordRuntimeRevision(ctx, *abandoned, egress.Access{})) require.NoError(t, store.PromoteRuntimeRevision(ctx, abandoned.Revision)) unreferenced, err = store.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) @@ -392,10 +389,10 @@ func (p *preparationTestBackend) UpsertAgentDefinition(ctx context.Context, defi return p.store.UpsertAgentDefinition(ctx, definition) } -func (p *preparationTestBackend) RecordRuntimeRevision(ctx context.Context, revision database.RuntimeRevision) error { +func (p *preparationTestBackend) RecordRuntimeRevision(ctx context.Context, revision database.RuntimeRevision, policy egress.Access) error { p.mu.Lock() defer p.mu.Unlock() - return p.store.RecordRuntimeRevision(ctx, revision) + return p.store.RecordRuntimeRevision(ctx, revision, policy) } func (p *preparationTestBackend) PromoteRuntimeRevision(ctx context.Context, revision string) error { @@ -560,6 +557,7 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections &atev1alpha1.WorkerPool{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "microvm"}, Spec: atev1alpha1.WorkerPoolSpec{SandboxClass: atev1alpha1.SandboxClassMicroVM}}, }) collections := Collections{ + EgressPolicies: krt.NewStaticCollection[*kagentv1alpha3.EgressPolicy](nil, nil, opts.WithName("EgressPolicies")...), AgentTemplates: krttest.GetMockCollection[*kagentv1alpha3.AgentTemplate](mock), Harnesses: harnesses, ResolvedModelConfigs: krttest.GetMockCollection[translator.ResolvedModelConfig](mock), @@ -574,7 +572,7 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections collections.Reconciliations = newAgentReconciliations(collections.Agents, translator.Collections{ Harnesses: collections.Harnesses, AgentTemplates: collections.AgentTemplates, ResolvedModelConfigs: collections.ResolvedModelConfigs, RemoteMCPServers: collections.RemoteMCPServers, ConfigMaps: collections.ConfigMaps, - Secrets: collections.Secrets, WorkerPools: collections.WorkerPools, + Secrets: collections.Secrets, WorkerPools: collections.WorkerPools, EgressPolicies: collections.EgressPolicies, }, collections.AgentRuntimeObservations, opts, testRuntimeDefaults(), nil) collections.AgentStatuses = newAgentStatuses(collections.Agents, collections.Reconciliations, opts) waitFor(t, func() bool { @@ -584,46 +582,38 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections return collections, harnesses } -func TestInitialRevisionWaitsForEgressPublication(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - collections, _ := newPreparationTestCollections(t, "gvisor") - initial := collections.Reconciliations.List()[0] - template := proto.CloneOf(initial.Target.ActorTemplate) - template.Metadata.Uid = "prepared" - template.Status = sandboxGoldenStatus() - store := &fakeRuntimeRevisionStore{promoteErr: database.ErrEgressPolicyNotPublished} - backend := &preparationTestBackend{templates: &fakeActorTemplates{template: template}, store: store} - queued := 0 - r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3(), func(revision string) { - backend.inspect(func() { - require.NotNil(t, store.revision) - require.Equal(t, store.revision.Revision, revision) - queued++ - }) - }) - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - var workers sync.WaitGroup - workers.Go(func() { r.Run(ctx.Done()) }) - defer func() { cancel(); workers.Wait() }() - synctest.Wait() - backend.inspect(func() { - require.Equal(t, 1, queued) - require.False(t, store.markedSuccessful, "enqueueing publication must not make the revision ready") - }) - require.NotNil(t, collections.Reconciliations.GetKey(initial.ResourceName()).PreparationFailure) - time.Sleep(500 * time.Millisecond) - synctest.Wait() - backend.inspect(func() { - require.Equal(t, 1, store.recordCalls, "pending publication must retry with backoff") - store.promoteErr = nil // Publication completes independently of preparation. - }) - time.Sleep(500 * time.Millisecond) - synctest.Wait() - backend.inspect(func() { - require.True(t, store.markedSuccessful, "preparation must recover without another Kubernetes event") - require.Equal(t, 2, store.recordCalls, "each attempt must record inputs only once") - }) - require.Nil(t, collections.Reconciliations.GetKey(initial.ResourceName()).PreparationFailure) +func TestPreparationRecordsCompiledPolicyBeforeEnqueueAndPromotion(t *testing.T) { + collections, _ := newPreparationTestCollections(t, "gvisor") + policy := egressTestPolicy("https://tickets.example.com", nil) + policy.Namespace = "team-a" + collections.EgressPolicies.(krt.StaticCollection[*kagentv1alpha3.EgressPolicy]).UpdateObject(policy) + agent := collections.Agents.List()[0].DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + collections.Agents.(krt.StaticCollection[*kagentv1alpha3.Agent]).UpdateObject(agent) + waitFor(t, func() bool { + state := collections.Reconciliations.GetKey("team-a/assistant") + return state.Target != nil && len(state.Target.Egress.Destinations) == 1 }) + initial := collections.Reconciliations.List()[0] + template := proto.CloneOf(initial.Target.ActorTemplate) + template.Metadata.Uid = "prepared" + template.Status = sandboxGoldenStatus() + want, err := egress.MergeAccess(egress.Access{Destinations: initial.Target.Revision.EgressDestinations, Credentials: initial.Target.Revision.Credentials}, initial.Target.Egress, initial.Target.Revision.TokenPassthroughOrigins) + require.NoError(t, err) + store := &fakeRuntimeRevisionStore{revisionErr: errors.New("record failed")} + queued := 0 + r := &Reconciler{collections: collections, templates: &fakeActorTemplates{template: template}, store: store, + enqueueEgress: func(revision string) { + queued++ + require.Equal(t, initial.Target.RevisionID.String(), revision) + require.Equal(t, want, store.policy, "initial policy must be recorded before enqueue") + require.False(t, store.markedSuccessful) + }, + } + require.ErrorIs(t, r.reconcileAgent(t.Context(), initial.ResourceName()), store.revisionErr) + require.Zero(t, queued, "failed recording must not enqueue publication") + store.revisionErr = nil + require.NoError(t, r.reconcileAgent(t.Context(), initial.ResourceName())) + require.Equal(t, 1, queued) + require.True(t, store.markedSuccessful, "initial readiness must not wait for the publication queue") } diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index ab92ae8c12..efebeab42a 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -26,6 +26,7 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/internal/translator" byotranslator "github.com/kagent-dev/kagent/go/core/internal/translator/byo" @@ -50,6 +51,7 @@ type AgentReconciliation struct { // construction all succeed. An absent target means there is no desired runtime. type compiledTarget struct { Revision translator.Revision + Egress egress.Access RevisionID translator.RevisionID ActorTemplate *ateapipb.ActorTemplate } @@ -59,7 +61,7 @@ func (c *compiledTarget) equals(other *compiledTarget) bool { return c == other } return c.RevisionID == other.RevisionID && c.Revision.Equals(other.Revision) && - proto.Equal(c.ActorTemplate, other.ActorTemplate) + proto.Equal(c.ActorTemplate, other.ActorTemplate) && reflect.DeepEqual(c.Egress, other.Egress) } func (r AgentReconciliation) ResourceName() string { return r.Agent.Namespace + "/" + r.Agent.Name } @@ -137,7 +139,7 @@ func newAgentReconciliations( state.CompilationFailure = &ReconciliationFailure{Condition: kagentv1alpha3.AgentConditionCompatible, Reason: "ActorTemplateInvalid", Message: err.Error()} return state } - state.Target = &compiledTarget{Revision: *revision, RevisionID: revisionID, ActorTemplate: actorTemplate} + state.Target = &compiledTarget{Revision: *revision, Egress: compilation.Egress, RevisionID: revisionID, ActorTemplate: actorTemplate} observed := krt.FetchOne(ctx, agentRuntimeObservations, krt.FilterKey(state.ResourceName())) if observed == nil || observed.RevisionID != state.Target.RevisionID { @@ -166,7 +168,7 @@ func newAgentReconciliations( // or a Session or checkpoint references them. type runtimeRevisionStore interface { UpsertAgentDefinition(context.Context, database.AgentDefinition) error - RecordRuntimeRevision(context.Context, database.RuntimeRevision) error + RecordRuntimeRevision(context.Context, database.RuntimeRevision, egress.Access) error PromoteRuntimeRevision(context.Context, string) error RetireAgentIdentities(ctx context.Context, namespace, name string, except *database.AgentDefinition) error } @@ -378,9 +380,15 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { ActorTemplateAtespace: observed.GetMetadata().GetAtespace(), ActorTemplateName: observed.GetMetadata().GetName(), ActorTemplateUID: observed.GetMetadata().GetUid(), } ready := observed.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() != nil - if err := r.store.RecordRuntimeRevision(ctx, revision); err != nil { + policy, err := egress.MergeAccess(egress.Access{Destinations: revision.EgressDestinations, Credentials: revision.Credentials}, target.Egress, revision.TokenPassthroughOrigins) + if err != nil { + return r.observePreparationError(*state, fmt.Errorf("merge runtime egress %s: %w", target.RevisionID, err)) + } + if err := r.store.RecordRuntimeRevision(ctx, revision, policy); err != nil { return r.observePreparationError(*state, fmt.Errorf("store runtime revision %s: %w", target.RevisionID, err)) } + // Catch grants that changed after compilation but before the revision was + // visible to source-event discovery. Preparation never replaces live policy. r.enqueueEgress(revision.Revision) if ready { if err := r.store.PromoteRuntimeRevision(ctx, revision.Revision); err != nil { diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index 1be613fdc1..4172998268 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -182,9 +182,6 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { Target: &compiledTarget{Revision: *revision, RevisionID: id, ActorTemplate: desired}, } states.UpdateObject(state) - require.ErrorIs(t, reconciler.reconcileAgent(ctx, state.ResourceName()), database.ErrEgressPolicyNotPublished) - _, err = store.SetRuntimeEgressPolicy(ctx, id.String(), egress.Access{}) - require.NoError(t, err) require.NoError(t, reconciler.reconcileAgent(ctx, state.ResourceName())) request := &apiv1.Session{ @@ -325,6 +322,7 @@ func (f *fakeActorTemplates) DeleteActorTemplate(context.Context, string, string type fakeRuntimeRevisionStore struct { pair *database.AgentDefinition revision *database.RuntimeRevision + policy egress.Access markedSuccessful bool retired string revisionErr error @@ -340,12 +338,13 @@ func (s *fakeRuntimeRevisionStore) UpsertAgentDefinition(_ context.Context, pair return s.pairErr } -func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision) error { +func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision, policy egress.Access) error { s.recordCalls++ if s.revisionErr != nil { return s.revisionErr } s.revision = &revision + s.policy = policy return nil } diff --git a/go/core/internal/controller/sandbox_collections.go b/go/core/internal/controller/sandbox_collections.go index 25a21be69a..52426f1a5a 100644 --- a/go/core/internal/controller/sandbox_collections.go +++ b/go/core/internal/controller/sandbox_collections.go @@ -10,6 +10,7 @@ import ( "istio.io/istio/pkg/kube/krt" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -25,6 +26,7 @@ type sandboxReconciliation struct { Template *kagentv1alpha3.SandboxTemplate RevisionID string SourceSnapshot json.RawMessage + Egress egress.Access DesiredActorTemplate *ateapipb.ActorTemplate ObservedActorTemplate *ateapipb.ActorTemplate CompilationError string @@ -89,12 +91,13 @@ func newSandboxCollections(inputs Collections, policy substrate.SandboxPolicy, o if pool == nil { state.Failure = &ReconciliationFailure{Reason: "WorkerPoolNotFound", Message: "The referenced sandbox WorkerPool does not exist"} } else { - _, err := translator.CompileEgressPolicies(ctx, translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets}, template.Namespace, template.Spec.EgressPolicyRefs) + grants, err := translator.CompileEgressPolicies(ctx, translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets}, template.Namespace, template.Spec.EgressPolicyRefs) if err != nil { state.CompilationError = err.Error() state.Failure = &ReconciliationFailure{Reason: kagentv1alpha3.SandboxTemplateReasonEgressPolicyResolutionFailed, Message: err.Error()} return state } + state.Egress = grants state.DesiredActorTemplate, state.RevisionID, state.SourceSnapshot, err = substrate.SandboxActorTemplate(template, (*pool).Spec.SandboxClass, policy) if err != nil { state.CompilationError = err.Error() diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index 996609c82b..f52f69fd52 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -21,6 +21,7 @@ import ( kagentclient "github.com/kagent-dev/kagent/go/api/clientset/versioned/typed/api/v1alpha3" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" ) @@ -33,7 +34,7 @@ const ( type sandboxRevisionStore interface { UpsertSandboxTemplateDefinition(context.Context, database.SandboxTemplateDefinition) error RetireSandboxTemplateIdentities(context.Context, string, string) error - RecordSandboxRevision(context.Context, database.SandboxRevision) error + RecordSandboxRevision(context.Context, database.SandboxRevision, egress.Access) error PromoteSandboxRevision(context.Context, string) error } @@ -177,7 +178,11 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), SourceSnapshot: state.SourceSnapshot, } - if err := s.store.RecordSandboxRevision(ctx, revision); err != nil { + policy, err := egress.MergeAccess(egress.Access{}, state.Egress, nil) + if err != nil { + return s.observeError(*state, fmt.Errorf("merge sandbox egress %s: %w", state.RevisionID, err), true) + } + if err := s.store.RecordSandboxRevision(ctx, revision, policy); err != nil { return s.observeError(*state, fmt.Errorf("store sandbox revision %s: %w", state.RevisionID, err), true) } s.enqueueEgress(revision.Revision) diff --git a/go/core/internal/controller/sandbox_reconciler_test.go b/go/core/internal/controller/sandbox_reconciler_test.go index 6c70a14063..6292406d89 100644 --- a/go/core/internal/controller/sandbox_reconciler_test.go +++ b/go/core/internal/controller/sandbox_reconciler_test.go @@ -25,6 +25,7 @@ import ( kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" ) @@ -33,6 +34,7 @@ type sandboxTestStore struct { mu sync.Mutex desired database.SandboxTemplateDefinition revision database.SandboxRevision + policy egress.Access ready bool retired bool retireAttempts int @@ -48,7 +50,7 @@ func (s *sandboxTestStore) UpsertSandboxTemplateDefinition(_ context.Context, va s.desired = value return nil } -func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision) error { +func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision, policy egress.Access) error { s.mu.Lock() defer s.mu.Unlock() s.recordCalls++ @@ -56,6 +58,7 @@ func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value databa return s.recordErr } s.revision = value + s.policy = policy return nil } func (s *sandboxTestStore) PromoteSandboxRevision(_ context.Context, revision string) error { @@ -312,25 +315,40 @@ func TestSandboxPendingStatusRecoversWithoutGraphEvent(t *testing.T) { }, 5*time.Second, 10*time.Millisecond) } -func TestInitialSandboxRevisionWaitsForEgressPublication(t *testing.T) { - s, templates, _ := newSandboxTestReconciler(t, "guest@sha256:"+strings.Repeat("b", 64)) - store := s.store.(*sandboxTestStore) - s.actors.(*sandboxTestActors).autoReady = true - require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) - syncSandboxTemplate(t, s, templates) - store.promoteErr = database.ErrEgressPolicyNotPublished - s.enqueueEgress = func(revision string) { - require.Equal(t, store.revision.Revision, revision) - require.False(t, store.ready, "initial policy must precede readiness") +func TestSandboxPreparationRecordsCompiledPolicyBeforeEnqueueAndPromotion(t *testing.T) { + ctx := t.Context() + inputs := newEgressTestInputs(ctx) + template := sandboxTestTemplate() + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + inputs.sandboxes.UpdateObject(template) + policy := egressTestPolicy("https://tickets.example.com", nil) + policy.Namespace = template.Namespace + inputs.policies.UpdateObject(policy) + inputs.runtime.Collections.WorkerPools = krt.NewStaticCollection(nil, []*atev1alpha1.WorkerPool{{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "default"}}}, inputs.runtime.Options.WithName("Pools")...) + store := &sandboxTestStore{} + actors := &sandboxTestActors{store: store, templates: map[string]*ateapipb.ActorTemplate{}, autoReady: true} + queued := 0 + s := &SandboxReconciler{ + collections: newSandboxCollections(inputs.runtime.Collections, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, inputs.runtime.Options), + store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3(), + enqueueEgress: func(revision string) { + queued++ + require.Equal(t, store.revision.Revision, revision) + require.Equal(t, []string{"https://tickets.example.com:443"}, store.policy.Destinations) + require.False(t, store.ready) + }, } - require.ErrorIs(t, s.reconcile(t.Context(), "team-a/scratch"), database.ErrEgressPolicyNotPublished) - require.False(t, store.ready) - waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch").Failure != nil }) - store.promoteErr = errors.New("promotion unavailable") - require.ErrorIs(t, s.reconcile(t.Context(), "team-a/scratch"), store.promoteErr) - require.False(t, store.ready) - store.promoteErr = nil - require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) - require.True(t, store.ready) - require.Equal(t, 3, store.recordCalls, "each attempt must record inputs only once") + waitFor(t, func() bool { + state := s.collections.states.GetKey("team-a/scratch") + return state != nil && state.DesiredActorTemplate != nil + }) + require.NoError(t, s.reconcile(ctx, "team-a/scratch")) // Add the finalizer. + syncSandboxTemplate(t, s, inputs.sandboxes) + store.recordErr = errors.New("record failed") + require.ErrorIs(t, s.reconcile(ctx, "team-a/scratch"), store.recordErr) + require.Zero(t, queued) + store.recordErr = nil + require.NoError(t, s.reconcile(ctx, "team-a/scratch")) + require.Equal(t, 1, queued) + require.True(t, store.ready, "initial readiness must not wait for the publication queue") } diff --git a/go/core/internal/database/client_session_test.go b/go/core/internal/database/client_session_test.go index 163f23b4b9..c7f1f4da69 100644 --- a/go/core/internal/database/client_session_test.go +++ b/go/core/internal/database/client_session_test.go @@ -378,11 +378,9 @@ func TestForkSessionCopiesBoundedHistory(t *testing.T) { if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision, egress.Access{}); err != nil { t.Fatal(err) } - _, policyErr := client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) - require.NoError(t, policyErr) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) source, _, err := client.CreateSession(ctx, &apiv1.Session{ @@ -540,11 +538,9 @@ func TestSessionCreateAndTransitions(t *testing.T) { if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision, egress.Access{}); err != nil { t.Fatal(err) } - _, policyErr := client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) - require.NoError(t, policyErr) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) storedRevision, err := client.GetRuntimeRevision(ctx, revision.Revision) @@ -645,11 +641,9 @@ func sessionFixture(t *testing.T, client *Client, ctx context.Context, namespace if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision, egress.Access{}); err != nil { t.Fatal(err) } - _, policyErr := client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) - require.NoError(t, policyErr) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) } diff --git a/go/core/internal/database/errors.go b/go/core/internal/database/errors.go index bb08026ef9..3184e03884 100644 --- a/go/core/internal/database/errors.go +++ b/go/core/internal/database/errors.go @@ -21,7 +21,3 @@ var ( // ErrObjectDeleting reports that deletion has started. The object is unavailable // to new callers, so it also matches ErrNotFound. var ErrObjectDeleting = fmt.Errorf("object is being deleted: %w", ErrNotFound) - -// ErrEgressPolicyNotPublished asks preparation to retry promotion after the -// controller has published the revision's initial intended policy. -var ErrEgressPolicyNotPublished = fmt.Errorf("egress policy is not published: %w", ErrFailedPrecondition) diff --git a/go/core/internal/database/protobuf_test.go b/go/core/internal/database/protobuf_test.go index 86b4005309..39d52af768 100644 --- a/go/core/internal/database/protobuf_test.go +++ b/go/core/internal/database/protobuf_test.go @@ -14,6 +14,7 @@ import ( "google.golang.org/protobuf/proto" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) func addUnknown(message proto.Message) { @@ -286,11 +287,11 @@ func TestShareAndAgentCardProtobufPersistence(t *testing.T) { require.NoError(t, err) addUnknown(card) revision := RuntimeRevision{Revision: "revision", Namespace: "team-a", AgentName: "assistant", AgentUID: "template", SourceSnapshot: []byte(`{}`), AgentCard: card, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "template"} - require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, egress.Access{})) // Reconciliation can update runtime identity, but the pinned card is immutable. revision.AgentCard = &a2apb.AgentCard{Name: "replacement"} revision.ActorTemplateUID = "new-uid" - require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, egress.Access{})) stored, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) require.True(t, proto.Equal(card, stored.AgentCard)) diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index 705afac869..2ee8051b04 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -1,10 +1,14 @@ package database import ( + "context" + "sync" "testing" + "time" a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" "github.com/stretchr/testify/require" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" @@ -16,9 +20,7 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { ctx := t.Context() revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", TokenPassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} require.NoError(t, client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", DesiredRevision: "agent"})) - require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) - _, policyErr := client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) - require.NoError(t, policyErr) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, egress.Access{})) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) instance, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "request") require.NoError(t, err) @@ -109,9 +111,10 @@ func TestRuntimeEgressPolicyIsMutableControllerOutput(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() revision := RuntimeRevision{Revision: "policy-revision", Namespace: "team-a", AgentName: "assistant", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", EgressDestinations: []string{"https://model.example.com:443"}} - require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) - _, err := client.GetRuntimeEgressPolicy(ctx, revision.Revision) - require.ErrorIs(t, err, ErrNotFound, "absence must not become an empty allowlist") + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, egress.Access{Destinations: revision.EgressDestinations})) + initial, err := client.GetRuntimeEgressPolicy(ctx, revision.Revision) + require.NoError(t, err) + require.Equal(t, revision.EgressDestinations, initial.Destinations) allowed := egress.Access{Destinations: []string{"https://model.example.com:443", "https://tools.example.com:443"}, Credentials: []egress.Credential{{Origin: "https://tools.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/auth/token"}}} changed, err := client.SetRuntimeEgressPolicy(ctx, revision.Revision, allowed) require.NoError(t, err) @@ -139,7 +142,7 @@ func TestRuntimeEgressPolicyIsMutableControllerOutput(t *testing.T) { require.NoError(t, err) require.True(t, changed) // Preparation observations must not restore an earlier policy. - require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, allowed)) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) got, err = replica.GetRuntimeEgressPolicy(ctx, revision.Revision) require.NoError(t, err) @@ -164,3 +167,63 @@ func TestRuntimeEgressPolicyIsMutableControllerOutput(t *testing.T) { _, err = client.SetRuntimeEgressPolicy(ctx, revision.Revision, allowed) require.ErrorIs(t, err, ErrNotFound, "deleting revisions must remain distinguishable from unchanged policies") } + +func TestRecordRevisionInitialPolicyIsAtomic(t *testing.T) { + for _, kind := range []RuntimeEgressSourceKind{RuntimeEgressAgent, RuntimeEgressSandbox} { + t.Run(string(kind), func(t *testing.T) { + pool := setupTestDB(t) + client := NewClient(pool, "public") + ctx := t.Context() + initial := egress.Access{Destinations: []string{"https://tickets.example.com"}} + record := func(client *Client, ctx context.Context, policy egress.Access) error { + if kind == RuntimeEgressAgent { + return client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, policy) + } + return client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, SandboxTemplateName: "source", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, policy) + } + barrier := &runtimeReferenceBarrier{query: "INSERT INTO runtime_egress_policy", afterQuery: true, reached: make(chan struct{}), resume: make(chan struct{})} + release := sync.OnceFunc(func() { close(barrier.resume) }) + config := pool.Config() + config.ConnConfig.Tracer = barrier + writerPool, err := pgxpool.NewWithConfig(ctx, config) + require.NoError(t, err) + defer writerPool.Close() + defer release() + writeCtx, cancel := context.WithCancel(ctx) + defer cancel() + written := make(chan error, 1) + go func() { written <- record(NewClient(writerPool, "public"), writeCtx, initial) }() + select { + case <-barrier.reached: + case <-time.After(5 * time.Second): + t.Fatal("initial policy was not inserted") + } + _, err = client.GetRuntimeEgress(ctx, "revision") + require.ErrorIs(t, err, ErrNotFound, "uncommitted inputs must not be visible") + _, err = client.GetRuntimeEgressPolicy(ctx, "revision") + require.ErrorIs(t, err, ErrNotFound, "uncommitted policy must not be visible") + cancel() + release() + require.ErrorIs(t, <-written, context.Canceled) + _, err = client.GetRuntimeEgressPolicy(ctx, "revision") + require.ErrorIs(t, err, ErrNotFound, "failed recording must roll back the initial policy") + _, err = client.GetRuntimeEgress(ctx, "revision") + require.ErrorIs(t, err, ErrNotFound, "failed recording must roll back the revision") + + require.NoError(t, record(client, ctx, initial)) + pinned, err := client.GetRuntimeEgress(ctx, "revision") + require.NoError(t, err) + require.Equal(t, kind, pinned.Kind) + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + require.Equal(t, []string{"https://tickets.example.com:443"}, policy.Destinations) + _, err = client.SetRuntimeEgressPolicy(ctx, "revision", egress.Access{}) + require.NoError(t, err) + require.NoError(t, record(client, ctx, initial)) + policy, err = client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + require.Empty(t, policy.Destinations, "preparation must not undo a live revocation") + require.Error(t, record(client, ctx, egress.Access{Destinations: []string{"invalid"}})) + }) + } +} diff --git a/go/core/internal/database/runtime_lifecycle_test.go b/go/core/internal/database/runtime_lifecycle_test.go index 81afb6ba2b..d9970c2ae0 100644 --- a/go/core/internal/database/runtime_lifecycle_test.go +++ b/go/core/internal/database/runtime_lifecycle_test.go @@ -6,6 +6,8 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/stretchr/testify/require" + + "github.com/kagent-dev/kagent/go/core/internal/egress" ) func TestRuntimeLifecycleStoresSharedValues(t *testing.T) { @@ -69,7 +71,7 @@ func TestAgentWritesRejectSandboxRevisions(t *testing.T) { revision, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) revision.Revision, revision.ActorTemplateUID = "sandbox-revision", "replacement-uid" - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision), ErrConflict) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision, egress.Access{}), ErrConflict) require.ErrorIs(t, client.PromoteRuntimeRevision(ctx, "sandbox-revision"), ErrConflict) require.ErrorIs(t, client.PromoteSandboxRevision(ctx, "revision"), ErrConflict) var uid string diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index 823b75a5fb..f9122602d5 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -67,10 +67,11 @@ func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefi }) } -// RecordRuntimeRevision stores revision inputs without making them available for -// new Sessions. Existing revisions retain immutable inputs; only their actor UID -// and update time are refreshed. Deleting revisions return ErrObjectDeleting. -func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevision) error { +// RecordRuntimeRevision atomically stores revision inputs and the initial policy +// without making them available for new Sessions. Existing inputs and policy are +// preserved; only the actor UID and update time are refreshed. Malformed policies +// are rejected. Deleting revisions return ErrObjectDeleting. +func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevision, policy egress.Access) error { if revision.AgentCard == nil { return fmt.Errorf("runtime revision %s has no Agent Card", revision.Revision) } @@ -83,7 +84,7 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi RuntimeArtifact: RuntimeArtifact{Revision: revision.Revision, Kind: runtimeKindAgent, Namespace: revision.Namespace, ActorTemplateAtespace: revision.ActorTemplateAtespace, ActorTemplateName: revision.ActorTemplateName, ActorTemplateUID: revision.ActorTemplateUID}, SourceSnapshot: revision.SourceSnapshot, EgressDestinations: revision.EgressDestinations, Credentials: revision.Credentials, TokenPassthroughOrigins: revision.TokenPassthroughOrigins, - }); err != nil { + }, policy); err != nil { return err } if err := execSQL(ctx, tx, ` @@ -97,10 +98,8 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi } // PromoteRuntimeRevision makes a recorded, prepared revision available only to -// the active Agent identity that still desires it. Without a published intended -// policy it returns ErrEgressPolicyNotPublished for retry. Missing/deleting -// revisions return ErrNotFound / ErrObjectDeleting; stale reports leave the -// definition unchanged. +// the active Agent identity that still desires it. Missing/deleting revisions return +// ErrNotFound / ErrObjectDeleting; stale reports leave the definition unchanged. func (c *Client) PromoteRuntimeRevision(ctx context.Context, revision string) error { return c.withTx(ctx, func(tx pgx.Tx) error { // Match GC finalization's lock order: definition before revision. @@ -115,30 +114,17 @@ func (c *Client) PromoteRuntimeRevision(ctx context.Context, revision string) er if err := lockRuntimeRevisionForReference(ctx, tx, revision, runtimeKindAgent); err != nil { return err } - published, err := queryOne(ctx, tx, ` - WITH published AS ( - SELECT revision FROM runtime_egress_policy WHERE revision = $1 - ), promoted AS ( - UPDATE agent_definition p - SET latest_successful_revision = r.revision, updated_at = NOW() - FROM agent_runtime_revision r - WHERE r.revision = $1 - AND p.namespace = r.namespace - AND p.agent_uid = r.agent_uid - AND p.desired_revision = r.revision - AND p.retired_at IS NULL - AND p.latest_successful_revision IS DISTINCT FROM r.revision - AND EXISTS (SELECT 1 FROM published) - ) - SELECT EXISTS (SELECT 1 FROM published) - `, pgx.RowTo[bool], revision) - if err != nil { - return err - } - if !published { - return ErrEgressPolicyNotPublished - } - return nil + return execSQL(ctx, tx, ` + UPDATE agent_definition p + SET latest_successful_revision = r.revision, updated_at = NOW() + FROM agent_runtime_revision r + WHERE r.revision = $1 + AND p.namespace = r.namespace + AND p.agent_uid = r.agent_uid + AND p.desired_revision = r.revision + AND p.retired_at IS NULL + AND p.latest_successful_revision IS DISTINCT FROM r.revision + `, revision) }) } @@ -253,10 +239,20 @@ type runtimeRevisionRecord struct { Credentials []egress.Credential } -// recordRuntimeRevision writes immutable common inputs with mutable artifact -// status. The caller writes its typed extension in the same transaction. -// Promotion is a separate transition after intended egress policy is published. -func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevisionRecord) error { +// recordRuntimeRevision writes common inputs, mutable artifact status, and an +// insert-only initial policy. The caller must write its typed extension in the +// same transaction. Preparation retries never replace a published live policy. +func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevisionRecord, policy egress.Access) error { + policy, err := egress.CanonicalAccess(policy.Destinations, policy.Credentials) + if err != nil { + return fmt.Errorf("validate initial runtime egress policy %s: %w", revision.Revision, err) + } + if policy.Destinations == nil { + policy.Destinations = []string{} + } + if policy.Credentials == nil { + policy.Credentials = []egress.Credential{} + } if revision.Credentials == nil { revision.Credentials = []egress.Credential{} } @@ -277,10 +273,16 @@ func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevis if err != nil { return fmt.Errorf("record runtime revision %s: %w", revision.Revision, err) } - if result.RowsAffected() != 0 { - return nil + if result.RowsAffected() == 0 { + if err := lockRuntimeRevisionForReference(ctx, tx, revision.Revision, revision.Kind); err != nil { + return err + } } - return lockRuntimeRevisionForReference(ctx, tx, revision.Revision, revision.Kind) + return execSQL(ctx, tx, ` + INSERT INTO runtime_egress_policy (revision, destinations, credentials) + VALUES ($1, $2, $3) + ON CONFLICT (revision) DO NOTHING + `, revision.Revision, policy.Destinations, policy.Credentials) } // BeginRuntimeRevisionDeletion marks an unreferenced revision as deleting, diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index 0e9d69c56c..3082ec3779 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -151,7 +151,7 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { revision.Revision = "new" revision.AgentUID = "replacement-uid" revision.ActorTemplateName = "new-actor-template" - require.NoError(t, client.RecordRuntimeRevision(ctx, *revision)) + require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, egress.Access{})) pair := AgentDefinition{ Namespace: "team-a", AgentName: "assistant", AgentUID: revision.AgentUID, DesiredRevision: "new", @@ -160,8 +160,6 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { request := newSessionRequest(uuid.NewString(), "assistant", "kagent", "") _, _, err = client.CreateSession(ctx, request, "session") require.ErrorIs(t, err, ErrNotFound, "a replacement must not select the previous template UID") - _, policyErr := client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) - require.NoError(t, policyErr) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) session, _, err := client.CreateSession(ctx, request, "session") require.NoError(t, err) @@ -343,7 +341,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) pair.AgentUID = "replacement-uid" pair.DesiredRevision = "revision" require.ErrorIs(t, client.UpsertAgentDefinition(ctx, pair), ErrObjectDeleting) - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original), ErrObjectDeleting) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, egress.Access{}), ErrObjectDeleting) require.ErrorIs(t, client.PromoteRuntimeRevision(ctx, original.Revision), ErrObjectDeleting) // A new client rediscovers and retries the committed claim after a crash. restarted := NewClient(pool, "public") @@ -360,7 +358,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.ErrorIs(t, err, ErrNotFound) // The same digest can be prepared again once cleanup has completed. original.ActorTemplateUID = "recreated-actor-uid" - require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original)) + require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, egress.Access{})) newClaim, err := restarted.BeginRuntimeRevisionDeletion(ctx, "revision") require.NoError(t, err) require.NotNil(t, newClaim) @@ -368,7 +366,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) _, err = restarted.GetRuntimeRevision(ctx, "revision") require.NoError(t, err, "a delayed collector must not finalize the new runtime") require.NoError(t, restarted.DeleteRuntimeRevision(ctx, "revision", "recreated-actor-uid")) - require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original)) + require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, egress.Access{})) require.NoError(t, restarted.UpsertAgentDefinition(ctx, pair)) } @@ -488,27 +486,20 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { require.Equal(t, want, session.PreparedRevision) } require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) - require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, egress.Access{})) assertAvailableRevision("") stored, err := c.GetRuntimeRevision(ctx, revision.Revision) require.NoError(t, err) require.Equal(t, "original", stored.AgentCard.Name) require.ErrorIs(t, c.PromoteRuntimeRevision(ctx, "missing"), ErrNotFound) - require.ErrorIs(t, c.PromoteRuntimeRevision(ctx, revision.Revision), ErrEgressPolicyNotPublished) - assertAvailableRevision("") - _, err = c.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) - require.NoError(t, err) require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("first") pair.DesiredRevision = "second" require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) revision.Revision, revision.ActorTemplateName = "second", "second" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) - require.ErrorIs(t, c.PromoteRuntimeRevision(ctx, revision.Revision), ErrEgressPolicyNotPublished) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, egress.Access{})) assertAvailableRevision("first") - _, err = c.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) - require.NoError(t, err) require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("second") // A delayed report for the old revision must not replace the newer ready revision. @@ -519,9 +510,7 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) require.NoError(t, c.RetireAgentIdentities(ctx, pair.Namespace, pair.AgentName, nil)) revision.Revision, revision.ActorTemplateName = "third", "third" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) - _, err = c.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) - require.NoError(t, err) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, egress.Access{})) require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("") // Reviving the pair must retain the last success, not a report made while retired. @@ -535,12 +524,12 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { func TestRuntimeRevisionPersistsCredentialBindings(t *testing.T) { client := NewClient(setupTestDB(t), "public") revision := RuntimeRevision{Revision: "credential-revision", Namespace: "team", AgentName: "agent", AgentUID: "agent", SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{"https://api.example.com:443"}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", Credentials: []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}}} - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision)) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, egress.Access{})) got, err := client.GetRuntimeRevision(t.Context(), revision.Revision) require.NoError(t, err) require.Equal(t, revision.Credentials, got.Credentials) revision.Credentials[0].URI = "ate-secret://k8s.io/default/team/other/token" - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision)) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, egress.Access{})) unchanged, err := client.GetRuntimeRevision(t.Context(), revision.Revision) require.NoError(t, err) require.Equal(t, got.Credentials, unchanged.Credentials, "revision credentials are immutable") diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index 9e1d16138c..c3f8724738 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -6,6 +6,8 @@ import ( "fmt" "github.com/jackc/pgx/v5" + + "github.com/kagent-dev/kagent/go/core/internal/egress" ) type SandboxRevision struct { @@ -69,12 +71,14 @@ func (c *Client) RetireSandboxTemplateIdentities(ctx context.Context, namespace, `, namespace, name) } -// RecordSandboxRevision stores immutable sandbox inputs and refreshes artifact -// identity without promoting the revision. Deleting revisions return ErrObjectDeleting. -func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevision) error { +// RecordSandboxRevision atomically stores sandbox inputs and the initial policy +// without promoting the revision. Existing inputs and policy are preserved; only +// artifact identity is refreshed. Malformed policies are rejected. Deleting +// revisions return ErrObjectDeleting. +func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevision, policy egress.Access) error { return c.withTx(ctx, func(tx pgx.Tx) error { revision.Kind = runtimeKindSandbox - if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{RuntimeArtifact: revision.RuntimeArtifact, SourceSnapshot: revision.SourceSnapshot}); err != nil { + if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{RuntimeArtifact: revision.RuntimeArtifact, SourceSnapshot: revision.SourceSnapshot}, policy); err != nil { return err } if err := execSQL(ctx, tx, ` @@ -88,10 +92,8 @@ func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevi } // PromoteSandboxRevision makes a recorded, prepared revision available only to -// the active template identity that still desires it. Without a published intended -// policy it returns ErrEgressPolicyNotPublished for retry. Missing/deleting -// revisions return ErrNotFound / ErrObjectDeleting; stale reports leave the -// definition unchanged. +// the active template identity that still desires it. Missing/deleting revisions return +// ErrNotFound / ErrObjectDeleting; stale reports leave the definition unchanged. func (c *Client) PromoteSandboxRevision(ctx context.Context, revision string) error { return c.withTx(ctx, func(tx pgx.Tx) error { // Match GC finalization's lock order: definition before revision. @@ -106,26 +108,13 @@ func (c *Client) PromoteSandboxRevision(ctx context.Context, revision string) er if err := lockRuntimeRevisionForReference(ctx, tx, revision, runtimeKindSandbox); err != nil { return err } - published, err := queryOne(ctx, tx, ` - WITH published AS ( - SELECT revision FROM runtime_egress_policy WHERE revision = $1 - ), promoted AS ( - UPDATE sandbox_template_definition p SET latest_successful_revision = r.revision, updated_at = NOW() - FROM runtime_revision r JOIN sandbox_revision s USING (revision) - WHERE r.revision = $1 AND p.namespace = r.namespace AND p.sandbox_template_uid = s.sandbox_template_uid - AND p.desired_revision = r.revision AND p.retired_at IS NULL - AND p.latest_successful_revision IS DISTINCT FROM r.revision - AND EXISTS (SELECT 1 FROM published) - ) - SELECT EXISTS (SELECT 1 FROM published) - `, pgx.RowTo[bool], revision) - if err != nil { - return err - } - if !published { - return ErrEgressPolicyNotPublished - } - return nil + return execSQL(ctx, tx, ` + UPDATE sandbox_template_definition p SET latest_successful_revision = r.revision, updated_at = NOW() + FROM runtime_revision r JOIN sandbox_revision s USING (revision) + WHERE r.revision = $1 AND p.namespace = r.namespace AND p.sandbox_template_uid = s.sandbox_template_uid + AND p.desired_revision = r.revision AND p.retired_at IS NULL + AND p.latest_successful_revision IS DISTINCT FROM r.revision + `, revision) }) } diff --git a/go/core/internal/database/sandboxes_test.go b/go/core/internal/database/sandboxes_test.go index d771df10e0..61c95ad6c6 100644 --- a/go/core/internal/database/sandboxes_test.go +++ b/go/core/internal/database/sandboxes_test.go @@ -19,9 +19,7 @@ func sandboxFixture(t *testing.T, client *Client, revision string) (*apiv1.Sandb ctx := t.Context() require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision})) require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: revision, Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: revision, ActorTemplateUID: "actor-uid"}, - SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)})) - _, policyErr := client.SetRuntimeEgressPolicy(ctx, revision, egress.Access{}) - require.NoError(t, policyErr) + SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, egress.Access{})) require.NoError(t, client.PromoteSandboxRevision(ctx, revision)) hash := sha256.Sum256([]byte("request")) return &apiv1.Sandbox{Id: uuid.NewString(), Creator: "alice", SandboxTemplate: &apiv1.ResourceReference{Namespace: "team-a", Name: "scratch"}, Name: "experiment"}, @@ -37,7 +35,7 @@ func TestPromoteSandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { revision, err := client.GetSandboxRevision(ctx, "first") require.NoError(t, err) revision.Revision, revision.ActorTemplateName = "second", "second" - require.NoError(t, client.RecordSandboxRevision(ctx, *revision)) + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, egress.Access{})) assertAvailable := func(want string) { t.Helper() input := proto.CloneOf(request) @@ -52,10 +50,6 @@ func TestPromoteSandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { } assertAvailable("first") require.ErrorIs(t, client.PromoteSandboxRevision(ctx, "missing"), ErrNotFound) - require.ErrorIs(t, client.PromoteSandboxRevision(ctx, "second"), ErrEgressPolicyNotPublished) - assertAvailable("first") - _, err = client.SetRuntimeEgressPolicy(ctx, "second", egress.Access{}) - require.NoError(t, err) require.NoError(t, client.PromoteSandboxRevision(ctx, "second")) assertAvailable("second") require.NoError(t, client.PromoteSandboxRevision(ctx, "first")) @@ -66,7 +60,7 @@ func TestPromoteSandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, definition)) assertAvailable("second") revision.Revision, revision.ActorTemplateName = "unused", "unused" - require.NoError(t, client.RecordSandboxRevision(ctx, *revision)) + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, egress.Access{})) claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "unused") require.NoError(t, err) require.NotNil(t, claimed) diff --git a/go/core/internal/grpcserver/scheduledrun_test.go b/go/core/internal/grpcserver/scheduledrun_test.go index 1eecccc5ed..d14be7d510 100644 --- a/go/core/internal/grpcserver/scheduledrun_test.go +++ b/go/core/internal/grpcserver/scheduledrun_test.go @@ -186,9 +186,7 @@ func scheduledRunTestServer(t *testing.T) (*database.Client, apiv1.ScheduledRunS Revision: pair.DesiredRevision, Namespace: pair.Namespace, AgentName: pair.AgentName, AgentUID: pair.AgentUID, SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: "runtime-uid", - })) - _, policyErr := store.SetRuntimeEgressPolicy(t.Context(), pair.DesiredRevision, egress.Access{}) - require.NoError(t, policyErr) + }, egress.Access{})) require.NoError(t, store.PromoteRuntimeRevision(t.Context(), pair.DesiredRevision)) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) diff --git a/go/core/internal/grpcserver/task_store_test.go b/go/core/internal/grpcserver/task_store_test.go index bbf5b1000e..a8b6ec665b 100644 --- a/go/core/internal/grpcserver/task_store_test.go +++ b/go/core/internal/grpcserver/task_store_test.go @@ -642,9 +642,7 @@ func createTaskStoreSession(t *testing.T, store *database.Client) *apiv1.Session Namespace: revision.Namespace, AgentName: revision.AgentName, AgentUID: revision.AgentUID, DesiredRevision: revision.Revision, })) - require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision)) - _, policyErr := store.SetRuntimeEgressPolicy(t.Context(), revision.Revision, egress.Access{}) - require.NoError(t, policyErr) + require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision, egress.Access{})) require.NoError(t, store.PromoteRuntimeRevision(t.Context(), revision.Revision)) session, _, err := store.CreateSession(t.Context(), &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", diff --git a/go/core/internal/service/sandbox/guest_test.go b/go/core/internal/service/sandbox/guest_test.go index 0b69885cee..b54e9865e9 100644 --- a/go/core/internal/service/sandbox/guest_test.go +++ b/go/core/internal/service/sandbox/guest_test.go @@ -134,9 +134,7 @@ func guestFixture(t *testing.T) (*sandboxservice.Service, context.Context, *test require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - })) - _, policyErr := store.SetRuntimeEgressPolicy(ctx, "revision", egress.Access{}) - require.NoError(t, policyErr) + }, egress.Access{})) require.NoError(t, store.PromoteSandboxRevision(ctx, "revision")) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) diff --git a/go/core/internal/service/sandbox/service_test.go b/go/core/internal/service/sandbox/service_test.go index 7a09fecd15..02426f7074 100644 --- a/go/core/internal/service/sandbox/service_test.go +++ b/go/core/internal/service/sandbox/service_test.go @@ -139,9 +139,7 @@ func serviceFixture(t *testing.T) (*Service, context.Context, *testActors) { require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - })) - _, policyErr := store.SetRuntimeEgressPolicy(ctx, "revision", egress.Access{}) - require.NoError(t, policyErr) + }, egress.Access{})) require.NoError(t, store.PromoteSandboxRevision(ctx, "revision")) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) @@ -299,9 +297,7 @@ func TestSandboxResolvesCurrentGrantsBeforeResume(t *testing.T) { } store := service.config.Store require.NoError(t, store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: revision.Revision})) - require.NoError(t, store.RecordSandboxRevision(ctx, revision)) - _, policyErr := store.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{}) - require.NoError(t, policyErr) + require.NoError(t, store.RecordSandboxRevision(ctx, revision, egress.Access{})) require.NoError(t, store.PromoteSandboxRevision(ctx, revision.Revision)) instance, err := service.Create(ctx, createRequest()) require.NoError(t, err) diff --git a/go/core/internal/service/session/workflow_test.go b/go/core/internal/service/session/workflow_test.go index 0ab947325f..c92a785cbc 100644 --- a/go/core/internal/service/session/workflow_test.go +++ b/go/core/internal/service/session/workflow_test.go @@ -149,9 +149,7 @@ func lifecycleFixture(t *testing.T) (*lifecycleTestStore, *apiv1.Session) { ActorTemplateAtespace: "team-a", ActorTemplateName: "assistant-kagent-revision", ActorTemplateUID: "actor-template-uid", } require.NoError(t, client.UpsertAgentDefinition(t.Context(), database.AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision})) - require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision)) - _, policyErr := client.SetRuntimeEgressPolicy(t.Context(), revision.Revision, egress.Access{}) - require.NoError(t, policyErr) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision, egress.Access{})) require.NoError(t, client.PromoteRuntimeRevision(t.Context(), revision.Revision)) session, _, err := client.CreateSession(t.Context(), &apiv1.Session{Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}}, uuid.NewString()) require.NoError(t, err) diff --git a/go/core/internal/translator/compiler.go b/go/core/internal/translator/compiler.go index 8020582cc6..ac0f22d8e1 100644 --- a/go/core/internal/translator/compiler.go +++ b/go/core/internal/translator/compiler.go @@ -178,6 +178,7 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agent *v1alpha3.Age if _, err := egress.MergeAccess(egress.Access{Destinations: result.EgressDestinations, Credentials: result.Credentials}, access, result.TokenPassthroughOrigins); err != nil { return nil, NewValidationError("%v", err) } + result.Egress = access workerKey := types.NamespacedName{Namespace: harness.Namespace, Name: harness.Spec.Substrate.WorkerPoolRef.Name} workerPool := krt.FetchOne(c.ctx, c.collections.WorkerPools, krt.FilterObjectName(workerKey)) if workerPool == nil { diff --git a/go/core/internal/translator/egress_test.go b/go/core/internal/translator/egress_test.go index e8408c02d6..b24030b541 100644 --- a/go/core/internal/translator/egress_test.go +++ b/go/core/internal/translator/egress_test.go @@ -86,6 +86,8 @@ func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { return result } result := compile() + require.Equal(t, []string{"https://tickets.example.com:443"}, result.Egress.Destinations) + require.Len(t, result.Egress.Credentials, 1) require.NotContains(t, result.EgressDestinations, "https://tickets.example.com:443") require.NotEmpty(t, result.EgressDestinations, "preserve inferred integration access") for _, binding := range result.Credentials { @@ -101,7 +103,9 @@ func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { require.NoError(t, err) require.Equal(t, original, rotated) policy.Spec.Destinations = append(policy.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://registry.npmjs.org"}) - updated, err := compile().Digest() + changed := compile() + require.Contains(t, changed.Egress.Destinations, "https://registry.npmjs.org:443") + updated, err := changed.Digest() require.NoError(t, err) require.Equal(t, original, updated) agent.Spec.EgressPolicyRefs = nil diff --git a/go/core/internal/translator/revision.go b/go/core/internal/translator/revision.go index da68522ef6..bb72d060b2 100644 --- a/go/core/internal/translator/revision.go +++ b/go/core/internal/translator/revision.go @@ -30,11 +30,12 @@ func (id RevisionID) Short() string { return hex.EncodeToString(id[:shortRevisio // IsZero reports whether compilation has not produced an identity. func (id RevisionID) IsZero() bool { return id == RevisionID{} } -// CompileResult contains one immutable runtime revision and the non-blocking -// diagnostics produced while compiling it. Diagnostics are deliberately kept -// outside Revision because they do not describe runtime behavior. +// CompileResult contains immutable runtime inputs, current explicit grants, +// and non-blocking diagnostics. Live grants and diagnostics do not enter the +// revision's identity. type CompileResult struct { Revision + Egress egress.Access Warnings []string } From 67cb4529a7b60366f5cb9d486e7b86e604215535 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 19:15:49 +0000 Subject: [PATCH 25/43] refactor: own live egress grants on source definitions Signed-off-by: Eitan Yarmush --- docs/architecture/egress-policies.md | 70 +- docs/architecture/sandboxes.md | 5 +- .../internal/controller/collections_test.go | 4 +- go/core/internal/controller/egress.go | 320 +++------- go/core/internal/controller/egress_test.go | 598 +++--------------- go/core/internal/controller/equality_test.go | 2 +- go/core/internal/controller/grants_test.go | 141 +++++ .../internal/controller/preparation_test.go | 59 +- go/core/internal/controller/reconciler.go | 78 ++- .../internal/controller/reconciler_test.go | 23 +- .../controller/sandbox_collections.go | 16 +- .../controller/sandbox_integration_test.go | 2 +- .../internal/controller/sandbox_reconciler.go | 32 +- .../controller/sandbox_reconciler_test.go | 51 +- .../internal/database/client_session_test.go | 13 +- go/core/internal/database/models.go | 1 + go/core/internal/database/protobuf_test.go | 5 +- .../database/runtime_creation_test.go | 6 +- go/core/internal/database/runtime_egress.go | 230 +++---- .../internal/database/runtime_egress_test.go | 266 ++++---- .../database/runtime_lifecycle_test.go | 6 +- .../internal/database/runtime_revisions.go | 98 +-- .../database/runtime_revisions_test.go | 60 +- .../internal/database/sandbox_revisions.go | 57 +- go/core/internal/database/sandboxes_test.go | 16 +- go/core/internal/egress/live.go | 20 +- go/core/internal/egress/live_test.go | 46 ++ .../internal/grpcserver/scheduledrun_test.go | 6 +- .../internal/grpcserver/task_store_test.go | 8 +- .../internal/service/sandbox/guest_test.go | 6 +- .../internal/service/sandbox/service_test.go | 11 +- .../internal/service/session/workflow_test.go | 5 +- go/core/internal/substrate/egress_test.go | 25 +- go/core/internal/translator/compiler.go | 1 - go/core/internal/translator/egress_test.go | 3 - go/core/internal/translator/revision.go | 6 +- go/core/pkg/app/app.go | 10 +- .../pkg/migrations/core/000001_initial.sql | 12 +- go/core/pkg/migrations/runner_test.go | 6 +- 39 files changed, 958 insertions(+), 1366 deletions(-) create mode 100644 go/core/internal/controller/grants_test.go create mode 100644 go/core/internal/egress/live_test.go diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index bfc2e40850..4fa571fa98 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -159,50 +159,52 @@ Explicit policy grants and policy references do not enter revision identity. Changing a grant or its Secret reference does not prepare a new runtime image. Secret values remain outside compiled inputs; the gateway handles value rotation. -The elected controller compiles current grants from cached, dependency-tracked -Kubernetes inputs and persists the intended policy for each retained revision. -This mutable output is stored separately from immutable revision inputs. -Preparation carries the explicit grants from the same KRT compilation as the -revision, merges them with pinned access, and records the revision and initial -policy in one transaction. Repeated preparation preserves the existing policy; -only the egress reconciler updates it. Promotion remains a separate step once -the runtime is prepared and does not wait for the publication queue. - -After recording commits, preparation enqueues the revision to catch grants that -changed while source-event discovery could not yet see the new revision. Source -events and sweeps use the same publication queue. It serializes writes for each -revision and reruns with the latest grants when an event arrives during -publication; unrelated revisions can publish concurrently. -Each revision merges its pinned integration access with the current grants of its -owning Agent or SandboxTemplate, even when that source's next runtime revision -cannot be prepared. - -Policy and Secret events update only revisions of the affected source and queue -their existing Actors when the intended policy changes. A five-minute sweep -repairs missed translations without rewriting unchanged policies or checking -their Actors. Controller startup checks all Actors to recover interrupted updates. -Actor updates retry with exponential backoff independently of successful -updates. Running and suspended Actors receive updates without replacement or a -new invocation. Missing or invalid policies and Secrets revoke explicit grants; -pinned integration access remains available. Grants that conflict with pinned -credentials or caller-token origins, or exceed the merged rule limit, are withheld -with rate-limited warnings. Recreating an Agent or SandboxTemplate does not -transfer its grants to revisions belonging to the previous Kubernetes UID. +The elected controller's Agent and SandboxTemplate reconcilers resolve current +grants independently of runtime compilation. A missing Harness, AgentTemplate, +or WorkerPool cannot freeze a revocation. Each preparation queue is the sole +writer of its source definition's grants in PostgreSQL. Definition updates +persist canonical grants and retire replaced Kubernetes UIDs atomically, before +recording or promoting a runtime revision. Failed runtime preparation preserves +the last successful revision while continuing to update source grants. + +Effective policy is computed when read: the revision's pinned integration access +is merged with grants from its active source UID. Retired or replaced identities +contribute no grants. There is no separately published per-revision policy and +no notification needed when a revision is recorded: every read joins against +the current persisted grants, including for newly recorded revisions. + +Changes to grants enqueue the source's actors across all retained revisions and +Kubernetes UIDs. Retirement and uncertain database write outcomes also enqueue +that source. Actor application runs in separate queues so slow Substrate calls +do not block preparation. The applier only reads effective policy; it does not +translate Kubernetes inputs or write grants. Startup recovery pages all allocated +actors within the controller's namespace watch scope. Failed applications retry +with exponential backoff. Preparation finalizers on both source kinds preserve +deletion events through controller downtime. + +Running and suspended actors receive updates without replacement or a new +invocation. Missing or invalid policy references and Secrets revoke explicit +grants. Grants that conflict with pinned credentials or caller-token origins, or +exceed the merged rule limit, are withheld while pinned integration access remains. +The applier reports these conflicts with warnings limited per revision; create +and resume apply the same fallback. Missing revisions or malformed persisted +inputs return errors and block execution. New Sessions and sandboxes pin the latest successful runtime revision, including while a configuration edit is unresolved, preparing, or failed. Initial creation still requires a successful revision. Existing instances and checkpoint forks retain their runtime configuration. -Create, creation retries, and resume load and apply the controller's persisted -intended policy. They do not read or translate Kubernetes policy. The workflow -creates a suspended Actor and installs its `default` egress policy before it can +Create, creation retries, and resume load and apply the effective policy from +persisted revision inputs and source grants. They do not read or translate +Kubernetes policy. The workflow creates a suspended Actor and installs its +`default` egress policy before it can run; a fork cannot restore an old policy from a snapshot. Applications read the Substrate policy version before loading intended policy, then write using its UID and version preconditions. Conflicts reload both. After applying, they recheck intended policy to catch updates whose actor discovery raced with creation. -Missing or malformed persisted policy blocks execution and is retried. No database -transaction or lock is held across Substrate calls. +Missing revisions or malformed persisted inputs block execution and are retried. +No database transaction or lock is held across Substrate calls. Policy convergence is asynchronous: a Kubernetes write is not a synchronous revocation barrier. Once the gateway observes the changed policy, subsequent diff --git a/docs/architecture/sandboxes.md b/docs/architecture/sandboxes.md index 98353aba1b..3a323cb83c 100644 --- a/docs/architecture/sandboxes.md +++ b/docs/architecture/sandboxes.md @@ -119,9 +119,10 @@ expiration. Sandboxes have no allowed egress destinations by default. Select same-namespace [EgressPolicy resources](egress-policies.md) through `SandboxTemplate.spec.egressPolicyRefs` to grant additional HTTP(S) origins and static gateway credentials. The controller -translates current grants and persists an intended policy per prepared revision. +persists current grants on the source definition; effective policy is merged with +pinned revision access when read. Policy edits update running and suspended sandboxes without replacing their Actor. -Creation and resume apply that persisted policy before execution, without +Creation and resume apply that effective policy before execution, without translating Kubernetes inputs. ## Lifecycle and recovery diff --git a/go/core/internal/controller/collections_test.go b/go/core/internal/controller/collections_test.go index 78c2bf3274..92fc7b931d 100644 --- a/go/core/internal/controller/collections_test.go +++ b/go/core/internal/controller/collections_test.go @@ -102,7 +102,7 @@ func TestReconciliationCollectionsCompileAndObserveRevision(t *testing.T) { observed.Metadata.Uid = "actor-template-uid" observed.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}}} store := &fakeRuntimeRevisionStore{} - reconciler := &Reconciler{enqueueEgress: noopEnqueueEgress, + reconciler := &Reconciler{enqueueActors: noopEnqueueActors, collections: collections, templates: &fakeActorTemplates{template: observed}, store: store, } require.NoError(t, reconciler.reconcileAgent(t.Context(), state.ResourceName())) @@ -525,7 +525,7 @@ func waitFor(t *testing.T, condition func() bool) { } func testAgent(template *kagentv1alpha3.AgentTemplate, harness *kagentv1alpha3.Harness) *kagentv1alpha3.Agent { - return &kagentv1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: template.Name, UID: template.UID}, Spec: kagentv1alpha3.AgentSpec{ + return &kagentv1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: template.Name, UID: template.UID, Finalizers: []string{agentPreparationFinalizer}}, Spec: kagentv1alpha3.AgentSpec{ Template: kagentv1alpha3.AgentTemplateSource{Ref: &corev1.LocalObjectReference{Name: template.Name}}, HarnessRef: corev1.LocalObjectReference{Name: harness.Name}, }} diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go index be63820881..bee572f65c 100644 --- a/go/core/internal/controller/egress.go +++ b/go/core/internal/controller/egress.go @@ -2,7 +2,6 @@ package controller import ( "context" - "errors" "slices" "sync" "time" @@ -11,63 +10,19 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "istio.io/istio/pkg/kube/controllers" - "istio.io/istio/pkg/kube/krt" "k8s.io/apimachinery/pkg/util/cache" - "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/manager" - kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" - "github.com/kagent-dev/kagent/go/core/internal/translator" "github.com/kagent-dev/kagent/go/pkg/logging" ) -type runtimeEgressStore interface { - GetRuntimeEgress(context.Context, string) (*database.RuntimeEgress, error) - SetRuntimeEgressPolicy(context.Context, string, egress.Access) (bool, error) - ListRuntimeEgressRevisions(context.Context, string, int) ([]string, error) - ListRuntimeEgressSourceRevisions(context.Context, database.RuntimeEgressSource, string, int) ([]string, error) - ListRuntimeEgressTargets(context.Context, string, string, int) ([]database.RuntimeEgressTarget, error) -} - -type policyGrants struct { - Source database.RuntimeEgressSource - UID string - Access egress.Access -} - -func (p policyGrants) ResourceName() string { - return string(p.Source.Kind) + "/" + p.Source.Namespace + "/" + p.Source.Name -} - -func newEgressGrants(runtime *Runtime) krt.Collection[policyGrants] { - inputs := runtime.Collections - collections := translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets} - agents := krt.NewCollection(inputs.Agents, func(ctx krt.HandlerContext, agent *kagentv1alpha3.Agent) *policyGrants { - grants := &policyGrants{Source: database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: agent.Namespace, Name: agent.Name}, UID: string(agent.UID)} - if agent.DeletionTimestamp.IsZero() { - // CompileEgressPolicies returns empty access on every failure. Runtime - // compilation reports resolution errors through the source's status. - access, err := translator.CompileEgressPolicies(ctx, collections, agent.Namespace, agent.Spec.EgressPolicyRefs) - if err == nil { - grants.Access = access - } - } - return grants - }, runtime.Options.WithName("AgentEgressGrants")...) - sandboxes := krt.NewCollection(inputs.SandboxTemplates, func(ctx krt.HandlerContext, template *kagentv1alpha3.SandboxTemplate) *policyGrants { - grants := &policyGrants{Source: database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: template.Namespace, Name: template.Name}, UID: string(template.UID)} - if template.DeletionTimestamp.IsZero() { - access, err := translator.CompileEgressPolicies(ctx, collections, template.Namespace, template.Spec.EgressPolicyRefs) - if err == nil { - grants.Access = access - } - } - return grants - }, runtime.Options.WithName("SandboxEgressGrants")...) - return krt.JoinCollection([]krt.Collection[policyGrants]{agents, sandboxes}, runtime.Options.WithName("RuntimeEgressGrants")...) +type actorEgressStore interface { + egress.PolicyStore + ListRuntimeEgressTargets(context.Context, string, int) ([]database.RuntimeEgressTarget, error) + ListRuntimeEgressSourceTargets(context.Context, database.RuntimeEgressSource, string, int) ([]database.RuntimeEgressTarget, error) } type actorEgressClient interface { @@ -75,21 +30,16 @@ type actorEgressClient interface { EnsureActorEgressPolicy(context.Context, string, string, string) error } -// EgressReconciler publishes KRT's current grants per retained revision. Actors -// and API replicas apply this persisted output without translating Kubernetes. -// Startup actor recovery has separate workers from live actor updates. -type EgressReconciler struct { - grants krt.Collection[policyGrants] - store runtimeEgressStore - actors actorEgressClient - namespaces []string - warnings *cache.LRUExpireCache - publications workqueue.TypedRateLimitingInterface[string] - sources controllers.Queue - pages controllers.Queue - updates controllers.Queue - recovery controllers.Queue - sweeps controllers.Queue +// ActorEgressApplier applies persisted effective policy independently of runtime +// preparation. It never translates Kubernetes inputs or writes source grants. +type ActorEgressApplier struct { + store actorEgressStore + actors actorEgressClient + namespaces []string + warnings *cache.LRUExpireCache + pages controllers.Queue + updates controllers.Queue + recovery controllers.Queue } type egressSourcePage struct { @@ -97,221 +47,103 @@ type egressSourcePage struct { After string } -type egressActorPage struct { - Revision string - After string - Recovery bool -} - -type egressSweepPage struct { - After string - RecoverActors bool -} +type egressRecoveryPage struct{ After string } -var _ manager.LeaderElectionRunnable = (*EgressReconciler)(nil) +var _ manager.LeaderElectionRunnable = (*ActorEgressApplier)(nil) -func NewEgressReconciler(ctx context.Context, runtime *Runtime, store runtimeEgressStore, actors actorEgressClient, namespaces []string) *EgressReconciler { - e := &EgressReconciler{grants: newEgressGrants(runtime), store: store, actors: actors, - namespaces: slices.Clone(namespaces), warnings: cache.NewLRUExpireCache(1024), - publications: workqueue.NewTypedRateLimitingQueueWithConfig( - workqueue.NewTypedItemExponentialFailureRateLimiter[string](time.Second, 30*time.Second), - workqueue.TypedRateLimitingQueueConfig[string]{Name: "live-egress-publications"}), - } +func NewActorEgressApplier(ctx context.Context, store actorEgressStore, actors actorEgressClient, namespaces []string) *ActorEgressApplier { + a := &ActorEgressApplier{store: store, actors: actors, namespaces: slices.Clone(namespaces), warnings: cache.NewLRUExpireCache(1024)} reconcile := func(item any) error { - err := e.reconcileActor(ctx, item.(database.RuntimeEgressTarget)) + err := a.reconcileActor(ctx, item.(database.RuntimeEgressTarget)) if status.Code(err) == codes.NotFound { return nil } return err } - e.updates = newReconciliationQueue("live-egress-actors", reconcile) - e.recovery = newReconciliationQueue("live-egress-recovery", reconcile) + a.updates = newReconciliationQueue("live-egress-actors", reconcile) + a.recovery = newReconciliationQueue("live-egress-recovery", reconcile) const pageSize = 100 - e.pages = newReconciliationQueue("live-egress-revision-actors", func(item any) error { - page := item.(egressActorPage) - // Startup recovery lists all revisions independently of publication. - // Check watch scope before expanding its first actor page. - if page.Recovery && page.After == "" && len(e.namespaces) > 0 { - pinned, err := e.store.GetRuntimeEgress(ctx, page.Revision) - if errors.Is(err, database.ErrNotFound) { + a.pages = newReconciliationQueue("live-egress-pages", func(item any) error { + switch page := item.(type) { + case egressSourcePage: + if !a.watches(page.Source.Namespace) { return nil } + targets, err := a.store.ListRuntimeEgressSourceTargets(ctx, page.Source, page.After, pageSize) if err != nil { return err } - if !slices.Contains(e.namespaces, pinned.Namespace) { - return nil + for _, target := range targets { + a.updates.Add(target) } - } - targets, err := e.store.ListRuntimeEgressTargets(ctx, page.Revision, page.After, pageSize) - if err != nil { - return err - } - queue := e.updates - if page.Recovery { - queue = e.recovery - } - for _, target := range targets { - queue.Add(target) - } - if len(targets) == pageSize { - page.After = targets[len(targets)-1].ID - e.pages.Add(page) - } - return nil - }) - e.sources = newReconciliationQueue("live-egress-sources", func(item any) error { - page := item.(egressSourcePage) - revisions, err := e.store.ListRuntimeEgressSourceRevisions(ctx, page.Source, page.After, pageSize) - if err != nil { - return err - } - for _, revision := range revisions { - e.EnqueueRevision(revision) - } - if len(revisions) == pageSize { - page.After = revisions[len(revisions)-1] - e.sources.Add(page) - } - return nil - }) - e.sweeps = newReconciliationQueue("live-egress-sweeps", func(item any) error { - page := item.(egressSweepPage) - revisions, err := e.store.ListRuntimeEgressRevisions(ctx, page.After, pageSize) - if err != nil { - return err - } - for _, revision := range revisions { - e.EnqueueRevision(revision) - if page.RecoverActors { - e.pages.Add(egressActorPage{Revision: revision, Recovery: true}) + if len(targets) == pageSize { + page.After = targets[len(targets)-1].ID + a.pages.Add(page) + } + case egressRecoveryPage: + targets, err := a.store.ListRuntimeEgressTargets(ctx, page.After, pageSize) + if err != nil { + return err + } + for _, target := range targets { + if a.watches(target.Namespace) { + a.recovery.Add(target) + } + } + if len(targets) == pageSize { + page.After = targets[len(targets)-1].ID + a.pages.Add(page) } - } - if len(revisions) == pageSize { - page.After = revisions[len(revisions)-1] - e.sweeps.Add(page) } return nil }) - return e + return a } -// EnqueueRevision schedules publication from the latest cached grants. The -// revision alone is the key: re-adding it during publication guarantees a rerun. -func (e *EgressReconciler) EnqueueRevision(revision string) { - e.publications.Add(revision) +func (a *ActorEgressApplier) watches(namespace string) bool { + return len(a.namespaces) == 0 || slices.Contains(a.namespaces, namespace) } -// publishRevision is called only by publication workers; the queue serializes -// cache lookup through persistence for each revision. -func (e *EgressReconciler) publishRevision(ctx context.Context, revision string) error { - pinned, err := e.store.GetRuntimeEgress(ctx, revision) - if err != nil { - return err - } - if len(e.namespaces) > 0 && !slices.Contains(e.namespaces, pinned.Namespace) { - return nil - } - base := egress.Access{Destinations: pinned.EgressDestinations, Credentials: pinned.Credentials} - var grants egress.Access - key := policyGrants{Source: database.RuntimeEgressSource{Kind: pinned.Kind, Namespace: pinned.Namespace, Name: pinned.SourceName}} - if current := e.grants.GetKey(key.ResourceName()); current != nil && current.UID == pinned.SourceUID { - grants = current.Access - } - access, err := egress.MergeAccess(base, grants, pinned.TokenPassthroughOrigins) - if err != nil { - // Bound warning state while letting every revision report its own failures. - if _, warned := e.warnings.Get(revision); !warned { - logging.FromContext(ctx).WarnContext(ctx, "explicit egress grants withheld", "revision", revision, "namespace", pinned.Namespace, "source", pinned.SourceName, "error", err) - e.warnings.Add(revision, struct{}{}, 30*time.Second) - } - access = base - } else { - e.warnings.Remove(revision) - } - changed, err := e.store.SetRuntimeEgressPolicy(ctx, revision, access) - if changed || err != nil { - // A failed response may hide a committed write. Queue actor application - // even then: a publication retry could observe an unchanged policy. - e.pages.Add(egressActorPage{Revision: revision}) - } - return err +// EnqueueSource schedules all allocated instances of this source, across UIDs. +func (a *ActorEgressApplier) EnqueueSource(source database.RuntimeEgressSource) { + a.pages.Add(egressSourcePage{Source: source}) } -func (e *EgressReconciler) runPublications(ctx context.Context) { +func (a *ActorEgressApplier) NeedLeaderElection() bool { return true } + +func (a *ActorEgressApplier) Start(ctx context.Context) error { var workers sync.WaitGroup - for range 4 { - workers.Go(func() { - for e.processPublication(ctx) { - } - }) - } - <-ctx.Done() - e.publications.ShutDown() + workers.Go(func() { a.updates.Run(ctx.Done()) }) + workers.Go(func() { a.recovery.Run(ctx.Done()) }) + a.pages.Add(egressRecoveryPage{}) + a.pages.Run(ctx.Done()) workers.Wait() + return nil } -func (e *EgressReconciler) processPublication(ctx context.Context) bool { - revision, shutdown := e.publications.Get() - if shutdown { - return false - } - defer e.publications.Done(revision) - if err := e.publishRevision(ctx, revision); err != nil && !errors.Is(err, database.ErrNotFound) && ctx.Err() == nil { - logging.FromContext(ctx).ErrorContext(ctx, "publish runtime egress policy", "revision", revision, "error", err) - e.publications.AddRateLimited(revision) - } else { - e.publications.Forget(revision) - } - return true -} - -func (e *EgressReconciler) NeedLeaderElection() bool { return true } - -func (e *EgressReconciler) Start(ctx context.Context) error { - handler := e.grants.RegisterBatch(func(events []krt.Event[policyGrants]) { - for _, event := range events { - e.sources.Add(egressSourcePage{Source: event.Latest().Source}) - } - }, false) - defer handler.UnregisterHandler() - queues := []controllers.Queue{e.sources, e.pages, e.updates, e.recovery, e.sweeps} - if !handler.WaitUntilSynced(ctx.Done()) { - e.publications.ShutDown() - for _, queue := range queues { - queue.ShutDownEarly() - } - return ctx.Err() - } - var workers sync.WaitGroup - workers.Go(func() { e.runPublications(ctx) }) - for _, queue := range queues { - workers.Go(func() { queue.Run(ctx.Done()) }) - } - defer workers.Wait() - e.sweeps.Add(egressSweepPage{RecoverActors: true}) - ticker := time.NewTicker(5 * time.Minute) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return nil - case <-ticker.C: - e.sweeps.Add(egressSweepPage{}) - } +func (a *ActorEgressApplier) reconcileActor(ctx context.Context, target database.RuntimeEgressTarget) error { + if !a.watches(target.Namespace) { + return nil } -} - -func (e *EgressReconciler) reconcileActor(ctx context.Context, target database.RuntimeEgressTarget) error { binding := substrate.ActorBinding{Atespace: target.Atespace, Name: substrate.ActorName(target.ID), TemplateAtespace: target.Atespace, TemplateName: target.TemplateName} - actor, err := e.actors.GetActor(ctx, binding.Atespace, binding.Name) + actor, err := a.actors.GetActor(ctx, binding.Atespace, binding.Name) if err != nil { return err } if !binding.Matches(actor) || (target.ActorUID != "" && target.ActorUID != actor.GetMetadata().GetUid()) { - // This queued identity no longer owns the actor. Recovery discovers its - // current binding; retrying this stale target cannot repair it. return nil } - return e.actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, target.Revision) + policy, err := a.store.GetRuntimeEgressPolicy(ctx, target.Revision) + if err != nil { + return err + } + if policy.WithheldGrants != nil { + if _, warned := a.warnings.Get(target.Revision); !warned { + logging.FromContext(ctx).WarnContext(ctx, "explicit egress grants withheld", "revision", target.Revision, "namespace", target.Namespace, "error", policy.WithheldGrants) + a.warnings.Add(target.Revision, struct{}{}, 30*time.Second) + } + } else { + a.warnings.Remove(target.Revision) + } + return a.actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, target.Revision) } diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go index d954324ba3..d09f0051b7 100644 --- a/go/core/internal/controller/egress_test.go +++ b/go/core/internal/controller/egress_test.go @@ -6,9 +6,7 @@ import ( "errors" "fmt" "log/slog" - "reflect" "slices" - "strings" "sync" "testing" "testing/synctest" @@ -16,206 +14,78 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/stretchr/testify/require" - "istio.io/istio/pkg/kube/controllers" - "istio.io/istio/pkg/kube/krt" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/cache" - clocktesting "k8s.io/utils/clock/testing" - "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/pkg/logging" ) -func noopEnqueueEgress(string) {} +func noopEnqueueActors(database.RuntimeEgressSource) {} type egressTestStore struct { - mu sync.Mutex - pinned map[string]database.RuntimeEgress - intended map[string]egress.Access - targets map[string][]database.RuntimeEgressTarget - listed chan database.RuntimeEgressSource - publishErr error - losePublishResponse bool - beforePublish func(context.Context, string, egress.Access) error + mu sync.Mutex + policies map[string]egress.Policy + targets map[database.RuntimeEgressSource][]database.RuntimeEgressTarget } -var _ runtimeEgressStore = (*egressTestStore)(nil) +var _ actorEgressStore = (*egressTestStore)(nil) -func (e *egressTestStore) GetRuntimeEgress(_ context.Context, revision string) (*database.RuntimeEgress, error) { +func (e *egressTestStore) GetRuntimeEgressPolicy(_ context.Context, revision string) (egress.Policy, error) { e.mu.Lock() defer e.mu.Unlock() - row, ok := e.pinned[revision] + policy, ok := e.policies[revision] if !ok { - return nil, database.ErrNotFound - } - return &row, nil -} -func (e *egressTestStore) SetRuntimeEgressPolicy(ctx context.Context, revision string, access egress.Access) (bool, error) { - if e.beforePublish != nil { - if err := e.beforePublish(ctx, revision, access); err != nil { - return false, err - } - } - e.mu.Lock() - defer e.mu.Unlock() - if e.publishErr != nil { - return false, e.publishErr - } - if e.intended == nil { - e.intended = map[string]egress.Access{} - } - if current, exists := e.intended[revision]; exists && reflect.DeepEqual(current, access) { - return false, nil - } - e.intended[revision] = access - if e.losePublishResponse { - e.losePublishResponse = false - return false, errors.New("lost publication response") - } - return true, nil -} -func (e *egressTestStore) GetRuntimeEgressPolicy(_ context.Context, revision string) (egress.Access, error) { - e.mu.Lock() - defer e.mu.Unlock() - policy, ok := e.intended[revision] - if !ok { - return egress.Access{}, database.ErrNotFound + return egress.Policy{}, database.ErrNotFound } return policy, nil } -func (e *egressTestStore) ListRuntimeEgressRevisions(_ context.Context, after string, limit int) ([]string, error) { +func (e *egressTestStore) ListRuntimeEgressSourceTargets(_ context.Context, source database.RuntimeEgressSource, after string, limit int) ([]database.RuntimeEgressTarget, error) { e.mu.Lock() defer e.mu.Unlock() - var revisions []string - for key := range e.pinned { - if key > after { - revisions = append(revisions, key) - } - } - slices.Sort(revisions) - return revisions[:min(limit, len(revisions))], nil + return egressTestPage(e.targets[source], after, limit), nil } -func (e *egressTestStore) ListRuntimeEgressSourceRevisions(_ context.Context, source database.RuntimeEgressSource, after string, limit int) ([]string, error) { +func (e *egressTestStore) ListRuntimeEgressTargets(_ context.Context, after string, limit int) ([]database.RuntimeEgressTarget, error) { e.mu.Lock() defer e.mu.Unlock() - if e.listed != nil { - e.listed <- source - } - var revisions []string - for key, pinned := range e.pinned { - if pinned.Kind == source.Kind && pinned.Namespace == source.Namespace && pinned.SourceName == source.Name && key > after { - revisions = append(revisions, key) - } + var targets []database.RuntimeEgressTarget + for _, rows := range e.targets { + targets = append(targets, rows...) } - slices.Sort(revisions) - return revisions[:min(limit, len(revisions))], nil + return egressTestPage(targets, after, limit), nil } -func (e *egressTestStore) ListRuntimeEgressTargets(_ context.Context, revision, after string, limit int) ([]database.RuntimeEgressTarget, error) { - e.mu.Lock() - defer e.mu.Unlock() - var targets []database.RuntimeEgressTarget - for _, target := range e.targets[revision] { - if target.ID > after { - targets = append(targets, target) +func egressTestPage(targets []database.RuntimeEgressTarget, after string, limit int) []database.RuntimeEgressTarget { + targets = slices.Clone(targets) + slices.SortFunc(targets, func(a, b database.RuntimeEgressTarget) int { + if a.ID < b.ID { + return -1 } - } - return targets[:min(limit, len(targets))], nil + if a.ID > b.ID { + return 1 + } + return 0 + }) + targets = slices.DeleteFunc(targets, func(target database.RuntimeEgressTarget) bool { return target.ID <= after }) + return targets[:min(len(targets), limit)] } -type egressTestInputs struct { - runtime *Runtime - agents krt.StaticCollection[*v1alpha3.Agent] - sandboxes krt.StaticCollection[*v1alpha3.SandboxTemplate] - policies krt.StaticCollection[*v1alpha3.EgressPolicy] - secrets krt.StaticCollection[*corev1.Secret] +type observingEgressActors struct { + store *egressTestStore + mu sync.Mutex + applied map[string]egress.Access + attempts map[string][]time.Time + failures int + mismatched bool } -func newEgressTestInputs(ctx context.Context) egressTestInputs { - opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) - inputs := egressTestInputs{ - agents: krt.NewStaticCollection[*v1alpha3.Agent](nil, nil, opts.WithName("Agents")...), - sandboxes: krt.NewStaticCollection[*v1alpha3.SandboxTemplate](nil, nil, opts.WithName("Sandboxes")...), - policies: krt.NewStaticCollection[*v1alpha3.EgressPolicy](nil, nil, opts.WithName("Policies")...), - secrets: krt.NewStaticCollection[*corev1.Secret](nil, nil, opts.WithName("Secrets")...), - } - inputs.runtime = &Runtime{Options: opts, Collections: Collections{Agents: inputs.agents, SandboxTemplates: inputs.sandboxes, EgressPolicies: inputs.policies, Secrets: inputs.secrets}} - return inputs -} -func (i egressTestInputs) source(kind database.RuntimeEgressSourceKind, uid string, refs []corev1.LocalObjectReference) { - metadata := metav1.ObjectMeta{Namespace: "team", Name: "source", UID: types.UID(uid)} - if kind == database.RuntimeEgressAgent { - i.agents.UpdateObject(&v1alpha3.Agent{ObjectMeta: metadata, Spec: v1alpha3.AgentSpec{EgressPolicyRefs: refs}}) - } else { - i.sandboxes.UpdateObject(&v1alpha3.SandboxTemplate{ObjectMeta: metadata, Spec: v1alpha3.SandboxTemplateSpec{EgressPolicyRefs: refs}}) - } -} -func egressTestPolicy(origin string, auth *v1alpha3.EgressAuthentication) *v1alpha3.EgressPolicy { - return &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: origin, Authentication: auth}}}} -} +var _ actorEgressClient = (*observingEgressActors)(nil) -func TestEgressTranslationUsesCachedDependencies(t *testing.T) { - for _, kind := range []database.RuntimeEgressSourceKind{database.RuntimeEgressAgent, database.RuntimeEgressSandbox} { - t.Run(string(kind), func(t *testing.T) { - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - inputs := newEgressTestInputs(ctx) - refs := []corev1.LocalObjectReference{{Name: "access"}} - inputs.source(kind, "original", refs) - base := egress.Access{Destinations: []string{"https://model.example.com:443"}} - store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"pinned": {Kind: kind, Namespace: "team", SourceName: "source", SourceUID: "original", EgressDestinations: base.Destinations}}} - controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) - done := make(chan error, 1) - go func() { done <- controller.Start(ctx) }() - defer func() { cancel(); require.NoError(t, <-done) }() - expect := func(want egress.Access) { - t.Helper() - require.Eventually(t, func() bool { - got, err := store.GetRuntimeEgressPolicy(ctx, "pinned") - return err == nil && reflect.DeepEqual(want, got) - }, time.Second, 10*time.Millisecond) - } - expect(base) // Missing refs and broken unrelated runtime inputs cannot preserve stale grants. - policy := egressTestPolicy("https://tickets.example.com", nil) - inputs.policies.UpdateObject(policy.DeepCopy()) - allowed := egress.Access{Destinations: []string{"https://model.example.com:443", "https://tickets.example.com:443"}} - expect(allowed) - inputs.source(kind, "original", nil) - expect(base) - inputs.source(kind, "original", refs) - expect(allowed) - inputs.policies.DeleteObject("team/access") - expect(base) - inputs.policies.UpdateObject(policy.DeepCopy()) - expect(allowed) - inputs.source(kind, "replacement", refs) - expect(base) - inputs.source(kind, "original", refs) - expect(allowed) - policy.Spec.Destinations[0].Authentication = &v1alpha3.EgressAuthentication{Header: "authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}, Key: "token"}} - inputs.policies.UpdateObject(policy.DeepCopy()) - expect(base) - inputs.secrets.UpdateObject(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "auth"}, Data: map[string][]byte{"token": []byte("secret")}}) - allowed.Credentials = []egress.Credential{{Origin: "https://tickets.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token"}} - expect(allowed) - inputs.secrets.DeleteObject("team/auth") - expect(base) - }) +func (a *observingEgressActors) GetActor(_ context.Context, space, name string) (*ateapipb.Actor, error) { + template := "runtime" + if a.mismatched { + template = "replacement" } + return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: "uid"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: template}}, nil } - -type observingEgressActors struct { - store *egressTestStore - mu sync.Mutex - applied map[string]egress.Access - attempts map[string][]time.Time - failing string - failures int -} - func (a *observingEgressActors) EnsureActorEgressPolicy(ctx context.Context, _, name, revision string) error { a.mu.Lock() defer a.mu.Unlock() @@ -223,366 +93,98 @@ func (a *observingEgressActors) EnsureActorEgressPolicy(ctx context.Context, _, a.attempts = map[string][]time.Time{} } a.attempts[name] = append(a.attempts[name], time.Now()) - if name == a.failing && len(a.attempts[name]) <= a.failures { + if a.failures > 0 { + a.failures-- return errors.New("backend unavailable") } - access, err := a.store.GetRuntimeEgressPolicy(ctx, revision) + policy, err := a.store.GetRuntimeEgressPolicy(ctx, revision) if err != nil { return err } if a.applied == nil { a.applied = map[string]egress.Access{} } - a.applied[name] = access + a.applied[name] = policy.Access return nil } -func (a *observingEgressActors) GetActor(_ context.Context, space, name string) (*ateapipb.Actor, error) { - return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: "uid"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: "runtime"}}, nil -} -func TestEgressUpdatesRetainedRevisionsAndRecoversAfterRestart(t *testing.T) { - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - inputs := newEgressTestInputs(ctx) - inputs.source(database.RuntimeEgressAgent, "uid", []corev1.LocalObjectReference{{Name: "access"}}) - inputs.policies.UpdateObject(egressTestPolicy("https://tickets.example.com", nil)) - pinned := database.RuntimeEgress{Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"} - old := pinned - old.EgressDestinations = []string{"https://old-model.example.com:443"} - store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"new": pinned, "old": old}, targets: map[string][]database.RuntimeEgressTarget{ - "new": {{ID: "new", Revision: "new", Atespace: "team", TemplateName: "runtime"}}, - "old": {{ID: "old", Revision: "old", Atespace: "team", TemplateName: "runtime"}}, - }} - actors := &observingEgressActors{store: store} - expect := func(granted bool) { - t.Helper() - require.Eventually(t, func() bool { - actors.mu.Lock() - defer actors.mu.Unlock() - current, ok := actors.applied["session-new"] - previous, oldOK := actors.applied["session-old"] - return ok && oldOK && slices.Contains(current.Destinations, "https://tickets.example.com:443") == granted && - slices.Contains(previous.Destinations, "https://tickets.example.com:443") == granted && slices.Contains(previous.Destinations, "https://old-model.example.com:443") - }, 3*time.Second, 10*time.Millisecond) - } - for i := range 2 { - runCtx, stop := context.WithCancel(ctx) - controller := NewEgressReconciler(runCtx, inputs.runtime, store, actors, nil) - done := make(chan error, 1) - go func() { done <- controller.Start(runCtx) }() - if i == 0 { - expect(true) - inputs.policies.DeleteObject("team/access") - expect(false) - } else { - expect(false) - } - stop() - require.NoError(t, <-done) - // Startup repairs existing actors without a new Kubernetes event/invocation. - actors.mu.Lock() - actors.applied = map[string]egress.Access{} - actors.mu.Unlock() - } -} - -func TestEgressSkipsUnchangedPoliciesAndRecoversLostWriteResponse(t *testing.T) { +func TestActorEgressSourcePagingAndStartupRecovery(t *testing.T) { synctest.Test(t, func(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) - inputs := newEgressTestInputs(ctx) - inputs.source(database.RuntimeEgressAgent, "uid", []corev1.LocalObjectReference{{Name: "access"}}) - inputs.policies.UpdateObject(egressTestPolicy("https://tickets.example.com", nil)) - store := &egressTestStore{ - pinned: map[string]database.RuntimeEgress{"revision": {Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"}}, - targets: map[string][]database.RuntimeEgressTarget{"revision": {{ID: "actor", Revision: "revision", Atespace: "team", TemplateName: "runtime"}}}, - losePublishResponse: true, - } - actors := &observingEgressActors{store: store} - controller := NewEgressReconciler(ctx, inputs.runtime, store, actors, nil) - defer controller.sources.ShutDownEarly() - var workers sync.WaitGroup - for _, queue := range []controllers.Queue{controller.pages, controller.updates, controller.recovery, controller.sweeps} { - workers.Go(func() { queue.Run(ctx.Done()) }) - } - workers.Go(func() { controller.runPublications(ctx) }) - defer func() { cancel(); workers.Wait() }() - require.True(t, controller.grants.WaitUntilSynced(ctx.Done())) - controller.EnqueueRevision("revision") - synctest.Wait() - require.Len(t, actors.attempts["session-actor"], 1, "an uncertain write must still apply the persisted policy") - require.Contains(t, actors.applied["session-actor"].Destinations, "https://tickets.example.com:443") - for range 3 { - controller.EnqueueRevision("revision") - controller.sweeps.Add(egressSweepPage{}) - synctest.Wait() - } - require.Len(t, actors.attempts["session-actor"], 1, "unchanged preparation and sweeps must not touch actors") - inputs.policies.DeleteObject("team/access") - synctest.Wait() - controller.sweeps.Add(egressSweepPage{}) - synctest.Wait() - require.Len(t, actors.attempts["session-actor"], 2, "a sweep must repair a missed policy event") - require.Empty(t, actors.applied["session-actor"].Destinations) - controller.sweeps.Add(egressSweepPage{RecoverActors: true}) - synctest.Wait() - require.Len(t, actors.attempts["session-actor"], 3, "startup must also check actors with unchanged policies") - }) -} - -func TestEgressStartupRecoveryRespectsWatchNamespaces(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - ctx, cancel := context.WithCancel(t.Context()) - inputs := newEgressTestInputs(ctx) - store := &egressTestStore{ - pinned: map[string]database.RuntimeEgress{ - "watched": {Kind: database.RuntimeEgressAgent, Namespace: "team"}, - "unwatched": {Kind: database.RuntimeEgressAgent, Namespace: "other"}, - }, - intended: map[string]egress.Access{ - "watched": {}, - "unwatched": {Destinations: []string{"https://tickets.example.com:443"}}, - }, - targets: map[string][]database.RuntimeEgressTarget{ - "watched": {{ID: "watched", Revision: "watched", Atespace: "team", TemplateName: "runtime"}}, - "unwatched": {{ID: "unwatched", Revision: "unwatched", Atespace: "other", TemplateName: "runtime"}}, - }, + source := database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team", Name: "source"} + foreign := database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: "other", Name: "source"} + store := &egressTestStore{policies: map[string]egress.Policy{"old": {}, "new": {}}, targets: map[database.RuntimeEgressSource][]database.RuntimeEgressTarget{}} + for i := range 101 { + revision := "old" + if i%2 == 0 { + revision = "new" + } + store.targets[source] = append(store.targets[source], database.RuntimeEgressTarget{ID: fmt.Sprintf("%03d", i), Revision: revision, Namespace: "team", Atespace: "team", TemplateName: "runtime"}) } + store.targets[foreign] = []database.RuntimeEgressTarget{{ID: "foreign", Revision: "new", Namespace: "other", Atespace: "other", TemplateName: "runtime"}} actors := &observingEgressActors{store: store} - controller := NewEgressReconciler(ctx, inputs.runtime, store, actors, []string{"team"}) + applier := NewActorEgressApplier(ctx, store, actors, []string{"team"}) var workers sync.WaitGroup - workers.Go(func() { require.NoError(t, controller.Start(ctx)) }) + workers.Go(func() { require.NoError(t, applier.Start(ctx)) }) defer func() { cancel(); workers.Wait() }() synctest.Wait() - require.Contains(t, actors.applied, "session-watched") - require.NotContains(t, actors.attempts, "session-unwatched") - unwatched, err := store.GetRuntimeEgressPolicy(ctx, "unwatched") - require.NoError(t, err) - require.Equal(t, []string{"https://tickets.example.com:443"}, unwatched.Destinations) - }) -} - -func TestEgressPublicationRequeuesRevocationDuringWrite(t *testing.T) { - for _, failFirst := range []bool{false, true} { - t.Run(fmt.Sprintf("fail_first=%t", failFirst), func(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - ctx, cancel := context.WithCancel(t.Context()) - inputs := newEgressTestInputs(ctx) - inputs.source(database.RuntimeEgressAgent, "uid", []corev1.LocalObjectReference{{Name: "access"}}) - inputs.policies.UpdateObject(egressTestPolicy("https://tickets.example.com", nil)) - pinned := database.RuntimeEgress{Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"} - entered, release := make(chan struct{}), make(chan struct{}) - store := &egressTestStore{ - pinned: map[string]database.RuntimeEgress{"revision": pinned, "other": pinned}, - beforePublish: func(ctx context.Context, revision string, access egress.Access) error { - if revision != "revision" || len(access.Destinations) == 0 { - return nil - } - close(entered) - select { - case <-release: - case <-ctx.Done(): - return ctx.Err() - } - if failFirst { - return errors.New("publication failed") - } - return nil - }, - } - controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) - var workers sync.WaitGroup - workers.Go(func() { controller.runPublications(ctx) }) - defer func() { - cancel() - workers.Wait() - for _, queue := range []controllers.Queue{controller.sources, controller.pages, controller.updates, controller.recovery, controller.sweeps} { - queue.ShutDownEarly() - } - }() - require.True(t, controller.grants.WaitUntilSynced(ctx.Done())) - controller.EnqueueRevision("revision") - <-entered - inputs.policies.DeleteObject("team/access") - synctest.Wait() - controller.EnqueueRevision("revision") - controller.EnqueueRevision("other") - synctest.Wait() - _, err := store.GetRuntimeEgressPolicy(ctx, "other") - require.NoError(t, err, "another revision must publish while the first database write is blocked") - _, err = store.GetRuntimeEgressPolicy(ctx, "revision") - require.ErrorIs(t, err, database.ErrNotFound, "the same revision must wait through commit") - close(release) - synctest.Wait() - for _, revision := range []string{"revision", "other"} { - access, err := store.GetRuntimeEgressPolicy(ctx, revision) - require.NoError(t, err) - require.Empty(t, access.Destinations, "a dirty revision must rerun with the revoked grants") - } - }) - }) - } -} - -func TestEgressPublicationRetriesWithoutAnotherEvent(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - ctx, cancel := context.WithCancel(t.Context()) - inputs := newEgressTestInputs(ctx) - store := &egressTestStore{ - pinned: map[string]database.RuntimeEgress{"revision": {Kind: database.RuntimeEgressAgent, Namespace: "team"}}, - publishErr: errors.New("database unavailable"), - } - controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) - var workers sync.WaitGroup - workers.Go(func() { require.NoError(t, controller.Start(ctx)) }) - defer func() { cancel(); workers.Wait() }() - synctest.Wait() - _, err := store.GetRuntimeEgressPolicy(ctx, "revision") - require.ErrorIs(t, err, database.ErrNotFound) + require.Len(t, actors.applied, 101, "startup recovery must traverse every page within watch scope") + updated := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} store.mu.Lock() - store.publishErr = nil + store.policies["new"] = egress.Policy{Access: updated} store.mu.Unlock() - time.Sleep(time.Second) + applier.EnqueueSource(source) + applier.EnqueueSource(foreign) synctest.Wait() - _, err = store.GetRuntimeEgressPolicy(ctx, "revision") - require.NoError(t, err) + for name, calls := range actors.attempts { + require.Len(t, calls, 2, name) + } + require.Equal(t, updated, actors.applied["session-000"]) + require.Empty(t, actors.applied["session-001"].Destinations, "old UID revisions retain their independently resolved policy") + require.NotContains(t, actors.applied, "session-foreign") }) } -func TestEgressPostCommitEnqueueRecoversEventBeforeRevisionExists(t *testing.T) { +func TestActorEgressRetriesAndRejectsStaleIdentity(t *testing.T) { synctest.Test(t, func(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) - inputs := newEgressTestInputs(ctx) - inputs.source(database.RuntimeEgressAgent, "uid", []corev1.LocalObjectReference{{Name: "access"}}) - inputs.policies.UpdateObject(egressTestPolicy("https://tickets.example.com", nil)) - store := &egressTestStore{listed: make(chan database.RuntimeEgressSource, 1)} - controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) - require.True(t, controller.grants.WaitUntilSynced(ctx.Done())) + source := database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: "team", Name: "scratch"} + target := database.RuntimeEgressTarget{ID: "actor", Revision: "revision", Namespace: "team", Atespace: "team", TemplateName: "runtime"} + store := &egressTestStore{policies: map[string]egress.Policy{"revision": {}}, targets: map[database.RuntimeEgressSource][]database.RuntimeEgressTarget{source: {target}}} + actors := &observingEgressActors{store: store, failures: 2} + applier := NewActorEgressApplier(ctx, store, actors, nil) var workers sync.WaitGroup - workers.Go(func() { require.NoError(t, controller.Start(ctx)) }) + workers.Go(func() { require.NoError(t, applier.Start(ctx)) }) defer func() { cancel(); workers.Wait() }() synctest.Wait() - // Compilation saw the grant, but its revision has not committed yet. - initial := controller.grants.GetKey("agent/team/source").Access - require.NotEmpty(t, initial.Destinations) - inputs.policies.DeleteObject("team/access") + time.Sleep(time.Second) synctest.Wait() - require.Len(t, store.listed, 1, "revocation discovery must run before the revision exists") - require.Equal(t, database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team", Name: "source"}, <-store.listed) - store.mu.Lock() - store.pinned = map[string]database.RuntimeEgress{"revision": {Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"}} - store.intended = map[string]egress.Access{"revision": initial} - store.mu.Unlock() - controller.EnqueueRevision("revision") + time.Sleep(2 * time.Second) synctest.Wait() - policy, err := store.GetRuntimeEgressPolicy(ctx, "revision") - require.NoError(t, err) - require.Empty(t, policy.Destinations, "post-commit publication must apply the missed revocation") + attempts := actors.attempts["session-actor"] + require.Len(t, attempts, 3) + require.Equal(t, time.Second, attempts[1].Sub(attempts[0])) + require.Equal(t, 2*time.Second, attempts[2].Sub(attempts[1])) + actors.mismatched = true + require.NoError(t, applier.reconcileActor(ctx, target)) + require.Len(t, actors.attempts["session-actor"], 3) }) } -func TestEgressTargetsSourceChangesAndBacksOffActorFailures(t *testing.T) { - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - inputs := newEgressTestInputs(ctx) - inputs.source(database.RuntimeEgressAgent, "uid", nil) - store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"revision": {Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"}}, - intended: map[string]egress.Access{"revision": {}}, - listed: make(chan database.RuntimeEgressSource, 20), targets: map[string][]database.RuntimeEgressTarget{"revision": { - {ID: "failing", Revision: "revision", Atespace: "team", TemplateName: "runtime"}, - {ID: "healthy", Revision: "revision", Atespace: "team", TemplateName: "runtime"}, - }}} - actors := &observingEgressActors{store: store, failing: "session-failing", failures: 2} - controller := NewEgressReconciler(ctx, inputs.runtime, store, actors, nil) - // Exercise recovery backoff independently of initial informer delivery. - require.True(t, controller.grants.WaitUntilSynced(ctx.Done())) - done := make(chan error, 1) - go func() { done <- controller.Start(ctx) }() - defer func() { cancel(); require.NoError(t, <-done) }() - require.Eventually(t, func() bool { - actors.mu.Lock() - defer actors.mu.Unlock() - return len(actors.attempts["session-failing"]) == 3 - }, 5*time.Second, 10*time.Millisecond) - actors.mu.Lock() - attempts := slices.Clone(actors.attempts["session-failing"]) - healthy := len(actors.attempts["session-healthy"]) - actors.mu.Unlock() - require.GreaterOrEqual(t, attempts[1].Sub(attempts[0]), time.Second) - require.GreaterOrEqual(t, attempts[2].Sub(attempts[1]), 2*time.Second) - require.Equal(t, 1, healthy) - inputs.agents.UpdateObject(&v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "unrelated", UID: "other"}}) - source := database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team", Name: "unrelated"} - select { - case got := <-store.listed: - require.Equal(t, source, got) - case <-ctx.Done(): - t.Fatal("source change was not queued") - } - inputs.agents.DeleteObject("team/unrelated") - select { - case got := <-store.listed: - require.Equal(t, source, got) - case <-ctx.Done(): - t.Fatal("source deletion was not queued") - } -} - -func TestEgressWithholdsConflictingGrantsAndWarns(t *testing.T) { - for _, reason := range []string{"credential", "passthrough", "rule limit"} { - t.Run(reason, func(t *testing.T) { - var logs bytes.Buffer - ctx := logging.IntoContext(t.Context(), slog.New(slog.NewJSONHandler(&logs, nil))) - inputs := newEgressTestInputs(ctx) - inputs.source(database.RuntimeEgressAgent, "uid", []corev1.LocalObjectReference{{Name: "access"}}) - base := egress.Access{Destinations: []string{"https://model.example.com:443"}} - pinned := database.RuntimeEgress{Kind: database.RuntimeEgressAgent, Namespace: "team", SourceName: "source", SourceUID: "uid"} - authentication := &v1alpha3.EgressAuthentication{Header: "authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "other"}, Key: "token"}} - switch reason { - case "credential": - base.Credentials = []egress.Credential{{Origin: base.Destinations[0], Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token"}} - case "passthrough": - pinned.TokenPassthroughOrigins = base.Destinations - case "rule limit": - base.Destinations = nil - for i := range 256 { - base.Destinations = append(base.Destinations, fmt.Sprintf("https://host-%03d.example.com:443", i)) - } - authentication = nil - } - pinned.EgressDestinations, pinned.Credentials = base.Destinations, base.Credentials - inputs.policies.UpdateObject(egressTestPolicy("https://model.example.com", authentication)) - inputs.secrets.UpdateObject(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "other"}, Data: map[string][]byte{"token": []byte("private-value")}}) - store := &egressTestStore{pinned: map[string]database.RuntimeEgress{"revision": pinned, "other": pinned}} - controller := NewEgressReconciler(ctx, inputs.runtime, store, &observingEgressActors{store: store}, nil) - defer controller.publications.ShutDown() - defer func() { - for _, queue := range []controllers.Queue{controller.sources, controller.pages, controller.updates, controller.recovery, controller.sweeps} { - queue.ShutDownEarly() - } - }() - require.True(t, controller.grants.WaitUntilSynced(ctx.Done())) - publish := func(revision string) { - controller.EnqueueRevision(revision) - require.True(t, controller.processPublication(ctx)) - } - clock := clocktesting.NewFakeClock(time.Now()) - controller.warnings = cache.NewLRUExpireCacheWithClock(1024, clock) - for range 3 { - publish("revision") - got, err := store.GetRuntimeEgressPolicy(ctx, "revision") - require.NoError(t, err) - require.Equal(t, base, got) - } - require.Equal(t, 1, strings.Count(logs.String(), "explicit egress grants withheld")) - publish("other") - require.Contains(t, logs.String(), `"revision":"other"`, "a noisy revision must not suppress another revision's warning") - require.Equal(t, 2, strings.Count(logs.String(), "explicit egress grants withheld")) - clock.Step(31 * time.Second) - publish("revision") - require.Equal(t, 3, strings.Count(logs.String(), "explicit egress grants withheld")) - require.Contains(t, logs.String(), `"level":"WARN"`) - require.Contains(t, logs.String(), `"revision":"revision"`) - require.NotContains(t, logs.String(), "private-value") - }) - } +func TestActorEgressWarningsAreLimitedPerRevision(t *testing.T) { + var logs bytes.Buffer + ctx := logging.IntoContext(t.Context(), slog.New(slog.NewTextHandler(&logs, nil))) + store := &egressTestStore{policies: map[string]egress.Policy{ + "first": {WithheldGrants: errors.New("credential conflict")}, "second": {WithheldGrants: errors.New("rule limit")}, + }} + actors := &observingEgressActors{store: store} + applier := NewActorEgressApplier(ctx, store, actors, nil) + defer applier.pages.ShutDownEarly() + defer applier.updates.ShutDownEarly() + defer applier.recovery.ShutDownEarly() + for _, revision := range []string{"first", "first", "second"} { + require.NoError(t, applier.reconcileActor(ctx, database.RuntimeEgressTarget{ID: revision, Revision: revision, Namespace: "team", Atespace: "team", TemplateName: "runtime"})) + } + require.Equal(t, 2, bytes.Count(logs.Bytes(), []byte("explicit egress grants withheld"))) + require.Len(t, actors.applied, 2, "warnings must not block fallback application") } diff --git a/go/core/internal/controller/equality_test.go b/go/core/internal/controller/equality_test.go index 373b37a3af..5acd2939c8 100644 --- a/go/core/internal/controller/equality_test.go +++ b/go/core/internal/controller/equality_test.go @@ -56,7 +56,7 @@ func TestAgentReconciliationEquality(t *testing.T) { }, equal: true}, {name: "template source", change: func(r *AgentReconciliation) { r.Agent.Generation++ }}, {name: "live grants", change: func(r *AgentReconciliation) { - r.Target.Egress.Destinations = []string{"https://tickets.example.com:443"} + r.Grants.Destinations = []string{"https://tickets.example.com:443"} }}, {name: "revision identity", change: func(r *AgentReconciliation) { r.Target.RevisionID[0]++ }}, {name: "revision inputs", change: func(r *AgentReconciliation) { r.Target.Revision.SandboxClass = "microvm" }}, diff --git a/go/core/internal/controller/grants_test.go b/go/core/internal/controller/grants_test.go new file mode 100644 index 0000000000..bcfc742991 --- /dev/null +++ b/go/core/internal/controller/grants_test.go @@ -0,0 +1,141 @@ +package controller + +import ( + "context" + "errors" + "sync" + "testing" + "testing/synctest" + + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/substrate" +) + +func TestAgentGrantChangesReconcileWithoutRuntimeChange(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + collections, harnesses := newPreparationTestCollections(t, "gvisor") + initial := collections.Reconciliations.List()[0] + template := proto.CloneOf(initial.Target.ActorTemplate) + template.Metadata.Uid = "prepared" + template.Status = sandboxGoldenStatus() + store := &fakeRuntimeRevisionStore{} + backend := &preparationTestBackend{templates: &fakeActorTemplates{template: template}, store: store} + var sources []database.RuntimeEgressSource + r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3(), func(source database.RuntimeEgressSource) { sources = append(sources, source) }) + ctx, cancel := context.WithCancel(t.Context()) + var workers sync.WaitGroup + workers.Go(func() { r.Run(ctx.Done()) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + sources = nil + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(policy) + agent := initial.Agent.DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + synctest.Wait() + backend.inspect(func() { + require.Equal(t, initial.Target.RevisionID.String(), store.pair.DesiredRevision) + require.Equal(t, []string{"https://tickets.example.com:443"}, store.pair.Grants.Destinations) + }) + require.Equal(t, []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}, sources) + sources = nil + harnesses.DeleteObject("team-a/byo") + synctest.Wait() + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.NotEmpty(t, store.pair.Grants.Destinations) + }) + require.Empty(t, sources, "unrelated runtime failure must not fan out unchanged grants") + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).DeleteObject("team-a/access") + synctest.Wait() + backend.inspect(func() { require.Empty(t, store.pair.DesiredRevision); require.Empty(t, store.pair.Grants.Destinations) }) + require.Len(t, sources, 1, "revocation must reconcile even while the runtime remains unresolvable") + }) +} + +func TestAgentGrantWriteFailureAndRetirementEnqueueActors(t *testing.T) { + collections, _ := newPreparationTestCollections(t, "gvisor") + state := collections.Reconciliations.List()[0] + store := &fakeRuntimeRevisionStore{pairErr: errors.New("response lost")} + var sources []database.RuntimeEgressSource + r := &Reconciler{collections: collections, store: store, enqueueActors: func(source database.RuntimeEgressSource) { sources = append(sources, source) }, status: kagentfake.NewSimpleClientset(state.Agent.DeepCopy()).ApiV1alpha3()} + require.ErrorIs(t, r.reconcileAgent(t.Context(), state.ResourceName()), store.pairErr) + require.Len(t, sources, 1, "a failed response can hide a committed grants update") + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).DeleteObject(state.ResourceName()) + waitFor(t, func() bool { return collections.Reconciliations.GetKey(state.ResourceName()) == nil }) + require.NoError(t, r.reconcileAgent(t.Context(), state.ResourceName())) + require.Len(t, sources, 2) + require.Equal(t, sources[0], sources[1]) +} + +func TestAgentFinalizerRetiresGrantsAfterOfflineDeletion(t *testing.T) { + collections, _ := newPreparationTestCollections(t, "gvisor") + agent := collections.Agents.List()[0].DeepCopy() + agent.Finalizers = nil + agents := collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]) + agents.UpdateObject(agent) + waitFor(t, func() bool { return len(collections.Reconciliations.GetKey("team-a/assistant").Agent.Finalizers) == 0 }) + client := kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3() + store := &fakeRuntimeRevisionStore{} + var sources []database.RuntimeEgressSource + r := &Reconciler{collections: collections, store: store, status: client, enqueueActors: func(source database.RuntimeEgressSource) { sources = append(sources, source) }} + require.NoError(t, r.reconcileAgent(t.Context(), "team-a/assistant")) + require.Zero(t, store.pairCalls, "deletion ownership must be durable before grants") + updated, err := client.Agents("team-a").Get(t.Context(), "assistant", metav1.GetOptions{}) + require.NoError(t, err) + require.Contains(t, updated.Finalizers, agentPreparationFinalizer) + updated.DeletionTimestamp = &metav1.Time{Time: metav1.Now().Time} + agents.UpdateObject(updated) + waitFor(t, func() bool { + return !collections.Reconciliations.GetKey("team-a/assistant").Agent.DeletionTimestamp.IsZero() + }) + require.NoError(t, r.reconcileAgent(t.Context(), "team-a/assistant")) + require.NotEmpty(t, store.retired) + require.Len(t, sources, 1) + persisted, err := client.Agents("team-a").Get(t.Context(), "assistant", metav1.GetOptions{}) + require.NoError(t, err) + require.NotContains(t, persisted.Finalizers, agentPreparationFinalizer) +} + +func TestSandboxGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { + opts := krt.NewOptionsBuilder(t.Context().Done(), "test-grants", nil) + template := sandboxTestTemplate() + template.Finalizers = []string{sandboxPreparationFinalizer} + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + templates := krt.NewStaticCollection(nil, []*v1alpha3.SandboxTemplate{template}, opts.WithName("Templates")...) + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} + policies := krt.NewStaticCollection(nil, []*v1alpha3.EgressPolicy{policy}, opts.WithName("Policies")...) + pools := krt.NewStaticCollection[*atev1alpha1.WorkerPool](nil, nil, opts.WithName("Pools")...) + store := &sandboxTestStore{} + var sources []database.RuntimeEgressSource + s := &SandboxReconciler{collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools, EgressPolicies: policies}, substrate.SandboxPolicy{}, opts), store: store, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3(), enqueueActors: func(source database.RuntimeEgressSource) { sources = append(sources, source) }} + waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch") != nil }) + require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) + require.Nil(t, s.collections.states.GetKey("team-a/scratch").DesiredActorTemplate, "missing WorkerPool prevents runtime compilation") + require.Equal(t, []string{"https://tickets.example.com:443"}, store.desired.Grants.Destinations) + require.Len(t, sources, 1) + policies.DeleteObject("team-a/access") + waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch").CompilationError != "" }) + require.Error(t, s.reconcile(t.Context(), "team-a/scratch")) + require.Empty(t, store.desired.Grants.Destinations) + require.Len(t, sources, 2, "revocation must reach existing actors during failed compilation") + store.upsertErr = errors.New("response lost") + require.ErrorIs(t, s.reconcile(t.Context(), "team-a/scratch"), store.upsertErr) + require.Len(t, sources, 3) + template = template.DeepCopy() + template.DeletionTimestamp = &metav1.Time{Time: metav1.Now().Time} + templates.UpdateObject(template) + waitFor(t, func() bool { return !s.collections.states.GetKey("team-a/scratch").Template.DeletionTimestamp.IsZero() }) + require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) + require.Len(t, sources, 4) +} diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index 63adb535eb..c86adc7a1a 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -27,7 +27,6 @@ import ( kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -55,7 +54,7 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { templates.template.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}, }} - reconciler := &Reconciler{enqueueEgress: noopEnqueueEgress, collections: collections, templates: templates, store: store} + reconciler := &Reconciler{enqueueActors: noopEnqueueActors, collections: collections, templates: templates, store: store} require.NoError(t, reconciler.reconcileAgent(ctx, initial.ResourceName())) updatedHarness := harnesses.List()[0].DeepCopy() @@ -98,7 +97,7 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { // A delayed success for the abandoned preparation must not replace A. abandoned, err := store.GetRuntimeRevision(ctx, preparing.Target.RevisionID.String()) require.NoError(t, err) - require.NoError(t, store.RecordRuntimeRevision(ctx, *abandoned, egress.Access{})) + require.NoError(t, store.RecordRuntimeRevision(ctx, *abandoned)) require.NoError(t, store.PromoteRuntimeRevision(ctx, abandoned.Revision)) unreferenced, err = store.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) @@ -157,7 +156,7 @@ func TestPreparationErrorsPublishStatusAndRecover(t *testing.T) { collections, _ := newPreparationTestCollections(t, "microvm") initial := collections.Reconciliations.List()[0] statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() - reconciler := &Reconciler{enqueueEgress: noopEnqueueEgress, collections: collections, templates: &test.templates, store: &test.store, status: statusClient} + reconciler := &Reconciler{enqueueActors: noopEnqueueActors, collections: collections, templates: &test.templates, store: &test.store, status: statusClient} err := reconciler.reconcileAgent(t.Context(), initial.ResourceName()) require.Error(t, err) require.Equal(t, test.code, status.Code(err)) @@ -238,7 +237,7 @@ func TestPreparationRevisionIsolation(t *testing.T) { collections, harnesses := newPreparationTestCollections(t, "microvm") initial := collections.Reconciliations.List()[0] templates := &fakeActorTemplates{createErr: status.Error(codes.FailedPrecondition, `SandboxConfig "microvm" not found`)} - reconciler := &Reconciler{enqueueEgress: noopEnqueueEgress, collections: collections, templates: templates, store: &fakeRuntimeRevisionStore{}} + reconciler := &Reconciler{enqueueActors: noopEnqueueActors, collections: collections, templates: templates, store: &fakeRuntimeRevisionStore{}} require.Error(t, reconciler.reconcileAgent(t.Context(), initial.ResourceName())) waitFor(t, func() bool { return collections.Reconciliations.GetKey(initial.ResourceName()).PreparationFailure != nil @@ -280,7 +279,7 @@ func TestPreparationQueueAndPollerBackoff(t *testing.T) { initial := collections.Reconciliations.List()[0] statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() backend := &preparationTestBackend{templates: &test.templates, store: &test.store} - r := newReconciler(collections, backend, backend, statusClient, noopEnqueueEgress) + r := newReconciler(collections, backend, backend, statusClient, noopEnqueueActors) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -383,16 +382,16 @@ func (p *preparationTestBackend) CreateActorTemplate(ctx context.Context, templa return p.templates.CreateActorTemplate(ctx, template) } -func (p *preparationTestBackend) UpsertAgentDefinition(ctx context.Context, definition database.AgentDefinition) error { +func (p *preparationTestBackend) UpsertAgentDefinition(ctx context.Context, definition database.AgentDefinition) (bool, error) { p.mu.Lock() defer p.mu.Unlock() return p.store.UpsertAgentDefinition(ctx, definition) } -func (p *preparationTestBackend) RecordRuntimeRevision(ctx context.Context, revision database.RuntimeRevision, policy egress.Access) error { +func (p *preparationTestBackend) RecordRuntimeRevision(ctx context.Context, revision database.RuntimeRevision) error { p.mu.Lock() defer p.mu.Unlock() - return p.store.RecordRuntimeRevision(ctx, revision, policy) + return p.store.RecordRuntimeRevision(ctx, revision) } func (p *preparationTestBackend) PromoteRuntimeRevision(ctx context.Context, revision string) error { @@ -413,7 +412,7 @@ func TestPreparationDesiredChangesBypassBackoff(t *testing.T) { initial := collections.Reconciliations.List()[0] templates := &fakeActorTemplates{createErr: status.Error(codes.FailedPrecondition, "missing config")} store := &fakeRuntimeRevisionStore{} - r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3(), noopEnqueueEgress) + r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3(), noopEnqueueActors) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -467,7 +466,7 @@ func TestPreparationTerminalFailures(t *testing.T) { } store := &fakeRuntimeRevisionStore{} statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() - r := newReconciler(collections, templates, store, statusClient, noopEnqueueEgress) + r := newReconciler(collections, templates, store, statusClient, noopEnqueueActors) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -526,7 +525,7 @@ func TestInvalidActorTemplateHasNoCompiledTarget(t *testing.T) { require.Nil(t, state.PreparationFailure) store := &fakeRuntimeRevisionStore{} templates := &fakeActorTemplates{getErr: errors.New("Substrate must not be called for an invalid target")} - r := &Reconciler{enqueueEgress: noopEnqueueEgress, collections: collections, templates: templates, store: store} + r := &Reconciler{enqueueActors: noopEnqueueActors, collections: collections, templates: templates, store: store} require.NoError(t, r.reconcileAgent(t.Context(), initial.ResourceName())) require.Empty(t, store.pair.DesiredRevision) require.Nil(t, templates.template) @@ -581,39 +580,3 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections }) return collections, harnesses } - -func TestPreparationRecordsCompiledPolicyBeforeEnqueueAndPromotion(t *testing.T) { - collections, _ := newPreparationTestCollections(t, "gvisor") - policy := egressTestPolicy("https://tickets.example.com", nil) - policy.Namespace = "team-a" - collections.EgressPolicies.(krt.StaticCollection[*kagentv1alpha3.EgressPolicy]).UpdateObject(policy) - agent := collections.Agents.List()[0].DeepCopy() - agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} - collections.Agents.(krt.StaticCollection[*kagentv1alpha3.Agent]).UpdateObject(agent) - waitFor(t, func() bool { - state := collections.Reconciliations.GetKey("team-a/assistant") - return state.Target != nil && len(state.Target.Egress.Destinations) == 1 - }) - initial := collections.Reconciliations.List()[0] - template := proto.CloneOf(initial.Target.ActorTemplate) - template.Metadata.Uid = "prepared" - template.Status = sandboxGoldenStatus() - want, err := egress.MergeAccess(egress.Access{Destinations: initial.Target.Revision.EgressDestinations, Credentials: initial.Target.Revision.Credentials}, initial.Target.Egress, initial.Target.Revision.TokenPassthroughOrigins) - require.NoError(t, err) - store := &fakeRuntimeRevisionStore{revisionErr: errors.New("record failed")} - queued := 0 - r := &Reconciler{collections: collections, templates: &fakeActorTemplates{template: template}, store: store, - enqueueEgress: func(revision string) { - queued++ - require.Equal(t, initial.Target.RevisionID.String(), revision) - require.Equal(t, want, store.policy, "initial policy must be recorded before enqueue") - require.False(t, store.markedSuccessful) - }, - } - require.ErrorIs(t, r.reconcileAgent(t.Context(), initial.ResourceName()), store.revisionErr) - require.Zero(t, queued, "failed recording must not enqueue publication") - store.revisionErr = nil - require.NoError(t, r.reconcileAgent(t.Context(), initial.ResourceName())) - require.Equal(t, 1, queued) - require.True(t, store.markedSuccessful, "initial readiness must not wait for the publication queue") -} diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index efebeab42a..0d2c86dab1 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "reflect" + "slices" "strings" "time" @@ -35,11 +36,14 @@ import ( kagenttranslator "github.com/kagent-dev/kagent/go/core/internal/translator/kagent" ) +const agentPreparationFinalizer = "kagent.dev/agent-preparation" + // AgentReconciliation is the complete desired and observed state for one // Agent. Compilation failures are data so invalid Agents still produce // status instead of disappearing from the graph. type AgentReconciliation struct { Agent *kagentv1alpha3.Agent + Grants egress.Access Target *compiledTarget Warnings []string CompilationFailure *ReconciliationFailure @@ -51,7 +55,6 @@ type AgentReconciliation struct { // construction all succeed. An absent target means there is no desired runtime. type compiledTarget struct { Revision translator.Revision - Egress egress.Access RevisionID translator.RevisionID ActorTemplate *ateapipb.ActorTemplate } @@ -61,7 +64,7 @@ func (c *compiledTarget) equals(other *compiledTarget) bool { return c == other } return c.RevisionID == other.RevisionID && c.Revision.Equals(other.Revision) && - proto.Equal(c.ActorTemplate, other.ActorTemplate) && reflect.DeepEqual(c.Egress, other.Egress) + proto.Equal(c.ActorTemplate, other.ActorTemplate) } func (r AgentReconciliation) ResourceName() string { return r.Agent.Namespace + "/" + r.Agent.Name } @@ -104,6 +107,15 @@ func newAgentReconciliations( ) krt.Collection[AgentReconciliation] { return krt.NewCollection(agents, func(ctx krt.HandlerContext, agent *kagentv1alpha3.Agent) *AgentReconciliation { state := &AgentReconciliation{Agent: agent} + if !agent.DeletionTimestamp.IsZero() { + return state + } + grants, grantErr := translator.CompileEgressPolicies(ctx, collections, agent.Namespace, agent.Spec.EgressPolicyRefs) + if grantErr != nil { + state.CompilationFailure = &ReconciliationFailure{Condition: kagentv1alpha3.AgentConditionResolvedRefs, Reason: "ReferenceResolutionFailed", Message: grantErr.Error()} + return state + } + state.Grants = grants compilation, err := translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings), @@ -139,7 +151,7 @@ func newAgentReconciliations( state.CompilationFailure = &ReconciliationFailure{Condition: kagentv1alpha3.AgentConditionCompatible, Reason: "ActorTemplateInvalid", Message: err.Error()} return state } - state.Target = &compiledTarget{Revision: *revision, Egress: compilation.Egress, RevisionID: revisionID, ActorTemplate: actorTemplate} + state.Target = &compiledTarget{Revision: *revision, RevisionID: revisionID, ActorTemplate: actorTemplate} observed := krt.FetchOne(ctx, agentRuntimeObservations, krt.FilterKey(state.ResourceName())) if observed == nil || observed.RevisionID != state.Target.RevisionID { @@ -167,8 +179,8 @@ func newAgentReconciliations( // Substrate owns ActorTemplates; the database retains revisions while an Agent // or a Session or checkpoint references them. type runtimeRevisionStore interface { - UpsertAgentDefinition(context.Context, database.AgentDefinition) error - RecordRuntimeRevision(context.Context, database.RuntimeRevision, egress.Access) error + UpsertAgentDefinition(context.Context, database.AgentDefinition) (bool, error) + RecordRuntimeRevision(context.Context, database.RuntimeRevision) error PromoteRuntimeRevision(context.Context, string) error RetireAgentIdentities(ctx context.Context, namespace, name string, except *database.AgentDefinition) error } @@ -185,7 +197,7 @@ type Reconciler struct { collections Collections templates actorTemplateClient store runtimeRevisionStore - enqueueEgress func(string) + enqueueActors func(database.RuntimeEgressSource) status kagentclient.ApiV1alpha3Interface agents controllers.Queue @@ -198,12 +210,12 @@ type Reconciler struct { // NewReconciler creates the Kubernetes and database write boundary. Run starts // its queues after the registered KRT handlers have received initial state. -func NewReconciler(config *rest.Config, collections Collections, store runtimeRevisionStore, templates actorTemplateClient, enqueueEgress func(string)) (*Reconciler, error) { +func NewReconciler(config *rest.Config, collections Collections, store runtimeRevisionStore, templates actorTemplateClient, enqueueActors func(database.RuntimeEgressSource)) (*Reconciler, error) { statusClient, err := kagentclient.NewForConfig(config) if err != nil { return nil, fmt.Errorf("create kagent status client: %w", err) } - return newReconciler(collections, templates, store, statusClient, enqueueEgress), nil + return newReconciler(collections, templates, store, statusClient, enqueueActors), nil } func newReconciler( @@ -211,13 +223,13 @@ func newReconciler( templates actorTemplateClient, store runtimeRevisionStore, status kagentclient.ApiV1alpha3Interface, - enqueueEgress func(string), + enqueueActors func(database.RuntimeEgressSource), ) *Reconciler { r := &Reconciler{ collections: collections, templates: templates, store: store, - enqueueEgress: enqueueEgress, + enqueueActors: enqueueActors, status: status, } r.agents = newReconciliationQueue("agents", func(item any) error { @@ -231,10 +243,11 @@ func newReconciler( }) r.agentHandler = collections.Reconciliations.Register(func(event krt.Event[AgentReconciliation]) { - // Observations and status writes must not bypass failure backoff. Only - // a changed desired runtime or Agent identity needs immediate work. + // Desired inputs and cleanup changes bypass failure backoff; observations + // and status writes do not. Grants must reconcile without a new revision. if event.Old != nil && event.New != nil && event.Old.Agent.UID == event.New.Agent.UID && - event.Old.desiredRevision() == event.New.desiredRevision() { + slices.Equal(event.Old.Agent.Finalizers, event.New.Agent.Finalizers) && event.Old.desiredRevision() == event.New.desiredRevision() && + reflect.DeepEqual(event.Old.Grants, event.New.Grants) && event.Old.Agent.DeletionTimestamp.Equal(event.New.Agent.DeletionTimestamp) { return } r.agents.Add(krt.GetKey(event.Latest())) @@ -312,26 +325,48 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { (state == nil || state.Target == nil || observation.RevisionID != state.Target.RevisionID) { r.collections.AgentRuntimeObservations.DeleteObject(key) } - if state == nil { + if state == nil || !state.Agent.DeletionTimestamp.IsZero() { parts := strings.Split(key, "/") if len(parts) != 2 { return fmt.Errorf("invalid Agent key %q", key) } - if err := r.store.RetireAgentIdentities(ctx, parts[0], parts[1], nil); err != nil { + err := r.store.RetireAgentIdentities(ctx, parts[0], parts[1], nil) + r.enqueueActors(database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: parts[0], Name: parts[1]}) + if err != nil { return fmt.Errorf("retire Agent %s: %w", key, err) } + if state != nil && slices.Contains(state.Agent.Finalizers, agentPreparationFinalizer) { + updated := state.Agent.DeepCopy() + updated.Finalizers = slices.DeleteFunc(updated.Finalizers, func(value string) bool { return value == agentPreparationFinalizer }) + if _, err := r.status.Agents(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("remove Agent %s finalizer: %w", key, err) + } + } + return nil + } + // Retain deletion events through controller downtime before persisting grants. + if !slices.Contains(state.Agent.Finalizers, agentPreparationFinalizer) { + updated := state.Agent.DeepCopy() + updated.Finalizers = append(updated.Finalizers, agentPreparationFinalizer) + if _, err := r.status.Agents(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("add Agent %s finalizer: %w", key, err) + } return nil } definition := database.AgentDefinition{ Namespace: state.Agent.Namespace, AgentName: state.Agent.Name, - AgentUID: string(state.Agent.UID), DesiredRevision: state.desiredRevision(), + AgentUID: string(state.Agent.UID), DesiredRevision: state.desiredRevision(), Grants: state.Grants, } // Store the desired edge before creating compute so a concurrent collector // cannot mistake the revision for abandoned state. Unresolved inputs clear // the old desired edge without inventing a runtime revision; // the upsert preserves the current UID's last-good runtime in either case. - if err := r.store.UpsertAgentDefinition(ctx, definition); err != nil { + changed, err := r.store.UpsertAgentDefinition(ctx, definition) + if changed || err != nil { + r.enqueueActors(database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: definition.Namespace, Name: definition.AgentName}) + } + if err != nil { return r.observePreparationError(*state, fmt.Errorf("store Agent %s: %w", key, err)) } if state.Target == nil { @@ -380,16 +415,9 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { ActorTemplateAtespace: observed.GetMetadata().GetAtespace(), ActorTemplateName: observed.GetMetadata().GetName(), ActorTemplateUID: observed.GetMetadata().GetUid(), } ready := observed.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() != nil - policy, err := egress.MergeAccess(egress.Access{Destinations: revision.EgressDestinations, Credentials: revision.Credentials}, target.Egress, revision.TokenPassthroughOrigins) - if err != nil { - return r.observePreparationError(*state, fmt.Errorf("merge runtime egress %s: %w", target.RevisionID, err)) - } - if err := r.store.RecordRuntimeRevision(ctx, revision, policy); err != nil { + if err := r.store.RecordRuntimeRevision(ctx, revision); err != nil { return r.observePreparationError(*state, fmt.Errorf("store runtime revision %s: %w", target.RevisionID, err)) } - // Catch grants that changed after compilation but before the revision was - // visible to source-event discovery. Preparation never replaces live policy. - r.enqueueEgress(revision.Revision) if ready { if err := r.store.PromoteRuntimeRevision(ctx, revision.Revision); err != nil { return r.observePreparationError(*state, fmt.Errorf("promote runtime revision %s: %w", target.RevisionID, err)) diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index 4172998268..310b5c680b 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -3,6 +3,7 @@ package controller import ( "context" "errors" + "reflect" "sync/atomic" "testing" "testing/synctest" @@ -26,7 +27,6 @@ import ( kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -34,7 +34,7 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { stop := make(chan struct{}) t.Cleanup(func() { close(stop) }) opts := krt.NewOptionsBuilder(stop, "test", nil) - template := &kagentv1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "assistant", UID: "template-uid"}} + template := &kagentv1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Finalizers: []string{agentPreparationFinalizer}, Namespace: "team-a", Name: "assistant", UID: "template-uid"}} desiredActor := &ateapipb.ActorTemplate{Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "assistant-kagent-revision"}} revision := &translator.Revision{AgentCard: &a2apb.AgentCard{Name: "assistant"}} revision.AgentCard.ProtoReflect().SetUnknown(protowire.AppendString(protowire.AppendTag(nil, 1000, protowire.BytesType), "future")) @@ -56,7 +56,7 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { store := &fakeRuntimeRevisionStore{} templates := &fakeActorTemplates{} statusClient := kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3() - reconciler := &Reconciler{enqueueEgress: noopEnqueueEgress, + reconciler := &Reconciler{enqueueActors: noopEnqueueActors, collections: Collections{ Agents: krttest.GetMockCollection[*kagentv1alpha3.Agent](mock), AgentRuntimeObservations: krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...), @@ -110,7 +110,7 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { } // A fresh reconciler must clean up observations without remembering earlier calls. - reconciler = &Reconciler{enqueueEgress: noopEnqueueEgress, collections: reconciler.collections, templates: templates, store: store, status: statusClient} + reconciler = &Reconciler{enqueueActors: noopEnqueueActors, collections: reconciler.collections, templates: templates, store: store, status: statusClient} nextTarget := *state.Target state.Target = &nextTarget state.Target.ActorTemplate = proto.CloneOf(state.Target.ActorTemplate) @@ -158,7 +158,7 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) states := krt.NewStaticCollection[AgentReconciliation](nil, nil, opts.WithName("Reconciliations")...) templates := &fakeActorTemplates{} - reconciler := &Reconciler{enqueueEgress: noopEnqueueEgress, + reconciler := &Reconciler{enqueueActors: noopEnqueueActors, collections: Collections{ AgentRuntimeObservations: krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...), Reconciliations: states, @@ -178,7 +178,7 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}, }} state := AgentReconciliation{ - Agent: &kagentv1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "assistant", UID: "agent-uid"}}, + Agent: &kagentv1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Finalizers: []string{agentPreparationFinalizer}, Namespace: "team-a", Name: "assistant", UID: "agent-uid"}}, Target: &compiledTarget{Revision: *revision, RevisionID: id, ActorTemplate: desired}, } states.UpdateObject(state) @@ -322,7 +322,6 @@ func (f *fakeActorTemplates) DeleteActorTemplate(context.Context, string, string type fakeRuntimeRevisionStore struct { pair *database.AgentDefinition revision *database.RuntimeRevision - policy egress.Access markedSuccessful bool retired string revisionErr error @@ -332,19 +331,19 @@ type fakeRuntimeRevisionStore struct { pairCalls int } -func (s *fakeRuntimeRevisionStore) UpsertAgentDefinition(_ context.Context, pair database.AgentDefinition) error { +func (s *fakeRuntimeRevisionStore) UpsertAgentDefinition(_ context.Context, pair database.AgentDefinition) (bool, error) { s.pairCalls++ + changed := s.pair == nil || !reflect.DeepEqual(s.pair.Grants, pair.Grants) s.pair = &pair - return s.pairErr + return changed, s.pairErr } -func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision, policy egress.Access) error { +func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision) error { s.recordCalls++ if s.revisionErr != nil { return s.revisionErr } s.revision = &revision - s.policy = policy return nil } @@ -407,7 +406,7 @@ func TestReconcilerUpdatesModelConfigStatusOnSecretHashChange(t *testing.T) { &fakeActorTemplates{}, &fakeRuntimeRevisionStore{}, statusClient, - noopEnqueueEgress, + noopEnqueueActors, ) go reconciler.Run(stop) diff --git a/go/core/internal/controller/sandbox_collections.go b/go/core/internal/controller/sandbox_collections.go index 52426f1a5a..656e6c008c 100644 --- a/go/core/internal/controller/sandbox_collections.go +++ b/go/core/internal/controller/sandbox_collections.go @@ -26,7 +26,7 @@ type sandboxReconciliation struct { Template *kagentv1alpha3.SandboxTemplate RevisionID string SourceSnapshot json.RawMessage - Egress egress.Access + Grants egress.Access DesiredActorTemplate *ateapipb.ActorTemplate ObservedActorTemplate *ateapipb.ActorTemplate CompilationError string @@ -87,17 +87,17 @@ func newSandboxCollections(inputs Collections, policy substrate.SandboxPolicy, o if !template.DeletionTimestamp.IsZero() { return state } + grants, err := translator.CompileEgressPolicies(ctx, translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets}, template.Namespace, template.Spec.EgressPolicyRefs) + if err != nil { + state.CompilationError = err.Error() + state.Failure = &ReconciliationFailure{Reason: kagentv1alpha3.SandboxTemplateReasonEgressPolicyResolutionFailed, Message: err.Error()} + return state + } + state.Grants = grants pool := krt.FetchOne(ctx, inputs.WorkerPools, krt.FilterKey(template.Namespace+"/"+template.Spec.Substrate.WorkerPoolRef.Name)) if pool == nil { state.Failure = &ReconciliationFailure{Reason: "WorkerPoolNotFound", Message: "The referenced sandbox WorkerPool does not exist"} } else { - grants, err := translator.CompileEgressPolicies(ctx, translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets}, template.Namespace, template.Spec.EgressPolicyRefs) - if err != nil { - state.CompilationError = err.Error() - state.Failure = &ReconciliationFailure{Reason: kagentv1alpha3.SandboxTemplateReasonEgressPolicyResolutionFailed, Message: err.Error()} - return state - } - state.Egress = grants state.DesiredActorTemplate, state.RevisionID, state.SourceSnapshot, err = substrate.SandboxActorTemplate(template, (*pool).Spec.SandboxClass, policy) if err != nil { state.CompilationError = err.Error() diff --git a/go/core/internal/controller/sandbox_integration_test.go b/go/core/internal/controller/sandbox_integration_test.go index 4e086ec507..b74adc5f11 100644 --- a/go/core/internal/controller/sandbox_integration_test.go +++ b/go/core/internal/controller/sandbox_integration_test.go @@ -65,7 +65,7 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { Secrets: typedCollection[*corev1.Secret](runtimeClient, []string{"team-a"}, "Secrets", options), SandboxTemplates: typedCollection[*kagentv1alpha3.SandboxTemplate](runtimeClient, []string{"team-a"}, "SandboxTemplates", options), WorkerPools: pools, }} - reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, noopEnqueueEgress) + reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, noopEnqueueActors) require.NoError(t, err) results := make(chan error, 2) go func() { results <- runtime.Start(ctx) }() diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index f52f69fd52..180cd276e3 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -21,7 +21,6 @@ import ( kagentclient "github.com/kagent-dev/kagent/go/api/clientset/versioned/typed/api/v1alpha3" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" ) @@ -32,9 +31,9 @@ const ( ) type sandboxRevisionStore interface { - UpsertSandboxTemplateDefinition(context.Context, database.SandboxTemplateDefinition) error + UpsertSandboxTemplateDefinition(context.Context, database.SandboxTemplateDefinition) (bool, error) RetireSandboxTemplateIdentities(context.Context, string, string) error - RecordSandboxRevision(context.Context, database.SandboxRevision, egress.Access) error + RecordSandboxRevision(context.Context, database.SandboxRevision) error PromoteSandboxRevision(context.Context, string) error } @@ -43,19 +42,19 @@ type sandboxRevisionStore interface { type SandboxReconciler struct { collections sandboxCollections store sandboxRevisionStore - enqueueEgress func(string) + enqueueActors func(database.RuntimeEgressSource) actors actorTemplateClient client kagentclient.ApiV1alpha3Interface } var _ manager.LeaderElectionRunnable = (*SandboxReconciler)(nil) -func NewSandboxReconciler(config *rest.Config, runtime *Runtime, store sandboxRevisionStore, actors actorTemplateClient, policy substrate.SandboxPolicy, enqueueEgress func(string)) (*SandboxReconciler, error) { +func NewSandboxReconciler(config *rest.Config, runtime *Runtime, store sandboxRevisionStore, actors actorTemplateClient, policy substrate.SandboxPolicy, enqueueActors func(database.RuntimeEgressSource)) (*SandboxReconciler, error) { client, err := kagentclient.NewForConfig(config) if err != nil { return nil, fmt.Errorf("create SandboxTemplate client: %w", err) } - return &SandboxReconciler{collections: newSandboxCollections(runtime.Collections, policy, runtime.Options), store: store, actors: actors, client: client, enqueueEgress: enqueueEgress}, nil + return &SandboxReconciler{collections: newSandboxCollections(runtime.Collections, policy, runtime.Options), store: store, actors: actors, client: client, enqueueActors: enqueueActors}, nil } func (s *SandboxReconciler) NeedLeaderElection() bool { return true } @@ -115,7 +114,9 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { } if state == nil || !state.Template.DeletionTimestamp.IsZero() { namespace, name, _ := strings.Cut(key, "/") - if err := s.store.RetireSandboxTemplateIdentities(ctx, namespace, name); err != nil { + err := s.store.RetireSandboxTemplateIdentities(ctx, namespace, name) + s.enqueueActors(database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: namespace, Name: name}) + if err != nil { return fmt.Errorf("retire SandboxTemplate %s: %w", key, err) } if state != nil && slices.Contains(state.Template.Finalizers, sandboxPreparationFinalizer) { @@ -137,9 +138,13 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { } return nil } - if err := s.store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{ - Namespace: state.Template.Namespace, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), DesiredRevision: state.desiredRevision(), - }); err != nil { + changed, err := s.store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{ + Namespace: state.Template.Namespace, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), DesiredRevision: state.desiredRevision(), Grants: state.Grants, + }) + if changed || err != nil { + s.enqueueActors(database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: state.Template.Namespace, Name: state.Template.Name}) + } + if err != nil { return s.observeError(*state, fmt.Errorf("store SandboxTemplate %s definition: %w", key, err), true) } if state.DesiredActorTemplate == nil { @@ -178,14 +183,9 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), SourceSnapshot: state.SourceSnapshot, } - policy, err := egress.MergeAccess(egress.Access{}, state.Egress, nil) - if err != nil { - return s.observeError(*state, fmt.Errorf("merge sandbox egress %s: %w", state.RevisionID, err), true) - } - if err := s.store.RecordSandboxRevision(ctx, revision, policy); err != nil { + if err := s.store.RecordSandboxRevision(ctx, revision); err != nil { return s.observeError(*state, fmt.Errorf("store sandbox revision %s: %w", state.RevisionID, err), true) } - s.enqueueEgress(revision.Revision) if ready { if err := s.store.PromoteSandboxRevision(ctx, revision.Revision); err != nil { return s.observeError(*state, fmt.Errorf("promote sandbox revision %s: %w", state.RevisionID, err), true) diff --git a/go/core/internal/controller/sandbox_reconciler_test.go b/go/core/internal/controller/sandbox_reconciler_test.go index 6292406d89..36907ff942 100644 --- a/go/core/internal/controller/sandbox_reconciler_test.go +++ b/go/core/internal/controller/sandbox_reconciler_test.go @@ -25,16 +25,15 @@ import ( kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" ) // These fakes are also used by the informer/queue integration test. type sandboxTestStore struct { + upsertErr error mu sync.Mutex desired database.SandboxTemplateDefinition revision database.SandboxRevision - policy egress.Access ready bool retired bool retireAttempts int @@ -44,13 +43,14 @@ type sandboxTestStore struct { recordCalls int } -func (s *sandboxTestStore) UpsertSandboxTemplateDefinition(_ context.Context, value database.SandboxTemplateDefinition) error { +func (s *sandboxTestStore) UpsertSandboxTemplateDefinition(_ context.Context, value database.SandboxTemplateDefinition) (bool, error) { s.mu.Lock() defer s.mu.Unlock() + changed := !reflect.DeepEqual(s.desired.Grants, value.Grants) s.desired = value - return nil + return changed, s.upsertErr } -func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision, policy egress.Access) error { +func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision) error { s.mu.Lock() defer s.mu.Unlock() s.recordCalls++ @@ -58,7 +58,6 @@ func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value databa return s.recordErr } s.revision = value - s.policy = policy return nil } func (s *sandboxTestStore) PromoteSandboxRevision(_ context.Context, revision string) error { @@ -144,7 +143,7 @@ func newSandboxTestReconciler(t *testing.T, guestImage string) (*SandboxReconcil actors := &sandboxTestActors{store: store, templates: map[string]*ateapipb.ActorTemplate{}} reconciler := &SandboxReconciler{ collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools}, substrate.SandboxPolicy{GuestImage: guestImage, CPU: "1", Memory: "1Gi"}, opts), - enqueueEgress: noopEnqueueEgress, + enqueueActors: noopEnqueueActors, store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3(), } waitFor(t, func() bool { return reconciler.collections.states.GetKey("team-a/scratch") != nil }) @@ -314,41 +313,3 @@ func TestSandboxPendingStatusRecoversWithoutGraphEvent(t *testing.T) { return err == nil && apimeta.IsStatusConditionTrue(current.Status.Conditions, "Ready") }, 5*time.Second, 10*time.Millisecond) } - -func TestSandboxPreparationRecordsCompiledPolicyBeforeEnqueueAndPromotion(t *testing.T) { - ctx := t.Context() - inputs := newEgressTestInputs(ctx) - template := sandboxTestTemplate() - template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} - inputs.sandboxes.UpdateObject(template) - policy := egressTestPolicy("https://tickets.example.com", nil) - policy.Namespace = template.Namespace - inputs.policies.UpdateObject(policy) - inputs.runtime.Collections.WorkerPools = krt.NewStaticCollection(nil, []*atev1alpha1.WorkerPool{{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "default"}}}, inputs.runtime.Options.WithName("Pools")...) - store := &sandboxTestStore{} - actors := &sandboxTestActors{store: store, templates: map[string]*ateapipb.ActorTemplate{}, autoReady: true} - queued := 0 - s := &SandboxReconciler{ - collections: newSandboxCollections(inputs.runtime.Collections, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, inputs.runtime.Options), - store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3(), - enqueueEgress: func(revision string) { - queued++ - require.Equal(t, store.revision.Revision, revision) - require.Equal(t, []string{"https://tickets.example.com:443"}, store.policy.Destinations) - require.False(t, store.ready) - }, - } - waitFor(t, func() bool { - state := s.collections.states.GetKey("team-a/scratch") - return state != nil && state.DesiredActorTemplate != nil - }) - require.NoError(t, s.reconcile(ctx, "team-a/scratch")) // Add the finalizer. - syncSandboxTemplate(t, s, inputs.sandboxes) - store.recordErr = errors.New("record failed") - require.ErrorIs(t, s.reconcile(ctx, "team-a/scratch"), store.recordErr) - require.Zero(t, queued) - store.recordErr = nil - require.NoError(t, s.reconcile(ctx, "team-a/scratch")) - require.Equal(t, 1, queued) - require.True(t, store.ready, "initial readiness must not wait for the publication queue") -} diff --git a/go/core/internal/database/client_session_test.go b/go/core/internal/database/client_session_test.go index c7f1f4da69..7353a33df6 100644 --- a/go/core/internal/database/client_session_test.go +++ b/go/core/internal/database/client_session_test.go @@ -15,7 +15,6 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" - "github.com/kagent-dev/kagent/go/core/internal/egress" ) func TestMalformedDatabaseIDsReturnErrors(t *testing.T) { @@ -375,10 +374,10 @@ func TestForkSessionCopiesBoundedHistory(t *testing.T) { Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision, } - if err := client.UpsertAgentDefinition(ctx, pair); err != nil { + if _, err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision, egress.Access{}); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision); err != nil { t.Fatal(err) } require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) @@ -535,10 +534,10 @@ func TestSessionCreateAndTransitions(t *testing.T) { Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision, } - if err := client.UpsertAgentDefinition(ctx, pair); err != nil { + if _, err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision, egress.Access{}); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision); err != nil { t.Fatal(err) } require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) @@ -638,10 +637,10 @@ func sessionFixture(t *testing.T, client *Client, ctx context.Context, namespace Namespace: namespace, AgentName: template, AgentUID: template + "-uid", DesiredRevision: revisionID, } - if err := client.UpsertAgentDefinition(ctx, pair); err != nil { + if _, err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision, egress.Access{}); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision); err != nil { t.Fatal(err) } require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) diff --git a/go/core/internal/database/models.go b/go/core/internal/database/models.go index f7d9a5a453..7bfafc8aea 100644 --- a/go/core/internal/database/models.go +++ b/go/core/internal/database/models.go @@ -51,6 +51,7 @@ type AgentMemorySearchResult struct { // AgentDefinition tracks the desired and last successful runtime for an Agent UID. type AgentDefinition struct { + Grants egress.Access Namespace string AgentName string AgentUID string diff --git a/go/core/internal/database/protobuf_test.go b/go/core/internal/database/protobuf_test.go index 39d52af768..86b4005309 100644 --- a/go/core/internal/database/protobuf_test.go +++ b/go/core/internal/database/protobuf_test.go @@ -14,7 +14,6 @@ import ( "google.golang.org/protobuf/proto" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" - "github.com/kagent-dev/kagent/go/core/internal/egress" ) func addUnknown(message proto.Message) { @@ -287,11 +286,11 @@ func TestShareAndAgentCardProtobufPersistence(t *testing.T) { require.NoError(t, err) addUnknown(card) revision := RuntimeRevision{Revision: "revision", Namespace: "team-a", AgentName: "assistant", AgentUID: "template", SourceSnapshot: []byte(`{}`), AgentCard: card, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "template"} - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, egress.Access{})) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) // Reconciliation can update runtime identity, but the pinned card is immutable. revision.AgentCard = &a2apb.AgentCard{Name: "replacement"} revision.ActorTemplateUID = "new-uid" - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, egress.Access{})) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) stored, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) require.True(t, proto.Equal(card, stored.AgentCard)) diff --git a/go/core/internal/database/runtime_creation_test.go b/go/core/internal/database/runtime_creation_test.go index b7d7624d49..bb23002f09 100644 --- a/go/core/internal/database/runtime_creation_test.go +++ b/go/core/internal/database/runtime_creation_test.go @@ -30,7 +30,8 @@ func TestRuntimeCreationUsesLatestSuccessfulRevision(t *testing.T) { return session.GetPreparedRevision(), created, err } desire = func(c *Client, revision string) error { - return c.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", DesiredRevision: revision}) + _, err := c.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", DesiredRevision: revision}) + return err } } else { input, options := sandboxFixture(t, client, "revision-1") @@ -44,7 +45,8 @@ func TestRuntimeCreationUsesLatestSuccessfulRevision(t *testing.T) { return instance.GetPreparedRevision(), created, err } desire = func(c *Client, revision string) error { - return c.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) + _, err := c.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) + return err } unresolved = "pending:uid:2" } diff --git a/go/core/internal/database/runtime_egress.go b/go/core/internal/database/runtime_egress.go index 864c8b7c16..7e1fc15bcf 100644 --- a/go/core/internal/database/runtime_egress.go +++ b/go/core/internal/database/runtime_egress.go @@ -9,42 +9,67 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/egress" ) -// RuntimeEgress retains integration access and the source identity of a pinned -// revision. Explicit policy grants are resolved from the current Kubernetes source. -type RuntimeEgress struct { - Kind RuntimeEgressSourceKind - Namespace string - SourceName string - SourceUID string - EgressDestinations []string - Credentials []egress.Credential - TokenPassthroughOrigins []string +// canonicalStoredGrants validates and normalizes source grants for NOT NULL storage. +func canonicalStoredGrants(grants egress.Access) (egress.Access, error) { + canonical, err := egress.CanonicalAccess(grants.Destinations, grants.Credentials) + if err != nil { + return egress.Access{}, err + } + if canonical.Destinations == nil { + canonical.Destinations = []string{} + } + if canonical.Credentials == nil { + canonical.Credentials = []egress.Credential{} + } + return canonical, nil } -// GetRuntimeEgress reads immutable access and source identity for either runtime -// kind. Missing revisions return ErrNotFound; malformed grants return an error. -func (c *Client) GetRuntimeEgress(ctx context.Context, revision string) (*RuntimeEgress, error) { +// GetRuntimeEgressPolicy merges pinned inputs with the active source UID's grants +// in one database snapshot. Missing/deleting revisions and malformed inputs fail +// closed. Retired or replaced identities contribute no grants. Merge conflicts +// retain pinned access and return a diagnostic, not a failed read. +func (c *Client) GetRuntimeEgressPolicy(ctx context.Context, revision string) (egress.Policy, error) { + type policyInputs struct { + EgressDestinations []string + Credentials []egress.Credential + TokenPassthroughOrigins []string + GrantDestinations []string + GrantCredentials []egress.Credential + } row, err := queryOne(ctx, c.db, ` - SELECT r.kind, r.namespace, r.egress_destinations, r.credentials, r.token_passthrough_origins, - COALESCE(a.agent_name, s.sandbox_template_name) AS source_name, - COALESCE(a.agent_uid, s.sandbox_template_uid) AS source_uid - FROM runtime_revision r - LEFT JOIN agent_revision a USING (revision) - LEFT JOIN sandbox_revision s USING (revision) - WHERE r.revision = $1 AND r.deleted_at IS NULL - `, pgx.RowToStructByName[RuntimeEgress], revision) + SELECT r.egress_destinations, r.credentials, r.token_passthrough_origins, + COALESCE(ad.grant_destinations, sd.grant_destinations, '{}') AS grant_destinations, + COALESCE(ad.grant_credentials, sd.grant_credentials, '[]') AS grant_credentials + FROM runtime_revision r + LEFT JOIN agent_revision a ON r.kind = 'agent' AND a.revision = r.revision + LEFT JOIN sandbox_revision s ON r.kind = 'sandbox' AND s.revision = r.revision + LEFT JOIN agent_definition ad ON ad.namespace = r.namespace AND ad.agent_uid = a.agent_uid AND ad.retired_at IS NULL + LEFT JOIN sandbox_template_definition sd ON sd.namespace = r.namespace AND sd.sandbox_template_uid = s.sandbox_template_uid AND sd.retired_at IS NULL + WHERE r.revision = $1 AND r.deleted_at IS NULL + AND (a.revision IS NOT NULL OR s.revision IS NOT NULL) + `, pgx.RowToStructByName[policyInputs], revision) + if err != nil { + return egress.Policy{}, fmt.Errorf("get runtime egress policy %s: %w", revision, notFoundOr(err)) + } + tokens, err := egress.CanonicalAccess(row.TokenPassthroughOrigins, nil) if err != nil { - return nil, fmt.Errorf("get runtime egress %s: %w", revision, notFoundOr(err)) + return egress.Policy{}, fmt.Errorf("decode runtime token origins %s: %w", revision, err) } - access, err := egress.MergeAccess(egress.Access{Destinations: row.EgressDestinations, Credentials: row.Credentials}, egress.Access{}, row.TokenPassthroughOrigins) + base, err := egress.MergeAccess(egress.Access{Destinations: row.EgressDestinations, Credentials: row.Credentials}, egress.Access{}, tokens.Destinations) if err != nil { - return nil, fmt.Errorf("decode runtime egress %s: %w", revision, err) + return egress.Policy{}, fmt.Errorf("decode runtime egress base %s: %w", revision, err) } - row.EgressDestinations, row.Credentials = access.Destinations, access.Credentials - return &row, nil + grants, err := egress.CanonicalAccess(row.GrantDestinations, row.GrantCredentials) + if err != nil { + return egress.Policy{}, fmt.Errorf("decode runtime egress grants %s: %w", revision, err) + } + access, withheld := egress.EffectiveAccess(base, grants, tokens.Destinations) + return egress.Policy{Access: access, WithheldGrants: withheld}, nil } -// RuntimeEgressSourceKind identifies the Kubernetes owner of a runtime revision. +var _ egress.PolicyStore = (*Client)(nil) + +// RuntimeEgressSourceKind identifies a Kubernetes owner of runtime instances. type RuntimeEgressSourceKind string const ( @@ -52,126 +77,73 @@ const ( RuntimeEgressSandbox RuntimeEgressSourceKind = runtimeKindSandbox ) -// RuntimeEgressSource selects all revisions of an Agent or SandboxTemplate, -// including former UIDs whose grants must be revoked after source recreation. +// RuntimeEgressSource includes former UIDs whose grants must be revoked on recreation. type RuntimeEgressSource struct { Kind RuntimeEgressSourceKind Namespace string Name string } -// ListRuntimeEgressRevisions lists retained revisions in key order for recovery, -// including revisions without actors. Deleting revisions are excluded. -func (c *Client) ListRuntimeEgressRevisions(ctx context.Context, after string, limit int) ([]string, error) { +// RuntimeEgressTarget identifies allocated instances, including suspended actors. +type RuntimeEgressTarget struct { + ID string + Revision string + Namespace string + Atespace string + TemplateName string + ActorUID string +} + +// ListRuntimeEgressTargets pages all allocated instances in ID order for startup +// recovery. Deleting instances and unissued creations are excluded; concurrent +// creation reads effective policy itself and rechecks it after applying. +func (c *Client) ListRuntimeEgressTargets(ctx context.Context, afterID string, limit int) ([]RuntimeEgressTarget, error) { return queryMany(ctx, c.db, ` - SELECT revision FROM runtime_revision - WHERE revision > $1 AND deleted_at IS NULL ORDER BY revision LIMIT $2 - `, pgx.RowTo[string], after, limit) + SELECT i.id::text AS id, i.prepared_revision AS revision, r.namespace, + r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, COALESCE(s.actor_uid, '') AS actor_uid + FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision + LEFT JOIN session s ON s.id = i.id + WHERE i.id > COALESCE(NULLIF($1, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) + AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' + ORDER BY i.id LIMIT $2 + `, pgx.RowToStructByName[RuntimeEgressTarget], afterID, limit) } -// ListRuntimeEgressSourceRevisions lists a source's retained revisions across -// Kubernetes UIDs. Empty and unsupported kinds are errors, never sweep selectors. -func (c *Client) ListRuntimeEgressSourceRevisions(ctx context.Context, source RuntimeEgressSource, after string, limit int) ([]string, error) { +// ListRuntimeEgressSourceTargets pages allocated instances across a source's UIDs +// in ID order, excluding deletions and unissued creations. Invalid sources return +// an error; a missing source yields an empty page. +func (c *Client) ListRuntimeEgressSourceTargets(ctx context.Context, source RuntimeEgressSource, afterID string, limit int) ([]RuntimeEgressTarget, error) { if source.Namespace == "" || source.Name == "" { return nil, fmt.Errorf("runtime egress source requires namespace and name") } switch source.Kind { case RuntimeEgressAgent: return queryMany(ctx, c.db, ` - SELECT r.revision FROM runtime_revision r JOIN agent_revision a USING (revision) - WHERE r.namespace = $1 AND a.agent_name = $2 AND r.revision > $3 - AND r.deleted_at IS NULL ORDER BY r.revision LIMIT $4 - `, pgx.RowTo[string], source.Namespace, source.Name, after, limit) + SELECT i.id::text AS id, i.prepared_revision AS revision, r.namespace, + r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, COALESCE(s.actor_uid, '') AS actor_uid + FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision + JOIN agent_revision a ON a.revision = r.revision + LEFT JOIN session s ON s.id = i.id + WHERE r.namespace = $1 AND a.agent_name = $2 + AND i.id > COALESCE(NULLIF($3, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) + AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' + ORDER BY i.id LIMIT $4 + `, pgx.RowToStructByName[RuntimeEgressTarget], source.Namespace, source.Name, afterID, limit) case RuntimeEgressSandbox: return queryMany(ctx, c.db, ` - SELECT r.revision FROM runtime_revision r JOIN sandbox_revision s USING (revision) - WHERE r.namespace = $1 AND s.sandbox_template_name = $2 AND r.revision > $3 - AND r.deleted_at IS NULL ORDER BY r.revision LIMIT $4 - `, pgx.RowTo[string], source.Namespace, source.Name, after, limit) + SELECT i.id::text AS id, i.prepared_revision AS revision, r.namespace, + r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, '' AS actor_uid + FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision + JOIN sandbox_revision s ON s.revision = r.revision + WHERE r.namespace = $1 AND s.sandbox_template_name = $2 + AND i.id > COALESCE(NULLIF($3, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) + AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' + ORDER BY i.id LIMIT $4 + `, pgx.RowToStructByName[RuntimeEgressTarget], source.Namespace, source.Name, afterID, limit) default: return nil, fmt.Errorf("unknown runtime egress source kind %q", source.Kind) } } - -// SetRuntimeEgressPolicy publishes controller output for a retained revision. -// It writes and returns true only when the canonical policy changes. -// It does not mutate the revision's pinned inputs or readiness. Missing/deleting -// revisions return ErrNotFound; malformed policy input is rejected before writing. -func (c *Client) SetRuntimeEgressPolicy(ctx context.Context, revision string, access egress.Access) (bool, error) { - canonical, err := egress.CanonicalAccess(access.Destinations, access.Credentials) - if err != nil { - return false, fmt.Errorf("validate runtime egress policy %s: %w", revision, err) - } - if canonical.Destinations == nil { - canonical.Destinations = []string{} - } - if canonical.Credentials == nil { - canonical.Credentials = []egress.Credential{} - } - changed, err := queryOne(ctx, c.db, ` - WITH retained AS ( - SELECT revision FROM runtime_revision WHERE revision = $1 AND deleted_at IS NULL - ), changed AS ( - INSERT INTO runtime_egress_policy (revision, destinations, credentials) - SELECT revision, $2, $3 FROM retained - ON CONFLICT (revision) DO UPDATE SET destinations = EXCLUDED.destinations, credentials = EXCLUDED.credentials - WHERE (runtime_egress_policy.destinations, runtime_egress_policy.credentials) - IS DISTINCT FROM (EXCLUDED.destinations, EXCLUDED.credentials) - RETURNING revision - ) - SELECT EXISTS (SELECT 1 FROM changed) FROM retained - `, pgx.RowTo[bool], revision, canonical.Destinations, canonical.Credentials) - if err != nil { - return false, fmt.Errorf("store runtime egress policy %s: %w", revision, notFoundOr(err)) - } - return changed, nil -} - -// GetRuntimeEgressPolicy reads the latest controller output, independent of any -// replica's Kubernetes cache. Missing and malformed policies fail closed. -func (c *Client) GetRuntimeEgressPolicy(ctx context.Context, revision string) (egress.Access, error) { - access, err := queryOne(ctx, c.db, ` - SELECT destinations, credentials FROM runtime_egress_policy WHERE revision = $1 - `, pgx.RowToStructByName[egress.Access], revision) - if err != nil { - return egress.Access{}, fmt.Errorf("get runtime egress policy %s: %w", revision, notFoundOr(err)) - } - canonical, err := egress.CanonicalAccess(access.Destinations, access.Credentials) - if err != nil { - return egress.Access{}, fmt.Errorf("decode runtime egress policy %s: %w", revision, err) - } - return canonical, nil -} - -var _ egress.PolicyStore = (*Client)(nil) - -// RuntimeEgressTarget identifies an allocated runtime, including suspended or -// partially created actors that still need live policy reconciliation. -type RuntimeEgressTarget struct { - ID string - Revision string - Atespace string - TemplateName string - ActorUID string -} - -// ListRuntimeEgressTargets lists a revision's allocated instances in ID order. -// Deleting instances and unissued creations are excluded. Concurrent creation -// reads the persisted policy itself and rechecks it after applying. -func (c *Client) ListRuntimeEgressTargets(ctx context.Context, revision, afterID string, limit int) ([]RuntimeEgressTarget, error) { - rows, err := queryMany(ctx, c.db, ` - SELECT i.id::text AS id, i.prepared_revision AS revision, r.actor_template_atespace AS atespace, - r.actor_template_name AS template_name, COALESCE(s.actor_uid, '') AS actor_uid - FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision - LEFT JOIN session s ON s.id = i.id - WHERE i.prepared_revision = $1 - AND i.id > COALESCE(NULLIF($2, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) - AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) - AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' - ORDER BY i.id LIMIT $3 - `, pgx.RowToStructByName[RuntimeEgressTarget], revision, afterID, limit) - if err != nil { - return nil, fmt.Errorf("list runtime egress targets: %w", err) - } - return rows, nil -} diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index 2ee8051b04..15bd726f9c 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -1,14 +1,11 @@ package database import ( - "context" - "sync" + "slices" "testing" - "time" a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgxpool" "github.com/stretchr/testify/require" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" @@ -19,8 +16,9 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", TokenPassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} - require.NoError(t, client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", DesiredRevision: "agent"})) - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, egress.Access{})) + _, upsertErr := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", DesiredRevision: "agent"}) + require.NoError(t, upsertErr) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) instance, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "request") require.NoError(t, err) @@ -41,12 +39,8 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { _, err = client.FinishSessionOperation(ctx, instance.Id, op.ID, executor, "", "", "") require.NoError(t, err) // A recreated Agent must not transfer authorization to an old pinned session. - require.NoError(t, client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "new-uid", DesiredRevision: "pending"})) - pinned, err := client.GetRuntimeEgress(ctx, instance.PreparedRevision) - require.NoError(t, err) - require.Equal(t, "old-uid", pinned.SourceUID) - require.Equal(t, RuntimeEgressAgent, pinned.Kind) - require.Equal(t, revision.TokenPassthroughOrigins, pinned.TokenPassthroughOrigins) + _, upsertErr = client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "new-uid", DesiredRevision: "pending"}) + require.NoError(t, upsertErr) sandbox, options := sandboxFixture(t, client, "sandbox") created, _, err := client.CreateSandbox(ctx, sandbox, "sandbox", options) require.NoError(t, err) @@ -55,175 +49,163 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { claimed, err = client.ClaimSandboxOperation(ctx, created.Id, sandboxOp.ID, uuid.New()) require.NoError(t, err) require.True(t, claimed) - pinned, err = client.GetRuntimeEgress(ctx, created.PreparedRevision) - require.NoError(t, err) - require.Equal(t, RuntimeEgressSandbox, pinned.Kind) - require.Equal(t, "scratch", pinned.SourceName) - require.Equal(t, "uid", pinned.SourceUID) for _, test := range []struct { source RuntimeEgressSource want string }{ - {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, instance.PreparedRevision}, - {RuntimeEgressSource{Kind: RuntimeEgressSandbox, Namespace: "team-a", Name: "scratch"}, created.PreparedRevision}, + {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, instance.Id}, + {RuntimeEgressSource{Kind: RuntimeEgressSandbox, Namespace: "team-a", Name: "scratch"}, created.Id}, {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "other", Name: "assistant"}, ""}, {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "unrelated"}, ""}, } { - revisions, err := client.ListRuntimeEgressSourceRevisions(ctx, test.source, "", 10) + targets, err := client.ListRuntimeEgressSourceTargets(ctx, test.source, "", 10) require.NoError(t, err) if test.want == "" { - require.Empty(t, revisions) + require.Empty(t, targets) } else { - require.Equal(t, []string{test.want}, revisions) + require.Len(t, targets, 1) + require.Equal(t, test.want, targets[0].ID) } } for _, kind := range []RuntimeEgressSourceKind{"", "typo"} { - _, err := client.ListRuntimeEgressSourceRevisions(ctx, RuntimeEgressSource{Kind: kind, Namespace: "team-a", Name: "assistant"}, "", 10) + _, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: kind, Namespace: "team-a", Name: "assistant"}, "", 10) require.Error(t, err) } - first, err := client.ListRuntimeEgressRevisions(ctx, "", 1) + first, err := client.ListRuntimeEgressTargets(ctx, "", 1) require.NoError(t, err) require.Len(t, first, 1) - second, err := client.ListRuntimeEgressRevisions(ctx, first[0], 1) + second, err := client.ListRuntimeEgressTargets(ctx, first[0].ID, 1) require.NoError(t, err) require.Len(t, second, 1) - require.ElementsMatch(t, []string{instance.PreparedRevision, created.PreparedRevision}, []string{first[0], second[0]}) - last, err := client.ListRuntimeEgressRevisions(ctx, second[0], 1) + require.ElementsMatch(t, []string{instance.Id, created.Id}, []string{first[0].ID, second[0].ID}) + last, err := client.ListRuntimeEgressTargets(ctx, second[0].ID, 1) require.NoError(t, err) require.Empty(t, last) - for _, test := range []struct{ revision, id string }{{instance.PreparedRevision, instance.Id}, {created.PreparedRevision, created.Id}} { - targets, err := client.ListRuntimeEgressTargets(ctx, test.revision, "", 1) - require.NoError(t, err) - require.Len(t, targets, 1) - require.Equal(t, test.id, targets[0].ID) - targets, err = client.ListRuntimeEgressTargets(ctx, test.revision, test.id, 1) - require.NoError(t, err) - require.Empty(t, targets) - } _, err = client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE) require.NoError(t, err) - remaining, err := client.ListRuntimeEgressTargets(ctx, instance.PreparedRevision, "", 10) + remaining, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, "", 10) require.NoError(t, err) require.Empty(t, remaining) } -func TestRuntimeEgressPolicyIsMutableControllerOutput(t *testing.T) { - client := NewClient(setupTestDB(t), "public") - ctx := t.Context() - revision := RuntimeRevision{Revision: "policy-revision", Namespace: "team-a", AgentName: "assistant", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", EgressDestinations: []string{"https://model.example.com:443"}} - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, egress.Access{Destinations: revision.EgressDestinations})) - initial, err := client.GetRuntimeEgressPolicy(ctx, revision.Revision) - require.NoError(t, err) - require.Equal(t, revision.EgressDestinations, initial.Destinations) - allowed := egress.Access{Destinations: []string{"https://model.example.com:443", "https://tools.example.com:443"}, Credentials: []egress.Credential{{Origin: "https://tools.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/auth/token"}}} - changed, err := client.SetRuntimeEgressPolicy(ctx, revision.Revision, allowed) - require.NoError(t, err) - require.True(t, changed) - var before string - require.NoError(t, client.db.QueryRow(ctx, `SELECT xmin::text FROM runtime_egress_policy WHERE revision = $1`, revision.Revision).Scan(&before)) - equivalent := egress.Access{Destinations: []string{"https://tools.example.com", "https://model.example.com", "https://tools.example.com:443"}, Credentials: allowed.Credentials} - changed, err = client.SetRuntimeEgressPolicy(ctx, revision.Revision, equivalent) - require.NoError(t, err) - require.False(t, changed, "equivalent canonical policies must not trigger actor updates") - var after string - require.NoError(t, client.db.QueryRow(ctx, `SELECT xmin::text FROM runtime_egress_policy WHERE revision = $1`, revision.Revision).Scan(&after)) - require.Equal(t, before, after, "unchanged policies must not rewrite the row") - // A different API replica reads the same intended policy without any K8s cache. - replica := NewClient(client.db, "public") - got, err := replica.GetRuntimeEgressPolicy(ctx, revision.Revision) - require.NoError(t, err) - require.Equal(t, allowed, got) - rotated := egress.Access{Destinations: allowed.Destinations, Credentials: []egress.Credential{allowed.Credentials[0]}} - rotated.Credentials[0].URI = "ate-secret://k8s.io/default/team-a/rotated/token" - changed, err = client.SetRuntimeEgressPolicy(ctx, revision.Revision, rotated) - require.NoError(t, err) - require.True(t, changed, "a credential-only change must update actors") - changed, err = client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{Destinations: revision.EgressDestinations}) - require.NoError(t, err) - require.True(t, changed) - // Preparation observations must not restore an earlier policy. - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, allowed)) - require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) - got, err = replica.GetRuntimeEgressPolicy(ctx, revision.Revision) - require.NoError(t, err) - require.Equal(t, revision.EgressDestinations, got.Destinations) - require.Empty(t, got.Credentials) - pinned, err := client.GetRuntimeEgress(ctx, revision.Revision) - require.NoError(t, err) - require.Equal(t, revision.EgressDestinations, pinned.EgressDestinations) - require.Empty(t, pinned.Credentials) - changed, err = client.SetRuntimeEgressPolicy(ctx, "missing", allowed) - require.ErrorIs(t, err, ErrNotFound) - require.False(t, changed) - _, err = client.SetRuntimeEgressPolicy(ctx, revision.Revision, egress.Access{Destinations: []string{"invalid"}}) - require.Error(t, err) - // Durable malformed data must fail rather than granting a partial policy. - _, err = client.db.Exec(ctx, `UPDATE runtime_egress_policy SET credentials = '[{"hostname":"legacy","header":"authorization"}]' WHERE revision = $1`, revision.Revision) - require.NoError(t, err) - _, err = replica.GetRuntimeEgressPolicy(ctx, revision.Revision) - require.Error(t, err) - _, err = client.BeginRuntimeRevisionDeletion(ctx, revision.Revision) - require.NoError(t, err) - _, err = client.SetRuntimeEgressPolicy(ctx, revision.Revision, allowed) - require.ErrorIs(t, err, ErrNotFound, "deleting revisions must remain distinguishable from unchanged policies") -} - -func TestRecordRevisionInitialPolicyIsAtomic(t *testing.T) { +func TestSourceGrantsDetermineEffectivePolicy(t *testing.T) { for _, kind := range []RuntimeEgressSourceKind{RuntimeEgressAgent, RuntimeEgressSandbox} { t.Run(string(kind), func(t *testing.T) { - pool := setupTestDB(t) - client := NewClient(pool, "public") + client := NewClient(setupTestDB(t), "public") ctx := t.Context() - initial := egress.Access{Destinations: []string{"https://tickets.example.com"}} - record := func(client *Client, ctx context.Context, policy egress.Access) error { + upsert := func(uid string, grants egress.Access) (bool, error) { if kind == RuntimeEgressAgent { - return client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, policy) + return client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: uid, Grants: grants}) } - return client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, SandboxTemplateName: "source", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, policy) + return client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team", SandboxTemplateName: "source", SandboxTemplateUID: uid, Grants: grants}) + } + base := egress.Access{} + if kind == RuntimeEgressAgent { + base = egress.Access{Destinations: []string{"https://model.example.com:443"}, Credentials: []egress.Credential{{Origin: "https://model.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"}}} } - barrier := &runtimeReferenceBarrier{query: "INSERT INTO runtime_egress_policy", afterQuery: true, reached: make(chan struct{}), resume: make(chan struct{})} - release := sync.OnceFunc(func() { close(barrier.resume) }) - config := pool.Config() - config.ConnConfig.Tracer = barrier - writerPool, err := pgxpool.NewWithConfig(ctx, config) + initial := egress.Access{Destinations: []string{"https://tickets.example.com"}, Credentials: []egress.Credential{{Origin: "https://tickets.example.com", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/tickets/token"}}} + changed, err := upsert("uid", egress.Access{}) + require.NoError(t, err) + require.False(t, changed) + changed, err = upsert("uid", initial) + require.NoError(t, err) + require.True(t, changed) + canonical, err := egress.CanonicalAccess(initial.Destinations, initial.Credentials) require.NoError(t, err) - defer writerPool.Close() - defer release() - writeCtx, cancel := context.WithCancel(ctx) - defer cancel() - written := make(chan error, 1) - go func() { written <- record(NewClient(writerPool, "public"), writeCtx, initial) }() - select { - case <-barrier.reached: - case <-time.After(5 * time.Second): - t.Fatal("initial policy was not inserted") + changed, err = upsert("uid", canonical) + require.NoError(t, err) + require.False(t, changed, "canonical equivalence must not trigger fan-out") + if kind == RuntimeEgressAgent { + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, EgressDestinations: base.Destinations, Credentials: base.Credentials, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"})) + } else { + require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, SandboxTemplateName: "source", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)})) } - _, err = client.GetRuntimeEgress(ctx, "revision") - require.ErrorIs(t, err, ErrNotFound, "uncommitted inputs must not be visible") - _, err = client.GetRuntimeEgressPolicy(ctx, "revision") - require.ErrorIs(t, err, ErrNotFound, "uncommitted policy must not be visible") - cancel() - release() - require.ErrorIs(t, <-written, context.Canceled) - _, err = client.GetRuntimeEgressPolicy(ctx, "revision") - require.ErrorIs(t, err, ErrNotFound, "failed recording must roll back the initial policy") - _, err = client.GetRuntimeEgress(ctx, "revision") - require.ErrorIs(t, err, ErrNotFound, "failed recording must roll back the revision") - - require.NoError(t, record(client, ctx, initial)) - pinned, err := client.GetRuntimeEgress(ctx, "revision") + assertPolicy := func(grants egress.Access) { + t.Helper() + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + expected, err := egress.MergeAccess(base, grants, nil) + require.NoError(t, err) + require.True(t, slices.Equal(expected.Destinations, policy.Destinations)) + require.True(t, slices.Equal(expected.Credentials, policy.Credentials)) + require.NoError(t, policy.WithheldGrants) + } + assertPolicy(canonical) + _, err = upsert("uid", egress.Access{Destinations: []string{"invalid"}}) + require.Error(t, err) + assertPolicy(canonical) + if kind == RuntimeEgressAgent { + conflict := egress.Access{Destinations: base.Destinations, Credentials: []egress.Credential{{Origin: base.Destinations[0], Header: "authorization", URI: "ate-secret://k8s.io/default/team/other/token"}}} + changed, err = upsert("uid", conflict) + require.NoError(t, err) + require.True(t, changed) + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + require.Error(t, policy.WithheldGrants) + require.Equal(t, base, policy.Access) + } + if kind == RuntimeEgressAgent { + require.NoError(t, client.RetireAgentIdentities(ctx, "team", "source", nil)) + } else { + require.NoError(t, client.RetireSandboxTemplateIdentities(ctx, "team", "source")) + } + assertPolicy(egress.Access{}) + changed, err = upsert("uid", canonical) require.NoError(t, err) - require.Equal(t, kind, pinned.Kind) - policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.True(t, changed, "reviving a source restores only its newly supplied grants") + assertPolicy(canonical) + changed, err = upsert("replacement", canonical) require.NoError(t, err) - require.Equal(t, []string{"https://tickets.example.com:443"}, policy.Destinations) - _, err = client.SetRuntimeEgressPolicy(ctx, "revision", egress.Access{}) + require.True(t, changed, "retiring the old UID requires actor fan-out") + assertPolicy(egress.Access{}) + changed, err = upsert("replacement", canonical) require.NoError(t, err) - require.NoError(t, record(client, ctx, initial)) - policy, err = client.GetRuntimeEgressPolicy(ctx, "revision") + require.False(t, changed) + _, err = client.GetRuntimeEgressPolicy(ctx, "missing") + require.ErrorIs(t, err, ErrNotFound) + }) + } +} + +func TestSourceGrantsRollbackWithInvalidRevision(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + grants := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} + definition := AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", Grants: grants} + _, err := client.UpsertAgentDefinition(ctx, definition) + require.NoError(t, err) + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "agent", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "agent"})) + require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "sandbox", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "sandbox"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "sandbox", SourceSnapshot: []byte(`{}`)})) + for _, uid := range []string{"uid", "replacement"} { + definition.AgentUID, definition.DesiredRevision, definition.Grants = uid, "sandbox", egress.Access{} + _, err = client.UpsertAgentDefinition(ctx, definition) + require.ErrorIs(t, err, ErrConflict) + policy, err := client.GetRuntimeEgressPolicy(ctx, "agent") + require.NoError(t, err) + require.Equal(t, grants.Destinations, policy.Destinations, "failed upsert must roll back grants and retirement") + require.Empty(t, policy.Credentials) + } +} + +func TestRuntimeEgressMalformedInputsFailClosed(t *testing.T) { + for _, test := range []struct{ name, statement string }{ + {"grant credentials", `UPDATE agent_definition SET grant_credentials = '[{"hostname":"legacy.example.com"}]'`}, + {"grant origin", `UPDATE agent_definition SET grant_destinations = ARRAY['invalid']`}, + {"pinned credentials", `UPDATE runtime_revision SET credentials = '[{"hostname":"legacy.example.com"}]'`}, + {"token origin", `UPDATE runtime_revision SET token_passthrough_origins = ARRAY['invalid']`}, + {"missing typed revision", `DELETE FROM agent_revision`}, + } { + t.Run(test.name, func(t *testing.T) { + pool := setupTestDB(t) + client := NewClient(pool, "public") + ctx := t.Context() + _, err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid"}) require.NoError(t, err) - require.Empty(t, policy.Destinations, "preparation must not undo a live revocation") - require.Error(t, record(client, ctx, egress.Access{Destinations: []string{"invalid"}})) + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"})) + _, err = pool.Exec(ctx, test.statement) + require.NoError(t, err) + _, err = client.GetRuntimeEgressPolicy(ctx, "revision") + require.Error(t, err) }) } } diff --git a/go/core/internal/database/runtime_lifecycle_test.go b/go/core/internal/database/runtime_lifecycle_test.go index d9970c2ae0..6efd46245c 100644 --- a/go/core/internal/database/runtime_lifecycle_test.go +++ b/go/core/internal/database/runtime_lifecycle_test.go @@ -6,8 +6,6 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/stretchr/testify/require" - - "github.com/kagent-dev/kagent/go/core/internal/egress" ) func TestRuntimeLifecycleStoresSharedValues(t *testing.T) { @@ -59,7 +57,7 @@ func TestAgentWritesRejectSandboxRevisions(t *testing.T) { `) require.NoError(t, err) // Failed validation must roll back preparation changes and preserve the ready revision. - err = client.UpsertAgentDefinition(ctx, AgentDefinition{ + _, err = client.UpsertAgentDefinition(ctx, AgentDefinition{ Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", DesiredRevision: "sandbox-revision", }) @@ -71,7 +69,7 @@ func TestAgentWritesRejectSandboxRevisions(t *testing.T) { revision, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) revision.Revision, revision.ActorTemplateUID = "sandbox-revision", "replacement-uid" - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision, egress.Access{}), ErrConflict) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision), ErrConflict) require.ErrorIs(t, client.PromoteRuntimeRevision(ctx, "sandbox-revision"), ErrConflict) require.ErrorIs(t, client.PromoteSandboxRevision(ctx, "revision"), ErrConflict) var uid string diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index f9122602d5..734b721540 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -15,29 +15,48 @@ import ( ) // UpsertAgentDefinition records the desired runtime revision for a -// Agent identity. Updating an existing definition revives it if retired and +// Kubernetes Agent identity. Updating an existing definition revives it if retired and // preserves its latest successful revision. It atomically retires older identities -// at the same names and rejects revisions whose deletion has started. -func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefinition) error { - return c.withTx(ctx, func(tx pgx.Tx) error { - if err := retireAgentIdentities(ctx, tx, definition.Namespace, definition.AgentName, &definition); err != nil { - return fmt.Errorf("retire replaced Agent: %w", err) +// at the same names and rejects revisions whose deletion has started. Grants +// are canonicalized before writing. The bool reports changed active grants or +// retirement of another UID; errors may include an uncertain commit outcome. +func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefinition) (bool, error) { + grants, err := canonicalStoredGrants(definition.Grants) + if err != nil { + return false, fmt.Errorf("validate source grants: %w", err) + } + changed := false + err = c.withTx(ctx, func(tx pgx.Tx) error { + retired, err := tx.Exec(ctx, ` + UPDATE agent_definition SET retired_at = NOW(), updated_at = NOW() + WHERE namespace = $1 AND agent_name = $2 AND agent_uid <> $3 AND retired_at IS NULL + `, definition.Namespace, definition.AgentName, definition.AgentUID) + if err != nil { + return err } - if err := execSQL(ctx, tx, ` - INSERT INTO agent_definition ( - namespace, agent_name, agent_uid, desired_revision, retired_at - ) VALUES ($1, $2, $3, $4, NULL) - ON CONFLICT (namespace, agent_uid) DO UPDATE SET - agent_name = EXCLUDED.agent_name, - desired_revision = EXCLUDED.desired_revision, - retired_at = NULL, - updated_at = NOW() - `, - definition.Namespace, definition.AgentName, definition.AgentUID, - definition.DesiredRevision, - ); err != nil { + changed = retired.RowsAffected() > 0 + grantsChanged, err := queryOne(ctx, tx, ` + WITH previous AS MATERIALIZED ( + SELECT grant_destinations, grant_credentials, retired_at FROM agent_definition + WHERE namespace = $1 AND agent_uid = $3 FOR UPDATE + ), updated AS ( + INSERT INTO agent_definition (namespace, agent_name, agent_uid, desired_revision, grant_destinations, grant_credentials) + SELECT $1, $2, $3, $4, $5, $6 FROM (SELECT COUNT(*) FROM previous) locked + ON CONFLICT (namespace, agent_uid) DO UPDATE SET + agent_name = EXCLUDED.agent_name, desired_revision = EXCLUDED.desired_revision, + grant_destinations = EXCLUDED.grant_destinations, grant_credentials = EXCLUDED.grant_credentials, + retired_at = NULL, updated_at = NOW() + RETURNING namespace + ) + SELECT COALESCE((SELECT grant_destinations FROM previous WHERE retired_at IS NULL), '{}'::text[]) IS DISTINCT FROM $5 + OR COALESCE((SELECT grant_credentials FROM previous WHERE retired_at IS NULL), '[]'::jsonb) IS DISTINCT FROM $6 + FROM updated + `, pgx.RowTo[bool], definition.Namespace, definition.AgentName, definition.AgentUID, definition.DesiredRevision, grants.Destinations, grants.Credentials) + if err != nil { return err } + + changed = changed || grantsChanged // Lock Agents before revisions. Validate the resulting definition, including // its retained last-good pointer when reactivating a retired identity. type revisionStatus struct { @@ -65,13 +84,13 @@ func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefi } return nil }) + return changed, err } -// RecordRuntimeRevision atomically stores revision inputs and the initial policy -// without making them available for new Sessions. Existing inputs and policy are -// preserved; only the actor UID and update time are refreshed. Malformed policies -// are rejected. Deleting revisions return ErrObjectDeleting. -func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevision, policy egress.Access) error { +// RecordRuntimeRevision stores immutable inputs without making them available for +// new Sessions. Existing inputs are preserved; actor UID and update time refresh. +// Deleting revisions return ErrObjectDeleting. +func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevision) error { if revision.AgentCard == nil { return fmt.Errorf("runtime revision %s has no Agent Card", revision.Revision) } @@ -84,7 +103,7 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi RuntimeArtifact: RuntimeArtifact{Revision: revision.Revision, Kind: runtimeKindAgent, Namespace: revision.Namespace, ActorTemplateAtespace: revision.ActorTemplateAtespace, ActorTemplateName: revision.ActorTemplateName, ActorTemplateUID: revision.ActorTemplateUID}, SourceSnapshot: revision.SourceSnapshot, EgressDestinations: revision.EgressDestinations, Credentials: revision.Credentials, TokenPassthroughOrigins: revision.TokenPassthroughOrigins, - }, policy); err != nil { + }); err != nil { return err } if err := execSQL(ctx, tx, ` @@ -239,20 +258,9 @@ type runtimeRevisionRecord struct { Credentials []egress.Credential } -// recordRuntimeRevision writes common inputs, mutable artifact status, and an -// insert-only initial policy. The caller must write its typed extension in the -// same transaction. Preparation retries never replace a published live policy. -func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevisionRecord, policy egress.Access) error { - policy, err := egress.CanonicalAccess(policy.Destinations, policy.Credentials) - if err != nil { - return fmt.Errorf("validate initial runtime egress policy %s: %w", revision.Revision, err) - } - if policy.Destinations == nil { - policy.Destinations = []string{} - } - if policy.Credentials == nil { - policy.Credentials = []egress.Credential{} - } +// recordRuntimeRevision writes immutable common inputs and mutable artifact status. +// The caller writes its typed extension in the same transaction. +func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevisionRecord) error { if revision.Credentials == nil { revision.Credentials = []egress.Credential{} } @@ -273,16 +281,10 @@ func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevis if err != nil { return fmt.Errorf("record runtime revision %s: %w", revision.Revision, err) } - if result.RowsAffected() == 0 { - if err := lockRuntimeRevisionForReference(ctx, tx, revision.Revision, revision.Kind); err != nil { - return err - } + if result.RowsAffected() != 0 { + return nil } - return execSQL(ctx, tx, ` - INSERT INTO runtime_egress_policy (revision, destinations, credentials) - VALUES ($1, $2, $3) - ON CONFLICT (revision) DO NOTHING - `, revision.Revision, policy.Destinations, policy.Credentials) + return lockRuntimeRevisionForReference(ctx, tx, revision.Revision, revision.Kind) } // BeginRuntimeRevisionDeletion marks an unreferenced revision as deleting, diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index 3082ec3779..aafec031ad 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -151,12 +151,13 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { revision.Revision = "new" revision.AgentUID = "replacement-uid" revision.ActorTemplateName = "new-actor-template" - require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, egress.Access{})) + require.NoError(t, client.RecordRuntimeRevision(ctx, *revision)) pair := AgentDefinition{ Namespace: "team-a", AgentName: "assistant", AgentUID: revision.AgentUID, DesiredRevision: "new", } - require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) + _, upsertErr := client.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) request := newSessionRequest(uuid.NewString(), "assistant", "kagent", "") _, _, err = client.CreateSession(ctx, request, "session") require.ErrorIs(t, err, ErrNotFound, "a replacement must not select the previous template UID") @@ -168,7 +169,8 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { // Reconciliation of the same identity must preserve its last success while // a new desired revision is still preparing. pair.DesiredRevision = "pending" - require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) + _, upsertErr = client.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) retried, created, err := client.CreateSession(ctx, request, "session") require.NoError(t, err) require.False(t, created) @@ -253,7 +255,8 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) pair.AgentUID = "replacement-uid" pair.DesiredRevision = "revision" } - created <- creating.UpsertAgentDefinition(ctx, pair) + _, upsertErr := creating.UpsertAgentDefinition(ctx, pair) + created <- upsertErr }() select { case <-barrier.reached: @@ -324,7 +327,8 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) // A skipped finalization must leave last-good intact for reactivation. require.NoError(t, client.DeleteRuntimeRevision(ctx, "revision", "revision-actor-uid")) - require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) + _, upsertErr := client.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) session, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "session") require.NoError(t, err) require.Equal(t, "revision", session.GetPreparedRevision()) @@ -337,11 +341,13 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.NotNil(t, claimed) // Reactivating a retired success pointer and adding a new desired edge are // both forbidden, as is overwriting a runtime while deletion is in flight. - require.ErrorIs(t, client.UpsertAgentDefinition(ctx, pair), ErrObjectDeleting) + _, upsertErr = client.UpsertAgentDefinition(ctx, pair) + require.ErrorIs(t, upsertErr, ErrObjectDeleting) pair.AgentUID = "replacement-uid" pair.DesiredRevision = "revision" - require.ErrorIs(t, client.UpsertAgentDefinition(ctx, pair), ErrObjectDeleting) - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, egress.Access{}), ErrObjectDeleting) + _, upsertErr = client.UpsertAgentDefinition(ctx, pair) + require.ErrorIs(t, upsertErr, ErrObjectDeleting) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original), ErrObjectDeleting) require.ErrorIs(t, client.PromoteRuntimeRevision(ctx, original.Revision), ErrObjectDeleting) // A new client rediscovers and retries the committed claim after a crash. restarted := NewClient(pool, "public") @@ -358,7 +364,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.ErrorIs(t, err, ErrNotFound) // The same digest can be prepared again once cleanup has completed. original.ActorTemplateUID = "recreated-actor-uid" - require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, egress.Access{})) + require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original)) newClaim, err := restarted.BeginRuntimeRevisionDeletion(ctx, "revision") require.NoError(t, err) require.NotNil(t, newClaim) @@ -366,8 +372,9 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) _, err = restarted.GetRuntimeRevision(ctx, "revision") require.NoError(t, err, "a delayed collector must not finalize the new runtime") require.NoError(t, restarted.DeleteRuntimeRevision(ctx, "revision", "recreated-actor-uid")) - require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, egress.Access{})) - require.NoError(t, restarted.UpsertAgentDefinition(ctx, pair)) + require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original)) + _, upsertErr = restarted.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) } func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { @@ -419,7 +426,8 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { created <- creating.PromoteRuntimeRevision(ctx, "revision") return } - created <- creating.UpsertAgentDefinition(ctx, pair) + _, upsertErr := creating.UpsertAgentDefinition(ctx, pair) + created <- upsertErr } finalize := func() { deleted <- deleting.DeleteRuntimeRevision(ctx, "revision", claimed.ActorTemplateUID) } if finalizeFirst { @@ -451,7 +459,8 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { require.ErrorIs(t, <-created, ErrObjectDeleting) } require.NoError(t, <-deleted, "finalization and pair writes must not deadlock") - require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) + _, upsertErr := client.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) }) } } @@ -485,8 +494,9 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { require.NoError(t, err) require.Equal(t, want, session.PreparedRevision) } - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, egress.Access{})) + _, upsertErr := c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) assertAvailableRevision("") stored, err := c.GetRuntimeRevision(ctx, revision.Revision) require.NoError(t, err) @@ -496,9 +506,10 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("first") pair.DesiredRevision = "second" - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) + _, upsertErr = c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) revision.Revision, revision.ActorTemplateName = "second", "second" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, egress.Access{})) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) assertAvailableRevision("first") require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("second") @@ -507,29 +518,32 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("second") pair.DesiredRevision = "third" - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) + _, upsertErr = c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) require.NoError(t, c.RetireAgentIdentities(ctx, pair.Namespace, pair.AgentName, nil)) revision.Revision, revision.ActorTemplateName = "third", "third" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, egress.Access{})) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("") // Reviving the pair must retain the last success, not a report made while retired. - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) + _, upsertErr = c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) assertAvailableRevision("second") pair.DesiredRevision = "second" - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) + _, upsertErr = c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) assertAvailableRevision("second") } func TestRuntimeRevisionPersistsCredentialBindings(t *testing.T) { client := NewClient(setupTestDB(t), "public") revision := RuntimeRevision{Revision: "credential-revision", Namespace: "team", AgentName: "agent", AgentUID: "agent", SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{"https://api.example.com:443"}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", Credentials: []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}}} - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, egress.Access{})) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision)) got, err := client.GetRuntimeRevision(t.Context(), revision.Revision) require.NoError(t, err) require.Equal(t, revision.Credentials, got.Credentials) revision.Credentials[0].URI = "ate-secret://k8s.io/default/team/other/token" - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, egress.Access{})) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision)) unchanged, err := client.GetRuntimeRevision(t.Context(), revision.Revision) require.NoError(t, err) require.Equal(t, got.Credentials, unchanged.Credentials, "revision credentials are immutable") diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index c3f8724738..da8f9c08fa 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -19,6 +19,7 @@ type SandboxRevision struct { // SandboxTemplateDefinition tracks the desired and last successful runtime for a SandboxTemplate UID. type SandboxTemplateDefinition struct { + Grants egress.Access Namespace string SandboxTemplateName string SandboxTemplateUID string @@ -27,22 +28,45 @@ type SandboxTemplateDefinition struct { // UpsertSandboxTemplateDefinition pins desired inputs before any backend mutation. A new // Kubernetes UID retires the former identity without inheriting its last success. -func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition SandboxTemplateDefinition) error { - return c.withTx(ctx, func(tx pgx.Tx) error { - if err := execSQL(ctx, tx, ` +// Grants are canonicalized before writing. The bool reports changed active grants +// or retirement of another UID; errors may include an uncertain commit outcome. +func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition SandboxTemplateDefinition) (bool, error) { + grants, err := canonicalStoredGrants(definition.Grants) + if err != nil { + return false, fmt.Errorf("validate source grants: %w", err) + } + changed := false + err = c.withTx(ctx, func(tx pgx.Tx) error { + retired, err := tx.Exec(ctx, ` UPDATE sandbox_template_definition SET retired_at = NOW(), updated_at = NOW() WHERE namespace = $1 AND sandbox_template_name = $2 AND sandbox_template_uid <> $3 AND retired_at IS NULL - `, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID); err != nil { + `, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID) + if err != nil { return err } - if err := execSQL(ctx, tx, ` - INSERT INTO sandbox_template_definition (namespace, sandbox_template_name, sandbox_template_uid, desired_revision) - VALUES ($1, $2, $3, $4) - ON CONFLICT (namespace, sandbox_template_uid) DO UPDATE SET - desired_revision = EXCLUDED.desired_revision, retired_at = NULL, updated_at = NOW() - `, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID, definition.DesiredRevision); err != nil { + changed = retired.RowsAffected() > 0 + grantsChanged, err := queryOne(ctx, tx, ` + WITH previous AS MATERIALIZED ( + SELECT grant_destinations, grant_credentials, retired_at FROM sandbox_template_definition + WHERE namespace = $1 AND sandbox_template_uid = $3 FOR UPDATE + ), updated AS ( + INSERT INTO sandbox_template_definition (namespace, sandbox_template_name, sandbox_template_uid, desired_revision, grant_destinations, grant_credentials) + SELECT $1, $2, $3, $4, $5, $6 FROM (SELECT COUNT(*) FROM previous) locked + ON CONFLICT (namespace, sandbox_template_uid) DO UPDATE SET + sandbox_template_name = EXCLUDED.sandbox_template_name, desired_revision = EXCLUDED.desired_revision, + grant_destinations = EXCLUDED.grant_destinations, grant_credentials = EXCLUDED.grant_credentials, + retired_at = NULL, updated_at = NOW() + RETURNING namespace + ) + SELECT COALESCE((SELECT grant_destinations FROM previous WHERE retired_at IS NULL), '{}'::text[]) IS DISTINCT FROM $5 + OR COALESCE((SELECT grant_credentials FROM previous WHERE retired_at IS NULL), '[]'::jsonb) IS DISTINCT FROM $6 + FROM updated + `, pgx.RowTo[bool], definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID, definition.DesiredRevision, grants.Destinations, grants.Credentials) + if err != nil { return err } + + changed = changed || grantsChanged revisions, err := queryMany(ctx, tx, ` SELECT r.revision, r.kind, r.namespace, r.actor_template_atespace, r.actor_template_name, r.actor_template_uid, r.deleted_at FROM runtime_revision r JOIN sandbox_template_definition p @@ -62,8 +86,11 @@ func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition } return nil }) + return changed, err } +// RetireSandboxTemplateIdentities retires active identities at this source name. +// Missing/already-retired identities are a no-op; retained actors lose live grants. func (c *Client) RetireSandboxTemplateIdentities(ctx context.Context, namespace, name string) error { return execSQL(ctx, c.db, ` UPDATE sandbox_template_definition SET retired_at = NOW(), updated_at = NOW() @@ -71,14 +98,12 @@ func (c *Client) RetireSandboxTemplateIdentities(ctx context.Context, namespace, `, namespace, name) } -// RecordSandboxRevision atomically stores sandbox inputs and the initial policy -// without promoting the revision. Existing inputs and policy are preserved; only -// artifact identity is refreshed. Malformed policies are rejected. Deleting -// revisions return ErrObjectDeleting. -func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevision, policy egress.Access) error { +// RecordSandboxRevision stores immutable sandbox inputs and refreshes artifact +// identity without promoting the revision. Deleting revisions return ErrObjectDeleting. +func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevision) error { return c.withTx(ctx, func(tx pgx.Tx) error { revision.Kind = runtimeKindSandbox - if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{RuntimeArtifact: revision.RuntimeArtifact, SourceSnapshot: revision.SourceSnapshot}, policy); err != nil { + if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{RuntimeArtifact: revision.RuntimeArtifact, SourceSnapshot: revision.SourceSnapshot}); err != nil { return err } if err := execSQL(ctx, tx, ` diff --git a/go/core/internal/database/sandboxes_test.go b/go/core/internal/database/sandboxes_test.go index 61c95ad6c6..cf8205dae3 100644 --- a/go/core/internal/database/sandboxes_test.go +++ b/go/core/internal/database/sandboxes_test.go @@ -11,15 +11,15 @@ import ( "google.golang.org/protobuf/proto" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" - "github.com/kagent-dev/kagent/go/core/internal/egress" ) func sandboxFixture(t *testing.T, client *Client, revision string) (*apiv1.Sandbox, SandboxCreateOptions) { t.Helper() ctx := t.Context() - require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision})) + _, upsertErr := client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) + require.NoError(t, upsertErr) require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: revision, Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: revision, ActorTemplateUID: "actor-uid"}, - SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, egress.Access{})) + SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)})) require.NoError(t, client.PromoteSandboxRevision(ctx, revision)) hash := sha256.Sum256([]byte("request")) return &apiv1.Sandbox{Id: uuid.NewString(), Creator: "alice", SandboxTemplate: &apiv1.ResourceReference{Namespace: "team-a", Name: "scratch"}, Name: "experiment"}, @@ -31,11 +31,12 @@ func TestPromoteSandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { ctx := t.Context() request, options := sandboxFixture(t, client, "first") definition := SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: "second"} - require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, definition)) + _, upsertErr := client.UpsertSandboxTemplateDefinition(ctx, definition) + require.NoError(t, upsertErr) revision, err := client.GetSandboxRevision(ctx, "first") require.NoError(t, err) revision.Revision, revision.ActorTemplateName = "second", "second" - require.NoError(t, client.RecordSandboxRevision(ctx, *revision, egress.Access{})) + require.NoError(t, client.RecordSandboxRevision(ctx, *revision)) assertAvailable := func(want string) { t.Helper() input := proto.CloneOf(request) @@ -57,10 +58,11 @@ func TestPromoteSandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { require.NoError(t, client.RetireSandboxTemplateIdentities(ctx, "team-a", "scratch")) require.NoError(t, client.PromoteSandboxRevision(ctx, "first")) assertAvailable("") - require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, definition)) + _, upsertErr = client.UpsertSandboxTemplateDefinition(ctx, definition) + require.NoError(t, upsertErr) assertAvailable("second") revision.Revision, revision.ActorTemplateName = "unused", "unused" - require.NoError(t, client.RecordSandboxRevision(ctx, *revision, egress.Access{})) + require.NoError(t, client.RecordSandboxRevision(ctx, *revision)) claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "unused") require.NoError(t, err) require.NotNil(t, claimed) diff --git a/go/core/internal/egress/live.go b/go/core/internal/egress/live.go index 066c89e72f..f08e6206da 100644 --- a/go/core/internal/egress/live.go +++ b/go/core/internal/egress/live.go @@ -6,10 +6,26 @@ import ( "slices" ) -// PolicyStore reads the controller's persisted intended policy for a revision. +// Policy is effective access plus a diagnostic when conflicting grants were withheld. +type Policy struct { + Access + WithheldGrants error +} + +// PolicyStore reads effective policy from persisted revision inputs and source grants. // A missing or malformed policy is an error, never permission to retain old access. type PolicyStore interface { - GetRuntimeEgressPolicy(context.Context, string) (Access, error) + GetRuntimeEgressPolicy(context.Context, string) (Policy, error) +} + +// EffectiveAccess merges validated pinned access and source grants. A conflict +// withholds all explicit grants while retaining the pinned base and its diagnostic. +func EffectiveAccess(base, grants Access, tokenPassthroughOrigins []string) (Access, error) { + access, err := MergeAccess(base, grants, tokenPassthroughOrigins) + if err != nil { + return base, err + } + return access, nil } // MergeAccess validates live grants against the pinned runtime's credentials and diff --git a/go/core/internal/egress/live_test.go b/go/core/internal/egress/live_test.go new file mode 100644 index 0000000000..ae718fdd69 --- /dev/null +++ b/go/core/internal/egress/live_test.go @@ -0,0 +1,46 @@ +package egress + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEffectiveAccess(t *testing.T) { + binding := Credential{Origin: "https://model.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"} + base := Access{Destinations: []string{binding.Origin}, Credentials: []Credential{binding}} + conflict := binding + conflict.URI = "ate-secret://k8s.io/default/team/other/token" + tooMany := Access{} + for i := range 256 { + tooMany.Destinations = append(tooMany.Destinations, fmt.Sprintf("https://host%d.example.com:443", i)) + } + for _, test := range []struct { + name string + base, grants Access + origins []string + withheld bool + }{ + {name: "no grants", base: base}, + {name: "additive", base: base, grants: Access{Destinations: []string{"https://tickets.example.com:443"}}}, + {name: "sandbox", grants: Access{Destinations: []string{"https://tickets.example.com:443"}}}, + {name: "duplicate", base: base, grants: base}, + {name: "credentials conflict", base: base, grants: Access{Destinations: base.Destinations, Credentials: []Credential{conflict}}, withheld: true}, + {name: "token passthrough", base: Access{Destinations: base.Destinations}, grants: base, origins: base.Destinations, withheld: true}, + {name: "rule limit", base: base, grants: tooMany, withheld: true}, + } { + t.Run(test.name, func(t *testing.T) { + access, err := EffectiveAccess(test.base, test.grants, test.origins) + if test.withheld { + require.Error(t, err) + require.Equal(t, test.base, access) + } else { + require.NoError(t, err) + for _, origin := range append(append([]string{}, test.base.Destinations...), test.grants.Destinations...) { + require.Contains(t, access.Destinations, origin) + } + } + }) + } +} diff --git a/go/core/internal/grpcserver/scheduledrun_test.go b/go/core/internal/grpcserver/scheduledrun_test.go index d14be7d510..11ed6cdd54 100644 --- a/go/core/internal/grpcserver/scheduledrun_test.go +++ b/go/core/internal/grpcserver/scheduledrun_test.go @@ -26,7 +26,6 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" - "github.com/kagent-dev/kagent/go/core/internal/egress" authimpl "github.com/kagent-dev/kagent/go/core/internal/httpserver/auth" "github.com/kagent-dev/kagent/go/core/internal/service/scheduledrun" sessionsvc "github.com/kagent-dev/kagent/go/core/internal/service/session" @@ -181,12 +180,13 @@ func scheduledRunTestServer(t *testing.T) (*database.Client, apiv1.ScheduledRunS t.Cleanup(db.Close) store := database.NewClient(db, "public") pair := database.AgentDefinition{Namespace: "team", AgentName: "report", AgentUID: "template-uid", DesiredRevision: "scheduled-revision"} - require.NoError(t, store.UpsertAgentDefinition(t.Context(), pair)) + _, upsertErr := store.UpsertAgentDefinition(t.Context(), pair) + require.NoError(t, upsertErr) require.NoError(t, store.RecordRuntimeRevision(t.Context(), database.RuntimeRevision{ Revision: pair.DesiredRevision, Namespace: pair.Namespace, AgentName: pair.AgentName, AgentUID: pair.AgentUID, SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: "runtime-uid", - }, egress.Access{})) + })) require.NoError(t, store.PromoteRuntimeRevision(t.Context(), pair.DesiredRevision)) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) diff --git a/go/core/internal/grpcserver/task_store_test.go b/go/core/internal/grpcserver/task_store_test.go index a8b6ec665b..30960c9baa 100644 --- a/go/core/internal/grpcserver/task_store_test.go +++ b/go/core/internal/grpcserver/task_store_test.go @@ -43,7 +43,6 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/a2agateway" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" - "github.com/kagent-dev/kagent/go/core/internal/egress" authimpl "github.com/kagent-dev/kagent/go/core/internal/httpserver/auth" sessionsvc "github.com/kagent-dev/kagent/go/core/internal/service/session" "github.com/kagent-dev/kagent/go/core/internal/service/taskstore" @@ -638,11 +637,12 @@ func createTaskStoreSession(t *testing.T, store *database.Client) *apiv1.Session AgentCard: &a2apb.AgentCard{Name: "assistant"}, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "assistant-kagent-revision", ActorTemplateUID: "actor-template-uid", } - require.NoError(t, store.UpsertAgentDefinition(t.Context(), database.AgentDefinition{ + _, upsertErr := store.UpsertAgentDefinition(t.Context(), database.AgentDefinition{ Namespace: revision.Namespace, AgentName: revision.AgentName, AgentUID: revision.AgentUID, DesiredRevision: revision.Revision, - })) - require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision, egress.Access{})) + }) + require.NoError(t, upsertErr) + require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision)) require.NoError(t, store.PromoteRuntimeRevision(t.Context(), revision.Revision)) session, _, err := store.CreateSession(t.Context(), &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", diff --git a/go/core/internal/service/sandbox/guest_test.go b/go/core/internal/service/sandbox/guest_test.go index b54e9865e9..b5f153971d 100644 --- a/go/core/internal/service/sandbox/guest_test.go +++ b/go/core/internal/service/sandbox/guest_test.go @@ -28,7 +28,6 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" - "github.com/kagent-dev/kagent/go/core/internal/egress" sandboxservice "github.com/kagent-dev/kagent/go/core/internal/service/sandbox" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/pkg/auth" @@ -130,11 +129,12 @@ func guestFixture(t *testing.T) (*sandboxservice.Service, context.Context, *test require.NoError(t, err) t.Cleanup(pool.Close) store := database.NewClient(pool, "public") - require.NoError(t, store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"})) + _, upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"}) + require.NoError(t, upsertErr) require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - }, egress.Access{})) + })) require.NoError(t, store.PromoteSandboxRevision(ctx, "revision")) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) diff --git a/go/core/internal/service/sandbox/service_test.go b/go/core/internal/service/sandbox/service_test.go index 02426f7074..54cfb92735 100644 --- a/go/core/internal/service/sandbox/service_test.go +++ b/go/core/internal/service/sandbox/service_test.go @@ -22,7 +22,6 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/pkg/auth" ) @@ -135,11 +134,12 @@ func serviceFixture(t *testing.T) (*Service, context.Context, *testActors) { require.NoError(t, err) t.Cleanup(pool.Close) store := database.NewClient(pool, "public") - require.NoError(t, store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"})) + _, upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"}) + require.NoError(t, upsertErr) require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - }, egress.Access{})) + })) require.NoError(t, store.PromoteSandboxRevision(ctx, "revision")) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) @@ -296,8 +296,9 @@ func TestSandboxResolvesCurrentGrantsBeforeResume(t *testing.T) { SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), } store := service.config.Store - require.NoError(t, store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: revision.Revision})) - require.NoError(t, store.RecordSandboxRevision(ctx, revision, egress.Access{})) + _, upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: revision.Revision}) + require.NoError(t, upsertErr) + require.NoError(t, store.RecordSandboxRevision(ctx, revision)) require.NoError(t, store.PromoteSandboxRevision(ctx, revision.Revision)) instance, err := service.Create(ctx, createRequest()) require.NoError(t, err) diff --git a/go/core/internal/service/session/workflow_test.go b/go/core/internal/service/session/workflow_test.go index c92a785cbc..d5719949e4 100644 --- a/go/core/internal/service/session/workflow_test.go +++ b/go/core/internal/service/session/workflow_test.go @@ -148,8 +148,9 @@ func lifecycleFixture(t *testing.T) (*lifecycleTestStore, *apiv1.Session) { AgentCard: &a2apb.AgentCard{Name: "assistant"}, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "assistant-kagent-revision", ActorTemplateUID: "actor-template-uid", } - require.NoError(t, client.UpsertAgentDefinition(t.Context(), database.AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision})) - require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision, egress.Access{})) + _, upsertErr := client.UpsertAgentDefinition(t.Context(), database.AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision}) + require.NoError(t, upsertErr) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision)) require.NoError(t, client.PromoteRuntimeRevision(t.Context(), revision.Revision)) session, _, err := client.CreateSession(t.Context(), &apiv1.Session{Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}}, uuid.NewString()) require.NoError(t, err) diff --git a/go/core/internal/substrate/egress_test.go b/go/core/internal/substrate/egress_test.go index 2b917691a8..14a61e9fed 100644 --- a/go/core/internal/substrate/egress_test.go +++ b/go/core/internal/substrate/egress_test.go @@ -15,9 +15,9 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/egress" ) -type egressPolicyStoreFunc func(context.Context, string) (egress.Access, error) +type egressPolicyStoreFunc func(context.Context, string) (egress.Policy, error) -func (f egressPolicyStoreFunc) GetRuntimeEgressPolicy(ctx context.Context, revision string) (egress.Access, error) { +func (f egressPolicyStoreFunc) GetRuntimeEgressPolicy(ctx context.Context, revision string) (egress.Policy, error) { return f(ctx, revision) } @@ -26,9 +26,9 @@ var _ egress.PolicyStore = egressPolicyStoreFunc(nil) func TestEnsureActorEgressPolicyUpdatesCurrentGrants(t *testing.T) { fake := &egressPolicyFake{createErr: context.DeadlineExceeded} access := egress.Access{Destinations: []string{"https://api.example.com:443"}} - client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(_ context.Context, revision string) (egress.Access, error) { + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(_ context.Context, revision string) (egress.Policy, error) { require.Equal(t, "pinned-revision", revision) - return access, nil + return egress.Policy{Access: access}, nil })}} reconcile := func() error { return client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "pinned-revision") } require.ErrorIs(t, reconcile(), context.DeadlineExceeded) @@ -57,7 +57,7 @@ func TestEnsureActorEgressPolicyUpdatesCurrentGrants(t *testing.T) { func TestEnsureActorEgressPolicyReloadsAfterConflict(t *testing.T) { fake := &egressPolicyFake{} access := egress.Access{} - client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Access, error) { return access, nil })}} + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Policy, error) { return egress.Policy{Access: access}, nil })}} reconcile := func() error { return client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "revision") } require.NoError(t, reconcile()) access.Destinations = []string{"https://revoked.example.com:443"} @@ -125,7 +125,7 @@ func (f *egressPolicyFake) UpdateActorEgressPolicy(_ context.Context, req *ateap func TestEnsureActorEgressPolicyCatchesUpdateDuringCreation(t *testing.T) { fake := &egressPolicyFake{} access := egress.Access{Destinations: []string{"https://revoked.example.com:443"}} - client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Access, error) { return access, nil })}} + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Policy, error) { return egress.Policy{Access: access}, nil })}} fake.afterCreate = func() { // The controller published revocation, but its actor discovery ran before // creation. There is no concurrent Substrate write to cause a CAS conflict. @@ -140,8 +140,19 @@ func TestEnsureActorEgressPolicyCatchesUpdateDuringCreation(t *testing.T) { func TestEnsureActorEgressPolicyRequiresPersistedPolicy(t *testing.T) { fake := &egressPolicyFake{} unavailable := errors.New("intended policy unavailable") - client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Access, error) { return egress.Access{}, unavailable })}} + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Policy, error) { return egress.Policy{}, unavailable })}} require.ErrorIs(t, client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision"), unavailable) require.Zero(t, fake.created) require.Zero(t, fake.updated) } + +func TestEnsureActorEgressPolicyAppliesFallbackDespiteWithheldGrants(t *testing.T) { + base := egress.Access{Destinations: []string{"https://model.example.com:443"}} + fake := &egressPolicyFake{} + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Policy, error) { + return egress.Policy{Access: base, WithheldGrants: errors.New("credential conflict")}, nil + })}} + require.NoError(t, client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision")) + require.Len(t, fake.policy.Rules, 1) + require.Equal(t, []string{"model.example.com"}, fake.policy.Rules[0].GetHttps().GetHostnames()) +} diff --git a/go/core/internal/translator/compiler.go b/go/core/internal/translator/compiler.go index ac0f22d8e1..8020582cc6 100644 --- a/go/core/internal/translator/compiler.go +++ b/go/core/internal/translator/compiler.go @@ -178,7 +178,6 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agent *v1alpha3.Age if _, err := egress.MergeAccess(egress.Access{Destinations: result.EgressDestinations, Credentials: result.Credentials}, access, result.TokenPassthroughOrigins); err != nil { return nil, NewValidationError("%v", err) } - result.Egress = access workerKey := types.NamespacedName{Namespace: harness.Namespace, Name: harness.Spec.Substrate.WorkerPoolRef.Name} workerPool := krt.FetchOne(c.ctx, c.collections.WorkerPools, krt.FilterObjectName(workerKey)) if workerPool == nil { diff --git a/go/core/internal/translator/egress_test.go b/go/core/internal/translator/egress_test.go index b24030b541..622556728b 100644 --- a/go/core/internal/translator/egress_test.go +++ b/go/core/internal/translator/egress_test.go @@ -86,8 +86,6 @@ func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { return result } result := compile() - require.Equal(t, []string{"https://tickets.example.com:443"}, result.Egress.Destinations) - require.Len(t, result.Egress.Credentials, 1) require.NotContains(t, result.EgressDestinations, "https://tickets.example.com:443") require.NotEmpty(t, result.EgressDestinations, "preserve inferred integration access") for _, binding := range result.Credentials { @@ -104,7 +102,6 @@ func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { require.Equal(t, original, rotated) policy.Spec.Destinations = append(policy.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://registry.npmjs.org"}) changed := compile() - require.Contains(t, changed.Egress.Destinations, "https://registry.npmjs.org:443") updated, err := changed.Digest() require.NoError(t, err) require.Equal(t, original, updated) diff --git a/go/core/internal/translator/revision.go b/go/core/internal/translator/revision.go index bb72d060b2..02dd438ee4 100644 --- a/go/core/internal/translator/revision.go +++ b/go/core/internal/translator/revision.go @@ -30,12 +30,10 @@ func (id RevisionID) Short() string { return hex.EncodeToString(id[:shortRevisio // IsZero reports whether compilation has not produced an identity. func (id RevisionID) IsZero() bool { return id == RevisionID{} } -// CompileResult contains immutable runtime inputs, current explicit grants, -// and non-blocking diagnostics. Live grants and diagnostics do not enter the -// revision's identity. +// CompileResult contains immutable runtime inputs and non-blocking diagnostics. +// Diagnostics do not enter the revision's identity. type CompileResult struct { Revision - Egress egress.Access Warnings []string } diff --git a/go/core/pkg/app/app.go b/go/core/pkg/app/app.go index 836d70d215..940140c7a2 100644 --- a/go/core/pkg/app/app.go +++ b/go/core/pkg/app/app.go @@ -287,11 +287,11 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er return err } defer actors.Close() - egressReconciler := controller.NewEgressReconciler(ctx, runtime, store, actors, watchNamespaces) - if err := manager.Add(egressReconciler); err != nil { - return fmt.Errorf("add egress reconciler: %w", err) + egressApplier := controller.NewActorEgressApplier(ctx, store, actors, watchNamespaces) + if err := manager.Add(egressApplier); err != nil { + return fmt.Errorf("add egress applier: %w", err) } - reconciler, err := controller.NewReconciler(kubeConfig, runtime.Collections, store, actors, egressReconciler.EnqueueRevision) + reconciler, err := controller.NewReconciler(kubeConfig, runtime.Collections, store, actors, egressApplier.EnqueueSource) if err != nil { return err } @@ -379,7 +379,7 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er CPU: cfg.GetSandboxes().GetCpu(), Memory: cfg.GetSandboxes().GetMemory(), } - preparation, err := controller.NewSandboxReconciler(kubeConfig, runtime, store, actors, policy, egressReconciler.EnqueueRevision) + preparation, err := controller.NewSandboxReconciler(kubeConfig, runtime, store, actors, policy, egressApplier.EnqueueSource) if err != nil { return err } diff --git a/go/core/pkg/migrations/core/000001_initial.sql b/go/core/pkg/migrations/core/000001_initial.sql index 71bc68c1b7..44702c7736 100644 --- a/go/core/pkg/migrations/core/000001_initial.sql +++ b/go/core/pkg/migrations/core/000001_initial.sql @@ -46,13 +46,6 @@ CREATE TABLE runtime_revision ( UNIQUE (actor_template_atespace, actor_template_name) ); --- Mutable controller output lives outside immutable runtime revision inputs. -CREATE TABLE runtime_egress_policy ( - revision TEXT PRIMARY KEY REFERENCES runtime_revision (revision) ON DELETE CASCADE, - destinations TEXT[] NOT NULL, - credentials JSONB NOT NULL CHECK (jsonb_typeof(credentials) = 'array') -); - CREATE TABLE agent_revision ( revision TEXT PRIMARY KEY, agent_name TEXT NOT NULL, @@ -71,6 +64,8 @@ CREATE TABLE sandbox_revision ( CREATE INDEX sandbox_revision_source_idx ON sandbox_revision (sandbox_template_name, revision); CREATE TABLE agent_definition ( + grant_destinations TEXT[] NOT NULL DEFAULT '{}', + grant_credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(grant_credentials) = 'array'), namespace TEXT NOT NULL, agent_name TEXT NOT NULL, agent_uid TEXT NOT NULL, @@ -85,6 +80,8 @@ CREATE UNIQUE INDEX agent_definition_active_name_idx ON agent_definition (namespace, agent_name) WHERE retired_at IS NULL; CREATE TABLE sandbox_template_definition ( + grant_destinations TEXT[] NOT NULL DEFAULT '{}', + grant_credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(grant_credentials) = 'array'), namespace TEXT NOT NULL, sandbox_template_name TEXT NOT NULL, sandbox_template_uid TEXT NOT NULL, @@ -402,7 +399,6 @@ DROP TABLE sandbox_template_definition; DROP TABLE agent_definition; DROP TABLE sandbox_revision; DROP TABLE agent_revision; -DROP TABLE runtime_egress_policy; DROP TABLE runtime_revision; DROP TABLE toolserver; DROP TABLE tool; diff --git a/go/core/pkg/migrations/runner_test.go b/go/core/pkg/migrations/runner_test.go index 5394bf9ba2..019b448a56 100644 --- a/go/core/pkg/migrations/runner_test.go +++ b/go/core/pkg/migrations/runner_test.go @@ -326,8 +326,10 @@ func TestBuiltinMigrationsRoundTrip(t *testing.T) { if !testColumnExists(t, dsn, "runtime_revision", "token_passthrough_origins") || !testColumnExists(t, dsn, "agent_runtime_revision", "token_passthrough_origins") { t.Fatal("runtime and agent revisions must expose passthrough origins") } - if !testTableExists(t, dsn, "runtime_egress_policy") { - t.Fatal("mutable runtime egress policies are missing") + for _, table := range []string{"agent_definition", "sandbox_template_definition"} { + if !testColumnExists(t, dsn, table, "grant_destinations") || !testColumnExists(t, dsn, table, "grant_credentials") { + t.Fatalf("%s source grants are missing", table) + } } if !testColumnExists(t, dsn, "session", "deletion_reason") { t.Fatal("session deletion reason is missing") From 15c9680189da2bfc49f8d78b1adc141096106357 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 19:21:25 +0000 Subject: [PATCH 26/43] test: cover agent grants during compilation failures Signed-off-by: Eitan Yarmush --- go/core/internal/controller/grants_test.go | 68 ++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/go/core/internal/controller/grants_test.go b/go/core/internal/controller/grants_test.go index bcfc742991..48f87035dd 100644 --- a/go/core/internal/controller/grants_test.go +++ b/go/core/internal/controller/grants_test.go @@ -63,6 +63,74 @@ func TestAgentGrantChangesReconcileWithoutRuntimeChange(t *testing.T) { }) } +func TestAgentGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { + for _, missing := range []string{"Harness", "AgentTemplate", "WorkerPool"} { + t.Run(missing, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + collections, harnesses := newPreparationTestCollections(t, "gvisor") + agent := collections.Agents.List()[0].DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + switch missing { + case "Harness": + agent.Spec.HarnessRef.Name = "missing" + case "AgentTemplate": + agent.Spec.Template = v1alpha3.AgentTemplateSource{Ref: &corev1.LocalObjectReference{Name: "missing"}} + case "WorkerPool": + harness := harnesses.List()[0].DeepCopy() + harness.Spec.Substrate.WorkerPoolRef.Name = "missing" + harnesses.UpdateObject(harness) + } + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + synctest.Wait() + store := &fakeRuntimeRevisionStore{} + backend := &preparationTestBackend{templates: &fakeActorTemplates{}, store: store} + var sources []database.RuntimeEgressSource + r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3(), func(source database.RuntimeEgressSource) { sources = append(sources, source) }) + ctx, cancel := context.WithCancel(t.Context()) + var workers sync.WaitGroup + workers.Go(func() { r.Run(ctx.Done()) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + sources = nil + + policies := collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]) + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} + policies.UpdateObject(policy) + synctest.Wait() + state := collections.Reconciliations.GetKey("team-a/assistant") + require.Nil(t, state.Target) + require.NotNil(t, state.CompilationFailure) + require.Contains(t, state.CompilationFailure.Message, missing) + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.Equal(t, []string{"https://tickets.example.com:443"}, store.pair.Grants.Destinations) + }) + wantSource := database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"} + require.Equal(t, []database.RuntimeEgressSource{wantSource}, sources) + + policy = policy.DeepCopy() + policy.Spec.Destinations[0].Origin = "https://updated.example.com" + policies.UpdateObject(policy) + synctest.Wait() + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.Equal(t, []string{"https://updated.example.com:443"}, store.pair.Grants.Destinations) + }) + require.Equal(t, []database.RuntimeEgressSource{wantSource, wantSource}, sources) + + policies.DeleteObject("team-a/access") + synctest.Wait() + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.Empty(t, store.pair.Grants.Destinations) + require.Zero(t, store.recordCalls, "grant changes must not require a compiled runtime") + }) + require.Equal(t, []database.RuntimeEgressSource{wantSource, wantSource, wantSource}, sources, "revocation must enqueue actors while compilation remains broken") + }) + }) + } +} + func TestAgentGrantWriteFailureAndRetirementEnqueueActors(t *testing.T) { collections, _ := newPreparationTestCollections(t, "gvisor") state := collections.Reconciliations.List()[0] From 97555337d7d91644dcf07e0348097b6291f4d2dc Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 19:35:55 +0000 Subject: [PATCH 27/43] fix: simplify egress reconciliation and preserve validation semantics Signed-off-by: Eitan Yarmush --- go/core/internal/controller/egress.go | 7 +- go/core/internal/controller/egress_test.go | 8 +- go/core/internal/controller/grants_test.go | 37 +++++++++ .../internal/controller/preparation_test.go | 4 +- go/core/internal/controller/reconciler.go | 36 +++++---- .../internal/controller/reconciler_test.go | 6 +- .../internal/database/client_session_test.go | 2 +- go/core/internal/database/models.go | 2 +- go/core/internal/database/runtime_egress.go | 65 +++++++--------- .../internal/database/runtime_egress_test.go | 76 ++++++++++++++++--- .../internal/database/runtime_revisions.go | 66 ++++++++-------- .../database/runtime_revisions_test.go | 63 ++++++--------- .../internal/database/sandbox_revisions.go | 55 ++++++++------ .../internal/database/scheduled_runs_test.go | 2 +- .../internal/service/sandbox/guest_test.go | 5 +- .../internal/service/sandbox/service_test.go | 5 +- .../internal/service/session/workflow_test.go | 4 +- go/core/internal/substrate/egress.go | 23 +++--- go/core/internal/substrate/egress_test.go | 28 +++++-- go/core/internal/substrate/lifecycle.go | 7 +- go/core/internal/substrate/lifecycle_test.go | 5 +- .../pkg/migrations/core/000001_initial.sql | 24 +++++- go/core/pkg/migrations/cross_track_test.go | 5 +- 23 files changed, 323 insertions(+), 212 deletions(-) diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go index bee572f65c..fb1dd7093b 100644 --- a/go/core/internal/controller/egress.go +++ b/go/core/internal/controller/egress.go @@ -20,14 +20,13 @@ import ( ) type actorEgressStore interface { - egress.PolicyStore ListRuntimeEgressTargets(context.Context, string, int) ([]database.RuntimeEgressTarget, error) ListRuntimeEgressSourceTargets(context.Context, database.RuntimeEgressSource, string, int) ([]database.RuntimeEgressTarget, error) } type actorEgressClient interface { GetActor(context.Context, string, string) (*ateapipb.Actor, error) - EnsureActorEgressPolicy(context.Context, string, string, string) error + EnsureActorEgressPolicy(context.Context, string, string, string) (egress.Policy, error) } // ActorEgressApplier applies persisted effective policy independently of runtime @@ -133,7 +132,7 @@ func (a *ActorEgressApplier) reconcileActor(ctx context.Context, target database if !binding.Matches(actor) || (target.ActorUID != "" && target.ActorUID != actor.GetMetadata().GetUid()) { return nil } - policy, err := a.store.GetRuntimeEgressPolicy(ctx, target.Revision) + policy, err := a.actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, target.Revision) if err != nil { return err } @@ -145,5 +144,5 @@ func (a *ActorEgressApplier) reconcileActor(ctx context.Context, target database } else { a.warnings.Remove(target.Revision) } - return a.actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, target.Revision) + return nil } diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go index d09f0051b7..54b9ed7e51 100644 --- a/go/core/internal/controller/egress_test.go +++ b/go/core/internal/controller/egress_test.go @@ -86,7 +86,7 @@ func (a *observingEgressActors) GetActor(_ context.Context, space, name string) } return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: "uid"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: template}}, nil } -func (a *observingEgressActors) EnsureActorEgressPolicy(ctx context.Context, _, name, revision string) error { +func (a *observingEgressActors) EnsureActorEgressPolicy(ctx context.Context, _, name, revision string) (egress.Policy, error) { a.mu.Lock() defer a.mu.Unlock() if a.attempts == nil { @@ -95,17 +95,17 @@ func (a *observingEgressActors) EnsureActorEgressPolicy(ctx context.Context, _, a.attempts[name] = append(a.attempts[name], time.Now()) if a.failures > 0 { a.failures-- - return errors.New("backend unavailable") + return egress.Policy{}, errors.New("backend unavailable") } policy, err := a.store.GetRuntimeEgressPolicy(ctx, revision) if err != nil { - return err + return egress.Policy{}, err } if a.applied == nil { a.applied = map[string]egress.Access{} } a.applied[name] = policy.Access - return nil + return policy, nil } func TestActorEgressSourcePagingAndStartupRecovery(t *testing.T) { diff --git a/go/core/internal/controller/grants_test.go b/go/core/internal/controller/grants_test.go index 48f87035dd..82a4c0e662 100644 --- a/go/core/internal/controller/grants_test.go +++ b/go/core/internal/controller/grants_test.go @@ -12,6 +12,7 @@ import ( "google.golang.org/protobuf/proto" "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" @@ -20,6 +21,42 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/substrate" ) +func TestAgentEgressErrorsUseCompilationClassification(t *testing.T) { + for _, test := range []struct { + name string + destinations []v1alpha3.EgressDestination + condition string + reason string + }{ + {name: "missing policy", condition: v1alpha3.AgentConditionResolvedRefs, reason: "ReferenceResolutionFailed"}, + {name: "invalid origin", destinations: []v1alpha3.EgressDestination{{Origin: "invalid"}}, condition: v1alpha3.AgentConditionCompatible, reason: "UnsupportedConfiguration"}, + {name: "missing secret", destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com", Authentication: &v1alpha3.EgressAuthentication{Header: "Authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "missing"}, Key: "token"}}}}, condition: v1alpha3.AgentConditionResolvedRefs, reason: "ReferenceResolutionFailed"}, + } { + t.Run(test.name, func(t *testing.T) { + collections, _ := newPreparationTestCollections(t, "gvisor") + if test.destinations != nil { + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(&v1alpha3.EgressPolicy{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, + Spec: v1alpha3.EgressPolicySpec{Destinations: test.destinations}, + }) + } + agent := collections.Agents.List()[0].DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + waitFor(t, func() bool { + observed := collections.AgentStatuses.GetKey("team-a/assistant") + return observed != nil && meta.IsStatusConditionFalse(observed.Status.Conditions, test.condition) + }) + condition := meta.FindStatusCondition(collections.AgentStatuses.GetKey("team-a/assistant").Status.Conditions, test.condition) + require.Equal(t, test.reason, condition.Reason) + state := collections.Reconciliations.GetKey("team-a/assistant") + require.Nil(t, state.Target) + require.Empty(t, state.Grants.Destinations) + require.Empty(t, state.Grants.Credentials) + }) + } +} + func TestAgentGrantChangesReconcileWithoutRuntimeChange(t *testing.T) { synctest.Test(t, func(t *testing.T) { collections, harnesses := newPreparationTestCollections(t, "gvisor") diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index c86adc7a1a..75f4bf20c4 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -400,10 +400,10 @@ func (p *preparationTestBackend) PromoteRuntimeRevision(ctx context.Context, rev return p.store.PromoteRuntimeRevision(ctx, revision) } -func (p *preparationTestBackend) RetireAgentIdentities(ctx context.Context, namespace, name string, except *database.AgentDefinition) error { +func (p *preparationTestBackend) RetireAgentIdentities(ctx context.Context, namespace, name string) error { p.mu.Lock() defer p.mu.Unlock() - return p.store.RetireAgentIdentities(ctx, namespace, name, except) + return p.store.RetireAgentIdentities(ctx, namespace, name) } func TestPreparationDesiredChangesBypassBackoff(t *testing.T) { diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index 0d2c86dab1..a6084cbc1b 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -112,7 +112,7 @@ func newAgentReconciliations( } grants, grantErr := translator.CompileEgressPolicies(ctx, collections, agent.Namespace, agent.Spec.EgressPolicyRefs) if grantErr != nil { - state.CompilationFailure = &ReconciliationFailure{Condition: kagentv1alpha3.AgentConditionResolvedRefs, Reason: "ReferenceResolutionFailed", Message: grantErr.Error()} + state.CompilationFailure = compilationFailure(grantErr) return state } state.Grants = grants @@ -123,19 +123,7 @@ func newAgentReconciliations( translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings), }, images).CompileAgent(context.Background(), agent) if err != nil { - condition, reason := kagentv1alpha3.AgentConditionResolvedRefs, "ReferenceResolutionFailed" - var validation *translator.ValidationError - var missingPool *translator.WorkerPoolNotFoundError - switch { - case errors.As(err, &validation): - // TODO(tim): Separate structural Agent validation from resolved - // compatibility failures so invalid sources do not report Accepted=True - // and ResolvedRefs=True. - condition, reason = kagentv1alpha3.AgentConditionCompatible, "UnsupportedConfiguration" - case errors.As(err, &missingPool): - reason = "WorkerPoolNotFound" - } - state.CompilationFailure = &ReconciliationFailure{Condition: condition, Reason: reason, Message: err.Error()} + state.CompilationFailure = compilationFailure(err) return state } revision := &compilation.Revision @@ -175,6 +163,22 @@ func newAgentReconciliations( }, opts.WithName("AgentReconciliations")...) } +func compilationFailure(err error) *ReconciliationFailure { + condition, reason := kagentv1alpha3.AgentConditionResolvedRefs, "ReferenceResolutionFailed" + var validation *translator.ValidationError + var missingPool *translator.WorkerPoolNotFoundError + switch { + case errors.As(err, &validation): + // TODO(tim): Separate structural Agent validation from resolved + // compatibility failures so invalid sources do not report Accepted=True + // and ResolvedRefs=True. + condition, reason = kagentv1alpha3.AgentConditionCompatible, "UnsupportedConfiguration" + case errors.As(err, &missingPool): + reason = "WorkerPoolNotFound" + } + return &ReconciliationFailure{Condition: condition, Reason: reason, Message: err.Error()} +} + // runtimeRevisionStore is the controller's narrow view of the shared database. // Substrate owns ActorTemplates; the database retains revisions while an Agent // or a Session or checkpoint references them. @@ -182,7 +186,7 @@ type runtimeRevisionStore interface { UpsertAgentDefinition(context.Context, database.AgentDefinition) (bool, error) RecordRuntimeRevision(context.Context, database.RuntimeRevision) error PromoteRuntimeRevision(context.Context, string) error - RetireAgentIdentities(ctx context.Context, namespace, name string, except *database.AgentDefinition) error + RetireAgentIdentities(ctx context.Context, namespace, name string) error } type actorTemplateClient interface { @@ -330,7 +334,7 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { if len(parts) != 2 { return fmt.Errorf("invalid Agent key %q", key) } - err := r.store.RetireAgentIdentities(ctx, parts[0], parts[1], nil) + err := r.store.RetireAgentIdentities(ctx, parts[0], parts[1]) r.enqueueActors(database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: parts[0], Name: parts[1]}) if err != nil { return fmt.Errorf("retire Agent %s: %w", key, err) diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index 310b5c680b..cf70506f0c 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -358,10 +358,8 @@ func (s *fakeRuntimeRevisionStore) PromoteRuntimeRevision(_ context.Context, rev return nil } -func (s *fakeRuntimeRevisionStore) RetireAgentIdentities(_ context.Context, namespace, template string, except *database.AgentDefinition) error { - if except == nil { - s.retired = namespace + "/" + template - } +func (s *fakeRuntimeRevisionStore) RetireAgentIdentities(_ context.Context, namespace, template string) error { + s.retired = namespace + "/" + template return nil } diff --git a/go/core/internal/database/client_session_test.go b/go/core/internal/database/client_session_test.go index 7353a33df6..67002c1806 100644 --- a/go/core/internal/database/client_session_test.go +++ b/go/core/internal/database/client_session_test.go @@ -955,7 +955,7 @@ func TestDeletedSessionPreservesRequestIdentityAndHidesAccess(t *testing.T) { _, _, err = client.CreateSession(ctx, request, "stable-request") require.ErrorIs(t, err, ErrIdempotencyConflict) // Tombstones release the revision FK and no longer block runtime cleanup. - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) require.Len(t, revisions, 1) diff --git a/go/core/internal/database/models.go b/go/core/internal/database/models.go index 7bfafc8aea..2aa500367c 100644 --- a/go/core/internal/database/models.go +++ b/go/core/internal/database/models.go @@ -51,10 +51,10 @@ type AgentMemorySearchResult struct { // AgentDefinition tracks the desired and last successful runtime for an Agent UID. type AgentDefinition struct { - Grants egress.Access Namespace string AgentName string AgentUID string + Grants egress.Access // DesiredRevision is a compiled runtime digest, or empty while inputs are unresolved. DesiredRevision string } diff --git a/go/core/internal/database/runtime_egress.go b/go/core/internal/database/runtime_egress.go index 7e1fc15bcf..f656f4a71b 100644 --- a/go/core/internal/database/runtime_egress.go +++ b/go/core/internal/database/runtime_egress.go @@ -3,12 +3,19 @@ package database import ( "context" "fmt" + "time" "github.com/jackc/pgx/v5" "github.com/kagent-dev/kagent/go/core/internal/egress" ) +type definitionGrantRow struct { + GrantDestinations []string + GrantCredentials []egress.Credential + RetiredAt *time.Time +} + // canonicalStoredGrants validates and normalizes source grants for NOT NULL storage. func canonicalStoredGrants(grants egress.Access) (egress.Access, error) { canonical, err := egress.CanonicalAccess(grants.Destinations, grants.Credentials) @@ -51,11 +58,14 @@ func (c *Client) GetRuntimeEgressPolicy(ctx context.Context, revision string) (e if err != nil { return egress.Policy{}, fmt.Errorf("get runtime egress policy %s: %w", revision, notFoundOr(err)) } - tokens, err := egress.CanonicalAccess(row.TokenPassthroughOrigins, nil) - if err != nil { - return egress.Policy{}, fmt.Errorf("decode runtime token origins %s: %w", revision, err) + for i, raw := range row.TokenPassthroughOrigins { + origin, err := egress.ParseOrigin(raw) + if err != nil { + return egress.Policy{}, fmt.Errorf("decode runtime token origins %s: %w", revision, err) + } + row.TokenPassthroughOrigins[i] = origin.String() } - base, err := egress.MergeAccess(egress.Access{Destinations: row.EgressDestinations, Credentials: row.Credentials}, egress.Access{}, tokens.Destinations) + base, err := egress.MergeAccess(egress.Access{Destinations: row.EgressDestinations, Credentials: row.Credentials}, egress.Access{}, row.TokenPassthroughOrigins) if err != nil { return egress.Policy{}, fmt.Errorf("decode runtime egress base %s: %w", revision, err) } @@ -63,7 +73,7 @@ func (c *Client) GetRuntimeEgressPolicy(ctx context.Context, revision string) (e if err != nil { return egress.Policy{}, fmt.Errorf("decode runtime egress grants %s: %w", revision, err) } - access, withheld := egress.EffectiveAccess(base, grants, tokens.Destinations) + access, withheld := egress.EffectiveAccess(base, grants, row.TokenPassthroughOrigins) return egress.Policy{Access: access, WithheldGrants: withheld}, nil } @@ -99,14 +109,10 @@ type RuntimeEgressTarget struct { // creation reads effective policy itself and rechecks it after applying. func (c *Client) ListRuntimeEgressTargets(ctx context.Context, afterID string, limit int) ([]RuntimeEgressTarget, error) { return queryMany(ctx, c.db, ` - SELECT i.id::text AS id, i.prepared_revision AS revision, r.namespace, - r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, COALESCE(s.actor_uid, '') AS actor_uid - FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision - LEFT JOIN session s ON s.id = i.id - WHERE i.id > COALESCE(NULLIF($1, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) - AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) - AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' - ORDER BY i.id LIMIT $2 + SELECT target.id::text AS id, revision, namespace, atespace, template_name, actor_uid + FROM runtime_egress_target target + WHERE target.id > COALESCE(NULLIF($1, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY target.id LIMIT $2 `, pgx.RowToStructByName[RuntimeEgressTarget], afterID, limit) } @@ -118,32 +124,15 @@ func (c *Client) ListRuntimeEgressSourceTargets(ctx context.Context, source Runt return nil, fmt.Errorf("runtime egress source requires namespace and name") } switch source.Kind { - case RuntimeEgressAgent: - return queryMany(ctx, c.db, ` - SELECT i.id::text AS id, i.prepared_revision AS revision, r.namespace, - r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, COALESCE(s.actor_uid, '') AS actor_uid - FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision - JOIN agent_revision a ON a.revision = r.revision - LEFT JOIN session s ON s.id = i.id - WHERE r.namespace = $1 AND a.agent_name = $2 - AND i.id > COALESCE(NULLIF($3, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) - AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) - AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' - ORDER BY i.id LIMIT $4 - `, pgx.RowToStructByName[RuntimeEgressTarget], source.Namespace, source.Name, afterID, limit) - case RuntimeEgressSandbox: - return queryMany(ctx, c.db, ` - SELECT i.id::text AS id, i.prepared_revision AS revision, r.namespace, - r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, '' AS actor_uid - FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision - JOIN sandbox_revision s ON s.revision = r.revision - WHERE r.namespace = $1 AND s.sandbox_template_name = $2 - AND i.id > COALESCE(NULLIF($3, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) - AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) - AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' - ORDER BY i.id LIMIT $4 - `, pgx.RowToStructByName[RuntimeEgressTarget], source.Namespace, source.Name, afterID, limit) + case RuntimeEgressAgent, RuntimeEgressSandbox: default: return nil, fmt.Errorf("unknown runtime egress source kind %q", source.Kind) } + return queryMany(ctx, c.db, ` + SELECT target.id::text AS id, revision, namespace, atespace, template_name, actor_uid + FROM runtime_egress_target target + WHERE source_kind = $1 AND namespace = $2 AND source_name = $3 + AND target.id > COALESCE(NULLIF($4, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY target.id LIMIT $5 + `, pgx.RowToStructByName[RuntimeEgressTarget], source.Kind, source.Namespace, source.Name, afterID, limit) } diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index 15bd726f9c..ac071bdb4d 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -1,6 +1,7 @@ package database import ( + "fmt" "slices" "testing" @@ -15,12 +16,12 @@ import ( func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() - revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", TokenPassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} - _, upsertErr := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "old-uid", DesiredRevision: "agent"}) + revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "scratch", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", TokenPassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} + _, upsertErr := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "scratch", AgentUID: "old-uid", DesiredRevision: "agent"}) require.NoError(t, upsertErr) require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) - instance, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "request") + instance, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "scratch", "kagent", ""), "request") require.NoError(t, err) op, err := client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) require.NoError(t, err) @@ -39,7 +40,7 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { _, err = client.FinishSessionOperation(ctx, instance.Id, op.ID, executor, "", "", "") require.NoError(t, err) // A recreated Agent must not transfer authorization to an old pinned session. - _, upsertErr = client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "new-uid", DesiredRevision: "pending"}) + _, upsertErr = client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "scratch", AgentUID: "new-uid", DesiredRevision: "pending"}) require.NoError(t, upsertErr) sandbox, options := sandboxFixture(t, client, "sandbox") created, _, err := client.CreateSandbox(ctx, sandbox, "sandbox", options) @@ -49,13 +50,15 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { claimed, err = client.ClaimSandboxOperation(ctx, created.Id, sandboxOp.ID, uuid.New()) require.NoError(t, err) require.True(t, claimed) + var sourceTargets []RuntimeEgressTarget + // Both kinds deliberately share a name; source filtering must preserve kind. for _, test := range []struct { source RuntimeEgressSource want string }{ - {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, instance.Id}, + {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "scratch"}, instance.Id}, {RuntimeEgressSource{Kind: RuntimeEgressSandbox, Namespace: "team-a", Name: "scratch"}, created.Id}, - {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "other", Name: "assistant"}, ""}, + {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "other", Name: "scratch"}, ""}, {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "unrelated"}, ""}, } { targets, err := client.ListRuntimeEgressSourceTargets(ctx, test.source, "", 10) @@ -65,10 +68,16 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { } else { require.Len(t, targets, 1) require.Equal(t, test.want, targets[0].ID) + if test.source.Kind == RuntimeEgressAgent { + require.Equal(t, "actor-uid", targets[0].ActorUID) + } else { + require.Empty(t, targets[0].ActorUID) + } + sourceTargets = append(sourceTargets, targets...) } } for _, kind := range []RuntimeEgressSourceKind{"", "typo"} { - _, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: kind, Namespace: "team-a", Name: "assistant"}, "", 10) + _, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: kind, Namespace: "team-a", Name: "scratch"}, "", 10) require.Error(t, err) } first, err := client.ListRuntimeEgressTargets(ctx, "", 1) @@ -78,16 +87,65 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { require.NoError(t, err) require.Len(t, second, 1) require.ElementsMatch(t, []string{instance.Id, created.Id}, []string{first[0].ID, second[0].ID}) + require.ElementsMatch(t, sourceTargets, append(first, second...), "source and recovery queries must return the same actor bindings") last, err := client.ListRuntimeEgressTargets(ctx, second[0].ID, 1) require.NoError(t, err) require.Empty(t, last) _, err = client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE) require.NoError(t, err) - remaining, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, "", 10) + remaining, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "scratch"}, "", 10) require.NoError(t, err) require.Empty(t, remaining) } +func TestRuntimeEgressTokenOriginsAreNotGrants(t *testing.T) { + many := make([]string, 257) + for i := range many { + many[i] = fmt.Sprintf("https://model-%d.example.com", i) + } + for _, test := range []struct { + name string + origins []string + }{ + {"more origins than policy rules", many}, + {"different protocols on the same port", []string{"http://model.example.com:443", "https://model.example.com:443"}}, + } { + t.Run(test.name, func(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{ + Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", + SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", + TokenPassthroughOrigins: test.origins, + })) + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + require.Empty(t, policy.Destinations, "token origins constrain credentials without granting network access") + require.NoError(t, policy.WithheldGrants) + }) + } +} + +func TestRuntimeEgressCanonicalTokenOriginWithholdsCredentials(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + _, err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", Grants: egress.Access{ + Destinations: []string{"https://model.example.com"}, + Credentials: []egress.Credential{{Origin: "https://model.example.com", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"}}, + }}) + require.NoError(t, err) + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{ + Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", + SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", + TokenPassthroughOrigins: []string{"https://MODEL.example.com."}, + })) + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + require.ErrorContains(t, policy.WithheldGrants, "caller-token passthrough") + require.Empty(t, policy.Destinations) + require.Empty(t, policy.Credentials) +} + func TestSourceGrantsDetermineEffectivePolicy(t *testing.T) { for _, kind := range []RuntimeEgressSourceKind{RuntimeEgressAgent, RuntimeEgressSandbox} { t.Run(string(kind), func(t *testing.T) { @@ -145,7 +203,7 @@ func TestSourceGrantsDetermineEffectivePolicy(t *testing.T) { require.Equal(t, base, policy.Access) } if kind == RuntimeEgressAgent { - require.NoError(t, client.RetireAgentIdentities(ctx, "team", "source", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team", "source")) } else { require.NoError(t, client.RetireSandboxTemplateIdentities(ctx, "team", "source")) } diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index 734b721540..8dc73e960b 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "time" a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" @@ -27,36 +28,32 @@ func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefi } changed := false err = c.withTx(ctx, func(tx pgx.Tx) error { - retired, err := tx.Exec(ctx, ` - UPDATE agent_definition SET retired_at = NOW(), updated_at = NOW() - WHERE namespace = $1 AND agent_name = $2 AND agent_uid <> $3 AND retired_at IS NULL - `, definition.Namespace, definition.AgentName, definition.AgentUID) + retired, err := retireAgentIdentities(ctx, tx, definition.Namespace, definition.AgentName, definition.AgentUID) if err != nil { return err } - changed = retired.RowsAffected() > 0 - grantsChanged, err := queryOne(ctx, tx, ` - WITH previous AS MATERIALIZED ( - SELECT grant_destinations, grant_credentials, retired_at FROM agent_definition - WHERE namespace = $1 AND agent_uid = $3 FOR UPDATE - ), updated AS ( - INSERT INTO agent_definition (namespace, agent_name, agent_uid, desired_revision, grant_destinations, grant_credentials) - SELECT $1, $2, $3, $4, $5, $6 FROM (SELECT COUNT(*) FROM previous) locked - ON CONFLICT (namespace, agent_uid) DO UPDATE SET + previous, err := queryOne(ctx, tx, ` + SELECT grant_destinations, grant_credentials, retired_at FROM agent_definition + WHERE namespace = $1 AND agent_uid = $2 FOR UPDATE + `, pgx.RowToStructByName[definitionGrantRow], definition.Namespace, definition.AgentUID) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return err + } + if previous.RetiredAt != nil { + previous = definitionGrantRow{} + } + if err := execSQL(ctx, tx, ` + INSERT INTO agent_definition (namespace, agent_name, agent_uid, desired_revision, grant_destinations, grant_credentials) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (namespace, agent_uid) DO UPDATE SET agent_name = EXCLUDED.agent_name, desired_revision = EXCLUDED.desired_revision, grant_destinations = EXCLUDED.grant_destinations, grant_credentials = EXCLUDED.grant_credentials, retired_at = NULL, updated_at = NOW() - RETURNING namespace - ) - SELECT COALESCE((SELECT grant_destinations FROM previous WHERE retired_at IS NULL), '{}'::text[]) IS DISTINCT FROM $5 - OR COALESCE((SELECT grant_credentials FROM previous WHERE retired_at IS NULL), '[]'::jsonb) IS DISTINCT FROM $6 - FROM updated - `, pgx.RowTo[bool], definition.Namespace, definition.AgentName, definition.AgentUID, definition.DesiredRevision, grants.Destinations, grants.Credentials) - if err != nil { + `, definition.Namespace, definition.AgentName, definition.AgentUID, definition.DesiredRevision, grants.Destinations, grants.Credentials); err != nil { return err } + changed = retired > 0 || !slices.Equal(previous.GrantDestinations, grants.Destinations) || !slices.Equal(previous.GrantCredentials, grants.Credentials) - changed = changed || grantsChanged // Lock Agents before revisions. Validate the resulting definition, including // its retained last-good pointer when reactivating a retired identity. type revisionStatus struct { @@ -184,26 +181,25 @@ func toRuntimeRevision(row runtimeRevisionRow) (*RuntimeRevision, error) { }, nil } -// RetireAgentIdentities retires identities at the given namespace/Agent -// names, except the supplied UID when non-nil. Existing sessions retain -// their pinned revisions. Missing and already-retired identities are a no-op. -func (c *Client) RetireAgentIdentities(ctx context.Context, namespace, name string, except *AgentDefinition) error { - return retireAgentIdentities(ctx, c.db, namespace, name, except) +// RetireAgentIdentities retires identities at the given namespace/Agent name. +// Existing sessions retain their pinned revisions. Missing and already-retired +// identities are a no-op. +func (c *Client) RetireAgentIdentities(ctx context.Context, namespace, name string) error { + _, err := retireAgentIdentities(ctx, c.db, namespace, name, "") + return err } // retireAgentIdentities uses the caller's executor so retirement can commit -// atomically with definition preparation. A nil exception retires every matching identity. -func retireAgentIdentities(ctx context.Context, db dbExecutor, namespace, name string, except *AgentDefinition) error { - var uid *string - if except != nil { - uid = &except.AgentUID - } - return execSQL(ctx, db, ` +// atomically with definition preparation and returns the number retired. An empty +// retainedUID retires every active identity at this source name. +func retireAgentIdentities(ctx context.Context, db dbExecutor, namespace, name, retainedUID string) (int64, error) { + result, err := db.Exec(ctx, ` UPDATE agent_definition SET retired_at = NOW(), updated_at = NOW() WHERE namespace = $1 AND agent_name = $2 - AND retired_at IS NULL AND agent_uid IS DISTINCT FROM $3::text - `, namespace, name, uid) + AND retired_at IS NULL AND agent_uid IS DISTINCT FROM NULLIF($3, '') + `, namespace, name, retainedUID) + return result.RowsAffected(), err } // ListUnreferencedRuntimeRevisions lists revisions unused by active Agents, diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index aafec031ad..24ce854ead 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -20,38 +20,19 @@ import ( ) func TestRetireAgentIdentities(t *testing.T) { - for _, test := range []struct { - name string - except *AgentDefinition - keep bool - }{ - {name: "all"}, - {name: "except current", except: &AgentDefinition{AgentUID: "assistant-uid"}, keep: true}, - {name: "replacement template", except: &AgentDefinition{AgentUID: "replacement-uid"}}, - } { - t.Run(test.name, func(t *testing.T) { - client := NewClient(setupTestDB(t), "public") - ctx := t.Context() - sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") - sessionFixture(t, client, ctx, "team-b", "other-namespace", "assistant", "kagent") - sessionFixture(t, client, ctx, "team-a", "other-template", "other", "kagent") - sessionFixture(t, client, ctx, "team-a", "other-agent", "another", "other") - for range 2 { - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", test.except)) - } - revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) - require.NoError(t, err) - if test.keep { - require.Empty(t, revisions) - session, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "session") - require.NoError(t, err) - require.Equal(t, "revision", session.GetPreparedRevision(), "the exception retains its last-good revision") - } else { - require.Len(t, revisions, 1, "retirement must stay within the requested namespace and names") - require.Equal(t, "revision", revisions[0].Revision) - } - }) + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") + sessionFixture(t, client, ctx, "team-b", "other-namespace", "assistant", "kagent") + sessionFixture(t, client, ctx, "team-a", "other-template", "other", "kagent") + sessionFixture(t, client, ctx, "team-a", "other-agent", "another", "other") + for range 2 { + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) } + revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) + require.NoError(t, err) + require.Len(t, revisions, 1, "retirement must stay within the requested namespace and name") + require.Equal(t, "revision", revisions[0].Revision) } func TestRuntimeRevisionCollectionAfterPairRetirement(t *testing.T) { @@ -67,7 +48,7 @@ func TestRuntimeRevisionCollectionAfterPairRetirement(t *testing.T) { _, err = client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) - err = client.RetireAgentIdentities(ctx, "team-a", "assistant", nil) + err = client.RetireAgentIdentities(ctx, "team-a", "assistant") require.NoError(t, err) revisions, err = client.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) @@ -90,7 +71,7 @@ func TestRuntimeRevisionCollectionPreservesSessionAndCheckpoint(t *testing.T) { require.NoError(t, err) _, err = markSessionReady(ctx, client, session.GetId(), "runtime") require.NoError(t, err) - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) assertRetained := func() { t.Helper() @@ -229,7 +210,7 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) query := "FROM agent_definition p" if source != "session" { query = "FOR UPDATE OF r" - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) } barrier := &runtimeReferenceBarrier{query: query, afterQuery: referenceFirst, reached: make(chan struct{}), resume: make(chan struct{})} var resume sync.Once @@ -269,13 +250,13 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) // so retirement must follow creation before GC can claim it. resume.Do(func() { close(barrier.resume) }) require.NoError(t, <-created) - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "revision") require.NoError(t, err) require.Nil(t, claimed) return } - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) } type claimResult struct { revision *RuntimeArtifact @@ -324,7 +305,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", DesiredRevision: "revision", } - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) // A skipped finalization must leave last-good intact for reactivation. require.NoError(t, client.DeleteRuntimeRevision(ctx, "revision", "revision-actor-uid")) _, upsertErr := client.UpsertAgentDefinition(ctx, pair) @@ -333,7 +314,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.NoError(t, err) require.Equal(t, "revision", session.GetPreparedRevision()) require.NoError(t, deleteSession(ctx, client, session.GetId())) - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) original, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "revision") @@ -386,7 +367,7 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "revision") require.NoError(t, err) require.NotNil(t, claimed) @@ -400,7 +381,7 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { waitingQuery := "ORDER BY namespace, agent_uid" if finalizeFirst { barrier.query, barrier.afterQuery = "FROM runtime_revision WHERE revision = $1 FOR UPDATE", true - waitingQuery = "INSERT INTO agent_definition" + waitingQuery = "WHERE namespace = $1 AND agent_uid = $2 FOR UPDATE" if operation == "promote" { waitingQuery = "FOR UPDATE OF p" } @@ -520,7 +501,7 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { pair.DesiredRevision = "third" _, upsertErr = c.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) - require.NoError(t, c.RetireAgentIdentities(ctx, pair.Namespace, pair.AgentName, nil)) + require.NoError(t, c.RetireAgentIdentities(ctx, pair.Namespace, pair.AgentName)) revision.Revision, revision.ActorTemplateName = "third", "third" require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index da8f9c08fa..a51e2e99ce 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -3,7 +3,9 @@ package database import ( "context" "encoding/json" + "errors" "fmt" + "slices" "github.com/jackc/pgx/v5" @@ -19,10 +21,10 @@ type SandboxRevision struct { // SandboxTemplateDefinition tracks the desired and last successful runtime for a SandboxTemplate UID. type SandboxTemplateDefinition struct { - Grants egress.Access Namespace string SandboxTemplateName string SandboxTemplateUID string + Grants egress.Access DesiredRevision string } @@ -37,36 +39,32 @@ func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition } changed := false err = c.withTx(ctx, func(tx pgx.Tx) error { - retired, err := tx.Exec(ctx, ` - UPDATE sandbox_template_definition SET retired_at = NOW(), updated_at = NOW() - WHERE namespace = $1 AND sandbox_template_name = $2 AND sandbox_template_uid <> $3 AND retired_at IS NULL - `, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID) + retired, err := retireSandboxTemplateIdentities(ctx, tx, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID) if err != nil { return err } - changed = retired.RowsAffected() > 0 - grantsChanged, err := queryOne(ctx, tx, ` - WITH previous AS MATERIALIZED ( - SELECT grant_destinations, grant_credentials, retired_at FROM sandbox_template_definition - WHERE namespace = $1 AND sandbox_template_uid = $3 FOR UPDATE - ), updated AS ( - INSERT INTO sandbox_template_definition (namespace, sandbox_template_name, sandbox_template_uid, desired_revision, grant_destinations, grant_credentials) - SELECT $1, $2, $3, $4, $5, $6 FROM (SELECT COUNT(*) FROM previous) locked - ON CONFLICT (namespace, sandbox_template_uid) DO UPDATE SET + previous, err := queryOne(ctx, tx, ` + SELECT grant_destinations, grant_credentials, retired_at FROM sandbox_template_definition + WHERE namespace = $1 AND sandbox_template_uid = $2 FOR UPDATE + `, pgx.RowToStructByName[definitionGrantRow], definition.Namespace, definition.SandboxTemplateUID) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return err + } + if previous.RetiredAt != nil { + previous = definitionGrantRow{} + } + if err := execSQL(ctx, tx, ` + INSERT INTO sandbox_template_definition (namespace, sandbox_template_name, sandbox_template_uid, desired_revision, grant_destinations, grant_credentials) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (namespace, sandbox_template_uid) DO UPDATE SET sandbox_template_name = EXCLUDED.sandbox_template_name, desired_revision = EXCLUDED.desired_revision, grant_destinations = EXCLUDED.grant_destinations, grant_credentials = EXCLUDED.grant_credentials, retired_at = NULL, updated_at = NOW() - RETURNING namespace - ) - SELECT COALESCE((SELECT grant_destinations FROM previous WHERE retired_at IS NULL), '{}'::text[]) IS DISTINCT FROM $5 - OR COALESCE((SELECT grant_credentials FROM previous WHERE retired_at IS NULL), '[]'::jsonb) IS DISTINCT FROM $6 - FROM updated - `, pgx.RowTo[bool], definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID, definition.DesiredRevision, grants.Destinations, grants.Credentials) - if err != nil { + `, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID, definition.DesiredRevision, grants.Destinations, grants.Credentials); err != nil { return err } + changed = retired > 0 || !slices.Equal(previous.GrantDestinations, grants.Destinations) || !slices.Equal(previous.GrantCredentials, grants.Credentials) - changed = changed || grantsChanged revisions, err := queryMany(ctx, tx, ` SELECT r.revision, r.kind, r.namespace, r.actor_template_atespace, r.actor_template_name, r.actor_template_uid, r.deleted_at FROM runtime_revision r JOIN sandbox_template_definition p @@ -92,10 +90,19 @@ func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition // RetireSandboxTemplateIdentities retires active identities at this source name. // Missing/already-retired identities are a no-op; retained actors lose live grants. func (c *Client) RetireSandboxTemplateIdentities(ctx context.Context, namespace, name string) error { - return execSQL(ctx, c.db, ` + _, err := retireSandboxTemplateIdentities(ctx, c.db, namespace, name, "") + return err +} + +// retireSandboxTemplateIdentities shares retirement with definition preparation +// and returns the number retired. An empty retainedUID retires all active identities. +func retireSandboxTemplateIdentities(ctx context.Context, db dbExecutor, namespace, name, retainedUID string) (int64, error) { + result, err := db.Exec(ctx, ` UPDATE sandbox_template_definition SET retired_at = NOW(), updated_at = NOW() WHERE namespace = $1 AND sandbox_template_name = $2 AND retired_at IS NULL - `, namespace, name) + AND sandbox_template_uid IS DISTINCT FROM NULLIF($3, '') + `, namespace, name, retainedUID) + return result.RowsAffected(), err } // RecordSandboxRevision stores immutable sandbox inputs and refreshes artifact diff --git a/go/core/internal/database/scheduled_runs_test.go b/go/core/internal/database/scheduled_runs_test.go index 80aa0e82a9..ca835dc53d 100644 --- a/go/core/internal/database/scheduled_runs_test.go +++ b/go/core/internal/database/scheduled_runs_test.go @@ -393,7 +393,7 @@ func TestScheduledExecutionWaitsForPreparedRevision(t *testing.T) { db := setupTestDB(t) c := NewClient(db, "public") schedule, _ := createTestSchedule(t, c) - require.NoError(t, c.RetireAgentIdentities(t.Context(), "team-a", "report", nil)) + require.NoError(t, c.RetireAgentIdentities(t.Context(), "team-a", "report")) execution, err := c.TriggerScheduledRun(t.Context(), uuid.MustParse(schedule.Id), "alice", "manual") require.NoError(t, err) _, err = c.ReserveScheduledRunExecutionSession(t.Context(), uuid.MustParse(execution.Id), "alice") diff --git a/go/core/internal/service/sandbox/guest_test.go b/go/core/internal/service/sandbox/guest_test.go index b5f153971d..6ed073f772 100644 --- a/go/core/internal/service/sandbox/guest_test.go +++ b/go/core/internal/service/sandbox/guest_test.go @@ -28,6 +28,7 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" sandboxservice "github.com/kagent-dev/kagent/go/core/internal/service/sandbox" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/pkg/auth" @@ -82,11 +83,11 @@ func (a *testActors) CreateActor(_ context.Context, space, name, templateSpace, Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}} return proto.CloneOf(a.actor), a.mutationErr } -func (a *testActors) EnsureActorEgressPolicy(context.Context, string, string, string) error { +func (a *testActors) EnsureActorEgressPolicy(context.Context, string, string, string) (egress.Policy, error) { a.mu.Lock() defer a.mu.Unlock() a.calls = append(a.calls, "policy") - return a.mutationErr + return egress.Policy{}, a.mutationErr } func (a *testActors) ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) { a.mu.Lock() diff --git a/go/core/internal/service/sandbox/service_test.go b/go/core/internal/service/sandbox/service_test.go index 54cfb92735..abcae41b2b 100644 --- a/go/core/internal/service/sandbox/service_test.go +++ b/go/core/internal/service/sandbox/service_test.go @@ -22,6 +22,7 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/pkg/auth" ) @@ -73,12 +74,12 @@ func (a *testActors) CreateActor(_ context.Context, space, name, templateSpace, Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}} return proto.CloneOf(a.actor), a.mutationErr } -func (a *testActors) EnsureActorEgressPolicy(_ context.Context, _, _ string, revision string) error { +func (a *testActors) EnsureActorEgressPolicy(_ context.Context, _, _ string, revision string) (egress.Policy, error) { a.mu.Lock() defer a.mu.Unlock() a.calls = append(a.calls, "policy") a.egressRevision = revision - return a.mutationErr + return egress.Policy{}, a.mutationErr } func (a *testActors) ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) { a.mu.Lock() diff --git a/go/core/internal/service/session/workflow_test.go b/go/core/internal/service/session/workflow_test.go index d5719949e4..6b27fcc9f6 100644 --- a/go/core/internal/service/session/workflow_test.go +++ b/go/core/internal/service/session/workflow_test.go @@ -286,13 +286,13 @@ func lifecycleForkFixture(t *testing.T, store *lifecycleTestStore, actors *lifec return fork, checkpoint.Id } -func (a *lifecycleTestActors) EnsureActorEgressPolicy(_ context.Context, atespace, name string, revision string) error { +func (a *lifecycleTestActors) EnsureActorEgressPolicy(_ context.Context, atespace, name string, revision string) (egress.Policy, error) { a.mu.Lock() defer a.mu.Unlock() a.policyActor = actorKey(atespace, name) a.policyRevision = revision a.policyCalls++ - return a.policyErr + return egress.Policy{}, a.policyErr } func TestActorCreationRetainsEgressPolicyFailure(t *testing.T) { diff --git a/go/core/internal/substrate/egress.go b/go/core/internal/substrate/egress.go index 27417b0d35..92232df0ca 100644 --- a/go/core/internal/substrate/egress.go +++ b/go/core/internal/substrate/egress.go @@ -9,32 +9,35 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + + "github.com/kagent-dev/kagent/go/core/internal/egress" ) // EnsureActorEgressPolicy installs current authorization without restarting the -// actor. Read the backend version before the controller's persisted intended +// actor and returns the effective policy it applied, including withheld-grant +// diagnostics. Read the backend version before the controller's persisted intended // policy so concurrent applications cannot overwrite a newer policy with an old // one. Conflicts reload both; lifecycle paths never translate Kubernetes. // Substrate deletes the policy with its Actor. -func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name, revision string) error { +func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name, revision string) (egress.Policy, error) { ctx, cancel := c.callCtx(ctx) defer cancel() if c.cfg.EgressPolicies == nil { - return fmt.Errorf("actor egress policy store is required") + return egress.Policy{}, fmt.Errorf("actor egress policy store is required") } actor := actorRef(atespace, name) for range 5 { existing, err := c.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) if err != nil && status.Code(err) != codes.NotFound { - return err + return egress.Policy{}, err } access, err := c.cfg.EgressPolicies.GetRuntimeEgressPolicy(ctx, revision) if err != nil { - return fmt.Errorf("load intended actor egress: %w", err) + return egress.Policy{}, fmt.Errorf("load intended actor egress: %w", err) } policy, err := ActorEgressPolicy(atespace, access.Destinations, access.Credentials) if err != nil { - return err + return egress.Policy{}, err } if existing == nil { _, err = c.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{Actor: actor, EgressPolicy: policy}) @@ -48,17 +51,17 @@ func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name, re continue } if err != nil { - return err + return egress.Policy{}, err } // A source update may have discovered this instance before its Actor // existed. Recheck after applying so creation cannot miss that update. latest, err := c.cfg.EgressPolicies.GetRuntimeEgressPolicy(ctx, revision) if err != nil { - return fmt.Errorf("reload intended actor egress: %w", err) + return egress.Policy{}, fmt.Errorf("reload intended actor egress: %w", err) } if slices.Equal(access.Destinations, latest.Destinations) && slices.Equal(access.Credentials, latest.Credentials) { - return nil + return latest, nil } } - return status.Error(codes.Aborted, "actor egress policy changed during reconciliation") + return egress.Policy{}, status.Error(codes.Aborted, "actor egress policy changed during reconciliation") } diff --git a/go/core/internal/substrate/egress_test.go b/go/core/internal/substrate/egress_test.go index 14a61e9fed..663b7766c7 100644 --- a/go/core/internal/substrate/egress_test.go +++ b/go/core/internal/substrate/egress_test.go @@ -30,7 +30,10 @@ func TestEnsureActorEgressPolicyUpdatesCurrentGrants(t *testing.T) { require.Equal(t, "pinned-revision", revision) return egress.Policy{Access: access}, nil })}} - reconcile := func() error { return client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "pinned-revision") } + reconcile := func() error { + _, err := client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "pinned-revision") + return err + } require.ErrorIs(t, reconcile(), context.DeadlineExceeded) require.NotNil(t, fake.policy, "the server committed before the response was lost") fake.createErr = nil @@ -58,7 +61,10 @@ func TestEnsureActorEgressPolicyReloadsAfterConflict(t *testing.T) { fake := &egressPolicyFake{} access := egress.Access{} client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Policy, error) { return egress.Policy{Access: access}, nil })}} - reconcile := func() error { return client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "revision") } + reconcile := func() error { + _, err := client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "revision") + return err + } require.NoError(t, reconcile()) access.Destinations = []string{"https://revoked.example.com:443"} fake.beforeUpdate = func() { @@ -131,7 +137,9 @@ func TestEnsureActorEgressPolicyCatchesUpdateDuringCreation(t *testing.T) { // creation. There is no concurrent Substrate write to cause a CAS conflict. access = egress.Access{} } - require.NoError(t, client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision")) + applied, err := client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision") + require.NoError(t, err) + require.Equal(t, access, applied.Access, "the returned policy must reflect the post-creation revocation") require.Empty(t, fake.policy.Rules) require.Equal(t, 1, fake.created) require.Equal(t, 1, fake.updated) @@ -141,7 +149,8 @@ func TestEnsureActorEgressPolicyRequiresPersistedPolicy(t *testing.T) { fake := &egressPolicyFake{} unavailable := errors.New("intended policy unavailable") client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Policy, error) { return egress.Policy{}, unavailable })}} - require.ErrorIs(t, client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision"), unavailable) + _, err := client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision") + require.ErrorIs(t, err, unavailable) require.Zero(t, fake.created) require.Zero(t, fake.updated) } @@ -149,10 +158,17 @@ func TestEnsureActorEgressPolicyRequiresPersistedPolicy(t *testing.T) { func TestEnsureActorEgressPolicyAppliesFallbackDespiteWithheldGrants(t *testing.T) { base := egress.Access{Destinations: []string{"https://model.example.com:443"}} fake := &egressPolicyFake{} + conflict := errors.New("credential conflict") + reads := 0 client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Policy, error) { - return egress.Policy{Access: base, WithheldGrants: errors.New("credential conflict")}, nil + reads++ + return egress.Policy{Access: base, WithheldGrants: conflict}, nil })}} - require.NoError(t, client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision")) + applied, err := client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision") + require.NoError(t, err) + require.Equal(t, base, applied.Access) + require.ErrorIs(t, applied.WithheldGrants, conflict) + require.Equal(t, 2, reads, "diagnostics must come from the existing apply and recheck reads") require.Len(t, fake.policy.Rules, 1) require.Equal(t, []string{"model.example.com"}, fake.policy.Rules[0].GetHttps().GetHostnames()) } diff --git a/go/core/internal/substrate/lifecycle.go b/go/core/internal/substrate/lifecycle.go index f42c40a7d9..cb8f8800ca 100644 --- a/go/core/internal/substrate/lifecycle.go +++ b/go/core/internal/substrate/lifecycle.go @@ -10,13 +10,14 @@ import ( "google.golang.org/protobuf/proto" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) type LifecycleClient interface { GetActor(context.Context, string, string) (*ateapipb.Actor, error) CreateActor(context.Context, string, string, string, string) (*ateapipb.Actor, error) CreateActorFromTag(context.Context, string, string, string, string, string, string) (*ateapipb.Actor, error) - EnsureActorEgressPolicy(context.Context, string, string, string) error + EnsureActorEgressPolicy(context.Context, string, string, string) (egress.Policy, error) ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) SuspendActor(context.Context, string, string) (*ateapipb.Actor, error) DeleteActor(context.Context, string, string) error @@ -144,7 +145,7 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio } } if err == nil { - err = actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, binding.Revision) + _, err = actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, binding.Revision) } if err == nil && creation.Resume && actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_RUNNING { actor, err = actors.ResumeActor(ctx, binding.Atespace, binding.Name) @@ -153,7 +154,7 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio } } case apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME: - if err := actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, binding.Revision); err != nil { + if _, err := actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, binding.Revision); err != nil { return err } actor, err = actors.ResumeActor(ctx, binding.Atespace, binding.Name) diff --git a/go/core/internal/substrate/lifecycle_test.go b/go/core/internal/substrate/lifecycle_test.go index a4e1301b25..a3a620d24b 100644 --- a/go/core/internal/substrate/lifecycle_test.go +++ b/go/core/internal/substrate/lifecycle_test.go @@ -12,6 +12,7 @@ import ( "google.golang.org/protobuf/proto" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) // Unexpected calls use the embedded nil interface and fail the test. @@ -41,9 +42,9 @@ func (a *transitionActors) CreateActor(_ context.Context, space, name, templateS return proto.CloneOf(a.actor), nil } -func (a *transitionActors) EnsureActorEgressPolicy(context.Context, string, string, string) error { +func (a *transitionActors) EnsureActorEgressPolicy(context.Context, string, string, string) (egress.Policy, error) { a.calls = append(a.calls, "policy") - return a.policyErr + return egress.Policy{}, a.policyErr } func (a *transitionActors) ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) { diff --git a/go/core/pkg/migrations/core/000001_initial.sql b/go/core/pkg/migrations/core/000001_initial.sql index 44702c7736..6beaa61641 100644 --- a/go/core/pkg/migrations/core/000001_initial.sql +++ b/go/core/pkg/migrations/core/000001_initial.sql @@ -64,11 +64,11 @@ CREATE TABLE sandbox_revision ( CREATE INDEX sandbox_revision_source_idx ON sandbox_revision (sandbox_template_name, revision); CREATE TABLE agent_definition ( - grant_destinations TEXT[] NOT NULL DEFAULT '{}', - grant_credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(grant_credentials) = 'array'), namespace TEXT NOT NULL, agent_name TEXT NOT NULL, agent_uid TEXT NOT NULL, + grant_destinations TEXT[] NOT NULL DEFAULT '{}', + grant_credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(grant_credentials) = 'array'), desired_revision TEXT NOT NULL, latest_successful_revision TEXT REFERENCES agent_revision(revision) ON DELETE RESTRICT, retired_at TIMESTAMPTZ, @@ -80,11 +80,11 @@ CREATE UNIQUE INDEX agent_definition_active_name_idx ON agent_definition (namespace, agent_name) WHERE retired_at IS NULL; CREATE TABLE sandbox_template_definition ( - grant_destinations TEXT[] NOT NULL DEFAULT '{}', - grant_credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(grant_credentials) = 'array'), namespace TEXT NOT NULL, sandbox_template_name TEXT NOT NULL, sandbox_template_uid TEXT NOT NULL, + grant_destinations TEXT[] NOT NULL DEFAULT '{}', + grant_credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(grant_credentials) = 'array'), desired_revision TEXT NOT NULL, latest_successful_revision TEXT REFERENCES sandbox_revision(revision) ON DELETE RESTRICT, retired_at TIMESTAMPTZ, @@ -328,6 +328,21 @@ CREATE VIEW sandbox_record AS s.data, s.namespace, s.sandbox_template_name, s.revision_receipt, s.request_hash, s.expires_at FROM runtime_instance r JOIN sandbox s USING (id) WHERE r.kind = 'sandbox'; +CREATE VIEW runtime_egress_target AS + SELECT i.id, i.prepared_revision AS revision, r.namespace, + source.kind AS source_kind, source.name AS source_name, + r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, + COALESCE(s.actor_uid, '') AS actor_uid + FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision + JOIN ( + SELECT revision, 'agent'::text AS kind, agent_name AS name FROM agent_revision + UNION ALL + SELECT revision, 'sandbox'::text AS kind, sandbox_template_name AS name FROM sandbox_revision + ) source ON source.revision = r.revision AND source.kind = r.kind + LEFT JOIN session s ON i.kind = 'agent' AND s.id = i.id + WHERE (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) + AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE'; + CREATE VIEW unreferenced_runtime_revision AS SELECT r.revision FROM runtime_revision r WHERE NOT EXISTS (SELECT 1 FROM agent_definition p WHERE p.retired_at IS NULL @@ -381,6 +396,7 @@ CREATE INDEX session_push_registration_lease ON session_push_registration (lease DROP TABLE session_push_registration; DROP VIEW unreferenced_runtime_revision; +DROP VIEW runtime_egress_target; DROP VIEW sandbox_record; DROP VIEW session_record; DROP VIEW agent_runtime_revision; diff --git a/go/core/pkg/migrations/cross_track_test.go b/go/core/pkg/migrations/cross_track_test.go index ab4401efc4..13f8585722 100644 --- a/go/core/pkg/migrations/cross_track_test.go +++ b/go/core/pkg/migrations/cross_track_test.go @@ -184,7 +184,7 @@ var schemaQualifiedChecks = []sqlCheck{ {"search_path", regexp.MustCompile(`(?i)\bsearch_path\b`)}, {"SET SCHEMA", regexp.MustCompile(`(?i)\bSET\s+SCHEMA\b`)}, {"schema-qualified table", regexp.MustCompile(`(?i)\b(?:CREATE|ALTER|DROP)\s+TABLE\s+(?:IF\s+(?:NOT\s+)?EXISTS\s+)?\w+\.\w+`)}, - {"schema-qualified index target", regexp.MustCompile(`(?i)\bON\s+\w+\.\w+`)}, + {"schema-qualified index target", regexp.MustCompile(`(?i)\bCREATE\s+(?:UNIQUE\s+)?INDEX\b[^;]*?\bON\s+(?:ONLY\s+)?\w+\.\w+`)}, {"schema-qualified reference", regexp.MustCompile(`(?i)\bREFERENCES\s+\w+\.\w+`)}, } @@ -240,12 +240,15 @@ func TestSchemaViolations(t *testing.T) { }{ {"unqualified table", `CREATE TABLE IF NOT EXISTS eval_set (id TEXT);`, nil}, {"unqualified index", `CREATE INDEX IF NOT EXISTS i ON eval_set(id);`, nil}, + {"join condition", `CREATE VIEW live AS SELECT a.id FROM actors a JOIN revisions r ON a.revision = r.id;`, nil}, + {"index followed by join", `CREATE INDEX i ON actors(id); CREATE VIEW live AS SELECT a.id FROM actors a JOIN revisions r ON a.revision = r.id;`, nil}, {"schema in comment", `-- create table myschema.foo here` + "\n" + `CREATE TABLE IF NOT EXISTS foo (id TEXT);`, nil}, {"qualified table", `CREATE TABLE myschema.eval_set (id TEXT);`, []string{"schema-qualified table"}}, {"create schema", `CREATE SCHEMA IF NOT EXISTS myschema;`, []string{"CREATE SCHEMA"}}, {"set search_path", `SET search_path TO myschema;`, []string{"search_path"}}, {"set schema", `ALTER TABLE foo SET SCHEMA myschema;`, []string{"SET SCHEMA"}}, {"qualified index target", `CREATE INDEX i ON myschema.foo(id);`, []string{"schema-qualified index target"}}, + {"qualified concurrent index target", `CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS i ON ONLY myschema.foo(id);`, []string{"schema-qualified index target"}}, {"qualified reference", `ALTER TABLE foo ADD CONSTRAINT fk FOREIGN KEY (b) REFERENCES myschema.bar(id);`, []string{"schema-qualified reference"}}, } for _, tt := range tests { From 0df9d881fdba2fe0f53dbd90fe8d5339d917bebc Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 19:48:37 +0000 Subject: [PATCH 28/43] refactor: apply live egress during source preparation Signed-off-by: Eitan Yarmush --- docs/architecture/egress-policies.md | 24 +- .../internal/controller/collections_test.go | 2 +- go/core/internal/controller/egress.go | 144 +++-------- go/core/internal/controller/egress_test.go | 237 +++++++----------- go/core/internal/controller/grants_test.go | 169 ++++++++++--- .../internal/controller/preparation_test.go | 29 ++- go/core/internal/controller/reconciler.go | 40 +-- .../internal/controller/reconciler_test.go | 15 +- .../controller/sandbox_integration_test.go | 2 +- .../internal/controller/sandbox_reconciler.go | 28 ++- .../controller/sandbox_reconciler_test.go | 12 +- .../internal/database/client_session_test.go | 6 +- .../database/runtime_creation_test.go | 4 +- go/core/internal/database/runtime_egress.go | 19 -- .../internal/database/runtime_egress_test.go | 58 ++--- .../database/runtime_lifecycle_test.go | 2 +- .../internal/database/runtime_revisions.go | 36 +-- .../database/runtime_revisions_test.go | 30 +-- .../internal/database/sandbox_revisions.go | 37 +-- go/core/internal/database/sandboxes_test.go | 6 +- .../internal/grpcserver/scheduledrun_test.go | 2 +- .../internal/grpcserver/task_store_test.go | 2 +- .../internal/service/sandbox/guest_test.go | 2 +- .../internal/service/sandbox/service_test.go | 4 +- .../internal/service/session/workflow_test.go | 2 +- go/core/pkg/app/app.go | 8 +- 26 files changed, 437 insertions(+), 483 deletions(-) diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index 4fa571fa98..ff47cbebea 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -173,22 +173,24 @@ contribute no grants. There is no separately published per-revision policy and no notification needed when a revision is recorded: every read joins against the current persisted grants, including for newly recorded revisions. -Changes to grants enqueue the source's actors across all retained revisions and -Kubernetes UIDs. Retirement and uncertain database write outcomes also enqueue -that source. Actor application runs in separate queues so slow Substrate calls -do not block preparation. The applier only reads effective policy; it does not -translate Kubernetes inputs or write grants. Startup recovery pages all allocated -actors within the controller's namespace watch scope. Failed applications retry -with exponential backoff. Preparation finalizers on both source kinds preserve -deletion events through controller downtime. +Every preparation persists the source definition, then pages its allocated actors +across all retained revisions and Kubernetes UIDs and applies effective policy +with at most 16 actor updates in parallel. Unchanged policies skip the Substrate +write. Failures return to the existing preparation queue, which retries the whole +source with exponential backoff even when grants have not changed. Slow actor +updates therefore delay further work on that preparation queue. Startup replays +every Agent and SandboxTemplate through preparation; no separate actor sweep is +needed. +Preparation finalizers preserve deletion through controller downtime and are +removed only after retiring the source and successfully updating its actors. Running and suspended actors receive updates without replacement or a new invocation. Missing or invalid policy references and Secrets revoke explicit grants. Grants that conflict with pinned credentials or caller-token origins, or exceed the merged rule limit, are withheld while pinned integration access remains. -The applier reports these conflicts with warnings limited per revision; create -and resume apply the same fallback. Missing revisions or malformed persisted -inputs return errors and block execution. +Preparation reports these conflicts with warnings; create and resume apply the +same fallback. Missing revisions or malformed persisted inputs return errors and +block execution. New Sessions and sandboxes pin the latest successful runtime revision, including while a configuration edit is unresolved, preparing, or failed. Initial creation diff --git a/go/core/internal/controller/collections_test.go b/go/core/internal/controller/collections_test.go index 92fc7b931d..af11b66679 100644 --- a/go/core/internal/controller/collections_test.go +++ b/go/core/internal/controller/collections_test.go @@ -102,7 +102,7 @@ func TestReconciliationCollectionsCompileAndObserveRevision(t *testing.T) { observed.Metadata.Uid = "actor-template-uid" observed.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}}} store := &fakeRuntimeRevisionStore{} - reconciler := &Reconciler{enqueueActors: noopEnqueueActors, + reconciler := &Reconciler{ collections: collections, templates: &fakeActorTemplates{template: observed}, store: store, } require.NoError(t, reconciler.reconcileAgent(t.Context(), state.ResourceName())) diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go index fb1dd7093b..47b5499caf 100644 --- a/go/core/internal/controller/egress.go +++ b/go/core/internal/controller/egress.go @@ -2,16 +2,13 @@ package controller import ( "context" - "slices" - "sync" - "time" + "fmt" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/hashicorp/go-multierror" + "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "istio.io/istio/pkg/kube/controllers" - "k8s.io/apimachinery/pkg/util/cache" - "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/egress" @@ -20,7 +17,6 @@ import ( ) type actorEgressStore interface { - ListRuntimeEgressTargets(context.Context, string, int) ([]database.RuntimeEgressTarget, error) ListRuntimeEgressSourceTargets(context.Context, database.RuntimeEgressSource, string, int) ([]database.RuntimeEgressTarget, error) } @@ -29,120 +25,60 @@ type actorEgressClient interface { EnsureActorEgressPolicy(context.Context, string, string, string) (egress.Policy, error) } -// ActorEgressApplier applies persisted effective policy independently of runtime -// preparation. It never translates Kubernetes inputs or writes source grants. -type ActorEgressApplier struct { - store actorEgressStore - actors actorEgressClient - namespaces []string - warnings *cache.LRUExpireCache - pages controllers.Queue - updates controllers.Queue - recovery controllers.Queue -} - -type egressSourcePage struct { - Source database.RuntimeEgressSource - After string -} - -type egressRecoveryPage struct{ After string } - -var _ manager.LeaderElectionRunnable = (*ActorEgressApplier)(nil) - -func NewActorEgressApplier(ctx context.Context, store actorEgressStore, actors actorEgressClient, namespaces []string) *ActorEgressApplier { - a := &ActorEgressApplier{store: store, actors: actors, namespaces: slices.Clone(namespaces), warnings: cache.NewLRUExpireCache(1024)} - reconcile := func(item any) error { - err := a.reconcileActor(ctx, item.(database.RuntimeEgressTarget)) - if status.Code(err) == codes.NotFound { - return nil - } - return err - } - a.updates = newReconciliationQueue("live-egress-actors", reconcile) - a.recovery = newReconciliationQueue("live-egress-recovery", reconcile) +// applySourceEgress updates every allocated actor, including suspended actors and +// former source UIDs. Preparation retries the source on failure, even when its +// grants are unchanged. Continue past individual failures so one unavailable +// actor cannot prevent revocation on the others. +func applySourceEgress(ctx context.Context, store actorEgressStore, actors actorEgressClient, source database.RuntimeEgressSource) error { const pageSize = 100 - a.pages = newReconciliationQueue("live-egress-pages", func(item any) error { - switch page := item.(type) { - case egressSourcePage: - if !a.watches(page.Source.Namespace) { - return nil - } - targets, err := a.store.ListRuntimeEgressSourceTargets(ctx, page.Source, page.After, pageSize) - if err != nil { - return err - } - for _, target := range targets { - a.updates.Add(target) - } - if len(targets) == pageSize { - page.After = targets[len(targets)-1].ID - a.pages.Add(page) - } - case egressRecoveryPage: - targets, err := a.store.ListRuntimeEgressTargets(ctx, page.After, pageSize) - if err != nil { - return err - } - for _, target := range targets { - if a.watches(target.Namespace) { - a.recovery.Add(target) + var failures *multierror.Error + after := "" + for { + targets, err := store.ListRuntimeEgressSourceTargets(ctx, source, after, pageSize) + if err != nil { + return multierror.Append(failures, fmt.Errorf("failed to list %s %s/%s actors: %w", source.Kind, source.Namespace, source.Name, err)).ErrorOrNil() + } + var group errgroup.Group + group.SetLimit(16) + for _, target := range targets { + group.Go(func() error { + if err := applyActorEgress(ctx, actors, target); err != nil { + return fmt.Errorf("failed to apply actor %s egress: %w", target.ID, err) } - } - if len(targets) == pageSize { - page.After = targets[len(targets)-1].ID - a.pages.Add(page) - } + return nil + }) } - return nil - }) - return a -} - -func (a *ActorEgressApplier) watches(namespace string) bool { - return len(a.namespaces) == 0 || slices.Contains(a.namespaces, namespace) -} - -// EnqueueSource schedules all allocated instances of this source, across UIDs. -func (a *ActorEgressApplier) EnqueueSource(source database.RuntimeEgressSource) { - a.pages.Add(egressSourcePage{Source: source}) -} - -func (a *ActorEgressApplier) NeedLeaderElection() bool { return true } - -func (a *ActorEgressApplier) Start(ctx context.Context) error { - var workers sync.WaitGroup - workers.Go(func() { a.updates.Run(ctx.Done()) }) - workers.Go(func() { a.recovery.Run(ctx.Done()) }) - a.pages.Add(egressRecoveryPage{}) - a.pages.Run(ctx.Done()) - workers.Wait() - return nil + if err := group.Wait(); err != nil { + failures = multierror.Append(failures, err) + } + if len(targets) < pageSize { + return failures.ErrorOrNil() + } + after = targets[len(targets)-1].ID + } } -func (a *ActorEgressApplier) reconcileActor(ctx context.Context, target database.RuntimeEgressTarget) error { - if !a.watches(target.Namespace) { +func applyActorEgress(ctx context.Context, actors actorEgressClient, target database.RuntimeEgressTarget) error { + binding := substrate.ActorBinding{Atespace: target.Atespace, Name: substrate.ActorName(target.ID), TemplateAtespace: target.Atespace, TemplateName: target.TemplateName} + actor, err := actors.GetActor(ctx, binding.Atespace, binding.Name) + if status.Code(err) == codes.NotFound { return nil } - binding := substrate.ActorBinding{Atespace: target.Atespace, Name: substrate.ActorName(target.ID), TemplateAtespace: target.Atespace, TemplateName: target.TemplateName} - actor, err := a.actors.GetActor(ctx, binding.Atespace, binding.Name) if err != nil { return err } if !binding.Matches(actor) || (target.ActorUID != "" && target.ActorUID != actor.GetMetadata().GetUid()) { return nil } - policy, err := a.actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, target.Revision) + policy, err := actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, target.Revision) + if status.Code(err) == codes.NotFound { + return nil + } if err != nil { return err } if policy.WithheldGrants != nil { - if _, warned := a.warnings.Get(target.Revision); !warned { - logging.FromContext(ctx).WarnContext(ctx, "explicit egress grants withheld", "revision", target.Revision, "namespace", target.Namespace, "error", policy.WithheldGrants) - a.warnings.Add(target.Revision, struct{}{}, 30*time.Second) - } - } else { - a.warnings.Remove(target.Revision) + logging.FromContext(ctx).WarnContext(ctx, "explicit egress grants withheld", "revision", target.Revision, "namespace", target.Namespace, "error", policy.WithheldGrants) } return nil } diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go index 54b9ed7e51..77f169b045 100644 --- a/go/core/internal/controller/egress_test.go +++ b/go/core/internal/controller/egress_test.go @@ -14,177 +14,130 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/pkg/logging" ) -func noopEnqueueActors(database.RuntimeEgressSource) {} - -type egressTestStore struct { - mu sync.Mutex - policies map[string]egress.Policy - targets map[database.RuntimeEgressSource][]database.RuntimeEgressTarget +type egressTargetStoreStub struct { + sources []database.RuntimeEgressSource + targets []database.RuntimeEgressTarget + listErr error } -var _ actorEgressStore = (*egressTestStore)(nil) +var _ actorEgressStore = (*egressTargetStoreStub)(nil) -func (e *egressTestStore) GetRuntimeEgressPolicy(_ context.Context, revision string) (egress.Policy, error) { - e.mu.Lock() - defer e.mu.Unlock() - policy, ok := e.policies[revision] - if !ok { - return egress.Policy{}, database.ErrNotFound - } - return policy, nil -} -func (e *egressTestStore) ListRuntimeEgressSourceTargets(_ context.Context, source database.RuntimeEgressSource, after string, limit int) ([]database.RuntimeEgressTarget, error) { - e.mu.Lock() - defer e.mu.Unlock() - return egressTestPage(e.targets[source], after, limit), nil -} -func (e *egressTestStore) ListRuntimeEgressTargets(_ context.Context, after string, limit int) ([]database.RuntimeEgressTarget, error) { - e.mu.Lock() - defer e.mu.Unlock() - var targets []database.RuntimeEgressTarget - for _, rows := range e.targets { - targets = append(targets, rows...) +func (s *egressTargetStoreStub) ListRuntimeEgressSourceTargets(_ context.Context, source database.RuntimeEgressSource, after string, limit int) ([]database.RuntimeEgressTarget, error) { + s.sources = append(s.sources, source) + if s.listErr != nil { + return nil, s.listErr } - return egressTestPage(targets, after, limit), nil -} -func egressTestPage(targets []database.RuntimeEgressTarget, after string, limit int) []database.RuntimeEgressTarget { - targets = slices.Clone(targets) - slices.SortFunc(targets, func(a, b database.RuntimeEgressTarget) int { - if a.ID < b.ID { - return -1 - } - if a.ID > b.ID { - return 1 - } - return 0 - }) - targets = slices.DeleteFunc(targets, func(target database.RuntimeEgressTarget) bool { return target.ID <= after }) - return targets[:min(len(targets), limit)] + targets := slices.DeleteFunc(slices.Clone(s.targets), func(target database.RuntimeEgressTarget) bool { return target.ID <= after }) + return targets[:min(len(targets), limit)], nil } -type observingEgressActors struct { - store *egressTestStore - mu sync.Mutex - applied map[string]egress.Access - attempts map[string][]time.Time - failures int - mismatched bool +type actorEgressStub struct { + getActor func(context.Context, string, string) (*ateapipb.Actor, error) + ensurePolicy func(context.Context, string, string, string) (egress.Policy, error) } -var _ actorEgressClient = (*observingEgressActors)(nil) +var _ actorEgressClient = (*actorEgressStub)(nil) -func (a *observingEgressActors) GetActor(_ context.Context, space, name string) (*ateapipb.Actor, error) { - template := "runtime" - if a.mismatched { - template = "replacement" +func (a *actorEgressStub) GetActor(ctx context.Context, space, name string) (*ateapipb.Actor, error) { + if a.getActor != nil { + return a.getActor(ctx, space, name) } - return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: "uid"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: template}}, nil + return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: "uid"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: "runtime"}}, nil } -func (a *observingEgressActors) EnsureActorEgressPolicy(ctx context.Context, _, name, revision string) (egress.Policy, error) { - a.mu.Lock() - defer a.mu.Unlock() - if a.attempts == nil { - a.attempts = map[string][]time.Time{} - } - a.attempts[name] = append(a.attempts[name], time.Now()) - if a.failures > 0 { - a.failures-- - return egress.Policy{}, errors.New("backend unavailable") - } - policy, err := a.store.GetRuntimeEgressPolicy(ctx, revision) - if err != nil { - return egress.Policy{}, err - } - if a.applied == nil { - a.applied = map[string]egress.Access{} - } - a.applied[name] = policy.Access - return policy, nil +func (a *actorEgressStub) EnsureActorEgressPolicy(ctx context.Context, space, name, revision string) (egress.Policy, error) { + return a.ensurePolicy(ctx, space, name, revision) } -func TestActorEgressSourcePagingAndStartupRecovery(t *testing.T) { +func TestSourceEgressAppliesEveryPageWithBoundedConcurrency(t *testing.T) { synctest.Test(t, func(t *testing.T) { - ctx, cancel := context.WithCancel(t.Context()) source := database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team", Name: "source"} - foreign := database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: "other", Name: "source"} - store := &egressTestStore{policies: map[string]egress.Policy{"old": {}, "new": {}}, targets: map[database.RuntimeEgressSource][]database.RuntimeEgressTarget{}} + store := &egressTargetStoreStub{} for i := range 101 { - revision := "old" - if i%2 == 0 { - revision = "new" - } - store.targets[source] = append(store.targets[source], database.RuntimeEgressTarget{ID: fmt.Sprintf("%03d", i), Revision: revision, Namespace: "team", Atespace: "team", TemplateName: "runtime"}) - } - store.targets[foreign] = []database.RuntimeEgressTarget{{ID: "foreign", Revision: "new", Namespace: "other", Atespace: "other", TemplateName: "runtime"}} - actors := &observingEgressActors{store: store} - applier := NewActorEgressApplier(ctx, store, actors, []string{"team"}) - var workers sync.WaitGroup - workers.Go(func() { require.NoError(t, applier.Start(ctx)) }) - defer func() { cancel(); workers.Wait() }() - synctest.Wait() - require.Len(t, actors.applied, 101, "startup recovery must traverse every page within watch scope") - updated := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} - store.mu.Lock() - store.policies["new"] = egress.Policy{Access: updated} - store.mu.Unlock() - applier.EnqueueSource(source) - applier.EnqueueSource(foreign) - synctest.Wait() - for name, calls := range actors.attempts { - require.Len(t, calls, 2, name) + store.targets = append(store.targets, database.RuntimeEgressTarget{ID: fmt.Sprintf("%03d", i), Revision: "revision", Namespace: "team", Atespace: "team", TemplateName: "runtime"}) } - require.Equal(t, updated, actors.applied["session-000"]) - require.Empty(t, actors.applied["session-001"].Destinations, "old UID revisions retain their independently resolved policy") - require.NotContains(t, actors.applied, "session-foreign") + var mu sync.Mutex + active, peak := 0, 0 + applied := map[string]int{} + backendErr := errors.New("backend unavailable") + actors := &actorEgressStub{ensurePolicy: func(ctx context.Context, _, name, _ string) (egress.Policy, error) { + mu.Lock() + active++ + peak = max(peak, active) + applied[name]++ + mu.Unlock() + time.Sleep(time.Second) + mu.Lock() + active-- + mu.Unlock() + if name == "session-000" { + return egress.Policy{}, backendErr + } + return egress.Policy{}, ctx.Err() + }} + require.ErrorIs(t, applySourceEgress(t.Context(), store, actors, source), backendErr) + require.Len(t, applied, 101, "a failed actor must not block subsequent actors or pages") + require.Greater(t, peak, 1) + require.LessOrEqual(t, peak, 16) + require.Equal(t, []database.RuntimeEgressSource{source, source}, store.sources) + store.listErr = errors.New("database unavailable") + require.ErrorIs(t, applySourceEgress(t.Context(), store, actors, source), store.listErr) }) } -func TestActorEgressRetriesAndRejectsStaleIdentity(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - ctx, cancel := context.WithCancel(t.Context()) - source := database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: "team", Name: "scratch"} - target := database.RuntimeEgressTarget{ID: "actor", Revision: "revision", Namespace: "team", Atespace: "team", TemplateName: "runtime"} - store := &egressTestStore{policies: map[string]egress.Policy{"revision": {}}, targets: map[database.RuntimeEgressSource][]database.RuntimeEgressTarget{source: {target}}} - actors := &observingEgressActors{store: store, failures: 2} - applier := NewActorEgressApplier(ctx, store, actors, nil) - var workers sync.WaitGroup - workers.Go(func() { require.NoError(t, applier.Start(ctx)) }) - defer func() { cancel(); workers.Wait() }() - synctest.Wait() - time.Sleep(time.Second) - synctest.Wait() - time.Sleep(2 * time.Second) - synctest.Wait() - attempts := actors.attempts["session-actor"] - require.Len(t, attempts, 3) - require.Equal(t, time.Second, attempts[1].Sub(attempts[0])) - require.Equal(t, 2*time.Second, attempts[2].Sub(attempts[1])) - actors.mismatched = true - require.NoError(t, applier.reconcileActor(ctx, target)) - require.Len(t, actors.attempts["session-actor"], 3) - }) +func TestActorEgressSkipsStaleOrMissingActors(t *testing.T) { + for _, test := range []struct { + name string + uid string + template string + getErr error + applyErr error + wantCalls int + wantErr bool + }{ + {name: "matching", uid: "uid", template: "runtime", wantCalls: 1}, + {name: "replaced template", uid: "uid", template: "replacement"}, + {name: "replaced actor", uid: "replacement", template: "runtime"}, + {name: "missing actor", getErr: status.Error(codes.NotFound, "gone")}, + {name: "deleted during apply", uid: "uid", template: "runtime", applyErr: status.Error(codes.NotFound, "gone"), wantCalls: 1}, + {name: "read failure", getErr: errors.New("unavailable"), wantErr: true}, + {name: "apply failure", uid: "uid", template: "runtime", applyErr: errors.New("unavailable"), wantCalls: 1, wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + calls := 0 + actors := &actorEgressStub{ + getActor: func(_ context.Context, space, name string) (*ateapipb.Actor, error) { + return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: test.uid}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: test.template}}, test.getErr + }, + ensurePolicy: func(context.Context, string, string, string) (egress.Policy, error) { + calls++ + return egress.Policy{}, test.applyErr + }, + } + err := applyActorEgress(t.Context(), actors, database.RuntimeEgressTarget{ID: "actor", Revision: "revision", Atespace: "team", TemplateName: "runtime", ActorUID: "uid"}) + if test.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, test.wantCalls, calls) + }) + } } -func TestActorEgressWarningsAreLimitedPerRevision(t *testing.T) { +func TestActorEgressReportsWithheldGrants(t *testing.T) { var logs bytes.Buffer ctx := logging.IntoContext(t.Context(), slog.New(slog.NewTextHandler(&logs, nil))) - store := &egressTestStore{policies: map[string]egress.Policy{ - "first": {WithheldGrants: errors.New("credential conflict")}, "second": {WithheldGrants: errors.New("rule limit")}, + actors := &actorEgressStub{ensurePolicy: func(context.Context, string, string, string) (egress.Policy, error) { + return egress.Policy{WithheldGrants: errors.New("credential conflict")}, nil }} - actors := &observingEgressActors{store: store} - applier := NewActorEgressApplier(ctx, store, actors, nil) - defer applier.pages.ShutDownEarly() - defer applier.updates.ShutDownEarly() - defer applier.recovery.ShutDownEarly() - for _, revision := range []string{"first", "first", "second"} { - require.NoError(t, applier.reconcileActor(ctx, database.RuntimeEgressTarget{ID: revision, Revision: revision, Namespace: "team", Atespace: "team", TemplateName: "runtime"})) - } - require.Equal(t, 2, bytes.Count(logs.Bytes(), []byte("explicit egress grants withheld"))) - require.Len(t, actors.applied, 2, "warnings must not block fallback application") + require.NoError(t, applyActorEgress(ctx, actors, database.RuntimeEgressTarget{ID: "actor", Revision: "revision", Atespace: "team", TemplateName: "runtime"})) + require.Contains(t, logs.String(), "explicit egress grants withheld") } diff --git a/go/core/internal/controller/grants_test.go b/go/core/internal/controller/grants_test.go index 82a4c0e662..d0fe278dcc 100644 --- a/go/core/internal/controller/grants_test.go +++ b/go/core/internal/controller/grants_test.go @@ -6,8 +6,10 @@ import ( "sync" "testing" "testing/synctest" + "time" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" "istio.io/istio/pkg/kube/krt" @@ -18,6 +20,7 @@ import ( kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" ) @@ -66,14 +69,13 @@ func TestAgentGrantChangesReconcileWithoutRuntimeChange(t *testing.T) { template.Status = sandboxGoldenStatus() store := &fakeRuntimeRevisionStore{} backend := &preparationTestBackend{templates: &fakeActorTemplates{template: template}, store: store} - var sources []database.RuntimeEgressSource - r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3(), func(source database.RuntimeEgressSource) { sources = append(sources, source) }) + r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3()) ctx, cancel := context.WithCancel(t.Context()) var workers sync.WaitGroup workers.Go(func() { r.Run(ctx.Done()) }) defer func() { cancel(); workers.Wait() }() synctest.Wait() - sources = nil + store.sources = nil policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(policy) agent := initial.Agent.DeepCopy() @@ -84,19 +86,20 @@ func TestAgentGrantChangesReconcileWithoutRuntimeChange(t *testing.T) { require.Equal(t, initial.Target.RevisionID.String(), store.pair.DesiredRevision) require.Equal(t, []string{"https://tickets.example.com:443"}, store.pair.Grants.Destinations) }) - require.Equal(t, []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}, sources) - sources = nil + require.Equal(t, []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}, store.sources) + store.sources = nil harnesses.DeleteObject("team-a/byo") synctest.Wait() backend.inspect(func() { require.Empty(t, store.pair.DesiredRevision) require.NotEmpty(t, store.pair.Grants.Destinations) }) - require.Empty(t, sources, "unrelated runtime failure must not fan out unchanged grants") + require.Len(t, store.sources, 1, "every preparation reapplies grants, including unchanged grants") + store.sources = nil collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).DeleteObject("team-a/access") synctest.Wait() backend.inspect(func() { require.Empty(t, store.pair.DesiredRevision); require.Empty(t, store.pair.Grants.Destinations) }) - require.Len(t, sources, 1, "revocation must reconcile even while the runtime remains unresolvable") + require.Len(t, store.sources, 1, "revocation must reconcile even while the runtime remains unresolvable") }) } @@ -121,14 +124,13 @@ func TestAgentGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { synctest.Wait() store := &fakeRuntimeRevisionStore{} backend := &preparationTestBackend{templates: &fakeActorTemplates{}, store: store} - var sources []database.RuntimeEgressSource - r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3(), func(source database.RuntimeEgressSource) { sources = append(sources, source) }) + r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3()) ctx, cancel := context.WithCancel(t.Context()) var workers sync.WaitGroup workers.Go(func() { r.Run(ctx.Done()) }) defer func() { cancel(); workers.Wait() }() synctest.Wait() - sources = nil + store.sources = nil policies := collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]) policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} @@ -143,7 +145,7 @@ func TestAgentGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { require.Equal(t, []string{"https://tickets.example.com:443"}, store.pair.Grants.Destinations) }) wantSource := database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"} - require.Equal(t, []database.RuntimeEgressSource{wantSource}, sources) + require.Equal(t, []database.RuntimeEgressSource{wantSource}, store.sources) policy = policy.DeepCopy() policy.Spec.Destinations[0].Origin = "https://updated.example.com" @@ -153,7 +155,7 @@ func TestAgentGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { require.Empty(t, store.pair.DesiredRevision) require.Equal(t, []string{"https://updated.example.com:443"}, store.pair.Grants.Destinations) }) - require.Equal(t, []database.RuntimeEgressSource{wantSource, wantSource}, sources) + require.Equal(t, []database.RuntimeEgressSource{wantSource, wantSource}, store.sources) policies.DeleteObject("team-a/access") synctest.Wait() @@ -162,25 +164,23 @@ func TestAgentGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { require.Empty(t, store.pair.Grants.Destinations) require.Zero(t, store.recordCalls, "grant changes must not require a compiled runtime") }) - require.Equal(t, []database.RuntimeEgressSource{wantSource, wantSource, wantSource}, sources, "revocation must enqueue actors while compilation remains broken") + require.Equal(t, []database.RuntimeEgressSource{wantSource, wantSource, wantSource}, store.sources, "revocation must apply actors while compilation remains broken") }) }) } } -func TestAgentGrantWriteFailureAndRetirementEnqueueActors(t *testing.T) { +func TestAgentGrantWriteFailureAndRetirementApplyActors(t *testing.T) { collections, _ := newPreparationTestCollections(t, "gvisor") state := collections.Reconciliations.List()[0] store := &fakeRuntimeRevisionStore{pairErr: errors.New("response lost")} - var sources []database.RuntimeEgressSource - r := &Reconciler{collections: collections, store: store, enqueueActors: func(source database.RuntimeEgressSource) { sources = append(sources, source) }, status: kagentfake.NewSimpleClientset(state.Agent.DeepCopy()).ApiV1alpha3()} + r := &Reconciler{collections: collections, store: store, status: kagentfake.NewSimpleClientset(state.Agent.DeepCopy()).ApiV1alpha3()} require.ErrorIs(t, r.reconcileAgent(t.Context(), state.ResourceName()), store.pairErr) - require.Len(t, sources, 1, "a failed response can hide a committed grants update") + require.Empty(t, store.sources, "a failed write retries preparation before applying actors") collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).DeleteObject(state.ResourceName()) waitFor(t, func() bool { return collections.Reconciliations.GetKey(state.ResourceName()) == nil }) require.NoError(t, r.reconcileAgent(t.Context(), state.ResourceName())) - require.Len(t, sources, 2) - require.Equal(t, sources[0], sources[1]) + require.Equal(t, []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}, store.sources) } func TestAgentFinalizerRetiresGrantsAfterOfflineDeletion(t *testing.T) { @@ -191,9 +191,13 @@ func TestAgentFinalizerRetiresGrantsAfterOfflineDeletion(t *testing.T) { agents.UpdateObject(agent) waitFor(t, func() bool { return len(collections.Reconciliations.GetKey("team-a/assistant").Agent.Finalizers) == 0 }) client := kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3() - store := &fakeRuntimeRevisionStore{} - var sources []database.RuntimeEgressSource - r := &Reconciler{collections: collections, store: store, status: client, enqueueActors: func(source database.RuntimeEgressSource) { sources = append(sources, source) }} + store := &fakeRuntimeRevisionStore{egressTargetStoreStub: egressTargetStoreStub{targets: []database.RuntimeEgressTarget{{ID: "actor", Revision: "old", Atespace: "team-a", TemplateName: "runtime"}}}} + pushErr := errors.New("backend unavailable") + actors := &fakeActorTemplates{actorEgressStub: actorEgressStub{ensurePolicy: func(context.Context, string, string, string) (egress.Policy, error) { + assert.NotEmpty(t, store.retired, "retire before applying revocation") + return egress.Policy{}, pushErr + }}} + r := &Reconciler{collections: collections, store: store, templates: actors, status: client} require.NoError(t, r.reconcileAgent(t.Context(), "team-a/assistant")) require.Zero(t, store.pairCalls, "deletion ownership must be durable before grants") updated, err := client.Agents("team-a").Get(t.Context(), "assistant", metav1.GetOptions{}) @@ -204,10 +208,15 @@ func TestAgentFinalizerRetiresGrantsAfterOfflineDeletion(t *testing.T) { waitFor(t, func() bool { return !collections.Reconciliations.GetKey("team-a/assistant").Agent.DeletionTimestamp.IsZero() }) + require.ErrorIs(t, r.reconcileAgent(t.Context(), "team-a/assistant"), pushErr) + persisted, err := client.Agents("team-a").Get(t.Context(), "assistant", metav1.GetOptions{}) + require.NoError(t, err) + require.Contains(t, persisted.Finalizers, agentPreparationFinalizer, "failed revocation must retain deletion ownership") + pushErr = nil require.NoError(t, r.reconcileAgent(t.Context(), "team-a/assistant")) require.NotEmpty(t, store.retired) - require.Len(t, sources, 1) - persisted, err := client.Agents("team-a").Get(t.Context(), "assistant", metav1.GetOptions{}) + require.Len(t, store.sources, 2) + persisted, err = client.Agents("team-a").Get(t.Context(), "assistant", metav1.GetOptions{}) require.NoError(t, err) require.NotContains(t, persisted.Finalizers, agentPreparationFinalizer) } @@ -222,25 +231,125 @@ func TestSandboxGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { policies := krt.NewStaticCollection(nil, []*v1alpha3.EgressPolicy{policy}, opts.WithName("Policies")...) pools := krt.NewStaticCollection[*atev1alpha1.WorkerPool](nil, nil, opts.WithName("Pools")...) store := &sandboxTestStore{} - var sources []database.RuntimeEgressSource - s := &SandboxReconciler{collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools, EgressPolicies: policies}, substrate.SandboxPolicy{}, opts), store: store, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3(), enqueueActors: func(source database.RuntimeEgressSource) { sources = append(sources, source) }} + s := &SandboxReconciler{collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools, EgressPolicies: policies}, substrate.SandboxPolicy{}, opts), store: store, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3()} waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch") != nil }) require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) require.Nil(t, s.collections.states.GetKey("team-a/scratch").DesiredActorTemplate, "missing WorkerPool prevents runtime compilation") require.Equal(t, []string{"https://tickets.example.com:443"}, store.desired.Grants.Destinations) - require.Len(t, sources, 1) + require.Len(t, store.sources, 1) policies.DeleteObject("team-a/access") waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch").CompilationError != "" }) require.Error(t, s.reconcile(t.Context(), "team-a/scratch")) require.Empty(t, store.desired.Grants.Destinations) - require.Len(t, sources, 2, "revocation must reach existing actors during failed compilation") + require.Len(t, store.sources, 2, "revocation must reach existing actors during failed compilation") store.upsertErr = errors.New("response lost") require.ErrorIs(t, s.reconcile(t.Context(), "team-a/scratch"), store.upsertErr) - require.Len(t, sources, 3) + require.Len(t, store.sources, 2, "failed writes retry before actor application") template = template.DeepCopy() template.DeletionTimestamp = &metav1.Time{Time: metav1.Now().Time} templates.UpdateObject(template) waitFor(t, func() bool { return !s.collections.states.GetKey("team-a/scratch").Template.DeletionTimestamp.IsZero() }) require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) - require.Len(t, sources, 4) + require.Len(t, store.sources, 3) +} + +// Registering the KRT handler must recover actors even when their grants already +// match persisted state. A failed push must retry without another graph event. +func TestPreparationReplaysAndRetriesUnchangedGrants(t *testing.T) { + for _, kind := range []string{"agent", "sandbox"} { + t.Run(kind, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + grants := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} + target := database.RuntimeEgressTarget{ID: "actor", Revision: "old", Atespace: "team-a", TemplateName: "runtime"} + var start func(context.Context) error + var persisted func() egress.Access + var actorStub *actorEgressStub + switch kind { + case "agent": + collections, harnesses := newPreparationTestCollections(t, "gvisor") + agent := collections.Agents.List()[0].DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(&v1alpha3.EgressPolicy{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: grants.Destinations[0]}}}, + }) + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + harnesses.DeleteObject("team-a/byo") + synctest.Wait() + store := &fakeRuntimeRevisionStore{pair: &database.AgentDefinition{Grants: grants}, egressTargetStoreStub: egressTargetStoreStub{targets: []database.RuntimeEgressTarget{target}}} + actors := &fakeActorTemplates{} + backend := &preparationTestBackend{templates: actors, store: store} + r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3()) + start = r.Start + persisted = func() egress.Access { backend.mu.Lock(); defer backend.mu.Unlock(); return store.pair.Grants } + actorStub = &actors.actorEgressStub + case "sandbox": + opts := krt.NewOptionsBuilder(ctx.Done(), "retry-grants", nil) + template := sandboxTestTemplate() + template.Finalizers = []string{sandboxPreparationFinalizer} + states := krt.NewStaticCollection(nil, []sandboxReconciliation{{Template: template, Grants: grants}}, opts.WithName("states")...) + store := &sandboxTestStore{desired: database.SandboxTemplateDefinition{Grants: grants}, egressTargetStoreStub: egressTargetStoreStub{targets: []database.RuntimeEgressTarget{target}}} + actors := &sandboxTestActors{} + s := &SandboxReconciler{collections: sandboxCollections{states: states, observations: krt.NewStaticCollection[sandboxRuntimeObservation](nil, nil, opts.WithName("observations")...)}, store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3()} + start = s.Start + persisted = func() egress.Access { store.mu.Lock(); defer store.mu.Unlock(); return store.desired.Grants } + actorStub = &actors.actorEgressStub + } + var mu sync.Mutex + var attempts []time.Time + var applied egress.Access + actorStub.ensurePolicy = func(context.Context, string, string, string) (egress.Policy, error) { + mu.Lock() + defer mu.Unlock() + attempts = append(attempts, time.Now()) + if len(attempts) == 1 { + return egress.Policy{}, errors.New("backend unavailable") + } + applied = persisted() + return egress.Policy{Access: applied}, nil + } + var workers sync.WaitGroup + workers.Go(func() { assert.NoError(t, start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + mu.Lock() + require.Len(t, attempts, 1, "startup must attempt actors with unchanged grants") + mu.Unlock() + time.Sleep(time.Second) + synctest.Wait() + mu.Lock() + defer mu.Unlock() + require.Len(t, attempts, 2, "the preparation queue must retry a failed push without input changes") + require.Equal(t, time.Second, attempts[1].Sub(attempts[0])) + require.Equal(t, grants.Destinations, applied.Destinations) + }) + }) + } +} + +func TestSandboxFinalizerWaitsForActorRevocation(t *testing.T) { + s, templates, _ := newSandboxTestReconciler(t, "") + const key = "team-a/scratch" + require.NoError(t, s.reconcile(t.Context(), key)) + template := syncSandboxTemplate(t, s, templates).DeepCopy() + template.DeletionTimestamp = &metav1.Time{Time: metav1.Now().Time} + templates.UpdateObject(template) + waitFor(t, func() bool { return !s.collections.states.GetKey(key).Template.DeletionTimestamp.IsZero() }) + store := s.store.(*sandboxTestStore) + store.targets = []database.RuntimeEgressTarget{{ID: "actor", Revision: "old", Atespace: "team-a", TemplateName: "runtime"}} + pushErr := errors.New("backend unavailable") + s.actors.(*sandboxTestActors).ensurePolicy = func(context.Context, string, string, string) (egress.Policy, error) { + assert.True(t, store.retired, "retire before applying revocation") + return egress.Policy{}, pushErr + } + require.ErrorIs(t, s.reconcile(t.Context(), key), pushErr) + persisted, err := s.client.SandboxTemplates("team-a").Get(t.Context(), "scratch", metav1.GetOptions{}) + require.NoError(t, err) + require.Contains(t, persisted.Finalizers, sandboxPreparationFinalizer) + pushErr = nil + require.NoError(t, s.reconcile(t.Context(), key)) + persisted, err = s.client.SandboxTemplates("team-a").Get(t.Context(), "scratch", metav1.GetOptions{}) + require.NoError(t, err) + require.NotContains(t, persisted.Finalizers, sandboxPreparationFinalizer) } diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index 75f4bf20c4..33fd25bfb0 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -27,6 +27,7 @@ import ( kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -54,7 +55,7 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { templates.template.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}, }} - reconciler := &Reconciler{enqueueActors: noopEnqueueActors, collections: collections, templates: templates, store: store} + reconciler := &Reconciler{collections: collections, templates: templates, store: store} require.NoError(t, reconciler.reconcileAgent(ctx, initial.ResourceName())) updatedHarness := harnesses.List()[0].DeepCopy() @@ -156,7 +157,7 @@ func TestPreparationErrorsPublishStatusAndRecover(t *testing.T) { collections, _ := newPreparationTestCollections(t, "microvm") initial := collections.Reconciliations.List()[0] statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() - reconciler := &Reconciler{enqueueActors: noopEnqueueActors, collections: collections, templates: &test.templates, store: &test.store, status: statusClient} + reconciler := &Reconciler{collections: collections, templates: &test.templates, store: &test.store, status: statusClient} err := reconciler.reconcileAgent(t.Context(), initial.ResourceName()) require.Error(t, err) require.Equal(t, test.code, status.Code(err)) @@ -237,7 +238,7 @@ func TestPreparationRevisionIsolation(t *testing.T) { collections, harnesses := newPreparationTestCollections(t, "microvm") initial := collections.Reconciliations.List()[0] templates := &fakeActorTemplates{createErr: status.Error(codes.FailedPrecondition, `SandboxConfig "microvm" not found`)} - reconciler := &Reconciler{enqueueActors: noopEnqueueActors, collections: collections, templates: templates, store: &fakeRuntimeRevisionStore{}} + reconciler := &Reconciler{collections: collections, templates: templates, store: &fakeRuntimeRevisionStore{}} require.Error(t, reconciler.reconcileAgent(t.Context(), initial.ResourceName())) waitFor(t, func() bool { return collections.Reconciliations.GetKey(initial.ResourceName()).PreparationFailure != nil @@ -279,7 +280,7 @@ func TestPreparationQueueAndPollerBackoff(t *testing.T) { initial := collections.Reconciliations.List()[0] statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() backend := &preparationTestBackend{templates: &test.templates, store: &test.store} - r := newReconciler(collections, backend, backend, statusClient, noopEnqueueActors) + r := newReconciler(collections, backend, backend, statusClient) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -382,7 +383,7 @@ func (p *preparationTestBackend) CreateActorTemplate(ctx context.Context, templa return p.templates.CreateActorTemplate(ctx, template) } -func (p *preparationTestBackend) UpsertAgentDefinition(ctx context.Context, definition database.AgentDefinition) (bool, error) { +func (p *preparationTestBackend) UpsertAgentDefinition(ctx context.Context, definition database.AgentDefinition) error { p.mu.Lock() defer p.mu.Unlock() return p.store.UpsertAgentDefinition(ctx, definition) @@ -412,7 +413,7 @@ func TestPreparationDesiredChangesBypassBackoff(t *testing.T) { initial := collections.Reconciliations.List()[0] templates := &fakeActorTemplates{createErr: status.Error(codes.FailedPrecondition, "missing config")} store := &fakeRuntimeRevisionStore{} - r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3(), noopEnqueueActors) + r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3()) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -466,7 +467,7 @@ func TestPreparationTerminalFailures(t *testing.T) { } store := &fakeRuntimeRevisionStore{} statusClient := kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3() - r := newReconciler(collections, templates, store, statusClient, noopEnqueueActors) + r := newReconciler(collections, templates, store, statusClient) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go r.Run(ctx.Done()) @@ -525,7 +526,7 @@ func TestInvalidActorTemplateHasNoCompiledTarget(t *testing.T) { require.Nil(t, state.PreparationFailure) store := &fakeRuntimeRevisionStore{} templates := &fakeActorTemplates{getErr: errors.New("Substrate must not be called for an invalid target")} - r := &Reconciler{enqueueActors: noopEnqueueActors, collections: collections, templates: templates, store: store} + r := &Reconciler{collections: collections, templates: templates, store: store} require.NoError(t, r.reconcileAgent(t.Context(), initial.ResourceName())) require.Empty(t, store.pair.DesiredRevision) require.Nil(t, templates.template) @@ -580,3 +581,15 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections }) return collections, harnesses } + +func (p *preparationTestBackend) ListRuntimeEgressSourceTargets(ctx context.Context, source database.RuntimeEgressSource, after string, limit int) ([]database.RuntimeEgressTarget, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.store.ListRuntimeEgressSourceTargets(ctx, source, after, limit) +} +func (p *preparationTestBackend) GetActor(ctx context.Context, atespace, name string) (*ateapipb.Actor, error) { + return p.templates.GetActor(ctx, atespace, name) +} +func (p *preparationTestBackend) EnsureActorEgressPolicy(ctx context.Context, atespace, name, revision string) (egress.Policy, error) { + return p.templates.EnsureActorEgressPolicy(ctx, atespace, name, revision) +} diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index a6084cbc1b..d8f36b9e66 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -183,13 +183,15 @@ func compilationFailure(err error) *ReconciliationFailure { // Substrate owns ActorTemplates; the database retains revisions while an Agent // or a Session or checkpoint references them. type runtimeRevisionStore interface { - UpsertAgentDefinition(context.Context, database.AgentDefinition) (bool, error) + actorEgressStore + UpsertAgentDefinition(context.Context, database.AgentDefinition) error RecordRuntimeRevision(context.Context, database.RuntimeRevision) error PromoteRuntimeRevision(context.Context, string) error RetireAgentIdentities(ctx context.Context, namespace, name string) error } type actorTemplateClient interface { + actorEgressClient EnsureAtespace(context.Context, string) error GetActorTemplate(context.Context, string, string) (*ateapipb.ActorTemplate, error) CreateActorTemplate(context.Context, *ateapipb.ActorTemplate) (*ateapipb.ActorTemplate, error) @@ -198,11 +200,10 @@ type actorTemplateClient interface { // Reconciler is the side-effect boundary for the pure KRT graph. Collection // handlers enqueue stable keys; retries always read the latest derived state. type Reconciler struct { - collections Collections - templates actorTemplateClient - store runtimeRevisionStore - enqueueActors func(database.RuntimeEgressSource) - status kagentclient.ApiV1alpha3Interface + collections Collections + templates actorTemplateClient + store runtimeRevisionStore + status kagentclient.ApiV1alpha3Interface agents controllers.Queue agentStatuses controllers.Queue @@ -214,12 +215,12 @@ type Reconciler struct { // NewReconciler creates the Kubernetes and database write boundary. Run starts // its queues after the registered KRT handlers have received initial state. -func NewReconciler(config *rest.Config, collections Collections, store runtimeRevisionStore, templates actorTemplateClient, enqueueActors func(database.RuntimeEgressSource)) (*Reconciler, error) { +func NewReconciler(config *rest.Config, collections Collections, store runtimeRevisionStore, templates actorTemplateClient) (*Reconciler, error) { statusClient, err := kagentclient.NewForConfig(config) if err != nil { return nil, fmt.Errorf("create kagent status client: %w", err) } - return newReconciler(collections, templates, store, statusClient, enqueueActors), nil + return newReconciler(collections, templates, store, statusClient), nil } func newReconciler( @@ -227,14 +228,12 @@ func newReconciler( templates actorTemplateClient, store runtimeRevisionStore, status kagentclient.ApiV1alpha3Interface, - enqueueActors func(database.RuntimeEgressSource), ) *Reconciler { r := &Reconciler{ - collections: collections, - templates: templates, - store: store, - enqueueActors: enqueueActors, - status: status, + collections: collections, + templates: templates, + store: store, + status: status, } r.agents = newReconciliationQueue("agents", func(item any) error { return r.reconcileAgent(context.Background(), item.(string)) @@ -335,10 +334,12 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { return fmt.Errorf("invalid Agent key %q", key) } err := r.store.RetireAgentIdentities(ctx, parts[0], parts[1]) - r.enqueueActors(database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: parts[0], Name: parts[1]}) if err != nil { return fmt.Errorf("retire Agent %s: %w", key, err) } + if err := applySourceEgress(ctx, r.store, r.templates, database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: parts[0], Name: parts[1]}); err != nil { + return err + } if state != nil && slices.Contains(state.Agent.Finalizers, agentPreparationFinalizer) { updated := state.Agent.DeepCopy() updated.Finalizers = slices.DeleteFunc(updated.Finalizers, func(value string) bool { return value == agentPreparationFinalizer }) @@ -366,13 +367,12 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { // cannot mistake the revision for abandoned state. Unresolved inputs clear // the old desired edge without inventing a runtime revision; // the upsert preserves the current UID's last-good runtime in either case. - changed, err := r.store.UpsertAgentDefinition(ctx, definition) - if changed || err != nil { - r.enqueueActors(database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: definition.Namespace, Name: definition.AgentName}) - } - if err != nil { + if err := r.store.UpsertAgentDefinition(ctx, definition); err != nil { return r.observePreparationError(*state, fmt.Errorf("store Agent %s: %w", key, err)) } + if err := applySourceEgress(ctx, r.store, r.templates, database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: definition.Namespace, Name: definition.AgentName}); err != nil { + return r.observePreparationError(*state, err) + } if state.Target == nil { return nil } diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index cf70506f0c..06a7cb0403 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -3,7 +3,6 @@ package controller import ( "context" "errors" - "reflect" "sync/atomic" "testing" "testing/synctest" @@ -56,7 +55,7 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { store := &fakeRuntimeRevisionStore{} templates := &fakeActorTemplates{} statusClient := kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3() - reconciler := &Reconciler{enqueueActors: noopEnqueueActors, + reconciler := &Reconciler{ collections: Collections{ Agents: krttest.GetMockCollection[*kagentv1alpha3.Agent](mock), AgentRuntimeObservations: krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...), @@ -110,7 +109,7 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { } // A fresh reconciler must clean up observations without remembering earlier calls. - reconciler = &Reconciler{enqueueActors: noopEnqueueActors, collections: reconciler.collections, templates: templates, store: store, status: statusClient} + reconciler = &Reconciler{collections: reconciler.collections, templates: templates, store: store, status: statusClient} nextTarget := *state.Target state.Target = &nextTarget state.Target.ActorTemplate = proto.CloneOf(state.Target.ActorTemplate) @@ -158,7 +157,7 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { opts := krt.NewOptionsBuilder(ctx.Done(), "test", nil) states := krt.NewStaticCollection[AgentReconciliation](nil, nil, opts.WithName("Reconciliations")...) templates := &fakeActorTemplates{} - reconciler := &Reconciler{enqueueActors: noopEnqueueActors, + reconciler := &Reconciler{ collections: Collections{ AgentRuntimeObservations: krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...), Reconciliations: states, @@ -279,6 +278,7 @@ func (s *failingFinalizationStore) DeleteRuntimeRevision(ctx context.Context, re } type fakeActorTemplates struct { + actorEgressStub template *ateapipb.ActorTemplate ensureErr error getErr error @@ -320,6 +320,7 @@ func (f *fakeActorTemplates) DeleteActorTemplate(context.Context, string, string } type fakeRuntimeRevisionStore struct { + egressTargetStoreStub pair *database.AgentDefinition revision *database.RuntimeRevision markedSuccessful bool @@ -331,11 +332,10 @@ type fakeRuntimeRevisionStore struct { pairCalls int } -func (s *fakeRuntimeRevisionStore) UpsertAgentDefinition(_ context.Context, pair database.AgentDefinition) (bool, error) { +func (s *fakeRuntimeRevisionStore) UpsertAgentDefinition(_ context.Context, pair database.AgentDefinition) error { s.pairCalls++ - changed := s.pair == nil || !reflect.DeepEqual(s.pair.Grants, pair.Grants) s.pair = &pair - return changed, s.pairErr + return s.pairErr } func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision) error { @@ -404,7 +404,6 @@ func TestReconcilerUpdatesModelConfigStatusOnSecretHashChange(t *testing.T) { &fakeActorTemplates{}, &fakeRuntimeRevisionStore{}, statusClient, - noopEnqueueActors, ) go reconciler.Run(stop) diff --git a/go/core/internal/controller/sandbox_integration_test.go b/go/core/internal/controller/sandbox_integration_test.go index b74adc5f11..c2a16ce1d4 100644 --- a/go/core/internal/controller/sandbox_integration_test.go +++ b/go/core/internal/controller/sandbox_integration_test.go @@ -65,7 +65,7 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { Secrets: typedCollection[*corev1.Secret](runtimeClient, []string{"team-a"}, "Secrets", options), SandboxTemplates: typedCollection[*kagentv1alpha3.SandboxTemplate](runtimeClient, []string{"team-a"}, "SandboxTemplates", options), WorkerPools: pools, }} - reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, noopEnqueueActors) + reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}) require.NoError(t, err) results := make(chan error, 2) go func() { results <- runtime.Start(ctx) }() diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index 180cd276e3..78bb895303 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -31,7 +31,8 @@ const ( ) type sandboxRevisionStore interface { - UpsertSandboxTemplateDefinition(context.Context, database.SandboxTemplateDefinition) (bool, error) + actorEgressStore + UpsertSandboxTemplateDefinition(context.Context, database.SandboxTemplateDefinition) error RetireSandboxTemplateIdentities(context.Context, string, string) error RecordSandboxRevision(context.Context, database.SandboxRevision) error PromoteSandboxRevision(context.Context, string) error @@ -40,21 +41,20 @@ type sandboxRevisionStore interface { // SandboxReconciler owns the queued side effects of SandboxTemplate preparation. // Sandbox instance lifecycle remains owned by the PostgreSQL-backed worker. type SandboxReconciler struct { - collections sandboxCollections - store sandboxRevisionStore - enqueueActors func(database.RuntimeEgressSource) - actors actorTemplateClient - client kagentclient.ApiV1alpha3Interface + collections sandboxCollections + store sandboxRevisionStore + actors actorTemplateClient + client kagentclient.ApiV1alpha3Interface } var _ manager.LeaderElectionRunnable = (*SandboxReconciler)(nil) -func NewSandboxReconciler(config *rest.Config, runtime *Runtime, store sandboxRevisionStore, actors actorTemplateClient, policy substrate.SandboxPolicy, enqueueActors func(database.RuntimeEgressSource)) (*SandboxReconciler, error) { +func NewSandboxReconciler(config *rest.Config, runtime *Runtime, store sandboxRevisionStore, actors actorTemplateClient, policy substrate.SandboxPolicy) (*SandboxReconciler, error) { client, err := kagentclient.NewForConfig(config) if err != nil { return nil, fmt.Errorf("create SandboxTemplate client: %w", err) } - return &SandboxReconciler{collections: newSandboxCollections(runtime.Collections, policy, runtime.Options), store: store, actors: actors, client: client, enqueueActors: enqueueActors}, nil + return &SandboxReconciler{collections: newSandboxCollections(runtime.Collections, policy, runtime.Options), store: store, actors: actors, client: client}, nil } func (s *SandboxReconciler) NeedLeaderElection() bool { return true } @@ -115,10 +115,12 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { if state == nil || !state.Template.DeletionTimestamp.IsZero() { namespace, name, _ := strings.Cut(key, "/") err := s.store.RetireSandboxTemplateIdentities(ctx, namespace, name) - s.enqueueActors(database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: namespace, Name: name}) if err != nil { return fmt.Errorf("retire SandboxTemplate %s: %w", key, err) } + if err := applySourceEgress(ctx, s.store, s.actors, database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: namespace, Name: name}); err != nil { + return err + } if state != nil && slices.Contains(state.Template.Finalizers, sandboxPreparationFinalizer) { updated := state.Template.DeepCopy() updated.Finalizers = slices.DeleteFunc(updated.Finalizers, func(value string) bool { return value == sandboxPreparationFinalizer }) @@ -138,15 +140,15 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { } return nil } - changed, err := s.store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{ + err := s.store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{ Namespace: state.Template.Namespace, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), DesiredRevision: state.desiredRevision(), Grants: state.Grants, }) - if changed || err != nil { - s.enqueueActors(database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: state.Template.Namespace, Name: state.Template.Name}) - } if err != nil { return s.observeError(*state, fmt.Errorf("store SandboxTemplate %s definition: %w", key, err), true) } + if err := applySourceEgress(ctx, s.store, s.actors, database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: state.Template.Namespace, Name: state.Template.Name}); err != nil { + return s.observeError(*state, err, true) + } if state.DesiredActorTemplate == nil { s.collections.observations.DeleteObject(key) } diff --git a/go/core/internal/controller/sandbox_reconciler_test.go b/go/core/internal/controller/sandbox_reconciler_test.go index 36907ff942..d0a29f559f 100644 --- a/go/core/internal/controller/sandbox_reconciler_test.go +++ b/go/core/internal/controller/sandbox_reconciler_test.go @@ -30,6 +30,7 @@ import ( // These fakes are also used by the informer/queue integration test. type sandboxTestStore struct { + egressTargetStoreStub upsertErr error mu sync.Mutex desired database.SandboxTemplateDefinition @@ -43,12 +44,11 @@ type sandboxTestStore struct { recordCalls int } -func (s *sandboxTestStore) UpsertSandboxTemplateDefinition(_ context.Context, value database.SandboxTemplateDefinition) (bool, error) { +func (s *sandboxTestStore) UpsertSandboxTemplateDefinition(_ context.Context, value database.SandboxTemplateDefinition) error { s.mu.Lock() defer s.mu.Unlock() - changed := !reflect.DeepEqual(s.desired.Grants, value.Grants) s.desired = value - return changed, s.upsertErr + return s.upsertErr } func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision) error { s.mu.Lock() @@ -86,6 +86,7 @@ func (s *sandboxTestStore) RetireSandboxTemplateIdentities(context.Context, stri var _ sandboxRevisionStore = (*sandboxTestStore)(nil) type sandboxTestActors struct { + actorEgressStub mu sync.Mutex store *sandboxTestStore templates map[string]*ateapipb.ActorTemplate @@ -142,9 +143,8 @@ func newSandboxTestReconciler(t *testing.T, guestImage string) (*SandboxReconcil store := &sandboxTestStore{} actors := &sandboxTestActors{store: store, templates: map[string]*ateapipb.ActorTemplate{}} reconciler := &SandboxReconciler{ - collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools}, substrate.SandboxPolicy{GuestImage: guestImage, CPU: "1", Memory: "1Gi"}, opts), - enqueueActors: noopEnqueueActors, - store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3(), + collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools}, substrate.SandboxPolicy{GuestImage: guestImage, CPU: "1", Memory: "1Gi"}, opts), + store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3(), } waitFor(t, func() bool { return reconciler.collections.states.GetKey("team-a/scratch") != nil }) return reconciler, templates, pools diff --git a/go/core/internal/database/client_session_test.go b/go/core/internal/database/client_session_test.go index 67002c1806..5d1181b105 100644 --- a/go/core/internal/database/client_session_test.go +++ b/go/core/internal/database/client_session_test.go @@ -374,7 +374,7 @@ func TestForkSessionCopiesBoundedHistory(t *testing.T) { Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision, } - if _, err := client.UpsertAgentDefinition(ctx, pair); err != nil { + if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } if err := client.RecordRuntimeRevision(ctx, revision); err != nil { @@ -534,7 +534,7 @@ func TestSessionCreateAndTransitions(t *testing.T) { Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision, } - if _, err := client.UpsertAgentDefinition(ctx, pair); err != nil { + if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } if err := client.RecordRuntimeRevision(ctx, revision); err != nil { @@ -637,7 +637,7 @@ func sessionFixture(t *testing.T, client *Client, ctx context.Context, namespace Namespace: namespace, AgentName: template, AgentUID: template + "-uid", DesiredRevision: revisionID, } - if _, err := client.UpsertAgentDefinition(ctx, pair); err != nil { + if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } if err := client.RecordRuntimeRevision(ctx, revision); err != nil { diff --git a/go/core/internal/database/runtime_creation_test.go b/go/core/internal/database/runtime_creation_test.go index bb23002f09..e3b83005df 100644 --- a/go/core/internal/database/runtime_creation_test.go +++ b/go/core/internal/database/runtime_creation_test.go @@ -30,7 +30,7 @@ func TestRuntimeCreationUsesLatestSuccessfulRevision(t *testing.T) { return session.GetPreparedRevision(), created, err } desire = func(c *Client, revision string) error { - _, err := c.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", DesiredRevision: revision}) + err := c.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", DesiredRevision: revision}) return err } } else { @@ -45,7 +45,7 @@ func TestRuntimeCreationUsesLatestSuccessfulRevision(t *testing.T) { return instance.GetPreparedRevision(), created, err } desire = func(c *Client, revision string) error { - _, err := c.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) + err := c.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) return err } unresolved = "pending:uid:2" diff --git a/go/core/internal/database/runtime_egress.go b/go/core/internal/database/runtime_egress.go index f656f4a71b..349f71607a 100644 --- a/go/core/internal/database/runtime_egress.go +++ b/go/core/internal/database/runtime_egress.go @@ -3,19 +3,12 @@ package database import ( "context" "fmt" - "time" "github.com/jackc/pgx/v5" "github.com/kagent-dev/kagent/go/core/internal/egress" ) -type definitionGrantRow struct { - GrantDestinations []string - GrantCredentials []egress.Credential - RetiredAt *time.Time -} - // canonicalStoredGrants validates and normalizes source grants for NOT NULL storage. func canonicalStoredGrants(grants egress.Access) (egress.Access, error) { canonical, err := egress.CanonicalAccess(grants.Destinations, grants.Credentials) @@ -104,18 +97,6 @@ type RuntimeEgressTarget struct { ActorUID string } -// ListRuntimeEgressTargets pages all allocated instances in ID order for startup -// recovery. Deleting instances and unissued creations are excluded; concurrent -// creation reads effective policy itself and rechecks it after applying. -func (c *Client) ListRuntimeEgressTargets(ctx context.Context, afterID string, limit int) ([]RuntimeEgressTarget, error) { - return queryMany(ctx, c.db, ` - SELECT target.id::text AS id, revision, namespace, atespace, template_name, actor_uid - FROM runtime_egress_target target - WHERE target.id > COALESCE(NULLIF($1, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) - ORDER BY target.id LIMIT $2 - `, pgx.RowToStructByName[RuntimeEgressTarget], afterID, limit) -} - // ListRuntimeEgressSourceTargets pages allocated instances across a source's UIDs // in ID order, excluding deletions and unissued creations. Invalid sources return // an error; a missing source yields an empty page. diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index ac071bdb4d..b25f222392 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -17,7 +17,7 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "scratch", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", TokenPassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} - _, upsertErr := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "scratch", AgentUID: "old-uid", DesiredRevision: "agent"}) + upsertErr := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "scratch", AgentUID: "old-uid", DesiredRevision: "agent"}) require.NoError(t, upsertErr) require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) @@ -40,7 +40,7 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { _, err = client.FinishSessionOperation(ctx, instance.Id, op.ID, executor, "", "", "") require.NoError(t, err) // A recreated Agent must not transfer authorization to an old pinned session. - _, upsertErr = client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "scratch", AgentUID: "new-uid", DesiredRevision: "pending"}) + upsertErr = client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "scratch", AgentUID: "new-uid", DesiredRevision: "pending"}) require.NoError(t, upsertErr) sandbox, options := sandboxFixture(t, client, "sandbox") created, _, err := client.CreateSandbox(ctx, sandbox, "sandbox", options) @@ -50,7 +50,6 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { claimed, err = client.ClaimSandboxOperation(ctx, created.Id, sandboxOp.ID, uuid.New()) require.NoError(t, err) require.True(t, claimed) - var sourceTargets []RuntimeEgressTarget // Both kinds deliberately share a name; source filtering must preserve kind. for _, test := range []struct { source RuntimeEgressSource @@ -73,22 +72,30 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { } else { require.Empty(t, targets[0].ActorUID) } - sourceTargets = append(sourceTargets, targets...) } } for _, kind := range []RuntimeEgressSourceKind{"", "typo"} { _, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: kind, Namespace: "team-a", Name: "scratch"}, "", 10) require.Error(t, err) } - first, err := client.ListRuntimeEgressTargets(ctx, "", 1) + // Page multiple allocated instances of one source without visiting other kinds. + sandbox.Id = uuid.NewString() + secondSandbox, _, err := client.CreateSandbox(ctx, sandbox, "second-sandbox", options) + require.NoError(t, err) + sandboxOp, err = client.BeginSandboxOperation(ctx, secondSandbox.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) + require.NoError(t, err) + claimed, err = client.ClaimSandboxOperation(ctx, secondSandbox.Id, sandboxOp.ID, uuid.New()) + require.NoError(t, err) + require.True(t, claimed) + source := RuntimeEgressSource{Kind: RuntimeEgressSandbox, Namespace: "team-a", Name: "scratch"} + first, err := client.ListRuntimeEgressSourceTargets(ctx, source, "", 1) require.NoError(t, err) require.Len(t, first, 1) - second, err := client.ListRuntimeEgressTargets(ctx, first[0].ID, 1) + second, err := client.ListRuntimeEgressSourceTargets(ctx, source, first[0].ID, 1) require.NoError(t, err) require.Len(t, second, 1) - require.ElementsMatch(t, []string{instance.Id, created.Id}, []string{first[0].ID, second[0].ID}) - require.ElementsMatch(t, sourceTargets, append(first, second...), "source and recovery queries must return the same actor bindings") - last, err := client.ListRuntimeEgressTargets(ctx, second[0].ID, 1) + require.ElementsMatch(t, []string{created.Id, secondSandbox.Id}, []string{first[0].ID, second[0].ID}) + last, err := client.ListRuntimeEgressSourceTargets(ctx, source, second[0].ID, 1) require.NoError(t, err) require.Empty(t, last) _, err = client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE) @@ -129,7 +136,7 @@ func TestRuntimeEgressTokenOriginsAreNotGrants(t *testing.T) { func TestRuntimeEgressCanonicalTokenOriginWithholdsCredentials(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() - _, err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", Grants: egress.Access{ + err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", Grants: egress.Access{ Destinations: []string{"https://model.example.com"}, Credentials: []egress.Credential{{Origin: "https://model.example.com", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"}}, }}) @@ -151,7 +158,7 @@ func TestSourceGrantsDetermineEffectivePolicy(t *testing.T) { t.Run(string(kind), func(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() - upsert := func(uid string, grants egress.Access) (bool, error) { + upsert := func(uid string, grants egress.Access) error { if kind == RuntimeEgressAgent { return client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: uid, Grants: grants}) } @@ -162,17 +169,14 @@ func TestSourceGrantsDetermineEffectivePolicy(t *testing.T) { base = egress.Access{Destinations: []string{"https://model.example.com:443"}, Credentials: []egress.Credential{{Origin: "https://model.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"}}} } initial := egress.Access{Destinations: []string{"https://tickets.example.com"}, Credentials: []egress.Credential{{Origin: "https://tickets.example.com", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/tickets/token"}}} - changed, err := upsert("uid", egress.Access{}) + err := upsert("uid", egress.Access{}) require.NoError(t, err) - require.False(t, changed) - changed, err = upsert("uid", initial) + err = upsert("uid", initial) require.NoError(t, err) - require.True(t, changed) canonical, err := egress.CanonicalAccess(initial.Destinations, initial.Credentials) require.NoError(t, err) - changed, err = upsert("uid", canonical) + err = upsert("uid", canonical) require.NoError(t, err) - require.False(t, changed, "canonical equivalence must not trigger fan-out") if kind == RuntimeEgressAgent { require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, EgressDestinations: base.Destinations, Credentials: base.Credentials, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"})) } else { @@ -189,14 +193,13 @@ func TestSourceGrantsDetermineEffectivePolicy(t *testing.T) { require.NoError(t, policy.WithheldGrants) } assertPolicy(canonical) - _, err = upsert("uid", egress.Access{Destinations: []string{"invalid"}}) + err = upsert("uid", egress.Access{Destinations: []string{"invalid"}}) require.Error(t, err) assertPolicy(canonical) if kind == RuntimeEgressAgent { conflict := egress.Access{Destinations: base.Destinations, Credentials: []egress.Credential{{Origin: base.Destinations[0], Header: "authorization", URI: "ate-secret://k8s.io/default/team/other/token"}}} - changed, err = upsert("uid", conflict) + err = upsert("uid", conflict) require.NoError(t, err) - require.True(t, changed) policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") require.NoError(t, err) require.Error(t, policy.WithheldGrants) @@ -208,17 +211,14 @@ func TestSourceGrantsDetermineEffectivePolicy(t *testing.T) { require.NoError(t, client.RetireSandboxTemplateIdentities(ctx, "team", "source")) } assertPolicy(egress.Access{}) - changed, err = upsert("uid", canonical) + err = upsert("uid", canonical) require.NoError(t, err) - require.True(t, changed, "reviving a source restores only its newly supplied grants") assertPolicy(canonical) - changed, err = upsert("replacement", canonical) + err = upsert("replacement", canonical) require.NoError(t, err) - require.True(t, changed, "retiring the old UID requires actor fan-out") assertPolicy(egress.Access{}) - changed, err = upsert("replacement", canonical) + err = upsert("replacement", canonical) require.NoError(t, err) - require.False(t, changed) _, err = client.GetRuntimeEgressPolicy(ctx, "missing") require.ErrorIs(t, err, ErrNotFound) }) @@ -230,13 +230,13 @@ func TestSourceGrantsRollbackWithInvalidRevision(t *testing.T) { ctx := t.Context() grants := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} definition := AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", Grants: grants} - _, err := client.UpsertAgentDefinition(ctx, definition) + err := client.UpsertAgentDefinition(ctx, definition) require.NoError(t, err) require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "agent", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "agent"})) require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "sandbox", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "sandbox"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "sandbox", SourceSnapshot: []byte(`{}`)})) for _, uid := range []string{"uid", "replacement"} { definition.AgentUID, definition.DesiredRevision, definition.Grants = uid, "sandbox", egress.Access{} - _, err = client.UpsertAgentDefinition(ctx, definition) + err = client.UpsertAgentDefinition(ctx, definition) require.ErrorIs(t, err, ErrConflict) policy, err := client.GetRuntimeEgressPolicy(ctx, "agent") require.NoError(t, err) @@ -257,7 +257,7 @@ func TestRuntimeEgressMalformedInputsFailClosed(t *testing.T) { pool := setupTestDB(t) client := NewClient(pool, "public") ctx := t.Context() - _, err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid"}) + err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid"}) require.NoError(t, err) require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"})) _, err = pool.Exec(ctx, test.statement) diff --git a/go/core/internal/database/runtime_lifecycle_test.go b/go/core/internal/database/runtime_lifecycle_test.go index 6efd46245c..81afb6ba2b 100644 --- a/go/core/internal/database/runtime_lifecycle_test.go +++ b/go/core/internal/database/runtime_lifecycle_test.go @@ -57,7 +57,7 @@ func TestAgentWritesRejectSandboxRevisions(t *testing.T) { `) require.NoError(t, err) // Failed validation must roll back preparation changes and preserve the ready revision. - _, err = client.UpsertAgentDefinition(ctx, AgentDefinition{ + err = client.UpsertAgentDefinition(ctx, AgentDefinition{ Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", DesiredRevision: "sandbox-revision", }) diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index 8dc73e960b..4410e58aaa 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "slices" "time" a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" @@ -19,29 +18,16 @@ import ( // Kubernetes Agent identity. Updating an existing definition revives it if retired and // preserves its latest successful revision. It atomically retires older identities // at the same names and rejects revisions whose deletion has started. Grants -// are canonicalized before writing. The bool reports changed active grants or -// retirement of another UID; errors may include an uncertain commit outcome. -func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefinition) (bool, error) { +// are canonicalized before writing; errors may include an uncertain commit outcome. +func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefinition) error { grants, err := canonicalStoredGrants(definition.Grants) if err != nil { - return false, fmt.Errorf("validate source grants: %w", err) + return fmt.Errorf("validate source grants: %w", err) } - changed := false - err = c.withTx(ctx, func(tx pgx.Tx) error { - retired, err := retireAgentIdentities(ctx, tx, definition.Namespace, definition.AgentName, definition.AgentUID) - if err != nil { - return err - } - previous, err := queryOne(ctx, tx, ` - SELECT grant_destinations, grant_credentials, retired_at FROM agent_definition - WHERE namespace = $1 AND agent_uid = $2 FOR UPDATE - `, pgx.RowToStructByName[definitionGrantRow], definition.Namespace, definition.AgentUID) - if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return c.withTx(ctx, func(tx pgx.Tx) error { + if err := retireAgentIdentities(ctx, tx, definition.Namespace, definition.AgentName, definition.AgentUID); err != nil { return err } - if previous.RetiredAt != nil { - previous = definitionGrantRow{} - } if err := execSQL(ctx, tx, ` INSERT INTO agent_definition (namespace, agent_name, agent_uid, desired_revision, grant_destinations, grant_credentials) VALUES ($1, $2, $3, $4, $5, $6) @@ -52,7 +38,6 @@ func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefi `, definition.Namespace, definition.AgentName, definition.AgentUID, definition.DesiredRevision, grants.Destinations, grants.Credentials); err != nil { return err } - changed = retired > 0 || !slices.Equal(previous.GrantDestinations, grants.Destinations) || !slices.Equal(previous.GrantCredentials, grants.Credentials) // Lock Agents before revisions. Validate the resulting definition, including // its retained last-good pointer when reactivating a retired identity. @@ -81,7 +66,6 @@ func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefi } return nil }) - return changed, err } // RecordRuntimeRevision stores immutable inputs without making them available for @@ -185,21 +169,19 @@ func toRuntimeRevision(row runtimeRevisionRow) (*RuntimeRevision, error) { // Existing sessions retain their pinned revisions. Missing and already-retired // identities are a no-op. func (c *Client) RetireAgentIdentities(ctx context.Context, namespace, name string) error { - _, err := retireAgentIdentities(ctx, c.db, namespace, name, "") - return err + return retireAgentIdentities(ctx, c.db, namespace, name, "") } // retireAgentIdentities uses the caller's executor so retirement can commit -// atomically with definition preparation and returns the number retired. An empty +// atomically with definition preparation. An empty // retainedUID retires every active identity at this source name. -func retireAgentIdentities(ctx context.Context, db dbExecutor, namespace, name, retainedUID string) (int64, error) { - result, err := db.Exec(ctx, ` +func retireAgentIdentities(ctx context.Context, db dbExecutor, namespace, name, retainedUID string) error { + return execSQL(ctx, db, ` UPDATE agent_definition SET retired_at = NOW(), updated_at = NOW() WHERE namespace = $1 AND agent_name = $2 AND retired_at IS NULL AND agent_uid IS DISTINCT FROM NULLIF($3, '') `, namespace, name, retainedUID) - return result.RowsAffected(), err } // ListUnreferencedRuntimeRevisions lists revisions unused by active Agents, diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index 24ce854ead..4abfacfac7 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -137,7 +137,7 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { Namespace: "team-a", AgentName: "assistant", AgentUID: revision.AgentUID, DesiredRevision: "new", } - _, upsertErr := client.UpsertAgentDefinition(ctx, pair) + upsertErr := client.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) request := newSessionRequest(uuid.NewString(), "assistant", "kagent", "") _, _, err = client.CreateSession(ctx, request, "session") @@ -150,7 +150,7 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { // Reconciliation of the same identity must preserve its last success while // a new desired revision is still preparing. pair.DesiredRevision = "pending" - _, upsertErr = client.UpsertAgentDefinition(ctx, pair) + upsertErr = client.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) retried, created, err := client.CreateSession(ctx, request, "session") require.NoError(t, err) @@ -236,7 +236,7 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) pair.AgentUID = "replacement-uid" pair.DesiredRevision = "revision" } - _, upsertErr := creating.UpsertAgentDefinition(ctx, pair) + upsertErr := creating.UpsertAgentDefinition(ctx, pair) created <- upsertErr }() select { @@ -308,7 +308,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) // A skipped finalization must leave last-good intact for reactivation. require.NoError(t, client.DeleteRuntimeRevision(ctx, "revision", "revision-actor-uid")) - _, upsertErr := client.UpsertAgentDefinition(ctx, pair) + upsertErr := client.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) session, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "session") require.NoError(t, err) @@ -322,11 +322,11 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.NotNil(t, claimed) // Reactivating a retired success pointer and adding a new desired edge are // both forbidden, as is overwriting a runtime while deletion is in flight. - _, upsertErr = client.UpsertAgentDefinition(ctx, pair) + upsertErr = client.UpsertAgentDefinition(ctx, pair) require.ErrorIs(t, upsertErr, ErrObjectDeleting) pair.AgentUID = "replacement-uid" pair.DesiredRevision = "revision" - _, upsertErr = client.UpsertAgentDefinition(ctx, pair) + upsertErr = client.UpsertAgentDefinition(ctx, pair) require.ErrorIs(t, upsertErr, ErrObjectDeleting) require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original), ErrObjectDeleting) require.ErrorIs(t, client.PromoteRuntimeRevision(ctx, original.Revision), ErrObjectDeleting) @@ -354,7 +354,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.NoError(t, err, "a delayed collector must not finalize the new runtime") require.NoError(t, restarted.DeleteRuntimeRevision(ctx, "revision", "recreated-actor-uid")) require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original)) - _, upsertErr = restarted.UpsertAgentDefinition(ctx, pair) + upsertErr = restarted.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) } @@ -381,7 +381,7 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { waitingQuery := "ORDER BY namespace, agent_uid" if finalizeFirst { barrier.query, barrier.afterQuery = "FROM runtime_revision WHERE revision = $1 FOR UPDATE", true - waitingQuery = "WHERE namespace = $1 AND agent_uid = $2 FOR UPDATE" + waitingQuery = "INSERT INTO agent_definition" if operation == "promote" { waitingQuery = "FOR UPDATE OF p" } @@ -407,7 +407,7 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { created <- creating.PromoteRuntimeRevision(ctx, "revision") return } - _, upsertErr := creating.UpsertAgentDefinition(ctx, pair) + upsertErr := creating.UpsertAgentDefinition(ctx, pair) created <- upsertErr } finalize := func() { deleted <- deleting.DeleteRuntimeRevision(ctx, "revision", claimed.ActorTemplateUID) } @@ -440,7 +440,7 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { require.ErrorIs(t, <-created, ErrObjectDeleting) } require.NoError(t, <-deleted, "finalization and pair writes must not deadlock") - _, upsertErr := client.UpsertAgentDefinition(ctx, pair) + upsertErr := client.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) }) } @@ -475,7 +475,7 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { require.NoError(t, err) require.Equal(t, want, session.PreparedRevision) } - _, upsertErr := c.UpsertAgentDefinition(ctx, pair) + upsertErr := c.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) assertAvailableRevision("") @@ -487,7 +487,7 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("first") pair.DesiredRevision = "second" - _, upsertErr = c.UpsertAgentDefinition(ctx, pair) + upsertErr = c.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) revision.Revision, revision.ActorTemplateName = "second", "second" require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) @@ -499,7 +499,7 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("second") pair.DesiredRevision = "third" - _, upsertErr = c.UpsertAgentDefinition(ctx, pair) + upsertErr = c.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) require.NoError(t, c.RetireAgentIdentities(ctx, pair.Namespace, pair.AgentName)) revision.Revision, revision.ActorTemplateName = "third", "third" @@ -507,11 +507,11 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) assertAvailableRevision("") // Reviving the pair must retain the last success, not a report made while retired. - _, upsertErr = c.UpsertAgentDefinition(ctx, pair) + upsertErr = c.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) assertAvailableRevision("second") pair.DesiredRevision = "second" - _, upsertErr = c.UpsertAgentDefinition(ctx, pair) + upsertErr = c.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) assertAvailableRevision("second") } diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index a51e2e99ce..425e6df85e 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -3,9 +3,7 @@ package database import ( "context" "encoding/json" - "errors" "fmt" - "slices" "github.com/jackc/pgx/v5" @@ -30,29 +28,16 @@ type SandboxTemplateDefinition struct { // UpsertSandboxTemplateDefinition pins desired inputs before any backend mutation. A new // Kubernetes UID retires the former identity without inheriting its last success. -// Grants are canonicalized before writing. The bool reports changed active grants -// or retirement of another UID; errors may include an uncertain commit outcome. -func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition SandboxTemplateDefinition) (bool, error) { +// Grants are canonicalized before writing; errors may include an uncertain commit outcome. +func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition SandboxTemplateDefinition) error { grants, err := canonicalStoredGrants(definition.Grants) if err != nil { - return false, fmt.Errorf("validate source grants: %w", err) + return fmt.Errorf("validate source grants: %w", err) } - changed := false - err = c.withTx(ctx, func(tx pgx.Tx) error { - retired, err := retireSandboxTemplateIdentities(ctx, tx, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID) - if err != nil { - return err - } - previous, err := queryOne(ctx, tx, ` - SELECT grant_destinations, grant_credentials, retired_at FROM sandbox_template_definition - WHERE namespace = $1 AND sandbox_template_uid = $2 FOR UPDATE - `, pgx.RowToStructByName[definitionGrantRow], definition.Namespace, definition.SandboxTemplateUID) - if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return c.withTx(ctx, func(tx pgx.Tx) error { + if err := retireSandboxTemplateIdentities(ctx, tx, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID); err != nil { return err } - if previous.RetiredAt != nil { - previous = definitionGrantRow{} - } if err := execSQL(ctx, tx, ` INSERT INTO sandbox_template_definition (namespace, sandbox_template_name, sandbox_template_uid, desired_revision, grant_destinations, grant_credentials) VALUES ($1, $2, $3, $4, $5, $6) @@ -63,7 +48,6 @@ func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition `, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID, definition.DesiredRevision, grants.Destinations, grants.Credentials); err != nil { return err } - changed = retired > 0 || !slices.Equal(previous.GrantDestinations, grants.Destinations) || !slices.Equal(previous.GrantCredentials, grants.Credentials) revisions, err := queryMany(ctx, tx, ` SELECT r.revision, r.kind, r.namespace, r.actor_template_atespace, r.actor_template_name, r.actor_template_uid, r.deleted_at @@ -84,25 +68,22 @@ func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition } return nil }) - return changed, err } // RetireSandboxTemplateIdentities retires active identities at this source name. // Missing/already-retired identities are a no-op; retained actors lose live grants. func (c *Client) RetireSandboxTemplateIdentities(ctx context.Context, namespace, name string) error { - _, err := retireSandboxTemplateIdentities(ctx, c.db, namespace, name, "") - return err + return retireSandboxTemplateIdentities(ctx, c.db, namespace, name, "") } // retireSandboxTemplateIdentities shares retirement with definition preparation -// and returns the number retired. An empty retainedUID retires all active identities. -func retireSandboxTemplateIdentities(ctx context.Context, db dbExecutor, namespace, name, retainedUID string) (int64, error) { - result, err := db.Exec(ctx, ` +// in the caller's transaction. An empty retainedUID retires all active identities. +func retireSandboxTemplateIdentities(ctx context.Context, db dbExecutor, namespace, name, retainedUID string) error { + return execSQL(ctx, db, ` UPDATE sandbox_template_definition SET retired_at = NOW(), updated_at = NOW() WHERE namespace = $1 AND sandbox_template_name = $2 AND retired_at IS NULL AND sandbox_template_uid IS DISTINCT FROM NULLIF($3, '') `, namespace, name, retainedUID) - return result.RowsAffected(), err } // RecordSandboxRevision stores immutable sandbox inputs and refreshes artifact diff --git a/go/core/internal/database/sandboxes_test.go b/go/core/internal/database/sandboxes_test.go index cf8205dae3..defcfe39e3 100644 --- a/go/core/internal/database/sandboxes_test.go +++ b/go/core/internal/database/sandboxes_test.go @@ -16,7 +16,7 @@ import ( func sandboxFixture(t *testing.T, client *Client, revision string) (*apiv1.Sandbox, SandboxCreateOptions) { t.Helper() ctx := t.Context() - _, upsertErr := client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) + upsertErr := client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) require.NoError(t, upsertErr) require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: revision, Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: revision, ActorTemplateUID: "actor-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)})) @@ -31,7 +31,7 @@ func TestPromoteSandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { ctx := t.Context() request, options := sandboxFixture(t, client, "first") definition := SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: "second"} - _, upsertErr := client.UpsertSandboxTemplateDefinition(ctx, definition) + upsertErr := client.UpsertSandboxTemplateDefinition(ctx, definition) require.NoError(t, upsertErr) revision, err := client.GetSandboxRevision(ctx, "first") require.NoError(t, err) @@ -58,7 +58,7 @@ func TestPromoteSandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { require.NoError(t, client.RetireSandboxTemplateIdentities(ctx, "team-a", "scratch")) require.NoError(t, client.PromoteSandboxRevision(ctx, "first")) assertAvailable("") - _, upsertErr = client.UpsertSandboxTemplateDefinition(ctx, definition) + upsertErr = client.UpsertSandboxTemplateDefinition(ctx, definition) require.NoError(t, upsertErr) assertAvailable("second") revision.Revision, revision.ActorTemplateName = "unused", "unused" diff --git a/go/core/internal/grpcserver/scheduledrun_test.go b/go/core/internal/grpcserver/scheduledrun_test.go index 11ed6cdd54..8cdd8e25f6 100644 --- a/go/core/internal/grpcserver/scheduledrun_test.go +++ b/go/core/internal/grpcserver/scheduledrun_test.go @@ -180,7 +180,7 @@ func scheduledRunTestServer(t *testing.T) (*database.Client, apiv1.ScheduledRunS t.Cleanup(db.Close) store := database.NewClient(db, "public") pair := database.AgentDefinition{Namespace: "team", AgentName: "report", AgentUID: "template-uid", DesiredRevision: "scheduled-revision"} - _, upsertErr := store.UpsertAgentDefinition(t.Context(), pair) + upsertErr := store.UpsertAgentDefinition(t.Context(), pair) require.NoError(t, upsertErr) require.NoError(t, store.RecordRuntimeRevision(t.Context(), database.RuntimeRevision{ Revision: pair.DesiredRevision, Namespace: pair.Namespace, AgentName: pair.AgentName, AgentUID: pair.AgentUID, diff --git a/go/core/internal/grpcserver/task_store_test.go b/go/core/internal/grpcserver/task_store_test.go index 30960c9baa..534ed9e144 100644 --- a/go/core/internal/grpcserver/task_store_test.go +++ b/go/core/internal/grpcserver/task_store_test.go @@ -637,7 +637,7 @@ func createTaskStoreSession(t *testing.T, store *database.Client) *apiv1.Session AgentCard: &a2apb.AgentCard{Name: "assistant"}, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "assistant-kagent-revision", ActorTemplateUID: "actor-template-uid", } - _, upsertErr := store.UpsertAgentDefinition(t.Context(), database.AgentDefinition{ + upsertErr := store.UpsertAgentDefinition(t.Context(), database.AgentDefinition{ Namespace: revision.Namespace, AgentName: revision.AgentName, AgentUID: revision.AgentUID, DesiredRevision: revision.Revision, }) diff --git a/go/core/internal/service/sandbox/guest_test.go b/go/core/internal/service/sandbox/guest_test.go index 6ed073f772..a24b8c3e0e 100644 --- a/go/core/internal/service/sandbox/guest_test.go +++ b/go/core/internal/service/sandbox/guest_test.go @@ -130,7 +130,7 @@ func guestFixture(t *testing.T) (*sandboxservice.Service, context.Context, *test require.NoError(t, err) t.Cleanup(pool.Close) store := database.NewClient(pool, "public") - _, upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"}) + upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"}) require.NoError(t, upsertErr) require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, diff --git a/go/core/internal/service/sandbox/service_test.go b/go/core/internal/service/sandbox/service_test.go index abcae41b2b..c1adc4cd80 100644 --- a/go/core/internal/service/sandbox/service_test.go +++ b/go/core/internal/service/sandbox/service_test.go @@ -135,7 +135,7 @@ func serviceFixture(t *testing.T) (*Service, context.Context, *testActors) { require.NoError(t, err) t.Cleanup(pool.Close) store := database.NewClient(pool, "public") - _, upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"}) + upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"}) require.NoError(t, upsertErr) require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, @@ -297,7 +297,7 @@ func TestSandboxResolvesCurrentGrantsBeforeResume(t *testing.T) { SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), } store := service.config.Store - _, upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: revision.Revision}) + upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: revision.Revision}) require.NoError(t, upsertErr) require.NoError(t, store.RecordSandboxRevision(ctx, revision)) require.NoError(t, store.PromoteSandboxRevision(ctx, revision.Revision)) diff --git a/go/core/internal/service/session/workflow_test.go b/go/core/internal/service/session/workflow_test.go index 6b27fcc9f6..9f4c73dcd8 100644 --- a/go/core/internal/service/session/workflow_test.go +++ b/go/core/internal/service/session/workflow_test.go @@ -148,7 +148,7 @@ func lifecycleFixture(t *testing.T) (*lifecycleTestStore, *apiv1.Session) { AgentCard: &a2apb.AgentCard{Name: "assistant"}, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "assistant-kagent-revision", ActorTemplateUID: "actor-template-uid", } - _, upsertErr := client.UpsertAgentDefinition(t.Context(), database.AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision}) + upsertErr := client.UpsertAgentDefinition(t.Context(), database.AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision}) require.NoError(t, upsertErr) require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision)) require.NoError(t, client.PromoteRuntimeRevision(t.Context(), revision.Revision)) diff --git a/go/core/pkg/app/app.go b/go/core/pkg/app/app.go index 940140c7a2..347a4fb1ec 100644 --- a/go/core/pkg/app/app.go +++ b/go/core/pkg/app/app.go @@ -287,11 +287,7 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er return err } defer actors.Close() - egressApplier := controller.NewActorEgressApplier(ctx, store, actors, watchNamespaces) - if err := manager.Add(egressApplier); err != nil { - return fmt.Errorf("add egress applier: %w", err) - } - reconciler, err := controller.NewReconciler(kubeConfig, runtime.Collections, store, actors, egressApplier.EnqueueSource) + reconciler, err := controller.NewReconciler(kubeConfig, runtime.Collections, store, actors) if err != nil { return err } @@ -379,7 +375,7 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er CPU: cfg.GetSandboxes().GetCpu(), Memory: cfg.GetSandboxes().GetMemory(), } - preparation, err := controller.NewSandboxReconciler(kubeConfig, runtime, store, actors, policy, egressApplier.EnqueueSource) + preparation, err := controller.NewSandboxReconciler(kubeConfig, runtime, store, actors, policy) if err != nil { return err } From 6c3046f9fe38d2ce5db4a666d698bbd39128f022 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 20:02:20 +0000 Subject: [PATCH 29/43] refactor: record and promote revisions atomically Signed-off-by: Eitan Yarmush --- docs/architecture/egress-policies.md | 5 +- .../internal/controller/preparation_test.go | 13 +-- go/core/internal/controller/reconciler.go | 11 +- .../internal/controller/reconciler_test.go | 15 +-- .../internal/controller/sandbox_reconciler.go | 11 +- .../controller/sandbox_reconciler_test.go | 16 +-- .../internal/database/client_session_test.go | 9 +- go/core/internal/database/protobuf_test.go | 4 +- .../internal/database/runtime_egress_test.go | 17 ++- .../database/runtime_lifecycle_test.go | 6 +- .../internal/database/runtime_revisions.go | 50 ++++----- .../database/runtime_revisions_test.go | 106 ++++++++++++++---- .../internal/database/sandbox_revisions.go | 44 ++++---- go/core/internal/database/sandboxes_test.go | 19 ++-- .../internal/grpcserver/scheduledrun_test.go | 3 +- .../internal/grpcserver/task_store_test.go | 3 +- .../internal/service/sandbox/guest_test.go | 3 +- .../internal/service/sandbox/service_test.go | 6 +- .../internal/service/session/workflow_test.go | 3 +- 19 files changed, 177 insertions(+), 167 deletions(-) diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index ff47cbebea..d64a9d3eb3 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -163,8 +163,9 @@ The elected controller's Agent and SandboxTemplate reconcilers resolve current grants independently of runtime compilation. A missing Harness, AgentTemplate, or WorkerPool cannot freeze a revocation. Each preparation queue is the sole writer of its source definition's grants in PostgreSQL. Definition updates -persist canonical grants and retire replaced Kubernetes UIDs atomically, before -recording or promoting a runtime revision. Failed runtime preparation preserves +persist canonical grants and retire replaced Kubernetes UIDs atomically. A single +store transaction records the runtime revision and, when ready, promotes it if +the active source still desires it. Failed runtime preparation preserves the last successful revision while continuing to update source grants. Effective policy is computed when read: the revision's pinned integration access diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index 33fd25bfb0..2ff25e48ba 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -98,8 +98,7 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { // A delayed success for the abandoned preparation must not replace A. abandoned, err := store.GetRuntimeRevision(ctx, preparing.Target.RevisionID.String()) require.NoError(t, err) - require.NoError(t, store.RecordRuntimeRevision(ctx, *abandoned)) - require.NoError(t, store.PromoteRuntimeRevision(ctx, abandoned.Revision)) + require.NoError(t, store.RecordRuntimeRevision(ctx, *abandoned, database.RecordRevisionOptions{Ready: true})) unreferenced, err = store.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) require.Len(t, unreferenced, 1) @@ -389,16 +388,10 @@ func (p *preparationTestBackend) UpsertAgentDefinition(ctx context.Context, defi return p.store.UpsertAgentDefinition(ctx, definition) } -func (p *preparationTestBackend) RecordRuntimeRevision(ctx context.Context, revision database.RuntimeRevision) error { +func (p *preparationTestBackend) RecordRuntimeRevision(ctx context.Context, revision database.RuntimeRevision, options database.RecordRevisionOptions) error { p.mu.Lock() defer p.mu.Unlock() - return p.store.RecordRuntimeRevision(ctx, revision) -} - -func (p *preparationTestBackend) PromoteRuntimeRevision(ctx context.Context, revision string) error { - p.mu.Lock() - defer p.mu.Unlock() - return p.store.PromoteRuntimeRevision(ctx, revision) + return p.store.RecordRuntimeRevision(ctx, revision, options) } func (p *preparationTestBackend) RetireAgentIdentities(ctx context.Context, namespace, name string) error { diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index d8f36b9e66..c89251a937 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -185,8 +185,7 @@ func compilationFailure(err error) *ReconciliationFailure { type runtimeRevisionStore interface { actorEgressStore UpsertAgentDefinition(context.Context, database.AgentDefinition) error - RecordRuntimeRevision(context.Context, database.RuntimeRevision) error - PromoteRuntimeRevision(context.Context, string) error + RecordRuntimeRevision(context.Context, database.RuntimeRevision, database.RecordRevisionOptions) error RetireAgentIdentities(ctx context.Context, namespace, name string) error } @@ -419,14 +418,10 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { ActorTemplateAtespace: observed.GetMetadata().GetAtespace(), ActorTemplateName: observed.GetMetadata().GetName(), ActorTemplateUID: observed.GetMetadata().GetUid(), } ready := observed.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() != nil - if err := r.store.RecordRuntimeRevision(ctx, revision); err != nil { + if err := r.store.RecordRuntimeRevision(ctx, revision, database.RecordRevisionOptions{Ready: ready}); err != nil { return r.observePreparationError(*state, fmt.Errorf("store runtime revision %s: %w", target.RevisionID, err)) } - if ready { - if err := r.store.PromoteRuntimeRevision(ctx, revision.Revision); err != nil { - return r.observePreparationError(*state, fmt.Errorf("promote runtime revision %s: %w", target.RevisionID, err)) - } - } + // This observation drives Kubernetes Ready status on a separate queue. // Publish it only after session creation can select the persisted revision. r.observePreparation(*state, observed, nil) diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index 06a7cb0403..babc835a89 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -326,7 +326,6 @@ type fakeRuntimeRevisionStore struct { markedSuccessful bool retired string revisionErr error - promoteErr error recordCalls int pairErr error pairCalls int @@ -338,23 +337,15 @@ func (s *fakeRuntimeRevisionStore) UpsertAgentDefinition(_ context.Context, pair return s.pairErr } -func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision) error { +func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision, options database.RecordRevisionOptions) error { s.recordCalls++ if s.revisionErr != nil { return s.revisionErr } s.revision = &revision - return nil -} - -func (s *fakeRuntimeRevisionStore) PromoteRuntimeRevision(_ context.Context, revision string) error { - if s.promoteErr != nil { - return s.promoteErr - } - if s.revision == nil || s.revision.Revision != revision { - return database.ErrNotFound + if options.Ready { + s.markedSuccessful = true } - s.markedSuccessful = true return nil } diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index 78bb895303..27b428dbee 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -34,8 +34,7 @@ type sandboxRevisionStore interface { actorEgressStore UpsertSandboxTemplateDefinition(context.Context, database.SandboxTemplateDefinition) error RetireSandboxTemplateIdentities(context.Context, string, string) error - RecordSandboxRevision(context.Context, database.SandboxRevision) error - PromoteSandboxRevision(context.Context, string) error + RecordSandboxRevision(context.Context, database.SandboxRevision, database.RecordRevisionOptions) error } // SandboxReconciler owns the queued side effects of SandboxTemplate preparation. @@ -185,14 +184,10 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), SourceSnapshot: state.SourceSnapshot, } - if err := s.store.RecordSandboxRevision(ctx, revision); err != nil { + if err := s.store.RecordSandboxRevision(ctx, revision, database.RecordRevisionOptions{Ready: ready}); err != nil { return s.observeError(*state, fmt.Errorf("store sandbox revision %s: %w", state.RevisionID, err), true) } - if ready { - if err := s.store.PromoteSandboxRevision(ctx, revision.Revision); err != nil { - return s.observeError(*state, fmt.Errorf("promote sandbox revision %s: %w", state.RevisionID, err), true) - } - } + // Kubernetes readiness can only follow a revision that callers can select // from the store. Failed persistence must never publish a ready observation. s.collections.observations.ConditionalUpdateObject(sandboxRuntimeObservation{Key: key, RevisionID: state.RevisionID, Template: observed}) diff --git a/go/core/internal/controller/sandbox_reconciler_test.go b/go/core/internal/controller/sandbox_reconciler_test.go index d0a29f559f..44b41fd51d 100644 --- a/go/core/internal/controller/sandbox_reconciler_test.go +++ b/go/core/internal/controller/sandbox_reconciler_test.go @@ -40,7 +40,6 @@ type sandboxTestStore struct { retireAttempts int retireErr error recordErr error - promoteErr error recordCalls int } @@ -50,7 +49,7 @@ func (s *sandboxTestStore) UpsertSandboxTemplateDefinition(_ context.Context, va s.desired = value return s.upsertErr } -func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision) error { +func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision, options database.RecordRevisionOptions) error { s.mu.Lock() defer s.mu.Unlock() s.recordCalls++ @@ -58,18 +57,9 @@ func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value databa return s.recordErr } s.revision = value - return nil -} -func (s *sandboxTestStore) PromoteSandboxRevision(_ context.Context, revision string) error { - s.mu.Lock() - defer s.mu.Unlock() - if s.promoteErr != nil { - return s.promoteErr - } - if s.revision.Revision != revision { - return database.ErrNotFound + if options.Ready { + s.ready = true } - s.ready = true return nil } func (s *sandboxTestStore) RetireSandboxTemplateIdentities(context.Context, string, string) error { diff --git a/go/core/internal/database/client_session_test.go b/go/core/internal/database/client_session_test.go index 5d1181b105..b06452fc49 100644 --- a/go/core/internal/database/client_session_test.go +++ b/go/core/internal/database/client_session_test.go @@ -377,10 +377,9 @@ func TestForkSessionCopiesBoundedHistory(t *testing.T) { if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true}); err != nil { t.Fatal(err) } - require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) source, _, err := client.CreateSession(ctx, &apiv1.Session{ Id: sourceID, Creator: "alice", Name: "Namespace explanation", @@ -537,10 +536,9 @@ func TestSessionCreateAndTransitions(t *testing.T) { if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true}); err != nil { t.Fatal(err) } - require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) storedRevision, err := client.GetRuntimeRevision(ctx, revision.Revision) if err != nil { @@ -640,10 +638,9 @@ func sessionFixture(t *testing.T, client *Client, ctx context.Context, namespace if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true}); err != nil { t.Fatal(err) } - require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) } func newSessionRequest(id, template, harness, name string) *apiv1.Session { diff --git a/go/core/internal/database/protobuf_test.go b/go/core/internal/database/protobuf_test.go index 86b4005309..ab3104ab59 100644 --- a/go/core/internal/database/protobuf_test.go +++ b/go/core/internal/database/protobuf_test.go @@ -286,11 +286,11 @@ func TestShareAndAgentCardProtobufPersistence(t *testing.T) { require.NoError(t, err) addUnknown(card) revision := RuntimeRevision{Revision: "revision", Namespace: "team-a", AgentName: "assistant", AgentUID: "template", SourceSnapshot: []byte(`{}`), AgentCard: card, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "template"} - require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{})) // Reconciliation can update runtime identity, but the pinned card is immutable. revision.AgentCard = &a2apb.AgentCard{Name: "replacement"} revision.ActorTemplateUID = "new-uid" - require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{})) stored, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) require.True(t, proto.Equal(card, stored.AgentCard)) diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index b25f222392..1f7751ae79 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -19,8 +19,7 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "scratch", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", TokenPassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} upsertErr := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "scratch", AgentUID: "old-uid", DesiredRevision: "agent"}) require.NoError(t, upsertErr) - require.NoError(t, client.RecordRuntimeRevision(ctx, revision)) - require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) instance, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "scratch", "kagent", ""), "request") require.NoError(t, err) op, err := client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) @@ -124,7 +123,7 @@ func TestRuntimeEgressTokenOriginsAreNotGrants(t *testing.T) { Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", TokenPassthroughOrigins: test.origins, - })) + }, RecordRevisionOptions{})) policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") require.NoError(t, err) require.Empty(t, policy.Destinations, "token origins constrain credentials without granting network access") @@ -145,7 +144,7 @@ func TestRuntimeEgressCanonicalTokenOriginWithholdsCredentials(t *testing.T) { Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", TokenPassthroughOrigins: []string{"https://MODEL.example.com."}, - })) + }, RecordRevisionOptions{})) policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") require.NoError(t, err) require.ErrorContains(t, policy.WithheldGrants, "caller-token passthrough") @@ -178,9 +177,9 @@ func TestSourceGrantsDetermineEffectivePolicy(t *testing.T) { err = upsert("uid", canonical) require.NoError(t, err) if kind == RuntimeEgressAgent { - require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, EgressDestinations: base.Destinations, Credentials: base.Credentials, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"})) + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, EgressDestinations: base.Destinations, Credentials: base.Credentials, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, RecordRevisionOptions{})) } else { - require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, SandboxTemplateName: "source", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)})) + require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, SandboxTemplateName: "source", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{})) } assertPolicy := func(grants egress.Access) { t.Helper() @@ -232,8 +231,8 @@ func TestSourceGrantsRollbackWithInvalidRevision(t *testing.T) { definition := AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", Grants: grants} err := client.UpsertAgentDefinition(ctx, definition) require.NoError(t, err) - require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "agent", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "agent"})) - require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "sandbox", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "sandbox"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "sandbox", SourceSnapshot: []byte(`{}`)})) + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "agent", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "agent"}, RecordRevisionOptions{})) + require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "sandbox", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "sandbox"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "sandbox", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{})) for _, uid := range []string{"uid", "replacement"} { definition.AgentUID, definition.DesiredRevision, definition.Grants = uid, "sandbox", egress.Access{} err = client.UpsertAgentDefinition(ctx, definition) @@ -259,7 +258,7 @@ func TestRuntimeEgressMalformedInputsFailClosed(t *testing.T) { ctx := t.Context() err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid"}) require.NoError(t, err) - require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"})) + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, RecordRevisionOptions{})) _, err = pool.Exec(ctx, test.statement) require.NoError(t, err) _, err = client.GetRuntimeEgressPolicy(ctx, "revision") diff --git a/go/core/internal/database/runtime_lifecycle_test.go b/go/core/internal/database/runtime_lifecycle_test.go index 81afb6ba2b..7c06ae747b 100644 --- a/go/core/internal/database/runtime_lifecycle_test.go +++ b/go/core/internal/database/runtime_lifecycle_test.go @@ -69,9 +69,9 @@ func TestAgentWritesRejectSandboxRevisions(t *testing.T) { revision, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) revision.Revision, revision.ActorTemplateUID = "sandbox-revision", "replacement-uid" - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision), ErrConflict) - require.ErrorIs(t, client.PromoteRuntimeRevision(ctx, "sandbox-revision"), ErrConflict) - require.ErrorIs(t, client.PromoteSandboxRevision(ctx, "revision"), ErrConflict) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{}), ErrConflict) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{Ready: true}), ErrConflict) + require.ErrorIs(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "scratch"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "scratch-uid", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{Ready: true}), ErrConflict) var uid string require.NoError(t, pool.QueryRow(ctx, "SELECT actor_template_uid FROM runtime_revision WHERE revision = 'sandbox-revision'").Scan(&uid)) require.Equal(t, "original-uid", uid) diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index 4410e58aaa..915082cb82 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -68,10 +68,17 @@ func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefi }) } -// RecordRuntimeRevision stores immutable inputs without making them available for -// new Sessions. Existing inputs are preserved; actor UID and update time refresh. -// Deleting revisions return ErrObjectDeleting. -func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevision) error { +// RecordRevisionOptions controls whether a prepared revision becomes available. +type RecordRevisionOptions struct { + // Ready promotes the revision only while its source is active and still desires it. + Ready bool +} + +// RecordRuntimeRevision atomically stores immutable inputs and, when Ready, +// promotes the revision for its active source if still desired. Existing inputs +// are preserved; actor UID and update time refresh. Stale reports cannot replace +// the last successful revision. Deleting revisions return ErrObjectDeleting. +func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevision, options RecordRevisionOptions) error { if revision.AgentCard == nil { return fmt.Errorf("runtime revision %s has no Agent Card", revision.Revision) } @@ -80,6 +87,15 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi return fmt.Errorf("encode runtime revision Agent Card: %w", err) } return c.withTx(ctx, func(tx pgx.Tx) error { + if options.Ready { + // Match definition preparation and GC: source before revision. + if _, err := queryMany(ctx, tx, ` + SELECT 1 FROM agent_definition p + WHERE p.namespace = $1 AND p.agent_uid = $2 FOR UPDATE OF p + `, pgx.RowTo[int], revision.Namespace, revision.AgentUID); err != nil { + return err + } + } if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{ RuntimeArtifact: RuntimeArtifact{Revision: revision.Revision, Kind: runtimeKindAgent, Namespace: revision.Namespace, ActorTemplateAtespace: revision.ActorTemplateAtespace, ActorTemplateName: revision.ActorTemplateName, ActorTemplateUID: revision.ActorTemplateUID}, @@ -93,38 +109,20 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi `, revision.Revision, revision.AgentName, revision.AgentUID, card); err != nil { return err } - return nil - }) -} - -// PromoteRuntimeRevision makes a recorded, prepared revision available only to -// the active Agent identity that still desires it. Missing/deleting revisions return -// ErrNotFound / ErrObjectDeleting; stale reports leave the definition unchanged. -func (c *Client) PromoteRuntimeRevision(ctx context.Context, revision string) error { - return c.withTx(ctx, func(tx pgx.Tx) error { - // Match GC finalization's lock order: definition before revision. - _, err := queryOne(ctx, tx, ` - SELECT 1 FROM agent_definition p - JOIN agent_runtime_revision r ON p.namespace = r.namespace AND p.agent_uid = r.agent_uid - WHERE r.revision = $1 FOR UPDATE OF p - `, pgx.RowTo[int], revision) - if err != nil && !errors.Is(err, pgx.ErrNoRows) { - return err - } - if err := lockRuntimeRevisionForReference(ctx, tx, revision, runtimeKindAgent); err != nil { - return err + if !options.Ready { + return nil } return execSQL(ctx, tx, ` UPDATE agent_definition p SET latest_successful_revision = r.revision, updated_at = NOW() FROM agent_runtime_revision r - WHERE r.revision = $1 + WHERE r.revision = $1 AND p.namespace = $2 AND p.agent_uid = $3 AND p.namespace = r.namespace AND p.agent_uid = r.agent_uid AND p.desired_revision = r.revision AND p.retired_at IS NULL AND p.latest_successful_revision IS DISTINCT FROM r.revision - `, revision) + `, revision.Revision, revision.Namespace, revision.AgentUID) }) } diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index 4abfacfac7..82af16b59e 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -132,7 +132,7 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { revision.Revision = "new" revision.AgentUID = "replacement-uid" revision.ActorTemplateName = "new-actor-template" - require.NoError(t, client.RecordRuntimeRevision(ctx, *revision)) + require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{})) pair := AgentDefinition{ Namespace: "team-a", AgentName: "assistant", AgentUID: revision.AgentUID, DesiredRevision: "new", @@ -142,7 +142,7 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { request := newSessionRequest(uuid.NewString(), "assistant", "kagent", "") _, _, err = client.CreateSession(ctx, request, "session") require.ErrorIs(t, err, ErrNotFound, "a replacement must not select the previous template UID") - require.NoError(t, client.PromoteRuntimeRevision(ctx, revision.Revision)) + require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{Ready: true})) session, _, err := client.CreateSession(ctx, request, "session") require.NoError(t, err) require.Equal(t, "new", session.GetPreparedRevision()) @@ -328,8 +328,8 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) pair.DesiredRevision = "revision" upsertErr = client.UpsertAgentDefinition(ctx, pair) require.ErrorIs(t, upsertErr, ErrObjectDeleting) - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original), ErrObjectDeleting) - require.ErrorIs(t, client.PromoteRuntimeRevision(ctx, original.Revision), ErrObjectDeleting) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, RecordRevisionOptions{}), ErrObjectDeleting) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, RecordRevisionOptions{Ready: true}), ErrObjectDeleting) // A new client rediscovers and retries the committed claim after a crash. restarted := NewClient(pool, "public") revisions, err := restarted.ListUnreferencedRuntimeRevisions(ctx) @@ -345,7 +345,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.ErrorIs(t, err, ErrNotFound) // The same digest can be prepared again once cleanup has completed. original.ActorTemplateUID = "recreated-actor-uid" - require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original)) + require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, RecordRevisionOptions{})) newClaim, err := restarted.BeginRuntimeRevisionDeletion(ctx, "revision") require.NoError(t, err) require.NotNil(t, newClaim) @@ -353,7 +353,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) _, err = restarted.GetRuntimeRevision(ctx, "revision") require.NoError(t, err, "a delayed collector must not finalize the new runtime") require.NoError(t, restarted.DeleteRuntimeRevision(ctx, "revision", "recreated-actor-uid")) - require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original)) + require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, RecordRevisionOptions{})) upsertErr = restarted.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) } @@ -367,6 +367,8 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") + revision, err := client.GetRuntimeRevision(ctx, "revision") + require.NoError(t, err) require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "revision") require.NoError(t, err) @@ -404,7 +406,7 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { created, deleted := make(chan error, 1), make(chan error, 1) create := func() { if operation == "promote" { - created <- creating.PromoteRuntimeRevision(ctx, "revision") + created <- creating.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{Ready: true}) return } upsertErr := creating.UpsertAgentDefinition(ctx, pair) @@ -432,9 +434,7 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { return err == nil && waiting }, 5*time.Second, 10*time.Millisecond) resume.Do(func() { close(barrier.resume) }) - if finalizeFirst && operation == "promote" { - require.ErrorIs(t, <-created, ErrNotFound) - } else if finalizeFirst { + if finalizeFirst { require.NoError(t, <-created) } else { require.ErrorIs(t, <-created, ErrObjectDeleting) @@ -447,7 +447,7 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { } } -func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { +func TestRecordReadyRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { pool := setupTestDB(t) c := NewClient(pool, "public") ctx := t.Context() @@ -477,34 +477,32 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { } upsertErr := c.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) - require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{})) assertAvailableRevision("") stored, err := c.GetRuntimeRevision(ctx, revision.Revision) require.NoError(t, err) require.Equal(t, "original", stored.AgentCard.Name) - require.ErrorIs(t, c.PromoteRuntimeRevision(ctx, "missing"), ErrNotFound) - require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) assertAvailableRevision("first") pair.DesiredRevision = "second" upsertErr = c.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) revision.Revision, revision.ActorTemplateName = "second", "second" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{})) assertAvailableRevision("first") - require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) assertAvailableRevision("second") // A delayed report for the old revision must not replace the newer ready revision. revision.Revision, revision.ActorTemplateName = "first", "first" - require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) assertAvailableRevision("second") pair.DesiredRevision = "third" upsertErr = c.UpsertAgentDefinition(ctx, pair) require.NoError(t, upsertErr) require.NoError(t, c.RetireAgentIdentities(ctx, pair.Namespace, pair.AgentName)) revision.Revision, revision.ActorTemplateName = "third", "third" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision)) - require.NoError(t, c.PromoteRuntimeRevision(ctx, revision.Revision)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) assertAvailableRevision("") // Reviving the pair must retain the last success, not a report made while retired. upsertErr = c.UpsertAgentDefinition(ctx, pair) @@ -519,12 +517,12 @@ func TestPromoteRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { func TestRuntimeRevisionPersistsCredentialBindings(t *testing.T) { client := NewClient(setupTestDB(t), "public") revision := RuntimeRevision{Revision: "credential-revision", Namespace: "team", AgentName: "agent", AgentUID: "agent", SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{"https://api.example.com:443"}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", Credentials: []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}}} - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision)) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, RecordRevisionOptions{})) got, err := client.GetRuntimeRevision(t.Context(), revision.Revision) require.NoError(t, err) require.Equal(t, revision.Credentials, got.Credentials) revision.Credentials[0].URI = "ate-secret://k8s.io/default/team/other/token" - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision)) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, RecordRevisionOptions{})) unchanged, err := client.GetRuntimeRevision(t.Context(), revision.Revision) require.NoError(t, err) require.Equal(t, got.Credentials, unchanged.Credentials, "revision credentials are immutable") @@ -541,3 +539,69 @@ func TestRuntimeRevisionRejectsMalformedStoredCredentials(t *testing.T) { require.ErrorContains(t, err, "exact HTTP(S) DNS origin") require.Nil(t, revision) } + +func TestRecordReadyRevisionRollsBackOnPromotionFailure(t *testing.T) { + for _, kind := range []string{"agent", "sandbox"} { + for _, existing := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/existing=%t", kind, existing), func(t *testing.T) { + pool := setupTestDB(t) + client := NewClient(pool, "public") + ctx := t.Context() + var record func(string, RecordRevisionOptions) error + var storedUID func() (string, error) + switch kind { + case "agent": + require.NoError(t, client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", DesiredRevision: "revision"})) + record = func(uid string, options RecordRevisionOptions) error { + return client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", AgentCard: &a2apb.AgentCard{}, SourceSnapshot: []byte(`{}`), ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: uid}, options) + } + storedUID = func() (string, error) { + revision, err := client.GetRuntimeRevision(ctx, "revision") + if err != nil { + return "", err + } + return revision.ActorTemplateUID, nil + } + // Force the final write to fail after the revision has been recorded. + _, err := pool.Exec(ctx, `ALTER TABLE agent_definition ADD CONSTRAINT reject_promotion CHECK (latest_successful_revision IS NULL)`) + require.NoError(t, err) + t.Cleanup(func() { + if _, err := pool.Exec(context.Background(), `ALTER TABLE agent_definition DROP CONSTRAINT reject_promotion`); err != nil { + t.Errorf("remove test constraint: %v", err) + } + }) + case "sandbox": + require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team", SandboxTemplateName: "source", SandboxTemplateUID: "uid", DesiredRevision: "revision"})) + record = func(uid string, options RecordRevisionOptions) error { + return client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: uid}, SandboxTemplateName: "source", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, options) + } + storedUID = func() (string, error) { + revision, err := client.GetSandboxRevision(ctx, "revision") + if err != nil { + return "", err + } + return revision.ActorTemplateUID, nil + } + _, err := pool.Exec(ctx, `ALTER TABLE sandbox_template_definition ADD CONSTRAINT reject_promotion CHECK (latest_successful_revision IS NULL)`) + require.NoError(t, err) + t.Cleanup(func() { + if _, err := pool.Exec(context.Background(), `ALTER TABLE sandbox_template_definition DROP CONSTRAINT reject_promotion`); err != nil { + t.Errorf("remove test constraint: %v", err) + } + }) + } + if existing { + require.NoError(t, record("original", RecordRevisionOptions{})) + } + require.ErrorContains(t, record("updated", RecordRevisionOptions{Ready: true}), "reject_promotion") + uid, err := storedUID() + if existing { + require.NoError(t, err) + require.Equal(t, "original", uid, "a failed promotion must roll back artifact updates") + } else { + require.ErrorIs(t, err, ErrNotFound, "a failed promotion must not leave a partially recorded revision") + } + }) + } + } +} diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index 425e6df85e..dc7254fb9a 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -86,10 +86,21 @@ func retireSandboxTemplateIdentities(ctx context.Context, db dbExecutor, namespa `, namespace, name, retainedUID) } -// RecordSandboxRevision stores immutable sandbox inputs and refreshes artifact -// identity without promoting the revision. Deleting revisions return ErrObjectDeleting. -func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevision) error { +// RecordSandboxRevision atomically stores immutable inputs and, when Ready, +// promotes the revision for its active source if still desired. Existing inputs +// are preserved; actor UID and update time refresh. Stale reports cannot replace +// the last successful revision. Deleting revisions return ErrObjectDeleting. +func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevision, options RecordRevisionOptions) error { return c.withTx(ctx, func(tx pgx.Tx) error { + if options.Ready { + // Match definition preparation and GC: source before revision. + if _, err := queryMany(ctx, tx, ` + SELECT 1 FROM sandbox_template_definition p + WHERE p.namespace = $1 AND p.sandbox_template_uid = $2 FOR UPDATE OF p + `, pgx.RowTo[int], revision.Namespace, revision.SandboxTemplateUID); err != nil { + return err + } + } revision.Kind = runtimeKindSandbox if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{RuntimeArtifact: revision.RuntimeArtifact, SourceSnapshot: revision.SourceSnapshot}); err != nil { return err @@ -100,34 +111,17 @@ func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevi `, revision.Revision, revision.SandboxTemplateName, revision.SandboxTemplateUID); err != nil { return err } - return nil - }) -} - -// PromoteSandboxRevision makes a recorded, prepared revision available only to -// the active template identity that still desires it. Missing/deleting revisions return -// ErrNotFound / ErrObjectDeleting; stale reports leave the definition unchanged. -func (c *Client) PromoteSandboxRevision(ctx context.Context, revision string) error { - return c.withTx(ctx, func(tx pgx.Tx) error { - // Match GC finalization's lock order: definition before revision. - if _, err := queryMany(ctx, tx, ` - SELECT 1 FROM sandbox_template_definition p - JOIN runtime_revision r ON p.namespace = r.namespace - JOIN sandbox_revision s ON r.revision = s.revision AND p.sandbox_template_uid = s.sandbox_template_uid - WHERE r.revision = $1 FOR UPDATE OF p - `, pgx.RowTo[int], revision); err != nil { - return err - } - if err := lockRuntimeRevisionForReference(ctx, tx, revision, runtimeKindSandbox); err != nil { - return err + if !options.Ready { + return nil } return execSQL(ctx, tx, ` UPDATE sandbox_template_definition p SET latest_successful_revision = r.revision, updated_at = NOW() FROM runtime_revision r JOIN sandbox_revision s USING (revision) - WHERE r.revision = $1 AND p.namespace = r.namespace AND p.sandbox_template_uid = s.sandbox_template_uid + WHERE r.revision = $1 AND p.namespace = $2 AND p.sandbox_template_uid = $3 + AND p.namespace = r.namespace AND p.sandbox_template_uid = s.sandbox_template_uid AND p.desired_revision = r.revision AND p.retired_at IS NULL AND p.latest_successful_revision IS DISTINCT FROM r.revision - `, revision) + `, revision.Revision, revision.Namespace, revision.SandboxTemplateUID) }) } diff --git a/go/core/internal/database/sandboxes_test.go b/go/core/internal/database/sandboxes_test.go index defcfe39e3..40bed380c9 100644 --- a/go/core/internal/database/sandboxes_test.go +++ b/go/core/internal/database/sandboxes_test.go @@ -19,14 +19,13 @@ func sandboxFixture(t *testing.T, client *Client, revision string) (*apiv1.Sandb upsertErr := client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) require.NoError(t, upsertErr) require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: revision, Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: revision, ActorTemplateUID: "actor-uid"}, - SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)})) - require.NoError(t, client.PromoteSandboxRevision(ctx, revision)) + SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{Ready: true})) hash := sha256.Sum256([]byte("request")) return &apiv1.Sandbox{Id: uuid.NewString(), Creator: "alice", SandboxTemplate: &apiv1.ResourceReference{Namespace: "team-a", Name: "scratch"}, Name: "experiment"}, SandboxCreateOptions{RequestHash: hash[:], TemplateUID: "uid", TTL: time.Hour} } -func TestPromoteSandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { +func TestRecordReadySandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() request, options := sandboxFixture(t, client, "first") @@ -36,7 +35,7 @@ func TestPromoteSandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { revision, err := client.GetSandboxRevision(ctx, "first") require.NoError(t, err) revision.Revision, revision.ActorTemplateName = "second", "second" - require.NoError(t, client.RecordSandboxRevision(ctx, *revision)) + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{})) assertAvailable := func(want string) { t.Helper() input := proto.CloneOf(request) @@ -50,23 +49,23 @@ func TestPromoteSandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { require.Equal(t, want, instance.PreparedRevision) } assertAvailable("first") - require.ErrorIs(t, client.PromoteSandboxRevision(ctx, "missing"), ErrNotFound) - require.NoError(t, client.PromoteSandboxRevision(ctx, "second")) + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{Ready: true})) assertAvailable("second") - require.NoError(t, client.PromoteSandboxRevision(ctx, "first")) + revision.Revision, revision.ActorTemplateName = "first", "first" + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{Ready: true})) assertAvailable("second") require.NoError(t, client.RetireSandboxTemplateIdentities(ctx, "team-a", "scratch")) - require.NoError(t, client.PromoteSandboxRevision(ctx, "first")) + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{Ready: true})) assertAvailable("") upsertErr = client.UpsertSandboxTemplateDefinition(ctx, definition) require.NoError(t, upsertErr) assertAvailable("second") revision.Revision, revision.ActorTemplateName = "unused", "unused" - require.NoError(t, client.RecordSandboxRevision(ctx, *revision)) + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{})) claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "unused") require.NoError(t, err) require.NotNil(t, claimed) - require.ErrorIs(t, client.PromoteSandboxRevision(ctx, "unused"), ErrObjectDeleting) + require.ErrorIs(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{Ready: true}), ErrObjectDeleting) } func finishSandbox(t *testing.T, client *Client, id string, kind apiv1.RuntimeOperation) *apiv1.Sandbox { diff --git a/go/core/internal/grpcserver/scheduledrun_test.go b/go/core/internal/grpcserver/scheduledrun_test.go index 8cdd8e25f6..453442b661 100644 --- a/go/core/internal/grpcserver/scheduledrun_test.go +++ b/go/core/internal/grpcserver/scheduledrun_test.go @@ -186,8 +186,7 @@ func scheduledRunTestServer(t *testing.T) (*database.Client, apiv1.ScheduledRunS Revision: pair.DesiredRevision, Namespace: pair.Namespace, AgentName: pair.AgentName, AgentUID: pair.AgentUID, SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: "runtime-uid", - })) - require.NoError(t, store.PromoteRuntimeRevision(t.Context(), pair.DesiredRevision)) + }, database.RecordRevisionOptions{Ready: true})) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) agent := &v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "report"}, Spec: v1alpha3.AgentSpec{ diff --git a/go/core/internal/grpcserver/task_store_test.go b/go/core/internal/grpcserver/task_store_test.go index 534ed9e144..7ae3ad4cdb 100644 --- a/go/core/internal/grpcserver/task_store_test.go +++ b/go/core/internal/grpcserver/task_store_test.go @@ -642,8 +642,7 @@ func createTaskStoreSession(t *testing.T, store *database.Client) *apiv1.Session DesiredRevision: revision.Revision, }) require.NoError(t, upsertErr) - require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision)) - require.NoError(t, store.PromoteRuntimeRevision(t.Context(), revision.Revision)) + require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision, database.RecordRevisionOptions{Ready: true})) session, _, err := store.CreateSession(t.Context(), &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", diff --git a/go/core/internal/service/sandbox/guest_test.go b/go/core/internal/service/sandbox/guest_test.go index a24b8c3e0e..a0c67cf652 100644 --- a/go/core/internal/service/sandbox/guest_test.go +++ b/go/core/internal/service/sandbox/guest_test.go @@ -135,8 +135,7 @@ func guestFixture(t *testing.T) (*sandboxservice.Service, context.Context, *test require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - })) - require.NoError(t, store.PromoteSandboxRevision(ctx, "revision")) + }, database.RecordRevisionOptions{Ready: true})) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&v1alpha3.SandboxTemplate{ diff --git a/go/core/internal/service/sandbox/service_test.go b/go/core/internal/service/sandbox/service_test.go index c1adc4cd80..77b67bd89e 100644 --- a/go/core/internal/service/sandbox/service_test.go +++ b/go/core/internal/service/sandbox/service_test.go @@ -140,8 +140,7 @@ func serviceFixture(t *testing.T) (*Service, context.Context, *testActors) { require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - })) - require.NoError(t, store.PromoteSandboxRevision(ctx, "revision")) + }, database.RecordRevisionOptions{Ready: true})) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&v1alpha3.SandboxTemplate{ @@ -299,8 +298,7 @@ func TestSandboxResolvesCurrentGrantsBeforeResume(t *testing.T) { store := service.config.Store upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: revision.Revision}) require.NoError(t, upsertErr) - require.NoError(t, store.RecordSandboxRevision(ctx, revision)) - require.NoError(t, store.PromoteSandboxRevision(ctx, revision.Revision)) + require.NoError(t, store.RecordSandboxRevision(ctx, revision, database.RecordRevisionOptions{Ready: true})) instance, err := service.Create(ctx, createRequest()) require.NoError(t, err) require.Equal(t, revision.Revision, instance.PreparedRevision) diff --git a/go/core/internal/service/session/workflow_test.go b/go/core/internal/service/session/workflow_test.go index 9f4c73dcd8..8ce3df2d6b 100644 --- a/go/core/internal/service/session/workflow_test.go +++ b/go/core/internal/service/session/workflow_test.go @@ -150,8 +150,7 @@ func lifecycleFixture(t *testing.T) (*lifecycleTestStore, *apiv1.Session) { } upsertErr := client.UpsertAgentDefinition(t.Context(), database.AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision}) require.NoError(t, upsertErr) - require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision)) - require.NoError(t, client.PromoteRuntimeRevision(t.Context(), revision.Revision)) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision, database.RecordRevisionOptions{Ready: true})) session, _, err := client.CreateSession(t.Context(), &apiv1.Session{Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}}, uuid.NewString()) require.NoError(t, err) return &lifecycleTestStore{Client: client, revision: revision}, session From 39b08564a067ac503b65f9169458976659060ea9 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 20:07:23 +0000 Subject: [PATCH 30/43] refactor: name source policy access EgressGrant Signed-off-by: Eitan Yarmush --- docs/architecture/egress-policies.md | 3 ++ go/core/internal/controller/equality_test.go | 2 +- go/core/internal/controller/grants_test.go | 33 ++++++++++--------- go/core/internal/controller/reconciler.go | 8 ++--- .../controller/sandbox_collections.go | 4 +-- .../internal/controller/sandbox_reconciler.go | 2 +- go/core/internal/database/models.go | 8 ++--- .../internal/database/runtime_egress_test.go | 10 +++--- .../internal/database/runtime_revisions.go | 2 +- .../internal/database/sandbox_revisions.go | 4 +-- 10 files changed, 41 insertions(+), 35 deletions(-) diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index d64a9d3eb3..0b4d9faf81 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -155,6 +155,9 @@ when dependencies change. Runtime revisions retain the integration access inferred from models and tools, including credential references and caller-token passthrough constraints. +The combined additional access from a source's referenced EgressPolicies is its +`EgressGrant`: destinations it may contact and credentials the gateway may inject. +Reconciliation and source definition structs carry it as `EgressGrant egress.Access`. Explicit policy grants and policy references do not enter revision identity. Changing a grant or its Secret reference does not prepare a new runtime image. Secret values remain outside compiled inputs; the gateway handles value rotation. diff --git a/go/core/internal/controller/equality_test.go b/go/core/internal/controller/equality_test.go index 5acd2939c8..3c03c03ea8 100644 --- a/go/core/internal/controller/equality_test.go +++ b/go/core/internal/controller/equality_test.go @@ -56,7 +56,7 @@ func TestAgentReconciliationEquality(t *testing.T) { }, equal: true}, {name: "template source", change: func(r *AgentReconciliation) { r.Agent.Generation++ }}, {name: "live grants", change: func(r *AgentReconciliation) { - r.Grants.Destinations = []string{"https://tickets.example.com:443"} + r.EgressGrant.Destinations = []string{"https://tickets.example.com:443"} }}, {name: "revision identity", change: func(r *AgentReconciliation) { r.Target.RevisionID[0]++ }}, {name: "revision inputs", change: func(r *AgentReconciliation) { r.Target.Revision.SandboxClass = "microvm" }}, diff --git a/go/core/internal/controller/grants_test.go b/go/core/internal/controller/grants_test.go index d0fe278dcc..024f73ae55 100644 --- a/go/core/internal/controller/grants_test.go +++ b/go/core/internal/controller/grants_test.go @@ -54,8 +54,8 @@ func TestAgentEgressErrorsUseCompilationClassification(t *testing.T) { require.Equal(t, test.reason, condition.Reason) state := collections.Reconciliations.GetKey("team-a/assistant") require.Nil(t, state.Target) - require.Empty(t, state.Grants.Destinations) - require.Empty(t, state.Grants.Credentials) + require.Empty(t, state.EgressGrant.Destinations) + require.Empty(t, state.EgressGrant.Credentials) }) } } @@ -84,7 +84,7 @@ func TestAgentGrantChangesReconcileWithoutRuntimeChange(t *testing.T) { synctest.Wait() backend.inspect(func() { require.Equal(t, initial.Target.RevisionID.String(), store.pair.DesiredRevision) - require.Equal(t, []string{"https://tickets.example.com:443"}, store.pair.Grants.Destinations) + require.Equal(t, []string{"https://tickets.example.com:443"}, store.pair.EgressGrant.Destinations) }) require.Equal(t, []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}, store.sources) store.sources = nil @@ -92,13 +92,16 @@ func TestAgentGrantChangesReconcileWithoutRuntimeChange(t *testing.T) { synctest.Wait() backend.inspect(func() { require.Empty(t, store.pair.DesiredRevision) - require.NotEmpty(t, store.pair.Grants.Destinations) + require.NotEmpty(t, store.pair.EgressGrant.Destinations) }) require.Len(t, store.sources, 1, "every preparation reapplies grants, including unchanged grants") store.sources = nil collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).DeleteObject("team-a/access") synctest.Wait() - backend.inspect(func() { require.Empty(t, store.pair.DesiredRevision); require.Empty(t, store.pair.Grants.Destinations) }) + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.Empty(t, store.pair.EgressGrant.Destinations) + }) require.Len(t, store.sources, 1, "revocation must reconcile even while the runtime remains unresolvable") }) } @@ -142,7 +145,7 @@ func TestAgentGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { require.Contains(t, state.CompilationFailure.Message, missing) backend.inspect(func() { require.Empty(t, store.pair.DesiredRevision) - require.Equal(t, []string{"https://tickets.example.com:443"}, store.pair.Grants.Destinations) + require.Equal(t, []string{"https://tickets.example.com:443"}, store.pair.EgressGrant.Destinations) }) wantSource := database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"} require.Equal(t, []database.RuntimeEgressSource{wantSource}, store.sources) @@ -153,7 +156,7 @@ func TestAgentGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { synctest.Wait() backend.inspect(func() { require.Empty(t, store.pair.DesiredRevision) - require.Equal(t, []string{"https://updated.example.com:443"}, store.pair.Grants.Destinations) + require.Equal(t, []string{"https://updated.example.com:443"}, store.pair.EgressGrant.Destinations) }) require.Equal(t, []database.RuntimeEgressSource{wantSource, wantSource}, store.sources) @@ -161,7 +164,7 @@ func TestAgentGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { synctest.Wait() backend.inspect(func() { require.Empty(t, store.pair.DesiredRevision) - require.Empty(t, store.pair.Grants.Destinations) + require.Empty(t, store.pair.EgressGrant.Destinations) require.Zero(t, store.recordCalls, "grant changes must not require a compiled runtime") }) require.Equal(t, []database.RuntimeEgressSource{wantSource, wantSource, wantSource}, store.sources, "revocation must apply actors while compilation remains broken") @@ -235,12 +238,12 @@ func TestSandboxGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch") != nil }) require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) require.Nil(t, s.collections.states.GetKey("team-a/scratch").DesiredActorTemplate, "missing WorkerPool prevents runtime compilation") - require.Equal(t, []string{"https://tickets.example.com:443"}, store.desired.Grants.Destinations) + require.Equal(t, []string{"https://tickets.example.com:443"}, store.desired.EgressGrant.Destinations) require.Len(t, store.sources, 1) policies.DeleteObject("team-a/access") waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch").CompilationError != "" }) require.Error(t, s.reconcile(t.Context(), "team-a/scratch")) - require.Empty(t, store.desired.Grants.Destinations) + require.Empty(t, store.desired.EgressGrant.Destinations) require.Len(t, store.sources, 2, "revocation must reach existing actors during failed compilation") store.upsertErr = errors.New("response lost") require.ErrorIs(t, s.reconcile(t.Context(), "team-a/scratch"), store.upsertErr) @@ -277,23 +280,23 @@ func TestPreparationReplaysAndRetriesUnchangedGrants(t *testing.T) { collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) harnesses.DeleteObject("team-a/byo") synctest.Wait() - store := &fakeRuntimeRevisionStore{pair: &database.AgentDefinition{Grants: grants}, egressTargetStoreStub: egressTargetStoreStub{targets: []database.RuntimeEgressTarget{target}}} + store := &fakeRuntimeRevisionStore{pair: &database.AgentDefinition{EgressGrant: grants}, egressTargetStoreStub: egressTargetStoreStub{targets: []database.RuntimeEgressTarget{target}}} actors := &fakeActorTemplates{} backend := &preparationTestBackend{templates: actors, store: store} r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3()) start = r.Start - persisted = func() egress.Access { backend.mu.Lock(); defer backend.mu.Unlock(); return store.pair.Grants } + persisted = func() egress.Access { backend.mu.Lock(); defer backend.mu.Unlock(); return store.pair.EgressGrant } actorStub = &actors.actorEgressStub case "sandbox": opts := krt.NewOptionsBuilder(ctx.Done(), "retry-grants", nil) template := sandboxTestTemplate() template.Finalizers = []string{sandboxPreparationFinalizer} - states := krt.NewStaticCollection(nil, []sandboxReconciliation{{Template: template, Grants: grants}}, opts.WithName("states")...) - store := &sandboxTestStore{desired: database.SandboxTemplateDefinition{Grants: grants}, egressTargetStoreStub: egressTargetStoreStub{targets: []database.RuntimeEgressTarget{target}}} + states := krt.NewStaticCollection(nil, []sandboxReconciliation{{Template: template, EgressGrant: grants}}, opts.WithName("states")...) + store := &sandboxTestStore{desired: database.SandboxTemplateDefinition{EgressGrant: grants}, egressTargetStoreStub: egressTargetStoreStub{targets: []database.RuntimeEgressTarget{target}}} actors := &sandboxTestActors{} s := &SandboxReconciler{collections: sandboxCollections{states: states, observations: krt.NewStaticCollection[sandboxRuntimeObservation](nil, nil, opts.WithName("observations")...)}, store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3()} start = s.Start - persisted = func() egress.Access { store.mu.Lock(); defer store.mu.Unlock(); return store.desired.Grants } + persisted = func() egress.Access { store.mu.Lock(); defer store.mu.Unlock(); return store.desired.EgressGrant } actorStub = &actors.actorEgressStub } var mu sync.Mutex diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index c89251a937..a746f585ba 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -43,7 +43,7 @@ const agentPreparationFinalizer = "kagent.dev/agent-preparation" // status instead of disappearing from the graph. type AgentReconciliation struct { Agent *kagentv1alpha3.Agent - Grants egress.Access + EgressGrant egress.Access Target *compiledTarget Warnings []string CompilationFailure *ReconciliationFailure @@ -115,7 +115,7 @@ func newAgentReconciliations( state.CompilationFailure = compilationFailure(grantErr) return state } - state.Grants = grants + state.EgressGrant = grants compilation, err := translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings), @@ -249,7 +249,7 @@ func newReconciler( // and status writes do not. Grants must reconcile without a new revision. if event.Old != nil && event.New != nil && event.Old.Agent.UID == event.New.Agent.UID && slices.Equal(event.Old.Agent.Finalizers, event.New.Agent.Finalizers) && event.Old.desiredRevision() == event.New.desiredRevision() && - reflect.DeepEqual(event.Old.Grants, event.New.Grants) && event.Old.Agent.DeletionTimestamp.Equal(event.New.Agent.DeletionTimestamp) { + reflect.DeepEqual(event.Old.EgressGrant, event.New.EgressGrant) && event.Old.Agent.DeletionTimestamp.Equal(event.New.Agent.DeletionTimestamp) { return } r.agents.Add(krt.GetKey(event.Latest())) @@ -359,7 +359,7 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { } definition := database.AgentDefinition{ Namespace: state.Agent.Namespace, AgentName: state.Agent.Name, - AgentUID: string(state.Agent.UID), DesiredRevision: state.desiredRevision(), Grants: state.Grants, + AgentUID: string(state.Agent.UID), DesiredRevision: state.desiredRevision(), EgressGrant: state.EgressGrant, } // Store the desired edge before creating compute so a concurrent collector diff --git a/go/core/internal/controller/sandbox_collections.go b/go/core/internal/controller/sandbox_collections.go index 656e6c008c..f0ec4144ac 100644 --- a/go/core/internal/controller/sandbox_collections.go +++ b/go/core/internal/controller/sandbox_collections.go @@ -26,7 +26,7 @@ type sandboxReconciliation struct { Template *kagentv1alpha3.SandboxTemplate RevisionID string SourceSnapshot json.RawMessage - Grants egress.Access + EgressGrant egress.Access DesiredActorTemplate *ateapipb.ActorTemplate ObservedActorTemplate *ateapipb.ActorTemplate CompilationError string @@ -93,7 +93,7 @@ func newSandboxCollections(inputs Collections, policy substrate.SandboxPolicy, o state.Failure = &ReconciliationFailure{Reason: kagentv1alpha3.SandboxTemplateReasonEgressPolicyResolutionFailed, Message: err.Error()} return state } - state.Grants = grants + state.EgressGrant = grants pool := krt.FetchOne(ctx, inputs.WorkerPools, krt.FilterKey(template.Namespace+"/"+template.Spec.Substrate.WorkerPoolRef.Name)) if pool == nil { state.Failure = &ReconciliationFailure{Reason: "WorkerPoolNotFound", Message: "The referenced sandbox WorkerPool does not exist"} diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index 27b428dbee..e68b0eb3e4 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -140,7 +140,7 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { return nil } err := s.store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{ - Namespace: state.Template.Namespace, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), DesiredRevision: state.desiredRevision(), Grants: state.Grants, + Namespace: state.Template.Namespace, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), DesiredRevision: state.desiredRevision(), EgressGrant: state.EgressGrant, }) if err != nil { return s.observeError(*state, fmt.Errorf("store SandboxTemplate %s definition: %w", key, err), true) diff --git a/go/core/internal/database/models.go b/go/core/internal/database/models.go index 2aa500367c..4702c64372 100644 --- a/go/core/internal/database/models.go +++ b/go/core/internal/database/models.go @@ -51,10 +51,10 @@ type AgentMemorySearchResult struct { // AgentDefinition tracks the desired and last successful runtime for an Agent UID. type AgentDefinition struct { - Namespace string - AgentName string - AgentUID string - Grants egress.Access + Namespace string + AgentName string + AgentUID string + EgressGrant egress.Access // DesiredRevision is a compiled runtime digest, or empty while inputs are unresolved. DesiredRevision string } diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index 1f7751ae79..9a15ca66cf 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -135,7 +135,7 @@ func TestRuntimeEgressTokenOriginsAreNotGrants(t *testing.T) { func TestRuntimeEgressCanonicalTokenOriginWithholdsCredentials(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() - err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", Grants: egress.Access{ + err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", EgressGrant: egress.Access{ Destinations: []string{"https://model.example.com"}, Credentials: []egress.Credential{{Origin: "https://model.example.com", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"}}, }}) @@ -159,9 +159,9 @@ func TestSourceGrantsDetermineEffectivePolicy(t *testing.T) { ctx := t.Context() upsert := func(uid string, grants egress.Access) error { if kind == RuntimeEgressAgent { - return client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: uid, Grants: grants}) + return client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: uid, EgressGrant: grants}) } - return client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team", SandboxTemplateName: "source", SandboxTemplateUID: uid, Grants: grants}) + return client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team", SandboxTemplateName: "source", SandboxTemplateUID: uid, EgressGrant: grants}) } base := egress.Access{} if kind == RuntimeEgressAgent { @@ -228,13 +228,13 @@ func TestSourceGrantsRollbackWithInvalidRevision(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() grants := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} - definition := AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", Grants: grants} + definition := AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", EgressGrant: grants} err := client.UpsertAgentDefinition(ctx, definition) require.NoError(t, err) require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "agent", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "agent"}, RecordRevisionOptions{})) require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "sandbox", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "sandbox"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "sandbox", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{})) for _, uid := range []string{"uid", "replacement"} { - definition.AgentUID, definition.DesiredRevision, definition.Grants = uid, "sandbox", egress.Access{} + definition.AgentUID, definition.DesiredRevision, definition.EgressGrant = uid, "sandbox", egress.Access{} err = client.UpsertAgentDefinition(ctx, definition) require.ErrorIs(t, err, ErrConflict) policy, err := client.GetRuntimeEgressPolicy(ctx, "agent") diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index 915082cb82..822b3472a6 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -20,7 +20,7 @@ import ( // at the same names and rejects revisions whose deletion has started. Grants // are canonicalized before writing; errors may include an uncertain commit outcome. func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefinition) error { - grants, err := canonicalStoredGrants(definition.Grants) + grants, err := canonicalStoredGrants(definition.EgressGrant) if err != nil { return fmt.Errorf("validate source grants: %w", err) } diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index dc7254fb9a..ec08e0df01 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -22,7 +22,7 @@ type SandboxTemplateDefinition struct { Namespace string SandboxTemplateName string SandboxTemplateUID string - Grants egress.Access + EgressGrant egress.Access DesiredRevision string } @@ -30,7 +30,7 @@ type SandboxTemplateDefinition struct { // Kubernetes UID retires the former identity without inheriting its last success. // Grants are canonicalized before writing; errors may include an uncertain commit outcome. func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition SandboxTemplateDefinition) error { - grants, err := canonicalStoredGrants(definition.Grants) + grants, err := canonicalStoredGrants(definition.EgressGrant) if err != nil { return fmt.Errorf("validate source grants: %w", err) } From 483e85e3c779026a3b40b176ef1f3092b6f07d0d Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 20:22:15 +0000 Subject: [PATCH 31/43] refactor: share harness skills paths with e2e tests Signed-off-by: Eitan Yarmush --- go/adk/cmd/main.go | 3 +-- go/adk/pkg/config/agentplugins.go | 3 +++ go/core/test/e2e/harness_test.go | 9 ++++++--- go/harness/claude/config/paths.go | 8 ++++++++ go/harness/claude/internal/adapter/adapter.go | 2 +- go/harness/codex/config/paths.go | 8 ++++++++ go/harness/codex/internal/adapter/adapter.go | 7 ++++--- 7 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 go/harness/claude/config/paths.go create mode 100644 go/harness/codex/config/paths.go diff --git a/go/adk/cmd/main.go b/go/adk/cmd/main.go index 1a18a9b0cc..bfa5d6b018 100644 --- a/go/adk/cmd/main.go +++ b/go/adk/cmd/main.go @@ -28,7 +28,6 @@ import ( const ( defaultPluginPackagesRoot = "/plugins" - defaultSkillsRoot = "/skills" defaultPluginDataRoot = "/data/plugins" ) @@ -73,7 +72,7 @@ func run(logLevel, host, port, configDir string) error { logging.IntoContext(context.Background(), logger), agentConfig, config.AgentPluginPaths{ Packages: defaultPluginPackagesRoot, - Skills: defaultSkillsRoot, + Skills: config.DefaultSkillsRoot, Data: defaultPluginDataRoot, }, ); err != nil { diff --git a/go/adk/pkg/config/agentplugins.go b/go/adk/pkg/config/agentplugins.go index db0ab56f01..f4bf1adb19 100644 --- a/go/adk/pkg/config/agentplugins.go +++ b/go/adk/pkg/config/agentplugins.go @@ -9,6 +9,9 @@ import ( "github.com/kagent-dev/kagent/go/core/pkg/agentplugins" ) +// DefaultSkillsRoot is where the ADK installs root-agent skills. +const DefaultSkillsRoot = "/skills" + // AgentPluginPaths contains the ADK runtime destinations for Agent Plugin // packages, selected skills, and mutable MCP server data. type AgentPluginPaths struct { diff --git a/go/core/test/e2e/harness_test.go b/go/core/test/e2e/harness_test.go index a66010e5cc..cb45e61592 100644 --- a/go/core/test/e2e/harness_test.go +++ b/go/core/test/e2e/harness_test.go @@ -6,7 +6,10 @@ import ( ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + adkconfig "github.com/kagent-dev/kagent/go/adk/pkg/config" "github.com/kagent-dev/kagent/go/api/v1alpha3" + claudeconfig "github.com/kagent-dev/kagent/go/harness/claude/config" + codexconfig "github.com/kagent-dev/kagent/go/harness/codex/config" ) type testHarness struct { @@ -60,10 +63,10 @@ func (h testHarness) createModel(t *testing.T, kube ctrlclient.Client, modelURL func (h testHarness) skillsRoot() string { switch h.name { case codexE2EHarness: - return "/data/codex/skills" + return codexconfig.SkillsRoot("/data") case claudeE2EHarness: - return "/data/generated/claude/.claude/skills" + return claudeconfig.SkillsRoot("/data") default: - return "/skills" + return adkconfig.DefaultSkillsRoot } } diff --git a/go/harness/claude/config/paths.go b/go/harness/claude/config/paths.go new file mode 100644 index 0000000000..2b1953ba29 --- /dev/null +++ b/go/harness/claude/config/paths.go @@ -0,0 +1,8 @@ +package config + +import "path/filepath" + +// SkillsRoot is the directory of selected skills in the generated Claude plugin. +func SkillsRoot(dataDir string) string { + return filepath.Join(dataDir, "generated", "claude", ".claude", "skills") +} diff --git a/go/harness/claude/internal/adapter/adapter.go b/go/harness/claude/internal/adapter/adapter.go index b6067d9803..1945cee735 100644 --- a/go/harness/claude/internal/adapter/adapter.go +++ b/go/harness/claude/internal/adapter/adapter.go @@ -59,7 +59,7 @@ func New(ctx context.Context, input Input) (*driver.ProcessDriver, error) { } var pluginDirs []string if cfg.SkillResources != nil { - skillsDir := filepath.Join(skillRoot, ".claude", "skills") + skillsDir := config.SkillsRoot(input.DurableDir) if err := utils.EnsurePrivateDir(skillsDir); err != nil { return nil, fmt.Errorf("prepare generated Claude skills directory: %w", err) } diff --git a/go/harness/codex/config/paths.go b/go/harness/codex/config/paths.go new file mode 100644 index 0000000000..0b76ba467a --- /dev/null +++ b/go/harness/codex/config/paths.go @@ -0,0 +1,8 @@ +package config + +import "path/filepath" + +// SkillsRoot is the directory where Codex discovers installed skills. +func SkillsRoot(dataDir string) string { + return filepath.Join(dataDir, "codex", "skills") +} diff --git a/go/harness/codex/internal/adapter/adapter.go b/go/harness/codex/internal/adapter/adapter.go index 1552695aa5..d92bb6ca02 100644 --- a/go/harness/codex/internal/adapter/adapter.go +++ b/go/harness/codex/internal/adapter/adapter.go @@ -41,7 +41,8 @@ func New(ctx context.Context, input Input) (*driver.ProcessDriver, error) { return nil, fmt.Errorf("workspace and durable directories must be absolute paths") } codexHome := filepath.Join(input.DurableDir, "codex") - for _, directory := range []string{input.Workspace, codexHome, filepath.Join(codexHome, "agents"), filepath.Join(codexHome, "skills")} { + skillsRoot := config.SkillsRoot(input.DurableDir) + for _, directory := range []string{input.Workspace, codexHome, filepath.Join(codexHome, "agents"), skillsRoot} { if err := utils.EnsurePrivateDir(directory); err != nil { return nil, fmt.Errorf("prepare Codex directory %q: %w", directory, err) } @@ -49,13 +50,13 @@ func New(ctx context.Context, input Input) (*driver.ProcessDriver, error) { if err := reconcileGeneratedDir(filepath.Join(codexHome, "agents"), agentFileNames(cfg.Agents)); err != nil { return nil, fmt.Errorf("reconcile Codex agents: %w", err) } - if err := reconcileGeneratedDir(filepath.Join(codexHome, "skills"), nil); err != nil { + if err := reconcileGeneratedDir(skillsRoot, nil); err != nil { return nil, fmt.Errorf("reconcile Codex skills: %w", err) } if cfg.SkillResources != nil { materialized, err := agentplugins.Materialize(ctx, *cfg.SkillResources, agentplugins.Paths{ Packages: filepath.Join(codexHome, "packages"), - Skills: filepath.Join(codexHome, "skills"), + Skills: skillsRoot, }) if err != nil { return nil, fmt.Errorf("materialize Codex skills: %w", err) From 029c8a9881c04b95e563daa100bfdf2cc7e491f9 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 20:42:23 +0000 Subject: [PATCH 32/43] fix: simplify egress ownership and allow HTTP credentials Recover offline Agent deletions through preparation instead of a finalizer. Hash compiled runtime inputs while retaining full source provenance, and keep source actor queries inline in the store. Allow credentials on exact HTTP origins and exercise delivery with a plain HTTP origin, removing shared gateway configuration changes from the e2e fixture. Signed-off-by: Eitan Yarmush --- docs/architecture/egress-policies.md | 16 +- go/api/v1alpha3/egresspolicy_cel_test.go | 3 +- go/api/v1alpha3/egresspolicy_types.go | 2 +- go/core/internal/controller/collections.go | 3 + .../internal/controller/collections_test.go | 2 +- go/core/internal/controller/grants_test.go | 94 ++++--- .../internal/controller/preparation_test.go | 13 +- go/core/internal/controller/reconciler.go | 59 +++-- .../internal/controller/reconciler_test.go | 29 ++- go/core/internal/database/runtime_egress.go | 37 ++- .../internal/database/runtime_egress_test.go | 7 + .../internal/database/runtime_revisions.go | 11 + .../database/runtime_revisions_test.go | 19 ++ go/core/internal/substrate/sandbox.go | 28 +- go/core/internal/substrate/sandbox_test.go | 6 +- go/core/internal/translator/compiler.go | 4 +- go/core/internal/translator/compiler_test.go | 8 + go/core/internal/translator/egress.go | 4 +- go/core/internal/translator/egress_test.go | 16 +- go/core/internal/translator/revision.go | 7 +- go/core/internal/translator/revision_test.go | 6 +- .../pkg/migrations/core/000001_initial.sql | 16 -- go/core/test/e2e/egress_credentials_test.go | 242 +----------------- .../api.kagent.dev_egresspolicies.yaml | 5 +- 24 files changed, 264 insertions(+), 373 deletions(-) diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index 0b4d9faf81..a209913fa9 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -81,7 +81,9 @@ access. A standalone sandbox with no references has no allowed egress. ## Call an API with a credential Policies authenticate requests using a header, an optional prefix, and a Secret -in the same namespace. Authentication requires an exact HTTPS origin: +in the same namespace. Authentication requires an exact HTTP(S) origin. HTTP +supports in-cluster services whose transport is secured by a service mesh; the gateway injects the +credential without exposing it to the guest: ```yaml apiVersion: api.kagent.dev/v1alpha3 @@ -158,7 +160,10 @@ including credential references and caller-token passthrough constraints. The combined additional access from a source's referenced EgressPolicies is its `EgressGrant`: destinations it may contact and credentials the gateway may inject. Reconciliation and source definition structs carry it as `EgressGrant egress.Access`. -Explicit policy grants and policy references do not enter revision identity. +Revision identity covers compiled runtime inputs and the owning Kubernetes UID. +The full source specification is preserved as provenance for the first preparation +of that revision. Explicit policy grants and policy references do not enter +revision identity. Changing a grant or its Secret reference does not prepare a new runtime image. Secret values remain outside compiled inputs; the gateway handles value rotation. @@ -185,8 +190,11 @@ source with exponential backoff even when grants have not changed. Slow actor updates therefore delay further work on that preparation queue. Startup replays every Agent and SandboxTemplate through preparation; no separate actor sweep is needed. -Preparation finalizers preserve deletion through controller downtime and are -removed only after retiring the source and successfully updating its actors. +After its input cache syncs, Agent preparation also enqueues stored definitions +whose Agents are absent from watched namespaces. This includes retired identities +so a restart retries revocations interrupted after retirement. Agent deletion +does not need a finalizer. SandboxTemplate preparation retains its existing +finalizer until retirement and actor updates succeed. Running and suspended actors receive updates without replacement or a new invocation. Missing or invalid policy references and Secrets revoke explicit diff --git a/go/api/v1alpha3/egresspolicy_cel_test.go b/go/api/v1alpha3/egresspolicy_cel_test.go index 8b5a9e54c0..d8f0fa0bd4 100644 --- a/go/api/v1alpha3/egresspolicy_cel_test.go +++ b/go/api/v1alpha3/egresspolicy_cel_test.go @@ -28,7 +28,8 @@ func testEgressPolicyValidation(t *testing.T, cl ctrlclient.Client, namespace st {"API key", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "X-API-Key", SecretKeyRef: secret}}, false}, {"empty prefix", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "X-API-Key", Prefix: new(""), SecretKeyRef: secret}}, false}, {"custom prefix", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "Authorization", Prefix: new("Token "), SecretKeyRef: secret}}, false}, - {"HTTP auth", EgressDestination{Origin: "http://api.example.com", Authentication: bearer}, true}, + {"HTTP auth", EgressDestination{Origin: "http://api.example.com", Authentication: bearer}, false}, + {"HTTP wildcard auth", EgressDestination{Origin: "http://*.example.com", Authentication: bearer}, true}, {"wildcard auth", EgressDestination{Origin: "https://*.example.com", Authentication: bearer}, true}, {"empty auth", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{}}, true}, {"missing header", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{SecretKeyRef: secret}}, true}, diff --git a/go/api/v1alpha3/egresspolicy_types.go b/go/api/v1alpha3/egresspolicy_types.go index 2cd00bf4a4..9fc67a54d6 100644 --- a/go/api/v1alpha3/egresspolicy_types.go +++ b/go/api/v1alpha3/egresspolicy_types.go @@ -21,7 +21,7 @@ type EgressPolicySpec struct { } // EgressDestination grants access to one HTTP(S) origin, including all its paths. -// +kubebuilder:validation:XValidation:rule="!has(self.authentication) || (self.origin.startsWith('https://') && !self.origin.contains('*'))",message="authentication requires an exact HTTPS origin" +// +kubebuilder:validation:XValidation:rule="!has(self.authentication) || !self.origin.contains('*')",message="authentication requires an exact HTTP(S) origin" type EgressDestination struct { // Origin is a scheme, DNS hostname, and optional port. A wildcard replaces // exactly one leftmost DNS label, for example https://*.example.com. diff --git a/go/core/internal/controller/collections.go b/go/core/internal/controller/collections.go index 068cecbcc3..b7c9dbe33c 100644 --- a/go/core/internal/controller/collections.go +++ b/go/core/internal/controller/collections.go @@ -2,6 +2,7 @@ package controller import ( "reflect" + "slices" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -19,6 +20,7 @@ import ( // Collections contains shared Kubernetes inputs and compiled Agent state. type Collections struct { + watchNamespaces []string EgressPolicies krt.Collection[*kagentv1alpha3.EgressPolicy] SandboxTemplates krt.Collection[*kagentv1alpha3.SandboxTemplate] Agents krt.Collection[*kagentv1alpha3.Agent] @@ -85,6 +87,7 @@ func NewCollections(client kube.Client, watchNamespaces []string, opts krt.Optio statuses := newAgentStatuses(agents, reconciliations, opts) return Collections{ + watchNamespaces: slices.Clone(watchNamespaces), EgressPolicies: egressPolicies, SandboxTemplates: sandboxTemplates, Agents: agents, diff --git a/go/core/internal/controller/collections_test.go b/go/core/internal/controller/collections_test.go index af11b66679..8e8ab3a99d 100644 --- a/go/core/internal/controller/collections_test.go +++ b/go/core/internal/controller/collections_test.go @@ -525,7 +525,7 @@ func waitFor(t *testing.T, condition func() bool) { } func testAgent(template *kagentv1alpha3.AgentTemplate, harness *kagentv1alpha3.Harness) *kagentv1alpha3.Agent { - return &kagentv1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: template.Name, UID: template.UID, Finalizers: []string{agentPreparationFinalizer}}, Spec: kagentv1alpha3.AgentSpec{ + return &kagentv1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: template.Name, UID: template.UID}, Spec: kagentv1alpha3.AgentSpec{ Template: kagentv1alpha3.AgentTemplateSource{Ref: &corev1.LocalObjectReference{Name: template.Name}}, HarnessRef: corev1.LocalObjectReference{Name: harness.Name}, }} diff --git a/go/core/internal/controller/grants_test.go b/go/core/internal/controller/grants_test.go index 024f73ae55..9c2d6bfcce 100644 --- a/go/core/internal/controller/grants_test.go +++ b/go/core/internal/controller/grants_test.go @@ -3,6 +3,7 @@ package controller import ( "context" "errors" + "fmt" "sync" "testing" "testing/synctest" @@ -67,14 +68,15 @@ func TestAgentGrantChangesReconcileWithoutRuntimeChange(t *testing.T) { template := proto.CloneOf(initial.Target.ActorTemplate) template.Metadata.Uid = "prepared" template.Status = sandboxGoldenStatus() - store := &fakeRuntimeRevisionStore{} + store := &fakeRuntimeRevisionStore{definitionSources: []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}} backend := &preparationTestBackend{templates: &fakeActorTemplates{template: template}, store: store} r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3()) ctx, cancel := context.WithCancel(t.Context()) var workers sync.WaitGroup - workers.Go(func() { r.Run(ctx.Done()) }) + workers.Go(func() { assert.NoError(t, r.Start(ctx)) }) defer func() { cancel(); workers.Wait() }() synctest.Wait() + require.Empty(t, store.retired, "startup must preserve a matching live Agent") store.sources = nil policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(policy) @@ -130,7 +132,7 @@ func TestAgentGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3()) ctx, cancel := context.WithCancel(t.Context()) var workers sync.WaitGroup - workers.Go(func() { r.Run(ctx.Done()) }) + workers.Go(func() { assert.NoError(t, r.Start(ctx)) }) defer func() { cancel(); workers.Wait() }() synctest.Wait() store.sources = nil @@ -186,42 +188,58 @@ func TestAgentGrantWriteFailureAndRetirementApplyActors(t *testing.T) { require.Equal(t, []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}, store.sources) } -func TestAgentFinalizerRetiresGrantsAfterOfflineDeletion(t *testing.T) { +func TestAgentStartupRecoversOfflineDeletions(t *testing.T) { + for _, namespaces := range [][]string{nil, {"team-a"}} { + t.Run(fmt.Sprint(namespaces), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + collections, _ := newPreparationTestCollections(t, "gvisor") + collections.watchNamespaces = namespaces + // Delete before registering handlers: no KRT event can recover it. + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).DeleteObject("team-a/assistant") + synctest.Wait() + sources := []database.RuntimeEgressSource{ + {Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, + {Kind: database.RuntimeEgressAgent, Namespace: "team-b", Name: "assistant"}, + } + store := &fakeRuntimeRevisionStore{definitionSources: sources, egressTargetStoreStub: egressTargetStoreStub{ + targets: []database.RuntimeEgressTarget{{ID: "actor", Revision: "old", Atespace: "team-a", TemplateName: "runtime"}}, + }} + attempts := 0 + actors := &fakeActorTemplates{actorEgressStub: actorEgressStub{ensurePolicy: func(context.Context, string, string, string) (egress.Policy, error) { + assert.NotEmpty(t, store.retired, "retire before applying revocation") + attempts++ + if attempts == 1 { + return egress.Policy{}, errors.New("backend unavailable") + } + return egress.Policy{}, nil + }}} + r := newReconciler(collections, actors, store, kagentfake.NewSimpleClientset().ApiV1alpha3()) + var workers sync.WaitGroup + workers.Go(func() { assert.NoError(t, r.Start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + time.Sleep(time.Second) + synctest.Wait() + want := []database.RuntimeEgressSource{sources[0], sources[0]} + if len(namespaces) == 0 { + want = append(want, sources[1]) + } + require.ElementsMatch(t, want, store.sources, "recovery must honor namespace scope and retry revocation") + require.Equal(t, len(want), attempts) + require.Zero(t, store.pairCalls, "missing Agents must not be recreated") + }) + }) + } +} + +func TestAgentStartupRecoveryReadFailure(t *testing.T) { collections, _ := newPreparationTestCollections(t, "gvisor") - agent := collections.Agents.List()[0].DeepCopy() - agent.Finalizers = nil - agents := collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]) - agents.UpdateObject(agent) - waitFor(t, func() bool { return len(collections.Reconciliations.GetKey("team-a/assistant").Agent.Finalizers) == 0 }) - client := kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3() - store := &fakeRuntimeRevisionStore{egressTargetStoreStub: egressTargetStoreStub{targets: []database.RuntimeEgressTarget{{ID: "actor", Revision: "old", Atespace: "team-a", TemplateName: "runtime"}}}} - pushErr := errors.New("backend unavailable") - actors := &fakeActorTemplates{actorEgressStub: actorEgressStub{ensurePolicy: func(context.Context, string, string, string) (egress.Policy, error) { - assert.NotEmpty(t, store.retired, "retire before applying revocation") - return egress.Policy{}, pushErr - }}} - r := &Reconciler{collections: collections, store: store, templates: actors, status: client} - require.NoError(t, r.reconcileAgent(t.Context(), "team-a/assistant")) - require.Zero(t, store.pairCalls, "deletion ownership must be durable before grants") - updated, err := client.Agents("team-a").Get(t.Context(), "assistant", metav1.GetOptions{}) - require.NoError(t, err) - require.Contains(t, updated.Finalizers, agentPreparationFinalizer) - updated.DeletionTimestamp = &metav1.Time{Time: metav1.Now().Time} - agents.UpdateObject(updated) - waitFor(t, func() bool { - return !collections.Reconciliations.GetKey("team-a/assistant").Agent.DeletionTimestamp.IsZero() - }) - require.ErrorIs(t, r.reconcileAgent(t.Context(), "team-a/assistant"), pushErr) - persisted, err := client.Agents("team-a").Get(t.Context(), "assistant", metav1.GetOptions{}) - require.NoError(t, err) - require.Contains(t, persisted.Finalizers, agentPreparationFinalizer, "failed revocation must retain deletion ownership") - pushErr = nil - require.NoError(t, r.reconcileAgent(t.Context(), "team-a/assistant")) - require.NotEmpty(t, store.retired) - require.Len(t, store.sources, 2) - persisted, err = client.Agents("team-a").Get(t.Context(), "assistant", metav1.GetOptions{}) - require.NoError(t, err) - require.NotContains(t, persisted.Finalizers, agentPreparationFinalizer) + store := &fakeRuntimeRevisionStore{sourcesErr: errors.New("database unavailable")} + r := newReconciler(collections, &fakeActorTemplates{}, store, kagentfake.NewSimpleClientset().ApiV1alpha3()) + require.ErrorIs(t, r.Start(t.Context()), store.sourcesErr) + require.Zero(t, store.pairCalls, "failed recovery must fail startup") } func TestSandboxGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index 2ff25e48ba..a09d4336f9 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -12,6 +12,7 @@ import ( atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -282,7 +283,7 @@ func TestPreparationQueueAndPollerBackoff(t *testing.T) { r := newReconciler(collections, backend, backend, statusClient) ctx, cancel := context.WithCancel(t.Context()) defer cancel() - go r.Run(ctx.Done()) + go func() { assert.NoError(t, r.Start(ctx)) }() synctest.Wait() backend.inspect(func() { require.Equal(t, 1, test.store.pairCalls, "publishing a failure must not immediately requeue it") @@ -382,6 +383,12 @@ func (p *preparationTestBackend) CreateActorTemplate(ctx context.Context, templa return p.templates.CreateActorTemplate(ctx, template) } +func (p *preparationTestBackend) ListAgentDefinitionSources(ctx context.Context) ([]database.RuntimeEgressSource, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.store.ListAgentDefinitionSources(ctx) +} + func (p *preparationTestBackend) UpsertAgentDefinition(ctx context.Context, definition database.AgentDefinition) error { p.mu.Lock() defer p.mu.Unlock() @@ -409,7 +416,7 @@ func TestPreparationDesiredChangesBypassBackoff(t *testing.T) { r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3()) ctx, cancel := context.WithCancel(t.Context()) defer cancel() - go r.Run(ctx.Done()) + go func() { assert.NoError(t, r.Start(ctx)) }() synctest.Wait() time.Sleep(10 * time.Second) synctest.Wait() @@ -463,7 +470,7 @@ func TestPreparationTerminalFailures(t *testing.T) { r := newReconciler(collections, templates, store, statusClient) ctx, cancel := context.WithCancel(t.Context()) defer cancel() - go r.Run(ctx.Done()) + go func() { assert.NoError(t, r.Start(ctx)) }() synctest.Wait() require.Equal(t, 1, store.pairCalls) state := collections.Reconciliations.GetKey(initial.ResourceName()) diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index a746f585ba..f9a4dced33 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -36,8 +36,6 @@ import ( kagenttranslator "github.com/kagent-dev/kagent/go/core/internal/translator/kagent" ) -const agentPreparationFinalizer = "kagent.dev/agent-preparation" - // AgentReconciliation is the complete desired and observed state for one // Agent. Compilation failures are data so invalid Agents still produce // status instead of disappearing from the graph. @@ -186,6 +184,7 @@ type runtimeRevisionStore interface { actorEgressStore UpsertAgentDefinition(context.Context, database.AgentDefinition) error RecordRuntimeRevision(context.Context, database.RuntimeRevision, database.RecordRevisionOptions) error + ListAgentDefinitionSources(context.Context) ([]database.RuntimeEgressSource, error) RetireAgentIdentities(ctx context.Context, namespace, name string) error } @@ -248,7 +247,7 @@ func newReconciler( // Desired inputs and cleanup changes bypass failure backoff; observations // and status writes do not. Grants must reconcile without a new revision. if event.Old != nil && event.New != nil && event.Old.Agent.UID == event.New.Agent.UID && - slices.Equal(event.Old.Agent.Finalizers, event.New.Agent.Finalizers) && event.Old.desiredRevision() == event.New.desiredRevision() && + event.Old.desiredRevision() == event.New.desiredRevision() && reflect.DeepEqual(event.Old.EgressGrant, event.New.EgressGrant) && event.Old.Agent.DeletionTimestamp.Equal(event.New.Agent.DeletionTimestamp) { return } @@ -281,19 +280,44 @@ func newReconciliationQueue(name string, reconcile func(any) error) controllers. ) } -// Run waits for the graph boundary to observe initial state, then processes -// Agent and status writes until stop closes. -func (r *Reconciler) Run(stop <-chan struct{}) { +// Start waits for initial Kubernetes state, recovers offline deletions, then +// processes Agent preparation and status writes until cancellation. +func (r *Reconciler) Start(ctx context.Context) error { + stop := ctx.Done() if !r.agentHandler.WaitUntilSynced(stop) || !r.agentStatusHandler.WaitUntilSynced(stop) || !r.modelConfigStatusHandler.WaitUntilSynced(stop) { r.agents.ShutDownEarly() r.agentStatuses.ShutDownEarly() r.modelConfigStatuses.ShutDownEarly() - return + return nil + } + if err := r.enqueueDeletedAgents(ctx); err != nil { + r.agents.ShutDownEarly() + r.agentStatuses.ShutDownEarly() + r.modelConfigStatuses.ShutDownEarly() + return err } go r.pollPendingTemplates(stop) go r.agentStatuses.Run(stop) go r.modelConfigStatuses.Run(stop) r.agents.Run(stop) + return nil +} + +func (r *Reconciler) enqueueDeletedAgents(ctx context.Context) error { + sources, err := r.store.ListAgentDefinitionSources(ctx) + if err != nil { + return fmt.Errorf("recover deleted Agents: %w", err) + } + for _, source := range sources { + if len(r.collections.watchNamespaces) != 0 && !slices.Contains(r.collections.watchNamespaces, source.Namespace) { + continue + } + key := source.Namespace + "/" + source.Name + if r.collections.Agents.GetKey(key) == nil { + r.agents.Add(key) + } + } + return nil } func (r *Reconciler) pollPendingTemplates(stop <-chan struct{}) { @@ -314,11 +338,6 @@ func (r *Reconciler) pollPendingTemplates(stop <-chan struct{}) { } } -func (r *Reconciler) Start(ctx context.Context) error { - r.Run(ctx.Done()) - return nil -} - func (r *Reconciler) NeedLeaderElection() bool { return true } func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { @@ -339,22 +358,6 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { if err := applySourceEgress(ctx, r.store, r.templates, database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: parts[0], Name: parts[1]}); err != nil { return err } - if state != nil && slices.Contains(state.Agent.Finalizers, agentPreparationFinalizer) { - updated := state.Agent.DeepCopy() - updated.Finalizers = slices.DeleteFunc(updated.Finalizers, func(value string) bool { return value == agentPreparationFinalizer }) - if _, err := r.status.Agents(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("remove Agent %s finalizer: %w", key, err) - } - } - return nil - } - // Retain deletion events through controller downtime before persisting grants. - if !slices.Contains(state.Agent.Finalizers, agentPreparationFinalizer) { - updated := state.Agent.DeepCopy() - updated.Finalizers = append(updated.Finalizers, agentPreparationFinalizer) - if _, err := r.status.Agents(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("add Agent %s finalizer: %w", key, err) - } return nil } definition := database.AgentDefinition{ diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index babc835a89..d812ef28ee 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -11,6 +11,7 @@ import ( a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -33,7 +34,7 @@ func TestReconcilerPersistsPairInOrder(t *testing.T) { stop := make(chan struct{}) t.Cleanup(func() { close(stop) }) opts := krt.NewOptionsBuilder(stop, "test", nil) - template := &kagentv1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Finalizers: []string{agentPreparationFinalizer}, Namespace: "team-a", Name: "assistant", UID: "template-uid"}} + template := &kagentv1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "assistant", UID: "template-uid"}} desiredActor := &ateapipb.ActorTemplate{Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "assistant-kagent-revision"}} revision := &translator.Revision{AgentCard: &a2apb.AgentCard{Name: "assistant"}} revision.AgentCard.ProtoReflect().SetUnknown(protowire.AppendString(protowire.AppendTag(nil, 1000, protowire.BytesType), "future")) @@ -177,7 +178,7 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden"}, }} state := AgentReconciliation{ - Agent: &kagentv1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Finalizers: []string{agentPreparationFinalizer}, Namespace: "team-a", Name: "assistant", UID: "agent-uid"}}, + Agent: &kagentv1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "assistant", UID: "agent-uid"}}, Target: &compiledTarget{Revision: *revision, RevisionID: id, ActorTemplate: desired}, } states.UpdateObject(state) @@ -321,14 +322,20 @@ func (f *fakeActorTemplates) DeleteActorTemplate(context.Context, string, string type fakeRuntimeRevisionStore struct { egressTargetStoreStub - pair *database.AgentDefinition - revision *database.RuntimeRevision - markedSuccessful bool - retired string - revisionErr error - recordCalls int - pairErr error - pairCalls int + pair *database.AgentDefinition + revision *database.RuntimeRevision + markedSuccessful bool + retired string + revisionErr error + recordCalls int + pairErr error + pairCalls int + definitionSources []database.RuntimeEgressSource + sourcesErr error +} + +func (s *fakeRuntimeRevisionStore) ListAgentDefinitionSources(context.Context) ([]database.RuntimeEgressSource, error) { + return s.definitionSources, s.sourcesErr } func (s *fakeRuntimeRevisionStore) UpsertAgentDefinition(_ context.Context, pair database.AgentDefinition) error { @@ -397,7 +404,7 @@ func TestReconcilerUpdatesModelConfigStatusOnSecretHashChange(t *testing.T) { statusClient, ) - go reconciler.Run(stop) + go func() { assert.NoError(t, reconciler.Start(t.Context())) }() var initialUpdate *kagentv1alpha3.ModelConfig var err error diff --git a/go/core/internal/database/runtime_egress.go b/go/core/internal/database/runtime_egress.go index 349f71607a..00c70b6050 100644 --- a/go/core/internal/database/runtime_egress.go +++ b/go/core/internal/database/runtime_egress.go @@ -105,15 +105,36 @@ func (c *Client) ListRuntimeEgressSourceTargets(ctx context.Context, source Runt return nil, fmt.Errorf("runtime egress source requires namespace and name") } switch source.Kind { - case RuntimeEgressAgent, RuntimeEgressSandbox: + case RuntimeEgressAgent: + return queryMany(ctx, c.db, ` + SELECT i.id::text AS id, i.prepared_revision AS revision, r.namespace, + r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, + COALESCE(s.actor_uid, '') AS actor_uid + FROM runtime_instance i + JOIN runtime_revision r ON r.revision = i.prepared_revision + JOIN agent_revision a ON a.revision = r.revision + JOIN session s ON s.id = i.id + WHERE i.kind = 'agent' AND r.kind = 'agent' AND r.namespace = $1 AND a.agent_name = $2 + AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) + AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' + AND i.id > COALESCE(NULLIF($3, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY i.id LIMIT $4 + `, pgx.RowToStructByName[RuntimeEgressTarget], source.Namespace, source.Name, afterID, limit) + case RuntimeEgressSandbox: + return queryMany(ctx, c.db, ` + SELECT i.id::text AS id, i.prepared_revision AS revision, r.namespace, + r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, + '' AS actor_uid + FROM runtime_instance i + JOIN runtime_revision r ON r.revision = i.prepared_revision + JOIN sandbox_revision s ON s.revision = r.revision + WHERE i.kind = 'sandbox' AND r.kind = 'sandbox' AND r.namespace = $1 AND s.sandbox_template_name = $2 + AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) + AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' + AND i.id > COALESCE(NULLIF($3, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY i.id LIMIT $4 + `, pgx.RowToStructByName[RuntimeEgressTarget], source.Namespace, source.Name, afterID, limit) default: return nil, fmt.Errorf("unknown runtime egress source kind %q", source.Kind) } - return queryMany(ctx, c.db, ` - SELECT target.id::text AS id, revision, namespace, atespace, template_name, actor_uid - FROM runtime_egress_target target - WHERE source_kind = $1 AND namespace = $2 AND source_name = $3 - AND target.id > COALESCE(NULLIF($4, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) - ORDER BY target.id LIMIT $5 - `, pgx.RowToStructByName[RuntimeEgressTarget], source.Kind, source.Namespace, source.Name, afterID, limit) } diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index 9a15ca66cf..cdbf56cd88 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -22,12 +22,19 @@ func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { require.NoError(t, client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) instance, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "scratch", "kagent", ""), "request") require.NoError(t, err) + pendingTargets, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "scratch"}, "", 10) + require.NoError(t, err) + require.Empty(t, pendingTargets, "unissued creations cannot have actors") op, err := client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) require.NoError(t, err) executor := uuid.New() claimed, err := client.ClaimSessionOperation(ctx, instance.Id, op.ID, executor) require.NoError(t, err) require.True(t, claimed) + pendingTargets, err = client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "scratch"}, "", 10) + require.NoError(t, err) + require.Len(t, pendingTargets, 1, "a claimed creation may already have allocated an actor") + require.Empty(t, pendingTargets[0].ActorUID, "creation has not yet persisted the actor UID") _, err = client.FinishSessionOperation(ctx, instance.Id, op.ID, executor, "actor.example.com", "actor-uid", "") require.NoError(t, err) op, err = client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND) diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index 822b3472a6..83714f82bd 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -68,6 +68,17 @@ func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefi }) } +// ListAgentDefinitionSources returns distinct Agent names across active and +// retired identities. Startup recovery includes retired rows because a crash +// may interrupt actor revocation after retirement commits. +func (c *Client) ListAgentDefinitionSources(ctx context.Context) ([]RuntimeEgressSource, error) { + return queryMany(ctx, c.db, ` + SELECT DISTINCT 'agent' AS kind, namespace, agent_name AS name + FROM agent_definition + ORDER BY namespace, name + `, pgx.RowToStructByName[RuntimeEgressSource]) +} + // RecordRevisionOptions controls whether a prepared revision becomes available. type RecordRevisionOptions struct { // Ready promotes the revision only while its source is active and still desires it. diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index 82af16b59e..4d99d95c75 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -35,6 +35,25 @@ func TestRetireAgentIdentities(t *testing.T) { require.Equal(t, "revision", revisions[0].Revision) } +func TestListAgentDefinitionSourcesIncludesRetiredIdentities(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + for _, definition := range []AgentDefinition{ + {Namespace: "team-a", AgentName: "assistant", AgentUID: "old"}, + {Namespace: "team-a", AgentName: "assistant", AgentUID: "replacement"}, + {Namespace: "team-b", AgentName: "assistant", AgentUID: "other"}, + } { + require.NoError(t, client.UpsertAgentDefinition(ctx, definition)) + } + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) + sources, err := client.ListAgentDefinitionSources(ctx) + require.NoError(t, err) + require.Equal(t, []RuntimeEgressSource{ + {Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, + {Kind: RuntimeEgressAgent, Namespace: "team-b", Name: "assistant"}, + }, sources, "retirement may commit before actor revocation; recovery must retain that source and deduplicate old UIDs") +} + func TestRuntimeRevisionCollectionAfterPairRetirement(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() diff --git a/go/core/internal/substrate/sandbox.go b/go/core/internal/substrate/sandbox.go index f73eb6d835..cfe27e2e28 100644 --- a/go/core/internal/substrate/sandbox.go +++ b/go/core/internal/substrate/sandbox.go @@ -9,6 +9,7 @@ import ( atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" @@ -25,7 +26,6 @@ type SandboxPolicy struct { type sandboxInputs struct { Kind string - Version int Namespace string Name string UID string @@ -67,18 +67,13 @@ func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1. if len(environment) > 32 { return nil, "", nil, fmt.Errorf("sandbox exceeds Substrate's 32 environment variables") } - spec := template.Spec.DeepCopy() - spec.EgressPolicyRefs = nil - // Version 2 accounts for namespace-scoped placement in immutable templates. - inputs := sandboxInputs{Kind: "sandbox", Version: 2, Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), Spec: *spec, Class: class, Policy: policy} + inputs := sandboxInputs{Kind: "sandbox", Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), Spec: template.Spec, Class: class, Policy: policy} snapshot, err := json.Marshal(inputs) if err != nil { return nil, "", nil, fmt.Errorf("encode sandbox inputs: %w", err) } - digest := sha256.Sum256(snapshot) - revision := hex.EncodeToString(digest[:]) result := &ateapipb.ActorTemplate{ - Metadata: &ateapipb.ResourceMetadata{Atespace: template.Namespace, Name: "sandbox-" + revision[:40]}, + Metadata: &ateapipb.ResourceMetadata{Atespace: template.Namespace}, SandboxConfig: config, WorkerSelector: workerSelectorForPool(types.NamespacedName{Namespace: template.Namespace, Name: template.Spec.Substrate.WorkerPoolRef.Name}), Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{{Name: "cpu", Quantity: policy.CPU}, {Name: "memory", Quantity: policy.Memory}}}, @@ -100,5 +95,22 @@ func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1. OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, }, } + // Hash the compiled template and source identity. Live policy references + // remain in provenance but do not affect immutable runtime contents. + compiled, err := proto.MarshalOptions{Deterministic: true}.Marshal(result) + if err != nil { + return nil, "", nil, fmt.Errorf("encode sandbox template: %w", err) + } + identity, err := json.Marshal(struct { + Name string + UID string + Template []byte + }{template.Name, string(template.UID), compiled}) + if err != nil { + return nil, "", nil, fmt.Errorf("encode sandbox revision: %w", err) + } + digest := sha256.Sum256(identity) + revision := hex.EncodeToString(digest[:]) + result.Metadata.Name = "sandbox-" + revision[:40] return result, revision, snapshot, nil } diff --git a/go/core/internal/substrate/sandbox_test.go b/go/core/internal/substrate/sandbox_test.go index 0502a59cb6..4c2f30ff64 100644 --- a/go/core/internal/substrate/sandbox_test.go +++ b/go/core/internal/substrate/sandbox_test.go @@ -1,6 +1,7 @@ package substrate import ( + "encoding/json" "strings" "testing" @@ -36,9 +37,12 @@ func TestSandboxPreparationPinsCompleteInputs(t *testing.T) { require.NoError(t, err) require.Equal(t, digest, same) template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "live"}} - _, unchanged, _, err := SandboxActorTemplate(template, "", policy) + _, unchanged, source, err := SandboxActorTemplate(template, "", policy) require.NoError(t, err) require.Equal(t, digest, unchanged, "policy references are live authorization") + var recorded sandboxInputs + require.NoError(t, json.Unmarshal(source, &recorded)) + require.Equal(t, template.Spec, recorded.Spec, "provenance retains the complete source spec") for _, change := range []struct { name string mutate func(*v1alpha3.SandboxTemplate, *SandboxPolicy) diff --git a/go/core/internal/translator/compiler.go b/go/core/internal/translator/compiler.go index 8020582cc6..e13dde24e3 100644 --- a/go/core/internal/translator/compiler.go +++ b/go/core/internal/translator/compiler.go @@ -131,14 +131,12 @@ func (c *Compiler) CompileAgent(ctx context.Context, agent *v1alpha3.Agent) (*Co return nil, err } result.AgentUID = string(agent.UID) - spec := agent.Spec.DeepCopy() - spec.EgressPolicyRefs = nil result.Provenance, err = json.Marshal(struct { AgentName string `json:"agentName"` AgentUID string `json:"agentUID"` Spec v1alpha3.AgentSpec `json:"spec"` Inputs json.RawMessage `json:"inputs"` - }{agent.Name, string(agent.UID), *spec, result.Provenance}) + }{agent.Name, string(agent.UID), agent.Spec, result.Provenance}) if err != nil { return nil, fmt.Errorf("encode Agent provenance: %w", err) } diff --git a/go/core/internal/translator/compiler_test.go b/go/core/internal/translator/compiler_test.go index 3115182abf..8ce2daebb1 100644 --- a/go/core/internal/translator/compiler_test.go +++ b/go/core/internal/translator/compiler_test.go @@ -406,6 +406,14 @@ func TestCompileAgentStructuredOutput(t *testing.T) { require.Empty(t, config.Agent.SubAgents[0].SessionDbUrl) require.Contains(t, string(revision.Provenance), `"kind":"ConfigMap"`) require.Equal(t, []string{"application/json"}, revision.AgentCard.DefaultOutputModes) + original, err := revision.Digest() + require.NoError(t, err) + configMap.Data["answer.json"] = `{"type":"object","properties":{"changed":{"type":"string"}}}` + changed, err := compiler(t, modelConfig(), configMap, child, harness).CompileAgent(t.Context(), inlineAgent(harness, root)) + require.NoError(t, err) + updated, err := changed.Digest() + require.NoError(t, err) + require.NotEqual(t, original, updated, "resolved ConfigMap contents must still affect compiled runtime identity") } func TestResolveModelConfigFoundryEndpoint(t *testing.T) { diff --git a/go/core/internal/translator/egress.go b/go/core/internal/translator/egress.go index a0b3e447d4..299c8e538a 100644 --- a/go/core/internal/translator/egress.go +++ b/go/core/internal/translator/egress.go @@ -64,8 +64,8 @@ func compileEgressPolicies(namespace string, refs []corev1.LocalObjectReference, if auth == nil { continue } - if origin.Scheme != "https" || strings.Contains(origin.Host, "*") { - return egress.Access{}, NewValidationError("EgressPolicy %q: authentication requires an exact HTTPS origin", key) + if strings.Contains(origin.Host, "*") { + return egress.Access{}, NewValidationError("EgressPolicy %q: authentication requires an exact HTTP(S) origin", key) } secret := auth.SecretKeyRef binding := egress.Credential{Origin: origin.String(), Header: auth.Header} diff --git a/go/core/internal/translator/egress_test.go b/go/core/internal/translator/egress_test.go index 622556728b..839c458725 100644 --- a/go/core/internal/translator/egress_test.go +++ b/go/core/internal/translator/egress_test.go @@ -50,6 +50,15 @@ func TestEgressPolicyCredentialPrefixes(t *testing.T) { } } +func TestEgressPolicyHTTPCredentials(t *testing.T) { + policy, secret := egressPolicy() + policy.Spec.Destinations[0].Origin = "http://tickets.example.com:8080" + access, err := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", []corev1.LocalObjectReference{{Name: policy.Name}}) + require.NoError(t, err) + require.Equal(t, []string{"http://tickets.example.com:8080"}, access.Destinations) + require.Equal(t, []egress.Credential{{Origin: "http://tickets.example.com:8080", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/test/ticket-auth/token"}}, access.Credentials) +} + func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { for _, kind := range []string{"kagent", "codex", "claude", "byo"} { t.Run(kind, func(t *testing.T) { @@ -86,6 +95,9 @@ func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { return result } result := compile() + var recorded struct{ Spec v1alpha3.AgentSpec } + require.NoError(t, json.Unmarshal(result.Provenance, &recorded)) + require.Equal(t, agent.Spec, recorded.Spec, "record the full source spec, including live policy references") require.NotContains(t, result.EgressDestinations, "https://tickets.example.com:443") require.NotEmpty(t, result.EgressDestinations, "preserve inferred integration access") for _, binding := range result.Credentials { @@ -136,7 +148,7 @@ func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { } func TestEgressPolicyResolutionFailsClosed(t *testing.T) { - for _, name := range []string{"missing policy", "other namespace", "missing secret", "missing key", "optional secret", "HTTP auth", "wildcard auth", "empty auth", "missing header", "protected header", "invalid prefix", "conflicting auth", "conflicting prefix"} { + for _, name := range []string{"missing policy", "other namespace", "missing secret", "missing key", "optional secret", "wildcard auth", "empty auth", "missing header", "protected header", "invalid prefix", "conflicting auth", "conflicting prefix"} { t.Run(name, func(t *testing.T) { policy, secret := egressPolicy() objects := []any{policy, secret} @@ -152,8 +164,6 @@ func TestEgressPolicyResolutionFailsClosed(t *testing.T) { secret.Data = nil case "optional secret": policy.Spec.Destinations[0].Authentication.SecretKeyRef.Optional = new(true) - case "HTTP auth": - policy.Spec.Destinations[0].Origin = "http://tickets.example.com" case "wildcard auth": policy.Spec.Destinations[0].Origin = "https://*.example.com" case "empty auth": diff --git a/go/core/internal/translator/revision.go b/go/core/internal/translator/revision.go index 02dd438ee4..37863518dc 100644 --- a/go/core/internal/translator/revision.go +++ b/go/core/internal/translator/revision.go @@ -60,8 +60,8 @@ type Revision struct { SandboxClass atev1alpha1.SandboxClass SnapshotLocation string - // Provenance identifies non-secret Kubernetes inputs. Gateway-fetched - // credential values are deliberately excluded from revision identity. + // Provenance records non-secret source configuration for diagnostics. Revision + // identity comes from compiled runtime inputs, not this source snapshot. Provenance json.RawMessage // Credentials contains references resolved by the egress gateway. Credentials []egress.Credential @@ -104,7 +104,6 @@ func (r *Revision) Digest() (RevisionID, error) { SkillImages []SkillImage `json:"skillImages,omitempty"` WorkerPoolName string `json:"workerPoolName"` SnapshotLocation string `json:"snapshotLocation"` - Provenance json.RawMessage `json:"provenance"` Credentials []egress.Credential `json:"credentials,omitempty"` TokenPassthroughOrigins []string `json:"tokenPassthroughOrigins,omitempty"` EgressDestinations []string `json:"egressDestinations"` @@ -117,7 +116,7 @@ func (r *Revision) Digest() (RevisionID, error) { AgentName: r.AgentName, AgentUID: r.AgentUID, Namespace: r.Namespace, Image: r.Image, Command: r.Command, Args: r.Args, Environment: r.Environment, ConfigJSON: r.ConfigJSON, SkillImages: r.SkillImages, - WorkerPoolName: r.WorkerPoolName, SnapshotLocation: r.SnapshotLocation, Provenance: r.Provenance, + WorkerPoolName: r.WorkerPoolName, SnapshotLocation: r.SnapshotLocation, TokenPassthroughOrigins: r.TokenPassthroughOrigins, Credentials: r.Credentials, EgressDestinations: r.EgressDestinations, SandboxClass: sandboxClass, WorkerPoolNamespace: r.Namespace, }) diff --git a/go/core/internal/translator/revision_test.go b/go/core/internal/translator/revision_test.go index ecde2c53c5..e817061e39 100644 --- a/go/core/internal/translator/revision_test.go +++ b/go/core/internal/translator/revision_test.go @@ -49,7 +49,7 @@ func TestRevisionDigestReplacesNameOnlyWorkerPoolTemplate(t *testing.T) { require.Equal(t, id, repeated) } -func TestRevisionDigestIncludesProvenance(t *testing.T) { +func TestRevisionDigestExcludesProvenance(t *testing.T) { revision := &Revision{Namespace: "agents", AgentName: "helper", Provenance: []byte(`[{"kind":"ConfigMap","hash":"first"}]`)} first, err := revision.Digest() if err != nil { @@ -60,8 +60,8 @@ func TestRevisionDigestIncludesProvenance(t *testing.T) { if err != nil { t.Fatal(err) } - if first == second { - t.Fatal("configuration change did not change runtime revision") + if first != second { + t.Fatal("source-only change altered runtime revision") } if len(first.Short()) != 12 || !strings.HasPrefix(first.String(), first.Short()) { t.Fatalf("short revision %q is not a prefix of %q", first.Short(), first.String()) diff --git a/go/core/pkg/migrations/core/000001_initial.sql b/go/core/pkg/migrations/core/000001_initial.sql index 6beaa61641..27b0c5e819 100644 --- a/go/core/pkg/migrations/core/000001_initial.sql +++ b/go/core/pkg/migrations/core/000001_initial.sql @@ -328,21 +328,6 @@ CREATE VIEW sandbox_record AS s.data, s.namespace, s.sandbox_template_name, s.revision_receipt, s.request_hash, s.expires_at FROM runtime_instance r JOIN sandbox s USING (id) WHERE r.kind = 'sandbox'; -CREATE VIEW runtime_egress_target AS - SELECT i.id, i.prepared_revision AS revision, r.namespace, - source.kind AS source_kind, source.name AS source_name, - r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, - COALESCE(s.actor_uid, '') AS actor_uid - FROM runtime_instance i JOIN runtime_revision r ON r.revision = i.prepared_revision - JOIN ( - SELECT revision, 'agent'::text AS kind, agent_name AS name FROM agent_revision - UNION ALL - SELECT revision, 'sandbox'::text AS kind, sandbox_template_name AS name FROM sandbox_revision - ) source ON source.revision = r.revision AND source.kind = r.kind - LEFT JOIN session s ON i.kind = 'agent' AND s.id = i.id - WHERE (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) - AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE'; - CREATE VIEW unreferenced_runtime_revision AS SELECT r.revision FROM runtime_revision r WHERE NOT EXISTS (SELECT 1 FROM agent_definition p WHERE p.retired_at IS NULL @@ -396,7 +381,6 @@ CREATE INDEX session_push_registration_lease ON session_push_registration (lease DROP TABLE session_push_registration; DROP VIEW unreferenced_runtime_revision; -DROP VIEW runtime_egress_target; DROP VIEW sandbox_record; DROP VIEW session_record; DROP VIEW agent_runtime_revision; diff --git a/go/core/test/e2e/egress_credentials_test.go b/go/core/test/e2e/egress_credentials_test.go index d435e1ab51..5573ed6a2e 100644 --- a/go/core/test/e2e/egress_credentials_test.go +++ b/go/core/test/e2e/egress_credentials_test.go @@ -2,45 +2,28 @@ package e2e_test import ( "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "fmt" "io" - "maps" - "math/big" "net" "net/http" "net/http/httptest" "net/url" "testing" - "time" - "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/util/wait" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/yaml" "github.com/kagent-dev/kagent/go/api/v1alpha3" ) -// Keep the parent sequential: the fixture installs a temporary HTTPS root in -// the gateway and restores its configuration before other parallel E2Es run. func TestEgressPolicyCredentialDelivery(t *testing.T) { + t.Parallel() target := interactionTarget(t) kube := interactionKubeClient(t) const token = "explicit-policy-test-token" - destination := startAuthenticatedEgressDestination(t, kube, token) + destination := startAuthenticatedEgressDestination(t, token) image := publishSkillImage(t) forEachHarness(t, func(t *testing.T, harness testHarness) { t.Parallel() @@ -75,13 +58,13 @@ func TestEgressPolicyCredentialDelivery(t *testing.T) { require.NoError(t, kube.Update(t.Context(), agent)) run.setPhase("authenticated") run.expect(t, "authenticated", "ALLOWED") - // The HTTPS server returns success only when both placeholders have been + // The HTTP server returns success only when both placeholders have been // replaced from the Secret, with and without a prefix respectively. finish() }) } -func startAuthenticatedEgressDestination(t *testing.T, kube ctrlclient.Client, token string) string { +func startAuthenticatedEgressDestination(t *testing.T, token string) string { t.Helper() server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer "+token || r.Header.Get("X-API-Key") != token { @@ -94,34 +77,14 @@ func startAuthenticatedEgressDestination(t *testing.T, kube ctrlclient.Client, t listener, err := net.Listen("tcp", "0.0.0.0:0") require.NoError(t, err) server.Listener = listener - origin := reachableServerURL(t, "https://"+listener.Addr().String(), "") + origin := reachableServerURL(t, "http://"+listener.Addr().String(), "") parsed, err := url.Parse(origin) require.NoError(t, err) - caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - require.NoError(t, err) - ca := &x509.Certificate{ - SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "kagent E2E CA"}, - NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), - IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, - } - caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) - require.NoError(t, err) - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - require.NoError(t, err) - certificate := &x509.Certificate{ - SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "kagent E2E origin"}, DNSNames: []string{parsed.Hostname()}, IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, - NotBefore: ca.NotBefore, NotAfter: ca.NotAfter, - BasicConstraintsValid: true, KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - } - der, err := x509.CreateCertificate(rand.Reader, certificate, ca, &key.PublicKey, caKey) - require.NoError(t, err) - server.TLS = &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{{Certificate: [][]byte{der, caDER}, PrivateKey: key}}} - server.StartTLS() + server.Start() t.Cleanup(server.Close) // Prove a successful fetch cannot be produced by a missing/unchanged header. for _, value := range []string{"", "kagent-credential-injected"} { - request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://"+net.JoinHostPort("127.0.0.1", parsed.Port()), nil) + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://"+net.JoinHostPort("127.0.0.1", parsed.Port()), nil) require.NoError(t, err) request.Header.Set("Authorization", "Bearer "+value) request.Header.Set("X-API-Key", value) @@ -130,196 +93,5 @@ func startAuthenticatedEgressDestination(t *testing.T, kube ctrlclient.Client, t require.NoError(t, response.Body.Close()) require.Equal(t, http.StatusForbidden, response.StatusCode) } - trustEgressTestOrigin(t, kube, string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}))) return origin } - -// Configure only the gateway's destination TLS trust. Actor identity, policy -// lookup, credential provider, and guest MITM trust use the normal installation. -func trustEgressTestOrigin(t *testing.T, kube ctrlclient.Client, certificate string) { - t.Helper() - deployments := &appsv1.DeploymentList{} - require.NoError(t, kube.List(t.Context(), deployments, ctrlclient.InNamespace("ate-system"), ctrlclient.MatchingLabels{"app": "atenet-egress"})) - require.Len(t, deployments.Items, 1) - deployment := &deployments.Items[0] - var configName string - for _, volume := range deployment.Spec.Template.Spec.Volumes { - if volume.Name == "config" && volume.ConfigMap != nil { - configName = volume.ConfigMap.Name - } - } - require.NotEmpty(t, configName) - config := &corev1.ConfigMap{} - require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKey{Namespace: deployment.Namespace, Name: configName}, config)) - original := config.DeepCopy() - patched, restored, err := egressGatewayTrustConfig(config.Data, certificate) - require.NoError(t, err) - config.Data = patched - rollout := func(ctx context.Context) error { - current := &appsv1.Deployment{} - deploymentKey := ctrlclient.ObjectKeyFromObject(deployment) - if err := kube.Get(ctx, deploymentKey, current); err != nil { - return err - } - before := current.DeepCopy() - if current.Spec.Template.Annotations == nil { - current.Spec.Template.Annotations = map[string]string{} - } - current.Spec.Template.Annotations["kagent.dev/egress-test-rollout"] = uuid.NewString() - if err := kube.Patch(ctx, current, ctrlclient.MergeFrom(before)); err != nil { - return err - } - return wait.PollUntilContextTimeout(ctx, time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { - observed := &appsv1.Deployment{} - if err := kube.Get(ctx, deploymentKey, observed); err != nil { - return false, err - } - return observed.Status.ObservedGeneration >= current.Generation && observed.Status.UpdatedReplicas == *observed.Spec.Replicas && observed.Status.AvailableReplicas == *observed.Spec.Replicas && observed.Status.Replicas == *observed.Spec.Replicas, nil - }) - } - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) - defer cancel() - current := &corev1.ConfigMap{} - if !assert.NoError(t, kube.Get(ctx, ctrlclient.ObjectKeyFromObject(original), current), "get gateway config for cleanup") { - return - } - before := current.DeepCopy() - current.Data["config.yaml"] = restored["config.yaml"] - delete(current.Data, egressTestRootFile) - if !assert.NoError(t, kube.Patch(ctx, current, ctrlclient.MergeFrom(before)), "restore gateway config") { - return - } - assert.NoError(t, rollout(ctx), "roll out restored gateway config") - }) - require.NoError(t, kube.Patch(t.Context(), config, ctrlclient.MergeFrom(original))) - require.NoError(t, rollout(t.Context())) -} - -const egressTestRootFile = "egress-test-root.pem" - -// Patch only dynamic HTTPS destinations, preserving the gateway's other policies. -// The restore configuration also removes test trust left by an interrupted run. -func egressGatewayTrustConfig(data map[string]string, certificate string) (patched, restored map[string]string, err error) { - var config map[string]any - if err := yaml.Unmarshal([]byte(data["config.yaml"]), &config); err != nil { - return nil, nil, fmt.Errorf("decode gateway config: %w", err) - } - var backends []map[string]any - binds, _ := config["binds"].([]any) - for _, value := range binds { - bind, _ := value.(map[string]any) - if bind["mode"] != "internal" { - continue - } - listeners, _ := bind["listeners"].([]any) - for _, value := range listeners { - listener, _ := value.(map[string]any) - if listener["protocol"] != "HTTPS" { - continue - } - routes, _ := listener["routes"].([]any) - for _, value := range routes { - route, _ := value.(map[string]any) - items, _ := route["backends"].([]any) - for _, value := range items { - backend, _ := value.(map[string]any) - if _, dynamic := backend["dynamic"]; dynamic { - backends = append(backends, backend) - } - } - } - } - } - if len(backends) != 1 { - return nil, nil, fmt.Errorf("expected one dynamic HTTPS backend, found %d", len(backends)) - } - backend := backends[0] - path := []string{"policies", "backendTLS", "root"} - root, _, err := unstructured.NestedString(backend, path...) - if err != nil { - return nil, nil, fmt.Errorf("read gateway TLS root: %w", err) - } - restored = maps.Clone(data) - delete(restored, egressTestRootFile) - const testRoot = "/etc/agentgateway/" + egressTestRootFile - if root == testRoot { - unstructured.RemoveNestedField(backend, path...) - cleaned, err := yaml.Marshal(config) - if err != nil { - return nil, nil, fmt.Errorf("encode restored gateway config: %w", err) - } - restored["config.yaml"] = string(cleaned) - } - if err := unstructured.SetNestedField(backend, testRoot, path...); err != nil { - return nil, nil, fmt.Errorf("set gateway TLS root: %w", err) - } - updated, err := yaml.Marshal(config) - if err != nil { - return nil, nil, fmt.Errorf("encode gateway config: %w", err) - } - patched = maps.Clone(restored) - patched["config.yaml"] = string(updated) - patched[egressTestRootFile] = certificate - return patched, restored, nil -} - -func TestEgressGatewayTrustConfig(t *testing.T) { - const gateway = `frontendPolicies: - substrateEgressActorResolution: - policies: - backendTLS: {root: /identity-ca.pem} -binds: -- mode: internal - listeners: - - protocol: HTTP - routes: - - backends: - - dynamic: {} - - protocol: HTTPS - tls: {mode: dynamicCa, cert: /mitm.pem} - routes: - - policies: - substrateEgress: - policies: - backendTLS: {root: /control-plane-ca.pem} - backends: - - dynamic: {} - policies: - backendTLS: %s -` - for _, tc := range []struct { - name, tls, restoredTLS string - }{ - {"clean", "{}", "{}"}, - {"interrupted", "{root: /etc/agentgateway/egress-test-root.pem}", "{}"}, - {"existing trust", "{root: /custom-ca.pem, hostname: example.com}", "{root: /custom-ca.pem, hostname: example.com}"}, - {"block format", "\n root: /etc/agentgateway/egress-test-root.pem", "{}"}, - } { - t.Run(tc.name, func(t *testing.T) { - input := map[string]string{"config.yaml": fmt.Sprintf(gateway, tc.tls), egressTestRootFile: "leftover CA", "other.pem": "keep"} - before := maps.Clone(input) - patched, restored, err := egressGatewayTrustConfig(input, "new CA") - require.NoError(t, err) - require.Equal(t, before, input, "patch must not mutate the saved config") - require.Equal(t, "new CA", patched[egressTestRootFile]) - require.Equal(t, "keep", patched["other.pem"]) - require.NotContains(t, restored, egressTestRootFile) - var want, got map[string]any - require.NoError(t, yaml.Unmarshal([]byte(fmt.Sprintf(gateway, tc.restoredTLS)), &want)) - require.NoError(t, yaml.Unmarshal([]byte(restored["config.yaml"]), &got)) - require.Equal(t, want, got, "cleanup must preserve unrelated TLS policy") - repeated, _, err := egressGatewayTrustConfig(patched, "replacement CA") - require.NoError(t, err) - require.Equal(t, patched["config.yaml"], repeated["config.yaml"], "reapplying trust must be idempotent") - require.Equal(t, "replacement CA", repeated[egressTestRootFile]) - require.Contains(t, patched["config.yaml"], "root: /etc/agentgateway/egress-test-root.pem") - require.Contains(t, patched["config.yaml"], "root: /identity-ca.pem") - require.Contains(t, patched["config.yaml"], "root: /control-plane-ca.pem") - }) - } - for _, input := range []string{"[invalid", "binds: []"} { - _, _, err := egressGatewayTrustConfig(map[string]string{"config.yaml": input}, "CA") - require.Error(t, err) - } -} diff --git a/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml b/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml index 6e2e978cc0..3945fc758b 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml @@ -122,9 +122,8 @@ spec: - origin type: object x-kubernetes-validations: - - message: authentication requires an exact HTTPS origin - rule: '!has(self.authentication) || (self.origin.startsWith(''https://'') - && !self.origin.contains(''*''))' + - message: authentication requires an exact HTTP(S) origin + rule: '!has(self.authentication) || !self.origin.contains(''*'')' maxItems: 64 minItems: 1 type: array From c8fec1ce7bf44976d847b5cc28283eb18b14467b Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 16 Oct 2026 23:53:52 +0000 Subject: [PATCH 33/43] fix: admit egress grants independently of runtime readiness Resolve policies independently and retain still-requested last-good grants when additions conflict. Filter only incompatible grants for older revisions, expose EgressPoliciesApplied on Agents and SandboxTemplates, and bound deleted-Agent recovery to retained instances. Signed-off-by: Eitan Yarmush --- docs/architecture/credential-injection.md | 11 +- docs/architecture/egress-policies.md | 69 ++++++--- go/api/v1alpha3/agent_types.go | 15 +- go/api/v1alpha3/egresspolicy_types.go | 9 ++ go/api/v1alpha3/sandboxtemplate_types.go | 9 +- go/core/internal/controller/egress.go | 7 +- go/core/internal/controller/egress_test.go | 13 -- .../internal/controller/grant_admission.go | 75 +++++++++ .../controller/grant_admission_test.go | 144 +++++++++++++++++ go/core/internal/controller/grants_test.go | 15 +- .../internal/controller/preparation_test.go | 6 + go/core/internal/controller/reconciler.go | 30 +++- .../internal/controller/reconciler_test.go | 8 + .../controller/sandbox_collections.go | 13 +- .../controller/sandbox_integration_test.go | 21 +-- .../internal/controller/sandbox_reconciler.go | 13 +- .../controller/sandbox_reconciler_test.go | 10 ++ go/core/internal/controller/status.go | 1 + go/core/internal/database/runtime_egress.go | 25 ++- .../internal/database/runtime_egress_test.go | 53 ++++++- .../internal/database/runtime_revisions.go | 27 +++- .../database/runtime_revisions_test.go | 16 +- .../internal/database/sandbox_revisions.go | 10 ++ go/core/internal/egress/live.go | 69 +++++++-- go/core/internal/egress/live_test.go | 107 +++++++++---- go/core/internal/substrate/egress.go | 4 +- go/core/internal/substrate/egress_test.go | 18 --- go/core/internal/translator/compiler.go | 11 -- go/core/internal/translator/credentials.go | 2 +- .../internal/translator/credentials_test.go | 15 +- go/core/internal/translator/egress.go | 145 +++++++++++------- go/core/internal/translator/egress_test.go | 78 ++++++++-- go/core/test/e2e/egress_test.go | 43 +++++- .../templates/api.kagent.dev_agents.yaml | 7 +- .../api.kagent.dev_sandboxtemplates.yaml | 6 +- 35 files changed, 857 insertions(+), 248 deletions(-) create mode 100644 go/core/internal/controller/grant_admission.go create mode 100644 go/core/internal/controller/grant_admission_test.go diff --git a/docs/architecture/credential-injection.md b/docs/architecture/credential-injection.md index 0df74d9e50..78def5dc58 100644 --- a/docs/architecture/credential-injection.md +++ b/docs/architecture/credential-injection.md @@ -1,8 +1,9 @@ # Runtime credential injection -Kagent requires Substrate **v0.5.0-alpha2**. The compiler turns ModelConfig API -keys, Secret-backed RemoteMCPServer headers, and explicit EgressPolicy -authentication into destination-scoped egress bindings. Substrate's gateway fetches the referenced Kubernetes Secret and +Kagent requires Substrate **v0.5.0-alpha2**. The runtime compiler turns +ModelConfig API keys and Secret-backed RemoteMCPServer headers into pinned +destination-scoped bindings. Independently, EgressPolicy admission supplies +additional live bindings. Substrate's gateway fetches the referenced Kubernetes Secret and replaces the outgoing HTTP header when the request carries a placeholder. SDKs receive an inert placeholder where they require an API key; real credentials never enter compiled environments, runtime configuration, or revision provenance. @@ -10,7 +11,9 @@ runtime configuration, or revision provenance. Inferred integration bindings are persisted with the prepared revision; their Secret names, keys, destinations, and headers affect revision identity. Explicit EgressPolicy bindings are resolved from current policies and update live actors -without changing that revision. Both are installed before execution, including +without changing that revision. Conflicts hold new grants and report the +independent [EgressPoliciesApplied condition](egress-policies.md#preparation-and-lifecycle); +they do not block runtime compilation. Both are installed before execution, including creation retries, resume, and checkpoint forks. Secret values and Secret UIDs do not affect revision identity. Rotation is handled by the gateway; its credential cache can take up to five minutes to refresh, without recompiling or restarting an agent. diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md index a209913fa9..e2616f9987 100644 --- a/docs/architecture/egress-policies.md +++ b/docs/architecture/egress-policies.md @@ -72,7 +72,7 @@ spec: ``` Each workload can select up to 16 policies, each containing 1–64 destinations. -The compiler combines them with inferred integration access, removes equivalent +Grant admission combines them with inferred integration access, removes equivalent origins, and orders the result deterministically. The final Actor policy is limited to 256 rules. References are additive; they cannot restrict inferred model, MCP, artifact, or control-plane access. No references means no additional @@ -112,7 +112,7 @@ The gateway sets the header value to `prefix + secretValue`. Prefixes are litera include the trailing space in `"Bearer "`. Omitting the prefix or setting it to an empty string uses the Secret value directly, as with an API-key header. -Create these Secrets separately. The compiler validates that each Secret and key +Create these Secrets separately. Policy resolution validates that each Secret and key exists, but persists only the reference. Substrate's credential provider fetches the value and the gateway replaces the request header. Scripts and SDKs must send that header, using an inert placeholder: @@ -132,8 +132,8 @@ CA setup, inferred SDK placeholders, and rotation behavior. Bindings use the complete origin and a case-insensitive header name. Different ports can use different credentials. Identical bindings merge; conflicting -Secret references or prefixes for the same origin/header fail preparation. -Explicit and inferred bindings follow the same rule. Caller-token passthrough +Secret references or prefixes for the same origin/header hold new grants. +Pinned integration bindings take precedence over additional grants. Caller-token passthrough cannot share its origin with static gateway credentials. Routing, framing, hop-by-hop, proxy-authentication, and Cookie headers cannot carry credentials. At most 16 credential headers are supported per origin. @@ -150,13 +150,32 @@ because Substrate cannot distinguish that rule selection. ## Preparation and lifecycle -KRT watches selected policies and their Secrets. Missing policies, missing -Secret keys, invalid destinations, and conflicting bindings prevent preparation. -The controller publishes the error on the workload's status and re-evaluates it -when dependencies change. +KRT watches selected policies and their Secrets independently of runtime +compilation. Each policy resolves separately. A missing or deleting policy, +missing Secret key, or invalid policy contributes no access; grants from the +remaining policies are still considered. Invalid reference names or more than +16 references revoke all additional access. Grant errors never block runtime +compilation or change revision identity. + +Both Agent and SandboxTemplate expose an independent `EgressPoliciesApplied` +condition: + +| Status | Reason | Meaning | +| --- | --- | --- | +| True | `Applied` | All requested grants are admitted. | +| False | `EgressPolicyNotFound` | A referenced policy is missing or deleting. | +| False | `InvalidEgressPolicy` | A policy, Secret reference, or reference list is invalid. | +| False | `EgressGrantConflict` | New grants are held; still-requested, compatible previous grants remain. | + +Messages identify affected policies and conflicts. A conflict takes precedence +when there are also resolution errors. This condition never affects `Ready` or +`ResolvedRefs`; missing policies no longer fail runtime reference resolution. +It reports admission from the controller's current inputs, not acknowledgement +that every actor has received the update. Runtime revisions retain the integration access inferred from models and tools, -including credential references and caller-token passthrough constraints. +including credential references and caller-token passthrough constraints. This +pinned access is the revision's base. The combined additional access from a source's referenced EgressPolicies is its `EgressGrant`: destinations it may contact and credentials the gateway may inject. Reconciliation and source definition structs carry it as `EgressGrant egress.Access`. @@ -171,8 +190,18 @@ The elected controller's Agent and SandboxTemplate reconcilers resolve current grants independently of runtime compilation. A missing Harness, AgentTemplate, or WorkerPool cannot freeze a revocation. Each preparation queue is the sole writer of its source definition's grants in PostgreSQL. Definition updates -persist canonical grants and retire replaced Kubernetes UIDs atomically. A single -store transaction records the runtime revision and, when ready, promotes it if +persist admitted grants and retire replaced Kubernetes UIDs atomically. + +The stored grant set is the last good set: valid on its own and compatible with +the desired revision's base when compilation succeeds. If the complete candidate +conflicts, preparation retains only previously admitted destinations and identical +credential bindings that are still requested, filtering any that conflict with +the desired base. Removals apply immediately; all additions wait until the +conflict is fixed. A new Kubernetes UID inherits no grants. When unrelated +compilation fails, admission checks grants on their own and still applies +revocations. + +A single store transaction records the runtime revision and, when ready, promotes it if the active source still desires it. Failed runtime preparation preserves the last successful revision while continuing to update source grants. @@ -191,17 +220,19 @@ updates therefore delay further work on that preparation queue. Startup replays every Agent and SandboxTemplate through preparation; no separate actor sweep is needed. After its input cache syncs, Agent preparation also enqueues stored definitions -whose Agents are absent from watched namespaces. This includes retired identities -so a restart retries revocations interrupted after retirement. Agent deletion -does not need a finalizer. SandboxTemplate preparation retains its existing +whose Agents are absent from watched namespaces. Retired identities are included +only while they have non-deleted instances, so a restart retries interrupted +revocations without re-enqueueing every historic source. Agent deletion does not +need a finalizer. SandboxTemplate preparation retains its existing finalizer until retirement and actor updates succeed. Running and suspended actors receive updates without replacement or a new -invocation. Missing or invalid policy references and Secrets revoke explicit -grants. Grants that conflict with pinned credentials or caller-token origins, or -exceed the merged rule limit, are withheld while pinned integration access remains. -Preparation reports these conflicts with warnings; create and resume apply the -same fallback. Missing revisions or malformed persisted inputs return errors and +invocation. Missing or invalid policies and Secrets revoke only the affected +policies' grants. Actors on older revisions can have different pinned access: +their effective policy drops only the destinations or credentials that conflict +with that base, caller-token origins, or combined limits. Nonconflicting grants +remain. The pinned base always wins; dropping a credential retains its network +destination if the destination itself is compatible. Missing revisions or malformed persisted inputs return errors and block execution. New Sessions and sandboxes pin the latest successful runtime revision, including diff --git a/go/api/v1alpha3/agent_types.go b/go/api/v1alpha3/agent_types.go index 61fb786bf0..d44f398b4f 100644 --- a/go/api/v1alpha3/agent_types.go +++ b/go/api/v1alpha3/agent_types.go @@ -80,10 +80,11 @@ type AgentTemplateSource struct { } const ( - AgentConditionAccepted = "Accepted" - AgentConditionResolvedRefs = "ResolvedRefs" - AgentConditionCompatible = "Compatible" - AgentConditionReady = "Ready" + AgentConditionEgressPoliciesApplied = "EgressPoliciesApplied" + AgentConditionAccepted = "Accepted" + AgentConditionResolvedRefs = "ResolvedRefs" + AgentConditionCompatible = "Compatible" + AgentConditionReady = "Ready" ) // AgentStatus reports compilation and preparation of an Agent. @@ -107,7 +108,11 @@ type AgentStatus struct { // +listType=set // +optional Warnings []string `json:"warnings,omitempty"` - // +kubebuilder:validation:MaxItems=4 + // Conditions report runtime readiness and independent egress grant admission. + // EgressPoliciesApplied is False for EgressPolicyNotFound, InvalidEgressPolicy, + // or EgressGrantConflict; Applied means all grants are admitted. Grant issues + // do not affect Ready. + // +kubebuilder:validation:MaxItems=5 // +listType=map // +listMapKey=type // +optional diff --git a/go/api/v1alpha3/egresspolicy_types.go b/go/api/v1alpha3/egresspolicy_types.go index 9fc67a54d6..deac70d7f2 100644 --- a/go/api/v1alpha3/egresspolicy_types.go +++ b/go/api/v1alpha3/egresspolicy_types.go @@ -86,3 +86,12 @@ func init() { return nil }) } + +// Reasons for the independent EgressPoliciesApplied condition on Agents and +// SandboxTemplates. Grant resolution and admission never block runtime readiness. +const ( + EgressPoliciesReasonApplied = "Applied" + EgressPoliciesReasonNotFound = "EgressPolicyNotFound" + EgressPoliciesReasonInvalidEgressPolicy = "InvalidEgressPolicy" + EgressPoliciesReasonConflict = "EgressGrantConflict" +) diff --git a/go/api/v1alpha3/sandboxtemplate_types.go b/go/api/v1alpha3/sandboxtemplate_types.go index 1c1796715c..0b563f7070 100644 --- a/go/api/v1alpha3/sandboxtemplate_types.go +++ b/go/api/v1alpha3/sandboxtemplate_types.go @@ -49,15 +49,18 @@ type SandboxTemplateSpec struct { EgressPolicyRefs []corev1.LocalObjectReference `json:"egressPolicyRefs,omitempty"` } -// SandboxTemplateReasonEgressPolicyResolutionFailed identifies an unresolved or invalid egress grant. -const SandboxTemplateReasonEgressPolicyResolutionFailed = "EgressPolicyResolutionFailed" +// SandboxTemplateConditionEgressPoliciesApplied reports grant admission independently of Ready. +const SandboxTemplateConditionEgressPoliciesApplied = "EgressPoliciesApplied" // SandboxTemplateStatus reports preparation of the current template inputs. type SandboxTemplateStatus struct { // ObservedGeneration is the generation considered by preparation. // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // Conditions describe whether a prepared revision is available. + // Conditions report runtime readiness and independent egress grant admission. + // EgressPoliciesApplied is False for EgressPolicyNotFound, InvalidEgressPolicy, + // or EgressGrantConflict; Applied means all grants are admitted. Grant issues + // do not affect Ready. // +optional // +listType=map // +listMapKey=type diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go index 47b5499caf..c5e2016723 100644 --- a/go/core/internal/controller/egress.go +++ b/go/core/internal/controller/egress.go @@ -13,7 +13,6 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" - "github.com/kagent-dev/kagent/go/pkg/logging" ) type actorEgressStore interface { @@ -70,15 +69,13 @@ func applyActorEgress(ctx context.Context, actors actorEgressClient, target data if !binding.Matches(actor) || (target.ActorUID != "" && target.ActorUID != actor.GetMetadata().GetUid()) { return nil } - policy, err := actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, target.Revision) + _, err = actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, target.Revision) if status.Code(err) == codes.NotFound { return nil } if err != nil { return err } - if policy.WithheldGrants != nil { - logging.FromContext(ctx).WarnContext(ctx, "explicit egress grants withheld", "revision", target.Revision, "namespace", target.Namespace, "error", policy.WithheldGrants) - } + return nil } diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go index 77f169b045..b4519f534f 100644 --- a/go/core/internal/controller/egress_test.go +++ b/go/core/internal/controller/egress_test.go @@ -1,11 +1,9 @@ package controller import ( - "bytes" "context" "errors" "fmt" - "log/slog" "slices" "sync" "testing" @@ -19,7 +17,6 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/egress" - "github.com/kagent-dev/kagent/go/pkg/logging" ) type egressTargetStoreStub struct { @@ -131,13 +128,3 @@ func TestActorEgressSkipsStaleOrMissingActors(t *testing.T) { }) } } - -func TestActorEgressReportsWithheldGrants(t *testing.T) { - var logs bytes.Buffer - ctx := logging.IntoContext(t.Context(), slog.New(slog.NewTextHandler(&logs, nil))) - actors := &actorEgressStub{ensurePolicy: func(context.Context, string, string, string) (egress.Policy, error) { - return egress.Policy{WithheldGrants: errors.New("credential conflict")}, nil - }} - require.NoError(t, applyActorEgress(ctx, actors, database.RuntimeEgressTarget{ID: "actor", Revision: "revision", Atespace: "team", TemplateName: "runtime"})) - require.Contains(t, logs.String(), "explicit egress grants withheld") -} diff --git a/go/core/internal/controller/grant_admission.go b/go/core/internal/controller/grant_admission.go new file mode 100644 index 0000000000..e4a155b896 --- /dev/null +++ b/go/core/internal/controller/grant_admission.go @@ -0,0 +1,75 @@ +package controller + +import ( + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/translator" +) + +// evaluateEgressGrant checks the complete candidate without reading persisted +// grants. On conflict, retain the raw union so preparation can apply removals. +func evaluateEgressGrant(resolution translator.EgressGrantResolution, base egress.Access, tokenOrigins []string) (egress.Access, []translator.EgressGrantIssue, bool) { + candidate, err := egress.CanonicalAccess(resolution.Access.Destinations, resolution.Access.Credentials) + if err == nil { + _, err = egress.MergeAccess(base, candidate, tokenOrigins) + } + if err == nil { + return candidate, resolution.Issues, false + } + issues := append(resolution.Issues, translator.EgressGrantIssue{ + Reason: v1alpha3.EgressPoliciesReasonConflict, Message: err.Error(), + }) + return resolution.Access, issues, true +} + +func retainedEgressGrant(previous, candidate, base egress.Access, tokenOrigins []string) egress.Access { + retained := egress.IntersectAccess(previous, candidate) + filtered, _ := egress.FilterConflicts(base, retained, tokenOrigins) + return filtered +} + +func egressPoliciesCondition(conditionType string, generation int64, issues []translator.EgressGrantIssue, conflict bool) metav1.Condition { + condition := metav1.Condition{ + Type: conditionType, ObservedGeneration: generation, + Status: metav1.ConditionTrue, Reason: v1alpha3.EgressPoliciesReasonApplied, + Message: "All requested egress grants are admitted", + } + if len(issues) == 0 && !conflict { + return condition + } + condition.Status = metav1.ConditionFalse + var messages []string + for _, issue := range issues { + message := issue.Message + if issue.Policy != "" { + message = fmt.Sprintf("%s: %s", issue.Policy, message) + } + messages = append(messages, message) + } + if conflict { + condition.Reason = v1alpha3.EgressPoliciesReasonConflict + condition.Message = "new grants are held, previously applied grants remain in effect; " + } else { + condition.Reason = issues[0].Reason + condition.Message = "" + } + condition.Message += strings.Join(messages, "; ") + // Stay comfortably below the Kubernetes condition message limit. + const maxMessage = 4096 + if len(condition.Message) > maxMessage { + condition.Message = strings.ToValidUTF8(condition.Message[:maxMessage-3], "") + "..." + } + return condition +} + +func (c *compiledTarget) egressBase() (egress.Access, []string) { + if c == nil { + return egress.Access{}, nil + } + return egress.Access{Destinations: c.Revision.EgressDestinations, Credentials: c.Revision.Credentials}, c.Revision.TokenPassthroughOrigins +} diff --git a/go/core/internal/controller/grant_admission_test.go b/go/core/internal/controller/grant_admission_test.go new file mode 100644 index 0000000000..f3da9908f8 --- /dev/null +++ b/go/core/internal/controller/grant_admission_test.go @@ -0,0 +1,144 @@ +package controller + +import ( + "strings" + "testing" + "testing/synctest" + + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/substrate" + "github.com/kagent-dev/kagent/go/core/internal/translator" +) + +func TestGrantAdmissionAcrossSources(t *testing.T) { + for _, kind := range []string{"agent", "sandbox"} { + t.Run(kind, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + opts := krt.NewOptionsBuilder(t.Context().Done(), "admission", nil) + var policies krt.StaticCollection[*v1alpha3.EgressPolicy] + var reconcile func() error + var stored func() egress.Access + var conditions func() []metav1.Condition + switch kind { + case "agent": + collections, _ := newPreparationTestCollections(t, "gvisor") + initial := collections.Reconciliations.List()[0] + agent := initial.Agent.DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "a"}, {Name: "b"}} + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + policies = collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]) + template := proto.CloneOf(initial.Target.ActorTemplate) + template.Metadata.Uid, template.Status = "prepared", sandboxGoldenStatus() + store := &fakeRuntimeRevisionStore{} + r := &Reconciler{collections: collections, templates: &fakeActorTemplates{template: template}, store: store} + reconcile = func() error { return r.reconcileAgent(t.Context(), initial.ResourceName()) } + stored = func() egress.Access { return store.pair.EgressGrant } + conditions = func() []metav1.Condition { + state := collections.Reconciliations.GetKey(initial.ResourceName()) + require.NotNil(t, state.Target, "grant conflicts must not block compilation") + require.Equal(t, initial.Target.RevisionID, state.Target.RevisionID) + return statusForAgent(*state, agent.Generation, "").Conditions + } + case "sandbox": + template := sandboxTestTemplate() + template.Finalizers = []string{sandboxPreparationFinalizer} + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "a"}, {Name: "b"}} + templates := krt.NewStaticCollection(nil, []*v1alpha3.SandboxTemplate{template}, opts.WithName("templates")...) + policies = krt.NewStaticCollection[*v1alpha3.EgressPolicy](nil, nil, opts.WithName("policies")...) + pools := krt.NewStaticCollection(nil, []*atev1alpha1.WorkerPool{{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "default"}}}, opts.WithName("pools")...) + store := &sandboxTestStore{} + actors := &sandboxTestActors{store: store, templates: map[string]*ateapipb.ActorTemplate{}, autoReady: true} + s := &SandboxReconciler{collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools, EgressPolicies: policies}, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, opts), + store: store, actors: actors, client: kagentfake.NewSimpleClientset(template).ApiV1alpha3()} + reconcile = func() error { return s.reconcile(t.Context(), "team-a/scratch") } + stored = func() egress.Access { return store.desired.EgressGrant } + conditions = func() []metav1.Condition { + state := s.collections.states.GetKey("team-a/scratch") + require.NotNil(t, state.DesiredActorTemplate) + return sandboxStatusWithTransitionTimes(*state).Conditions + } + } + policy := func(name string, origins ...string) *v1alpha3.EgressPolicy { + p := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: name}} + for _, origin := range origins { + p.Spec.Destinations = append(p.Spec.Destinations, v1alpha3.EgressDestination{Origin: origin}) + } + return p + } + check := func(reason string, origins ...string) { + t.Helper() + synctest.Wait() + require.NoError(t, reconcile()) + synctest.Wait() + require.Equal(t, origins, stored().Destinations) + observed := conditions() + require.True(t, meta.IsStatusConditionTrue(observed, "Ready"), "%v", observed) + condition := meta.FindStatusCondition(observed, "EgressPoliciesApplied") + require.NotNil(t, condition) + require.Equal(t, reason, condition.Reason) + if reason == v1alpha3.EgressPoliciesReasonApplied { + require.Equal(t, metav1.ConditionTrue, condition.Status) + } else { + require.Equal(t, metav1.ConditionFalse, condition.Status) + } + } + const a = "https://a.example.com:443" + const x = "https://x.example.com:443" + policies.UpdateObject(policy("a", a)) + policies.UpdateObject(policy("b", x)) + check(v1alpha3.EgressPoliciesReasonApplied, a, x) + // The new HTTP rule conflicts with a. An unrelated addition is held + // too, and the removed x is revoked immediately. + policies.UpdateObject(policy("b", "http://a.example.com:443", "https://new.example.com")) + check(v1alpha3.EgressPoliciesReasonConflict, a) + // Repeating preparation cannot admit previously held additions. + check(v1alpha3.EgressPoliciesReasonConflict, a) + policies.DeleteObject("team-a/b") + check(v1alpha3.EgressPoliciesReasonNotFound, a) + policies.UpdateObject(policy("b", "https://new.example.com")) + check(v1alpha3.EgressPoliciesReasonApplied, a, "https://new.example.com:443") + }) + }) + } +} + +func TestGrantAdmissionWithChangedBase(t *testing.T) { + const a = "https://a.example.com:443" + const model = "https://model.example.com:443" + old := egress.Access{Destinations: []string{a, model}} + base := egress.Access{Destinations: []string{"http://model.example.com:443"}} + candidate, issues, conflict := evaluateEgressGrant(translator.EgressGrantResolution{Access: old}, base, nil) + require.True(t, conflict) + require.Len(t, issues, 1) + require.Equal(t, egress.Access{Destinations: []string{a}}, retainedEgressGrant(old, candidate, base, nil)) + // With no compiled target, admission can still revoke grants safely. + require.Equal(t, old, retainedEgressGrant(old, candidate, egress.Access{}, nil)) + require.Empty(t, retainedEgressGrant(egress.Access{}, candidate, base, nil).Destinations, "new identities inherit no grants") +} + +func TestEgressConditionSurvivesRuntimeFailureAndBoundsMessages(t *testing.T) { + issues := []translator.EgressGrantIssue{ + {Policy: "team/missing", Reason: v1alpha3.EgressPoliciesReasonNotFound, Message: "missing"}, + {Reason: v1alpha3.EgressPoliciesReasonConflict, Message: strings.Repeat("conflict", 1000)}, + } + state := AgentReconciliation{EgressIssues: issues, EgressConflict: true, + CompilationFailure: &ReconciliationFailure{Condition: v1alpha3.AgentConditionResolvedRefs, Reason: "ReferenceResolutionFailed", Message: "Harness is missing"}} + status := statusForAgent(state, 1, "") + condition := meta.FindStatusCondition(status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied) + require.Equal(t, v1alpha3.EgressPoliciesReasonConflict, condition.Reason) + require.Contains(t, condition.Message, "new grants are held") + require.Contains(t, condition.Message, "team/missing") + require.LessOrEqual(t, len(condition.Message), 4096) + require.False(t, meta.IsStatusConditionTrue(status.Conditions, v1alpha3.AgentConditionReady)) +} diff --git a/go/core/internal/controller/grants_test.go b/go/core/internal/controller/grants_test.go index 9c2d6bfcce..1c2f20a557 100644 --- a/go/core/internal/controller/grants_test.go +++ b/go/core/internal/controller/grants_test.go @@ -25,16 +25,16 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/substrate" ) -func TestAgentEgressErrorsUseCompilationClassification(t *testing.T) { +func TestAgentEgressErrorsDoNotBlockCompilation(t *testing.T) { for _, test := range []struct { name string destinations []v1alpha3.EgressDestination condition string reason string }{ - {name: "missing policy", condition: v1alpha3.AgentConditionResolvedRefs, reason: "ReferenceResolutionFailed"}, - {name: "invalid origin", destinations: []v1alpha3.EgressDestination{{Origin: "invalid"}}, condition: v1alpha3.AgentConditionCompatible, reason: "UnsupportedConfiguration"}, - {name: "missing secret", destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com", Authentication: &v1alpha3.EgressAuthentication{Header: "Authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "missing"}, Key: "token"}}}}, condition: v1alpha3.AgentConditionResolvedRefs, reason: "ReferenceResolutionFailed"}, + {name: "missing policy", condition: v1alpha3.AgentConditionEgressPoliciesApplied, reason: v1alpha3.EgressPoliciesReasonNotFound}, + {name: "invalid origin", destinations: []v1alpha3.EgressDestination{{Origin: "invalid"}}, condition: v1alpha3.AgentConditionEgressPoliciesApplied, reason: v1alpha3.EgressPoliciesReasonInvalidEgressPolicy}, + {name: "missing secret", destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com", Authentication: &v1alpha3.EgressAuthentication{Header: "Authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "missing"}, Key: "token"}}}}, condition: v1alpha3.AgentConditionEgressPoliciesApplied, reason: v1alpha3.EgressPoliciesReasonInvalidEgressPolicy}, } { t.Run(test.name, func(t *testing.T) { collections, _ := newPreparationTestCollections(t, "gvisor") @@ -54,7 +54,8 @@ func TestAgentEgressErrorsUseCompilationClassification(t *testing.T) { condition := meta.FindStatusCondition(collections.AgentStatuses.GetKey("team-a/assistant").Status.Conditions, test.condition) require.Equal(t, test.reason, condition.Reason) state := collections.Reconciliations.GetKey("team-a/assistant") - require.Nil(t, state.Target) + require.NotNil(t, state.Target) + require.Nil(t, state.CompilationFailure) require.Empty(t, state.EgressGrant.Destinations) require.Empty(t, state.EgressGrant.Credentials) }) @@ -259,8 +260,8 @@ func TestSandboxGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { require.Equal(t, []string{"https://tickets.example.com:443"}, store.desired.EgressGrant.Destinations) require.Len(t, store.sources, 1) policies.DeleteObject("team-a/access") - waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch").CompilationError != "" }) - require.Error(t, s.reconcile(t.Context(), "team-a/scratch")) + waitFor(t, func() bool { return len(s.collections.states.GetKey("team-a/scratch").EgressIssues) != 0 }) + require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) require.Empty(t, store.desired.EgressGrant.Destinations) require.Len(t, store.sources, 2, "revocation must reach existing actors during failed compilation") store.upsertErr = errors.New("response lost") diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index a09d4336f9..4bd467a12c 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -593,3 +593,9 @@ func (p *preparationTestBackend) GetActor(ctx context.Context, atespace, name st func (p *preparationTestBackend) EnsureActorEgressPolicy(ctx context.Context, atespace, name, revision string) (egress.Policy, error) { return p.templates.EnsureActorEgressPolicy(ctx, atespace, name, revision) } + +func (p *preparationTestBackend) GetAgentEgressGrant(ctx context.Context, namespace, uid string) (egress.Access, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.store.GetAgentEgressGrant(ctx, namespace, uid) +} diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index f9a4dced33..fcd21740d6 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -42,6 +42,8 @@ import ( type AgentReconciliation struct { Agent *kagentv1alpha3.Agent EgressGrant egress.Access + EgressIssues []translator.EgressGrantIssue + EgressConflict bool Target *compiledTarget Warnings []string CompilationFailure *ReconciliationFailure @@ -108,12 +110,12 @@ func newAgentReconciliations( if !agent.DeletionTimestamp.IsZero() { return state } - grants, grantErr := translator.CompileEgressPolicies(ctx, collections, agent.Namespace, agent.Spec.EgressPolicyRefs) - if grantErr != nil { - state.CompilationFailure = compilationFailure(grantErr) - return state - } - state.EgressGrant = grants + resolution := translator.CompileEgressPolicies(ctx, collections, agent.Namespace, agent.Spec.EgressPolicyRefs) + // Admission still runs when an unrelated runtime input fails to compile. + defer func() { + base, tokenOrigins := state.Target.egressBase() + state.EgressGrant, state.EgressIssues, state.EgressConflict = evaluateEgressGrant(resolution, base, tokenOrigins) + }() compilation, err := translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings), @@ -183,6 +185,7 @@ func compilationFailure(err error) *ReconciliationFailure { type runtimeRevisionStore interface { actorEgressStore UpsertAgentDefinition(context.Context, database.AgentDefinition) error + GetAgentEgressGrant(context.Context, string, string) (egress.Access, error) RecordRuntimeRevision(context.Context, database.RuntimeRevision, database.RecordRevisionOptions) error ListAgentDefinitionSources(context.Context) ([]database.RuntimeEgressSource, error) RetireAgentIdentities(ctx context.Context, namespace, name string) error @@ -248,7 +251,9 @@ func newReconciler( // and status writes do not. Grants must reconcile without a new revision. if event.Old != nil && event.New != nil && event.Old.Agent.UID == event.New.Agent.UID && event.Old.desiredRevision() == event.New.desiredRevision() && - reflect.DeepEqual(event.Old.EgressGrant, event.New.EgressGrant) && event.Old.Agent.DeletionTimestamp.Equal(event.New.Agent.DeletionTimestamp) { + reflect.DeepEqual(event.Old.EgressGrant, event.New.EgressGrant) && + reflect.DeepEqual(event.Old.EgressIssues, event.New.EgressIssues) && + event.Old.EgressConflict == event.New.EgressConflict && event.Old.Agent.DeletionTimestamp.Equal(event.New.Agent.DeletionTimestamp) { return } r.agents.Add(krt.GetKey(event.Latest())) @@ -360,9 +365,18 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { } return nil } + grants := state.EgressGrant + if state.EgressConflict { + previous, err := r.store.GetAgentEgressGrant(ctx, state.Agent.Namespace, string(state.Agent.UID)) + if err != nil { + return r.observePreparationError(*state, fmt.Errorf("read Agent %s grants: %w", key, err)) + } + base, tokenOrigins := state.Target.egressBase() + grants = retainedEgressGrant(previous, grants, base, tokenOrigins) + } definition := database.AgentDefinition{ Namespace: state.Agent.Namespace, AgentName: state.Agent.Name, - AgentUID: string(state.Agent.UID), DesiredRevision: state.desiredRevision(), EgressGrant: state.EgressGrant, + AgentUID: string(state.Agent.UID), DesiredRevision: state.desiredRevision(), EgressGrant: grants, } // Store the desired edge before creating compute so a concurrent collector diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index d812ef28ee..730151ffac 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -27,6 +27,7 @@ import ( kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -467,3 +468,10 @@ func TestReconciliationQueueRetriesWithBackoff(t *testing.T) { require.NoError(t, queue.WaitForClose(time.Second)) }) } + +func (s *fakeRuntimeRevisionStore) GetAgentEgressGrant(_ context.Context, namespace, uid string) (egress.Access, error) { + if s.pair != nil && s.pair.Namespace == namespace && s.pair.AgentUID == uid && s.retired == "" { + return s.pair.EgressGrant, nil + } + return egress.Access{}, nil +} diff --git a/go/core/internal/controller/sandbox_collections.go b/go/core/internal/controller/sandbox_collections.go index f0ec4144ac..ebf6102120 100644 --- a/go/core/internal/controller/sandbox_collections.go +++ b/go/core/internal/controller/sandbox_collections.go @@ -27,6 +27,8 @@ type sandboxReconciliation struct { RevisionID string SourceSnapshot json.RawMessage EgressGrant egress.Access + EgressIssues []translator.EgressGrantIssue + EgressConflict bool DesiredActorTemplate *ateapipb.ActorTemplate ObservedActorTemplate *ateapipb.ActorTemplate CompilationError string @@ -87,17 +89,14 @@ func newSandboxCollections(inputs Collections, policy substrate.SandboxPolicy, o if !template.DeletionTimestamp.IsZero() { return state } - grants, err := translator.CompileEgressPolicies(ctx, translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets}, template.Namespace, template.Spec.EgressPolicyRefs) - if err != nil { - state.CompilationError = err.Error() - state.Failure = &ReconciliationFailure{Reason: kagentv1alpha3.SandboxTemplateReasonEgressPolicyResolutionFailed, Message: err.Error()} - return state - } - state.EgressGrant = grants + resolution := translator.CompileEgressPolicies(ctx, translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets}, template.Namespace, template.Spec.EgressPolicyRefs) + // Sandboxes have no pinned model or tool access. + state.EgressGrant, state.EgressIssues, state.EgressConflict = evaluateEgressGrant(resolution, egress.Access{}, nil) pool := krt.FetchOne(ctx, inputs.WorkerPools, krt.FilterKey(template.Namespace+"/"+template.Spec.Substrate.WorkerPoolRef.Name)) if pool == nil { state.Failure = &ReconciliationFailure{Reason: "WorkerPoolNotFound", Message: "The referenced sandbox WorkerPool does not exist"} } else { + var err error state.DesiredActorTemplate, state.RevisionID, state.SourceSnapshot, err = substrate.SandboxActorTemplate(template, (*pool).Spec.SandboxClass, policy) if err != nil { state.CompilationError = err.Error() diff --git a/go/core/internal/controller/sandbox_integration_test.go b/go/core/internal/controller/sandbox_integration_test.go index c2a16ce1d4..c9cb9c079c 100644 --- a/go/core/internal/controller/sandbox_integration_test.go +++ b/go/core/internal/controller/sandbox_integration_test.go @@ -92,24 +92,25 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { } return apimeta.IsStatusConditionTrue(current.Status.Conditions, "Ready") } - waitUnresolved := func(message string) { + waitUnresolved := func(reason, message string) { t.Helper() require.Eventually(t, func() bool { current, err := client.ApiV1alpha3().SandboxTemplates(template.Namespace).Get(t.Context(), template.Name, metav1.GetOptions{}) if err != nil { return false } - condition := apimeta.FindStatusCondition(current.Status.Conditions, "Ready") - return condition != nil && condition.Status == metav1.ConditionFalse && condition.Reason == kagentv1alpha3.SandboxTemplateReasonEgressPolicyResolutionFailed && strings.Contains(condition.Message, message) + condition := apimeta.FindStatusCondition(current.Status.Conditions, kagentv1alpha3.SandboxTemplateConditionEgressPoliciesApplied) + return condition != nil && condition.Status == metav1.ConditionFalse && condition.Reason == reason && strings.Contains(condition.Message, message) }, 10*time.Second, 10*time.Millisecond) } - waitUnresolved("EgressPolicy") + waitUnresolved(kagentv1alpha3.EgressPoliciesReasonNotFound, "tickets") + require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) grant := &kagentv1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "tickets"}, Spec: kagentv1alpha3.EgressPolicySpec{ Destinations: []kagentv1alpha3.EgressDestination{{Origin: "https://tickets.example.com", Authentication: &kagentv1alpha3.EgressAuthentication{Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "ticket-auth"}, Key: "token"}}}}, }} grant, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Create(t.Context(), grant, metav1.CreateOptions{}) require.NoError(t, err) - waitUnresolved("Secret") + waitUnresolved(kagentv1alpha3.EgressPoliciesReasonInvalidEgressPolicy, "Secret") secret, err := kube.Kube().CoreV1().Secrets(template.Namespace).Create(t.Context(), &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "ticket-auth"}, Data: map[string][]byte{"token": []byte("original-token")}}, metav1.CreateOptions{}) require.NoError(t, err) require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) @@ -126,12 +127,13 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { require.Empty(t, other.Finalizers, "namespace filters must apply to the new informer") require.Empty(t, other.Status.Conditions) - // A missing key invalidates preparation; restoring it with a new value - // resolves the same pinned revision without serializing either value. + // A missing key revokes its policy without changing runtime readiness. + // Restoring it admits grants without serializing either value. secret.Data = nil secret, err = kube.Kube().CoreV1().Secrets(template.Namespace).Update(t.Context(), secret, metav1.UpdateOptions{}) require.NoError(t, err) - waitUnresolved("missing key") + waitUnresolved(kagentv1alpha3.EgressPoliciesReasonInvalidEgressPolicy, "missing key") + require.True(t, ready()) secret.Data = map[string][]byte{"token": []byte("rotated-token")} _, err = kube.Kube().CoreV1().Secrets(template.Namespace).Update(t.Context(), secret, metav1.UpdateOptions{}) require.NoError(t, err) @@ -143,7 +145,8 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { grant, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Update(t.Context(), grant, metav1.UpdateOptions{}) require.NoError(t, err) require.NoError(t, client.ApiV1alpha3().EgressPolicies(template.Namespace).Delete(t.Context(), grant.Name, metav1.DeleteOptions{})) - waitUnresolved("EgressPolicy") + waitUnresolved(kagentv1alpha3.EgressPoliciesReasonNotFound, "tickets") + require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) grant.ResourceVersion, grant.UID = "", "" _, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Create(t.Context(), grant, metav1.CreateOptions{}) require.NoError(t, err) diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index e68b0eb3e4..ddec7857c0 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -21,6 +21,7 @@ import ( kagentclient "github.com/kagent-dev/kagent/go/api/clientset/versioned/typed/api/v1alpha3" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" ) @@ -33,6 +34,7 @@ const ( type sandboxRevisionStore interface { actorEgressStore UpsertSandboxTemplateDefinition(context.Context, database.SandboxTemplateDefinition) error + GetSandboxEgressGrant(context.Context, string, string) (egress.Access, error) RetireSandboxTemplateIdentities(context.Context, string, string) error RecordSandboxRevision(context.Context, database.SandboxRevision, database.RecordRevisionOptions) error } @@ -139,8 +141,16 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { } return nil } + grants := state.EgressGrant + if state.EgressConflict { + previous, err := s.store.GetSandboxEgressGrant(ctx, state.Template.Namespace, string(state.Template.UID)) + if err != nil { + return s.observeError(*state, fmt.Errorf("read SandboxTemplate %s grants: %w", key, err), true) + } + grants = retainedEgressGrant(previous, grants, egress.Access{}, nil) + } err := s.store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{ - Namespace: state.Template.Namespace, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), DesiredRevision: state.desiredRevision(), EgressGrant: state.EgressGrant, + Namespace: state.Template.Namespace, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), DesiredRevision: state.desiredRevision(), EgressGrant: grants, }) if err != nil { return s.observeError(*state, fmt.Errorf("store SandboxTemplate %s definition: %w", key, err), true) @@ -231,5 +241,6 @@ func sandboxStatusWithTransitionTimes(state sandboxReconciliation) kagentv1alpha } desired.ObservedGeneration = state.Template.Generation apimeta.SetStatusCondition(&desired.Conditions, condition) + apimeta.SetStatusCondition(&desired.Conditions, egressPoliciesCondition(kagentv1alpha3.SandboxTemplateConditionEgressPoliciesApplied, state.Template.Generation, state.EgressIssues, state.EgressConflict)) return desired } diff --git a/go/core/internal/controller/sandbox_reconciler_test.go b/go/core/internal/controller/sandbox_reconciler_test.go index 44b41fd51d..bd22e0d564 100644 --- a/go/core/internal/controller/sandbox_reconciler_test.go +++ b/go/core/internal/controller/sandbox_reconciler_test.go @@ -25,6 +25,7 @@ import ( kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" ) @@ -303,3 +304,12 @@ func TestSandboxPendingStatusRecoversWithoutGraphEvent(t *testing.T) { return err == nil && apimeta.IsStatusConditionTrue(current.Status.Conditions, "Ready") }, 5*time.Second, 10*time.Millisecond) } + +func (s *sandboxTestStore) GetSandboxEgressGrant(_ context.Context, namespace, uid string) (egress.Access, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.desired.Namespace == namespace && s.desired.SandboxTemplateUID == uid && !s.retired { + return s.desired.EgressGrant, nil + } + return egress.Access{}, nil +} diff --git a/go/core/internal/controller/status.go b/go/core/internal/controller/status.go index 47828ad737..dff6ce40d0 100644 --- a/go/core/internal/controller/status.go +++ b/go/core/internal/controller/status.go @@ -23,6 +23,7 @@ func statusForAgent(state AgentReconciliation, generation int64, latestSuccessfu status := kagentv1alpha3.AgentStatus{ ObservedGeneration: generation, DesiredRevision: state.desiredRevision(), LatestSuccessfulRevision: latestSuccessful, } + status.Conditions = append(status.Conditions, egressPoliciesCondition(kagentv1alpha3.AgentConditionEgressPoliciesApplied, generation, state.EgressIssues, state.EgressConflict)) status.Warnings = append([]string(nil), state.Warnings...) if state.Target != nil { status.WorkloadImage = new(state.Target.Revision.Image) diff --git a/go/core/internal/database/runtime_egress.go b/go/core/internal/database/runtime_egress.go index 00c70b6050..2dbc3d4d68 100644 --- a/go/core/internal/database/runtime_egress.go +++ b/go/core/internal/database/runtime_egress.go @@ -2,6 +2,7 @@ package database import ( "context" + "errors" "fmt" "github.com/jackc/pgx/v5" @@ -26,8 +27,8 @@ func canonicalStoredGrants(grants egress.Access) (egress.Access, error) { // GetRuntimeEgressPolicy merges pinned inputs with the active source UID's grants // in one database snapshot. Missing/deleting revisions and malformed inputs fail -// closed. Retired or replaced identities contribute no grants. Merge conflicts -// retain pinned access and return a diagnostic, not a failed read. +// closed. Retired or replaced identities contribute no grants. Older revisions +// retain their pinned base and only the grants compatible with it. func (c *Client) GetRuntimeEgressPolicy(ctx context.Context, revision string) (egress.Policy, error) { type policyInputs struct { EgressDestinations []string @@ -66,8 +67,8 @@ func (c *Client) GetRuntimeEgressPolicy(ctx context.Context, revision string) (e if err != nil { return egress.Policy{}, fmt.Errorf("decode runtime egress grants %s: %w", revision, err) } - access, withheld := egress.EffectiveAccess(base, grants, row.TokenPassthroughOrigins) - return egress.Policy{Access: access, WithheldGrants: withheld}, nil + access, err := egress.EffectiveAccess(base, grants, row.TokenPassthroughOrigins) + return egress.Policy{Access: access}, err } var _ egress.PolicyStore = (*Client)(nil) @@ -138,3 +139,19 @@ func (c *Client) ListRuntimeEgressSourceTargets(ctx context.Context, source Runt return nil, fmt.Errorf("unknown runtime egress source kind %q", source.Kind) } } + +// decodeEgressGrant treats absent definitions as empty grants and rejects invalid +// durable rules. Callers scope the read to an active Kubernetes source identity. +func decodeEgressGrant(grants egress.Access, err error) (egress.Access, error) { + if errors.Is(err, pgx.ErrNoRows) { + return egress.Access{}, nil + } + if err != nil { + return egress.Access{}, err + } + canonical, err := egress.CanonicalAccess(grants.Destinations, grants.Credentials) + if err != nil { + return egress.Access{}, fmt.Errorf("decode source egress grants: %w", err) + } + return canonical, nil +} diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go index cdbf56cd88..fdb8bf2ca5 100644 --- a/go/core/internal/database/runtime_egress_test.go +++ b/go/core/internal/database/runtime_egress_test.go @@ -1,6 +1,7 @@ package database import ( + "context" "fmt" "slices" "testing" @@ -134,7 +135,6 @@ func TestRuntimeEgressTokenOriginsAreNotGrants(t *testing.T) { policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") require.NoError(t, err) require.Empty(t, policy.Destinations, "token origins constrain credentials without granting network access") - require.NoError(t, policy.WithheldGrants) }) } } @@ -154,8 +154,7 @@ func TestRuntimeEgressCanonicalTokenOriginWithholdsCredentials(t *testing.T) { }, RecordRevisionOptions{})) policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") require.NoError(t, err) - require.ErrorContains(t, policy.WithheldGrants, "caller-token passthrough") - require.Empty(t, policy.Destinations) + require.Equal(t, []string{"https://model.example.com:443"}, policy.Destinations) require.Empty(t, policy.Credentials) } @@ -196,20 +195,20 @@ func TestSourceGrantsDetermineEffectivePolicy(t *testing.T) { require.NoError(t, err) require.True(t, slices.Equal(expected.Destinations, policy.Destinations)) require.True(t, slices.Equal(expected.Credentials, policy.Credentials)) - require.NoError(t, policy.WithheldGrants) } assertPolicy(canonical) err = upsert("uid", egress.Access{Destinations: []string{"invalid"}}) require.Error(t, err) assertPolicy(canonical) if kind == RuntimeEgressAgent { - conflict := egress.Access{Destinations: base.Destinations, Credentials: []egress.Credential{{Origin: base.Destinations[0], Header: "authorization", URI: "ate-secret://k8s.io/default/team/other/token"}}} + conflict := egress.Access{Destinations: append(slices.Clone(base.Destinations), canonical.Destinations...), Credentials: append(slices.Clone(canonical.Credentials), egress.Credential{Origin: base.Destinations[0], Header: "authorization", URI: "ate-secret://k8s.io/default/team/other/token"})} err = upsert("uid", conflict) require.NoError(t, err) policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") require.NoError(t, err) - require.Error(t, policy.WithheldGrants) - require.Equal(t, base, policy.Access) + expected, err := egress.MergeAccess(base, canonical, nil) + require.NoError(t, err) + require.Equal(t, expected, policy.Access, "older revisions retain nonconflicting grants") } if kind == RuntimeEgressAgent { require.NoError(t, client.RetireAgentIdentities(ctx, "team", "source")) @@ -273,3 +272,43 @@ func TestRuntimeEgressMalformedInputsFailClosed(t *testing.T) { }) } } + +func TestReadAdmittedGrantsScopesIdentity(t *testing.T) { + for _, kind := range []RuntimeEgressSourceKind{RuntimeEgressAgent, RuntimeEgressSandbox} { + t.Run(string(kind), func(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + var read func(context.Context, string, string) (egress.Access, error) + var retire func(context.Context, string, string) error + upsert := func(uid string, grants egress.Access) error { + if kind == RuntimeEgressAgent { + return client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: uid, EgressGrant: grants}) + } + return client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team", SandboxTemplateName: "source", SandboxTemplateUID: uid, EgressGrant: grants}) + } + if kind == RuntimeEgressAgent { + read, retire = client.GetAgentEgressGrant, client.RetireAgentIdentities + } else { + read, retire = client.GetSandboxEgressGrant, client.RetireSandboxTemplateIdentities + } + grants := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} + require.NoError(t, upsert("uid", grants)) + got, err := read(ctx, "team", "uid") + require.NoError(t, err) + require.Equal(t, grants.Destinations, got.Destinations) + for _, scope := range [][2]string{{"team", "new"}, {"other", "uid"}} { + got, err = read(ctx, scope[0], scope[1]) + require.NoError(t, err) + require.Empty(t, got.Destinations) + } + require.NoError(t, retire(ctx, "team", "source")) + got, err = read(ctx, "team", "uid") + require.NoError(t, err) + require.Empty(t, got.Destinations, "retired grants must not be resurrected") + require.NoError(t, upsert("replacement", grants)) + got, err = read(ctx, "team", "uid") + require.NoError(t, err) + require.Empty(t, got.Destinations, "grants belong to the exact source UID") + }) + } +} diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index 83714f82bd..041cfdcf9f 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -68,13 +68,30 @@ func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefi }) } -// ListAgentDefinitionSources returns distinct Agent names across active and -// retired identities. Startup recovery includes retired rows because a crash -// may interrupt actor revocation after retirement commits. +// GetAgentEgressGrant returns the active identity's last admitted grants. +// Missing or retired identities return empty access; malformed stored grants fail. +func (c *Client) GetAgentEgressGrant(ctx context.Context, namespace, uid string) (egress.Access, error) { + grants, err := queryOne(ctx, c.db, ` + SELECT grant_destinations AS destinations, grant_credentials AS credentials + FROM agent_definition WHERE namespace = $1 AND agent_uid = $2 AND retired_at IS NULL + `, pgx.RowToStructByName[egress.Access], namespace, uid) + return decodeEgressGrant(grants, err) +} + +// ListAgentDefinitionSources returns active Agent names and retired identities +// that still have non-deleted instances. Startup recovery includes these retired +// rows because a crash may interrupt actor revocation after retirement commits. func (c *Client) ListAgentDefinitionSources(ctx context.Context) ([]RuntimeEgressSource, error) { return queryMany(ctx, c.db, ` - SELECT DISTINCT 'agent' AS kind, namespace, agent_name AS name - FROM agent_definition + SELECT DISTINCT 'agent' AS kind, d.namespace, d.agent_name AS name + FROM agent_definition d + WHERE d.retired_at IS NULL OR EXISTS ( + SELECT 1 FROM agent_revision a + JOIN runtime_revision r ON r.revision = a.revision + JOIN runtime_instance i ON i.prepared_revision = r.revision + WHERE r.namespace = d.namespace AND a.agent_uid = d.agent_uid + AND i.state <> 'RUNTIME_STATE_DELETED' + ) ORDER BY namespace, name `, pgx.RowToStructByName[RuntimeEgressSource]) } diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index 4d99d95c75..5bcdfa1655 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -35,23 +35,33 @@ func TestRetireAgentIdentities(t *testing.T) { require.Equal(t, "revision", revisions[0].Revision) } -func TestListAgentDefinitionSourcesIncludesRetiredIdentities(t *testing.T) { +func TestListAgentDefinitionSourcesBoundsRetiredIdentities(t *testing.T) { client := NewClient(setupTestDB(t), "public") ctx := t.Context() + sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") + session, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "request") + require.NoError(t, err) + _, err = finishSessionOperation(ctx, client, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, "actor.example.com") + require.NoError(t, err) for _, definition := range []AgentDefinition{ - {Namespace: "team-a", AgentName: "assistant", AgentUID: "old"}, {Namespace: "team-a", AgentName: "assistant", AgentUID: "replacement"}, + {Namespace: "team-a", AgentName: "unused", AgentUID: "unused"}, {Namespace: "team-b", AgentName: "assistant", AgentUID: "other"}, } { require.NoError(t, client.UpsertAgentDefinition(ctx, definition)) } require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "unused")) sources, err := client.ListAgentDefinitionSources(ctx) require.NoError(t, err) require.Equal(t, []RuntimeEgressSource{ {Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, {Kind: RuntimeEgressAgent, Namespace: "team-b", Name: "assistant"}, - }, sources, "retirement may commit before actor revocation; recovery must retain that source and deduplicate old UIDs") + }, sources, "include only active sources and retired identities that still have actors") + require.NoError(t, deleteSession(ctx, client, session.Id)) + sources, err = client.ListAgentDefinitionSources(ctx) + require.NoError(t, err) + require.Equal(t, []RuntimeEgressSource{{Kind: RuntimeEgressAgent, Namespace: "team-b", Name: "assistant"}}, sources) } func TestRuntimeRevisionCollectionAfterPairRetirement(t *testing.T) { diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index ec08e0df01..6efbd1faee 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -70,6 +70,16 @@ func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition }) } +// GetSandboxEgressGrant returns the active identity's last admitted grants. +// Missing or retired identities return empty access; malformed stored grants fail. +func (c *Client) GetSandboxEgressGrant(ctx context.Context, namespace, uid string) (egress.Access, error) { + grants, err := queryOne(ctx, c.db, ` + SELECT grant_destinations AS destinations, grant_credentials AS credentials + FROM sandbox_template_definition WHERE namespace = $1 AND sandbox_template_uid = $2 AND retired_at IS NULL + `, pgx.RowToStructByName[egress.Access], namespace, uid) + return decodeEgressGrant(grants, err) +} + // RetireSandboxTemplateIdentities retires active identities at this source name. // Missing/already-retired identities are a no-op; retained actors lose live grants. func (c *Client) RetireSandboxTemplateIdentities(ctx context.Context, namespace, name string) error { diff --git a/go/core/internal/egress/live.go b/go/core/internal/egress/live.go index f08e6206da..e07b66df87 100644 --- a/go/core/internal/egress/live.go +++ b/go/core/internal/egress/live.go @@ -6,10 +6,9 @@ import ( "slices" ) -// Policy is effective access plus a diagnostic when conflicting grants were withheld. +// Policy is the effective access for one runtime revision. type Policy struct { Access - WithheldGrants error } // PolicyStore reads effective policy from persisted revision inputs and source grants. @@ -18,14 +17,11 @@ type PolicyStore interface { GetRuntimeEgressPolicy(context.Context, string) (Policy, error) } -// EffectiveAccess merges validated pinned access and source grants. A conflict -// withholds all explicit grants while retaining the pinned base and its diagnostic. +// EffectiveAccess retains the base and all grants compatible with this revision. +// Stored grants can conflict with an older actor's pinned base. func EffectiveAccess(base, grants Access, tokenPassthroughOrigins []string) (Access, error) { - access, err := MergeAccess(base, grants, tokenPassthroughOrigins) - if err != nil { - return base, err - } - return access, nil + filtered, _ := FilterConflicts(base, grants, tokenPassthroughOrigins) + return MergeAccess(base, filtered, tokenPassthroughOrigins) } // MergeAccess validates live grants against the pinned runtime's credentials and @@ -42,3 +38,58 @@ func MergeAccess(base, grants Access, tokenPassthroughOrigins []string) (Access, } return access, nil } + +// IntersectAccess retains only destinations and complete credential bindings +// present in both inputs. Inputs contain canonical rules; b may be an invalid +// union of otherwise valid policies. Taking a subset of valid a remains valid. +func IntersectAccess(a, b Access) Access { + var result Access + for _, origin := range a.Destinations { + if slices.Contains(b.Destinations, origin) { + result.Destinations = append(result.Destinations, origin) + } + } + for _, credential := range a.Credentials { + if slices.Contains(result.Destinations, credential.Origin) && slices.Contains(b.Credentials, credential) { + result.Credentials = append(result.Credentials, credential) + } + } + return result +} + +// FilterConflicts retains grants compatible with a valid base and caller-token +// origins. Inputs must be individually canonical and valid. The base wins; +// sorted grant order determines which rules fit when the combined limits are +// exceeded. A conflicting credential does not remove its network destination. +// Neither input is modified; dropped origins are sorted and deduplicated. +func FilterConflicts(base, grants Access, tokenOrigins []string) (Access, []string) { + if _, err := MergeAccess(base, grants, tokenOrigins); err == nil { + return grants, nil + } + var result Access + var dropped []string + for _, origin := range grants.Destinations { + candidate := Access{Destinations: append(slices.Clone(result.Destinations), origin)} + if _, err := MergeAccess(base, candidate, tokenOrigins); err != nil { + dropped = append(dropped, origin) + } else { + result = candidate + } + } + for _, credential := range grants.Credentials { + if !slices.Contains(result.Destinations, credential.Origin) { + continue + } + candidate := Access{ + Destinations: result.Destinations, + Credentials: append(slices.Clone(result.Credentials), credential), + } + if _, err := MergeAccess(base, candidate, tokenOrigins); err != nil { + dropped = append(dropped, credential.Origin) + } else { + result = candidate + } + } + slices.Sort(dropped) + return result, slices.Compact(dropped) +} diff --git a/go/core/internal/egress/live_test.go b/go/core/internal/egress/live_test.go index ae718fdd69..ebe2f2c23d 100644 --- a/go/core/internal/egress/live_test.go +++ b/go/core/internal/egress/live_test.go @@ -2,45 +2,98 @@ package egress import ( "fmt" + "slices" "testing" "github.com/stretchr/testify/require" ) -func TestEffectiveAccess(t *testing.T) { +func canonicalTestAccess(t *testing.T, destinations []string, credentials ...Credential) Access { + t.Helper() + access, err := CanonicalAccess(destinations, credentials) + require.NoError(t, err) + return access +} + +func cloneAccess(a Access) Access { + return Access{Destinations: slices.Clone(a.Destinations), Credentials: slices.Clone(a.Credentials)} +} + +func TestIntersectAccess(t *testing.T) { + binding := Credential{Origin: "https://a.example.com", Header: "authorization", URI: "ate-secret://k8s.io/default/team/key/token"} + a := canonicalTestAccess(t, []string{"https://a.example.com:443", "https://removed.example.com"}, binding) + changed := a.Credentials[0] + changed.Prefix = "Bearer " + b := Access{Destinations: []string{"https://a.example.com:443", "https://new.example.com:443"}, Credentials: []Credential{changed, a.Credentials[0]}} + beforeA, beforeB := cloneAccess(a), cloneAccess(b) + require.Equal(t, Access{Destinations: []string{a.Destinations[0]}, Credentials: a.Credentials}, IntersectAccess(a, b)) + b.Credentials = []Credential{changed} + require.Empty(t, IntersectAccess(a, b).Credentials, "changed credential values are additions") + b.Credentials = beforeB.Credentials + require.Equal(t, beforeA, a) + require.Equal(t, beforeB, b) +} + +func TestFilterConflicts(t *testing.T) { binding := Credential{Origin: "https://model.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"} - base := Access{Destinations: []string{binding.Origin}, Credentials: []Credential{binding}} + base := canonicalTestAccess(t, []string{binding.Origin}, binding) conflict := binding conflict.URI = "ate-secret://k8s.io/default/team/other/token" - tooMany := Access{} - for i := range 256 { - tooMany.Destinations = append(tooMany.Destinations, fmt.Sprintf("https://host%d.example.com:443", i)) - } + const extra = "https://tickets.example.com:443" for _, test := range []struct { - name string - base, grants Access - origins []string - withheld bool + name string + base, grants, want Access + tokens, dropped []string }{ - {name: "no grants", base: base}, - {name: "additive", base: base, grants: Access{Destinations: []string{"https://tickets.example.com:443"}}}, - {name: "sandbox", grants: Access{Destinations: []string{"https://tickets.example.com:443"}}}, - {name: "duplicate", base: base, grants: base}, - {name: "credentials conflict", base: base, grants: Access{Destinations: base.Destinations, Credentials: []Credential{conflict}}, withheld: true}, - {name: "token passthrough", base: Access{Destinations: base.Destinations}, grants: base, origins: base.Destinations, withheld: true}, - {name: "rule limit", base: base, grants: tooMany, withheld: true}, + {name: "additive", base: base, grants: canonicalTestAccess(t, []string{extra}), want: canonicalTestAccess(t, []string{extra})}, + {name: "duplicate", base: base, grants: base, want: base}, + {name: "credential conflict keeps network access", base: base, grants: canonicalTestAccess(t, []string{binding.Origin, extra}, conflict), + want: canonicalTestAccess(t, []string{binding.Origin, extra}), dropped: []string{binding.Origin}}, + {name: "protocol conflict keeps other grants", base: base, grants: canonicalTestAccess(t, []string{"http://model.example.com:443", extra}), + want: canonicalTestAccess(t, []string{extra}), dropped: []string{"http://model.example.com:443"}}, + {name: "token passthrough", base: canonicalTestAccess(t, []string{binding.Origin}), tokens: []string{binding.Origin}, + grants: canonicalTestAccess(t, []string{binding.Origin, extra}, binding), want: canonicalTestAccess(t, []string{binding.Origin, extra}), dropped: []string{binding.Origin}}, } { t.Run(test.name, func(t *testing.T) { - access, err := EffectiveAccess(test.base, test.grants, test.origins) - if test.withheld { - require.Error(t, err) - require.Equal(t, test.base, access) - } else { - require.NoError(t, err) - for _, origin := range append(append([]string{}, test.base.Destinations...), test.grants.Destinations...) { - require.Contains(t, access.Destinations, origin) - } - } + beforeBase, beforeGrants := cloneAccess(test.base), cloneAccess(test.grants) + filtered, dropped := FilterConflicts(test.base, test.grants, test.tokens) + require.Equal(t, test.want, filtered) + require.Equal(t, test.dropped, dropped) + effective, err := EffectiveAccess(test.base, test.grants, test.tokens) + require.NoError(t, err) + want, err := MergeAccess(test.base, test.want, test.tokens) + require.NoError(t, err) + require.Equal(t, want, effective) + require.Equal(t, beforeBase, test.base) + require.Equal(t, beforeGrants, test.grants) }) } } + +func TestFilterConflictsLimits(t *testing.T) { + var destinations []string + for i := range 255 { + destinations = append(destinations, fmt.Sprintf("https://host%03d.example.com:443", i)) + } + base := canonicalTestAccess(t, destinations) + for _, input := range [][]string{ + {"https://z.example.com", "https://a.example.com"}, + {"https://a.example.com", "https://z.example.com"}, + } { + filtered, dropped := FilterConflicts(base, canonicalTestAccess(t, input), nil) + require.Equal(t, []string{"https://a.example.com:443"}, filtered.Destinations) + require.Equal(t, []string{"https://z.example.com:443"}, dropped) + } + var credentials []Credential + for i := range 16 { + credentials = append(credentials, Credential{Origin: "https://model.example.com:443", Header: fmt.Sprintf("x-header-%02d", i), URI: "ate-secret://k8s.io/default/team/auth/token"}) + } + base = canonicalTestAccess(t, []string{credentials[0].Origin}, credentials...) + extra := credentials[0] + extra.Header = "x-new" + grants := canonicalTestAccess(t, base.Destinations, extra) + filtered, dropped := FilterConflicts(base, grants, nil) + require.Empty(t, filtered.Credentials) + require.Equal(t, base.Destinations, filtered.Destinations) + require.Equal(t, base.Destinations, dropped) +} diff --git a/go/core/internal/substrate/egress.go b/go/core/internal/substrate/egress.go index 92232df0ca..67faa1b7b9 100644 --- a/go/core/internal/substrate/egress.go +++ b/go/core/internal/substrate/egress.go @@ -14,8 +14,8 @@ import ( ) // EnsureActorEgressPolicy installs current authorization without restarting the -// actor and returns the effective policy it applied, including withheld-grant -// diagnostics. Read the backend version before the controller's persisted intended +// actor and returns the effective policy it applied. Read the backend version +// before the controller's persisted intended // policy so concurrent applications cannot overwrite a newer policy with an old // one. Conflicts reload both; lifecycle paths never translate Kubernetes. // Substrate deletes the policy with its Actor. diff --git a/go/core/internal/substrate/egress_test.go b/go/core/internal/substrate/egress_test.go index 663b7766c7..71a7b4acd5 100644 --- a/go/core/internal/substrate/egress_test.go +++ b/go/core/internal/substrate/egress_test.go @@ -154,21 +154,3 @@ func TestEnsureActorEgressPolicyRequiresPersistedPolicy(t *testing.T) { require.Zero(t, fake.created) require.Zero(t, fake.updated) } - -func TestEnsureActorEgressPolicyAppliesFallbackDespiteWithheldGrants(t *testing.T) { - base := egress.Access{Destinations: []string{"https://model.example.com:443"}} - fake := &egressPolicyFake{} - conflict := errors.New("credential conflict") - reads := 0 - client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Policy, error) { - reads++ - return egress.Policy{Access: base, WithheldGrants: conflict}, nil - })}} - applied, err := client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision") - require.NoError(t, err) - require.Equal(t, base, applied.Access) - require.ErrorIs(t, applied.WithheldGrants, conflict) - require.Equal(t, 2, reads, "diagnostics must come from the existing apply and recheck reads") - require.Len(t, fake.policy.Rules, 1) - require.Equal(t, []string{"model.example.com"}, fake.policy.Rules[0].GetHttps().GetHostnames()) -} diff --git a/go/core/internal/translator/compiler.go b/go/core/internal/translator/compiler.go index e13dde24e3..cc7d8f2195 100644 --- a/go/core/internal/translator/compiler.go +++ b/go/core/internal/translator/compiler.go @@ -12,7 +12,6 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" ) // Compiler resolves public API objects into a complete, immutable runtime @@ -62,7 +61,6 @@ type resolvedTemplateBinding struct { // HarnessInput contains the Kubernetes inputs needed by a harness compiler. type HarnessInput struct { - Egress egress.Access // AgentName is the runnable identity, independent of inline or referenced inputs. AgentName string Harness *HarnessConfiguration @@ -146,10 +144,6 @@ func (c *Compiler) CompileAgent(ctx context.Context, agent *v1alpha3.Agent) (*Co // compileConfiguration resolves inputs and compiles configuration for the Agent. // The Agent name owns runtime identity; template and Harness names are provenance. func (c *Compiler) compileConfiguration(ctx context.Context, agent *v1alpha3.Agent, harness *HarnessConfiguration, template *TemplateConfiguration) (*CompileResult, error) { - access, err := CompileEgressPolicies(c.ctx, c.collections, agent.Namespace, agent.Spec.EgressPolicyRefs) - if err != nil { - return nil, err - } harnessCompiler := c.harnessCompilers[harnessType(harness)] if harnessCompiler == nil { return nil, NewValidationError("Harness runtime is not supported by any compiler") @@ -163,7 +157,6 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agent *v1alpha3.Age return nil, err } input.AgentName = agent.Name - input.Egress = access image, err := harnessImage(harness, c.images) if err != nil { return nil, err @@ -172,10 +165,6 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agent *v1alpha3.Age if err != nil { return nil, err } - // Explicit grants are live authorization, not immutable runtime inputs. - if _, err := egress.MergeAccess(egress.Access{Destinations: result.EgressDestinations, Credentials: result.Credentials}, access, result.TokenPassthroughOrigins); err != nil { - return nil, NewValidationError("%v", err) - } workerKey := types.NamespacedName{Namespace: harness.Namespace, Name: harness.Spec.Substrate.WorkerPoolRef.Name} workerPool := krt.FetchOne(c.ctx, c.collections.WorkerPools, krt.FilterObjectName(workerKey)) if workerPool == nil { diff --git a/go/core/internal/translator/credentials.go b/go/core/internal/translator/credentials.go index f10a9d4a3b..416f60cb43 100644 --- a/go/core/internal/translator/credentials.go +++ b/go/core/internal/translator/credentials.go @@ -110,7 +110,7 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, for _, binding := range bindings { destinations = append(destinations, binding.Origin) } - if _, err := egress.MergeAccess(egress.Access{Destinations: destinations, Credentials: bindings}, input.Egress, tokenPassthroughOrigins); err != nil { + if _, err := egress.MergeAccess(egress.Access{Destinations: destinations, Credentials: bindings}, egress.Access{}, tokenPassthroughOrigins); err != nil { return nil, nil, nil, NewValidationError("%v", err) } diff --git a/go/core/internal/translator/credentials_test.go b/go/core/internal/translator/credentials_test.go index 1cf5d36ca6..752362029d 100644 --- a/go/core/internal/translator/credentials_test.go +++ b/go/core/internal/translator/credentials_test.go @@ -8,7 +8,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" ) func TestCompileCredentialDestinations(t *testing.T) { @@ -175,15 +174,11 @@ func TestModelCredentialTargetOllamaCloud(t *testing.T) { } } -func TestPassthroughCredentialConflictUsesCanonicalOrigin(t *testing.T) { +func TestPassthroughCredentialUsesCanonicalOrigin(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: true, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://API.OPENAI.COM.:0443/v1"}}) - input.Egress.Destinations = []string{"https://api.openai.com"} - input.Egress.Credentials = []egress.Credential{{Origin: "https://api.openai.com", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token"}} - environment, _, _, err := CompileCredentials(input, nil, nil) - require.ErrorContains(t, err, "cannot combine caller-token passthrough") - require.Nil(t, environment) - input.Root.ResolvedModelConfig.Config.Spec.OpenAI.BaseURL = "https://api.openai.com:8443/v1" - environment, _, _, err = CompileCredentials(input, nil, nil) + environment, credentials, origins, err := CompileCredentials(input, nil, nil) + require.NoError(t, err) require.Empty(t, environment) - require.NoError(t, err, "a different port is a distinct credential scope") + require.Empty(t, credentials) + require.Equal(t, []string{"https://api.openai.com:443"}, origins) } diff --git a/go/core/internal/translator/egress.go b/go/core/internal/translator/egress.go index 299c8e538a..346d291e98 100644 --- a/go/core/internal/translator/egress.go +++ b/go/core/internal/translator/egress.go @@ -2,6 +2,7 @@ package translator import ( "fmt" + "slices" "strings" "istio.io/istio/pkg/kube/krt" @@ -14,10 +15,24 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/egress" ) -// CompileEgressPolicies resolves same-namespace grants for either an Agent or a -// SandboxTemplate. KRT tracks policy and Secret dependencies; only references -// enter the result, so Secret value rotation does not change a revision. -func CompileEgressPolicies(ctx krt.HandlerContext, collections Collections, namespace string, refs []corev1.LocalObjectReference) (egress.Access, error) { +// EgressGrantIssue describes one policy resolution error or admission conflict. +type EgressGrantIssue struct { + Policy string + Reason string + Message string +} + +// EgressGrantResolution contains the union of individually valid policies. +// Admission checks conflicts across this union and the desired revision's base. +type EgressGrantResolution struct { + Access egress.Access + Issues []EgressGrantIssue +} + +// CompileEgressPolicies resolves each same-namespace policy independently. +// KRT tracks policy and Secret dependencies. An unresolved policy contributes +// no grants, without removing grants supplied by the remaining policies. +func CompileEgressPolicies(ctx krt.HandlerContext, collections Collections, namespace string, refs []corev1.LocalObjectReference) EgressGrantResolution { return compileEgressPolicies(namespace, refs, func(name string) (*v1alpha3.EgressPolicy, error) { policy := krt.FetchOne(ctx, collections.EgressPolicies, krt.FilterKey(namespace+"/"+name)) if policy == nil { @@ -33,65 +48,89 @@ func CompileEgressPolicies(ctx krt.HandlerContext, collections Collections, name }) } -func compileEgressPolicies(namespace string, refs []corev1.LocalObjectReference, getPolicy func(string) (*v1alpha3.EgressPolicy, error), getSecret func(string) (*corev1.Secret, error)) (egress.Access, error) { +func compileEgressPolicies(namespace string, refs []corev1.LocalObjectReference, getPolicy func(string) (*v1alpha3.EgressPolicy, error), getSecret func(string) (*corev1.Secret, error)) EgressGrantResolution { + var result EgressGrantResolution + var invalidRefs string if len(refs) > 16 { - return egress.Access{}, NewValidationError("at most 16 egressPolicyRefs are supported") + invalidRefs = "at most 16 egressPolicyRefs are supported" } - var destinations []string - var credentials []egress.Credential for _, ref := range refs { if len(validation.IsDNS1123Subdomain(ref.Name)) != 0 { - return egress.Access{}, NewValidationError("invalid EgressPolicy reference %q", ref.Name) - } - key := namespace + "/" + ref.Name - policy, err := getPolicy(ref.Name) - if err != nil { - return egress.Access{}, fmt.Errorf("resolve EgressPolicy %q: %w", key, err) - } - if !policy.DeletionTimestamp.IsZero() { - return egress.Access{}, NewValidationError("EgressPolicy %q is being deleted", key) + invalidRefs = fmt.Sprintf("invalid EgressPolicy reference %q", ref.Name) + break } - if len(policy.Spec.Destinations) == 0 || len(policy.Spec.Destinations) > 64 { - return egress.Access{}, NewValidationError("EgressPolicy %q requires 1 to 64 destinations", key) + } + for _, ref := range refs { + issue := EgressGrantIssue{Policy: namespace + "/" + ref.Name, Reason: v1alpha3.EgressPoliciesReasonInvalidEgressPolicy} + if invalidRefs != "" { + issue.Message = invalidRefs + result.Issues = append(result.Issues, issue) + continue } - for _, destination := range policy.Spec.Destinations { - origin, err := egress.ParseOrigin(destination.Origin) - if err != nil { - return egress.Access{}, NewValidationError("EgressPolicy %q: %v", key, err) - } - destinations = append(destinations, origin.String()) - auth := destination.Authentication - if auth == nil { - continue - } - if strings.Contains(origin.Host, "*") { - return egress.Access{}, NewValidationError("EgressPolicy %q: authentication requires an exact HTTP(S) origin", key) - } - secret := auth.SecretKeyRef - binding := egress.Credential{Origin: origin.String(), Header: auth.Header} - if auth.Prefix != nil { - binding.Prefix = *auth.Prefix - } - if secret.Name == "" || secret.Key == "" || (secret.Optional != nil && *secret.Optional) { - return egress.Access{}, NewValidationError("EgressPolicy %q requires a non-optional Secret name and key", key) - } - binding.URI = "ate-secret://k8s.io/default/" + namespace + "/" + secret.Name + "/" + secret.Key - if _, err := egress.CanonicalCredentials([]egress.Credential{binding}); err != nil { - return egress.Access{}, NewValidationError("EgressPolicy %q: %v", key, err) + policy, err := getPolicy(ref.Name) + if err != nil || !policy.DeletionTimestamp.IsZero() { + if err == nil || apierrors.IsNotFound(err) { + issue.Reason = v1alpha3.EgressPoliciesReasonNotFound } - found, err := getSecret(secret.Name) + issue.Message = "EgressPolicy is being deleted" if err != nil { - return egress.Access{}, fmt.Errorf("EgressPolicy %q: Secret %q: %w", key, secret.Name, err) + issue.Message = err.Error() } - if _, ok := found.Data[secret.Key]; !ok { - return egress.Access{}, NewValidationError("EgressPolicy %q: Secret %q is missing key %q", key, secret.Name, secret.Key) - } - credentials = append(credentials, binding) + result.Issues = append(result.Issues, issue) + continue + } + access, err := compileEgressPolicy(namespace, policy, getSecret) + if err != nil { + issue.Message = err.Error() + result.Issues = append(result.Issues, issue) + continue } + result.Access.Destinations = append(result.Access.Destinations, access.Destinations...) + result.Access.Credentials = append(result.Access.Credentials, access.Credentials...) } - access, err := egress.CanonicalAccess(destinations, credentials) - if err != nil { - return egress.Access{}, NewValidationError("%v", err) + slices.SortFunc(result.Issues, func(a, b EgressGrantIssue) int { return strings.Compare(a.Policy, b.Policy) }) + return result +} + +func compileEgressPolicy(namespace string, policy *v1alpha3.EgressPolicy, getSecret func(string) (*corev1.Secret, error)) (egress.Access, error) { + if len(policy.Spec.Destinations) == 0 || len(policy.Spec.Destinations) > 64 { + return egress.Access{}, NewValidationError("requires 1 to 64 destinations") + } + var destinations []string + var credentials []egress.Credential + for _, destination := range policy.Spec.Destinations { + origin, err := egress.ParseOrigin(destination.Origin) + if err != nil { + return egress.Access{}, err + } + destinations = append(destinations, origin.String()) + auth := destination.Authentication + if auth == nil { + continue + } + if strings.Contains(origin.Host, "*") { + return egress.Access{}, NewValidationError("authentication requires an exact HTTP(S) origin") + } + secret := auth.SecretKeyRef + binding := egress.Credential{Origin: origin.String(), Header: auth.Header} + if auth.Prefix != nil { + binding.Prefix = *auth.Prefix + } + if secret.Name == "" || secret.Key == "" || (secret.Optional != nil && *secret.Optional) { + return egress.Access{}, NewValidationError("requires a non-optional Secret name and key") + } + binding.URI = "ate-secret://k8s.io/default/" + namespace + "/" + secret.Name + "/" + secret.Key + if _, err := egress.CanonicalCredentials([]egress.Credential{binding}); err != nil { + return egress.Access{}, err + } + found, err := getSecret(secret.Name) + if err != nil { + return egress.Access{}, fmt.Errorf("resolve Secret %q: %w", secret.Name, err) + } + if _, ok := found.Data[secret.Key]; !ok { + return egress.Access{}, NewValidationError("Secret %q is missing key %q", secret.Name, secret.Key) + } + credentials = append(credentials, binding) } - return access, nil + return egress.CanonicalAccess(destinations, credentials) } diff --git a/go/core/internal/translator/egress_test.go b/go/core/internal/translator/egress_test.go index 839c458725..37c70e3741 100644 --- a/go/core/internal/translator/egress_test.go +++ b/go/core/internal/translator/egress_test.go @@ -39,8 +39,9 @@ func TestEgressPolicyCredentialPrefixes(t *testing.T) { policy, secret := egressPolicy() auth := policy.Spec.Destinations[0].Authentication auth.Header, auth.Prefix = tc.header, tc.prefix - access, err := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", []corev1.LocalObjectReference{{Name: policy.Name}}) - require.NoError(t, err) + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", []corev1.LocalObjectReference{{Name: policy.Name}}) + require.Empty(t, resolution.Issues) + access := resolution.Access wantHeader := "authorization" if tc.header == "X-API-Key" { wantHeader = "x-api-key" @@ -53,8 +54,9 @@ func TestEgressPolicyCredentialPrefixes(t *testing.T) { func TestEgressPolicyHTTPCredentials(t *testing.T) { policy, secret := egressPolicy() policy.Spec.Destinations[0].Origin = "http://tickets.example.com:8080" - access, err := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", []corev1.LocalObjectReference{{Name: policy.Name}}) - require.NoError(t, err) + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", []corev1.LocalObjectReference{{Name: policy.Name}}) + require.Empty(t, resolution.Issues) + access := resolution.Access require.Equal(t, []string{"http://tickets.example.com:8080"}, access.Destinations) require.Equal(t, []egress.Credential{{Origin: "http://tickets.example.com:8080", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/test/ticket-auth/token"}}, access.Credentials) } @@ -123,7 +125,7 @@ func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { require.Equal(t, original, unreferenced) agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} require.NotContains(t, result.EgressDestinations, "https://registry.npmjs.org:443", "previous revision remains pinned") - // Explicit policy credentials cannot silently replace inferred model credentials. + // Conflicting and missing policies are independent of runtime compilation. policy.Spec.Destinations[0].Origin = "https://api.openai.com" auth := policy.Spec.Destinations[0].Authentication if kind == "claude" { @@ -134,7 +136,13 @@ func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { secret.Data["alternate"] = []byte("different-token") model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = secret.Name, "token" _, err = compiler(t, model, harness, policy, secret).CompileAgent(t.Context(), agent) - require.ErrorContains(t, err, "conflicting credentials") + require.NoError(t, err) + withConflict, err := compile().Digest() + require.NoError(t, err) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "missing"}} + missing, err := compile().Digest() + require.NoError(t, err) + require.Equal(t, withConflict, missing) // Codex and Claude reject caller-token passthrough independently. if kind == "claude" || kind == "codex" { return @@ -142,7 +150,7 @@ func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = "", "" model.Spec.APIKeyPassthrough = true _, err = compiler(t, model, harness, policy, secret).CompileAgent(t.Context(), agent) - require.ErrorContains(t, err, "cannot combine caller-token passthrough") + require.NoError(t, err) }) } } @@ -186,8 +194,15 @@ func TestEgressPolicyResolutionFailsClosed(t *testing.T) { objects = append(objects, other) refs = append(refs, corev1.LocalObjectReference{Name: other.Name}) } - _, err := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, objects...), "test", refs) - require.Error(t, err) + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, objects...), "test", refs) + if name == "conflicting auth" || name == "conflicting prefix" { + require.Empty(t, resolution.Issues, "individually valid policies reach admission") + _, err := egress.CanonicalAccess(resolution.Access.Destinations, resolution.Access.Credentials) + require.Error(t, err) + } else { + require.Len(t, resolution.Issues, 1) + require.Empty(t, resolution.Access.Destinations) + } }) } } @@ -201,13 +216,52 @@ func TestEgressPoliciesMergeCredentialsAndOrigins(t *testing.T) { }) collections := mockCollections(t, policy, other, secret) refs := []corev1.LocalObjectReference{{Name: policy.Name}, {Name: other.Name}} - access, err := translator.CompileEgressPolicies(krt.TestingDummyContext{}, collections, "test", refs) + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, collections, "test", refs) + require.Empty(t, resolution.Issues) + access, err := egress.CanonicalAccess(resolution.Access.Destinations, resolution.Access.Credentials) require.NoError(t, err) require.Len(t, access.Destinations, 3) require.Len(t, access.Credentials, 2) require.Equal(t, "x-api-key", access.Credentials[1].Header) refs[0], refs[1] = refs[1], refs[0] - reversed, err := translator.CompileEgressPolicies(krt.TestingDummyContext{}, collections, "test", refs) + reversed := translator.CompileEgressPolicies(krt.TestingDummyContext{}, collections, "test", refs) + require.Empty(t, reversed.Issues) + canonical, err := egress.CanonicalAccess(reversed.Access.Destinations, reversed.Access.Credentials) require.NoError(t, err) - require.Equal(t, access, reversed) + require.Equal(t, access, canonical) +} + +func TestEgressPoliciesResolveIndependently(t *testing.T) { + policy, secret := egressPolicy() + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", + []corev1.LocalObjectReference{{Name: "z-missing"}, {Name: policy.Name}, {Name: "a-missing"}}) + require.Equal(t, []string{"https://tickets.example.com:443"}, resolution.Access.Destinations) + require.Len(t, resolution.Access.Credentials, 1) + require.Len(t, resolution.Issues, 2) + require.Equal(t, "test/a-missing", resolution.Issues[0].Policy) + require.Equal(t, v1alpha3.EgressPoliciesReasonNotFound, resolution.Issues[0].Reason) + require.Equal(t, "test/z-missing", resolution.Issues[1].Policy) +} + +func TestInvalidEgressReferencesRevokeAll(t *testing.T) { + for _, name := range []string{"too many", "invalid name"} { + t.Run(name, func(t *testing.T) { + policy, secret := egressPolicy() + refs := []corev1.LocalObjectReference{{Name: policy.Name}} + if name == "invalid name" { + refs = append(refs, corev1.LocalObjectReference{Name: "Invalid!"}) + } else { + for range 16 { + refs = append(refs, corev1.LocalObjectReference{Name: policy.Name}) + } + } + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", refs) + require.Empty(t, resolution.Access.Destinations) + require.Empty(t, resolution.Access.Credentials) + require.Len(t, resolution.Issues, len(refs)) + for _, issue := range resolution.Issues { + require.Equal(t, v1alpha3.EgressPoliciesReasonInvalidEgressPolicy, issue.Reason) + } + }) + } } diff --git a/go/core/test/e2e/egress_test.go b/go/core/test/e2e/egress_test.go index 37272619e1..075ecf00f5 100644 --- a/go/core/test/e2e/egress_test.go +++ b/go/core/test/e2e/egress_test.go @@ -95,7 +95,7 @@ func TestSessionEgressPolicyUpdatesLiveActors(t *testing.T) { } run := runs("live") - finish := runEgressSkill(t, fixture, model, run, "initial_DENIED", "allowed_ALLOWED", "revoked_DENIED") + finish := runEgressSkill(t, fixture, model, run, "initial_DENIED", "allowed_ALLOWED", "conflict_ALLOWED", "fixed_ALLOWED", "revoked_DENIED") run.expect(t, "initial", "DENIED") require.Zero(t, requests("live_initial"), "ungranted requests reached the destination") require.NoError(t, kube.Get(t.Context(), key, agent)) @@ -104,6 +104,41 @@ func TestSessionEgressPolicyUpdatesLiveActors(t *testing.T) { run.setPhase("allowed") run.expect(t, "allowed", "ALLOWED") require.Positive(t, requests("live_allowed")) + waitEgressCondition := func(reason string) { + t.Helper() + require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, agent); err != nil { + return false, err + } + condition := apimeta.FindStatusCondition(agent.Status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied) + return condition != nil && condition.Reason == reason && + condition.ObservedGeneration == agent.Generation && + apimeta.IsStatusConditionTrue(agent.Status.Conditions, v1alpha3.AgentConditionReady), nil + })) + require.Equal(t, revision, agent.Status.LatestSuccessfulRevision) + } + waitEgressCondition(v1alpha3.EgressPoliciesReasonApplied) + // The opposite protocol on the model's exact host/port conflicts with + // pinned access. Existing grants must keep working in this invocation. + modelOrigin, err := url.Parse(modelURL) + require.NoError(t, err) + require.Equal(t, "http", modelOrigin.Scheme) + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKeyFromObject(grant), grant)) + grant.Spec.Destinations = append(grant.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://" + modelOrigin.Host}) + require.NoError(t, kube.Update(t.Context(), grant)) + waitEgressCondition(v1alpha3.EgressPoliciesReasonConflict) + require.True(t, apimeta.IsStatusConditionFalse(agent.Status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied)) + run.setPhase("conflict") + run.expect(t, "conflict", "ALLOWED") + require.Positive(t, requests("live_conflict")) + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKeyFromObject(grant), grant)) + grant.Spec.Destinations = grant.Spec.Destinations[:1] + require.NoError(t, kube.Update(t.Context(), grant)) + waitEgressCondition(v1alpha3.EgressPoliciesReasonApplied) + require.True(t, apimeta.IsStatusConditionTrue(agent.Status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied)) + run.setPhase("fixed") + run.expect(t, "fixed", "ALLOWED") + setGrant("https://unused.example.com") run.setPhase("revoked") run.expect(t, "revoked", "DENIED") @@ -136,7 +171,8 @@ func TestSessionEgressPolicyUpdatesLiveActors(t *testing.T) { if err := kube.Get(ctx, key, agent); err != nil { return false, err } - return apimeta.IsStatusConditionFalse(agent.Status.Conditions, v1alpha3.AgentConditionReady), nil + condition := apimeta.FindStatusCondition(agent.Status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied) + return condition != nil && condition.Reason == v1alpha3.EgressPoliciesReasonNotFound && apimeta.IsStatusConditionTrue(agent.Status.Conditions, v1alpha3.AgentConditionReady), nil })) require.Equal(t, revision, agent.Status.LatestSuccessfulRevision, "policy edits must not replace the runtime revision") created, err := fixture.sessions.CreateSession(fixture.ctx, &apiv1.CreateSessionRequest{Agent: &apiv1.ResourceReference{Namespace: key.Namespace, Name: key.Name}, RequestId: uuid.NewString()}) @@ -451,7 +487,8 @@ func TestSandboxEgressPolicyUpdatesLiveActor(t *testing.T) { if err := kube.Get(ctx, key, template); err != nil { return false, err } - return apimeta.IsStatusConditionFalse(template.Status.Conditions, "Ready"), nil + condition := apimeta.FindStatusCondition(template.Status.Conditions, v1alpha3.SandboxTemplateConditionEgressPoliciesApplied) + return condition != nil && condition.Reason == v1alpha3.EgressPoliciesReasonNotFound && apimeta.IsStatusConditionTrue(template.Status.Conditions, "Ready"), nil })) original := instance instance = f.create(t, 5*time.Minute) diff --git a/helm/kagent-crds/templates/api.kagent.dev_agents.yaml b/helm/kagent-crds/templates/api.kagent.dev_agents.yaml index 09a81343e1..6ba119ed9f 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_agents.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_agents.yaml @@ -517,6 +517,11 @@ spec: description: AgentStatus reports compilation and preparation of an Agent. properties: conditions: + description: |- + Conditions report runtime readiness and independent egress grant admission. + EgressPoliciesApplied is False for EgressPolicyNotFound, InvalidEgressPolicy, + or EgressGrantConflict; Applied means all grants are admitted. Grant issues + do not affect Ready. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -571,7 +576,7 @@ spec: - status - type type: object - maxItems: 4 + maxItems: 5 type: array x-kubernetes-list-map-keys: - type diff --git a/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml b/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml index b49df3f122..bf2fd9509d 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml @@ -160,7 +160,11 @@ spec: template inputs. properties: conditions: - description: Conditions describe whether a prepared revision is available. + description: |- + Conditions report runtime readiness and independent egress grant admission. + EgressPoliciesApplied is False for EgressPolicyNotFound, InvalidEgressPolicy, + or EgressGrantConflict; Applied means all grants are admitted. Grant issues + do not affect Ready. items: description: Condition contains details for one aspect of the current state of this API Resource. From 15001b9110f0696a2db69c410c11bd10acfe9ffd Mon Sep 17 00:00:00 2001 From: Cody Hartsook Date: Wed, 14 Oct 2026 12:58:14 -0700 Subject: [PATCH 34/43] feat: add outbound credential binding extension Signed-off-by: Cody Hartsook --- docs/architecture/credential-injection.md | 30 +++ go/core/internal/controller/collections.go | 5 +- .../internal/controller/collections_test.go | 12 +- .../internal/controller/preparation_test.go | 2 +- go/core/internal/controller/reconciler.go | 10 +- go/core/internal/controller/runtime.go | 5 +- .../database/runtime_revisions_test.go | 26 +-- go/core/internal/egress/credentials.go | 38 +++- go/core/internal/egress/credentials_test.go | 30 ++- go/core/internal/substrate/policy_test.go | 3 +- go/core/internal/translator/adkconfig/mcp.go | 28 +++ .../internal/translator/adkconfig/mcp_test.go | 15 ++ go/core/internal/translator/byo/compiler.go | 34 ++-- .../internal/translator/byo/compiler_test.go | 6 +- .../internal/translator/claude/compiler.go | 47 +++-- .../translator/claude/compiler_test.go | 40 ++-- go/core/internal/translator/codex/compiler.go | 47 +++-- .../translator/codex/compiler_test.go | 20 +- go/core/internal/translator/compiler_test.go | 10 +- .../translator/credential_runtime_test.go | 185 ++++++++++++++++++ go/core/internal/translator/credentials.go | 86 ++++++-- .../internal/translator/credentials_test.go | 37 +++- .../internal/translator/egress_credentials.go | 152 ++++++++++++++ .../translator/egress_credentials_test.go | 181 +++++++++++++++++ .../translator/kagent/agentcard_test.go | 2 +- .../internal/translator/kagent/compiler.go | 34 ++-- go/core/pkg/app/app.go | 6 +- go/core/pkg/egressext/credentials.go | 24 +++ 28 files changed, 957 insertions(+), 158 deletions(-) create mode 100644 go/core/internal/translator/credential_runtime_test.go create mode 100644 go/core/internal/translator/egress_credentials.go create mode 100644 go/core/internal/translator/egress_credentials_test.go create mode 100644 go/core/pkg/egressext/credentials.go diff --git a/docs/architecture/credential-injection.md b/docs/architecture/credential-injection.md index 78def5dc58..155a95e2d0 100644 --- a/docs/architecture/credential-injection.md +++ b/docs/architecture/credential-injection.md @@ -84,6 +84,36 @@ Caller-token passthrough retains its existing behavior. A passthrough model cannot share an origin with static gateway credentials, which would override the caller's authentication. +## Controller credential extension + +Embedded controllers can supply `app.Options.EgressCredentialBinder` to add +credential references during compilation. Nil preserves the built-in behavior. +The [CredentialBinder contract](../../go/core/pkg/egressext/credentials.go) defines +`Bindings(targets, existing)` and its concurrency, determinism, and no-I/O requirements. + +Targets are exact HTTPS origins from resolved models and MCP servers. EgressPolicy +grants are live authorization, not pinned targets. The compiler excludes origins with passthrough, model default +headers, literal Harness model keys, or literal or ConfigMap MCP headers. +Bindings cannot expand network access or replace existing bindings. Model bindings +must match the supplied header and prefix. Invalid results fail Agent preparation. + +References use `ate-secret:///` with a lowercase, +port-free provider authority. Kubernetes references retain the +`k8s.io/default///` path. Substrate owns provider +registration, gateway authentication, and caching. The binder returns references, +never credential values. + +The built-in compilers add model and MCP placeholders for contributed bindings. +For keyless models, this selects API-key or bearer-token authentication, including +Bedrock and Foundry. Arbitrary clients must send the bound header themselves. +Headerless requests do not trigger credential resolution. Changed bindings create +a new revision. Existing sessions retain their pinned bindings. + +Keyless Gemini support is deferred. Its origins are excluded until runtime +support is added. Secret-backed Gemini keeps its existing behavior. Controller +and telemetry destinations, direct A2A delegation, and standalone sandboxes are +outside this extension's target scope. + ## Deferred API fields The source retains commented declarations for Harness and SandboxTemplate diff --git a/go/core/internal/controller/collections.go b/go/core/internal/controller/collections.go index b7c9dbe33c..b3047bb918 100644 --- a/go/core/internal/controller/collections.go +++ b/go/core/internal/controller/collections.go @@ -16,6 +16,7 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) // Collections contains shared Kubernetes inputs and compiled Agent state. @@ -65,7 +66,7 @@ func (p AgentRuntimeObservation) Equals(other AgentRuntimeObservation) bool { // NewCollections creates the complete read-only input graph. An empty // watchNamespaces list watches all namespaces. -func NewCollections(client kube.Client, watchNamespaces []string, opts krt.OptionsBuilder, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig) Collections { +func NewCollections(client kube.Client, watchNamespaces []string, opts krt.OptionsBuilder, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig, binder egressext.CredentialBinder) Collections { egressPolicies := typedCollection[*kagentv1alpha3.EgressPolicy](client, watchNamespaces, "EgressPolicies", opts) sandboxTemplates := typedCollection[*kagentv1alpha3.SandboxTemplate](client, watchNamespaces, "SandboxTemplates", opts) agents := typedCollection[*kagentv1alpha3.Agent](client, watchNamespaces, "Agents", opts) @@ -83,7 +84,7 @@ func NewCollections(client kube.Client, watchNamespaces []string, opts krt.Optio Harnesses: harnesses, AgentTemplates: agentTemplates, ResolvedModelConfigs: resolvedModelConfigs, RemoteMCPServers: remoteMCPServers, ConfigMaps: configMaps, Secrets: secrets, WorkerPools: workerPools, } - reconciliations := newAgentReconciliations(agents, compilerCollections, agentRuntimeObservations, opts, settings, images) + reconciliations := newAgentReconciliations(agents, compilerCollections, agentRuntimeObservations, opts, settings, images, binder) statuses := newAgentStatuses(agents, reconciliations, opts) return Collections{ diff --git a/go/core/internal/controller/collections_test.go b/go/core/internal/controller/collections_test.go index 8e8ab3a99d..50b9c928bc 100644 --- a/go/core/internal/controller/collections_test.go +++ b/go/core/internal/controller/collections_test.go @@ -64,7 +64,7 @@ func TestReconciliationCollectionsCompileAndObserveRevision(t *testing.T) { Harnesses: collections.Harnesses, AgentTemplates: collections.AgentTemplates, ResolvedModelConfigs: collections.ResolvedModelConfigs, RemoteMCPServers: collections.RemoteMCPServers, ConfigMaps: collections.ConfigMaps, Secrets: collections.Secrets, WorkerPools: collections.WorkerPools, - }, collections.AgentRuntimeObservations, opts, testRuntimeDefaults(), nil, + }, collections.AgentRuntimeObservations, opts, testRuntimeDefaults(), nil, nil, ) collections.AgentStatuses = newAgentStatuses(collections.Agents, collections.Reconciliations, opts) @@ -160,7 +160,7 @@ func TestReconciliationReportsRuntimeValidationAsUnsupported(t *testing.T) { Harnesses: krttest.GetMockCollection[*kagentv1alpha3.Harness](mock), AgentTemplates: krttest.GetMockCollection[*kagentv1alpha3.AgentTemplate](mock), ResolvedModelConfigs: resolvedModels, ConfigMaps: configMaps, Secrets: secrets, RemoteMCPServers: krttest.GetMockCollection[*kagentv1alpha3.RemoteMCPServer](mock), WorkerPools: krttest.GetMockCollection[*atev1alpha1.WorkerPool](mock), - }, krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("Observations")...), opts, settings, nil, + }, krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("Observations")...), opts, settings, nil, nil, ) waitFor(t, func() bool { return states.GetKey("team-a/assistant") != nil }) state := states.GetKey("team-a/assistant") @@ -230,7 +230,7 @@ func TestReconciliationWorkerPoolSandboxClass(t *testing.T) { Harnesses: krttest.GetMockCollection[*kagentv1alpha3.Harness](mock), AgentTemplates: templates, ResolvedModelConfigs: resolvedModels, RemoteMCPServers: krttest.GetMockCollection[*kagentv1alpha3.RemoteMCPServer](mock), ConfigMaps: configMaps, Secrets: secrets, WorkerPools: workerPools, - }, observations, opts, testRuntimeDefaults(), nil) + }, observations, opts, testRuntimeDefaults(), nil, nil) key := "team-a/assistant" waitFor(t, func() bool { state := reconciliations.GetKey(key) @@ -371,7 +371,7 @@ func TestClaudeReconciliationCompilesActorTemplate(t *testing.T) { RemoteMCPServers: krttest.GetMockCollection[*kagentv1alpha3.RemoteMCPServer](mock), ConfigMaps: configMaps, Secrets: secrets, WorkerPools: krttest.GetMockCollection[*atev1alpha1.WorkerPool](mock), - }, krttest.GetMockCollection[AgentRuntimeObservation](mock), opts, testRuntimeDefaults(), nil, + }, krttest.GetMockCollection[AgentRuntimeObservation](mock), opts, testRuntimeDefaults(), nil, nil, ) waitFor(t, func() bool { states := reconciliations.List() @@ -427,7 +427,7 @@ func TestCodexReconciliationCompilesActorTemplate(t *testing.T) { RemoteMCPServers: krttest.GetMockCollection[*kagentv1alpha3.RemoteMCPServer](mock), ConfigMaps: configMaps, Secrets: secrets, WorkerPools: krttest.GetMockCollection[*atev1alpha1.WorkerPool](mock), - }, krttest.GetMockCollection[AgentRuntimeObservation](mock), opts, testRuntimeDefaults(), nil, + }, krttest.GetMockCollection[AgentRuntimeObservation](mock), opts, testRuntimeDefaults(), nil, nil, ) waitFor(t, func() bool { states := reconciliations.List() @@ -480,7 +480,7 @@ func TestReconciliationTracksSharedAgentTemplate(t *testing.T) { RemoteMCPServers: krttest.GetMockCollection[*kagentv1alpha3.RemoteMCPServer](mock), ConfigMaps: configMaps, Secrets: secrets, WorkerPools: krttest.GetMockCollection[*atev1alpha1.WorkerPool](mock), - }, krttest.GetMockCollection[AgentRuntimeObservation](mock), opts, testRuntimeDefaults(), nil, + }, krttest.GetMockCollection[AgentRuntimeObservation](mock), opts, testRuntimeDefaults(), nil, nil, ) var initial string waitFor(t, func() bool { diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index 4bd467a12c..058b38e11d 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -573,7 +573,7 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections Harnesses: collections.Harnesses, AgentTemplates: collections.AgentTemplates, ResolvedModelConfigs: collections.ResolvedModelConfigs, RemoteMCPServers: collections.RemoteMCPServers, ConfigMaps: collections.ConfigMaps, Secrets: collections.Secrets, WorkerPools: collections.WorkerPools, EgressPolicies: collections.EgressPolicies, - }, collections.AgentRuntimeObservations, opts, testRuntimeDefaults(), nil) + }, collections.AgentRuntimeObservations, opts, testRuntimeDefaults(), nil, nil) collections.AgentStatuses = newAgentStatuses(collections.Agents, collections.Reconciliations, opts) waitFor(t, func() bool { states := collections.Reconciliations.List() diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index fcd21740d6..54cc29a92b 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -34,6 +34,7 @@ import ( claudetranslator "github.com/kagent-dev/kagent/go/core/internal/translator/claude" codextranslator "github.com/kagent-dev/kagent/go/core/internal/translator/codex" kagenttranslator "github.com/kagent-dev/kagent/go/core/internal/translator/kagent" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) // AgentReconciliation is the complete desired and observed state for one @@ -104,6 +105,7 @@ func newAgentReconciliations( opts krt.OptionsBuilder, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig, + binder egressext.CredentialBinder, ) krt.Collection[AgentReconciliation] { return krt.NewCollection(agents, func(ctx krt.HandlerContext, agent *kagentv1alpha3.Agent) *AgentReconciliation { state := &AgentReconciliation{Agent: agent} @@ -117,10 +119,10 @@ func newAgentReconciliations( state.EgressGrant, state.EgressIssues, state.EgressConflict = evaluateEgressGrant(resolution, base, tokenOrigins) }() compilation, err := translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ - translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), - translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings), - translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings), - translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings, binder), + translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings, binder), + translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings, binder), + translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings, binder), }, images).CompileAgent(context.Background(), agent) if err != nil { state.CompilationFailure = compilationFailure(err) diff --git a/go/core/internal/controller/runtime.go b/go/core/internal/controller/runtime.go index 5250425f98..81331ac399 100644 --- a/go/core/internal/controller/runtime.go +++ b/go/core/internal/controller/runtime.go @@ -10,6 +10,7 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/core/internal/controller/apiclient" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) // Runtime owns the Kubernetes client and common options shared by the KRT @@ -23,13 +24,13 @@ type Runtime struct { // NewRuntime creates the shared KRT client and collection graph. Handlers are // deliberately registered separately at the eventual application boundary. -func NewRuntime(config *rest.Config, watchNamespaces []string, stop <-chan struct{}, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig) (*Runtime, error) { +func NewRuntime(config *rest.Config, watchNamespaces []string, stop <-chan struct{}, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig, binder egressext.CredentialBinder) (*Runtime, error) { client, err := apiclient.New(config) if err != nil { return nil, fmt.Errorf("create KRT Kubernetes client: %w", err) } options := krt.NewOptionsBuilder(stop, "kagent", krt.GlobalDebugHandler) - collections := NewCollections(client, watchNamespaces, options, settings, images) + collections := NewCollections(client, watchNamespaces, options, settings, images, binder) return &Runtime{Client: client, Options: options, Collections: collections}, nil } diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index 5bcdfa1655..7cc451a3a4 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -544,17 +544,21 @@ func TestRecordReadyRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { } func TestRuntimeRevisionPersistsCredentialBindings(t *testing.T) { - client := NewClient(setupTestDB(t), "public") - revision := RuntimeRevision{Revision: "credential-revision", Namespace: "team", AgentName: "agent", AgentUID: "agent", SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{"https://api.example.com:443"}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", Credentials: []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}}} - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, RecordRevisionOptions{})) - got, err := client.GetRuntimeRevision(t.Context(), revision.Revision) - require.NoError(t, err) - require.Equal(t, revision.Credentials, got.Credentials) - revision.Credentials[0].URI = "ate-secret://k8s.io/default/team/other/token" - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, RecordRevisionOptions{})) - unchanged, err := client.GetRuntimeRevision(t.Context(), revision.Revision) - require.NoError(t, err) - require.Equal(t, got.Credentials, unchanged.Credentials, "revision credentials are immutable") + for _, provider := range []string{"k8s.io", "provider.example"} { + t.Run(provider, func(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + revision := RuntimeRevision{Revision: "credential-revision-" + provider, Namespace: "team", AgentName: "agent", AgentUID: "agent", SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{"https://api.example.com:443"}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", Credentials: []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://" + provider + "/default/team/auth/token"}}} + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, RecordRevisionOptions{})) + got, err := client.GetRuntimeRevision(t.Context(), revision.Revision) + require.NoError(t, err) + require.Equal(t, revision.Credentials, got.Credentials) + revision.Credentials[0].URI = "ate-secret://k8s.io/default/team/other/token" + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, RecordRevisionOptions{})) + unchanged, err := client.GetRuntimeRevision(t.Context(), revision.Revision) + require.NoError(t, err) + require.Equal(t, got.Credentials, unchanged.Credentials, "revision credentials are immutable") + }) + } } func TestRuntimeRevisionRejectsMalformedStoredCredentials(t *testing.T) { diff --git a/go/core/internal/egress/credentials.go b/go/core/internal/egress/credentials.go index f226807c19..3979ee92d6 100644 --- a/go/core/internal/egress/credentials.go +++ b/go/core/internal/egress/credentials.go @@ -10,7 +10,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation" ) -// Credential binds a Secret reference to an exact destination and HTTP header. +// Credential binds a provider reference to an exact destination and HTTP header. // Values are fetched by the gateway and never enter a runtime revision. type Credential struct { Origin string `json:"origin"` @@ -33,13 +33,8 @@ func CanonicalCredentials(bindings []Credential) ([]Credential, error) { if !validCredentialHeader(c.Header) || !httpguts.ValidHeaderFieldValue(c.Prefix) { return nil, fmt.Errorf("invalid credential injection header %q", c.Header) } - u, err = url.Parse(c.URI) - if err != nil || u.Scheme != "ate-secret" || u.Host != "k8s.io" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { - return nil, fmt.Errorf("invalid Kubernetes credential URI") - } - parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") - if len(parts) != 4 || parts[0] != "default" || len(validation.IsDNS1123Label(parts[1])) != 0 || len(validation.IsDNS1123Subdomain(parts[2])) != 0 || parts[3] == "" || len(validation.IsConfigMapKey(parts[3])) != 0 { - return nil, fmt.Errorf("credential URI must identify the default locator, namespace, Secret and key") + if err := validCredentialURI(c.URI); err != nil { + return nil, err } } slices.SortFunc(result, func(a, b Credential) int { @@ -62,6 +57,33 @@ func CanonicalCredentials(bindings []Credential) ([]Credential, error) { return result, nil } +// validCredentialURI accepts ate-secret:///. Provider names are +// lowercase registration keys without ports. k8s.io paths name one Secret key. +func validCredentialURI(raw string) error { + u, err := url.Parse(raw) + if err != nil || u.Scheme != "ate-secret" || u.Opaque != "" { + return fmt.Errorf("invalid credential URI") + } + if u.User != nil || u.ForceQuery || u.RawQuery != "" || strings.Contains(raw, "#") { + return fmt.Errorf("credential URI cannot carry user information, a query or a fragment") + } + if u.Host == "" || u.Host != strings.ToLower(u.Hostname()) { + return fmt.Errorf("credential provider must be a lowercase name without a port") + } + if u.Host != "k8s.io" { + return nil + } + parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") + if len(parts) != 4 || parts[0] != "default" { + return fmt.Errorf("credential URI must identify the default locator, namespace, Secret and key") + } + namespace, secret, key := parts[1], parts[2], parts[3] + if len(validation.IsDNS1123Label(namespace)) != 0 || len(validation.IsDNS1123Subdomain(secret)) != 0 || len(validation.IsConfigMapKey(key)) != 0 { + return fmt.Errorf("credential URI must identify the default locator, namespace, Secret and key") + } + return nil +} + // validCredentialHeader excludes routing, framing, hop-by-hop and cookie headers. func validCredentialHeader(header string) bool { if len(header) > 256 || !httpguts.ValidHeaderFieldName(header) { diff --git a/go/core/internal/egress/credentials_test.go b/go/core/internal/egress/credentials_test.go index 453d663065..6692f51a72 100644 --- a/go/core/internal/egress/credentials_test.go +++ b/go/core/internal/egress/credentials_test.go @@ -22,7 +22,20 @@ func TestCanonicalCredentials(t *testing.T) { {"prefix", func(c *Credential) { c.Prefix = "Bearer\n" }}, {"provider", func(c *Credential) { c.URI = "https://example.com/secret" }}, {"missing key", func(c *Credential) { c.URI = "ate-secret://k8s.io/default/team/auth" }}, - {"old provider", func(c *Credential) { c.URI = "ate-secret://kubernetes.io/team/auth/token" }}, + {"empty key", func(c *Credential) { c.URI = "ate-secret://k8s.io/default/team/auth/" }}, + {"mixed case Kubernetes provider", func(c *Credential) { c.URI = "ate-secret://K8s.io/default/team/auth/token" }}, + {"Kubernetes provider port", func(c *Credential) { c.URI = "ate-secret://k8s.io:443/default/team/auth/token" }}, + {"mixed case external provider", func(c *Credential) { c.URI = "ate-secret://Provider.example/token" }}, + {"external provider port", func(c *Credential) { c.URI = "ate-secret://provider.example:443/token" }}, + {"empty provider port", func(c *Credential) { c.URI = "ate-secret://provider.example:/token" }}, + {"IPv6 provider", func(c *Credential) { c.URI = "ate-secret://[2001:db8::1]/token" }}, + {"missing provider", func(c *Credential) { c.URI = "ate-secret:///team/auth/token" }}, + {"userinfo", func(c *Credential) { c.URI = "ate-secret://user@provider.example/token" }}, + {"opaque", func(c *Credential) { c.URI = "ate-secret:provider.example/token" }}, + {"empty query", func(c *Credential) { c.URI += "?" }}, + {"fragment", func(c *Credential) { c.URI += "#token" }}, + {"empty fragment", func(c *Credential) { c.URI += "#" }}, + {"invalid escape", func(c *Credential) { c.URI = "ate-secret://provider.example/%xx" }}, {"missing locator", func(c *Credential) { c.URI = "ate-secret://k8s.io/team/auth/token" }}, {"unsupported locator", func(c *Credential) { c.URI = "ate-secret://k8s.io/remote/team/auth/token" }}, {"query", func(c *Credential) { c.URI += "?key=other" }}, @@ -39,3 +52,18 @@ func TestCanonicalCredentials(t *testing.T) { _, err = CanonicalCredentials([]Credential{base, other}) require.ErrorContains(t, err, "conflicting credentials") } + +func TestCanonicalCredentialsExternalProvider(t *testing.T) { + for _, uri := range []string{ + "ate-secret://provider.example/session-token", + "ate-secret://caller-token.kagent.dev/token", + "ate-secret://provider.example/default/https%3A%2F%2Fapi.example.com%3A8443", + } { + t.Run(uri, func(t *testing.T) { + binding := Credential{Origin: "https://api.example.com:8443", Header: "authorization", Prefix: "Bearer ", URI: uri} + got, err := CanonicalCredentials([]Credential{binding}) + require.NoError(t, err) + require.Equal(t, []Credential{binding}, got) + }) + } +} diff --git a/go/core/internal/substrate/policy_test.go b/go/core/internal/substrate/policy_test.go index 525efb3b89..fb2f992ab6 100644 --- a/go/core/internal/substrate/policy_test.go +++ b/go/core/internal/substrate/policy_test.go @@ -11,7 +11,7 @@ import ( func TestActorPolicyKeepsCredentialsOnExactOrigin(t *testing.T) { policy, err := ActorEgressPolicy("team", []string{"https://*.example.com", "https://api.example.com:443", "https://api.example.com:8443", "http://api.example.com"}, []egress.Credential{ {Origin: "https://api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}, - {Origin: "https://api.example.com:8443", Header: "x-api-key", URI: "ate-secret://k8s.io/default/team/other/key"}, + {Origin: "https://api.example.com:8443", Header: "x-api-key", URI: "ate-secret://provider.example/token"}, }) require.NoError(t, err) require.Len(t, policy.Rules, 4) @@ -26,4 +26,5 @@ func TestActorPolicyKeepsCredentialsOnExactOrigin(t *testing.T) { } require.Equal(t, []int32{443}, policy.Rules[2].GetHttps().GetPorts().GetNumbers()) require.Equal(t, []int32{8443}, policy.Rules[3].GetHttps().GetPorts().GetNumbers()) + require.Equal(t, "ate-secret://provider.example/token", policy.Rules[3].GetHttps().GetEffects().GetReplaceHeaders()[0].GetCredentialUri()) } diff --git a/go/core/internal/translator/adkconfig/mcp.go b/go/core/internal/translator/adkconfig/mcp.go index 04c354bbcc..771b31a990 100644 --- a/go/core/internal/translator/adkconfig/mcp.go +++ b/go/core/internal/translator/adkconfig/mcp.go @@ -5,8 +5,36 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/translator" ) +// ApplyCredentialHeaders adds gateway placeholders to the root and shared MCP clients. +func ApplyCredentialHeaders(config *configv1.ADKAgentConfig, additions []egress.Credential) error { + for i := range config.HttpTools { + params := config.HttpTools[i].Params + var err error + params.Headers, err = translator.CredentialHeaders(params.Url, params.Headers, additions) + if err != nil { + return err + } + } + for i := range config.SseTools { + params := config.SseTools[i].Params + var err error + params.Headers, err = translator.CredentialHeaders(params.Url, params.Headers, additions) + if err != nil { + return err + } + } + for _, child := range config.SubAgents { + if err := ApplyCredentialHeaders(child, additions); err != nil { + return err + } + } + return nil +} + // addRemoteMCPServer translates the two remote protocols supported by the ADK. // This path intentionally has no proxy URL or egress-gateway indirection. func (c *Builder) addRemoteMCPServer(config *configv1.ADKAgentConfig, server *v1alpha3.RemoteMCPServer, tools []string, requireApproval bool, headers map[string]string) error { diff --git a/go/core/internal/translator/adkconfig/mcp_test.go b/go/core/internal/translator/adkconfig/mcp_test.go index b7839cce56..d9352df838 100644 --- a/go/core/internal/translator/adkconfig/mcp_test.go +++ b/go/core/internal/translator/adkconfig/mcp_test.go @@ -7,6 +7,8 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/translator" ) func TestAddRemoteMCPServerPreservesBindingApproval(t *testing.T) { @@ -52,3 +54,16 @@ func TestAddRemoteMCPServerPreservesBindingApproval(t *testing.T) { }) } } + +func TestApplyCredentialHeadersToSharedAndSSEClients(t *testing.T) { + config := &configv1.ADKAgentConfig{ + HttpTools: []*configv1.ADKHTTPMCPServerConfig{{Params: &configv1.ADKHTTPConnectionParams{Url: "https://tools.example.com/http", Headers: map[string]string{"X-Trace": "keep"}}}}, + SseTools: []*configv1.ADKSSEMCPServerConfig{{Params: &configv1.ADKSSEConnectionParams{Url: "https://tools.example.com/sse"}}}, + SubAgents: []*configv1.ADKAgentConfig{{HttpTools: []*configv1.ADKHTTPMCPServerConfig{{Params: &configv1.ADKHTTPConnectionParams{Url: "https://tools.example.com/child"}}}}}, + } + require.NoError(t, ApplyCredentialHeaders(config, []egress.Credential{{Origin: "https://tools.example.com:443", Header: "authorization", URI: "ate-secret://provider.example/token"}})) + require.Equal(t, "keep", config.HttpTools[0].Params.Headers["X-Trace"]) + require.Equal(t, translator.CredentialPlaceholder, config.HttpTools[0].Params.Headers["authorization"]) + require.Equal(t, translator.CredentialPlaceholder, config.SseTools[0].Params.Headers["authorization"]) + require.Equal(t, translator.CredentialPlaceholder, config.SubAgents[0].HttpTools[0].Params.Headers["authorization"]) +} diff --git a/go/core/internal/translator/byo/compiler.go b/go/core/internal/translator/byo/compiler.go index 53f90f7fb9..4db2840a47 100644 --- a/go/core/internal/translator/byo/compiler.go +++ b/go/core/internal/translator/byo/compiler.go @@ -17,19 +17,21 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/core/internal/translator" "github.com/kagent-dev/kagent/go/core/internal/translator/adkconfig" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" "github.com/kagent-dev/kagent/go/pkg/tracing" ) // Compiler translates resolved inputs into a BYO A2A runtime revision. type Compiler struct { + binder egressext.CredentialBinder config *adkconfig.Builder settings *configv1.RuntimeDefaultsConfig } var _ translator.HarnessCompiler = (*Compiler)(nil) -func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig) *Compiler { - return &Compiler{settings: settings, config: adkconfig.NewBuilder(ctx, collections)} +func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig, binder egressext.CredentialBinder) *Compiler { + return &Compiler{settings: settings, binder: binder, config: adkconfig.NewBuilder(ctx, collections)} } func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) (*translator.CompileResult, error) { @@ -38,17 +40,6 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) return nil, err } template, harness := input.Root.Template, input.Harness - configJSON, err := runtimeconfig.Marshal(&configv1.ADKConfig{ - Runtime: translator.RuntimeConfig(c.settings, input.AgentName, template.Namespace), - Agent: compiled.Config, - }) - if err != nil { - var invalid *protovalidate.ValidationError - if errors.As(err, &invalid) { - return nil, translator.NewValidationError("invalid agent config: %w", err) - } - return nil, fmt.Errorf("marshal agent config: %w", err) - } card, err := pbconv.ToProtoAgentCard(agentTemplateCard(input.AgentName, template)) if err != nil { return nil, fmt.Errorf("convert agent card: %w", err) @@ -70,10 +61,25 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build revision provenance: %w", err) } - environment, credentials, tokenPassthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment) + environment, credentials, additions, tokenPassthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment, c.binder) if err != nil { return nil, err } + if err := adkconfig.ApplyCredentialHeaders(compiled.Config, additions); err != nil { + return nil, err + } + + configJSON, err := runtimeconfig.Marshal(&configv1.ADKConfig{ + Runtime: translator.RuntimeConfig(c.settings, input.AgentName, template.Namespace), + Agent: compiled.Config, + }) + if err != nil { + var invalid *protovalidate.ValidationError + if errors.As(err, &invalid) { + return nil, translator.NewValidationError("invalid agent config: %w", err) + } + return nil, fmt.Errorf("marshal agent config: %w", err) + } compiled.Egress = append(compiled.Egress, telemetryConfig.Destinations(environment)...) compiled.Egress = append(compiled.Egress, c.settings.GetControlPlaneUrl()) slices.Sort(compiled.Egress) diff --git a/go/core/internal/translator/byo/compiler_test.go b/go/core/internal/translator/byo/compiler_test.go index f9dd5b1dd3..762dfba0ab 100644 --- a/go/core/internal/translator/byo/compiler_test.go +++ b/go/core/internal/translator/byo/compiler_test.go @@ -30,7 +30,7 @@ func TestCompileOpaqueImage(t *testing.T) { Description: "custom A2A agent", SystemPrompt: "be helpful", }} - revision, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, testRuntimeDefaults()).Compile(context.Background(), &translator.HarnessInput{ + revision, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, testRuntimeDefaults(), nil).Compile(context.Background(), &translator.HarnessInput{ AgentName: "runnable-agent", Harness: harness, Root: &translator.AgentInput{Template: template, Instruction: template.Spec.SystemPrompt}, }) @@ -66,7 +66,7 @@ func TestCompileOpaqueImageKeepsItsOwnTelemetry(t *testing.T) { }} template := &translator.TemplateConfiguration{Name: "custom-agent", Namespace: "test", Source: &metav1.ObjectMeta{Name: "custom-agent", Namespace: "test"}} - revision, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, settings).Compile(context.Background(), &translator.HarnessInput{ + revision, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, settings, nil).Compile(context.Background(), &translator.HarnessInput{ AgentName: "runnable-agent", Harness: harness, Root: &translator.AgentInput{Template: template}, }) @@ -103,7 +103,7 @@ func TestCompiledTelemetryFitsTheActorEnvironmentBudget(t *testing.T) { harness.Spec.Env = append(harness.Spec.Env, v1alpha3.RuntimeEnvVar{Name: fmt.Sprintf("CUSTOM_%d", i), Value: "value"}) } template := &translator.TemplateConfiguration{Name: "custom-agent", Namespace: "test", Source: &metav1.ObjectMeta{Name: "custom-agent", Namespace: "test"}} - revision, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, settings).Compile(t.Context(), &translator.HarnessInput{ + revision, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, settings, nil).Compile(t.Context(), &translator.HarnessInput{ AgentName: "runnable-agent", Harness: harness, Root: &translator.AgentInput{Template: template}, }) require.NoError(t, err) diff --git a/go/core/internal/translator/claude/compiler.go b/go/core/internal/translator/claude/compiler.go index d5bb297486..90450831f7 100644 --- a/go/core/internal/translator/claude/compiler.go +++ b/go/core/internal/translator/claude/compiler.go @@ -21,18 +21,20 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" claudeconfig "github.com/kagent-dev/kagent/go/harness/claude/config" "github.com/kagent-dev/kagent/go/pkg/tracing" ) type Compiler struct { + binder egressext.CredentialBinder settings *configv1.RuntimeDefaultsConfig ctx krt.HandlerContext collections translator.Collections } -func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig) *Compiler { - return &Compiler{settings: settings, ctx: ctx, collections: collections} +func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig, binder egressext.CredentialBinder) *Compiler { + return &Compiler{settings: settings, binder: binder, ctx: ctx, collections: collections} } func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) (*translator.CompileResult, error) { @@ -95,13 +97,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) config.SkillResources = &skills.Resources } config.MCPServers = mcp.servers - if err := config.Validate(); err != nil { - return nil, translator.NewValidationError("invalid compiled Claude configuration: %v", err) - } - configJSON, err := json.Marshal(config) - if err != nil { - return nil, fmt.Errorf("marshal Claude config: %w", err) - } + card, err := pbconv.ToProtoAgentCard(translator.ManagedAgentCard(input.AgentName, input.Root.Template)) if err != nil { return nil, fmt.Errorf("convert Claude agent card: %w", err) @@ -110,10 +106,25 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build Claude revision provenance: %w", err) } - environment, credentials, tokenPassthroughOrigins, err := translator.CompileCredentials(input, nil, environment) + environment, credentials, additions, tokenPassthroughOrigins, err := translator.CompileCredentials(input, nil, environment, c.binder) if err != nil { return nil, err } + for name, server := range config.MCPServers { + server.Headers, err = translator.CredentialHeaders(server.URL, server.Headers, additions) + if err != nil { + return nil, err + } + config.MCPServers[name] = server + } + if err := config.Validate(); err != nil { + return nil, translator.NewValidationError("invalid compiled Claude configuration: %v", err) + } + + configJSON, err := json.Marshal(config) + if err != nil { + return nil, fmt.Errorf("marshal Claude config: %w", err) + } egress = append(egress, skills.Egress...) egress = append(egress, mcp.egress...) @@ -191,10 +202,13 @@ func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig) ([ return nil, nil, translator.NewValidationError("Claude does not support Anthropic provider options beyond baseUrl yet") } } - if err := c.requireSecretKey(ctx, model, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey, false); err != nil { - return nil, nil, err + environment := []corev1.EnvVar{{Name: claudeconfig.AnthropicAPIKeyEnvName, Value: translator.CredentialPlaceholder}} + if model.Spec.APIKeySecret != "" || c.binder == nil { + if err := c.requireSecretKey(ctx, model, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey, false); err != nil { + return nil, nil, err + } + environment[0] = secretEnvironment(claudeconfig.AnthropicAPIKeyEnvName, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey) } - environment := []corev1.EnvVar{secretEnvironment(claudeconfig.AnthropicAPIKeyEnvName, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey)} egress := []string{"https://api.anthropic.com:443"} if baseURL != "" { hostname, err := anthropicBaseURLOrigin(baseURL) @@ -219,6 +233,13 @@ func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig) ([ if !reflect.DeepEqual(options, v1alpha3.BedrockConfig{}) { return nil, nil, translator.NewValidationError("Claude does not support Bedrock provider options beyond region yet") } + if model.Spec.APIKeySecret == "" && c.binder != nil { + return []corev1.EnvVar{ + {Name: claudeconfig.UseBedrockEnvName, Value: "1"}, + {Name: claudeconfig.AWSRegionEnvName, Value: model.Spec.Bedrock.Region}, + {Name: claudeconfig.AWSBedrockTokenEnvName, Value: translator.CredentialPlaceholder}, + }, []string{"https://bedrock-runtime." + model.Spec.Bedrock.Region + ".amazonaws.com:443"}, nil + } if model.Spec.APIKeySecret == "" { return nil, nil, translator.NewValidationError("Claude Bedrock requires apiKeySecret with AWS credentials") } diff --git a/go/core/internal/translator/claude/compiler_test.go b/go/core/internal/translator/claude/compiler_test.go index d67f2604e0..2de9217833 100644 --- a/go/core/internal/translator/claude/compiler_test.go +++ b/go/core/internal/translator/claude/compiler_test.go @@ -82,7 +82,7 @@ func TestCompileProviderCredentials(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { input, reader := testInput(t, tt.model, tt.secretData) - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if tt.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tt.wantErr) { t.Fatalf("expected unsupported credential error, got %v", err) @@ -124,7 +124,7 @@ func TestCompileProviderCredentials(t *testing.T) { t.Fatalf("provenance contains gateway-managed Secret: %s", revision.Provenance) } - again, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + again, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil || !reflect.DeepEqual(revision, again) { t.Fatalf("compilation is not deterministic: %v", err) } @@ -151,7 +151,7 @@ func TestCompileRejectsUnsupportedConfiguration(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { input, reader := testInput(t, tt.model, map[string][]byte{"api-key": []byte("secret"), claudeconfig.AWSAccessKeyEnvName: []byte("access"), claudeconfig.AWSSecretKeyEnvName: []byte("secret"), "credentials.json": []byte(`{"type":"service_account"}`)}) - _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) var validation *translator.ValidationError if !errors.As(err, &validation) { t.Fatalf("Compile() error = %v, want validation error", err) @@ -168,7 +168,7 @@ func TestCompileRejectsProviderOwnedHarnessEnvironment(t *testing.T) { input, reader := testInput(t, model, map[string][]byte{"api-key": []byte("secret")}) value := "http://mock.example.com" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: claudeconfig.AnthropicBaseURLEnvName, Value: value}} - _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) var validation *translator.ValidationError if !errors.As(err, &validation) { t.Fatalf("Compile() error = %v, want validation error", err) @@ -184,7 +184,7 @@ func TestCompileAllowsRuntimeOTELEndpoint(t *testing.T) { value := "http://other-collector:4317" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: value}} - result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } @@ -202,7 +202,7 @@ func TestCompileAllowsRuntimeTelemetryOverrides(t *testing.T) { value := "department=engineering" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OTEL_RESOURCE_ATTRIBUTES", Value: value}} - result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } @@ -227,7 +227,7 @@ func TestCompileRootSkillsAndPluginSelections(t *testing.T) { Skills: []string{"deploy"}, }} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } @@ -245,7 +245,7 @@ func TestCompileRootSkillsAndPluginSelections(t *testing.T) { } input.Root.Template.Spec.Plugins[0].Skills = []string{"review"} - if _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input); err == nil || !strings.Contains(err.Error(), "duplicate skill name") { + if _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input); err == nil || !strings.Contains(err.Error(), "duplicate skill name") { t.Fatalf("duplicate skill Compile() error = %v", err) } } @@ -279,7 +279,7 @@ func TestCompileDirectWholeServerMCP(t *testing.T) { }}} input.Root.MCPTools = []translator.ResolvedMCPTool{{Binding: *input.Root.Template.Spec.Tools[0].MCP.DeepCopy(), Server: server}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } @@ -330,7 +330,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { } binding := v1alpha3.MCPToolBinding{Server: corev1.TypedLocalObjectReference{Kind: "RemoteMCPServer", Name: server.Name}} input.Root.MCPTools = []translator.ResolvedMCPTool{{Binding: binding, Server: server}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatalf("omitted selection Compile() error = %v", err) } @@ -339,7 +339,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { } input.Root.MCPTools[0].Binding.Tools = []string{"one"} - revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatalf("partial selection Compile() error = %v", err) } @@ -348,7 +348,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { } server.Status.ObservedGeneration = 0 - revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatalf("stale discovery Compile() error = %v", err) } @@ -362,7 +362,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { server.Spec.TLS = &v1alpha3.TLSConfig{DisableVerify: true} server.Spec.Timeout = &metav1.Duration{Duration: time.Minute} server.Spec.TerminateOnClose = &terminateOnClose - revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatalf("unsupported MCP options Compile() error = %v", err) } @@ -376,7 +376,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { } server.Spec.Protocol = v1alpha3.RemoteMCPServerProtocol("STDIO") - if _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input); err == nil || !strings.Contains(err.Error(), "unsupported protocol") { + if _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input); err == nil || !strings.Contains(err.Error(), "unsupported protocol") { t.Fatalf("unsupported MCP protocol Compile() error = %v", err) } @@ -386,7 +386,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { server.Spec.TerminateOnClose = nil input.Root.MCPTools[0].Binding.Tools = []string{"one"} input.Root.MCPTools[0].Binding.RequireApproval = true - revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatalf("approval-required MCP binding Compile() error = %v", err) } @@ -433,7 +433,7 @@ func TestCompileLocalSharedAgent(t *testing.T) { Name: "specialist", Description: "Handles specialist requests", Agent: child, }} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } @@ -495,7 +495,7 @@ func TestCompileRejectsUnsupportedLocalAgentConfiguration(t *testing.T) { } tt.mutate(&binding) input.Root.Shared = []translator.AgentInputBinding{binding} - _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err == nil || !strings.Contains(err.Error(), tt.want) { t.Fatalf("Compile() error = %v, want containing %q", err, tt.want) } @@ -531,7 +531,7 @@ func TestCompileRuntimeTelemetry(t *testing.T) { input.Harness.Name = "fast" settings := testRuntimeDefaults() - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, settings).Compile(context.Background(), input) + revision, err := NewCompiler(krt.TestingDummyContext{}, reader, settings, nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } @@ -552,7 +552,7 @@ func TestCompileRuntimeTelemetry(t *testing.T) { } settings.Telemetry = &configv1.TelemetryConfig{CaptureContent: true, MaxCaptureBytes: new(int32(4096))} - captured, err := NewCompiler(krt.TestingDummyContext{}, reader, settings).Compile(context.Background(), input) + captured, err := NewCompiler(krt.TestingDummyContext{}, reader, settings, nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } @@ -604,7 +604,7 @@ func TestCompiledTelemetryFitsTheActorEnvironmentBudget(t *testing.T) { }}} input.Root.MCPTools = []translator.ResolvedMCPTool{{Binding: *input.Root.Template.Spec.Tools[0].MCP.DeepCopy(), Server: server}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, settings).Compile(context.Background(), input) + revision, err := NewCompiler(krt.TestingDummyContext{}, reader, settings, nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } diff --git a/go/core/internal/translator/codex/compiler.go b/go/core/internal/translator/codex/compiler.go index 4300916d8f..5a74f59b67 100644 --- a/go/core/internal/translator/codex/compiler.go +++ b/go/core/internal/translator/codex/compiler.go @@ -20,6 +20,7 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" codexconfig "github.com/kagent-dev/kagent/go/harness/codex/config" "github.com/kagent-dev/kagent/go/pkg/tracing" ) @@ -41,13 +42,14 @@ var ownedEnvironment = map[string]struct{}{ } type Compiler struct { + binder egressext.CredentialBinder settings *configv1.RuntimeDefaultsConfig ctx krt.HandlerContext collections translator.Collections } -func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig) *Compiler { - return &Compiler{settings: settings, ctx: ctx, collections: collections} +func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig, binder egressext.CredentialBinder) *Compiler { + return &Compiler{settings: settings, binder: binder, ctx: ctx, collections: collections} } func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) (*translator.CompileResult, error) { @@ -115,13 +117,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if len(skills.Resources.Skills) != 0 || len(skills.Resources.Plugins) != 0 { cfg.SkillResources = &skills.Resources } - if err := cfg.Validate(); err != nil { - return nil, translator.NewValidationError("invalid compiled Codex configuration: %v", err) - } - configJSON, err := json.Marshal(cfg) - if err != nil { - return nil, fmt.Errorf("marshal Codex config: %w", err) - } + card, err := pbconv.ToProtoAgentCard(translator.ManagedAgentCard(input.AgentName, input.Root.Template)) if err != nil { return nil, fmt.Errorf("convert Codex agent card: %w", err) @@ -130,10 +126,25 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build Codex revision provenance: %w", err) } - environment, credentials, tokenPassthroughOrigins, err := translator.CompileCredentials(input, nil, environment) + environment, credentials, additions, tokenPassthroughOrigins, err := translator.CompileCredentials(input, nil, environment, c.binder) if err != nil { return nil, err } + for name, server := range cfg.MCPServers { + server.Headers, err = translator.CredentialHeaders(server.URL, server.Headers, additions) + if err != nil { + return nil, err + } + cfg.MCPServers[name] = server + } + if err := cfg.Validate(); err != nil { + return nil, translator.NewValidationError("invalid compiled Codex configuration: %v", err) + } + + configJSON, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("marshal Codex config: %w", err) + } egress = append(egress, skills.Egress...) egress = append(egress, mcp.egress...) egress = append(egress, telemetryConfig.Destinations(environment)...) @@ -164,8 +175,12 @@ func (c *Compiler) compileProvider(ctx context.Context, model *v1alpha3.ModelCon if !reflect.DeepEqual(options, v1alpha3.OpenAIConfig{}) { return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex does not support OpenAI provider options beyond baseUrl and apiFormat responses") } - if err := c.requireSecretKey(ctx, model.Namespace, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey); err != nil { - return codexconfig.Provider{}, nil, nil, err + environment := []corev1.EnvVar{{Name: openAIAPIKeyEnv, Value: translator.CredentialPlaceholder}} + if model.Spec.APIKeySecret != "" || c.binder == nil { + if err := c.requireSecretKey(ctx, model.Namespace, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey); err != nil { + return codexconfig.Provider{}, nil, nil, err + } + environment[0] = secretEnvironment(openAIAPIKeyEnv, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey) } provider := codexconfig.Provider{Name: "openai", BaseURL: baseURL} egress := []string{"https://api.openai.com:443"} @@ -176,7 +191,7 @@ func (c *Compiler) compileProvider(ctx context.Context, model *v1alpha3.ModelCon } egress = []string{host} } - return provider, []corev1.EnvVar{secretEnvironment(openAIAPIKeyEnv, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey)}, egress, nil + return provider, environment, egress, nil case v1alpha3.ModelProviderBedrock: if model.Spec.Bedrock == nil || strings.TrimSpace(model.Spec.Bedrock.Region) == "" { return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex Bedrock requires bedrock.region") @@ -193,6 +208,12 @@ func (c *Compiler) compileProvider(ctx context.Context, model *v1alpha3.ModelCon if !reflect.DeepEqual(options, v1alpha3.BedrockConfig{}) { return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex does not support Bedrock provider options beyond region") } + if model.Spec.APIKeySecret == "" && c.binder != nil { + return codexconfig.Provider{Name: "amazon-bedrock"}, []corev1.EnvVar{ + {Name: awsRegionEnv, Value: region}, + {Name: awsBedrockTokenEnv, Value: translator.CredentialPlaceholder}, + }, []string{"https://bedrock-runtime." + region + ".amazonaws.com:443"}, nil + } if model.Spec.APIKeySecret == "" || model.Spec.APIKeySecretKey != "" { return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex Bedrock requires apiKeySecret and an empty apiKeySecretKey") } diff --git a/go/core/internal/translator/codex/compiler_test.go b/go/core/internal/translator/codex/compiler_test.go index 2cc9eab92a..dc99c6d2fa 100644 --- a/go/core/internal/translator/codex/compiler_test.go +++ b/go/core/internal/translator/codex/compiler_test.go @@ -57,7 +57,7 @@ func TestCompileProviderCredentials(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { input, reader := testInput(t, test.model, test.secret) - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if test.wantErr != "" { if err == nil || !strings.Contains(err.Error(), test.wantErr) { t.Fatalf("expected unsupported credential error, got %v", err) @@ -92,7 +92,7 @@ func TestCompileProviderCredentials(t *testing.T) { if bytes.Contains(revision.ConfigJSON, []byte(credentialValue)) || bytes.Contains(revision.Provenance, []byte(credentialValue)) { t.Fatal("credential leaked into immutable revision") } - again, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + again, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil || !reflect.DeepEqual(revision, again) { t.Fatalf("compilation is not deterministic: %v", err) } @@ -111,7 +111,7 @@ func TestCompileAllowsRuntimeOTELEndpoint(t *testing.T) { value := "http://other-collector:4317" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: value}} - result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } @@ -131,7 +131,7 @@ func TestCompileAllowsRuntimeTelemetryOverrides(t *testing.T) { value := "x-tenant=team-a" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OTEL_EXPORTER_OTLP_HEADERS", Value: value}} - result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } @@ -150,7 +150,7 @@ func TestCompileRejectsUnsupportedProviderConfiguration(t *testing.T) { } for _, model := range tests { input, reader := testInput(t, model, map[string][]byte{"api-key": []byte("secret"), awsAccessKeyEnv: []byte("access"), awsSecretKeyEnv: []byte("secret")}) - _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) var validation *translator.ValidationError if !errors.As(err, &validation) { t.Errorf("Compile(%s) error = %v, want validation", model.Provider, err) @@ -175,7 +175,7 @@ func TestCompileMCPAndSharedAgent(t *testing.T) { }, Instruction: "Review carefully", }}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } @@ -218,7 +218,7 @@ func TestCompileMCPCompatibilityWarnings(t *testing.T) { } input.Root.MCPTools = []translator.ResolvedMCPTool{{Binding: v1alpha3.MCPToolBinding{RequireApproval: true}, Server: server}} - compilation, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + compilation, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) if err != nil { t.Fatalf("Compile() error = %v", err) } @@ -239,7 +239,7 @@ func TestCompileMCPCompatibilityWarnings(t *testing.T) { } server.Spec.Protocol = v1alpha3.RemoteMCPServerProtocolSse - if _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input); err == nil || !strings.Contains(err.Error(), "requires Streamable HTTP") { + if _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input); err == nil || !strings.Contains(err.Error(), "requires Streamable HTTP") { t.Fatalf("unsupported MCP protocol Compile() error = %v", err) } } @@ -275,7 +275,7 @@ func TestCompileRuntimeTelemetry(t *testing.T) { settings := testRuntimeDefaults() settings.Telemetry = &configv1.TelemetryConfig{Traces: &configv1.OTLPConfig{Endpoint: "http://collector:4317", Protocol: new("grpc")}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, settings).Compile(context.Background(), input) + revision, err := NewCompiler(krt.TestingDummyContext{}, reader, settings, nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } @@ -297,7 +297,7 @@ func TestCompileRuntimeTelemetry(t *testing.T) { } settings.Telemetry.CaptureContent = true - captured, err := NewCompiler(krt.TestingDummyContext{}, reader, settings).Compile(context.Background(), input) + captured, err := NewCompiler(krt.TestingDummyContext{}, reader, settings, nil).Compile(context.Background(), input) if err != nil { t.Fatal(err) } diff --git a/go/core/internal/translator/compiler_test.go b/go/core/internal/translator/compiler_test.go index 8ce2daebb1..26b49fbcee 100644 --- a/go/core/internal/translator/compiler_test.go +++ b/go/core/internal/translator/compiler_test.go @@ -212,10 +212,10 @@ func compilerWithImages(t *testing.T, images *configv1.HarnessImagesConfig, obje collections := mockCollections(t, append(objects, defaultWorkerPool())...) ctx := krt.TestingDummyContext{} return translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ - translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), - translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), - translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), - translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, testRuntimeDefaults(), nil), + translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, testRuntimeDefaults(), nil), + translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, testRuntimeDefaults(), nil), + translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, testRuntimeDefaults(), nil), }, images) } @@ -635,7 +635,7 @@ func TestCompileAgentInjectsCredentialsAtGateway(t *testing.T) { rotated.UID = "replacement-secret" rotated.Data["token"] = []byte("rotated-token") rotatedCompiler := translator.NewCompiler(krt.TestingDummyContext{}, mockCollections(t, modelConfig(), server, secondServer, rotated, secondSecret, defaultWorkerPool(), harness), map[translator.HarnessType]translator.HarnessCompiler{ - translator.HarnessTypeKagent: kagenttranslator.NewCompiler(krt.TestingDummyContext{}, mockCollections(t, modelConfig(), server, secondServer, rotated, secondSecret), testRuntimeDefaults()), + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(krt.TestingDummyContext{}, mockCollections(t, modelConfig(), server, secondServer, rotated, secondSecret), testRuntimeDefaults(), nil), }, testHarnessImages) next, err := rotatedCompiler.CompileAgent(t.Context(), inlineAgent(harness, template)) require.NoError(t, err) diff --git a/go/core/internal/translator/credential_runtime_test.go b/go/core/internal/translator/credential_runtime_test.go new file mode 100644 index 0000000000..7b3a0940cd --- /dev/null +++ b/go/core/internal/translator/credential_runtime_test.go @@ -0,0 +1,185 @@ +package translator_test + +import ( + "encoding/json" + "slices" + "testing" + + "github.com/stretchr/testify/require" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + runtimeconfig "github.com/kagent-dev/kagent/go/adk/pkg/config" + configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/translator" + byotranslator "github.com/kagent-dev/kagent/go/core/internal/translator/byo" + claudetranslator "github.com/kagent-dev/kagent/go/core/internal/translator/claude" + codextranslator "github.com/kagent-dev/kagent/go/core/internal/translator/codex" + kagenttranslator "github.com/kagent-dev/kagent/go/core/internal/translator/kagent" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" + claudeconfig "github.com/kagent-dev/kagent/go/harness/claude/config" + codexconfig "github.com/kagent-dev/kagent/go/harness/codex/config" +) + +// This consumer uses only the public extension contract. +type defaultCredentialBinder struct{ uri string } + +var _ egressext.CredentialBinder = defaultCredentialBinder{} + +func (d defaultCredentialBinder) Bindings(targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { + if d.uri == "" { + return nil, nil + } + var added []egressext.Credential + for _, target := range targets { + if slices.ContainsFunc(existing, func(c egressext.Credential) bool { return c.Origin == target.Origin }) { + continue + } + header, prefix := target.Header, target.Prefix + if header == "" { + header, prefix = "authorization", "Bearer " + } + added = append(added, egressext.Credential{Origin: target.Origin, Header: header, Prefix: prefix, URI: d.uri}) + } + return added, nil +} + +func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { + for _, test := range []struct { + kind translator.HarnessType + provider v1alpha3.ModelProvider + }{ + {translator.HarnessTypeKagent, v1alpha3.ModelProviderOpenAI}, + {translator.HarnessTypeBYO, v1alpha3.ModelProviderOpenAI}, + {translator.HarnessTypeClaude, v1alpha3.ModelProviderAnthropic}, + {translator.HarnessTypeCodex, v1alpha3.ModelProviderOpenAI}, + {translator.HarnessTypeClaude, v1alpha3.ModelProviderBedrock}, + {translator.HarnessTypeCodex, v1alpha3.ModelProviderBedrock}, + } { + kind := test.kind + t.Run(string(kind)+"/"+string(test.provider), func(t *testing.T) { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ + Workload: v1alpha3.HarnessWorkload{Image: new("example.com/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + model := modelConfig() + key := "OPENAI_API_KEY" + switch kind { + case translator.HarnessTypeKagent: + harness.Spec.Kagent = &v1alpha3.KagentHarness{} + case translator.HarnessTypeBYO: + harness.Spec.BYO = &v1alpha3.BYOHarness{} + case translator.HarnessTypeClaude: + harness.Spec.Claude = &v1alpha3.ClaudeHarness{} + model.Spec.Provider = v1alpha3.ModelProviderAnthropic + key = "ANTHROPIC_API_KEY" + case translator.HarnessTypeCodex: + harness.Spec.Codex = &v1alpha3.CodexHarness{} + model.Spec.OpenAI = &v1alpha3.OpenAIConfig{APIFormat: new(v1alpha3.OpenAIAPIFormatResponses)} + } + model.Spec.Provider = test.provider + switch test.provider { + case v1alpha3.ModelProviderBedrock: + model.Spec.Model = "gpt-5.2" + model.Spec.OpenAI = nil + model.Spec.Bedrock = &v1alpha3.BedrockConfig{Region: "us-east-1"} + key = "AWS_BEARER_TOKEN_BEDROCK" + } + server := &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Name: "tools", Namespace: "test"}, Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://tools.example.com/mcp"}} + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Name: "extra", Namespace: "test"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com:8443"}}}} + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help", Tools: []v1alpha3.ToolBinding{{MCP: &v1alpha3.MCPToolBinding{Server: corev1.TypedLocalObjectReference{Kind: "RemoteMCPServer", Name: server.Name}}}}, + }} + agent := inlineAgent(harness, template) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + collections := mockCollections(t, harness, model, server, policy, defaultWorkerPool()) + compile := func(binder egressext.CredentialBinder) (*translator.CompileResult, error) { + settings := testRuntimeDefaults() + ctx := krt.TestingDummyContext{} + compilers := map[translator.HarnessType]translator.HarnessCompiler{ + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings, binder), + translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings, binder), + translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings, binder), + translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings, binder), + } + return translator.NewCompiler(ctx, collections, compilers, nil).CompileAgent(t.Context(), agent) + } + result, err := compile(defaultCredentialBinder{uri: "ate-secret://provider.example/token"}) + require.NoError(t, err) + require.Len(t, result.Credentials, 2, "live EgressPolicy grants are not credential targets") + require.Contains(t, result.Environment, corev1.EnvVar{Name: key, Value: translator.CredentialPlaceholder}) + for _, variable := range result.Environment { + require.Nil(t, variable.ValueFrom) + } + switch kind { + case translator.HarnessTypeKagent, translator.HarnessTypeBYO: + var config configv1.ADKConfig + require.NoError(t, runtimeconfig.Unmarshal(result.ConfigJSON, &config)) + require.Equal(t, translator.CredentialPlaceholder, config.Agent.HttpTools[0].Params.Headers["authorization"]) + case translator.HarnessTypeClaude: + var config claudeconfig.Config + require.NoError(t, json.Unmarshal(result.ConfigJSON, &config)) + require.Equal(t, translator.CredentialPlaceholder, config.MCPServers["tools"].Headers["authorization"]) + case translator.HarnessTypeCodex: + var config codexconfig.Config + require.NoError(t, json.Unmarshal(result.ConfigJSON, &config)) + require.Equal(t, translator.CredentialPlaceholder, config.MCPServers["tools"].Headers["authorization"]) + } + first, err := result.Digest() + require.NoError(t, err) + again, err := compile(defaultCredentialBinder{uri: "ate-secret://provider.example/token"}) + require.NoError(t, err) + same, err := again.Digest() + require.NoError(t, err) + require.Equal(t, first, same) + changed, err := compile(defaultCredentialBinder{uri: "ate-secret://provider.example/next"}) + require.NoError(t, err) + second, err := changed.Digest() + require.NoError(t, err) + require.NotEqual(t, first, second) + require.Equal(t, result.EgressDestinations, changed.EgressDestinations) + if kind == translator.HarnessTypeKagent || kind == translator.HarnessTypeBYO { + baseline, err := compile(nil) + require.NoError(t, err) + require.Equal(t, baseline.EgressDestinations, result.EgressDestinations) + require.Empty(t, baseline.Credentials) + noop, err := compile(defaultCredentialBinder{}) + require.NoError(t, err) + require.Equal(t, baseline.Revision, noop.Revision) + nilDigest, err := baseline.Digest() + require.NoError(t, err) + noopDigest, err := noop.Digest() + require.NoError(t, err) + require.Equal(t, nilDigest, noopDigest) + require.NotContains(t, string(baseline.ConfigJSON), translator.CredentialPlaceholder) + } else { + _, err := compile(nil) + require.Error(t, err, "credentials are still required without an extension") + _, err = compile(defaultCredentialBinder{}) + require.ErrorContains(t, err, "requires a gateway credential") + } + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = "model-auth", "token" + secretKey := "token" + if test.provider == v1alpha3.ModelProviderBedrock { + model.Spec.APIKeySecretKey, secretKey = "", "AWS_BEARER_TOKEN_BEDROCK" + } + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "model-auth", Namespace: "test"}, Data: map[string][]byte{secretKey: []byte("model-secret")}} + collections = mockCollections(t, harness, model, server, policy, defaultWorkerPool(), secret) + baseline, err := compile(nil) + require.NoError(t, err) + noop, err := compile(defaultCredentialBinder{}) + require.NoError(t, err) + require.Equal(t, baseline.Revision, noop.Revision) + defaultDigest, err := baseline.Digest() + require.NoError(t, err) + noopDigest, err := noop.Digest() + require.NoError(t, err) + require.Equal(t, defaultDigest, noopDigest) + withDefault, err := compile(defaultCredentialBinder{uri: "ate-secret://provider.example/token"}) + require.NoError(t, err) + require.Contains(t, withDefault.Credentials, baseline.Credentials[0], "default credentials preserve explicit model authentication") + }) + } +} diff --git a/go/core/internal/translator/credentials.go b/go/core/internal/translator/credentials.go index 416f60cb43..226f632e73 100644 --- a/go/core/internal/translator/credentials.go +++ b/go/core/internal/translator/credentials.go @@ -2,7 +2,7 @@ package translator import ( "fmt" - "net/url" + "slices" "strings" corev1 "k8s.io/api/core/v1" @@ -10,6 +10,7 @@ import ( "github.com/kagent-dev/kagent/go/adk/pkg/models" "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" "github.com/kagent-dev/kagent/go/core/pkg/env" ) @@ -20,18 +21,19 @@ const CredentialPlaceholder = "kagent-credential-injected" // CompileCredentials replaces secret-backed environment values with inert SDK // placeholders and compiles their destination-scoped gateway bindings. Models // outside the agent tree (such as memory embeddings) are supplied separately. -func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, environment []corev1.EnvVar) ([]corev1.EnvVar, []egress.Credential, []string, error) { +// The third result contains only contributed bindings that need MCP placeholders. +func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, environment []corev1.EnvVar, binder egressext.CredentialBinder) ([]corev1.EnvVar, []egress.Credential, []egress.Credential, []string, error) { var bindings []egress.Credential var tokenPassthroughOrigins []string boundModels := map[string]bool{} boundMCP := map[string]bool{} bind := func(rawURL, header, prefix, namespace, name, key string) error { - u, err := url.Parse(strings.TrimSpace(rawURL)) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.Fragment != "" { - return NewValidationError("credential destination must be an absolute HTTP(S) URL without user information or fragment") + origin, err := credentialOrigin(rawURL) + if err != nil { + return NewValidationError("invalid credential destination: %v", err) } uri := "ate-secret://k8s.io/default/" + namespace + "/" + name + "/" + key - bindings = append(bindings, egress.Credential{Origin: egress.Origin(u), Header: header, Prefix: prefix, URI: uri}) + bindings = append(bindings, egress.Credential{Origin: origin, Header: header, Prefix: prefix, URI: uri}) return nil } models := append([]*ResolvedModelConfig(nil), extraModels...) @@ -58,7 +60,7 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, return nil } if err := visit(input.Root); err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } for _, resolved := range models { model := resolved.Config @@ -83,38 +85,44 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, } } if err := bind(endpoint, header, prefix, model.Namespace, model.Spec.APIKeySecret, key); err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } boundModels[name+"\x00"+model.Spec.APIKeySecret+"\x00"+key] = true } bindings, err := egress.CanonicalCredentials(bindings) if err != nil { - return nil, nil, nil, NewValidationError("%v", err) + return nil, nil, nil, nil, NewValidationError("%v", err) + } + additions, err := contributeCredentials(input, models, bindings, binder) + if err != nil { + return nil, nil, nil, nil, err + } + bindings, err = egress.CanonicalCredentials(append(bindings, additions...)) + if err != nil { + return nil, nil, nil, nil, NewValidationError("%v", err) } for _, resolved := range models { if !resolved.Config.Spec.APIKeyPassthrough { continue } _, endpoint, _, _ := modelCredentialTarget(resolved) - u, err := url.Parse(endpoint) - if err != nil { - return nil, nil, nil, NewValidationError("invalid passthrough credential destination") - } - origin, err := egress.ParseOrigin(egress.Origin(u)) + origin, err := credentialOrigin(endpoint) if err != nil { - return nil, nil, nil, NewValidationError("invalid passthrough credential destination: %v", err) + return nil, nil, nil, nil, NewValidationError("invalid passthrough credential destination: %v", err) } - tokenPassthroughOrigins = append(tokenPassthroughOrigins, origin.String()) + tokenPassthroughOrigins = append(tokenPassthroughOrigins, origin) } var destinations []string for _, binding := range bindings { destinations = append(destinations, binding.Origin) } if _, err := egress.MergeAccess(egress.Access{Destinations: destinations, Credentials: bindings}, egress.Access{}, tokenPassthroughOrigins); err != nil { - return nil, nil, nil, NewValidationError("%v", err) + return nil, nil, nil, nil, NewValidationError("%v", err) + } + result, err := applyModelCredentialPlaceholders(models, environment, bindings, additions) + if err != nil { + return nil, nil, nil, nil, err } - - result := append([]corev1.EnvVar(nil), environment...) for i, variable := range result { if variable.ValueFrom == nil { continue @@ -122,11 +130,47 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, ref := variable.ValueFrom.SecretKeyRef isMCP := strings.HasPrefix(variable.Name, "KAGENT_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CODEX_MCP_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CLAUDE_MCP_CREDENTIAL_") if ref == nil || (!boundModels[variable.Name+"\x00"+ref.Name+"\x00"+ref.Key] && (!isMCP || !boundMCP[ref.Name+"\x00"+ref.Key])) { - return nil, nil, nil, NewValidationError("environment credential %q cannot use gateway header injection; local signing and arbitrary secret environment variables are unsupported", variable.Name) + return nil, nil, nil, nil, NewValidationError("environment credential %q cannot use gateway header injection; local signing and arbitrary secret environment variables are unsupported", variable.Name) } result[i].Value, result[i].ValueFrom = CredentialPlaceholder, nil } - return result, bindings, tokenPassthroughOrigins, nil + return result, bindings, additions, tokenPassthroughOrigins, nil +} + +// Check requirements against the original environment, before Secret references +// or another model introduce placeholders for the same SDK variable. +func applyModelCredentialPlaceholders(models []*ResolvedModelConfig, environment []corev1.EnvVar, bindings, additions []egress.Credential) ([]corev1.EnvVar, error) { + result := append([]corev1.EnvVar(nil), environment...) + for _, resolved := range models { + model := resolved.Config + if model.Spec.APIKeySecret != "" || model.Spec.APIKeyPassthrough { + continue + } + name, endpoint, header, prefix := modelCredentialTarget(resolved) + if name == "" { + continue + } + origin, err := credentialOrigin(endpoint) + if err != nil { + continue + } + matches := func(binding egress.Credential) bool { + return binding.Origin == origin && binding.Header == header && binding.Prefix == prefix + } + bound := slices.ContainsFunc(bindings, matches) + i := slices.IndexFunc(result, func(variable corev1.EnvVar) bool { return variable.Name == name }) + required := slices.ContainsFunc(environment, func(variable corev1.EnvVar) bool { + return variable.Name == name && variable.Value == CredentialPlaceholder + }) + if required && !bound { + return nil, NewValidationError("model %q requires a gateway credential for %s header %s", model.Name, origin, header) + } + contributed := slices.ContainsFunc(additions, matches) + if contributed && i < 0 { + result = append(result, corev1.EnvVar{Name: name, Value: CredentialPlaceholder}) + } + } + return result, nil } func modelCredentialTarget(resolved *ResolvedModelConfig) (name, endpoint, header, prefix string) { diff --git a/go/core/internal/translator/credentials_test.go b/go/core/internal/translator/credentials_test.go index 752362029d..aeaf42aab6 100644 --- a/go/core/internal/translator/credentials_test.go +++ b/go/core/internal/translator/credentials_test.go @@ -34,7 +34,7 @@ func TestCompileCredentialDestinations(t *testing.T) { } input := credentialInput(test.spec) environment := []corev1.EnvVar{credentialEnv(test.env, "auth", test.spec.APIKeySecretKey)} - got, bindings, _, err := CompileCredentials(input, nil, environment) + got, bindings, _, _, err := CompileCredentials(input, nil, environment, nil) require.NoError(t, err) require.Equal(t, CredentialPlaceholder, got[0].Value) require.Nil(t, got[0].ValueFrom) @@ -54,31 +54,31 @@ func TestCompileCredentialsRejectsConflictingSharedModels(t *testing.T) { spec.APIKeySecret = "child" input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(spec).Root}} // The ADK environment is deduplicated, but every model still needs its own binding. - environment, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) + environment, _, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}, nil) require.ErrorContains(t, err, "conflicting credentials") require.Nil(t, environment) input.Root.Shared[0].Agent.ResolvedModelConfig.Config.Spec.OpenAI = &v1alpha3.OpenAIConfig{BaseURL: "https://child.example.com/v1"} - _, bindings, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) + _, bindings, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}, nil) require.NoError(t, err) require.Len(t, bindings, 2) } func TestCompileCredentialsRejectsLocalSecrets(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{}) - environment, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("AWS_SECRET_ACCESS_KEY", "auth", "token")}) + environment, _, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("AWS_SECRET_ACCESS_KEY", "auth", "token")}, nil) require.ErrorContains(t, err, "cannot use gateway header injection") require.Nil(t, environment) } func TestCompileCredentialsPreservesPassthrough(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: true}) - _, bindings, _, err := CompileCredentials(input, nil, nil) + _, bindings, _, _, err := CompileCredentials(input, nil, nil, nil) require.NoError(t, err) require.Empty(t, bindings) input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(v1alpha3.ModelConfigSpec{ Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "auth", APIKeySecretKey: "token", }).Root}} - environment, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}) + environment, _, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}, nil) require.ErrorContains(t, err, "cannot combine caller-token passthrough") require.Nil(t, environment) } @@ -176,9 +176,32 @@ func TestModelCredentialTargetOllamaCloud(t *testing.T) { func TestPassthroughCredentialUsesCanonicalOrigin(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: true, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://API.OPENAI.COM.:0443/v1"}}) - environment, credentials, origins, err := CompileCredentials(input, nil, nil) + environment, credentials, additions, origins, err := CompileCredentials(input, nil, nil, nil) require.NoError(t, err) require.Empty(t, environment) require.Empty(t, credentials) + require.Empty(t, additions) require.Equal(t, []string{"https://api.openai.com:443"}, origins) } + +func TestCompileCredentialsRejectsInvalidOrigins(t *testing.T) { + for _, endpoint := range []string{ + "ftp://api.example.com", "https://192.0.2.1", + "https://user@api.example.com", "https://api.example.com/#fragment", + } { + t.Run(endpoint, func(t *testing.T) { + for _, passthrough := range []bool{false, true} { + spec := v1alpha3.ModelConfigSpec{ + Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: passthrough, + OpenAI: &v1alpha3.OpenAIConfig{BaseURL: endpoint}, + } + if !passthrough { + spec.APIKeySecret, spec.APIKeySecretKey = "auth", "token" + } + _, bindings, _, _, err := CompileCredentials(credentialInput(spec), nil, nil, nil) + require.Error(t, err) + require.Empty(t, bindings) + } + }) + } +} diff --git a/go/core/internal/translator/egress_credentials.go b/go/core/internal/translator/egress_credentials.go new file mode 100644 index 0000000000..69ef50219d --- /dev/null +++ b/go/core/internal/translator/egress_credentials.go @@ -0,0 +1,152 @@ +package translator + +import ( + "fmt" + "net/url" + "slices" + "strings" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" +) + +func credentialOrigin(raw string) (string, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.User != nil || u.Fragment != "" { + return "", fmt.Errorf("invalid credential destination") + } + origin, err := egress.ParseOrigin(egress.Origin(u)) + if err != nil { + return "", err + } + return origin.String(), nil +} + +// contributeCredentials calls the extension with eligible destinations and validates its result. +func contributeCredentials(input *HarnessInput, models []*ResolvedModelConfig, existing []egress.Credential, binder egressext.CredentialBinder) ([]egress.Credential, error) { + if binder == nil { + return nil, nil + } + targets := eligibleCredentialTargets(input, models) + current := make([]egressext.Credential, len(existing)) + for i, binding := range existing { + current[i] = egressext.Credential(binding) + } + // Extension mutations must not change the compiler's bindings or validation targets. + contributed, err := binder.Bindings(slices.Clone(targets), current) + if err != nil { + return nil, fmt.Errorf("failed to contribute credentials: %w", err) + } + return validateContributedCredentials(contributed, targets) +} + +// eligibleCredentialTargets selects exact HTTPS destinations that permit additional authentication. +func eligibleCredentialTargets(input *HarnessInput, models []*ResolvedModelConfig) []egressext.CredentialTarget { + targets := map[string][]egressext.CredentialTarget{} + reserved := map[string]bool{} + add := func(raw, header, prefix string, preserve bool) { + origin, err := credentialOrigin(raw) + if err != nil || !strings.HasPrefix(origin, "https://") || strings.Contains(origin, "*") { + return + } + // Header replacement applies to the whole origin, so any explicit auth reserves it. + reserved[origin] = reserved[origin] || preserve + targets[origin] = append(targets[origin], egressext.CredentialTarget{Origin: origin, Header: header, Prefix: prefix}) + } + for _, model := range models { + name, endpoint, header, prefix := modelCredentialTarget(model) + literalKey := slices.ContainsFunc(input.Harness.Spec.Env, func(variable v1alpha3.RuntimeEnvVar) bool { return name != "" && variable.Name == name }) + // Keyless Gemini needs runtime support before a binding can supply its key. + keylessGemini := model.Config.Spec.Provider == v1alpha3.ModelProviderGemini && model.Config.Spec.APIKeySecret == "" + add(endpoint, header, prefix, literalKey || keylessGemini || model.Config.Spec.APIKeyPassthrough || len(model.Config.Spec.DefaultHeaders) != 0) + } + var visit func(*AgentInput) + visit = func(agent *AgentInput) { + for _, tool := range agent.MCPTools { + // Any literal or ConfigMap header can carry authentication. + preserve := slices.ContainsFunc(tool.Server.Spec.HeadersFrom, func(ref v1alpha3.ValueRef) bool { + return ref.ValueFrom == nil || ref.ValueFrom.Type != v1alpha3.SecretValueSource + }) + add(tool.Server.Spec.URL, "", "", preserve) + } + for _, child := range agent.Shared { + visit(child.Agent) + } + } + visit(input.Root) + var eligible []egressext.CredentialTarget + for origin, formats := range targets { + if reserved[origin] { + continue + } + // A generic MCP target must not relax a model's required header format. + known := slices.ContainsFunc(formats, func(target egressext.CredentialTarget) bool { return target.Header != "" }) + for _, target := range formats { + if target.Header != "" || !known { + eligible = append(eligible, target) + } + } + } + // Keep extension inputs stable across map iteration and duplicate references. + slices.SortFunc(eligible, func(a, b egressext.CredentialTarget) int { + return strings.Compare(a.Origin+"\x00"+a.Header+"\x00"+a.Prefix, b.Origin+"\x00"+b.Header+"\x00"+b.Prefix) + }) + return slices.Compact(eligible) +} + +// validateContributedCredentials converts bindings and checks their origins and header formats. +func validateContributedCredentials(contributed []egressext.Credential, targets []egressext.CredentialTarget) ([]egress.Credential, error) { + added := make([]egress.Credential, len(contributed)) + for i, binding := range contributed { + added[i] = egress.Credential(binding) + } + added, err := egress.CanonicalCredentials(added) + if err != nil { + return nil, NewValidationError("%v", err) + } + for _, binding := range added { + matches := slices.ContainsFunc(targets, func(target egressext.CredentialTarget) bool { + if target.Origin != binding.Origin { + return false + } + return target.Header == "" || (target.Header == binding.Header && target.Prefix == binding.Prefix) + }) + if matches { + continue + } + allowed := slices.ContainsFunc(targets, func(target egressext.CredentialTarget) bool { return target.Origin == binding.Origin }) + if !allowed { + return nil, NewValidationError("credential destination %q is not an eligible application target", binding.Origin) + } + return nil, NewValidationError("credential for %q must match a model header and prefix", binding.Origin) + } + return added, nil +} + +// CredentialHeaders adds placeholders for contributed credentials to compiled MCP headers. +func CredentialHeaders(rawURL string, headers map[string]string, additions []egress.Credential) (map[string]string, error) { + if len(additions) == 0 { + return headers, nil + } + origin, err := credentialOrigin(rawURL) + if err != nil { + return nil, NewValidationError("invalid MCP credential destination: %v", err) + } + for _, binding := range additions { + if binding.Origin != origin { + continue + } + present := false + for name := range headers { + present = present || strings.EqualFold(name, binding.Header) + } + if !present { + if headers == nil { + headers = map[string]string{} + } + headers[binding.Header] = CredentialPlaceholder + } + } + return headers, nil +} diff --git a/go/core/internal/translator/egress_credentials_test.go b/go/core/internal/translator/egress_credentials_test.go new file mode 100644 index 0000000000..bc70960b63 --- /dev/null +++ b/go/core/internal/translator/egress_credentials_test.go @@ -0,0 +1,181 @@ +package translator + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" +) + +type credentialBinderFunc func([]egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) + +func (f credentialBinderFunc) Bindings(targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { + return f(targets, existing) +} + +var _ egressext.CredentialBinder = credentialBinderFunc(nil) + +func TestEgressCredentialBinderTargets(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "model-key", APIKeySecretKey: "token"}) + input.Root.MCPTools = []ResolvedMCPTool{ + {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://MCP.Example.com./tools"}}}, + {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "http://local.example.com/tools"}}}, + } + input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAnthropic}).Root}} + binder := credentialBinderFunc(func(targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { + require.Equal(t, []egressext.CredentialTarget{ + {Origin: "https://api.anthropic.com:443", Header: "x-api-key"}, + {Origin: "https://api.openai.com:443", Header: "authorization", Prefix: "Bearer "}, + {Origin: "https://mcp.example.com:443"}, + }, targets) + require.Len(t, existing, 1) + require.Equal(t, egressext.Credential{ + Origin: "https://api.openai.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/model-key/token", + }, existing[0]) + // A binder cannot mutate the bindings used by the compiler. + existing[0].URI = "ate-secret://provider.example/changed" + return []egressext.Credential{{Origin: targets[0].Origin, Header: targets[0].Header, URI: "ate-secret://provider.example/token"}}, nil + }) + environment, bindings, added, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "model-key", "token")}, binder) + require.NoError(t, err) + require.Len(t, bindings, 2) + require.Equal(t, []egress.Credential{{ + Origin: "https://api.anthropic.com:443", Header: "x-api-key", URI: "ate-secret://provider.example/token", + }}, added) + require.Equal(t, "ate-secret://k8s.io/default/team/model-key/token", bindings[1].URI) + require.Contains(t, environment, corev1.EnvVar{Name: "ANTHROPIC_API_KEY", Value: CredentialPlaceholder}) +} + +func TestEgressCredentialBinderPreservesExplicitAuthentication(t *testing.T) { + for _, test := range []struct { + name string + configure func(*HarnessInput) + }{ + {"passthrough", func(input *HarnessInput) { input.Root.ResolvedModelConfig.Config.Spec.APIKeyPassthrough = true }}, + {"model headers", func(input *HarnessInput) { + input.Root.ResolvedModelConfig.Config.Spec.DefaultHeaders = map[string]string{"Authorization": "literal"} + }}, + {"harness key", func(input *HarnessInput) { + input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OPENAI_API_KEY", Value: "literal"}} + }}, + {"literal MCP header", func(input *HarnessInput) { + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.openai.com/mcp", HeadersFrom: []v1alpha3.ValueRef{{Name: "Authorization", Value: "literal"}}}}}} + }}, + {"ConfigMap MCP header", func(input *HarnessInput) { + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.openai.com/mcp", HeadersFrom: []v1alpha3.ValueRef{{Name: "Authorization", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.ConfigMapValueSource, Name: "auth", Key: "token"}}}}}}} + }}, + } { + t.Run(test.name, func(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + test.configure(input) + binder := credentialBinderFunc(func(targets []egressext.CredentialTarget, _ []egressext.Credential) ([]egressext.Credential, error) { + require.Empty(t, targets) + return nil, nil + }) + _, bindings, _, _, err := CompileCredentials(input, nil, nil, binder) + require.NoError(t, err) + require.Empty(t, bindings) + binder = credentialBinderFunc(func([]egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) { + return []egressext.Credential{{Origin: "https://api.openai.com", Header: "authorization", URI: "ate-secret://provider.example/token"}}, nil + }) + _, bindings, _, _, err = CompileCredentials(input, nil, nil, binder) + require.ErrorContains(t, err, "not an eligible application target") + require.Empty(t, bindings) + }) + } +} + +func TestEgressCredentialBinderRejectsInvalidBindings(t *testing.T) { + base := egressext.Credential{Origin: "https://api.openai.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/token"} + for _, test := range []struct { + name string + change func(*egressext.Credential) + want string + }{ + {"unknown origin", func(c *egressext.Credential) { c.Origin = "https://unrelated.example.com" }, "not an eligible application target"}, + {"different port", func(c *egressext.Credential) { c.Origin = "https://api.openai.com:8443" }, "not an eligible application target"}, + {"HTTP", func(c *egressext.Credential) { c.Origin = "http://api.openai.com" }, "not an eligible application target"}, + {"wildcard", func(c *egressext.Credential) { c.Origin = "https://*.openai.com" }, "exact HTTP(S) DNS origin"}, + {"routing header", func(c *egressext.Credential) { c.Header = "Host" }, "invalid credential injection header"}, + {"invalid prefix", func(c *egressext.Credential) { c.Prefix = "Bearer\n" }, "invalid credential injection header"}, + {"invalid URI", func(c *egressext.Credential) { c.URI = "https://provider.example/token" }, "invalid credential URI"}, + {"model header mismatch", func(c *egressext.Credential) { c.Header = "x-api-key" }, "must match a model header and prefix"}, + {"model prefix mismatch", func(c *egressext.Credential) { c.Prefix = "Basic " }, "must match a model header and prefix"}, + {"conflict", func(c *egressext.Credential) {}, "conflicting credentials"}, + } { + t.Run(test.name, func(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "auth", APIKeySecretKey: "token"}) + binding := base + test.change(&binding) + binder := credentialBinderFunc(func([]egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) { + return []egressext.Credential{binding}, nil + }) + _, bindings, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}, binder) + require.ErrorContains(t, err, test.want) + require.Empty(t, bindings) + }) + } + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + failure := errors.New("binder unavailable") + _, bindings, _, _, err := CompileCredentials(input, nil, nil, credentialBinderFunc(func([]egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) { + return nil, failure + })) + require.ErrorIs(t, err, failure) + require.Empty(t, bindings) +} + +func TestEgressCredentialBinderExtraModels(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + embedding := &ResolvedModelConfig{Config: &v1alpha3.ModelConfig{ObjectMeta: metav1.ObjectMeta{Name: "embedding", Namespace: "team"}, Spec: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://embeddings.example.com/v1"}}}} + binder := credentialBinderFunc(func(targets []egressext.CredentialTarget, _ []egressext.Credential) ([]egressext.Credential, error) { + require.Equal(t, []egressext.CredentialTarget{{Origin: "https://api.openai.com:443", Header: "authorization", Prefix: "Bearer "}, {Origin: "https://embeddings.example.com:443", Header: "authorization", Prefix: "Bearer "}}, targets) + return nil, nil + }) + _, bindings, _, _, err := CompileCredentials(input, []*ResolvedModelConfig{input.Root.ResolvedModelConfig, embedding}, nil, binder) + require.NoError(t, err) + require.Empty(t, bindings) +} + +func TestEgressCredentialBinderGeminiTargets(t *testing.T) { + for _, secret := range []string{"", "model-key"} { + t.Run("secret="+secret, func(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderGemini, APIKeySecret: secret}) + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://generativelanguage.googleapis.com/mcp"}}}} + targets := eligibleCredentialTargets(input, []*ResolvedModelConfig{input.Root.ResolvedModelConfig}) + if secret == "" { + require.Empty(t, targets, "MCP targets must not bypass the keyless Gemini exclusion") + } else { + require.Equal(t, []egressext.CredentialTarget{{Origin: "https://generativelanguage.googleapis.com:443", Header: "x-goog-api-key"}}, targets) + } + }) + } +} + +func TestEgressCredentialBinderCannotMutateValidationTargets(t *testing.T) { + for _, test := range []struct { + name string + change func(*egressext.CredentialTarget) + want string + }{ + {"origin", func(target *egressext.CredentialTarget) { target.Origin = "https://other.example.com:443" }, "not an eligible application target"}, + {"header", func(target *egressext.CredentialTarget) { target.Header = "x-other-key" }, "must match a model header and prefix"}, + } { + t.Run(test.name, func(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + binder := credentialBinderFunc(func(targets []egressext.CredentialTarget, _ []egressext.Credential) ([]egressext.Credential, error) { + require.Len(t, targets, 1) + test.change(&targets[0]) + return []egressext.Credential{{Origin: targets[0].Origin, Header: targets[0].Header, Prefix: targets[0].Prefix, URI: "ate-secret://provider.example/token"}}, nil + }) + _, bindings, _, _, err := CompileCredentials(input, nil, nil, binder) + require.ErrorContains(t, err, test.want) + require.Empty(t, bindings) + }) + } +} diff --git a/go/core/internal/translator/kagent/agentcard_test.go b/go/core/internal/translator/kagent/agentcard_test.go index d3c09986b9..93d5bf8247 100644 --- a/go/core/internal/translator/kagent/agentcard_test.go +++ b/go/core/internal/translator/kagent/agentcard_test.go @@ -15,7 +15,7 @@ import ( ) func TestCompilerRequiresModelConfig(t *testing.T) { - _, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, testRuntimeDefaults()).Compile(context.Background(), &translator.HarnessInput{ + _, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, testRuntimeDefaults(), nil).Compile(context.Background(), &translator.HarnessInput{ Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}}}, Root: &translator.AgentInput{Template: &translator.TemplateConfiguration{}}, }) diff --git a/go/core/internal/translator/kagent/compiler.go b/go/core/internal/translator/kagent/compiler.go index 8fd46c88df..9d384e87f8 100644 --- a/go/core/internal/translator/kagent/compiler.go +++ b/go/core/internal/translator/kagent/compiler.go @@ -14,18 +14,20 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/core/internal/translator" "github.com/kagent-dev/kagent/go/core/internal/translator/adkconfig" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) // Compiler translates resolved inputs into a kagent runtime revision. type Compiler struct { + binder egressext.CredentialBinder settings *configv1.RuntimeDefaultsConfig config *adkconfig.Builder } var _ translator.HarnessCompiler = (*Compiler)(nil) -func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig) *Compiler { - return &Compiler{settings: settings, config: adkconfig.NewBuilder(ctx, collections)} +func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig, binder egressext.CredentialBinder) *Compiler { + return &Compiler{settings: settings, binder: binder, config: adkconfig.NewBuilder(ctx, collections)} } func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) (*translator.CompileResult, error) { @@ -38,17 +40,6 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) return nil, err } template, harness := input.Root.Template, input.Harness - configJSON, err := runtimeconfig.Marshal(&configv1.ADKConfig{ - Runtime: translator.RuntimeConfig(c.settings, input.AgentName, template.Namespace), - Agent: compiled.Config, - }) - if err != nil { - var invalid *protovalidate.ValidationError - if errors.As(err, &invalid) { - return nil, translator.NewValidationError("invalid agent config: %w", err) - } - return nil, fmt.Errorf("marshal agent config: %w", err) - } card, err := pbconv.ToProtoAgentCard(translator.ManagedAgentCard(input.AgentName, template)) if err != nil { return nil, fmt.Errorf("convert agent card: %w", err) @@ -61,10 +52,25 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build revision provenance: %w", err) } - environment, credentials, tokenPassthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment) + environment, credentials, additions, tokenPassthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment, c.binder) if err != nil { return nil, err } + if err := adkconfig.ApplyCredentialHeaders(compiled.Config, additions); err != nil { + return nil, err + } + + configJSON, err := runtimeconfig.Marshal(&configv1.ADKConfig{ + Runtime: translator.RuntimeConfig(c.settings, input.AgentName, template.Namespace), + Agent: compiled.Config, + }) + if err != nil { + var invalid *protovalidate.ValidationError + if errors.As(err, &invalid) { + return nil, translator.NewValidationError("invalid agent config: %w", err) + } + return nil, fmt.Errorf("marshal agent config: %w", err) + } compiled.Egress = append(compiled.Egress, telemetryConfig.Destinations(environment)...) compiled.Egress = append(compiled.Egress, c.settings.GetControlPlaneUrl()) slices.Sort(compiled.Egress) diff --git a/go/core/pkg/app/app.go b/go/core/pkg/app/app.go index 347a4fb1ec..c174047d11 100644 --- a/go/core/pkg/app/app.go +++ b/go/core/pkg/app/app.go @@ -68,6 +68,7 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/version" "github.com/kagent-dev/kagent/go/core/pkg/auth" "github.com/kagent-dev/kagent/go/core/pkg/config" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" "github.com/kagent-dev/kagent/go/core/pkg/migrations" "github.com/kagent-dev/kagent/go/core/pkg/postgres" "github.com/kagent-dev/kagent/go/pkg/logging" @@ -83,6 +84,9 @@ import ( // every service that takes one — so a library consumer cannot narrow where // its own policy applies. type Options struct { + // EgressCredentialBinder tells the egress gateway which credentials to inject + // into an Agent's outbound requests. Nil adds none. + EgressCredentialBinder egressext.CredentialBinder // Authenticator identifies the caller. Nil selects InsecureAuthenticator, // which admits every request. Authenticator auth.AuthProvider @@ -272,7 +276,7 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er if err != nil { return fmt.Errorf("create controller manager: %w", err) } - runtime, err := controller.NewRuntime(kubeConfig, watchNamespaces, ctx.Done(), cfg.GetRuntimeDefaults(), cfg.GetHarnessImages()) + runtime, err := controller.NewRuntime(kubeConfig, watchNamespaces, ctx.Done(), cfg.GetRuntimeDefaults(), cfg.GetHarnessImages(), opts.EgressCredentialBinder) if err != nil { return err } diff --git a/go/core/pkg/egressext/credentials.go b/go/core/pkg/egressext/credentials.go new file mode 100644 index 0000000000..684e4819f3 --- /dev/null +++ b/go/core/pkg/egressext/credentials.go @@ -0,0 +1,24 @@ +// Package egressext defines outbound credential extensions. +package egressext + +// CredentialTarget is an eligible HTTPS origin. +// A nonempty Header requires the supplied header and prefix. +type CredentialTarget struct { + Origin string + Header string + Prefix string +} + +// Credential binds a provider reference to an outbound origin and header. +type Credential struct { + Origin string + Header string + Prefix string + URI string +} + +// CredentialBinder adds bindings during compilation for eligible targets without replacing existing bindings. +// Implementations must be deterministic and safe for concurrent calls, with no I/O or input mutation. +type CredentialBinder interface { + Bindings(targets []CredentialTarget, existing []Credential) ([]Credential, error) +} From 921be1e8ed72fbcaadc6dd47c4cbb6d94bb86a3c Mon Sep 17 00:00:00 2001 From: Cody Hartsook Date: Wed, 14 Oct 2026 13:08:30 -0700 Subject: [PATCH 35/43] docs: shorten credential extension overview Signed-off-by: Cody Hartsook --- docs/architecture/credential-injection.md | 36 ++++++----------------- 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/docs/architecture/credential-injection.md b/docs/architecture/credential-injection.md index 155a95e2d0..717c50bca3 100644 --- a/docs/architecture/credential-injection.md +++ b/docs/architecture/credential-injection.md @@ -86,33 +86,15 @@ the caller's authentication. ## Controller credential extension -Embedded controllers can supply `app.Options.EgressCredentialBinder` to add -credential references during compilation. Nil preserves the built-in behavior. -The [CredentialBinder contract](../../go/core/pkg/egressext/credentials.go) defines -`Bindings(targets, existing)` and its concurrency, determinism, and no-I/O requirements. - -Targets are exact HTTPS origins from resolved models and MCP servers. EgressPolicy -grants are live authorization, not pinned targets. The compiler excludes origins with passthrough, model default -headers, literal Harness model keys, or literal or ConfigMap MCP headers. -Bindings cannot expand network access or replace existing bindings. Model bindings -must match the supplied header and prefix. Invalid results fail Agent preparation. - -References use `ate-secret:///` with a lowercase, -port-free provider authority. Kubernetes references retain the -`k8s.io/default///` path. Substrate owns provider -registration, gateway authentication, and caching. The binder returns references, -never credential values. - -The built-in compilers add model and MCP placeholders for contributed bindings. -For keyless models, this selects API-key or bearer-token authentication, including -Bedrock and Foundry. Arbitrary clients must send the bound header themselves. -Headerless requests do not trigger credential resolution. Changed bindings create -a new revision. Existing sessions retain their pinned bindings. - -Keyless Gemini support is deferred. Its origins are excluded until runtime -support is added. Secret-backed Gemini keeps its existing behavior. Controller -and telemetry destinations, direct A2A delegation, and standalone sandboxes are -outside this extension's target scope. +Embedded controllers can set `app.Options.EgressCredentialBinder` to add gateway +credential references for eligible model and MCP destinations. EgressPolicy +grants are live authorization, not targets. Nil preserves built-in behavior. See [CredentialBinder](../../go/core/pkg/egressext/credentials.go) +for the contract. Core validates contributions, preserves explicit authentication, +and adds model/MCP placeholders without expanding network access. + +References use `ate-secret:///` with a lowercase, port-free +provider name registered in Substrate. Arbitrary clients must send the bound +header to trigger injection. Keyless Gemini support remains a follow-up. ## Deferred API fields From 09e6b6befb06fca62d462488cd4adfbef174c327 Mon Sep 17 00:00:00 2001 From: Cody Hartsook Date: Wed, 14 Oct 2026 13:58:57 -0700 Subject: [PATCH 36/43] feat: pass agent identity to credential binders Signed-off-by: Cody Hartsook --- .../translator/credential_runtime_test.go | 54 ++++++++++++++++++- .../internal/translator/egress_credentials.go | 5 +- .../translator/egress_credentials_test.go | 21 ++++---- go/core/pkg/egressext/credentials.go | 5 +- 4 files changed, 72 insertions(+), 13 deletions(-) diff --git a/go/core/internal/translator/credential_runtime_test.go b/go/core/internal/translator/credential_runtime_test.go index 7b3a0940cd..12b4446407 100644 --- a/go/core/internal/translator/credential_runtime_test.go +++ b/go/core/internal/translator/credential_runtime_test.go @@ -9,6 +9,7 @@ import ( "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" runtimeconfig "github.com/kagent-dev/kagent/go/adk/pkg/config" configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" @@ -28,7 +29,7 @@ type defaultCredentialBinder struct{ uri string } var _ egressext.CredentialBinder = defaultCredentialBinder{} -func (d defaultCredentialBinder) Bindings(targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { +func (d defaultCredentialBinder) Bindings(_ types.NamespacedName, targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { if d.uri == "" { return nil, nil } @@ -46,6 +47,57 @@ func (d defaultCredentialBinder) Bindings(targets []egressext.CredentialTarget, return added, nil } +type agentCredentialBinder map[types.NamespacedName]string + +var _ egressext.CredentialBinder = agentCredentialBinder(nil) + +func (a agentCredentialBinder) Bindings(agent types.NamespacedName, targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { + return defaultCredentialBinder{uri: a[agent]}.Bindings(agent, targets, existing) +} + +func TestCompileCredentialsUsesRunnableAgentIdentity(t *testing.T) { + binder := agentCredentialBinder{ + {Namespace: "team-a", Name: "researcher"}: "ate-secret://provider.example/team-a/researcher", + {Namespace: "team-a", Name: "reviewer"}: "ate-secret://provider.example/team-a/reviewer", + {Namespace: "team-b", Name: "researcher"}: "ate-secret://provider.example/team-b/researcher", + {Namespace: "team-b", Name: "reviewer"}: "ate-secret://provider.example/team-b/reviewer", + } + for _, namespace := range []string{"team-a", "team-b"} { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: namespace}, Spec: v1alpha3.HarnessSpec{ + Kagent: &v1alpha3.KagentHarness{}, Workload: v1alpha3.HarnessWorkload{Image: new("example.com/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + model := modelConfig() + model.Namespace = namespace + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "shared-behavior", Namespace: namespace}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help", + }} + pool := defaultWorkerPool() + pool.Namespace = namespace + collections := mockCollections(t, harness, model, template, pool) + settings := testRuntimeDefaults() + ctx := krt.TestingDummyContext{} + compiler := translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings, binder), + }, nil) + for _, source := range []string{"inline", "reference"} { + for _, name := range []string{"researcher", "reviewer"} { + t.Run(namespace+"/"+source+"/"+name, func(t *testing.T) { + agent := inlineAgent(harness, template) + agent.Name = name + if source == "reference" { + agent.Spec.Template = v1alpha3.AgentTemplateSource{Ref: &corev1.LocalObjectReference{Name: template.Name}} + } + result, err := compiler.CompileAgent(t.Context(), agent) + require.NoError(t, err) + require.Len(t, result.Credentials, 1) + require.Equal(t, binder[types.NamespacedName{Namespace: namespace, Name: name}], result.Credentials[0].URI) + }) + } + } + } +} + func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { for _, test := range []struct { kind translator.HarnessType diff --git a/go/core/internal/translator/egress_credentials.go b/go/core/internal/translator/egress_credentials.go index 69ef50219d..300b42a749 100644 --- a/go/core/internal/translator/egress_credentials.go +++ b/go/core/internal/translator/egress_credentials.go @@ -6,6 +6,8 @@ import ( "slices" "strings" + "k8s.io/apimachinery/pkg/types" + "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/pkg/egressext" @@ -33,8 +35,9 @@ func contributeCredentials(input *HarnessInput, models []*ResolvedModelConfig, e for i, binding := range existing { current[i] = egressext.Credential(binding) } + agent := types.NamespacedName{Namespace: input.Harness.Namespace, Name: input.AgentName} // Extension mutations must not change the compiler's bindings or validation targets. - contributed, err := binder.Bindings(slices.Clone(targets), current) + contributed, err := binder.Bindings(agent, slices.Clone(targets), current) if err != nil { return nil, fmt.Errorf("failed to contribute credentials: %w", err) } diff --git a/go/core/internal/translator/egress_credentials_test.go b/go/core/internal/translator/egress_credentials_test.go index bc70960b63..b9f692c3fd 100644 --- a/go/core/internal/translator/egress_credentials_test.go +++ b/go/core/internal/translator/egress_credentials_test.go @@ -7,16 +7,17 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) -type credentialBinderFunc func([]egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) +type credentialBinderFunc func(types.NamespacedName, []egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) -func (f credentialBinderFunc) Bindings(targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { - return f(targets, existing) +func (f credentialBinderFunc) Bindings(agent types.NamespacedName, targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { + return f(agent, targets, existing) } var _ egressext.CredentialBinder = credentialBinderFunc(nil) @@ -28,7 +29,7 @@ func TestEgressCredentialBinderTargets(t *testing.T) { {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "http://local.example.com/tools"}}}, } input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAnthropic}).Root}} - binder := credentialBinderFunc(func(targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { + binder := credentialBinderFunc(func(_ types.NamespacedName, targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { require.Equal(t, []egressext.CredentialTarget{ {Origin: "https://api.anthropic.com:443", Header: "x-api-key"}, {Origin: "https://api.openai.com:443", Header: "authorization", Prefix: "Bearer "}, @@ -74,14 +75,14 @@ func TestEgressCredentialBinderPreservesExplicitAuthentication(t *testing.T) { t.Run(test.name, func(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) test.configure(input) - binder := credentialBinderFunc(func(targets []egressext.CredentialTarget, _ []egressext.Credential) ([]egressext.Credential, error) { + binder := credentialBinderFunc(func(_ types.NamespacedName, targets []egressext.CredentialTarget, _ []egressext.Credential) ([]egressext.Credential, error) { require.Empty(t, targets) return nil, nil }) _, bindings, _, _, err := CompileCredentials(input, nil, nil, binder) require.NoError(t, err) require.Empty(t, bindings) - binder = credentialBinderFunc(func([]egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) { + binder = credentialBinderFunc(func(types.NamespacedName, []egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) { return []egressext.Credential{{Origin: "https://api.openai.com", Header: "authorization", URI: "ate-secret://provider.example/token"}}, nil }) _, bindings, _, _, err = CompileCredentials(input, nil, nil, binder) @@ -113,7 +114,7 @@ func TestEgressCredentialBinderRejectsInvalidBindings(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "auth", APIKeySecretKey: "token"}) binding := base test.change(&binding) - binder := credentialBinderFunc(func([]egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) { + binder := credentialBinderFunc(func(types.NamespacedName, []egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) { return []egressext.Credential{binding}, nil }) _, bindings, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}, binder) @@ -123,7 +124,7 @@ func TestEgressCredentialBinderRejectsInvalidBindings(t *testing.T) { } input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) failure := errors.New("binder unavailable") - _, bindings, _, _, err := CompileCredentials(input, nil, nil, credentialBinderFunc(func([]egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) { + _, bindings, _, _, err := CompileCredentials(input, nil, nil, credentialBinderFunc(func(types.NamespacedName, []egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) { return nil, failure })) require.ErrorIs(t, err, failure) @@ -133,7 +134,7 @@ func TestEgressCredentialBinderRejectsInvalidBindings(t *testing.T) { func TestEgressCredentialBinderExtraModels(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) embedding := &ResolvedModelConfig{Config: &v1alpha3.ModelConfig{ObjectMeta: metav1.ObjectMeta{Name: "embedding", Namespace: "team"}, Spec: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://embeddings.example.com/v1"}}}} - binder := credentialBinderFunc(func(targets []egressext.CredentialTarget, _ []egressext.Credential) ([]egressext.Credential, error) { + binder := credentialBinderFunc(func(_ types.NamespacedName, targets []egressext.CredentialTarget, _ []egressext.Credential) ([]egressext.Credential, error) { require.Equal(t, []egressext.CredentialTarget{{Origin: "https://api.openai.com:443", Header: "authorization", Prefix: "Bearer "}, {Origin: "https://embeddings.example.com:443", Header: "authorization", Prefix: "Bearer "}}, targets) return nil, nil }) @@ -168,7 +169,7 @@ func TestEgressCredentialBinderCannotMutateValidationTargets(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) - binder := credentialBinderFunc(func(targets []egressext.CredentialTarget, _ []egressext.Credential) ([]egressext.Credential, error) { + binder := credentialBinderFunc(func(_ types.NamespacedName, targets []egressext.CredentialTarget, _ []egressext.Credential) ([]egressext.Credential, error) { require.Len(t, targets, 1) test.change(&targets[0]) return []egressext.Credential{{Origin: targets[0].Origin, Header: targets[0].Header, Prefix: targets[0].Prefix, URI: "ate-secret://provider.example/token"}}, nil diff --git a/go/core/pkg/egressext/credentials.go b/go/core/pkg/egressext/credentials.go index 684e4819f3..73b447a2a7 100644 --- a/go/core/pkg/egressext/credentials.go +++ b/go/core/pkg/egressext/credentials.go @@ -1,6 +1,8 @@ // Package egressext defines outbound credential extensions. package egressext +import "k8s.io/apimachinery/pkg/types" + // CredentialTarget is an eligible HTTPS origin. // A nonempty Header requires the supplied header and prefix. type CredentialTarget struct { @@ -20,5 +22,6 @@ type Credential struct { // CredentialBinder adds bindings during compilation for eligible targets without replacing existing bindings. // Implementations must be deterministic and safe for concurrent calls, with no I/O or input mutation. type CredentialBinder interface { - Bindings(targets []CredentialTarget, existing []Credential) ([]Credential, error) + // agent identifies the runnable Agent, not its referenced AgentTemplate. + Bindings(agent types.NamespacedName, targets []CredentialTarget, existing []Credential) ([]Credential, error) } From 9c3b06e35c157e3f3325ad6853613d7511efc5fc Mon Sep 17 00:00:00 2001 From: Cody Hartsook Date: Wed, 14 Oct 2026 20:45:28 -0700 Subject: [PATCH 37/43] refactor: resolve egress policy before harness rendering Signed-off-by: Cody Hartsook --- docs/architecture/credential-injection.md | 13 +- go/core/internal/controller/collections.go | 4 +- go/core/internal/controller/reconciler.go | 12 +- go/core/internal/controller/runtime.go | 4 +- .../internal/translator/adkconfig/builder.go | 126 +------- .../translator/adkconfig/builder_test.go | 142 +-------- .../internal/translator/adkconfig/egress.go | 72 +++++ .../translator/adkconfig/egress_test.go | 31 ++ go/core/internal/translator/adkconfig/mcp.go | 28 -- .../internal/translator/adkconfig/mcp_test.go | 40 ++- .../internal/translator/adkconfig/model.go | 2 +- .../translator/adkconfig/model_test.go | 4 +- .../internal/translator/adkconfig/runtime.go | 25 +- go/core/internal/translator/byo/compiler.go | 18 +- .../internal/translator/byo/compiler_test.go | 6 +- .../internal/translator/claude/compiler.go | 100 +++--- .../translator/claude/compiler_test.go | 80 +++-- go/core/internal/translator/claude/mcp.go | 21 +- go/core/internal/translator/codex/compiler.go | 84 ++--- .../translator/codex/compiler_test.go | 64 +++- go/core/internal/translator/codex/mcp.go | 12 +- go/core/internal/translator/compiler.go | 67 +++- go/core/internal/translator/compiler_test.go | 29 +- go/core/internal/translator/credentials.go | 234 -------------- .../internal/translator/egress_credentials.go | 155 ---------- .../translator/egress_credentials_test.go | 182 ----------- go/core/internal/translator/egress_policy.go | 291 ++++++++++++++++++ ..._test.go => egress_policy_runtime_test.go} | 182 +++++++++-- .../internal/translator/egress_policy_test.go | 269 ++++++++++++++++ .../translator/kagent/agentcard_test.go | 2 +- .../internal/translator/kagent/compiler.go | 18 +- go/core/internal/translator/model_egress.go | 128 ++++++++ .../internal/translator/model_egress_test.go | 86 ++++++ .../internal/translator/resolved_policy.go | 77 +++++ ...ntials_test.go => resolved_policy_test.go} | 135 +++----- go/core/pkg/app/app.go | 8 +- go/core/pkg/egressext/credentials.go | 27 -- go/core/pkg/egressext/policy.go | 40 +++ 38 files changed, 1557 insertions(+), 1261 deletions(-) create mode 100644 go/core/internal/translator/adkconfig/egress.go create mode 100644 go/core/internal/translator/adkconfig/egress_test.go delete mode 100644 go/core/internal/translator/credentials.go delete mode 100644 go/core/internal/translator/egress_credentials.go delete mode 100644 go/core/internal/translator/egress_credentials_test.go create mode 100644 go/core/internal/translator/egress_policy.go rename go/core/internal/translator/{credential_runtime_test.go => egress_policy_runtime_test.go} (52%) create mode 100644 go/core/internal/translator/egress_policy_test.go create mode 100644 go/core/internal/translator/model_egress.go create mode 100644 go/core/internal/translator/model_egress_test.go create mode 100644 go/core/internal/translator/resolved_policy.go rename go/core/internal/translator/{credentials_test.go => resolved_policy_test.go} (60%) delete mode 100644 go/core/pkg/egressext/credentials.go create mode 100644 go/core/pkg/egressext/policy.go diff --git a/docs/architecture/credential-injection.md b/docs/architecture/credential-injection.md index 717c50bca3..2bf9f76bfb 100644 --- a/docs/architecture/credential-injection.md +++ b/docs/architecture/credential-injection.md @@ -86,11 +86,14 @@ the caller's authentication. ## Controller credential extension -Embedded controllers can set `app.Options.EgressCredentialBinder` to add gateway -credential references for eligible model and MCP destinations. EgressPolicy -grants are live authorization, not targets. Nil preserves built-in behavior. See [CredentialBinder](../../go/core/pkg/egressext/credentials.go) -for the contract. Core validates contributions, preserves explicit authentication, -and adds model/MCP placeholders without expanding network access. +Embedded controllers can set `app.Options.EgressPolicyTranslator` to add gateway +credential references before a Harness renders client configuration. See +[PolicyTranslator](../../go/core/pkg/egressext/policy.go) for the contract. +The shared compiler builds a draft from model and MCP inputs. EgressPolicy +grants are live authorization and stay out of the draft. The extension can only +add bindings for eligible destinations in that draft. Core validates the result, +preserves explicit authentication, and pins it in the revision. Nil adds no +bindings. References use `ate-secret:///` with a lowercase, port-free provider name registered in Substrate. Arbitrary clients must send the bound diff --git a/go/core/internal/controller/collections.go b/go/core/internal/controller/collections.go index b3047bb918..c15cc91ef5 100644 --- a/go/core/internal/controller/collections.go +++ b/go/core/internal/controller/collections.go @@ -66,7 +66,7 @@ func (p AgentRuntimeObservation) Equals(other AgentRuntimeObservation) bool { // NewCollections creates the complete read-only input graph. An empty // watchNamespaces list watches all namespaces. -func NewCollections(client kube.Client, watchNamespaces []string, opts krt.OptionsBuilder, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig, binder egressext.CredentialBinder) Collections { +func NewCollections(client kube.Client, watchNamespaces []string, opts krt.OptionsBuilder, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig, policyTranslator egressext.PolicyTranslator) Collections { egressPolicies := typedCollection[*kagentv1alpha3.EgressPolicy](client, watchNamespaces, "EgressPolicies", opts) sandboxTemplates := typedCollection[*kagentv1alpha3.SandboxTemplate](client, watchNamespaces, "SandboxTemplates", opts) agents := typedCollection[*kagentv1alpha3.Agent](client, watchNamespaces, "Agents", opts) @@ -84,7 +84,7 @@ func NewCollections(client kube.Client, watchNamespaces []string, opts krt.Optio Harnesses: harnesses, AgentTemplates: agentTemplates, ResolvedModelConfigs: resolvedModelConfigs, RemoteMCPServers: remoteMCPServers, ConfigMaps: configMaps, Secrets: secrets, WorkerPools: workerPools, } - reconciliations := newAgentReconciliations(agents, compilerCollections, agentRuntimeObservations, opts, settings, images, binder) + reconciliations := newAgentReconciliations(agents, compilerCollections, agentRuntimeObservations, opts, settings, images, policyTranslator) statuses := newAgentStatuses(agents, reconciliations, opts) return Collections{ diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index 54cc29a92b..0628e4edec 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -105,7 +105,7 @@ func newAgentReconciliations( opts krt.OptionsBuilder, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig, - binder egressext.CredentialBinder, + policyTranslator egressext.PolicyTranslator, ) krt.Collection[AgentReconciliation] { return krt.NewCollection(agents, func(ctx krt.HandlerContext, agent *kagentv1alpha3.Agent) *AgentReconciliation { state := &AgentReconciliation{Agent: agent} @@ -119,11 +119,11 @@ func newAgentReconciliations( state.EgressGrant, state.EgressIssues, state.EgressConflict = evaluateEgressGrant(resolution, base, tokenOrigins) }() compilation, err := translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ - translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings, binder), - translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings, binder), - translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings, binder), - translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings, binder), - }, images).CompileAgent(context.Background(), agent) + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings), + }, images, policyTranslator).CompileAgent(context.Background(), agent) if err != nil { state.CompilationFailure = compilationFailure(err) return state diff --git a/go/core/internal/controller/runtime.go b/go/core/internal/controller/runtime.go index 81331ac399..cf9743175b 100644 --- a/go/core/internal/controller/runtime.go +++ b/go/core/internal/controller/runtime.go @@ -24,13 +24,13 @@ type Runtime struct { // NewRuntime creates the shared KRT client and collection graph. Handlers are // deliberately registered separately at the eventual application boundary. -func NewRuntime(config *rest.Config, watchNamespaces []string, stop <-chan struct{}, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig, binder egressext.CredentialBinder) (*Runtime, error) { +func NewRuntime(config *rest.Config, watchNamespaces []string, stop <-chan struct{}, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig, policyTranslator egressext.PolicyTranslator) (*Runtime, error) { client, err := apiclient.New(config) if err != nil { return nil, fmt.Errorf("create KRT Kubernetes client: %w", err) } options := krt.NewOptionsBuilder(stop, "kagent", krt.GlobalDebugHandler) - collections := NewCollections(client, watchNamespaces, options, settings, images, binder) + collections := NewCollections(client, watchNamespaces, options, settings, images, policyTranslator) return &Runtime{Client: client, Options: options, Collections: collections}, nil } diff --git a/go/core/internal/translator/adkconfig/builder.go b/go/core/internal/translator/adkconfig/builder.go index a7e5e54ed0..acea3d8dbc 100644 --- a/go/core/internal/translator/adkconfig/builder.go +++ b/go/core/internal/translator/adkconfig/builder.go @@ -5,20 +5,16 @@ import ( "crypto/sha256" "encoding/json" "fmt" - "net/url" "slices" "strings" - "google.golang.org/protobuf/encoding/protojson" "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" runtimeconfig "github.com/kagent-dev/kagent/go/adk/pkg/config" - "github.com/kagent-dev/kagent/go/adk/pkg/models" configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -71,7 +67,7 @@ func (c *Builder) Build(ctx context.Context, input *translator.HarnessInput) (*R return nil, err } } - result, err := c.compileAgent(ctx, input.Root) + result, err := c.compileAgent(ctx, input.Root, input.Policy) if err != nil { return nil, err } @@ -79,10 +75,10 @@ func (c *Builder) Build(ctx context.Context, input *translator.HarnessInput) (*R if err := applyOutputSchema(result.Config, input.OutputSchema); err != nil { return nil, err } - if err := c.applyCompaction(result, input.Harness, input.Root.Template); err != nil { + if err := applyCompaction(result, input.Harness, input.Root.Template, input.SummarizerModel); err != nil { return nil, err } - if err := c.applyMemory(result, input.Harness); err != nil { + if err := applyMemory(result, input.Harness, input.MemoryModel); err != nil { return nil, err } } @@ -92,7 +88,7 @@ func (c *Builder) Build(ctx context.Context, input *translator.HarnessInput) (*R return result, nil } -func (c *Builder) compileAgent(ctx context.Context, input *translator.AgentInput) (*Result, error) { +func (c *Builder) compileAgent(ctx context.Context, input *translator.AgentInput, policy translator.ResolvedPolicy) (*Result, error) { modelRuntime := &modelRuntime{} var modelConfig *v1alpha3.ModelConfig if input.ResolvedModelConfig != nil { @@ -123,6 +119,10 @@ func (c *Builder) compileAgent(ctx context.Context, input *translator.AgentInput if err != nil { return nil, fmt.Errorf("resolve %s %q: %w", tool.Binding.Server.Kind, tool.Binding.Server.Name, err) } + headers, err = policy.MCPHeaders(tool.Server.Spec.URL, headers) + if err != nil { + return nil, err + } server := tool.Server.DeepCopy() server.Spec.HeadersFrom = nil if err := c.addRemoteMCPServer(cfg, server, tool.Binding.Tools, tool.Binding.RequireApproval, headers); err != nil { @@ -133,14 +133,14 @@ func (c *Builder) compileAgent(ctx context.Context, input *translator.AgentInput result := &Result{ Config: cfg, Templates: []*translator.TemplateConfiguration{input.Template}, Environment: modelRuntime.Environment, - Egress: append(agentConfigDestinations(cfg, modelConfig, modelRuntime.Model), plugins.Egress...), + Egress: plugins.Egress, SkillImages: plugins.Images, } if modelConfig != nil { result.Models = []*translator.ResolvedModelConfig{input.ResolvedModelConfig} } for _, binding := range input.Shared { - child, err := c.compileAgent(ctx, binding.Agent) + child, err := c.compileAgent(ctx, binding.Agent, policy) if err != nil { return nil, err } @@ -303,109 +303,3 @@ func DedupeEnv(values []corev1.EnvVar) []corev1.EnvVar { } return result } - -// agentConfigDestinations extracts the network allowlist required by the -// resolved model and MCP configuration. Provider defaults are included when -// no explicit endpoint appears in the serialized model. -func agentConfigDestinations(cfg *configv1.ADKAgentConfig, modelConfig *v1alpha3.ModelConfig, model *configv1.ADKModelConfig) []string { - destinations := make([]string, 0, len(cfg.HttpTools)+len(cfg.SseTools)+1) - for _, tool := range cfg.HttpTools { - destinations = appendURLOrigin(destinations, tool.Params.Url) - } - for _, tool := range cfg.SseTools { - destinations = appendURLOrigin(destinations, tool.Params.Url) - } - modelJSON, _ := protojson.Marshal(model) - var values any - if json.Unmarshal(modelJSON, &values) == nil { - destinations = appendURLValues(destinations, values) - } - if modelConfig == nil { - slices.Sort(destinations) - return slices.Compact(destinations) - } - switch modelConfig.Spec.Provider { - case v1alpha3.ModelProviderOpenAI: - destinations = append(destinations, "https://api.openai.com:443") - case v1alpha3.ModelProviderAnthropic: - destinations = append(destinations, "https://api.anthropic.com:443") - case v1alpha3.ModelProviderGemini: - destinations = append(destinations, "https://generativelanguage.googleapis.com:443") - case v1alpha3.ModelProviderMistral: - if mistral := modelConfig.Spec.Mistral; mistral == nil || mistral.BaseURL == nil || *mistral.BaseURL == "" { - destinations = append(destinations, "https://api.mistral.ai:443") - } - case v1alpha3.ModelProviderOllama: - // Ollama's endpoint is the provider's own field and is not part of the - // serialized model, so the walk above never sees it. Unlike the - // providers above there is no default to fall back on: the host is the - // operator's, so it has to be read from the spec. - if ollama := modelConfig.Spec.Ollama; ollama != nil { - if ollama.Host != "" { - destinations = appendURLOrigin(destinations, withDefaultScheme(ollama.Host)) - } - // A cloud model with a key and no explicit host reaches - // api.ollama.com, so the agent needs that host allowed or the call - // is denied by egress policy. The condition is shared with the key - // reference and credential binding (models.OllamaReachesCloud). - // - // This used to add the host whenever a cloud model had any key, - // ignoring the operator's host. That over-allowed egress: with a - // host set the request goes to that host, so api.ollama.com never - // needs to be reachable. - hasCredential := modelConfig.Spec.APIKeySecret != "" || modelConfig.Spec.APIKeyPassthrough - if models.OllamaReachesCloud(modelConfig.Spec.Model, ollama.Host, hasCredential) { - destinations = append(destinations, "https://api.ollama.com:443") - } - } - } - slices.Sort(destinations) - return slices.Compact(destinations) -} - -// withDefaultScheme makes a bare host:port an absolute URL. The Ollama host -// field accepts either form, but net/url reads the bare one as a scheme, so a -// caller that wants a hostname from it has to normalize first. -// -// A bare host defaults to http, which is right for a daemon (host:port on a -// private address). It is wrong for ollama.com's API, which serves HTTPS only: -// Cloudflare answers the http form with a redirect, and Go turns a 301 into a -// GET, so the POST /api/chat that should carry the request comes back 405 -// Method Not Allowed. The cloud endpoint therefore has to be normalized to -// https here, which is what the runtime's own copy of this does — the two -// disagreed, and the actor was pinned to http://api.ollama.com. -func withDefaultScheme(host string) string { - if strings.HasPrefix(host, "http://") || strings.HasPrefix(host, "https://") { - return host - } - if models.IsOllamaCloudEndpoint(host) { - return "https://" + host - } - return "http://" + host -} - -// appendURLValues walks serialized provider config because endpoint fields are -// provider-specific but all URLs reduce to HTTP(S) origins. -func appendURLValues(destinations []string, value any) []string { - switch value := value.(type) { - case string: - return appendURLOrigin(destinations, value) - case []any: - for _, item := range value { - destinations = appendURLValues(destinations, item) - } - case map[string]any: - for _, item := range value { - destinations = appendURLValues(destinations, item) - } - } - return destinations -} - -func appendURLOrigin(destinations []string, raw string) []string { - parsed, err := url.Parse(raw) - if err == nil && parsed.Hostname() != "" { - return append(destinations, egress.Origin(parsed)) - } - return destinations -} diff --git a/go/core/internal/translator/adkconfig/builder_test.go b/go/core/internal/translator/adkconfig/builder_test.go index 2babf5a0cb..c8ec6c290b 100644 --- a/go/core/internal/translator/adkconfig/builder_test.go +++ b/go/core/internal/translator/adkconfig/builder_test.go @@ -85,6 +85,12 @@ func TestBuildCompaction(t *testing.T) { builder := NewBuilder(krt.TestingDummyContext{}, collections) return builder.Build(context.Background(), &translator.HarnessInput{ Harness: harness, Root: &translator.AgentInput{Template: template, ResolvedModelConfig: agentModel}, + SummarizerModel: func() *translator.ResolvedModelConfig { + if c := harness.Spec.Kagent.Compaction; c != nil && c.Summarizer != nil && c.Summarizer.ModelConfigRef != nil && c.Summarizer.ModelConfigRef.Name == "summarizer" { + return resolvedModel(t, collections, "summarizer") + } + return nil + }(), }) } @@ -123,7 +129,7 @@ func TestBuildCompaction(t *testing.T) { require.Equal(t, "https://summarizer.example.com/v1", summarizer.BaseUrl) require.Len(t, result.Models, 2) require.Equal(t, "summarizer", result.Models[1].Config.Name) - require.Contains(t, result.Egress, "https://summarizer.example.com:443") + var credentials []string for _, variable := range result.Environment { if variable.ValueFrom != nil && variable.ValueFrom.SecretKeyRef != nil { @@ -153,131 +159,6 @@ func openAIModel(name, baseURL string) *v1alpha3.ModelConfig { } } -func TestMistralEgressDestination(t *testing.T) { - for _, test := range []struct { - name string - mistral *v1alpha3.MistralConfig - want string - }{ - {name: "default endpoint", want: "https://api.mistral.ai:443"}, - {name: "base URL override", mistral: &v1alpha3.MistralConfig{BaseURL: new("https://mistral.example.com/v1")}, want: "https://mistral.example.com:443"}, - } { - t.Run(test.name, func(t *testing.T) { - model := &v1alpha3.ModelConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "mistral", Namespace: "test"}, - Spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderMistral, Model: "mistral-large-latest", Mistral: test.mistral, - }, - } - collections := contextTestCollections(t, model) - result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(context.Background(), - &translator.HarnessInput{ - Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}}}, - Root: &translator.AgentInput{ - Template: &translator.TemplateConfiguration{Name: "pi", Namespace: "test"}, - ResolvedModelConfig: resolvedModel(t, collections, "mistral"), - }}) - require.NoError(t, err) - require.Equal(t, []string{test.want}, result.Egress) - }) - } -} - -// TestOllamaEgressDestination covers the one provider whose endpoint is not -// reachable through the serialized model: it lives on the ModelConfig's own -// Ollama field, so the generic walk over the model never sees it. Without an -// explicit case the allowlist came back empty, which compiles into a deny-all -// Actor egress policy and fails the model call with a 403. -func TestOllamaEgressDestination(t *testing.T) { - for _, host := range []string{"http://host.docker.internal:11434", "host.docker.internal:11434"} { - t.Run(host, func(t *testing.T) { - model := &v1alpha3.ModelConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "ollama", Namespace: "test"}, - Spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "llama3.2", - Ollama: &v1alpha3.OllamaConfig{Host: host}, - }, - } - collections := contextTestCollections(t, model) - result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(context.Background(), - &translator.HarnessInput{ - Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}}}, - Root: &translator.AgentInput{ - Template: &translator.TemplateConfiguration{Name: "pi", Namespace: "test"}, - ResolvedModelConfig: resolvedModel(t, collections, "ollama"), - }}) - require.NoError(t, err) - require.Equal(t, []string{"http://host.docker.internal:11434"}, result.Egress) - }) - } - - t.Run("no host configured", func(t *testing.T) { - model := &v1alpha3.ModelConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "ollama", Namespace: "test"}, - Spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "llama3.2", - Ollama: &v1alpha3.OllamaConfig{}, - }, - } - collections := contextTestCollections(t, model) - result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(context.Background(), - &translator.HarnessInput{ - Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}}}, - Root: &translator.AgentInput{ - Template: &translator.TemplateConfiguration{Name: "pi", Namespace: "test"}, - ResolvedModelConfig: resolvedModel(t, collections, "ollama"), - }}) - require.NoError(t, err) - require.Empty(t, result.Egress) - }) - - // A cloud-tagged model with a key bypasses the operator's host, so the cloud - // host has to be allowed as well or egress policy denies the call. - t.Run("cloud model with a secret allows api.ollama.com", func(t *testing.T) { - model := &v1alpha3.ModelConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "ollama", Namespace: "test"}, - Spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "deepseek-v4-flash:0731-cloud", - APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", - Ollama: &v1alpha3.OllamaConfig{}, - }, - } - collections := contextTestCollections(t, model) - result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(context.Background(), - &translator.HarnessInput{ - Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}}}, - Root: &translator.AgentInput{ - Template: &translator.TemplateConfiguration{Name: "pi", Namespace: "test"}, - ResolvedModelConfig: resolvedModel(t, collections, "ollama"), - }}) - require.NoError(t, err) - require.Contains(t, result.Egress, "https://api.ollama.com:443") - }) - - // A local model must never pick up the cloud host, even with a key present: - // that is the rerouting regression the routing rules exist to prevent. - t.Run("local model with a key does not allow the cloud host", func(t *testing.T) { - model := &v1alpha3.ModelConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "ollama", Namespace: "test"}, - Spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "llama3.2", - APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", - Ollama: &v1alpha3.OllamaConfig{Host: "host.docker.internal:11434"}, - }, - } - collections := contextTestCollections(t, model) - result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(context.Background(), - &translator.HarnessInput{ - Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}}}, - Root: &translator.AgentInput{ - Template: &translator.TemplateConfiguration{Name: "pi", Namespace: "test"}, - ResolvedModelConfig: resolvedModel(t, collections, "ollama"), - }}) - require.NoError(t, err) - require.Equal(t, []string{"http://host.docker.internal:11434"}, result.Egress) - }) -} - func contextTestCollections(t *testing.T, models ...*v1alpha3.ModelConfig) translator.Collections { t.Helper() objects := make([]any, 0, 2*len(models)) @@ -339,7 +220,12 @@ func TestBuildMemory(t *testing.T) { ModelConfigRef: corev1.LocalObjectReference{Name: tc.modelName}, TTLDays: 7, }}}, } - result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(t.Context(), &translator.HarnessInput{Harness: harness, Root: root}) + result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(t.Context(), &translator.HarnessInput{Harness: harness, Root: root, MemoryModel: func() *translator.ResolvedModelConfig { + if tc.modelName == "embedding" { + return resolvedModel(t, collections, "embedding") + } + return nil + }()}) if tc.wantErr != "" { require.ErrorContains(t, err, tc.wantErr) require.Nil(t, result) @@ -350,7 +236,7 @@ func TestBuildMemory(t *testing.T) { require.Equal(t, "https://embedding.example.com/v1", result.Config.Memory.Embedding.BaseUrl) require.Len(t, result.Models, 2) require.Equal(t, "embedding", result.Models[1].Config.Name) - require.Contains(t, result.Egress, "https://embedding.example.com:443") + require.Len(t, result.Environment, 2) require.Equal(t, "embedding-auth", result.Environment[1].ValueFrom.SecretKeyRef.Name) }) diff --git a/go/core/internal/translator/adkconfig/egress.go b/go/core/internal/translator/adkconfig/egress.go new file mode 100644 index 0000000000..d5274f2df0 --- /dev/null +++ b/go/core/internal/translator/adkconfig/egress.go @@ -0,0 +1,72 @@ +package adkconfig + +import ( + "net/url" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/translator" +) + +// AdditionalModelDestinations preserves ADK grants beyond the effective model endpoint. +// Retained defaults and metadata origins do not create credential targets. +func AdditionalModelDestinations(resolved *translator.ResolvedModelConfig) []string { + spec := resolved.Config.Spec + var destinations []string + switch spec.Provider { + case v1alpha3.ModelProviderOpenAI: + destinations = append(destinations, "https://api.openai.com:443") + case v1alpha3.ModelProviderAnthropic: + destinations = append(destinations, "https://api.anthropic.com:443") + } + // The ADK config walk also allowed URL-valued metadata. Preserve those grants + // without rendering a client or scanning inactive provider configurations. + modelName := spec.Model + if spec.Provider == v1alpha3.ModelProviderAzureOpenAI && spec.AzureOpenAI != nil { + modelName = spec.AzureOpenAI.DeploymentName + } + destinations = appendModelMetadataOrigin(destinations, modelName) + for _, value := range spec.DefaultHeaders { + destinations = appendModelMetadataOrigin(destinations, value) + } + var values []string + switch spec.Provider { + case v1alpha3.ModelProviderAzureOpenAI: + if spec.AzureOpenAI != nil { + values = []string{spec.AzureOpenAI.APIVersion} + } + case v1alpha3.ModelProviderOllama: + if spec.Ollama != nil { + for _, value := range spec.Ollama.Options { + values = append(values, value) + } + } + case v1alpha3.ModelProviderBedrock: + if spec.Bedrock != nil { + values = []string{spec.Bedrock.Region, spec.Bedrock.CacheTTL} + if guardrail := spec.Bedrock.Guardrail; guardrail != nil { + values = append(values, guardrail.Identifier, guardrail.Version, guardrail.Trace) + } + } + case v1alpha3.ModelProviderSAPAICore: + if spec.SAPAICore != nil { + values = []string{spec.SAPAICore.ResourceGroup} + } + case v1alpha3.ModelProviderFoundry: + if spec.Foundry != nil { + values = []string{spec.Foundry.Deployment, spec.Foundry.APIVersion} + } + } + for _, value := range values { + destinations = appendModelMetadataOrigin(destinations, value) + } + return destinations +} + +func appendModelMetadataOrigin(destinations []string, value string) []string { + parsed, err := url.Parse(value) + if err == nil && parsed.Hostname() != "" { + return append(destinations, egress.Origin(parsed)) + } + return destinations +} diff --git a/go/core/internal/translator/adkconfig/egress_test.go b/go/core/internal/translator/adkconfig/egress_test.go new file mode 100644 index 0000000000..e0ca775c37 --- /dev/null +++ b/go/core/internal/translator/adkconfig/egress_test.go @@ -0,0 +1,31 @@ +package adkconfig + +import ( + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/stretchr/testify/require" + "slices" + "testing" +) + +func TestRetainedADKMetadataDestinations(t *testing.T) { + for _, test := range []struct { + name string + spec v1alpha3.ModelConfigSpec + want []string + }{ + {"model and headers", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, Model: "https://model.example.com/name", DefaultHeaders: map[string]string{"HTTP-Referer": "https://app.example.com/page"}, Ollama: &v1alpha3.OllamaConfig{Options: map[string]string{"inactive": "https://unused.example.com"}}}, []string{"https://api.openai.com:443", "https://app.example.com:443", "https://model.example.com:443"}}, + {"Azure deployment replaces model name", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAzureOpenAI, Model: "https://unused.example.com", AzureOpenAI: &v1alpha3.AzureOpenAIConfig{DeploymentName: "https://deployment.example.com", APIVersion: "https://version.example.com"}}, []string{"https://deployment.example.com:443", "https://version.example.com:443"}}, + {"Ollama options", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Ollama: &v1alpha3.OllamaConfig{Options: map[string]string{"url": "https://options.example.com/path"}}}, []string{"https://options.example.com:443"}}, + {"Bedrock guardrail", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderBedrock, Bedrock: &v1alpha3.BedrockConfig{Region: "us-east-1", Guardrail: &v1alpha3.BedrockGuardrailConfig{Identifier: "https://guardrail.example.com/id", Version: "DRAFT"}}}, []string{"https://guardrail.example.com:443"}}, + {"SAP resource group", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderSAPAICore, SAPAICore: &v1alpha3.SAPAICoreConfig{ResourceGroup: "https://group.example.com"}}, []string{"https://group.example.com:443"}}, + {"Foundry metadata", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderFoundry, Foundry: &v1alpha3.FoundryConfig{Deployment: "https://deployment.example.com", APIVersion: "https://version.example.com"}}, []string{"https://deployment.example.com:443", "https://version.example.com:443"}}, + } { + t.Run(test.name, func(t *testing.T) { + model := &translator.ResolvedModelConfig{Config: &v1alpha3.ModelConfig{Spec: test.spec}} + got := AdditionalModelDestinations(model) + slices.Sort(got) + require.Equal(t, test.want, got) + }) + } +} diff --git a/go/core/internal/translator/adkconfig/mcp.go b/go/core/internal/translator/adkconfig/mcp.go index 771b31a990..04c354bbcc 100644 --- a/go/core/internal/translator/adkconfig/mcp.go +++ b/go/core/internal/translator/adkconfig/mcp.go @@ -5,36 +5,8 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" - "github.com/kagent-dev/kagent/go/core/internal/translator" ) -// ApplyCredentialHeaders adds gateway placeholders to the root and shared MCP clients. -func ApplyCredentialHeaders(config *configv1.ADKAgentConfig, additions []egress.Credential) error { - for i := range config.HttpTools { - params := config.HttpTools[i].Params - var err error - params.Headers, err = translator.CredentialHeaders(params.Url, params.Headers, additions) - if err != nil { - return err - } - } - for i := range config.SseTools { - params := config.SseTools[i].Params - var err error - params.Headers, err = translator.CredentialHeaders(params.Url, params.Headers, additions) - if err != nil { - return err - } - } - for _, child := range config.SubAgents { - if err := ApplyCredentialHeaders(child, additions); err != nil { - return err - } - } - return nil -} - // addRemoteMCPServer translates the two remote protocols supported by the ADK. // This path intentionally has no proxy URL or egress-gateway indirection. func (c *Builder) addRemoteMCPServer(config *configv1.ADKAgentConfig, server *v1alpha3.RemoteMCPServer, tools []string, requireApproval bool, headers map[string]string) error { diff --git a/go/core/internal/translator/adkconfig/mcp_test.go b/go/core/internal/translator/adkconfig/mcp_test.go index d9352df838..05ea91bcb3 100644 --- a/go/core/internal/translator/adkconfig/mcp_test.go +++ b/go/core/internal/translator/adkconfig/mcp_test.go @@ -4,11 +4,13 @@ import ( "testing" "github.com/stretchr/testify/require" + "istio.io/istio/pkg/kube/krt" + "k8s.io/apimachinery/pkg/types" configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) func TestAddRemoteMCPServerPreservesBindingApproval(t *testing.T) { @@ -55,15 +57,31 @@ func TestAddRemoteMCPServerPreservesBindingApproval(t *testing.T) { } } -func TestApplyCredentialHeadersToSharedAndSSEClients(t *testing.T) { - config := &configv1.ADKAgentConfig{ - HttpTools: []*configv1.ADKHTTPMCPServerConfig{{Params: &configv1.ADKHTTPConnectionParams{Url: "https://tools.example.com/http", Headers: map[string]string{"X-Trace": "keep"}}}}, - SseTools: []*configv1.ADKSSEMCPServerConfig{{Params: &configv1.ADKSSEConnectionParams{Url: "https://tools.example.com/sse"}}}, - SubAgents: []*configv1.ADKAgentConfig{{HttpTools: []*configv1.ADKHTTPMCPServerConfig{{Params: &configv1.ADKHTTPConnectionParams{Url: "https://tools.example.com/child"}}}}}, +// The extension runs before Build, including shared-agent and SSE clients. +type mcpPolicyTranslator struct{} + +var _ egressext.PolicyTranslator = mcpPolicyTranslator{} + +func (mcpPolicyTranslator) Translate(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://tools.example.com:443", Header: "authorization", URI: "ate-secret://provider.example/token"}) + return draft, nil +} + +func TestBuildResolvedCredentialHeaders(t *testing.T) { + server := func(path string, protocol v1alpha3.RemoteMCPServerProtocol) translator.ResolvedMCPTool { + return translator.ResolvedMCPTool{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://tools.example.com/" + path, Protocol: protocol}}} + } + input := &translator.HarnessInput{ + Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{BYO: &v1alpha3.BYOHarness{}}}, + Root: &translator.AgentInput{Template: &translator.TemplateConfiguration{}, MCPTools: []translator.ResolvedMCPTool{server("http", v1alpha3.RemoteMCPServerProtocolStreamableHttp)}, Shared: []translator.AgentInputBinding{ + {Name: "child", Agent: &translator.AgentInput{Template: &translator.TemplateConfiguration{}, MCPTools: []translator.ResolvedMCPTool{server("sse", v1alpha3.RemoteMCPServerProtocolSse)}}}, + }}, } - require.NoError(t, ApplyCredentialHeaders(config, []egress.Credential{{Origin: "https://tools.example.com:443", Header: "authorization", URI: "ate-secret://provider.example/token"}})) - require.Equal(t, "keep", config.HttpTools[0].Params.Headers["X-Trace"]) - require.Equal(t, translator.CredentialPlaceholder, config.HttpTools[0].Params.Headers["authorization"]) - require.Equal(t, translator.CredentialPlaceholder, config.SseTools[0].Params.Headers["authorization"]) - require.Equal(t, translator.CredentialPlaceholder, config.SubAgents[0].HttpTools[0].Params.Headers["authorization"]) + var err error + input.Policy, err = translator.ResolvePolicy(input, nil, mcpPolicyTranslator{}) + require.NoError(t, err) + result, err := NewBuilder(krt.TestingDummyContext{}, translator.Collections{}).Build(t.Context(), input) + require.NoError(t, err) + require.Equal(t, translator.CredentialPlaceholder, result.Config.HttpTools[0].Params.Headers["authorization"]) + require.Equal(t, translator.CredentialPlaceholder, result.Config.SubAgents[0].SseTools[0].Params.Headers["authorization"]) } diff --git a/go/core/internal/translator/adkconfig/model.go b/go/core/internal/translator/adkconfig/model.go index 1ede7b2b68..137ba54f2e 100644 --- a/go/core/internal/translator/adkconfig/model.go +++ b/go/core/internal/translator/adkconfig/model.go @@ -326,7 +326,7 @@ func translateModel(resolved *translator.ResolvedModelConfig) (*configv1.ADKMode if model.Spec.Ollama.Host != "" { modelDeploymentData.EnvVars = append(modelDeploymentData.EnvVars, corev1.EnvVar{ Name: env.OllamaAPIBase.Name(), - Value: withDefaultScheme(model.Spec.Ollama.Host), + Value: translator.OllamaURL(model.Spec.Ollama.Host), }) } // Bind the Secret only when the model reaches api.ollama.com; the diff --git a/go/core/internal/translator/adkconfig/model_test.go b/go/core/internal/translator/adkconfig/model_test.go index 41049692c7..29e2c6541e 100644 --- a/go/core/internal/translator/adkconfig/model_test.go +++ b/go/core/internal/translator/adkconfig/model_test.go @@ -139,7 +139,7 @@ func TestTranslateOllamaEnvironment(t *testing.T) { // The chart ships a default host, so this is the ordinary case for // an operator who follows the values.yaml comment introducing // apiKeySecretRef. The request goes to that host, so no credential - // binding exists, and mounting the key made CompileCredentials + // binding exists, and mounting the key made credential rendering // reject the whole revision ("cannot use gateway header injection") // instead of simply using the daemon. name: "an explicit host does not mount the key even for a cloud model", @@ -232,7 +232,7 @@ func TestWithDefaultScheme(t *testing.T) { } for _, tt := range tests { t.Run(tt.host, func(t *testing.T) { - require.Equal(t, tt.want, withDefaultScheme(tt.host)) + require.Equal(t, tt.want, translator.OllamaURL(tt.host)) }) } } diff --git a/go/core/internal/translator/adkconfig/runtime.go b/go/core/internal/translator/adkconfig/runtime.go index 82a9f133e4..975b1b4e83 100644 --- a/go/core/internal/translator/adkconfig/runtime.go +++ b/go/core/internal/translator/adkconfig/runtime.go @@ -4,9 +4,7 @@ import ( "encoding/json" "fmt" - "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" runtimeconfig "github.com/kagent-dev/kagent/go/adk/pkg/config" adkoutputschema "github.com/kagent-dev/kagent/go/adk/pkg/outputschema" @@ -19,20 +17,12 @@ type modelResult struct { Resolved *translator.ResolvedModelConfig Model *configv1.ADKModelConfig Environment []corev1.EnvVar - Egress []string } // buildModel translates a standalone ModelConfig without building an agent. -func (c *Builder) buildModel(namespace, name string) (*modelResult, error) { - resolved := krt.FetchOne(c.ctx, c.collections.ResolvedModelConfigs, krt.FilterObjectName(types.NamespacedName{Namespace: namespace, Name: name})) +func buildModel(resolved *translator.ResolvedModelConfig) (*modelResult, error) { if resolved == nil { - return nil, fmt.Errorf("model config %q not found", name) - } - if failures := resolved.SemanticFailures; len(failures) > 0 { - return nil, translator.NewValidationError("ModelConfig %q: %s", name, failures[0].Message) - } - if failures := resolved.ReferenceFailures; len(failures) > 0 { - return nil, fmt.Errorf("ModelConfig %q: %s", name, failures[0].Message) + return nil, fmt.Errorf("auxiliary model must be resolved before rendering") } runtime, err := resolveModel(resolved) if err != nil { @@ -43,7 +33,6 @@ func (c *Builder) buildModel(namespace, name string) (*modelResult, error) { } return &modelResult{ Resolved: resolved, Model: runtime.Model, Environment: runtime.Environment, - Egress: agentConfigDestinations(&configv1.ADKAgentConfig{}, resolved.Config, runtime.Model), }, nil } @@ -57,7 +46,7 @@ func (c *Builder) buildModel(namespace, name string) (*modelResult, error) { // and egress join the revision, and it joins the provenance so a change to it // compiles a new revision. The agent's own model is left out because the // runtime already summarizes with it by default. -func (c *Builder) applyCompaction(result *Result, harness *translator.HarnessConfiguration, template *translator.TemplateConfiguration) error { +func applyCompaction(result *Result, harness *translator.HarnessConfiguration, template *translator.TemplateConfiguration, resolved *translator.ResolvedModelConfig) error { spec := harness.Spec.Kagent.Compaction if spec == nil { return nil @@ -71,14 +60,13 @@ func (c *Builder) applyCompaction(result *Result, harness *translator.HarnessCon if summarizer := spec.Summarizer; summarizer != nil { compaction.PromptTemplate = summarizer.PromptTemplate if ref := summarizer.ModelConfigRef; ref != nil && !isAgentModel(template, ref.Name) { - model, err := c.buildModel(harness.Namespace, ref.Name) + model, err := buildModel(resolved) if err != nil { return fmt.Errorf("resolve summarizer ModelConfig %q: %w", ref.Name, err) } compaction.SummarizerModel = model.Model result.Models = append(result.Models, model.Resolved) result.Environment = append(result.Environment, model.Environment...) - result.Egress = append(result.Egress, model.Egress...) } } result.Config.ContextConfig = &configv1.ADKContextConfig{Compaction: compaction} @@ -119,19 +107,18 @@ func requireModels(input *translator.AgentInput) error { return nil } -func (c *Builder) applyMemory(result *Result, harness *translator.HarnessConfiguration) error { +func applyMemory(result *Result, harness *translator.HarnessConfiguration, resolved *translator.ResolvedModelConfig) error { memory := harness.Spec.Kagent.Memory if memory == nil { return nil } name := memory.ModelConfigRef.Name - model, err := c.buildModel(harness.Namespace, name) + model, err := buildModel(resolved) if err != nil { return fmt.Errorf("resolve memory ModelConfig %q: %w", name, err) } result.Config.Memory = &configv1.ADKMemoryConfig{TtlDays: int64(memory.TTLDays), Embedding: runtimeconfig.ModelToEmbeddingConfig(model.Model)} result.Models = append(result.Models, model.Resolved) result.Environment = append(result.Environment, model.Environment...) - result.Egress = append(result.Egress, model.Egress...) return nil } diff --git a/go/core/internal/translator/byo/compiler.go b/go/core/internal/translator/byo/compiler.go index 4db2840a47..46e73dad89 100644 --- a/go/core/internal/translator/byo/compiler.go +++ b/go/core/internal/translator/byo/compiler.go @@ -17,21 +17,19 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/core/internal/translator" "github.com/kagent-dev/kagent/go/core/internal/translator/adkconfig" - "github.com/kagent-dev/kagent/go/core/pkg/egressext" "github.com/kagent-dev/kagent/go/pkg/tracing" ) // Compiler translates resolved inputs into a BYO A2A runtime revision. type Compiler struct { - binder egressext.CredentialBinder config *adkconfig.Builder settings *configv1.RuntimeDefaultsConfig } var _ translator.HarnessCompiler = (*Compiler)(nil) -func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig, binder egressext.CredentialBinder) *Compiler { - return &Compiler{settings: settings, binder: binder, config: adkconfig.NewBuilder(ctx, collections)} +func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig) *Compiler { + return &Compiler{settings: settings, config: adkconfig.NewBuilder(ctx, collections)} } func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) (*translator.CompileResult, error) { @@ -61,13 +59,10 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build revision provenance: %w", err) } - environment, credentials, additions, tokenPassthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment, c.binder) + environment, err = input.Policy.RenderEnvironment(environment) if err != nil { return nil, err } - if err := adkconfig.ApplyCredentialHeaders(compiled.Config, additions); err != nil { - return nil, err - } configJSON, err := runtimeconfig.Marshal(&configv1.ADKConfig{ Runtime: translator.RuntimeConfig(c.settings, input.AgentName, template.Namespace), @@ -90,7 +85,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) Environment: environment, ConfigJSON: configJSON, AgentCard: card, SkillImages: compiled.SkillImages, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - TokenPassthroughOrigins: tokenPassthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: slices.Compact(compiled.Egress), + Provenance: provenance, EgressDestinations: slices.Compact(compiled.Egress), }}, nil } @@ -102,3 +97,8 @@ func agentTemplateCard(agentName string, template *translator.TemplateConfigurat DefaultInputModes: []string{"text"}, DefaultOutputModes: []string{"text"}, } } + +// ModelRequirements declares authentication and grants before client rendering. +func (*Compiler) ModelRequirements(model *translator.ResolvedModelConfig) translator.ModelRequirements { + return translator.ModelRequirements{Destinations: adkconfig.AdditionalModelDestinations(model)} +} diff --git a/go/core/internal/translator/byo/compiler_test.go b/go/core/internal/translator/byo/compiler_test.go index 762dfba0ab..f9dd5b1dd3 100644 --- a/go/core/internal/translator/byo/compiler_test.go +++ b/go/core/internal/translator/byo/compiler_test.go @@ -30,7 +30,7 @@ func TestCompileOpaqueImage(t *testing.T) { Description: "custom A2A agent", SystemPrompt: "be helpful", }} - revision, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, testRuntimeDefaults(), nil).Compile(context.Background(), &translator.HarnessInput{ + revision, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, testRuntimeDefaults()).Compile(context.Background(), &translator.HarnessInput{ AgentName: "runnable-agent", Harness: harness, Root: &translator.AgentInput{Template: template, Instruction: template.Spec.SystemPrompt}, }) @@ -66,7 +66,7 @@ func TestCompileOpaqueImageKeepsItsOwnTelemetry(t *testing.T) { }} template := &translator.TemplateConfiguration{Name: "custom-agent", Namespace: "test", Source: &metav1.ObjectMeta{Name: "custom-agent", Namespace: "test"}} - revision, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, settings, nil).Compile(context.Background(), &translator.HarnessInput{ + revision, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, settings).Compile(context.Background(), &translator.HarnessInput{ AgentName: "runnable-agent", Harness: harness, Root: &translator.AgentInput{Template: template}, }) @@ -103,7 +103,7 @@ func TestCompiledTelemetryFitsTheActorEnvironmentBudget(t *testing.T) { harness.Spec.Env = append(harness.Spec.Env, v1alpha3.RuntimeEnvVar{Name: fmt.Sprintf("CUSTOM_%d", i), Value: "value"}) } template := &translator.TemplateConfiguration{Name: "custom-agent", Namespace: "test", Source: &metav1.ObjectMeta{Name: "custom-agent", Namespace: "test"}} - revision, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, settings, nil).Compile(t.Context(), &translator.HarnessInput{ + revision, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, settings).Compile(t.Context(), &translator.HarnessInput{ AgentName: "runnable-agent", Harness: harness, Root: &translator.AgentInput{Template: template}, }) require.NoError(t, err) diff --git a/go/core/internal/translator/claude/compiler.go b/go/core/internal/translator/claude/compiler.go index 90450831f7..eb60aa5312 100644 --- a/go/core/internal/translator/claude/compiler.go +++ b/go/core/internal/translator/claude/compiler.go @@ -19,22 +19,19 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" - "github.com/kagent-dev/kagent/go/core/pkg/egressext" claudeconfig "github.com/kagent-dev/kagent/go/harness/claude/config" "github.com/kagent-dev/kagent/go/pkg/tracing" ) type Compiler struct { - binder egressext.CredentialBinder settings *configv1.RuntimeDefaultsConfig ctx krt.HandlerContext collections translator.Collections } -func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig, binder egressext.CredentialBinder) *Compiler { - return &Compiler{settings: settings, binder: binder, ctx: ctx, collections: collections} +func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig) *Compiler { + return &Compiler{settings: settings, ctx: ctx, collections: collections} } func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) (*translator.CompileResult, error) { @@ -58,7 +55,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) runtimeTelemetry := telemetryConfig.RuntimeTelemetry( tracing.RuntimeClaude, input.AgentName, template.Namespace, model.Spec) - providerEnvironment, egress, err := c.provider(ctx, model) + providerEnvironment, err := c.provider(ctx, model, input.Policy.ModelUsesGateway(input.Root.ResolvedModelConfig)) if err != nil { return nil, err } @@ -66,7 +63,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, err } - mcp, err := c.compileMCP(ctx, input.Root.Template.Namespace, input.Root.MCPTools) + mcp, err := c.compileMCP(ctx, input.Root.Template.Namespace, input.Root.MCPTools, input.Policy) if err != nil { return nil, err } @@ -106,17 +103,10 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build Claude revision provenance: %w", err) } - environment, credentials, additions, tokenPassthroughOrigins, err := translator.CompileCredentials(input, nil, environment, c.binder) + environment, err = input.Policy.RenderEnvironment(environment) if err != nil { return nil, err } - for name, server := range config.MCPServers { - server.Headers, err = translator.CredentialHeaders(server.URL, server.Headers, additions) - if err != nil { - return nil, err - } - config.MCPServers[name] = server - } if err := config.Validate(); err != nil { return nil, translator.NewValidationError("invalid compiled Claude configuration: %v", err) } @@ -126,8 +116,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) return nil, fmt.Errorf("marshal Claude config: %w", err) } - egress = append(egress, skills.Egress...) - egress = append(egress, mcp.egress...) + egress := skills.Egress egress = append(egress, telemetryConfig.Destinations(environment)...) egress = append(egress, c.settings.GetControlPlaneUrl()) slices.Sort(egress) @@ -138,9 +127,9 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) SkillImages: skills.Images, Environment: environment, ConfigJSON: configJSON, AgentCard: card, - WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, - SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - TokenPassthroughOrigins: tokenPassthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: egress, + WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, + SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, + Provenance: provenance, EgressDestinations: egress, }, Warnings: mcp.warnings, }, nil @@ -186,7 +175,7 @@ func sameProviderConfiguration(root, child v1alpha3.ModelConfigSpec) bool { return reflect.DeepEqual(root, child) } -func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig) ([]corev1.EnvVar, []string, error) { +func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig, gateway bool) ([]corev1.EnvVar, error) { switch model.Spec.Provider { case v1alpha3.ModelProviderAnthropic: var baseURL string @@ -199,30 +188,28 @@ func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig) ([ options.CacheTTL = "" } if !reflect.DeepEqual(options, v1alpha3.AnthropicConfig{}) { - return nil, nil, translator.NewValidationError("Claude does not support Anthropic provider options beyond baseUrl yet") + return nil, translator.NewValidationError("Claude does not support Anthropic provider options beyond baseUrl yet") } } environment := []corev1.EnvVar{{Name: claudeconfig.AnthropicAPIKeyEnvName, Value: translator.CredentialPlaceholder}} - if model.Spec.APIKeySecret != "" || c.binder == nil { + if model.Spec.APIKeySecret != "" || !gateway { if err := c.requireSecretKey(ctx, model, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey, false); err != nil { - return nil, nil, err + return nil, err } environment[0] = secretEnvironment(claudeconfig.AnthropicAPIKeyEnvName, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey) } - egress := []string{"https://api.anthropic.com:443"} if baseURL != "" { - hostname, err := anthropicBaseURLOrigin(baseURL) + err := validateAnthropicBaseURL(baseURL) if err != nil { - return nil, nil, err + return nil, err } environment = append(environment, corev1.EnvVar{Name: claudeconfig.AnthropicBaseURLEnvName, Value: baseURL}) - egress = []string{hostname} } - return environment, egress, nil + return environment, nil case v1alpha3.ModelProviderBedrock: if model.Spec.Bedrock == nil || strings.TrimSpace(model.Spec.Bedrock.Region) == "" { - return nil, nil, translator.NewValidationError("Claude Bedrock requires bedrock.region") + return nil, translator.NewValidationError("Claude Bedrock requires bedrock.region") } options := *model.Spec.Bedrock options.Region = "" @@ -231,24 +218,24 @@ func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig) ([ options.CacheTTL = "" } if !reflect.DeepEqual(options, v1alpha3.BedrockConfig{}) { - return nil, nil, translator.NewValidationError("Claude does not support Bedrock provider options beyond region yet") + return nil, translator.NewValidationError("Claude does not support Bedrock provider options beyond region yet") } - if model.Spec.APIKeySecret == "" && c.binder != nil { + if model.Spec.APIKeySecret == "" && gateway { return []corev1.EnvVar{ {Name: claudeconfig.UseBedrockEnvName, Value: "1"}, {Name: claudeconfig.AWSRegionEnvName, Value: model.Spec.Bedrock.Region}, {Name: claudeconfig.AWSBedrockTokenEnvName, Value: translator.CredentialPlaceholder}, - }, []string{"https://bedrock-runtime." + model.Spec.Bedrock.Region + ".amazonaws.com:443"}, nil + }, nil } if model.Spec.APIKeySecret == "" { - return nil, nil, translator.NewValidationError("Claude Bedrock requires apiKeySecret with AWS credentials") + return nil, translator.NewValidationError("Claude Bedrock requires apiKeySecret with AWS credentials") } if model.Spec.APIKeySecretKey != "" { - return nil, nil, translator.NewValidationError("Claude Bedrock reads standard AWS keys from apiKeySecret; apiKeySecretKey must be empty") + return nil, translator.NewValidationError("Claude Bedrock reads standard AWS keys from apiKeySecret; apiKeySecretKey must be empty") } secret, err := c.secret(ctx, model.Namespace, model.Spec.APIKeySecret) if err != nil { - return nil, nil, err + return nil, err } environment := []corev1.EnvVar{{Name: claudeconfig.UseBedrockEnvName, Value: "1"}, {Name: claudeconfig.AWSRegionEnvName, Value: model.Spec.Bedrock.Region}} if value := secret.Data[claudeconfig.AWSBedrockTokenEnvName]; len(value) != 0 { @@ -256,7 +243,7 @@ func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig) ([ } else { for _, key := range []string{claudeconfig.AWSAccessKeyEnvName, claudeconfig.AWSSecretKeyEnvName} { if len(secret.Data[key]) == 0 { - return nil, nil, translator.NewValidationError("Claude Bedrock Secret %q requires %s and %s, or %s", secret.Name, claudeconfig.AWSAccessKeyEnvName, claudeconfig.AWSSecretKeyEnvName, claudeconfig.AWSBedrockTokenEnvName) + return nil, translator.NewValidationError("Claude Bedrock Secret %q requires %s and %s, or %s", secret.Name, claudeconfig.AWSAccessKeyEnvName, claudeconfig.AWSSecretKeyEnvName, claudeconfig.AWSBedrockTokenEnvName) } environment = append(environment, secretEnvironment(key, secret.Name, key)) } @@ -264,36 +251,36 @@ func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig) ([ environment = append(environment, secretEnvironment(claudeconfig.AWSSessionTokenEnvName, secret.Name, claudeconfig.AWSSessionTokenEnvName)) } } - return environment, []string{"https://bedrock-runtime." + model.Spec.Bedrock.Region + ".amazonaws.com:443"}, nil + return environment, nil case v1alpha3.ModelProviderAnthropicVertexAI: if model.Spec.AnthropicVertexAI == nil || strings.TrimSpace(model.Spec.AnthropicVertexAI.ProjectID) == "" || strings.TrimSpace(model.Spec.AnthropicVertexAI.Location) == "" { - return nil, nil, translator.NewValidationError("Claude Vertex requires anthropicVertexAI.projectID and location") + return nil, translator.NewValidationError("Claude Vertex requires anthropicVertexAI.projectID and location") } options := *model.Spec.AnthropicVertexAI options.ProjectID, options.Location = "", "" if !reflect.DeepEqual(options, v1alpha3.AnthropicVertexAIConfig{}) { - return nil, nil, translator.NewValidationError("Claude does not support AnthropicVertexAI provider options beyond projectID and location yet") + return nil, translator.NewValidationError("Claude does not support AnthropicVertexAI provider options beyond projectID and location yet") } if err := c.requireGoogleCredentials(ctx, model); err != nil { - return nil, nil, err + return nil, err } cfg := model.Spec.AnthropicVertexAI return []corev1.EnvVar{ {Name: claudeconfig.UseVertexEnvName, Value: "1"}, {Name: claudeconfig.VertexProjectEnvName, Value: cfg.ProjectID}, {Name: claudeconfig.VertexRegionEnvName, Value: cfg.Location}, secretEnvironment(claudeconfig.GoogleCredentialsJSONEnvName, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey), - }, []string{"https://" + vertexHostname(cfg.Location) + ":443", "https://oauth2.googleapis.com:443"}, nil + }, nil default: - return nil, nil, translator.NewValidationError("Claude does not support ModelConfig provider %q", model.Spec.Provider) + return nil, translator.NewValidationError("Claude does not support ModelConfig provider %q", model.Spec.Provider) } } -func anthropicBaseURLOrigin(raw string) (string, error) { +func validateAnthropicBaseURL(raw string) error { parsed, err := url.Parse(raw) if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { - return "", translator.NewValidationError("Claude Anthropic baseUrl must be an absolute HTTP(S) URL without credentials, query, or fragment") + return translator.NewValidationError("Claude Anthropic baseUrl must be an absolute HTTP(S) URL without credentials, query, or fragment") } - return egress.Origin(parsed), nil + return nil } func (c *Compiler) requireSecretKey(ctx context.Context, model *v1alpha3.ModelConfig, name, key string, requireJSON bool) error { @@ -359,17 +346,6 @@ func secretEnvironment(environmentName, secretName, key string) corev1.EnvVar { }}} } -func vertexHostname(location string) string { - switch location { - case "global": - return "aiplatform.googleapis.com" - case "us", "eu": - return "aiplatform." + location + ".rep.googleapis.com" - default: - return location + "-aiplatform.googleapis.com" - } -} - type provenanceEntry struct { APIVersion string `json:"apiVersion"` Kind string `json:"kind"` @@ -462,3 +438,13 @@ func objectProvenance(apiVersion, kind, name string, uid types.UID, generation i } var _ translator.HarnessCompiler = (*Compiler)(nil) + +// ModelRequirements declares authentication and grants before client rendering. +func (*Compiler) ModelRequirements(model *translator.ResolvedModelConfig) translator.ModelRequirements { + switch model.Config.Spec.Provider { + case v1alpha3.ModelProviderAnthropic, v1alpha3.ModelProviderBedrock: + return translator.ModelRequirements{CredentialRequired: true} + default: + return translator.ModelRequirements{} + } +} diff --git a/go/core/internal/translator/claude/compiler_test.go b/go/core/internal/translator/claude/compiler_test.go index 2de9217833..6a9989c026 100644 --- a/go/core/internal/translator/claude/compiler_test.go +++ b/go/core/internal/translator/claude/compiler_test.go @@ -32,7 +32,6 @@ func TestCompileProviderCredentials(t *testing.T) { model v1alpha3.ModelConfigSpec secretData map[string][]byte wantEnv map[string]string - wantEgress []string wantErr string }{ { @@ -41,7 +40,6 @@ func TestCompileProviderCredentials(t *testing.T) { APIKeySecret: "model-auth", APIKeySecretKey: "api-key"}, secretData: map[string][]byte{"api-key": []byte(credentialValue)}, wantEnv: map[string]string{claudeconfig.AnthropicAPIKeyEnvName: translator.CredentialPlaceholder}, - wantEgress: []string{"http://kagent-controller.kagent:8083", "https://api.anthropic.com:443"}, }, { name: "Anthropic gateway", @@ -52,7 +50,6 @@ func TestCompileProviderCredentials(t *testing.T) { secretData: map[string][]byte{"api-key": []byte(credentialValue)}, wantEnv: map[string]string{claudeconfig.AnthropicAPIKeyEnvName: translator.CredentialPlaceholder, claudeconfig.AnthropicBaseURLEnvName: "http://host.docker.internal:8090/anthropic"}, - wantEgress: []string{"http://host.docker.internal:8090", "http://kagent-controller.kagent:8083"}, }, { name: "Bedrock IAM", @@ -67,7 +64,6 @@ func TestCompileProviderCredentials(t *testing.T) { APIKeySecret: "model-auth", Bedrock: &v1alpha3.BedrockConfig{Region: "us-west-2"}}, secretData: map[string][]byte{claudeconfig.AWSBedrockTokenEnvName: []byte(credentialValue)}, wantEnv: map[string]string{claudeconfig.UseBedrockEnvName: "1", claudeconfig.AWSRegionEnvName: "us-west-2", claudeconfig.AWSBedrockTokenEnvName: translator.CredentialPlaceholder}, - wantEgress: []string{"http://kagent-controller.kagent:8083", "https://bedrock-runtime.us-west-2.amazonaws.com:443"}, }, { name: "Anthropic Vertex AI", @@ -82,7 +78,7 @@ func TestCompileProviderCredentials(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { input, reader := testInput(t, tt.model, tt.secretData) - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if tt.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tt.wantErr) { t.Fatalf("expected unsupported credential error, got %v", err) @@ -114,7 +110,7 @@ func TestCompileProviderCredentials(t *testing.T) { if gotEnvironment[claudeconfig.SandboxEnvName] != "1" { t.Errorf("environment[%s] = %q, want %q", claudeconfig.SandboxEnvName, gotEnvironment[claudeconfig.SandboxEnvName], "1") } - if !reflect.DeepEqual(revision.EgressDestinations, tt.wantEgress) { + if !reflect.DeepEqual(revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083"}) { t.Errorf("egress = %v", revision.EgressDestinations) } if bytes.Contains(revision.ConfigJSON, []byte(credentialValue)) || bytes.Contains(revision.Provenance, []byte(credentialValue)) { @@ -124,7 +120,7 @@ func TestCompileProviderCredentials(t *testing.T) { t.Fatalf("provenance contains gateway-managed Secret: %s", revision.Provenance) } - again, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + again, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil || !reflect.DeepEqual(revision, again) { t.Fatalf("compilation is not deterministic: %v", err) } @@ -151,7 +147,7 @@ func TestCompileRejectsUnsupportedConfiguration(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { input, reader := testInput(t, tt.model, map[string][]byte{"api-key": []byte("secret"), claudeconfig.AWSAccessKeyEnvName: []byte("access"), claudeconfig.AWSSecretKeyEnvName: []byte("secret"), "credentials.json": []byte(`{"type":"service_account"}`)}) - _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) var validation *translator.ValidationError if !errors.As(err, &validation) { t.Fatalf("Compile() error = %v, want validation error", err) @@ -168,7 +164,7 @@ func TestCompileRejectsProviderOwnedHarnessEnvironment(t *testing.T) { input, reader := testInput(t, model, map[string][]byte{"api-key": []byte("secret")}) value := "http://mock.example.com" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: claudeconfig.AnthropicBaseURLEnvName, Value: value}} - _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) var validation *translator.ValidationError if !errors.As(err, &validation) { t.Fatalf("Compile() error = %v, want validation error", err) @@ -184,7 +180,7 @@ func TestCompileAllowsRuntimeOTELEndpoint(t *testing.T) { value := "http://other-collector:4317" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: value}} - result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + result, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -202,7 +198,7 @@ func TestCompileAllowsRuntimeTelemetryOverrides(t *testing.T) { value := "department=engineering" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OTEL_RESOURCE_ATTRIBUTES", Value: value}} - result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + result, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -227,7 +223,7 @@ func TestCompileRootSkillsAndPluginSelections(t *testing.T) { Skills: []string{"deploy"}, }} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -239,13 +235,13 @@ func TestCompileRootSkillsAndPluginSelections(t *testing.T) { len(cfg.SkillResources.Plugins) != 1 || !reflect.DeepEqual(cfg.SkillResources.Plugins[0].Skills, []string{"deploy"}) { t.Fatalf("compiled skills = %#v", cfg.SkillResources) } - wantEgress := []string{"http://kagent-controller.kagent:8083", "https://api.anthropic.com:443", "https://git.example.com:443"} + wantEgress := []string{"http://kagent-controller.kagent:8083", "https://git.example.com:443"} if !reflect.DeepEqual(revision.EgressDestinations, wantEgress) { t.Fatalf("egress = %v, want %v", revision.EgressDestinations, wantEgress) } input.Root.Template.Spec.Plugins[0].Skills = []string{"review"} - if _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input); err == nil || !strings.Contains(err.Error(), "duplicate skill name") { + if _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input); err == nil || !strings.Contains(err.Error(), "duplicate skill name") { t.Fatalf("duplicate skill Compile() error = %v", err) } } @@ -279,7 +275,7 @@ func TestCompileDirectWholeServerMCP(t *testing.T) { }}} input.Root.MCPTools = []translator.ResolvedMCPTool{{Binding: *input.Root.Template.Spec.Tools[0].MCP.DeepCopy(), Server: server}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -307,7 +303,7 @@ func TestCompileDirectWholeServerMCP(t *testing.T) { if !foundSecret { t.Fatalf("MCP credential environment missing: %#v", revision.Environment) } - if !reflect.DeepEqual(revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083", "https://api.anthropic.com:443", "https://mcp.example.com:443"}) { + if !reflect.DeepEqual(revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083"}) { t.Fatalf("egress = %v", revision.EgressDestinations) } if !bytes.Contains(revision.Provenance, []byte(`"kind":"RemoteMCPServer"`)) { @@ -330,7 +326,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { } binding := v1alpha3.MCPToolBinding{Server: corev1.TypedLocalObjectReference{Kind: "RemoteMCPServer", Name: server.Name}} input.Root.MCPTools = []translator.ResolvedMCPTool{{Binding: binding, Server: server}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatalf("omitted selection Compile() error = %v", err) } @@ -339,7 +335,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { } input.Root.MCPTools[0].Binding.Tools = []string{"one"} - revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + revision, err = compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatalf("partial selection Compile() error = %v", err) } @@ -348,7 +344,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { } server.Status.ObservedGeneration = 0 - revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + revision, err = compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatalf("stale discovery Compile() error = %v", err) } @@ -362,7 +358,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { server.Spec.TLS = &v1alpha3.TLSConfig{DisableVerify: true} server.Spec.Timeout = &metav1.Duration{Duration: time.Minute} server.Spec.TerminateOnClose = &terminateOnClose - revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + revision, err = compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatalf("unsupported MCP options Compile() error = %v", err) } @@ -376,7 +372,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { } server.Spec.Protocol = v1alpha3.RemoteMCPServerProtocol("STDIO") - if _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input); err == nil || !strings.Contains(err.Error(), "unsupported protocol") { + if _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input); err == nil || !strings.Contains(err.Error(), "unsupported protocol") { t.Fatalf("unsupported MCP protocol Compile() error = %v", err) } @@ -386,7 +382,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { server.Spec.TerminateOnClose = nil input.Root.MCPTools[0].Binding.Tools = []string{"one"} input.Root.MCPTools[0].Binding.RequireApproval = true - revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + revision, err = compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatalf("approval-required MCP binding Compile() error = %v", err) } @@ -433,7 +429,7 @@ func TestCompileLocalSharedAgent(t *testing.T) { Name: "specialist", Description: "Handles specialist requests", Agent: child, }} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -466,7 +462,7 @@ func TestCompileRejectsUnsupportedLocalAgentConfiguration(t *testing.T) { }{ {name: "provider configuration", mutate: func(binding *translator.AgentInputBinding) { binding.Agent.ResolvedModelConfig.Config.Spec.APIKeySecret = "different-auth" - }, want: "root agent's provider"}, + }, want: "conflicting credentials"}, {name: "nested tools", mutate: func(binding *translator.AgentInputBinding) { binding.Agent.Template.Spec.Tools = []v1alpha3.ToolBinding{{MCP: &v1alpha3.MCPToolBinding{}}} }, want: "cannot contain MCP or nested agent tools"}, @@ -495,7 +491,7 @@ func TestCompileRejectsUnsupportedLocalAgentConfiguration(t *testing.T) { } tt.mutate(&binding) input.Root.Shared = []translator.AgentInputBinding{binding} - _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err == nil || !strings.Contains(err.Error(), tt.want) { t.Fatalf("Compile() error = %v, want containing %q", err, tt.want) } @@ -531,7 +527,7 @@ func TestCompileRuntimeTelemetry(t *testing.T) { input.Harness.Name = "fast" settings := testRuntimeDefaults() - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, settings, nil).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, settings), input) if err != nil { t.Fatal(err) } @@ -552,7 +548,7 @@ func TestCompileRuntimeTelemetry(t *testing.T) { } settings.Telemetry = &configv1.TelemetryConfig{CaptureContent: true, MaxCaptureBytes: new(int32(4096))} - captured, err := NewCompiler(krt.TestingDummyContext{}, reader, settings, nil).Compile(context.Background(), input) + captured, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, settings), input) if err != nil { t.Fatal(err) } @@ -604,7 +600,7 @@ func TestCompiledTelemetryFitsTheActorEnvironmentBudget(t *testing.T) { }}} input.Root.MCPTools = []translator.ResolvedMCPTool{{Binding: *input.Root.Template.Spec.Tools[0].MCP.DeepCopy(), Server: server}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, settings, nil).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, settings), input) if err != nil { t.Fatal(err) } @@ -622,3 +618,31 @@ func TestCompiledTelemetryFitsTheActorEnvironmentBudget(t *testing.T) { func testRuntimeDefaults() *configv1.RuntimeDefaultsConfig { return &configv1.RuntimeDefaultsConfig{ControlPlaneUrl: new("http://kagent-controller.kagent:8083"), GatewayUrl: new("http://kagent-controller.kagent:8083"), Logging: &configv1.LoggingConfig{Level: new("info")}} } + +func compileTestInput(ctx context.Context, compiler *Compiler, input *translator.HarnessInput) (*translator.CompileResult, error) { + var resolve func(*translator.AgentInput) error + resolve = func(agent *translator.AgentInput) error { + if agent.ResolvedModelConfig != nil { + model, err := translator.ResolveModelConfig(compiler.ctx, compiler.collections, agent.ResolvedModelConfig.Config) + if err != nil { + return err + } + agent.ResolvedModelConfig = model + } + for _, child := range agent.Shared { + if err := resolve(child.Agent); err != nil { + return err + } + } + return nil + } + if err := resolve(input.Root); err != nil { + return nil, err + } + policy, err := translator.ResolvePolicy(input, compiler, nil) + if err != nil { + return nil, err + } + input.Policy = policy + return compiler.Compile(ctx, input) +} diff --git a/go/core/internal/translator/claude/mcp.go b/go/core/internal/translator/claude/mcp.go index 15d2f2ad16..e6f85e6fbc 100644 --- a/go/core/internal/translator/claude/mcp.go +++ b/go/core/internal/translator/claude/mcp.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "fmt" - "net/url" "slices" "strings" "time" @@ -14,7 +13,6 @@ import ( "k8s.io/apimachinery/pkg/types" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" claudeconfig "github.com/kagent-dev/kagent/go/harness/claude/config" ) @@ -22,7 +20,6 @@ import ( type mcpCompilation struct { servers map[string]claudeconfig.MCPServer environment []corev1.EnvVar - egress []string warnings []string } @@ -30,6 +27,7 @@ func (c *Compiler) compileMCP( ctx context.Context, namespace string, tools []translator.ResolvedMCPTool, + policy translator.ResolvedPolicy, ) (mcpCompilation, error) { if len(tools) == 0 { return mcpCompilation{}, nil @@ -60,20 +58,19 @@ func (c *Compiler) compileMCP( if compatibilityWarning != "" { result.warnings = append(result.warnings, compatibilityWarning) } - hostname, err := mcpOrigin(server.Spec.URL) - if err != nil { - return mcpCompilation{}, err - } headers, headerEnvironment, err := c.compileMCPHeaders(ctx, namespace, server.Spec.HeadersFrom) if err != nil { return mcpCompilation{}, fmt.Errorf("compile RemoteMCPServer %q headers: %w", server.Name, err) } + headers, err = policy.MCPHeaders(server.Spec.URL, headers) + if err != nil { + return mcpCompilation{}, err + } result.servers[name] = claudeconfig.MCPServer{ Type: transport, URL: server.Spec.URL, Headers: headers, RequireApproval: tool.Binding.RequireApproval, } result.environment = append(result.environment, headerEnvironment...) - result.egress = append(result.egress, hostname) } return result, nil } @@ -144,14 +141,6 @@ func claudeMCPTransport(server *v1alpha3.RemoteMCPServer) (string, string, error } } -func mcpOrigin(raw string) (string, error) { - parsed, err := url.Parse(raw) - if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" || parsed.User != nil || parsed.Fragment != "" { - return "", translator.NewValidationError("Claude RemoteMCPServer URL must be absolute HTTP(S) without credentials or fragment") - } - return egress.Origin(parsed), nil -} - func (c *Compiler) compileMCPHeaders(ctx context.Context, namespace string, refs []v1alpha3.ValueRef) (map[string]string, []corev1.EnvVar, error) { if len(refs) == 0 { return nil, nil, nil diff --git a/go/core/internal/translator/codex/compiler.go b/go/core/internal/translator/codex/compiler.go index 5a74f59b67..183c3ed364 100644 --- a/go/core/internal/translator/codex/compiler.go +++ b/go/core/internal/translator/codex/compiler.go @@ -6,7 +6,6 @@ import ( "crypto/sha256" "encoding/json" "fmt" - "net/url" "reflect" "slices" "strings" @@ -18,9 +17,7 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" - "github.com/kagent-dev/kagent/go/core/pkg/egressext" codexconfig "github.com/kagent-dev/kagent/go/harness/codex/config" "github.com/kagent-dev/kagent/go/pkg/tracing" ) @@ -42,14 +39,13 @@ var ownedEnvironment = map[string]struct{}{ } type Compiler struct { - binder egressext.CredentialBinder settings *configv1.RuntimeDefaultsConfig ctx krt.HandlerContext collections translator.Collections } -func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig, binder egressext.CredentialBinder) *Compiler { - return &Compiler{settings: settings, binder: binder, ctx: ctx, collections: collections} +func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig) *Compiler { + return &Compiler{settings: settings, ctx: ctx, collections: collections} } func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) (*translator.CompileResult, error) { @@ -74,7 +70,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) runtimeTelemetry := telemetryConfig.RuntimeTelemetry( tracing.RuntimeCodex, input.AgentName, template.Namespace, model.Spec) - provider, providerEnvironment, egress, err := c.compileProvider(ctx, model) + provider, providerEnvironment, err := c.compileProvider(ctx, model, input.Policy.ModelUsesGateway(input.Root.ResolvedModelConfig)) if err != nil { return nil, err } @@ -82,7 +78,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, err } - mcp, err := c.compileMCP(ctx, input.Root.Template.Namespace, input.Root.MCPTools) + mcp, err := c.compileMCP(ctx, input.Root.Template.Namespace, input.Root.MCPTools, input.Policy) if err != nil { return nil, err } @@ -126,17 +122,10 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build Codex revision provenance: %w", err) } - environment, credentials, additions, tokenPassthroughOrigins, err := translator.CompileCredentials(input, nil, environment, c.binder) + environment, err = input.Policy.RenderEnvironment(environment) if err != nil { return nil, err } - for name, server := range cfg.MCPServers { - server.Headers, err = translator.CredentialHeaders(server.URL, server.Headers, additions) - if err != nil { - return nil, err - } - cfg.MCPServers[name] = server - } if err := cfg.Validate(); err != nil { return nil, translator.NewValidationError("invalid compiled Codex configuration: %v", err) } @@ -145,8 +134,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("marshal Codex config: %w", err) } - egress = append(egress, skills.Egress...) - egress = append(egress, mcp.egress...) + egress := skills.Egress egress = append(egress, telemetryConfig.Destinations(environment)...) egress = append(egress, c.settings.GetControlPlaneUrl()) slices.Sort(egress) @@ -157,69 +145,61 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) SkillImages: skills.Images, Environment: environment, ConfigJSON: configJSON, AgentCard: card, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - TokenPassthroughOrigins: tokenPassthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: egress, + Provenance: provenance, EgressDestinations: egress, }, Warnings: mcp.warnings, }, nil } -func (c *Compiler) compileProvider(ctx context.Context, model *v1alpha3.ModelConfig) (codexconfig.Provider, []corev1.EnvVar, []string, error) { +func (c *Compiler) compileProvider(ctx context.Context, model *v1alpha3.ModelConfig, gateway bool) (codexconfig.Provider, []corev1.EnvVar, error) { switch model.Spec.Provider { case v1alpha3.ModelProviderOpenAI: if model.Spec.OpenAI == nil || model.Spec.OpenAI.APIFormat == nil || *model.Spec.OpenAI.APIFormat != v1alpha3.OpenAIAPIFormatResponses { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex OpenAI requires openAI.apiFormat responses") + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex OpenAI requires openAI.apiFormat responses") } options := *model.Spec.OpenAI baseURL := strings.TrimSpace(options.BaseURL) options.BaseURL, options.APIFormat = "", nil if !reflect.DeepEqual(options, v1alpha3.OpenAIConfig{}) { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex does not support OpenAI provider options beyond baseUrl and apiFormat responses") + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex does not support OpenAI provider options beyond baseUrl and apiFormat responses") } environment := []corev1.EnvVar{{Name: openAIAPIKeyEnv, Value: translator.CredentialPlaceholder}} - if model.Spec.APIKeySecret != "" || c.binder == nil { + if model.Spec.APIKeySecret != "" || !gateway { if err := c.requireSecretKey(ctx, model.Namespace, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey); err != nil { - return codexconfig.Provider{}, nil, nil, err + return codexconfig.Provider{}, nil, err } environment[0] = secretEnvironment(openAIAPIKeyEnv, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey) } provider := codexconfig.Provider{Name: "openai", BaseURL: baseURL} - egress := []string{"https://api.openai.com:443"} - if baseURL != "" { - host, err := absoluteHTTPOrigin(baseURL) - if err != nil { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex OpenAI baseUrl %v", err) - } - egress = []string{host} - } - return provider, environment, egress, nil + return provider, environment, nil case v1alpha3.ModelProviderBedrock: if model.Spec.Bedrock == nil || strings.TrimSpace(model.Spec.Bedrock.Region) == "" { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex Bedrock requires bedrock.region") + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex Bedrock requires bedrock.region") } options := *model.Spec.Bedrock region := strings.TrimSpace(options.Region) if !strings.HasPrefix(model.Spec.Model, "gpt-") { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex Bedrock supports only OpenAI gpt-* model IDs") + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex Bedrock supports only OpenAI gpt-* model IDs") } options.Region = "" if options.CacheTTL == "5m" { options.CacheTTL = "" } if !reflect.DeepEqual(options, v1alpha3.BedrockConfig{}) { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex does not support Bedrock provider options beyond region") + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex does not support Bedrock provider options beyond region") } - if model.Spec.APIKeySecret == "" && c.binder != nil { + if model.Spec.APIKeySecret == "" && gateway { return codexconfig.Provider{Name: "amazon-bedrock"}, []corev1.EnvVar{ {Name: awsRegionEnv, Value: region}, {Name: awsBedrockTokenEnv, Value: translator.CredentialPlaceholder}, - }, []string{"https://bedrock-runtime." + region + ".amazonaws.com:443"}, nil + }, nil } if model.Spec.APIKeySecret == "" || model.Spec.APIKeySecretKey != "" { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex Bedrock requires apiKeySecret and an empty apiKeySecretKey") + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex Bedrock requires apiKeySecret and an empty apiKeySecretKey") } secret, err := c.secret(ctx, model.Namespace, model.Spec.APIKeySecret) if err != nil { - return codexconfig.Provider{}, nil, nil, err + return codexconfig.Provider{}, nil, err } environment := []corev1.EnvVar{{Name: awsRegionEnv, Value: region}} if len(secret.Data[awsBedrockTokenEnv]) != 0 { @@ -227,7 +207,7 @@ func (c *Compiler) compileProvider(ctx context.Context, model *v1alpha3.ModelCon } else { for _, key := range []string{awsAccessKeyEnv, awsSecretKeyEnv} { if len(secret.Data[key]) == 0 { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex Bedrock Secret %q requires %s and %s, or %s", secret.Name, awsAccessKeyEnv, awsSecretKeyEnv, awsBedrockTokenEnv) + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex Bedrock Secret %q requires %s and %s, or %s", secret.Name, awsAccessKeyEnv, awsSecretKeyEnv, awsBedrockTokenEnv) } environment = append(environment, secretEnvironment(key, secret.Name, key)) } @@ -235,9 +215,9 @@ func (c *Compiler) compileProvider(ctx context.Context, model *v1alpha3.ModelCon environment = append(environment, secretEnvironment(awsSessionTokenEnv, secret.Name, awsSessionTokenEnv)) } } - return codexconfig.Provider{Name: "amazon-bedrock"}, environment, []string{"https://bedrock-runtime." + region + ".amazonaws.com:443"}, nil + return codexconfig.Provider{Name: "amazon-bedrock"}, environment, nil default: - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex does not support ModelConfig provider %q", model.Spec.Provider) + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex does not support ModelConfig provider %q", model.Spec.Provider) } } @@ -298,14 +278,6 @@ func secretEnvironment(environmentName, secretName, key string) corev1.EnvVar { }}} } -func absoluteHTTPOrigin(raw string) (string, error) { - parsed, err := url.Parse(raw) - if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" || parsed.User != nil || parsed.Fragment != "" { - return "", fmt.Errorf("must be an absolute HTTP(S) URL without credentials or fragment") - } - return egress.Origin(parsed), nil -} - type provenanceEntry struct { APIVersion string `json:"apiVersion"` Kind string `json:"kind"` @@ -404,3 +376,13 @@ func objectProvenance(apiVersion, kind, name string, uid types.UID, generation i } var _ translator.HarnessCompiler = (*Compiler)(nil) + +// ModelRequirements declares authentication and grants before client rendering. +func (*Compiler) ModelRequirements(model *translator.ResolvedModelConfig) translator.ModelRequirements { + switch model.Config.Spec.Provider { + case v1alpha3.ModelProviderOpenAI, v1alpha3.ModelProviderBedrock: + return translator.ModelRequirements{CredentialRequired: true} + default: + return translator.ModelRequirements{} + } +} diff --git a/go/core/internal/translator/codex/compiler_test.go b/go/core/internal/translator/codex/compiler_test.go index dc99c6d2fa..a9a3dcb359 100644 --- a/go/core/internal/translator/codex/compiler_test.go +++ b/go/core/internal/translator/codex/compiler_test.go @@ -34,20 +34,19 @@ func TestCompileProviderCredentials(t *testing.T) { secret map[string][]byte provider string environment map[string]string - egress []string wantErr string }{ { name: "OpenAI", model: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, Model: "gpt-5.2-codex", APIKeySecret: "model-auth", APIKeySecretKey: "api-key", OpenAI: &v1alpha3.OpenAIConfig{APIFormat: &responses}}, - secret: map[string][]byte{"api-key": []byte(credentialValue)}, provider: "openai", environment: map[string]string{openAIAPIKeyEnv: translator.CredentialPlaceholder}, egress: []string{"http://kagent-controller.kagent:8083", "https://api.openai.com:443"}, + secret: map[string][]byte{"api-key": []byte(credentialValue)}, provider: "openai", environment: map[string]string{openAIAPIKeyEnv: translator.CredentialPlaceholder}, }, { name: "OpenAI gateway", model: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, Model: "gpt", APIKeySecret: "model-auth", APIKeySecretKey: "api-key", OpenAI: &v1alpha3.OpenAIConfig{APIFormat: &responses, BaseURL: "https://gateway.example.com/v1"}}, - secret: map[string][]byte{"api-key": []byte(credentialValue)}, provider: "openai", environment: map[string]string{openAIAPIKeyEnv: translator.CredentialPlaceholder}, egress: []string{"http://kagent-controller.kagent:8083", "https://gateway.example.com:443"}, + secret: map[string][]byte{"api-key": []byte(credentialValue)}, provider: "openai", environment: map[string]string{openAIAPIKeyEnv: translator.CredentialPlaceholder}, }, { name: "Bedrock API key", model: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderBedrock, Model: "gpt-5.2", APIKeySecret: "model-auth", Bedrock: &v1alpha3.BedrockConfig{Region: "us-east-1", CacheTTL: "5m"}}, - secret: map[string][]byte{awsBedrockTokenEnv: []byte(credentialValue)}, provider: "amazon-bedrock", environment: map[string]string{awsRegionEnv: "us-east-1", awsBedrockTokenEnv: translator.CredentialPlaceholder}, egress: []string{"http://kagent-controller.kagent:8083", "https://bedrock-runtime.us-east-1.amazonaws.com:443"}, + secret: map[string][]byte{awsBedrockTokenEnv: []byte(credentialValue)}, provider: "amazon-bedrock", environment: map[string]string{awsRegionEnv: "us-east-1", awsBedrockTokenEnv: translator.CredentialPlaceholder}, }, { name: "Bedrock IAM", model: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderBedrock, Model: "gpt-5.2", APIKeySecret: "model-auth", Bedrock: &v1alpha3.BedrockConfig{Region: "us-west-2"}}, @@ -57,7 +56,7 @@ func TestCompileProviderCredentials(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { input, reader := testInput(t, test.model, test.secret) - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if test.wantErr != "" { if err == nil || !strings.Contains(err.Error(), test.wantErr) { t.Fatalf("expected unsupported credential error, got %v", err) @@ -67,6 +66,9 @@ func TestCompileProviderCredentials(t *testing.T) { if err != nil { t.Fatal(err) } + if len(revision.Credentials) != 0 { + t.Fatal("renderer must leave final credentials to the shared compiler") + } var cfg codexconfig.Config if err := json.Unmarshal(revision.ConfigJSON, &cfg); err != nil { t.Fatal(err) @@ -86,13 +88,13 @@ func TestCompileProviderCredentials(t *testing.T) { t.Errorf("environment[%s] = %q, want %q", name, gotEnvironment[name], value) } } - if !reflect.DeepEqual(revision.EgressDestinations, test.egress) { - t.Errorf("egress = %v, want %v", revision.EgressDestinations, test.egress) + if !reflect.DeepEqual(revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083"}) { + t.Errorf("egress = %v, want %v", revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083"}) } if bytes.Contains(revision.ConfigJSON, []byte(credentialValue)) || bytes.Contains(revision.Provenance, []byte(credentialValue)) { t.Fatal("credential leaked into immutable revision") } - again, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + again, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil || !reflect.DeepEqual(revision, again) { t.Fatalf("compilation is not deterministic: %v", err) } @@ -111,7 +113,7 @@ func TestCompileAllowsRuntimeOTELEndpoint(t *testing.T) { value := "http://other-collector:4317" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: value}} - result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + result, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -131,7 +133,7 @@ func TestCompileAllowsRuntimeTelemetryOverrides(t *testing.T) { value := "x-tenant=team-a" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OTEL_EXPORTER_OTLP_HEADERS", Value: value}} - result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + result, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -150,7 +152,7 @@ func TestCompileRejectsUnsupportedProviderConfiguration(t *testing.T) { } for _, model := range tests { input, reader := testInput(t, model, map[string][]byte{"api-key": []byte("secret"), awsAccessKeyEnv: []byte("access"), awsSecretKeyEnv: []byte("secret")}) - _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) var validation *translator.ValidationError if !errors.As(err, &validation) { t.Errorf("Compile(%s) error = %v, want validation", model.Provider, err) @@ -175,7 +177,7 @@ func TestCompileMCPAndSharedAgent(t *testing.T) { }, Instruction: "Review carefully", }}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -192,7 +194,7 @@ func TestCompileMCPAndSharedAgent(t *testing.T) { if !strings.HasPrefix(cfg.MCPServers["tools"].Headers["Authorization"], "${"+mcpCredentialPrefix) { t.Fatalf("MCP headers = %#v", cfg.MCPServers["tools"].Headers) } - if !reflect.DeepEqual(revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083", "https://api.openai.com:443", "https://mcp.example.com:443"}) { + if !reflect.DeepEqual(revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083"}) { t.Fatalf("egress = %v", revision.EgressDestinations) } } @@ -218,7 +220,7 @@ func TestCompileMCPCompatibilityWarnings(t *testing.T) { } input.Root.MCPTools = []translator.ResolvedMCPTool{{Binding: v1alpha3.MCPToolBinding{RequireApproval: true}, Server: server}} - compilation, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input) + compilation, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatalf("Compile() error = %v", err) } @@ -239,7 +241,7 @@ func TestCompileMCPCompatibilityWarnings(t *testing.T) { } server.Spec.Protocol = v1alpha3.RemoteMCPServerProtocolSse - if _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults(), nil).Compile(context.Background(), input); err == nil || !strings.Contains(err.Error(), "requires Streamable HTTP") { + if _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input); err == nil || !strings.Contains(err.Error(), "requires Streamable HTTP") { t.Fatalf("unsupported MCP protocol Compile() error = %v", err) } } @@ -275,7 +277,7 @@ func TestCompileRuntimeTelemetry(t *testing.T) { settings := testRuntimeDefaults() settings.Telemetry = &configv1.TelemetryConfig{Traces: &configv1.OTLPConfig{Endpoint: "http://collector:4317", Protocol: new("grpc")}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, settings, nil).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, settings), input) if err != nil { t.Fatal(err) } @@ -297,7 +299,7 @@ func TestCompileRuntimeTelemetry(t *testing.T) { } settings.Telemetry.CaptureContent = true - captured, err := NewCompiler(krt.TestingDummyContext{}, reader, settings, nil).Compile(context.Background(), input) + captured, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, settings), input) if err != nil { t.Fatal(err) } @@ -334,3 +336,31 @@ func TestCompileRuntimeTelemetry(t *testing.T) { func testRuntimeDefaults() *configv1.RuntimeDefaultsConfig { return &configv1.RuntimeDefaultsConfig{ControlPlaneUrl: new("http://kagent-controller.kagent:8083"), GatewayUrl: new("http://kagent-controller.kagent:8083"), Logging: &configv1.LoggingConfig{Level: new("info")}} } + +func compileTestInput(ctx context.Context, compiler *Compiler, input *translator.HarnessInput) (*translator.CompileResult, error) { + var resolve func(*translator.AgentInput) error + resolve = func(agent *translator.AgentInput) error { + if agent.ResolvedModelConfig != nil { + model, err := translator.ResolveModelConfig(compiler.ctx, compiler.collections, agent.ResolvedModelConfig.Config) + if err != nil { + return err + } + agent.ResolvedModelConfig = model + } + for _, child := range agent.Shared { + if err := resolve(child.Agent); err != nil { + return err + } + } + return nil + } + if err := resolve(input.Root); err != nil { + return nil, err + } + policy, err := translator.ResolvePolicy(input, compiler, nil) + if err != nil { + return nil, err + } + input.Policy = policy + return compiler.Compile(ctx, input) +} diff --git a/go/core/internal/translator/codex/mcp.go b/go/core/internal/translator/codex/mcp.go index dc4726f781..0f50ec8809 100644 --- a/go/core/internal/translator/codex/mcp.go +++ b/go/core/internal/translator/codex/mcp.go @@ -20,11 +20,10 @@ import ( type mcpCompilation struct { servers map[string]codexconfig.MCPServer environment []corev1.EnvVar - egress []string warnings []string } -func (c *Compiler) compileMCP(ctx context.Context, namespace string, tools []translator.ResolvedMCPTool) (mcpCompilation, error) { +func (c *Compiler) compileMCP(ctx context.Context, namespace string, tools []translator.ResolvedMCPTool, policy translator.ResolvedPolicy) (mcpCompilation, error) { if len(tools) == 0 { return mcpCompilation{}, nil } @@ -49,14 +48,14 @@ func (c *Compiler) compileMCP(ctx context.Context, namespace string, tools []tra if warning := codexMCPCompatibilityWarning(server); warning != "" { result.warnings = append(result.warnings, warning) } - host, err := absoluteHTTPOrigin(server.Spec.URL) - if err != nil { - return mcpCompilation{}, translator.NewValidationError("Codex RemoteMCPServer %q URL %v", server.Name, err) - } headers, environment, err := c.compileMCPHeaders(ctx, namespace, server.Spec.HeadersFrom) if err != nil { return mcpCompilation{}, fmt.Errorf("compile RemoteMCPServer %q headers: %w", server.Name, err) } + headers, err = policy.MCPHeaders(server.Spec.URL, headers) + if err != nil { + return mcpCompilation{}, err + } selected := append([]string(nil), tool.Binding.Tools...) slices.Sort(selected) selected = slices.Compact(selected) @@ -65,7 +64,6 @@ func (c *Compiler) compileMCP(ctx context.Context, namespace string, tools []tra RequireApproval: tool.Binding.RequireApproval, } result.environment = append(result.environment, environment...) - result.egress = append(result.egress, host) } return result, nil } diff --git a/go/core/internal/translator/compiler.go b/go/core/internal/translator/compiler.go index cc7d8f2195..3e08a12197 100644 --- a/go/core/internal/translator/compiler.go +++ b/go/core/internal/translator/compiler.go @@ -12,11 +12,14 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) // Compiler resolves public API objects into a complete, immutable runtime // revision. It owns the translation boundary from API objects to runtime inputs. type Compiler struct { + policyTranslator egressext.PolicyTranslator ctx krt.HandlerContext collections Collections harnessCompilers map[HarnessType]HarnessCompiler @@ -38,6 +41,13 @@ const ( // revision and its user-facing diagnostics. type HarnessCompiler interface { Compile(context.Context, *HarnessInput) (*CompileResult, error) + ModelRequirements(*ResolvedModelConfig) ModelRequirements +} + +// ModelRequirements declares integration-specific grants and authentication before rendering. +type ModelRequirements struct { + Destinations []string + CredentialRequired bool } // resolvedTemplateTree is the validated AgentTemplate topology for one Harness. @@ -61,6 +71,9 @@ type resolvedTemplateBinding struct { // HarnessInput contains the Kubernetes inputs needed by a harness compiler. type HarnessInput struct { + Policy ResolvedPolicy + SummarizerModel *ResolvedModelConfig + MemoryModel *ResolvedModelConfig // AgentName is the runnable identity, independent of inline or referenced inputs. AgentName string Harness *HarnessConfiguration @@ -91,8 +104,8 @@ type AgentInputBinding struct { } // NewCompiler constructs the runtime compiler. -func NewCompiler(ctx krt.HandlerContext, collections Collections, harnessCompilers map[HarnessType]HarnessCompiler, images *configv1.HarnessImagesConfig) *Compiler { - return &Compiler{ctx: ctx, collections: collections, harnessCompilers: maps.Clone(harnessCompilers), images: images} +func NewCompiler(ctx krt.HandlerContext, collections Collections, harnessCompilers map[HarnessType]HarnessCompiler, images *configv1.HarnessImagesConfig, policyTranslator egressext.PolicyTranslator) *Compiler { + return &Compiler{ctx: ctx, collections: collections, harnessCompilers: maps.Clone(harnessCompilers), images: images, policyTranslator: policyTranslator} } // CompileAgent resolves inline or referenced behavior with its referenced @@ -157,6 +170,13 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agent *v1alpha3.Age return nil, err } input.AgentName = agent.Name + if err := c.resolveAuxiliaryModels(input); err != nil { + return nil, err + } + input.Policy, err = ResolvePolicy(input, harnessCompiler, c.policyTranslator) + if err != nil { + return nil, err + } image, err := harnessImage(harness, c.images) if err != nil { return nil, err @@ -165,6 +185,12 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agent *v1alpha3.Age if err != nil { return nil, err } + // Pin integration access. + pinned, err := egress.CanonicalAccess(append(result.EgressDestinations, input.Policy.access.Destinations...), input.Policy.access.Credentials) + if err != nil { + return nil, NewValidationError("%v", err) + } + result.EgressDestinations, result.Credentials, result.TokenPassthroughOrigins = pinned.Destinations, pinned.Credentials, input.Policy.tokenPassthroughOrigins workerKey := types.NamespacedName{Namespace: harness.Namespace, Name: harness.Spec.Substrate.WorkerPoolRef.Name} workerPool := krt.FetchOne(c.ctx, c.collections.WorkerPools, krt.FilterObjectName(workerKey)) if workerPool == nil { @@ -331,3 +357,40 @@ func (c *Compiler) buildInputs(ctx context.Context, tree *resolvedTemplateTree) } return &HarnessInput{Harness: tree.Harness, Root: root, OutputSchema: outputSchema}, nil } + +// resolveAuxiliaryModels tracks every ADK model before policy resolution and rendering. +func (c *Compiler) resolveAuxiliaryModels(input *HarnessInput) error { + if input.Harness.Spec.Kagent == nil { + return nil + } + resolve := func(name string) (*ResolvedModelConfig, error) { + model := krt.FetchOne(c.ctx, c.collections.ResolvedModelConfigs, krt.FilterObjectName(types.NamespacedName{Namespace: input.Harness.Namespace, Name: name})) + if model == nil { + return nil, fmt.Errorf("model config %q not found", name) + } + if failure := model.Failure(); failure != nil { + if len(model.SemanticFailures) != 0 { + return nil, NewValidationError("ModelConfig %q: %s", name, failure.Message) + } + return nil, fmt.Errorf("ModelConfig %q: %s", name, failure.Message) + } + return model, nil + } + var err error + if compaction := input.Harness.Spec.Kagent.Compaction; compaction != nil && compaction.Summarizer != nil && compaction.Summarizer.ModelConfigRef != nil { + name := compaction.Summarizer.ModelConfigRef.Name + if input.Root.Template.Spec.ModelConfig == nil || input.Root.Template.Spec.ModelConfig.Name != name { + input.SummarizerModel, err = resolve(name) + if err != nil { + return err + } + } + } + if memory := input.Harness.Spec.Kagent.Memory; memory != nil { + input.MemoryModel, err = resolve(memory.ModelConfigRef.Name) + if err != nil { + return err + } + } + return nil +} diff --git a/go/core/internal/translator/compiler_test.go b/go/core/internal/translator/compiler_test.go index 26b49fbcee..cbb3fac792 100644 --- a/go/core/internal/translator/compiler_test.go +++ b/go/core/internal/translator/compiler_test.go @@ -212,11 +212,11 @@ func compilerWithImages(t *testing.T, images *configv1.HarnessImagesConfig, obje collections := mockCollections(t, append(objects, defaultWorkerPool())...) ctx := krt.TestingDummyContext{} return translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ - translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, testRuntimeDefaults(), nil), - translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, testRuntimeDefaults(), nil), - translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, testRuntimeDefaults(), nil), - translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, testRuntimeDefaults(), nil), - }, images) + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), + translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), + translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), + translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), + }, images, nil) } func defaultWorkerPool() *atev1alpha1.WorkerPool { @@ -469,8 +469,13 @@ func TestResolveModelConfigFoundryEndpoint(t *testing.T) { type testHarnessCompiler struct{ input *translator.HarnessInput } +func (*testHarnessCompiler) ModelRequirements(*translator.ResolvedModelConfig) translator.ModelRequirements { + return translator.ModelRequirements{} +} + func (c *testHarnessCompiler) Compile(_ context.Context, input *translator.HarnessInput) (*translator.CompileResult, error) { - c.input = input + snapshot := *input + c.input = &snapshot return &translator.CompileResult{Revision: translator.Revision{AgentName: input.AgentName}}, nil } @@ -488,7 +493,7 @@ func TestCompilerAcceptsExternalHarnessCompiler(t *testing.T) { revision, err := translator.NewCompiler(krt.TestingDummyContext{}, collections, map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeCodex: adapter, - }, testHarnessImages).CompileAgent(context.Background(), inlineAgent(harness, template)) + }, testHarnessImages, nil).CompileAgent(context.Background(), inlineAgent(harness, template)) require.NoError(t, err) require.Equal(t, "runnable-agent", revision.AgentName) require.Equal(t, testHarnessImages.GetCodex(), revision.Image) @@ -514,7 +519,7 @@ func TestCompilerRejectsStructuredOutputForUnsupportedHarness(t *testing.T) { _, err := translator.NewCompiler(krt.TestingDummyContext{}, collections, map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeCodex: adapter, - }, testHarnessImages).CompileAgent(context.Background(), inlineAgent(harness, template)) + }, testHarnessImages, nil).CompileAgent(context.Background(), inlineAgent(harness, template)) require.ErrorContains(t, err, `Harness runtime "codex" does not support structured output`) require.Nil(t, adapter.input) } @@ -533,7 +538,7 @@ func TestCompilerRejectsUnusableModelConfigBeforeHarnessCompiler(t *testing.T) { _, err := translator.NewCompiler(krt.TestingDummyContext{}, collections, map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeCodex: adapter, - }, testHarnessImages).CompileAgent(context.Background(), inlineAgent(harness, template)) + }, testHarnessImages, nil).CompileAgent(context.Background(), inlineAgent(harness, template)) require.ErrorContains(t, err, `resolve ModelConfig "default-model": secret missing not found`) require.Nil(t, adapter.input) } @@ -549,7 +554,7 @@ func TestCompilerPermitsBYOWithoutModelConfig(t *testing.T) { _, err := translator.NewCompiler(krt.TestingDummyContext{}, mockCollections(t, defaultWorkerPool(), harness), map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeBYO: adapter, - }, testHarnessImages).CompileAgent(context.Background(), inlineAgent(harness, template)) + }, testHarnessImages, nil).CompileAgent(context.Background(), inlineAgent(harness, template)) require.NoError(t, err) require.Nil(t, adapter.input.Root.ResolvedModelConfig) } @@ -635,8 +640,8 @@ func TestCompileAgentInjectsCredentialsAtGateway(t *testing.T) { rotated.UID = "replacement-secret" rotated.Data["token"] = []byte("rotated-token") rotatedCompiler := translator.NewCompiler(krt.TestingDummyContext{}, mockCollections(t, modelConfig(), server, secondServer, rotated, secondSecret, defaultWorkerPool(), harness), map[translator.HarnessType]translator.HarnessCompiler{ - translator.HarnessTypeKagent: kagenttranslator.NewCompiler(krt.TestingDummyContext{}, mockCollections(t, modelConfig(), server, secondServer, rotated, secondSecret), testRuntimeDefaults(), nil), - }, testHarnessImages) + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(krt.TestingDummyContext{}, mockCollections(t, modelConfig(), server, secondServer, rotated, secondSecret), testRuntimeDefaults()), + }, testHarnessImages, nil) next, err := rotatedCompiler.CompileAgent(t.Context(), inlineAgent(harness, template)) require.NoError(t, err) nextDigest, err := next.Digest() diff --git a/go/core/internal/translator/credentials.go b/go/core/internal/translator/credentials.go deleted file mode 100644 index 226f632e73..0000000000 --- a/go/core/internal/translator/credentials.go +++ /dev/null @@ -1,234 +0,0 @@ -package translator - -import ( - "fmt" - "slices" - "strings" - - corev1 "k8s.io/api/core/v1" - - "github.com/kagent-dev/kagent/go/adk/pkg/models" - "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" - "github.com/kagent-dev/kagent/go/core/pkg/egressext" - "github.com/kagent-dev/kagent/go/core/pkg/env" -) - -// CredentialPlaceholder satisfies SDKs that require a configured API key. -// The gateway overwrites the corresponding header with the current Secret. -const CredentialPlaceholder = "kagent-credential-injected" - -// CompileCredentials replaces secret-backed environment values with inert SDK -// placeholders and compiles their destination-scoped gateway bindings. Models -// outside the agent tree (such as memory embeddings) are supplied separately. -// The third result contains only contributed bindings that need MCP placeholders. -func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, environment []corev1.EnvVar, binder egressext.CredentialBinder) ([]corev1.EnvVar, []egress.Credential, []egress.Credential, []string, error) { - var bindings []egress.Credential - var tokenPassthroughOrigins []string - boundModels := map[string]bool{} - boundMCP := map[string]bool{} - bind := func(rawURL, header, prefix, namespace, name, key string) error { - origin, err := credentialOrigin(rawURL) - if err != nil { - return NewValidationError("invalid credential destination: %v", err) - } - uri := "ate-secret://k8s.io/default/" + namespace + "/" + name + "/" + key - bindings = append(bindings, egress.Credential{Origin: origin, Header: header, Prefix: prefix, URI: uri}) - return nil - } - models := append([]*ResolvedModelConfig(nil), extraModels...) - var visit func(*AgentInput) error - visit = func(agent *AgentInput) error { - if len(extraModels) == 0 && agent.ResolvedModelConfig != nil { - models = append(models, agent.ResolvedModelConfig) - } - for _, tool := range agent.MCPTools { - for _, ref := range tool.Server.Spec.HeadersFrom { - if ref.ValueFrom != nil && ref.ValueFrom.Type == v1alpha3.SecretValueSource { - boundMCP[ref.ValueFrom.Name+"\x00"+ref.ValueFrom.Key] = true - if err := bind(tool.Server.Spec.URL, ref.Name, "", tool.Server.Namespace, ref.ValueFrom.Name, ref.ValueFrom.Key); err != nil { - return err - } - } - } - } - for _, child := range agent.Shared { - if err := visit(child.Agent); err != nil { - return err - } - } - return nil - } - if err := visit(input.Root); err != nil { - return nil, nil, nil, nil, err - } - for _, resolved := range models { - model := resolved.Config - if model.Spec.APIKeyPassthrough || model.Spec.APIKeySecret == "" { - continue - } - name, endpoint, header, prefix := modelCredentialTarget(resolved) - if name == "" { - continue - } - key := model.Spec.APIKeySecretKey - if model.Spec.Provider == v1alpha3.ModelProviderBedrock { - key = env.AWSBearerTokenBedrock.Name() - bearer := false - for _, variable := range environment { - if variable.Name == name && variable.ValueFrom != nil && variable.ValueFrom.SecretKeyRef != nil && variable.ValueFrom.SecretKeyRef.Name == model.Spec.APIKeySecret && variable.ValueFrom.SecretKeyRef.Key == key { - bearer = true - } - } - if !bearer { - continue - } - } - if err := bind(endpoint, header, prefix, model.Namespace, model.Spec.APIKeySecret, key); err != nil { - return nil, nil, nil, nil, err - } - boundModels[name+"\x00"+model.Spec.APIKeySecret+"\x00"+key] = true - } - bindings, err := egress.CanonicalCredentials(bindings) - if err != nil { - return nil, nil, nil, nil, NewValidationError("%v", err) - } - additions, err := contributeCredentials(input, models, bindings, binder) - if err != nil { - return nil, nil, nil, nil, err - } - bindings, err = egress.CanonicalCredentials(append(bindings, additions...)) - if err != nil { - return nil, nil, nil, nil, NewValidationError("%v", err) - } - for _, resolved := range models { - if !resolved.Config.Spec.APIKeyPassthrough { - continue - } - _, endpoint, _, _ := modelCredentialTarget(resolved) - origin, err := credentialOrigin(endpoint) - if err != nil { - return nil, nil, nil, nil, NewValidationError("invalid passthrough credential destination: %v", err) - } - tokenPassthroughOrigins = append(tokenPassthroughOrigins, origin) - } - var destinations []string - for _, binding := range bindings { - destinations = append(destinations, binding.Origin) - } - if _, err := egress.MergeAccess(egress.Access{Destinations: destinations, Credentials: bindings}, egress.Access{}, tokenPassthroughOrigins); err != nil { - return nil, nil, nil, nil, NewValidationError("%v", err) - } - result, err := applyModelCredentialPlaceholders(models, environment, bindings, additions) - if err != nil { - return nil, nil, nil, nil, err - } - for i, variable := range result { - if variable.ValueFrom == nil { - continue - } - ref := variable.ValueFrom.SecretKeyRef - isMCP := strings.HasPrefix(variable.Name, "KAGENT_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CODEX_MCP_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CLAUDE_MCP_CREDENTIAL_") - if ref == nil || (!boundModels[variable.Name+"\x00"+ref.Name+"\x00"+ref.Key] && (!isMCP || !boundMCP[ref.Name+"\x00"+ref.Key])) { - return nil, nil, nil, nil, NewValidationError("environment credential %q cannot use gateway header injection; local signing and arbitrary secret environment variables are unsupported", variable.Name) - } - result[i].Value, result[i].ValueFrom = CredentialPlaceholder, nil - } - return result, bindings, additions, tokenPassthroughOrigins, nil -} - -// Check requirements against the original environment, before Secret references -// or another model introduce placeholders for the same SDK variable. -func applyModelCredentialPlaceholders(models []*ResolvedModelConfig, environment []corev1.EnvVar, bindings, additions []egress.Credential) ([]corev1.EnvVar, error) { - result := append([]corev1.EnvVar(nil), environment...) - for _, resolved := range models { - model := resolved.Config - if model.Spec.APIKeySecret != "" || model.Spec.APIKeyPassthrough { - continue - } - name, endpoint, header, prefix := modelCredentialTarget(resolved) - if name == "" { - continue - } - origin, err := credentialOrigin(endpoint) - if err != nil { - continue - } - matches := func(binding egress.Credential) bool { - return binding.Origin == origin && binding.Header == header && binding.Prefix == prefix - } - bound := slices.ContainsFunc(bindings, matches) - i := slices.IndexFunc(result, func(variable corev1.EnvVar) bool { return variable.Name == name }) - required := slices.ContainsFunc(environment, func(variable corev1.EnvVar) bool { - return variable.Name == name && variable.Value == CredentialPlaceholder - }) - if required && !bound { - return nil, NewValidationError("model %q requires a gateway credential for %s header %s", model.Name, origin, header) - } - contributed := slices.ContainsFunc(additions, matches) - if contributed && i < 0 { - result = append(result, corev1.EnvVar{Name: name, Value: CredentialPlaceholder}) - } - } - return result, nil -} - -func modelCredentialTarget(resolved *ResolvedModelConfig) (name, endpoint, header, prefix string) { - spec := resolved.Config.Spec - switch spec.Provider { - case v1alpha3.ModelProviderOpenAI: - name, endpoint, header, prefix = env.OpenAIAPIKey.Name(), "https://api.openai.com", "authorization", "Bearer " - if spec.OpenAI != nil && spec.OpenAI.BaseURL != "" { - endpoint = spec.OpenAI.BaseURL - } - case v1alpha3.ModelProviderAnthropic: - name, endpoint, header = env.AnthropicAPIKey.Name(), "https://api.anthropic.com", "x-api-key" - if spec.Anthropic != nil && spec.Anthropic.BaseURL != "" { - endpoint = spec.Anthropic.BaseURL - } - case v1alpha3.ModelProviderAzureOpenAI: - name, header = env.AzureOpenAIAPIKey.Name(), "api-key" - if spec.AzureOpenAI != nil { - endpoint = spec.AzureOpenAI.Endpoint - } - case v1alpha3.ModelProviderGemini: - name, endpoint, header = env.GoogleAPIKey.Name(), "https://generativelanguage.googleapis.com", "x-goog-api-key" - case v1alpha3.ModelProviderMistral: - name, endpoint, header, prefix = env.MistralAPIKey.Name(), "https://api.mistral.ai", "authorization", "Bearer " - if spec.Mistral != nil && spec.Mistral.BaseURL != nil && *spec.Mistral.BaseURL != "" { - endpoint = *spec.Mistral.BaseURL - } - case v1alpha3.ModelProviderBedrock: - name, header, prefix = env.AWSBearerTokenBedrock.Name(), "authorization", "Bearer " - if spec.Bedrock != nil { - endpoint = fmt.Sprintf("https://bedrock-runtime.%s.amazonaws.com", spec.Bedrock.Region) - } - case v1alpha3.ModelProviderOllama: - // Ollama Cloud is the only keyed Ollama endpoint, so a binding is - // declared exactly when the model reaches it. That condition lives in - // models.OllamaReachesCloud, which the compiler's Secret reference and - // the egress list also use: a cloud model picked from the catalog, an - // explicit host, and a missing key were each judged differently in each - // place, so a valid configuration could compile with an env var that had - // no binding to match, or with an egress list that omitted the host it - // was about to call. - // - // Without a binding this leaves the name empty, and the env var is - // treated as a plain secret reference, which is correct — no request - // leaves for api.ollama.com. - if spec.Ollama == nil { - break - } - hasCredential := spec.APIKeySecret != "" || spec.APIKeyPassthrough - if !models.OllamaReachesCloud(spec.Model, spec.Ollama.Host, hasCredential) { - break - } - name, endpoint, header, prefix = env.OllamaAPIKey.Name(), "https://api.ollama.com", "authorization", "Bearer " - case v1alpha3.ModelProviderFoundry: - name, endpoint, header = env.FoundryAPIKey.Name(), resolved.FoundryEndpoint, "api-key" - if spec.Foundry != nil && spec.Foundry.APIFormat == v1alpha3.FoundryAPIFormatAnthropic { - header = "x-api-key" - } - } - return name, endpoint, header, prefix -} diff --git a/go/core/internal/translator/egress_credentials.go b/go/core/internal/translator/egress_credentials.go deleted file mode 100644 index 300b42a749..0000000000 --- a/go/core/internal/translator/egress_credentials.go +++ /dev/null @@ -1,155 +0,0 @@ -package translator - -import ( - "fmt" - "net/url" - "slices" - "strings" - - "k8s.io/apimachinery/pkg/types" - - "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" - "github.com/kagent-dev/kagent/go/core/pkg/egressext" -) - -func credentialOrigin(raw string) (string, error) { - u, err := url.Parse(strings.TrimSpace(raw)) - if err != nil || u.User != nil || u.Fragment != "" { - return "", fmt.Errorf("invalid credential destination") - } - origin, err := egress.ParseOrigin(egress.Origin(u)) - if err != nil { - return "", err - } - return origin.String(), nil -} - -// contributeCredentials calls the extension with eligible destinations and validates its result. -func contributeCredentials(input *HarnessInput, models []*ResolvedModelConfig, existing []egress.Credential, binder egressext.CredentialBinder) ([]egress.Credential, error) { - if binder == nil { - return nil, nil - } - targets := eligibleCredentialTargets(input, models) - current := make([]egressext.Credential, len(existing)) - for i, binding := range existing { - current[i] = egressext.Credential(binding) - } - agent := types.NamespacedName{Namespace: input.Harness.Namespace, Name: input.AgentName} - // Extension mutations must not change the compiler's bindings or validation targets. - contributed, err := binder.Bindings(agent, slices.Clone(targets), current) - if err != nil { - return nil, fmt.Errorf("failed to contribute credentials: %w", err) - } - return validateContributedCredentials(contributed, targets) -} - -// eligibleCredentialTargets selects exact HTTPS destinations that permit additional authentication. -func eligibleCredentialTargets(input *HarnessInput, models []*ResolvedModelConfig) []egressext.CredentialTarget { - targets := map[string][]egressext.CredentialTarget{} - reserved := map[string]bool{} - add := func(raw, header, prefix string, preserve bool) { - origin, err := credentialOrigin(raw) - if err != nil || !strings.HasPrefix(origin, "https://") || strings.Contains(origin, "*") { - return - } - // Header replacement applies to the whole origin, so any explicit auth reserves it. - reserved[origin] = reserved[origin] || preserve - targets[origin] = append(targets[origin], egressext.CredentialTarget{Origin: origin, Header: header, Prefix: prefix}) - } - for _, model := range models { - name, endpoint, header, prefix := modelCredentialTarget(model) - literalKey := slices.ContainsFunc(input.Harness.Spec.Env, func(variable v1alpha3.RuntimeEnvVar) bool { return name != "" && variable.Name == name }) - // Keyless Gemini needs runtime support before a binding can supply its key. - keylessGemini := model.Config.Spec.Provider == v1alpha3.ModelProviderGemini && model.Config.Spec.APIKeySecret == "" - add(endpoint, header, prefix, literalKey || keylessGemini || model.Config.Spec.APIKeyPassthrough || len(model.Config.Spec.DefaultHeaders) != 0) - } - var visit func(*AgentInput) - visit = func(agent *AgentInput) { - for _, tool := range agent.MCPTools { - // Any literal or ConfigMap header can carry authentication. - preserve := slices.ContainsFunc(tool.Server.Spec.HeadersFrom, func(ref v1alpha3.ValueRef) bool { - return ref.ValueFrom == nil || ref.ValueFrom.Type != v1alpha3.SecretValueSource - }) - add(tool.Server.Spec.URL, "", "", preserve) - } - for _, child := range agent.Shared { - visit(child.Agent) - } - } - visit(input.Root) - var eligible []egressext.CredentialTarget - for origin, formats := range targets { - if reserved[origin] { - continue - } - // A generic MCP target must not relax a model's required header format. - known := slices.ContainsFunc(formats, func(target egressext.CredentialTarget) bool { return target.Header != "" }) - for _, target := range formats { - if target.Header != "" || !known { - eligible = append(eligible, target) - } - } - } - // Keep extension inputs stable across map iteration and duplicate references. - slices.SortFunc(eligible, func(a, b egressext.CredentialTarget) int { - return strings.Compare(a.Origin+"\x00"+a.Header+"\x00"+a.Prefix, b.Origin+"\x00"+b.Header+"\x00"+b.Prefix) - }) - return slices.Compact(eligible) -} - -// validateContributedCredentials converts bindings and checks their origins and header formats. -func validateContributedCredentials(contributed []egressext.Credential, targets []egressext.CredentialTarget) ([]egress.Credential, error) { - added := make([]egress.Credential, len(contributed)) - for i, binding := range contributed { - added[i] = egress.Credential(binding) - } - added, err := egress.CanonicalCredentials(added) - if err != nil { - return nil, NewValidationError("%v", err) - } - for _, binding := range added { - matches := slices.ContainsFunc(targets, func(target egressext.CredentialTarget) bool { - if target.Origin != binding.Origin { - return false - } - return target.Header == "" || (target.Header == binding.Header && target.Prefix == binding.Prefix) - }) - if matches { - continue - } - allowed := slices.ContainsFunc(targets, func(target egressext.CredentialTarget) bool { return target.Origin == binding.Origin }) - if !allowed { - return nil, NewValidationError("credential destination %q is not an eligible application target", binding.Origin) - } - return nil, NewValidationError("credential for %q must match a model header and prefix", binding.Origin) - } - return added, nil -} - -// CredentialHeaders adds placeholders for contributed credentials to compiled MCP headers. -func CredentialHeaders(rawURL string, headers map[string]string, additions []egress.Credential) (map[string]string, error) { - if len(additions) == 0 { - return headers, nil - } - origin, err := credentialOrigin(rawURL) - if err != nil { - return nil, NewValidationError("invalid MCP credential destination: %v", err) - } - for _, binding := range additions { - if binding.Origin != origin { - continue - } - present := false - for name := range headers { - present = present || strings.EqualFold(name, binding.Header) - } - if !present { - if headers == nil { - headers = map[string]string{} - } - headers[binding.Header] = CredentialPlaceholder - } - } - return headers, nil -} diff --git a/go/core/internal/translator/egress_credentials_test.go b/go/core/internal/translator/egress_credentials_test.go deleted file mode 100644 index b9f692c3fd..0000000000 --- a/go/core/internal/translator/egress_credentials_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package translator - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" - "github.com/kagent-dev/kagent/go/core/pkg/egressext" -) - -type credentialBinderFunc func(types.NamespacedName, []egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) - -func (f credentialBinderFunc) Bindings(agent types.NamespacedName, targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { - return f(agent, targets, existing) -} - -var _ egressext.CredentialBinder = credentialBinderFunc(nil) - -func TestEgressCredentialBinderTargets(t *testing.T) { - input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "model-key", APIKeySecretKey: "token"}) - input.Root.MCPTools = []ResolvedMCPTool{ - {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://MCP.Example.com./tools"}}}, - {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "http://local.example.com/tools"}}}, - } - input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAnthropic}).Root}} - binder := credentialBinderFunc(func(_ types.NamespacedName, targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { - require.Equal(t, []egressext.CredentialTarget{ - {Origin: "https://api.anthropic.com:443", Header: "x-api-key"}, - {Origin: "https://api.openai.com:443", Header: "authorization", Prefix: "Bearer "}, - {Origin: "https://mcp.example.com:443"}, - }, targets) - require.Len(t, existing, 1) - require.Equal(t, egressext.Credential{ - Origin: "https://api.openai.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/model-key/token", - }, existing[0]) - // A binder cannot mutate the bindings used by the compiler. - existing[0].URI = "ate-secret://provider.example/changed" - return []egressext.Credential{{Origin: targets[0].Origin, Header: targets[0].Header, URI: "ate-secret://provider.example/token"}}, nil - }) - environment, bindings, added, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "model-key", "token")}, binder) - require.NoError(t, err) - require.Len(t, bindings, 2) - require.Equal(t, []egress.Credential{{ - Origin: "https://api.anthropic.com:443", Header: "x-api-key", URI: "ate-secret://provider.example/token", - }}, added) - require.Equal(t, "ate-secret://k8s.io/default/team/model-key/token", bindings[1].URI) - require.Contains(t, environment, corev1.EnvVar{Name: "ANTHROPIC_API_KEY", Value: CredentialPlaceholder}) -} - -func TestEgressCredentialBinderPreservesExplicitAuthentication(t *testing.T) { - for _, test := range []struct { - name string - configure func(*HarnessInput) - }{ - {"passthrough", func(input *HarnessInput) { input.Root.ResolvedModelConfig.Config.Spec.APIKeyPassthrough = true }}, - {"model headers", func(input *HarnessInput) { - input.Root.ResolvedModelConfig.Config.Spec.DefaultHeaders = map[string]string{"Authorization": "literal"} - }}, - {"harness key", func(input *HarnessInput) { - input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OPENAI_API_KEY", Value: "literal"}} - }}, - {"literal MCP header", func(input *HarnessInput) { - input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.openai.com/mcp", HeadersFrom: []v1alpha3.ValueRef{{Name: "Authorization", Value: "literal"}}}}}} - }}, - {"ConfigMap MCP header", func(input *HarnessInput) { - input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.openai.com/mcp", HeadersFrom: []v1alpha3.ValueRef{{Name: "Authorization", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.ConfigMapValueSource, Name: "auth", Key: "token"}}}}}}} - }}, - } { - t.Run(test.name, func(t *testing.T) { - input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) - test.configure(input) - binder := credentialBinderFunc(func(_ types.NamespacedName, targets []egressext.CredentialTarget, _ []egressext.Credential) ([]egressext.Credential, error) { - require.Empty(t, targets) - return nil, nil - }) - _, bindings, _, _, err := CompileCredentials(input, nil, nil, binder) - require.NoError(t, err) - require.Empty(t, bindings) - binder = credentialBinderFunc(func(types.NamespacedName, []egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) { - return []egressext.Credential{{Origin: "https://api.openai.com", Header: "authorization", URI: "ate-secret://provider.example/token"}}, nil - }) - _, bindings, _, _, err = CompileCredentials(input, nil, nil, binder) - require.ErrorContains(t, err, "not an eligible application target") - require.Empty(t, bindings) - }) - } -} - -func TestEgressCredentialBinderRejectsInvalidBindings(t *testing.T) { - base := egressext.Credential{Origin: "https://api.openai.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/token"} - for _, test := range []struct { - name string - change func(*egressext.Credential) - want string - }{ - {"unknown origin", func(c *egressext.Credential) { c.Origin = "https://unrelated.example.com" }, "not an eligible application target"}, - {"different port", func(c *egressext.Credential) { c.Origin = "https://api.openai.com:8443" }, "not an eligible application target"}, - {"HTTP", func(c *egressext.Credential) { c.Origin = "http://api.openai.com" }, "not an eligible application target"}, - {"wildcard", func(c *egressext.Credential) { c.Origin = "https://*.openai.com" }, "exact HTTP(S) DNS origin"}, - {"routing header", func(c *egressext.Credential) { c.Header = "Host" }, "invalid credential injection header"}, - {"invalid prefix", func(c *egressext.Credential) { c.Prefix = "Bearer\n" }, "invalid credential injection header"}, - {"invalid URI", func(c *egressext.Credential) { c.URI = "https://provider.example/token" }, "invalid credential URI"}, - {"model header mismatch", func(c *egressext.Credential) { c.Header = "x-api-key" }, "must match a model header and prefix"}, - {"model prefix mismatch", func(c *egressext.Credential) { c.Prefix = "Basic " }, "must match a model header and prefix"}, - {"conflict", func(c *egressext.Credential) {}, "conflicting credentials"}, - } { - t.Run(test.name, func(t *testing.T) { - input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "auth", APIKeySecretKey: "token"}) - binding := base - test.change(&binding) - binder := credentialBinderFunc(func(types.NamespacedName, []egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) { - return []egressext.Credential{binding}, nil - }) - _, bindings, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}, binder) - require.ErrorContains(t, err, test.want) - require.Empty(t, bindings) - }) - } - input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) - failure := errors.New("binder unavailable") - _, bindings, _, _, err := CompileCredentials(input, nil, nil, credentialBinderFunc(func(types.NamespacedName, []egressext.CredentialTarget, []egressext.Credential) ([]egressext.Credential, error) { - return nil, failure - })) - require.ErrorIs(t, err, failure) - require.Empty(t, bindings) -} - -func TestEgressCredentialBinderExtraModels(t *testing.T) { - input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) - embedding := &ResolvedModelConfig{Config: &v1alpha3.ModelConfig{ObjectMeta: metav1.ObjectMeta{Name: "embedding", Namespace: "team"}, Spec: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://embeddings.example.com/v1"}}}} - binder := credentialBinderFunc(func(_ types.NamespacedName, targets []egressext.CredentialTarget, _ []egressext.Credential) ([]egressext.Credential, error) { - require.Equal(t, []egressext.CredentialTarget{{Origin: "https://api.openai.com:443", Header: "authorization", Prefix: "Bearer "}, {Origin: "https://embeddings.example.com:443", Header: "authorization", Prefix: "Bearer "}}, targets) - return nil, nil - }) - _, bindings, _, _, err := CompileCredentials(input, []*ResolvedModelConfig{input.Root.ResolvedModelConfig, embedding}, nil, binder) - require.NoError(t, err) - require.Empty(t, bindings) -} - -func TestEgressCredentialBinderGeminiTargets(t *testing.T) { - for _, secret := range []string{"", "model-key"} { - t.Run("secret="+secret, func(t *testing.T) { - input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderGemini, APIKeySecret: secret}) - input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://generativelanguage.googleapis.com/mcp"}}}} - targets := eligibleCredentialTargets(input, []*ResolvedModelConfig{input.Root.ResolvedModelConfig}) - if secret == "" { - require.Empty(t, targets, "MCP targets must not bypass the keyless Gemini exclusion") - } else { - require.Equal(t, []egressext.CredentialTarget{{Origin: "https://generativelanguage.googleapis.com:443", Header: "x-goog-api-key"}}, targets) - } - }) - } -} - -func TestEgressCredentialBinderCannotMutateValidationTargets(t *testing.T) { - for _, test := range []struct { - name string - change func(*egressext.CredentialTarget) - want string - }{ - {"origin", func(target *egressext.CredentialTarget) { target.Origin = "https://other.example.com:443" }, "not an eligible application target"}, - {"header", func(target *egressext.CredentialTarget) { target.Header = "x-other-key" }, "must match a model header and prefix"}, - } { - t.Run(test.name, func(t *testing.T) { - input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) - binder := credentialBinderFunc(func(_ types.NamespacedName, targets []egressext.CredentialTarget, _ []egressext.Credential) ([]egressext.Credential, error) { - require.Len(t, targets, 1) - test.change(&targets[0]) - return []egressext.Credential{{Origin: targets[0].Origin, Header: targets[0].Header, Prefix: targets[0].Prefix, URI: "ate-secret://provider.example/token"}}, nil - }) - _, bindings, _, _, err := CompileCredentials(input, nil, nil, binder) - require.ErrorContains(t, err, test.want) - require.Empty(t, bindings) - }) - } -} diff --git a/go/core/internal/translator/egress_policy.go b/go/core/internal/translator/egress_policy.go new file mode 100644 index 0000000000..d53c9da67b --- /dev/null +++ b/go/core/internal/translator/egress_policy.go @@ -0,0 +1,291 @@ +package translator + +import ( + "cmp" + "fmt" + "net/url" + "slices" + "strings" + + "k8s.io/apimachinery/pkg/types" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" +) + +func credentialOrigin(raw string) (string, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.User != nil || u.Fragment != "" { + return "", fmt.Errorf("invalid credential destination") + } + origin, err := egress.ParseOrigin(egress.Origin(u)) + if err != nil { + return "", err + } + return origin.String(), nil +} + +func policyModels(input *HarnessInput) []*ResolvedModelConfig { + var models []*ResolvedModelConfig + var visit func(*AgentInput) + visit = func(agent *AgentInput) { + if agent.ResolvedModelConfig != nil { + models = append(models, agent.ResolvedModelConfig) + } + for _, child := range agent.Shared { + visit(child.Agent) + } + } + visit(input.Root) + if input.SummarizerModel != nil { + models = append(models, input.SummarizerModel) + } + if input.MemoryModel != nil { + models = append(models, input.MemoryModel) + } + return models +} + +// ResolvePolicy resolves authentication before any harness renders client configuration. +func ResolvePolicy(input *HarnessInput, compiler HarnessCompiler, extension egressext.PolicyTranslator) (ResolvedPolicy, error) { + models := policyModels(input) + requirements := make(map[*ResolvedModelConfig]ModelRequirements, len(models)) + if compiler != nil { + for _, model := range models { + requirements[model] = compiler.ModelRequirements(model) + } + } + draft, err := policyDraft(input, models, requirements) + if err != nil { + return ResolvedPolicy{}, err + } + original := draft + if extension != nil { + draft, err = extension.Translate(types.NamespacedName{Namespace: input.Harness.Namespace, Name: input.AgentName}, clonePolicyDraft(draft)) + if err != nil { + return ResolvedPolicy{}, fmt.Errorf("failed to translate egress policy: %w", err) + } + } + access, err := validatePolicyDraft(original, draft) + if err != nil { + return ResolvedPolicy{}, err + } + result := ResolvedPolicy{access: access, modelBindings: map[*ResolvedModelConfig]bool{}, secretModels: map[string]bool{}, secretMCP: map[string]bool{}, headers: map[string][]string{}} + var visit func(*AgentInput) + visit = func(agent *AgentInput) { + for _, tool := range agent.MCPTools { + for _, ref := range tool.Server.Spec.HeadersFrom { + if ref.ValueFrom != nil && ref.ValueFrom.Type == v1alpha3.SecretValueSource { + result.secretMCP[ref.ValueFrom.Name+"\x00"+ref.ValueFrom.Key] = true + } + } + } + for _, child := range agent.Shared { + visit(child.Agent) + } + } + visit(input.Root) + for _, binding := range access.Credentials { + if !slices.Contains(original.Credentials, egressext.Credential(binding)) { + result.headers[binding.Origin] = append(result.headers[binding.Origin], binding.Header) + } + } + for _, resolved := range models { + model := resolved.Config + target := modelCredentialTarget(resolved) + name, endpoint, header, prefix := target.Environment, target.Endpoint, target.Header, target.Prefix + if name == "" { + continue + } + origin, err := credentialOrigin(endpoint) + if err != nil { + return ResolvedPolicy{}, NewValidationError("invalid credential destination: %v", err) + } + matches := func(binding egress.Credential) bool { + return binding.Origin == origin && binding.Header == header && binding.Prefix == prefix + } + bound := slices.ContainsFunc(access.Credentials, matches) + if model.Spec.APIKeyPassthrough { + if slices.ContainsFunc(access.Credentials, func(binding egress.Credential) bool { return binding.Origin == origin }) { + return ResolvedPolicy{}, NewValidationError("destination %q cannot combine caller-token passthrough with gateway credentials", origin) + } + result.tokenPassthroughOrigins = append(result.tokenPassthroughOrigins, origin) + continue + } + if model.Spec.APIKeySecret != "" { + if bound { + result.modelBindings[resolved] = true + result.secretModels[name+"\x00"+model.Spec.APIKeySecret+"\x00"+modelSecretKey(resolved)] = true + } + continue + } + // Optional SDK authentication keeps its default mode unless a binding is added. + selected := slices.Contains(result.headers[origin], header) + if requirements[resolved].CredentialRequired { + if !bound { + return ResolvedPolicy{}, NewValidationError("model %q requires apiKeySecret or a resolved gateway credential for %s", model.Name, origin) + } + selected = true + } + if selected && bound { + result.modelBindings[resolved] = true + result.modelKeys = append(result.modelKeys, name) + } + } + return result, nil +} + +func policyDraft(input *HarnessInput, models []*ResolvedModelConfig, requirementsByModel map[*ResolvedModelConfig]ModelRequirements) (egressext.PolicyDraft, error) { + var destinations []string + var bindings []egress.Credential + var requirements []egressext.AuthenticationRequirement + add := func(raw, header, prefix string, preserve, required bool) error { + origin, err := credentialOrigin(raw) + if err != nil { + return err + } + destinations = append(destinations, origin) + if strings.HasPrefix(origin, "https://") && !strings.Contains(origin, "*") { + requirements = append(requirements, egressext.AuthenticationRequirement{Origin: origin, Header: header, Prefix: prefix, Preserve: preserve, Required: required}) + } + return nil + } + for _, model := range models { + for _, endpoint := range modelEndpoints(model) { + origin, err := credentialOrigin(endpoint) + if err != nil { + return egressext.PolicyDraft{}, NewValidationError("invalid model destination: %v", err) + } + destinations = append(destinations, origin) + } + destinations = append(destinations, requirementsByModel[model].Destinations...) + target := modelCredentialTarget(model) + name, endpoint, header, prefix := target.Environment, target.Endpoint, target.Header, target.Prefix + if name == "" { + continue + } + spec := model.Config.Spec + literal := slices.ContainsFunc(input.Harness.Spec.Env, func(variable v1alpha3.RuntimeEnvVar) bool { return variable.Name == name }) + preserve := literal || spec.APIKeyPassthrough || len(spec.DefaultHeaders) != 0 || (spec.Provider == v1alpha3.ModelProviderGemini && spec.APIKeySecret == "") + if err := add(endpoint, header, prefix, preserve, requirementsByModel[model].CredentialRequired && spec.APIKeySecret == "" && !preserve); err != nil { + return egressext.PolicyDraft{}, NewValidationError("invalid model credential destination: %v", err) + } + if spec.APIKeySecret == "" || spec.APIKeyPassthrough { + continue + } + key := modelSecretKey(model) + if spec.Provider == v1alpha3.ModelProviderBedrock && key == "" { + continue + } + origin, _ := credentialOrigin(endpoint) // Validated above. + bindings = append(bindings, egress.Credential{Origin: origin, Header: header, Prefix: prefix, URI: "ate-secret://k8s.io/default/" + model.Config.Namespace + "/" + spec.APIKeySecret + "/" + key}) + } + var visit func(*AgentInput) error + visit = func(agent *AgentInput) error { + for _, tool := range agent.MCPTools { + preserve := slices.ContainsFunc(tool.Server.Spec.HeadersFrom, func(ref v1alpha3.ValueRef) bool { + return ref.ValueFrom == nil || ref.ValueFrom.Type != v1alpha3.SecretValueSource + }) + if err := add(tool.Server.Spec.URL, "", "", preserve, false); err != nil { + return NewValidationError("invalid MCP credential destination: %v", err) + } + origin, _ := credentialOrigin(tool.Server.Spec.URL) // Validated above. + for _, ref := range tool.Server.Spec.HeadersFrom { + if ref.ValueFrom != nil && ref.ValueFrom.Type == v1alpha3.SecretValueSource { + bindings = append(bindings, egress.Credential{Origin: origin, Header: ref.Name, URI: "ate-secret://k8s.io/default/" + tool.Server.Namespace + "/" + ref.ValueFrom.Name + "/" + ref.ValueFrom.Key}) + } + } + } + for _, child := range agent.Shared { + if err := visit(child.Agent); err != nil { + return err + } + } + return nil + } + if err := visit(input.Root); err != nil { + return egressext.PolicyDraft{}, err + } + access, err := egress.CanonicalAccess(destinations, bindings) + if err != nil { + return egressext.PolicyDraft{}, NewValidationError("%v", err) + } + // Requirements apply to the whole origin, including overlapping resource references. + for i := range requirements { + for _, other := range requirements { + if other.Origin != requirements[i].Origin { + continue + } + requirements[i].Preserve = requirements[i].Preserve || other.Preserve + if other.Header == requirements[i].Header && other.Prefix == requirements[i].Prefix { + requirements[i].Required = requirements[i].Required || other.Required + } + } + } + slices.SortFunc(requirements, func(a, b egressext.AuthenticationRequirement) int { + return cmp.Or(strings.Compare(a.Origin, b.Origin), strings.Compare(a.Header, b.Header), strings.Compare(a.Prefix, b.Prefix)) + }) + requirements = slices.Compact(requirements) + credentials := make([]egressext.Credential, len(access.Credentials)) + for i, binding := range access.Credentials { + credentials[i] = egressext.Credential(binding) + } + return egressext.PolicyDraft{Destinations: access.Destinations, Requirements: requirements, Credentials: credentials}, nil +} + +func clonePolicyDraft(draft egressext.PolicyDraft) egressext.PolicyDraft { + return egressext.PolicyDraft{Destinations: slices.Clone(draft.Destinations), Requirements: slices.Clone(draft.Requirements), Credentials: slices.Clone(draft.Credentials)} +} + +func validatePolicyDraft(original, translated egressext.PolicyDraft) (egress.Access, error) { + if !slices.Equal(original.Destinations, translated.Destinations) || !slices.Equal(original.Requirements, translated.Requirements) { + return egress.Access{}, NewValidationError("policy extension cannot change destinations or authentication requirements") + } + bindings := make([]egress.Credential, len(translated.Credentials)) + for i, binding := range translated.Credentials { + bindings[i] = egress.Credential(binding) + } + access, err := egress.CanonicalAccess(translated.Destinations, bindings) + if err != nil { + return egress.Access{}, NewValidationError("%v", err) + } + for _, binding := range original.Credentials { + if !slices.Contains(access.Credentials, egress.Credential(binding)) { + return egress.Access{}, NewValidationError("policy extension cannot remove or replace existing credentials") + } + } + for _, binding := range access.Credentials { + if slices.Contains(original.Credentials, egressext.Credential(binding)) { + continue + } + allowed, known, matches := false, false, false + for _, requirement := range original.Requirements { + if requirement.Origin != binding.Origin { + continue + } + if requirement.Preserve { + return egress.Access{}, NewValidationError("credential destination %q preserves explicit authentication", binding.Origin) + } + allowed = true + if requirement.Header != "" { + known = true + matches = matches || (requirement.Header == binding.Header && requirement.Prefix == binding.Prefix) + } + } + if !allowed { + return egress.Access{}, NewValidationError("credential destination %q is not an eligible application target", binding.Origin) + } + if known && !matches { + return egress.Access{}, NewValidationError("credential for %q must match a model header and prefix", binding.Origin) + } + } + for _, requirement := range original.Requirements { + if requirement.Required && !requirement.Preserve && !slices.ContainsFunc(access.Credentials, func(binding egress.Credential) bool { + return binding.Origin == requirement.Origin && binding.Header == requirement.Header && binding.Prefix == requirement.Prefix + }) { + return egress.Access{}, NewValidationError("model requires a gateway credential for %s header %s; configure apiKeySecret", requirement.Origin, requirement.Header) + } + } + return access, nil +} diff --git a/go/core/internal/translator/credential_runtime_test.go b/go/core/internal/translator/egress_policy_runtime_test.go similarity index 52% rename from go/core/internal/translator/credential_runtime_test.go rename to go/core/internal/translator/egress_policy_runtime_test.go index 12b4446407..4f7520e3a4 100644 --- a/go/core/internal/translator/credential_runtime_test.go +++ b/go/core/internal/translator/egress_policy_runtime_test.go @@ -14,6 +14,7 @@ import ( runtimeconfig "github.com/kagent-dev/kagent/go/adk/pkg/config" configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" byotranslator "github.com/kagent-dev/kagent/go/core/internal/translator/byo" claudetranslator "github.com/kagent-dev/kagent/go/core/internal/translator/claude" @@ -25,38 +26,120 @@ import ( ) // This consumer uses only the public extension contract. -type defaultCredentialBinder struct{ uri string } +type defaultPolicyTranslator struct{ uri string } -var _ egressext.CredentialBinder = defaultCredentialBinder{} +var _ egressext.PolicyTranslator = defaultPolicyTranslator{} -func (d defaultCredentialBinder) Bindings(_ types.NamespacedName, targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { +func (d defaultPolicyTranslator) Translate(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { if d.uri == "" { - return nil, nil + return draft, nil } - var added []egressext.Credential - for _, target := range targets { - if slices.ContainsFunc(existing, func(c egressext.Credential) bool { return c.Origin == target.Origin }) { + for _, target := range draft.Requirements { + if target.Preserve || slices.ContainsFunc(draft.Credentials, func(c egressext.Credential) bool { return c.Origin == target.Origin }) { continue } header, prefix := target.Header, target.Prefix + // A generic requirement must not override the model format at a shared origin. if header == "" { + if slices.ContainsFunc(draft.Requirements, func(r egressext.AuthenticationRequirement) bool { return r.Origin == target.Origin && r.Header != "" }) { + continue + } header, prefix = "authorization", "Bearer " } - added = append(added, egressext.Credential{Origin: target.Origin, Header: header, Prefix: prefix, URI: d.uri}) + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: target.Origin, Header: header, Prefix: prefix, URI: d.uri}) } - return added, nil + return draft, nil } -type agentCredentialBinder map[types.NamespacedName]string +type agentPolicyTranslator map[types.NamespacedName]string -var _ egressext.CredentialBinder = agentCredentialBinder(nil) +var _ egressext.PolicyTranslator = agentPolicyTranslator(nil) -func (a agentCredentialBinder) Bindings(agent types.NamespacedName, targets []egressext.CredentialTarget, existing []egressext.Credential) ([]egressext.Credential, error) { - return defaultCredentialBinder{uri: a[agent]}.Bindings(agent, targets, existing) +func (a agentPolicyTranslator) Translate(agent types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + return defaultPolicyTranslator{uri: a[agent]}.Translate(agent, draft) } -func TestCompileCredentialsUsesRunnableAgentIdentity(t *testing.T) { - binder := agentCredentialBinder{ +type recordingPolicyTranslator struct{ draft egressext.PolicyDraft } + +func (r *recordingPolicyTranslator) Translate(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + r.draft = draft + return draft, nil +} + +func TestPolicyDraftMatchesPersistedApplicationGrant(t *testing.T) { + for _, test := range []struct { + kind translator.HarnessType + provider v1alpha3.ModelProvider + retainedDefault string + }{ + {translator.HarnessTypeKagent, v1alpha3.ModelProviderOpenAI, "https://api.openai.com:443"}, + {translator.HarnessTypeBYO, v1alpha3.ModelProviderOpenAI, "https://api.openai.com:443"}, + {translator.HarnessTypeKagent, v1alpha3.ModelProviderAnthropic, "https://api.anthropic.com:443"}, + {translator.HarnessTypeBYO, v1alpha3.ModelProviderAnthropic, "https://api.anthropic.com:443"}, + {translator.HarnessTypeClaude, v1alpha3.ModelProviderAnthropic, ""}, + {translator.HarnessTypeCodex, v1alpha3.ModelProviderOpenAI, ""}, + } { + t.Run(string(test.kind)+"/"+string(test.provider), func(t *testing.T) { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ + Workload: v1alpha3.HarnessWorkload{Image: new("example.com/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + switch test.kind { + case translator.HarnessTypeKagent: + harness.Spec.Kagent = &v1alpha3.KagentHarness{} + case translator.HarnessTypeBYO: + harness.Spec.BYO = &v1alpha3.BYOHarness{} + case translator.HarnessTypeClaude: + harness.Spec.Claude = &v1alpha3.ClaudeHarness{} + case translator.HarnessTypeCodex: + harness.Spec.Codex = &v1alpha3.CodexHarness{} + } + model := modelConfig() + model.Spec.Provider = test.provider + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = "model-auth", "token" + if test.provider == v1alpha3.ModelProviderOpenAI { + model.Spec.OpenAI = &v1alpha3.OpenAIConfig{BaseURL: "https://models.example.com/v1", APIFormat: new(v1alpha3.OpenAIAPIFormatResponses)} + } else { + model.Spec.Anthropic = &v1alpha3.AnthropicConfig{BaseURL: "https://models.example.com/v1"} + } + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "model-auth", Namespace: "test"}, Data: map[string][]byte{"token": []byte("secret")}} + server := remoteMCPServer("tools", "https://tools.example.com/mcp") + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help", + Tools: []v1alpha3.ToolBinding{{MCP: &v1alpha3.MCPToolBinding{Server: corev1.TypedLocalObjectReference{Kind: "RemoteMCPServer", Name: server.Name}}}}, + }} + collections := mockCollections(t, harness, model, server, secret, defaultWorkerPool()) + ctx := krt.TestingDummyContext{} + settings := testRuntimeDefaults() + adapters := map[translator.HarnessType]translator.HarnessCompiler{ + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings), + } + recorder := &recordingPolicyTranslator{} + result, err := translator.NewCompiler(ctx, collections, adapters, nil, recorder).CompileAgent(t.Context(), inlineAgent(harness, template)) + require.NoError(t, err) + want := []string{"https://models.example.com:443", "https://tools.example.com:443"} + if test.retainedDefault != "" { + want = append(want, test.retainedDefault) + } + slices.Sort(want) + require.Equal(t, want, recorder.draft.Destinations) + for _, requirement := range recorder.draft.Requirements { + require.NotEqual(t, test.retainedDefault, requirement.Origin, "retaining a destination must not make it a credential target") + } + want = append(want, settings.GetControlPlaneUrl()) + slices.Sort(want) + require.Equal(t, want, result.EgressDestinations) + require.Len(t, result.Credentials, 1) + require.Equal(t, recorder.draft.Credentials[0].URI, result.Credentials[0].URI) + }) + } +} + +func TestPolicyUsesRunnableAgentIdentity(t *testing.T) { + policyTranslator := agentPolicyTranslator{ {Namespace: "team-a", Name: "researcher"}: "ate-secret://provider.example/team-a/researcher", {Namespace: "team-a", Name: "reviewer"}: "ate-secret://provider.example/team-a/reviewer", {Namespace: "team-b", Name: "researcher"}: "ate-secret://provider.example/team-b/researcher", @@ -78,8 +161,8 @@ func TestCompileCredentialsUsesRunnableAgentIdentity(t *testing.T) { settings := testRuntimeDefaults() ctx := krt.TestingDummyContext{} compiler := translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ - translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings, binder), - }, nil) + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), + }, nil, policyTranslator) for _, source := range []string{"inline", "reference"} { for _, name := range []string{"researcher", "reviewer"} { t.Run(namespace+"/"+source+"/"+name, func(t *testing.T) { @@ -91,7 +174,7 @@ func TestCompileCredentialsUsesRunnableAgentIdentity(t *testing.T) { result, err := compiler.CompileAgent(t.Context(), agent) require.NoError(t, err) require.Len(t, result.Credentials, 1) - require.Equal(t, binder[types.NamespacedName{Namespace: namespace, Name: name}], result.Credentials[0].URI) + require.Equal(t, policyTranslator[types.NamespacedName{Namespace: namespace, Name: name}], result.Credentials[0].URI) }) } } @@ -107,6 +190,8 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { {translator.HarnessTypeBYO, v1alpha3.ModelProviderOpenAI}, {translator.HarnessTypeClaude, v1alpha3.ModelProviderAnthropic}, {translator.HarnessTypeCodex, v1alpha3.ModelProviderOpenAI}, + {translator.HarnessTypeKagent, v1alpha3.ModelProviderBedrock}, + {translator.HarnessTypeBYO, v1alpha3.ModelProviderBedrock}, {translator.HarnessTypeClaude, v1alpha3.ModelProviderBedrock}, {translator.HarnessTypeCodex, v1alpha3.ModelProviderBedrock}, } { @@ -147,18 +232,18 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { agent := inlineAgent(harness, template) agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} collections := mockCollections(t, harness, model, server, policy, defaultWorkerPool()) - compile := func(binder egressext.CredentialBinder) (*translator.CompileResult, error) { + compile := func(policyTranslator egressext.PolicyTranslator) (*translator.CompileResult, error) { settings := testRuntimeDefaults() ctx := krt.TestingDummyContext{} compilers := map[translator.HarnessType]translator.HarnessCompiler{ - translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings, binder), - translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings, binder), - translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings, binder), - translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings, binder), + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings), } - return translator.NewCompiler(ctx, collections, compilers, nil).CompileAgent(t.Context(), agent) + return translator.NewCompiler(ctx, collections, compilers, nil, policyTranslator).CompileAgent(t.Context(), agent) } - result, err := compile(defaultCredentialBinder{uri: "ate-secret://provider.example/token"}) + result, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/token"}) require.NoError(t, err) require.Len(t, result.Credentials, 2, "live EgressPolicy grants are not credential targets") require.Contains(t, result.Environment, corev1.EnvVar{Name: key, Value: translator.CredentialPlaceholder}) @@ -181,12 +266,12 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { } first, err := result.Digest() require.NoError(t, err) - again, err := compile(defaultCredentialBinder{uri: "ate-secret://provider.example/token"}) + again, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/token"}) require.NoError(t, err) same, err := again.Digest() require.NoError(t, err) require.Equal(t, first, same) - changed, err := compile(defaultCredentialBinder{uri: "ate-secret://provider.example/next"}) + changed, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/next"}) require.NoError(t, err) second, err := changed.Digest() require.NoError(t, err) @@ -197,7 +282,7 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { require.NoError(t, err) require.Equal(t, baseline.EgressDestinations, result.EgressDestinations) require.Empty(t, baseline.Credentials) - noop, err := compile(defaultCredentialBinder{}) + noop, err := compile(defaultPolicyTranslator{}) require.NoError(t, err) require.Equal(t, baseline.Revision, noop.Revision) nilDigest, err := baseline.Digest() @@ -208,8 +293,8 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { require.NotContains(t, string(baseline.ConfigJSON), translator.CredentialPlaceholder) } else { _, err := compile(nil) - require.Error(t, err, "credentials are still required without an extension") - _, err = compile(defaultCredentialBinder{}) + require.ErrorContains(t, err, "configure apiKeySecret") + _, err = compile(defaultPolicyTranslator{}) require.ErrorContains(t, err, "requires a gateway credential") } model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = "model-auth", "token" @@ -221,7 +306,13 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { collections = mockCollections(t, harness, model, server, policy, defaultWorkerPool(), secret) baseline, err := compile(nil) require.NoError(t, err) - noop, err := compile(defaultCredentialBinder{}) + if test.provider == v1alpha3.ModelProviderBedrock { + require.Equal(t, []egress.Credential{{ + Origin: "https://bedrock-runtime.us-east-1.amazonaws.com:443", Header: "authorization", Prefix: "Bearer ", + URI: "ate-secret://k8s.io/default/test/model-auth/AWS_BEARER_TOKEN_BEDROCK", + }}, baseline.Credentials) + } + noop, err := compile(defaultPolicyTranslator{}) require.NoError(t, err) require.Equal(t, baseline.Revision, noop.Revision) defaultDigest, err := baseline.Digest() @@ -229,9 +320,36 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { noopDigest, err := noop.Digest() require.NoError(t, err) require.Equal(t, defaultDigest, noopDigest) - withDefault, err := compile(defaultCredentialBinder{uri: "ate-secret://provider.example/token"}) + withDefault, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/token"}) require.NoError(t, err) require.Contains(t, withDefault.Credentials, baseline.Credentials[0], "default credentials preserve explicit model authentication") }) } } + +func TestPolicyResolvesAuxiliaryModelsBeforeHarnessRendering(t *testing.T) { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ + Kagent: &v1alpha3.KagentHarness{ + Memory: &v1alpha3.KagentHarnessMemory{ModelConfigRef: corev1.LocalObjectReference{Name: "embedding"}}, + Compaction: &v1alpha3.KagentHarnessCompaction{Summarizer: &v1alpha3.KagentHarnessSummarizer{ModelConfigRef: &corev1.LocalObjectReference{Name: "summary"}}}, + }, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}}, + }} + root, embedding, summary := modelConfig(), modelConfig(), modelConfig() + embedding.Name, summary.Name = "embedding", "summary" + embedding.Spec.OpenAI = &v1alpha3.OpenAIConfig{BaseURL: "https://embeddings.example.com/v1"} + summary.Spec.OpenAI = &v1alpha3.OpenAIConfig{BaseURL: "https://summary.example.com/v1"} + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ModelConfig: &corev1.LocalObjectReference{Name: root.Name}}} + collections := mockCollections(t, harness, root, embedding, summary, defaultWorkerPool()) + adapter := &testHarnessCompiler{} + compiler := translator.NewCompiler(krt.TestingDummyContext{}, collections, map[translator.HarnessType]translator.HarnessCompiler{translator.HarnessTypeKagent: adapter}, testHarnessImages, defaultPolicyTranslator{uri: "ate-secret://provider.example/token"}) + result, err := compiler.CompileAgent(t.Context(), inlineAgent(harness, template)) + require.NoError(t, err) + require.NotNil(t, adapter.input.SummarizerModel) + require.NotNil(t, adapter.input.MemoryModel) + require.True(t, adapter.input.Policy.ModelUsesGateway(adapter.input.Root.ResolvedModelConfig), "authentication must be resolved when rendering starts") + require.True(t, adapter.input.Policy.ModelUsesGateway(adapter.input.SummarizerModel)) + require.True(t, adapter.input.Policy.ModelUsesGateway(adapter.input.MemoryModel)) + require.Len(t, result.Credentials, 3) + require.Equal(t, []string{"https://api.openai.com:443", "https://embeddings.example.com:443", "https://summary.example.com:443"}, result.EgressDestinations) +} diff --git a/go/core/internal/translator/egress_policy_test.go b/go/core/internal/translator/egress_policy_test.go new file mode 100644 index 0000000000..1f0c76e077 --- /dev/null +++ b/go/core/internal/translator/egress_policy_test.go @@ -0,0 +1,269 @@ +package translator + +import ( + "context" + "errors" + "slices" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" +) + +type policyTranslatorFunc func(types.NamespacedName, egressext.PolicyDraft) (egressext.PolicyDraft, error) + +func (f policyTranslatorFunc) Translate(agent types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + return f(agent, draft) +} + +var _ egressext.PolicyTranslator = policyTranslatorFunc(nil) + +func TestPolicyDraftTargets(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "model-key", APIKeySecretKey: "token"}) + input.Root.MCPTools = []ResolvedMCPTool{ + {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://MCP.Example.com./tools"}}}, + {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "http://local.example.com/tools"}}}, + } + input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAnthropic}).Root}} + extension := policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + require.Len(t, draft.Credentials, 1) + require.Contains(t, draft.Requirements, egressext.AuthenticationRequirement{Origin: "https://api.anthropic.com:443", Header: "x-api-key"}) + require.Contains(t, draft.Requirements, egressext.AuthenticationRequirement{Origin: "https://mcp.example.com:443"}) + require.Contains(t, draft.Destinations, "http://local.example.com:80") + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://api.anthropic.com", Header: "x-api-key", URI: "ate-secret://provider.example/token"}) + return draft, nil + }) + policy, err := ResolvePolicy(input, nil, extension) + require.NoError(t, err) + require.Len(t, policy.access.Credentials, 2) + rendered, err := policy.RenderEnvironment([]corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "model-key", "token")}) + require.NoError(t, err) + require.Contains(t, rendered, corev1.EnvVar{Name: "ANTHROPIC_API_KEY", Value: CredentialPlaceholder}) + require.Equal(t, "ate-secret://k8s.io/default/team/model-key/token", policy.access.Credentials[1].URI) +} + +func TestPolicyPreservesExplicitAuthentication(t *testing.T) { + for _, test := range []struct { + name string + configure func(*HarnessInput) + }{ + {"passthrough", func(input *HarnessInput) { input.Root.ResolvedModelConfig.Config.Spec.APIKeyPassthrough = true }}, + {"model headers", func(input *HarnessInput) { + input.Root.ResolvedModelConfig.Config.Spec.DefaultHeaders = map[string]string{"Authorization": "literal"} + }}, + {"harness key", func(input *HarnessInput) { + input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OPENAI_API_KEY", Value: "literal"}} + }}, + {"literal MCP header", func(input *HarnessInput) { + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.openai.com/mcp", HeadersFrom: []v1alpha3.ValueRef{{Name: "X-Trace", Value: "literal"}}}}}} + }}, + {"ConfigMap MCP header", func(input *HarnessInput) { + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.openai.com/mcp", HeadersFrom: []v1alpha3.ValueRef{{Name: "Authorization", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.ConfigMapValueSource, Name: "auth", Key: "token"}}}}}}} + }}, + } { + t.Run(test.name, func(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + test.configure(input) + _, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + for _, requirement := range draft.Requirements { + require.True(t, requirement.Preserve) + } + return draft, nil + })) + require.NoError(t, err) + _, err = ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://api.openai.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/token"}) + return draft, nil + })) + require.ErrorContains(t, err, "preserves explicit authentication") + }) + } +} + +func TestPolicyRejectsInvalidBindings(t *testing.T) { + base := egressext.Credential{Origin: "https://api.openai.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/token"} + for _, test := range []struct { + name string + change func(*egressext.Credential) + want string + }{ + {"unknown origin", func(c *egressext.Credential) { c.Origin = "https://unrelated.example.com" }, "not explicitly allowed"}, + {"different port", func(c *egressext.Credential) { c.Origin = "https://api.openai.com:8443" }, "not explicitly allowed"}, + {"HTTP", func(c *egressext.Credential) { c.Origin = "http://api.openai.com" }, "not explicitly allowed"}, + {"wildcard", func(c *egressext.Credential) { c.Origin = "https://*.openai.com" }, "exact HTTP(S) DNS origin"}, + {"routing header", func(c *egressext.Credential) { c.Header = "Host" }, "invalid credential injection header"}, + {"invalid prefix", func(c *egressext.Credential) { c.Prefix = "Bearer\n" }, "invalid credential injection header"}, + {"invalid URI", func(c *egressext.Credential) { c.URI = "https://provider.example/token" }, "invalid credential URI"}, + {"model header mismatch", func(c *egressext.Credential) { c.Header = "x-api-key" }, "must match a model header and prefix"}, + {"model prefix mismatch", func(c *egressext.Credential) { c.Prefix = "Basic " }, "must match a model header and prefix"}, + } { + t.Run(test.name, func(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.openai.com/mcp"}}}} + binding := base + test.change(&binding) + _, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + draft.Credentials = append(draft.Credentials, binding) + return draft, nil + })) + require.ErrorContains(t, err, test.want) + }) + } + failure := errors.New("policy unavailable") + _, err := ResolvePolicy(credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}), nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + return draft, failure + })) + require.ErrorIs(t, err, failure) +} + +func TestPolicyCannotWeakenDraft(t *testing.T) { + for _, test := range []struct { + name string + change func(*egressext.PolicyDraft) + want string + }{ + {"destinations", func(d *egressext.PolicyDraft) { d.Destinations[0] = "https://other.example.com:443" }, "cannot change destinations"}, + {"header", func(d *egressext.PolicyDraft) { d.Requirements[0].Header = "x-other" }, "cannot change destinations"}, + {"preserve", func(d *egressext.PolicyDraft) { d.Requirements[0].Preserve = true }, "cannot change destinations"}, + {"required", func(d *egressext.PolicyDraft) { d.Requirements[0].Required = true }, "cannot change destinations"}, + {"remove binding", func(d *egressext.PolicyDraft) { d.Credentials = nil }, "cannot remove or replace"}, + {"replace binding", func(d *egressext.PolicyDraft) { d.Credentials[0].URI = "ate-secret://provider.example/changed" }, "cannot remove or replace"}, + {"conflict", func(d *egressext.PolicyDraft) { + binding := d.Credentials[0] + binding.URI = "ate-secret://provider.example/changed" + d.Credentials = append(d.Credentials, binding) + }, "conflicting credentials"}, + } { + t.Run(test.name, func(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "auth", APIKeySecretKey: "token"}) + before, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + _, err = ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + test.change(&draft) + return draft, nil + })) + require.ErrorContains(t, err, test.want) + after, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + require.Equal(t, before, after, "extension mutation must not affect subsequent compilation") + }) + } +} + +func TestPolicyAuxiliaryModels(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + input.MemoryModel = &ResolvedModelConfig{Config: &v1alpha3.ModelConfig{ObjectMeta: metav1.ObjectMeta{Name: "embedding", Namespace: "team"}, Spec: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://embeddings.example.com/v1"}}}} + input.SummarizerModel = &ResolvedModelConfig{Config: &v1alpha3.ModelConfig{ObjectMeta: metav1.ObjectMeta{Name: "summary", Namespace: "team"}, Spec: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAnthropic}}} + _, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + require.Len(t, draft.Requirements, 3) + require.Contains(t, draft.Destinations, "https://embeddings.example.com:443") + require.Contains(t, draft.Destinations, "https://api.anthropic.com:443") + return draft, nil + })) + require.NoError(t, err) +} + +func TestPolicyGeminiRequirements(t *testing.T) { + for _, secret := range []string{"", "model-key"} { + t.Run("secret="+secret, func(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderGemini, APIKeySecret: secret, APIKeySecretKey: "token"}) + draft, err := policyDraft(input, policyModels(input), nil) + require.NoError(t, err) + require.NotEmpty(t, draft.Requirements) + for _, r := range draft.Requirements { + require.Equal(t, secret == "", r.Preserve) + } + }) + } +} + +func TestPolicyRequiredAuthenticationIsNotAPlaceholderSentinel(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OPENAI_API_KEY", Value: CredentialPlaceholder}} + policy, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + _, err = policy.RenderEnvironment([]corev1.EnvVar{{Name: "OPENAI_API_KEY", Value: CredentialPlaceholder}}) + require.NoError(t, err, "a literal value must not create an authentication requirement") + input.Harness.Spec.Env = nil + input.Harness.Spec.Codex = &v1alpha3.CodexHarness{} + _, err = ResolvePolicy(input, requiredModelCredentials{v1alpha3.ModelProviderOpenAI}, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + require.True(t, slices.ContainsFunc(draft.Requirements, func(r egressext.AuthenticationRequirement) bool { return r.Required })) + return draft, nil + })) + require.ErrorContains(t, err, "requires a gateway credential") +} + +func TestPolicyRetainsDestinationsWithoutCredentialRequirements(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Ollama: &v1alpha3.OllamaConfig{Host: "daemon.example.com:11434"}}) + policy, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + require.Equal(t, []string{"http://daemon.example.com:11434"}, draft.Destinations) + require.Empty(t, draft.Requirements) + return draft, nil + })) + require.NoError(t, err) + require.Equal(t, []string{"http://daemon.example.com:11434"}, policy.access.Destinations) +} + +func TestPolicyModelDestinations(t *testing.T) { + for _, test := range []struct { + name string + spec v1alpha3.ModelConfigSpec + want []string + }{ + {"Mistral default", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderMistral}, []string{"https://api.mistral.ai:443"}}, + {"Mistral override", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderMistral, Mistral: &v1alpha3.MistralConfig{BaseURL: new("https://mistral.example.com/v1")}}, []string{"https://mistral.example.com:443"}}, + {"Ollama bare host", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Ollama: &v1alpha3.OllamaConfig{Host: "daemon.example.com:11434"}}, []string{"http://daemon.example.com:11434"}}, + {"Ollama explicit scheme", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Ollama: &v1alpha3.OllamaConfig{Host: "http://daemon.example.com:11434"}}, []string{"http://daemon.example.com:11434"}}, + {"Ollama no host", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Ollama: &v1alpha3.OllamaConfig{}}, nil}, + {"Ollama cloud", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", APIKeySecret: "auth", APIKeySecretKey: "token", Ollama: &v1alpha3.OllamaConfig{}}, []string{"https://api.ollama.com:443"}}, + {"Ollama local with key", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Model: "llama3.2", APIKeySecret: "auth", APIKeySecretKey: "token", Ollama: &v1alpha3.OllamaConfig{Host: "daemon.example.com:11434"}}, []string{"http://daemon.example.com:11434"}}, + {"SAP endpoints", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderSAPAICore, SAPAICore: &v1alpha3.SAPAICoreConfig{BaseURL: "https://sap.example.com/v1", AuthURL: "https://auth.example.com/token"}}, []string{"https://auth.example.com:443", "https://sap.example.com:443"}}, + } { + t.Run(test.name, func(t *testing.T) { + input := credentialInput(test.spec) + policy, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + require.Equal(t, test.want, draft.Destinations) + return draft, nil + })) + require.NoError(t, err) + require.Equal(t, test.want, policy.access.Destinations) + }) + } +} + +func TestPolicyPreservesExistingBindingClientBehavior(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAnthropic}) + input.Root.MCPTools = []ResolvedMCPTool{ + {Server: &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Namespace: "team"}, Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.anthropic.com/mcp", HeadersFrom: []v1alpha3.ValueRef{ + {Name: "x-api-key", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "auth", Key: "token"}}, + }}}}, + {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.anthropic.com/other"}}}, + } + policy, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + environment, err := policy.RenderEnvironment(nil) + require.NoError(t, err) + require.Empty(t, environment, "existing bindings do not select SDK authentication") + headers, err := policy.MCPHeaders("https://api.anthropic.com/other", nil) + require.NoError(t, err) + require.Empty(t, headers, "existing bindings do not add client headers") + unchanged, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + return draft, nil + })) + require.NoError(t, err) + require.Equal(t, policy, unchanged) +} + +type requiredModelCredentials []v1alpha3.ModelProvider + +func (r requiredModelCredentials) ModelRequirements(model *ResolvedModelConfig) ModelRequirements { + return ModelRequirements{CredentialRequired: slices.Contains(r, model.Config.Spec.Provider)} +} +func (requiredModelCredentials) Compile(context.Context, *HarnessInput) (*CompileResult, error) { + panic("policy resolution must not render a client") +} diff --git a/go/core/internal/translator/kagent/agentcard_test.go b/go/core/internal/translator/kagent/agentcard_test.go index 93d5bf8247..d3c09986b9 100644 --- a/go/core/internal/translator/kagent/agentcard_test.go +++ b/go/core/internal/translator/kagent/agentcard_test.go @@ -15,7 +15,7 @@ import ( ) func TestCompilerRequiresModelConfig(t *testing.T) { - _, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, testRuntimeDefaults(), nil).Compile(context.Background(), &translator.HarnessInput{ + _, err := NewCompiler(krt.TestingDummyContext{}, translator.Collections{}, testRuntimeDefaults()).Compile(context.Background(), &translator.HarnessInput{ Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}}}, Root: &translator.AgentInput{Template: &translator.TemplateConfiguration{}}, }) diff --git a/go/core/internal/translator/kagent/compiler.go b/go/core/internal/translator/kagent/compiler.go index 9d384e87f8..9e8c513d0b 100644 --- a/go/core/internal/translator/kagent/compiler.go +++ b/go/core/internal/translator/kagent/compiler.go @@ -14,20 +14,18 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/core/internal/translator" "github.com/kagent-dev/kagent/go/core/internal/translator/adkconfig" - "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) // Compiler translates resolved inputs into a kagent runtime revision. type Compiler struct { - binder egressext.CredentialBinder settings *configv1.RuntimeDefaultsConfig config *adkconfig.Builder } var _ translator.HarnessCompiler = (*Compiler)(nil) -func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig, binder egressext.CredentialBinder) *Compiler { - return &Compiler{settings: settings, binder: binder, config: adkconfig.NewBuilder(ctx, collections)} +func NewCompiler(ctx krt.HandlerContext, collections translator.Collections, settings *configv1.RuntimeDefaultsConfig) *Compiler { + return &Compiler{settings: settings, config: adkconfig.NewBuilder(ctx, collections)} } func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) (*translator.CompileResult, error) { @@ -52,13 +50,10 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build revision provenance: %w", err) } - environment, credentials, additions, tokenPassthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment, c.binder) + environment, err = input.Policy.RenderEnvironment(environment) if err != nil { return nil, err } - if err := adkconfig.ApplyCredentialHeaders(compiled.Config, additions); err != nil { - return nil, err - } configJSON, err := runtimeconfig.Marshal(&configv1.ADKConfig{ Runtime: translator.RuntimeConfig(c.settings, input.AgentName, template.Namespace), @@ -81,6 +76,11 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) Environment: environment, ConfigJSON: configJSON, AgentCard: card, SkillImages: compiled.SkillImages, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - TokenPassthroughOrigins: tokenPassthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: compiled.Egress, + Provenance: provenance, EgressDestinations: compiled.Egress, }}, nil } + +// ModelRequirements declares authentication and grants before client rendering. +func (*Compiler) ModelRequirements(model *translator.ResolvedModelConfig) translator.ModelRequirements { + return translator.ModelRequirements{Destinations: adkconfig.AdditionalModelDestinations(model)} +} diff --git a/go/core/internal/translator/model_egress.go b/go/core/internal/translator/model_egress.go new file mode 100644 index 0000000000..ba4e5abbeb --- /dev/null +++ b/go/core/internal/translator/model_egress.go @@ -0,0 +1,128 @@ +package translator + +import ( + "fmt" + "slices" + "strings" + + "k8s.io/apimachinery/pkg/types" + + "github.com/kagent-dev/kagent/go/adk/pkg/models" + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/pkg/env" +) + +func modelSecretKey(model *ResolvedModelConfig) string { + if model.Config.Spec.Provider != v1alpha3.ModelProviderBedrock { + return model.Config.Spec.APIKeySecretKey + } + secret := types.NamespacedName{Namespace: model.Config.Namespace, Name: model.Config.Spec.APIKeySecret} + if slices.ContainsFunc(model.References, func(ref ModelConfigReference) bool { + return ref.Kind == "Secret" && ref.NamespacedName == secret && ref.Key == env.AWSBearerTokenBedrock.Name() + }) { + return env.AWSBearerTokenBedrock.Name() + } + return "" +} + +type modelCredential struct { + Environment string + Endpoint string + Header string + Prefix string +} + +func modelCredentialTarget(resolved *ResolvedModelConfig) modelCredential { + var name, endpoint, header, prefix string + spec := resolved.Config.Spec + switch spec.Provider { + case v1alpha3.ModelProviderOpenAI: + name, endpoint, header, prefix = env.OpenAIAPIKey.Name(), "https://api.openai.com", "authorization", "Bearer " + if spec.OpenAI != nil && spec.OpenAI.BaseURL != "" { + endpoint = spec.OpenAI.BaseURL + } + case v1alpha3.ModelProviderAnthropic: + name, endpoint, header = env.AnthropicAPIKey.Name(), "https://api.anthropic.com", "x-api-key" + if spec.Anthropic != nil && spec.Anthropic.BaseURL != "" { + endpoint = spec.Anthropic.BaseURL + } + case v1alpha3.ModelProviderAzureOpenAI: + name, header = env.AzureOpenAIAPIKey.Name(), "api-key" + if spec.AzureOpenAI != nil { + endpoint = spec.AzureOpenAI.Endpoint + } + case v1alpha3.ModelProviderGemini: + name, endpoint, header = env.GoogleAPIKey.Name(), "https://generativelanguage.googleapis.com", "x-goog-api-key" + case v1alpha3.ModelProviderMistral: + name, endpoint, header, prefix = env.MistralAPIKey.Name(), "https://api.mistral.ai", "authorization", "Bearer " + if spec.Mistral != nil && spec.Mistral.BaseURL != nil && *spec.Mistral.BaseURL != "" { + endpoint = *spec.Mistral.BaseURL + } + case v1alpha3.ModelProviderBedrock: + name, header, prefix = env.AWSBearerTokenBedrock.Name(), "authorization", "Bearer " + if spec.Bedrock != nil { + endpoint = fmt.Sprintf("https://bedrock-runtime.%s.amazonaws.com", spec.Bedrock.Region) + } + case v1alpha3.ModelProviderOllama: + // Ollama Cloud is the only keyed Ollama endpoint, so a binding is + // declared exactly when the model reaches it. That condition lives in + // models.OllamaReachesCloud, which the compiler's Secret reference and + // the egress list also use: a cloud model picked from the catalog, an + // explicit host, and a missing key were each judged differently in each + // place, so a valid configuration could compile with an env var that had + // no binding to match, or with an egress list that omitted the host it + // was about to call. + // + // Without a binding this leaves the name empty, and the env var is + // treated as a plain secret reference, which is correct — no request + // leaves for api.ollama.com. + if spec.Ollama == nil { + break + } + hasCredential := spec.APIKeySecret != "" || spec.APIKeyPassthrough + if !models.OllamaReachesCloud(spec.Model, spec.Ollama.Host, hasCredential) { + break + } + name, endpoint, header, prefix = env.OllamaAPIKey.Name(), "https://api.ollama.com", "authorization", "Bearer " + case v1alpha3.ModelProviderFoundry: + name, endpoint, header = env.FoundryAPIKey.Name(), resolved.FoundryEndpoint, "api-key" + if spec.Foundry != nil && spec.Foundry.APIFormat == v1alpha3.FoundryAPIFormatAnthropic { + header = "x-api-key" + } + } + return modelCredential{Environment: name, Endpoint: endpoint, Header: header, Prefix: prefix} +} + +// modelEndpoints returns application endpoints before client configuration is rendered. +func modelEndpoints(resolved *ResolvedModelConfig) []string { + if resolved == nil { + return nil + } + spec := resolved.Config.Spec + endpoint := modelCredentialTarget(resolved).Endpoint + var endpoints []string + if endpoint != "" { + endpoints = append(endpoints, endpoint) + } + if spec.Provider == v1alpha3.ModelProviderOllama && spec.Ollama != nil && spec.Ollama.Host != "" { + endpoints = append(endpoints, OllamaURL(spec.Ollama.Host)) + } + if spec.Provider == v1alpha3.ModelProviderSAPAICore && spec.SAPAICore != nil { + endpoints = append(endpoints, spec.SAPAICore.BaseURL) + if spec.SAPAICore.AuthURL != "" { + endpoints = append(endpoints, spec.SAPAICore.AuthURL) + } + } + return endpoints +} + +// OllamaURL uses HTTPS for the cloud endpoint and HTTP for bare daemon addresses. +func OllamaURL(host string) string { + if strings.HasPrefix(host, "http://") || strings.HasPrefix(host, "https://") { + return host + } + if models.IsOllamaCloudEndpoint(host) { + return "https://" + host + } + return "http://" + host +} diff --git a/go/core/internal/translator/model_egress_test.go b/go/core/internal/translator/model_egress_test.go new file mode 100644 index 0000000000..15ee43ee67 --- /dev/null +++ b/go/core/internal/translator/model_egress_test.go @@ -0,0 +1,86 @@ +package translator + +import ( + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" +) + +// A cloud-tagged Ollama model reaches api.ollama.com with the key, which is a +// gateway-injected credential like any other — without this binding the +// translator rejects OLLAMA_API_KEY as an unsupported local secret and the +// AgentTemplate goes Compatible=False. +func TestModelCredentialTargetOllamaCloud(t *testing.T) { + secret := v1alpha3.ModelConfigSpec{ + Provider: v1alpha3.ModelProviderOllama, + Model: "deepseek-v4-flash:0731-cloud", + APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", + Ollama: &v1alpha3.OllamaConfig{}, + } + + tests := []struct { + name string + spec v1alpha3.ModelConfigSpec + wantName string + wantEndpoint string + }{ + { + name: "cloud model with a secret binds the cloud", + spec: secret, + wantName: "OLLAMA_API_KEY", + wantEndpoint: "https://api.ollama.com", + }, + { + name: "cloud model with passthrough binds the cloud", + spec: v1alpha3.ModelConfigSpec{ + Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", + APIKeyPassthrough: true, Ollama: &v1alpha3.OllamaConfig{}, + }, + wantName: "OLLAMA_API_KEY", + wantEndpoint: "https://api.ollama.com", + }, + { + // A local model must declare no binding: no request leaves for the + // cloud, so there is nothing to inject a header into. + name: "local model declares no binding", + spec: v1alpha3.ModelConfigSpec{ + Provider: v1alpha3.ModelProviderOllama, Model: "llama3.2", + APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", + Ollama: &v1alpha3.OllamaConfig{}, + }, + }, + { + // An explicit host wins over the cloud route, so the key stays a + // plain secret reference rather than a gateway binding. + name: "explicit host declares no binding", + spec: v1alpha3.ModelConfigSpec{ + Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", + APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", + Ollama: &v1alpha3.OllamaConfig{Host: "host.docker.internal:11434"}, + }, + }, + { + name: "cloud model without any key declares no binding", + spec: v1alpha3.ModelConfigSpec{ + Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", + Ollama: &v1alpha3.OllamaConfig{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resolved := &ResolvedModelConfig{ + Config: &v1alpha3.ModelConfig{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test"}, + Spec: tt.spec, + }, + } + target := modelCredentialTarget(resolved) + require.Equal(t, tt.wantName, target.Environment) + require.Equal(t, tt.wantEndpoint, target.Endpoint) + }) + } +} diff --git a/go/core/internal/translator/resolved_policy.go b/go/core/internal/translator/resolved_policy.go new file mode 100644 index 0000000000..571f8b8cbb --- /dev/null +++ b/go/core/internal/translator/resolved_policy.go @@ -0,0 +1,77 @@ +package translator + +import ( + "slices" + "strings" + + corev1 "k8s.io/api/core/v1" + + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +// CredentialPlaceholder satisfies SDKs that require a configured API key. +const CredentialPlaceholder = "kagent-credential-injected" + +// ResolvedPolicy contains pinned integration access and the authentication used to render clients. +// Its private fields prevent renderers from changing authentication decisions. +type ResolvedPolicy struct { + access egress.Access + modelKeys []string + modelBindings map[*ResolvedModelConfig]bool + secretModels map[string]bool + secretMCP map[string]bool + headers map[string][]string + // tokenPassthroughOrigins cannot receive gateway credentials, including live grants. + tokenPassthroughOrigins []string +} + +// ModelUsesGateway reports the resolved authentication mode of a model. +func (p ResolvedPolicy) ModelUsesGateway(model *ResolvedModelConfig) bool { + return p.modelBindings[model] +} + +// RenderEnvironment replaces supported Secret references and emits resolved SDK keys. +func (p ResolvedPolicy) RenderEnvironment(environment []corev1.EnvVar) ([]corev1.EnvVar, error) { + result := slices.Clone(environment) + for i, variable := range result { + if variable.ValueFrom == nil { + continue + } + ref := variable.ValueFrom.SecretKeyRef + isMCP := strings.HasPrefix(variable.Name, "KAGENT_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CODEX_MCP_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CLAUDE_MCP_CREDENTIAL_") + if ref == nil || (!p.secretModels[variable.Name+"\x00"+ref.Name+"\x00"+ref.Key] && (!isMCP || !p.secretMCP[ref.Name+"\x00"+ref.Key])) { + return nil, NewValidationError("environment credential %q cannot use gateway header injection; local signing and arbitrary secret environment variables are unsupported", variable.Name) + } + result[i].Value, result[i].ValueFrom = CredentialPlaceholder, nil + } + for _, name := range p.modelKeys { + if !slices.ContainsFunc(result, func(variable corev1.EnvVar) bool { return variable.Name == name }) { + result = append(result, corev1.EnvVar{Name: name, Value: CredentialPlaceholder}) + } + } + return result, nil +} + +// MCPHeaders renders missing headers from the resolved client authentication. +func (p ResolvedPolicy) MCPHeaders(rawURL string, headers map[string]string) (map[string]string, error) { + if len(p.headers) == 0 { + return headers, nil + } + origin, err := credentialOrigin(rawURL) + if err != nil { + return nil, NewValidationError("invalid MCP credential destination: %v", err) + } + for _, header := range p.headers[origin] { + present := false + for name := range headers { + present = present || strings.EqualFold(name, header) + } + if !present { + if headers == nil { + headers = map[string]string{} + } + headers[header] = CredentialPlaceholder + } + } + return headers, nil +} diff --git a/go/core/internal/translator/credentials_test.go b/go/core/internal/translator/resolved_policy_test.go similarity index 60% rename from go/core/internal/translator/credentials_test.go rename to go/core/internal/translator/resolved_policy_test.go index aeaf42aab6..37fc282cf6 100644 --- a/go/core/internal/translator/credentials_test.go +++ b/go/core/internal/translator/resolved_policy_test.go @@ -6,11 +6,13 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) -func TestCompileCredentialDestinations(t *testing.T) { +func TestResolvedPolicyCredentialDestinations(t *testing.T) { for _, test := range []struct { name string spec v1alpha3.ModelConfigSpec @@ -34,7 +36,7 @@ func TestCompileCredentialDestinations(t *testing.T) { } input := credentialInput(test.spec) environment := []corev1.EnvVar{credentialEnv(test.env, "auth", test.spec.APIKeySecretKey)} - got, bindings, _, _, err := CompileCredentials(input, nil, environment, nil) + got, bindings, err := renderTestCredentials(input, environment) require.NoError(t, err) require.Equal(t, CredentialPlaceholder, got[0].Value) require.Nil(t, got[0].ValueFrom) @@ -48,43 +50,46 @@ func TestCompileCredentialDestinations(t *testing.T) { } } -func TestCompileCredentialsRejectsConflictingSharedModels(t *testing.T) { +func TestResolvedPolicyRejectsConflictingSharedModels(t *testing.T) { spec := v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "root", APIKeySecretKey: "token"} input := credentialInput(spec) spec.APIKeySecret = "child" input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(spec).Root}} // The ADK environment is deduplicated, but every model still needs its own binding. - environment, _, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}, nil) + environment, _, err := renderTestCredentials(input, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) require.ErrorContains(t, err, "conflicting credentials") require.Nil(t, environment) input.Root.Shared[0].Agent.ResolvedModelConfig.Config.Spec.OpenAI = &v1alpha3.OpenAIConfig{BaseURL: "https://child.example.com/v1"} - _, bindings, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}, nil) + _, bindings, err := renderTestCredentials(input, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) require.NoError(t, err) require.Len(t, bindings, 2) } -func TestCompileCredentialsRejectsLocalSecrets(t *testing.T) { +func TestResolvedPolicyRejectsLocalSecrets(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{}) - environment, _, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("AWS_SECRET_ACCESS_KEY", "auth", "token")}, nil) + environment, _, err := renderTestCredentials(input, []corev1.EnvVar{credentialEnv("AWS_SECRET_ACCESS_KEY", "auth", "token")}) require.ErrorContains(t, err, "cannot use gateway header injection") require.Nil(t, environment) } -func TestCompileCredentialsPreservesPassthrough(t *testing.T) { +func TestResolvedPolicyPreservesPassthrough(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: true}) - _, bindings, _, _, err := CompileCredentials(input, nil, nil, nil) + _, bindings, err := renderTestCredentials(input, nil) require.NoError(t, err) require.Empty(t, bindings) input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(v1alpha3.ModelConfigSpec{ Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "auth", APIKeySecretKey: "token", }).Root}} - environment, _, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}, nil) + environment, _, err := renderTestCredentials(input, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}) require.ErrorContains(t, err, "cannot combine caller-token passthrough") require.Nil(t, environment) } func credentialInput(spec v1alpha3.ModelConfigSpec) *HarnessInput { resolved := &ResolvedModelConfig{Config: &v1alpha3.ModelConfig{ObjectMeta: metav1.ObjectMeta{Namespace: "team"}, Spec: spec}} + if spec.Provider == v1alpha3.ModelProviderBedrock && spec.APIKeySecretKey == "AWS_BEARER_TOKEN_BEDROCK" { + resolved.References = []ModelConfigReference{{Kind: "Secret", NamespacedName: types.NamespacedName{Namespace: "team", Name: spec.APIKeySecret}, Key: spec.APIKeySecretKey}} + } if spec.Foundry != nil { resolved.FoundryEndpoint = spec.Foundry.Endpoint } @@ -98,93 +103,21 @@ func credentialEnv(name, secret, key string) corev1.EnvVar { return corev1.EnvVar{Name: name, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secret}, Key: key}}} } -// A cloud-tagged Ollama model reaches api.ollama.com with the key, which is a -// gateway-injected credential like any other — without this binding the -// translator rejects OLLAMA_API_KEY as an unsupported local secret and the -// AgentTemplate goes Compatible=False. -func TestModelCredentialTargetOllamaCloud(t *testing.T) { - secret := v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, - Model: "deepseek-v4-flash:0731-cloud", - APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", - Ollama: &v1alpha3.OllamaConfig{}, - } - - tests := []struct { - name string - spec v1alpha3.ModelConfigSpec - wantName string - wantEndpoint string - }{ - { - name: "cloud model with a secret binds the cloud", - spec: secret, - wantName: "OLLAMA_API_KEY", - wantEndpoint: "https://api.ollama.com", - }, - { - name: "cloud model with passthrough binds the cloud", - spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", - APIKeyPassthrough: true, Ollama: &v1alpha3.OllamaConfig{}, - }, - wantName: "OLLAMA_API_KEY", - wantEndpoint: "https://api.ollama.com", - }, - { - // A local model must declare no binding: no request leaves for the - // cloud, so there is nothing to inject a header into. - name: "local model declares no binding", - spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "llama3.2", - APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", - Ollama: &v1alpha3.OllamaConfig{}, - }, - }, - { - // An explicit host wins over the cloud route, so the key stays a - // plain secret reference rather than a gateway binding. - name: "explicit host declares no binding", - spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", - APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", - Ollama: &v1alpha3.OllamaConfig{Host: "host.docker.internal:11434"}, - }, - }, - { - name: "cloud model without any key declares no binding", - spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", - Ollama: &v1alpha3.OllamaConfig{}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - resolved := &ResolvedModelConfig{ - Config: &v1alpha3.ModelConfig{ - ObjectMeta: metav1.ObjectMeta{Namespace: "test"}, - Spec: tt.spec, - }, - } - name, endpoint, _, _ := modelCredentialTarget(resolved) - require.Equal(t, tt.wantName, name) - require.Equal(t, tt.wantEndpoint, endpoint) - }) - } -} - -func TestPassthroughCredentialUsesCanonicalOrigin(t *testing.T) { +func TestPassthroughCredentialConflictUsesCanonicalOrigin(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: true, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://API.OPENAI.COM.:0443/v1"}}) - environment, credentials, additions, origins, err := CompileCredentials(input, nil, nil, nil) - require.NoError(t, err) - require.Empty(t, environment) - require.Empty(t, credentials) - require.Empty(t, additions) - require.Equal(t, []string{"https://api.openai.com:443"}, origins) + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Namespace: "team"}, Spec: v1alpha3.RemoteMCPServerSpec{ + URL: "https://api.openai.com/mcp", HeadersFrom: []v1alpha3.ValueRef{{Name: "authorization", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "auth", Key: "token"}}}, + }}}} + environment, _, err := renderTestCredentials(input, nil) + require.ErrorContains(t, err, "cannot combine caller-token passthrough") + require.Nil(t, environment) + input.Root.ResolvedModelConfig.Config.Spec.OpenAI.BaseURL = "https://api.openai.com:8443/v1" + policy, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err, "a different port is a distinct credential scope") + require.Equal(t, []string{"https://api.openai.com:8443"}, policy.tokenPassthroughOrigins) } -func TestCompileCredentialsRejectsInvalidOrigins(t *testing.T) { +func TestResolvedPolicyRejectsInvalidOrigins(t *testing.T) { for _, endpoint := range []string{ "ftp://api.example.com", "https://192.0.2.1", "https://user@api.example.com", "https://api.example.com/#fragment", @@ -198,10 +131,22 @@ func TestCompileCredentialsRejectsInvalidOrigins(t *testing.T) { if !passthrough { spec.APIKeySecret, spec.APIKeySecretKey = "auth", "token" } - _, bindings, _, _, err := CompileCredentials(credentialInput(spec), nil, nil, nil) + _, bindings, err := renderTestCredentials(credentialInput(spec), nil) require.Error(t, err) require.Empty(t, bindings) } }) } } + +func renderTestCredentials(input *HarnessInput, environment []corev1.EnvVar) ([]corev1.EnvVar, []egress.Credential, error) { + policy, err := ResolvePolicy(input, nil, nil) + if err != nil { + return nil, nil, err + } + rendered, err := policy.RenderEnvironment(environment) + if err != nil { + return nil, nil, err + } + return rendered, policy.access.Credentials, nil +} diff --git a/go/core/pkg/app/app.go b/go/core/pkg/app/app.go index c174047d11..4f03d81f68 100644 --- a/go/core/pkg/app/app.go +++ b/go/core/pkg/app/app.go @@ -84,9 +84,9 @@ import ( // every service that takes one — so a library consumer cannot narrow where // its own policy applies. type Options struct { - // EgressCredentialBinder tells the egress gateway which credentials to inject - // into an Agent's outbound requests. Nil adds none. - EgressCredentialBinder egressext.CredentialBinder + // EgressPolicyTranslator resolves additional outbound credentials before client rendering. + // Nil preserves the built-in authentication rules. + EgressPolicyTranslator egressext.PolicyTranslator // Authenticator identifies the caller. Nil selects InsecureAuthenticator, // which admits every request. Authenticator auth.AuthProvider @@ -276,7 +276,7 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er if err != nil { return fmt.Errorf("create controller manager: %w", err) } - runtime, err := controller.NewRuntime(kubeConfig, watchNamespaces, ctx.Done(), cfg.GetRuntimeDefaults(), cfg.GetHarnessImages(), opts.EgressCredentialBinder) + runtime, err := controller.NewRuntime(kubeConfig, watchNamespaces, ctx.Done(), cfg.GetRuntimeDefaults(), cfg.GetHarnessImages(), opts.EgressPolicyTranslator) if err != nil { return err } diff --git a/go/core/pkg/egressext/credentials.go b/go/core/pkg/egressext/credentials.go deleted file mode 100644 index 73b447a2a7..0000000000 --- a/go/core/pkg/egressext/credentials.go +++ /dev/null @@ -1,27 +0,0 @@ -// Package egressext defines outbound credential extensions. -package egressext - -import "k8s.io/apimachinery/pkg/types" - -// CredentialTarget is an eligible HTTPS origin. -// A nonempty Header requires the supplied header and prefix. -type CredentialTarget struct { - Origin string - Header string - Prefix string -} - -// Credential binds a provider reference to an outbound origin and header. -type Credential struct { - Origin string - Header string - Prefix string - URI string -} - -// CredentialBinder adds bindings during compilation for eligible targets without replacing existing bindings. -// Implementations must be deterministic and safe for concurrent calls, with no I/O or input mutation. -type CredentialBinder interface { - // agent identifies the runnable Agent, not its referenced AgentTemplate. - Bindings(agent types.NamespacedName, targets []CredentialTarget, existing []Credential) ([]Credential, error) -} diff --git a/go/core/pkg/egressext/policy.go b/go/core/pkg/egressext/policy.go new file mode 100644 index 0000000000..f17c6d8dc1 --- /dev/null +++ b/go/core/pkg/egressext/policy.go @@ -0,0 +1,40 @@ +// Package egressext defines outbound policy extensions. +package egressext + +import "k8s.io/apimachinery/pkg/types" + +// AuthenticationRequirement describes an application's authentication at an origin. +// A nonempty Header requires that header and prefix for new bindings. +type AuthenticationRequirement struct { + Origin string + Header string + Prefix string + // Preserve prevents additional bindings at this origin. + Preserve bool + // Required requires a gateway binding before the client can be rendered. + Required bool +} + +// Credential binds a provider reference to an outbound origin and header. +type Credential struct { + Origin string + Header string + Prefix string + URI string +} + +// PolicyDraft contains application destinations, authentication requirements and credential references. +// It contains no credential values. The extension can only add Credentials. +type PolicyDraft struct { + Destinations []string + Requirements []AuthenticationRequirement + Credentials []Credential +} + +// PolicyTranslator resolves policy before harness configuration is rendered. +// Implementations must be deterministic and safe for concurrent calls, with no I/O. +// Core rejects changed destinations, requirements, and existing bindings. +type PolicyTranslator interface { + // agent identifies the runnable Agent, not its referenced AgentTemplate. + Translate(agent types.NamespacedName, draft PolicyDraft) (PolicyDraft, error) +} From 59be74fc5d72d34385890b929aa31bf731145abd Mon Sep 17 00:00:00 2001 From: Cody Hartsook Date: Wed, 14 Oct 2026 21:24:44 -0700 Subject: [PATCH 38/43] refactor: split egress policy resolution into helpers Signed-off-by: Cody Hartsook --- go/core/internal/translator/egress_policy.go | 407 +++++++++++------- .../internal/translator/egress_policy_test.go | 17 + 2 files changed, 260 insertions(+), 164 deletions(-) diff --git a/go/core/internal/translator/egress_policy.go b/go/core/internal/translator/egress_policy.go index d53c9da67b..c4c36b31b6 100644 --- a/go/core/internal/translator/egress_policy.go +++ b/go/core/internal/translator/egress_policy.go @@ -26,23 +26,56 @@ func credentialOrigin(raw string) (string, error) { return origin.String(), nil } +func secretURI(namespace, name, key string) string { + return "ate-secret://k8s.io/default/" + namespace + "/" + name + "/" + key +} + +func isSecretHeader(ref v1alpha3.ValueRef) bool { + return ref.ValueFrom != nil && ref.ValueFrom.Type == v1alpha3.SecretValueSource +} + +func bindsTarget(origin, header, prefix string) func(egress.Credential) bool { + return func(binding egress.Credential) bool { + return binding.Origin == origin && binding.Header == header && binding.Prefix == prefix + } +} + +func publicCredentials(bindings []egress.Credential) []egressext.Credential { + result := make([]egressext.Credential, len(bindings)) + for i, binding := range bindings { + result[i] = egressext.Credential(binding) + } + return result +} + +func internalCredentials(bindings []egressext.Credential) []egress.Credential { + result := make([]egress.Credential, len(bindings)) + for i, binding := range bindings { + result[i] = egress.Credential(binding) + } + return result +} + +// policyAgents returns the root and its shared agents in depth-first order. +func policyAgents(root *AgentInput) []*AgentInput { + agents := []*AgentInput{root} + for _, child := range root.Shared { + agents = append(agents, policyAgents(child.Agent)...) + } + return agents +} + func policyModels(input *HarnessInput) []*ResolvedModelConfig { var models []*ResolvedModelConfig - var visit func(*AgentInput) - visit = func(agent *AgentInput) { + for _, agent := range policyAgents(input.Root) { if agent.ResolvedModelConfig != nil { models = append(models, agent.ResolvedModelConfig) } - for _, child := range agent.Shared { - visit(child.Agent) - } } - visit(input.Root) - if input.SummarizerModel != nil { - models = append(models, input.SummarizerModel) - } - if input.MemoryModel != nil { - models = append(models, input.MemoryModel) + for _, model := range []*ResolvedModelConfig{input.SummarizerModel, input.MemoryModel} { + if model != nil { + models = append(models, model) + } } return models } @@ -71,167 +104,200 @@ func ResolvePolicy(input *HarnessInput, compiler HarnessCompiler, extension egre if err != nil { return ResolvedPolicy{}, err } - result := ResolvedPolicy{access: access, modelBindings: map[*ResolvedModelConfig]bool{}, secretModels: map[string]bool{}, secretMCP: map[string]bool{}, headers: map[string][]string{}} - var visit func(*AgentInput) - visit = func(agent *AgentInput) { + result := ResolvedPolicy{ + access: access, modelBindings: map[*ResolvedModelConfig]bool{}, secretModels: map[string]bool{}, + secretMCP: secretMCPRefs(input.Root), headers: extensionHeaders(original, access), + } + for _, model := range models { + if err := result.resolveModel(model, requirements[model].CredentialRequired); err != nil { + return ResolvedPolicy{}, err + } + } + return result, nil +} + +// secretMCPRefs lists the Secret keys that MCP headers may reference. +func secretMCPRefs(root *AgentInput) map[string]bool { + refs := map[string]bool{} + for _, agent := range policyAgents(root) { for _, tool := range agent.MCPTools { for _, ref := range tool.Server.Spec.HeadersFrom { - if ref.ValueFrom != nil && ref.ValueFrom.Type == v1alpha3.SecretValueSource { - result.secretMCP[ref.ValueFrom.Name+"\x00"+ref.ValueFrom.Key] = true + if isSecretHeader(ref) { + refs[ref.ValueFrom.Name+"\x00"+ref.ValueFrom.Key] = true } } } - for _, child := range agent.Shared { - visit(child.Agent) - } } - visit(input.Root) + return refs +} + +// extensionHeaders lists, by origin, the headers of bindings the extension added. +func extensionHeaders(original egressext.PolicyDraft, access egress.Access) map[string][]string { + headers := map[string][]string{} for _, binding := range access.Credentials { if !slices.Contains(original.Credentials, egressext.Credential(binding)) { - result.headers[binding.Origin] = append(result.headers[binding.Origin], binding.Header) + headers[binding.Origin] = append(headers[binding.Origin], binding.Header) } } - for _, resolved := range models { - model := resolved.Config - target := modelCredentialTarget(resolved) - name, endpoint, header, prefix := target.Environment, target.Endpoint, target.Header, target.Prefix - if name == "" { - continue - } - origin, err := credentialOrigin(endpoint) - if err != nil { - return ResolvedPolicy{}, NewValidationError("invalid credential destination: %v", err) - } - matches := func(binding egress.Credential) bool { - return binding.Origin == origin && binding.Header == header && binding.Prefix == prefix - } - bound := slices.ContainsFunc(access.Credentials, matches) - if model.Spec.APIKeyPassthrough { - if slices.ContainsFunc(access.Credentials, func(binding egress.Credential) bool { return binding.Origin == origin }) { - return ResolvedPolicy{}, NewValidationError("destination %q cannot combine caller-token passthrough with gateway credentials", origin) - } - result.tokenPassthroughOrigins = append(result.tokenPassthroughOrigins, origin) - continue - } - if model.Spec.APIKeySecret != "" { - if bound { - result.modelBindings[resolved] = true - result.secretModels[name+"\x00"+model.Spec.APIKeySecret+"\x00"+modelSecretKey(resolved)] = true - } - continue - } - // Optional SDK authentication keeps its default mode unless a binding is added. - selected := slices.Contains(result.headers[origin], header) - if requirements[resolved].CredentialRequired { - if !bound { - return ResolvedPolicy{}, NewValidationError("model %q requires apiKeySecret or a resolved gateway credential for %s", model.Name, origin) - } - selected = true + return headers +} + +// resolveModel records whether a model client authenticates through the gateway. +func (p *ResolvedPolicy) resolveModel(resolved *ResolvedModelConfig, required bool) error { + spec := resolved.Config.Spec + target := modelCredentialTarget(resolved) + if target.Environment == "" { + return nil + } + origin, err := credentialOrigin(target.Endpoint) + if err != nil { + return NewValidationError("invalid credential destination: %v", err) + } + bound := slices.ContainsFunc(p.access.Credentials, bindsTarget(origin, target.Header, target.Prefix)) + switch { + case spec.APIKeyPassthrough: + if slices.ContainsFunc(p.access.Credentials, func(binding egress.Credential) bool { return binding.Origin == origin }) { + return NewValidationError("destination %q cannot combine caller-token passthrough with gateway credentials", origin) } - if selected && bound { - result.modelBindings[resolved] = true - result.modelKeys = append(result.modelKeys, name) + p.tokenPassthroughOrigins = append(p.tokenPassthroughOrigins, origin) + case spec.APIKeySecret != "": + if bound { + p.modelBindings[resolved] = true + p.secretModels[target.Environment+"\x00"+spec.APIKeySecret+"\x00"+modelSecretKey(resolved)] = true } + case required && !bound: + return NewValidationError("model %q requires apiKeySecret or a resolved gateway credential for %s", resolved.Config.Name, origin) + // Optional SDK authentication keeps its default mode unless the extension adds a binding. + case bound && (required || slices.Contains(p.headers[origin], target.Header)): + p.modelBindings[resolved] = true + p.modelKeys = append(p.modelKeys, target.Environment) } - return result, nil + return nil } -func policyDraft(input *HarnessInput, models []*ResolvedModelConfig, requirementsByModel map[*ResolvedModelConfig]ModelRequirements) (egressext.PolicyDraft, error) { - var destinations []string - var bindings []egress.Credential - var requirements []egressext.AuthenticationRequirement - add := func(raw, header, prefix string, preserve, required bool) error { - origin, err := credentialOrigin(raw) - if err != nil { - return err - } - destinations = append(destinations, origin) - if strings.HasPrefix(origin, "https://") && !strings.Contains(origin, "*") { - requirements = append(requirements, egressext.AuthenticationRequirement{Origin: origin, Header: header, Prefix: prefix, Preserve: preserve, Required: required}) +// draftBuilder collects application destinations, bindings and authentication requirements. +type draftBuilder struct { + destinations []string + bindings []egress.Credential + requirements []egressext.AuthenticationRequirement +} + +func (b *draftBuilder) allow(raw string) (string, error) { + origin, err := credentialOrigin(raw) + if err != nil { + return "", err + } + b.destinations = append(b.destinations, origin) + return origin, nil +} + +// allowTarget allows an origin that can carry credentials. Only exact HTTPS origins get a requirement. +func (b *draftBuilder) allowTarget(raw string, requirement egressext.AuthenticationRequirement) (string, error) { + origin, err := b.allow(raw) + if err != nil { + return "", err + } + if strings.HasPrefix(origin, "https://") && !strings.Contains(origin, "*") { + requirement.Origin = origin + b.requirements = append(b.requirements, requirement) + } + return origin, nil +} + +func (b *draftBuilder) addModel(model *ResolvedModelConfig, harnessEnv []v1alpha3.RuntimeEnvVar, requirements ModelRequirements) error { + for _, endpoint := range modelEndpoints(model) { + if _, err := b.allow(endpoint); err != nil { + return NewValidationError("invalid model destination: %v", err) } + } + b.destinations = append(b.destinations, requirements.Destinations...) + target := modelCredentialTarget(model) + if target.Environment == "" { return nil } - for _, model := range models { - for _, endpoint := range modelEndpoints(model) { - origin, err := credentialOrigin(endpoint) - if err != nil { - return egressext.PolicyDraft{}, NewValidationError("invalid model destination: %v", err) - } - destinations = append(destinations, origin) - } - destinations = append(destinations, requirementsByModel[model].Destinations...) - target := modelCredentialTarget(model) - name, endpoint, header, prefix := target.Environment, target.Endpoint, target.Header, target.Prefix - if name == "" { - continue - } - spec := model.Config.Spec - literal := slices.ContainsFunc(input.Harness.Spec.Env, func(variable v1alpha3.RuntimeEnvVar) bool { return variable.Name == name }) - preserve := literal || spec.APIKeyPassthrough || len(spec.DefaultHeaders) != 0 || (spec.Provider == v1alpha3.ModelProviderGemini && spec.APIKeySecret == "") - if err := add(endpoint, header, prefix, preserve, requirementsByModel[model].CredentialRequired && spec.APIKeySecret == "" && !preserve); err != nil { - return egressext.PolicyDraft{}, NewValidationError("invalid model credential destination: %v", err) - } - if spec.APIKeySecret == "" || spec.APIKeyPassthrough { - continue + spec := model.Config.Spec + preserve := modelPreservesAuth(spec, harnessEnv, target.Environment) + origin, err := b.allowTarget(target.Endpoint, egressext.AuthenticationRequirement{ + Header: target.Header, Prefix: target.Prefix, Preserve: preserve, + Required: requirements.CredentialRequired && spec.APIKeySecret == "" && !preserve, + }) + if err != nil { + return NewValidationError("invalid model credential destination: %v", err) + } + if spec.APIKeySecret == "" || spec.APIKeyPassthrough { + return nil + } + key := modelSecretKey(model) + if spec.Provider == v1alpha3.ModelProviderBedrock && key == "" { + return nil + } + b.bindings = append(b.bindings, egress.Credential{Origin: origin, Header: target.Header, Prefix: target.Prefix, URI: secretURI(model.Config.Namespace, spec.APIKeySecret, key)}) + return nil +} + +// modelPreservesAuth reports explicit model authentication that new bindings must not override. +// Keyless Gemini is included until its runtime can use a gateway binding. +func modelPreservesAuth(spec v1alpha3.ModelConfigSpec, harnessEnv []v1alpha3.RuntimeEnvVar, environment string) bool { + literal := slices.ContainsFunc(harnessEnv, func(variable v1alpha3.RuntimeEnvVar) bool { return variable.Name == environment }) + keylessGemini := spec.Provider == v1alpha3.ModelProviderGemini && spec.APIKeySecret == "" + return literal || spec.APIKeyPassthrough || len(spec.DefaultHeaders) != 0 || keylessGemini +} + +func (b *draftBuilder) addMCPServer(server *v1alpha3.RemoteMCPServer) error { + // Literal and ConfigMap headers can carry authentication. + preserve := slices.ContainsFunc(server.Spec.HeadersFrom, func(ref v1alpha3.ValueRef) bool { return !isSecretHeader(ref) }) + origin, err := b.allowTarget(server.Spec.URL, egressext.AuthenticationRequirement{Preserve: preserve}) + if err != nil { + return NewValidationError("invalid MCP credential destination: %v", err) + } + for _, ref := range server.Spec.HeadersFrom { + if isSecretHeader(ref) { + b.bindings = append(b.bindings, egress.Credential{Origin: origin, Header: ref.Name, URI: secretURI(server.Namespace, ref.ValueFrom.Name, ref.ValueFrom.Key)}) } - key := modelSecretKey(model) - if spec.Provider == v1alpha3.ModelProviderBedrock && key == "" { - continue + } + return nil +} + +func policyDraft(input *HarnessInput, models []*ResolvedModelConfig, requirementsByModel map[*ResolvedModelConfig]ModelRequirements) (egressext.PolicyDraft, error) { + var b draftBuilder + for _, model := range models { + if err := b.addModel(model, input.Harness.Spec.Env, requirementsByModel[model]); err != nil { + return egressext.PolicyDraft{}, err } - origin, _ := credentialOrigin(endpoint) // Validated above. - bindings = append(bindings, egress.Credential{Origin: origin, Header: header, Prefix: prefix, URI: "ate-secret://k8s.io/default/" + model.Config.Namespace + "/" + spec.APIKeySecret + "/" + key}) } - var visit func(*AgentInput) error - visit = func(agent *AgentInput) error { + for _, agent := range policyAgents(input.Root) { for _, tool := range agent.MCPTools { - preserve := slices.ContainsFunc(tool.Server.Spec.HeadersFrom, func(ref v1alpha3.ValueRef) bool { - return ref.ValueFrom == nil || ref.ValueFrom.Type != v1alpha3.SecretValueSource - }) - if err := add(tool.Server.Spec.URL, "", "", preserve, false); err != nil { - return NewValidationError("invalid MCP credential destination: %v", err) - } - origin, _ := credentialOrigin(tool.Server.Spec.URL) // Validated above. - for _, ref := range tool.Server.Spec.HeadersFrom { - if ref.ValueFrom != nil && ref.ValueFrom.Type == v1alpha3.SecretValueSource { - bindings = append(bindings, egress.Credential{Origin: origin, Header: ref.Name, URI: "ate-secret://k8s.io/default/" + tool.Server.Namespace + "/" + ref.ValueFrom.Name + "/" + ref.ValueFrom.Key}) - } + if err := b.addMCPServer(tool.Server); err != nil { + return egressext.PolicyDraft{}, err } } - for _, child := range agent.Shared { - if err := visit(child.Agent); err != nil { - return err - } - } - return nil - } - if err := visit(input.Root); err != nil { - return egressext.PolicyDraft{}, err } - access, err := egress.CanonicalAccess(destinations, bindings) + access, err := egress.CanonicalAccess(b.destinations, b.bindings) if err != nil { return egressext.PolicyDraft{}, NewValidationError("%v", err) } - // Requirements apply to the whole origin, including overlapping resource references. - for i := range requirements { + return egressext.PolicyDraft{Destinations: access.Destinations, Requirements: mergeRequirements(b.requirements), Credentials: publicCredentials(access.Credentials)}, nil +} + +// mergeRequirements applies preservation to a whole origin, including overlapping +// resource references, and merges requirements that share a header format. +func mergeRequirements(requirements []egressext.AuthenticationRequirement) []egressext.AuthenticationRequirement { + merged := slices.Clone(requirements) + for i := range merged { for _, other := range requirements { - if other.Origin != requirements[i].Origin { + if other.Origin != merged[i].Origin { continue } - requirements[i].Preserve = requirements[i].Preserve || other.Preserve - if other.Header == requirements[i].Header && other.Prefix == requirements[i].Prefix { - requirements[i].Required = requirements[i].Required || other.Required + merged[i].Preserve = merged[i].Preserve || other.Preserve + if other.Header == merged[i].Header && other.Prefix == merged[i].Prefix { + merged[i].Required = merged[i].Required || other.Required } } } - slices.SortFunc(requirements, func(a, b egressext.AuthenticationRequirement) int { + slices.SortFunc(merged, func(a, b egressext.AuthenticationRequirement) int { return cmp.Or(strings.Compare(a.Origin, b.Origin), strings.Compare(a.Header, b.Header), strings.Compare(a.Prefix, b.Prefix)) }) - requirements = slices.Compact(requirements) - credentials := make([]egressext.Credential, len(access.Credentials)) - for i, binding := range access.Credentials { - credentials[i] = egressext.Credential(binding) - } - return egressext.PolicyDraft{Destinations: access.Destinations, Requirements: requirements, Credentials: credentials}, nil + return slices.Compact(merged) } func clonePolicyDraft(draft egressext.PolicyDraft) egressext.PolicyDraft { @@ -242,11 +308,7 @@ func validatePolicyDraft(original, translated egressext.PolicyDraft) (egress.Acc if !slices.Equal(original.Destinations, translated.Destinations) || !slices.Equal(original.Requirements, translated.Requirements) { return egress.Access{}, NewValidationError("policy extension cannot change destinations or authentication requirements") } - bindings := make([]egress.Credential, len(translated.Credentials)) - for i, binding := range translated.Credentials { - bindings[i] = egress.Credential(binding) - } - access, err := egress.CanonicalAccess(translated.Destinations, bindings) + access, err := egress.CanonicalAccess(translated.Destinations, internalCredentials(translated.Credentials)) if err != nil { return egress.Access{}, NewValidationError("%v", err) } @@ -259,33 +321,50 @@ func validatePolicyDraft(original, translated egressext.PolicyDraft) (egress.Acc if slices.Contains(original.Credentials, egressext.Credential(binding)) { continue } - allowed, known, matches := false, false, false - for _, requirement := range original.Requirements { - if requirement.Origin != binding.Origin { - continue - } - if requirement.Preserve { - return egress.Access{}, NewValidationError("credential destination %q preserves explicit authentication", binding.Origin) - } - allowed = true - if requirement.Header != "" { - known = true - matches = matches || (requirement.Header == binding.Header && requirement.Prefix == binding.Prefix) - } + if err := validateAddedCredential(binding, original.Requirements); err != nil { + return egress.Access{}, err + } + } + if err := requireCredentials(original.Requirements, access.Credentials); err != nil { + return egress.Access{}, err + } + return access, nil +} + +// validateAddedCredential checks an extension binding against the requirements at its origin. +func validateAddedCredential(binding egress.Credential, requirements []egressext.AuthenticationRequirement) error { + allowed, known, matches := false, false, false + for _, requirement := range requirements { + if requirement.Origin != binding.Origin { + continue } - if !allowed { - return egress.Access{}, NewValidationError("credential destination %q is not an eligible application target", binding.Origin) + if requirement.Preserve { + return NewValidationError("credential destination %q preserves explicit authentication", binding.Origin) } - if known && !matches { - return egress.Access{}, NewValidationError("credential for %q must match a model header and prefix", binding.Origin) + allowed = true + if requirement.Header != "" { + known = true + matches = matches || bindsTarget(requirement.Origin, requirement.Header, requirement.Prefix)(binding) } } - for _, requirement := range original.Requirements { - if requirement.Required && !requirement.Preserve && !slices.ContainsFunc(access.Credentials, func(binding egress.Credential) bool { - return binding.Origin == requirement.Origin && binding.Header == requirement.Header && binding.Prefix == requirement.Prefix - }) { - return egress.Access{}, NewValidationError("model requires a gateway credential for %s header %s; configure apiKeySecret", requirement.Origin, requirement.Header) + if !allowed { + return NewValidationError("credential destination %q is not an eligible application target", binding.Origin) + } + if known && !matches { + return NewValidationError("credential for %q must match a model header and prefix", binding.Origin) + } + return nil +} + +// requireCredentials checks that every required, unpreserved requirement has a binding. +func requireCredentials(requirements []egressext.AuthenticationRequirement, credentials []egress.Credential) error { + for _, requirement := range requirements { + if !requirement.Required || requirement.Preserve { + continue + } + if !slices.ContainsFunc(credentials, bindsTarget(requirement.Origin, requirement.Header, requirement.Prefix)) { + return NewValidationError("model requires a gateway credential for %s header %s; configure apiKeySecret", requirement.Origin, requirement.Header) } } - return access, nil + return nil } diff --git a/go/core/internal/translator/egress_policy_test.go b/go/core/internal/translator/egress_policy_test.go index 1f0c76e077..e1ee39b144 100644 --- a/go/core/internal/translator/egress_policy_test.go +++ b/go/core/internal/translator/egress_policy_test.go @@ -267,3 +267,20 @@ func (r requiredModelCredentials) ModelRequirements(model *ResolvedModelConfig) func (requiredModelCredentials) Compile(context.Context, *HarnessInput) (*CompileResult, error) { panic("policy resolution must not render a client") } + +func TestMergeRequirements(t *testing.T) { + const origin, other = "https://models.example.com:443", "https://tools.example.com:443" + merged := mergeRequirements([]egressext.AuthenticationRequirement{ + {Origin: other}, + {Origin: origin, Header: "authorization", Prefix: "Bearer ", Required: true}, + {Origin: origin, Preserve: true}, + {Origin: origin, Header: "authorization", Prefix: "Bearer "}, + {Origin: origin, Header: "x-api-key"}, + }) + require.Equal(t, []egressext.AuthenticationRequirement{ + {Origin: origin, Preserve: true}, + {Origin: origin, Header: "authorization", Prefix: "Bearer ", Preserve: true, Required: true}, + {Origin: origin, Header: "x-api-key", Preserve: true}, + {Origin: other}, + }, merged) +} From 4934ee2e40f1379b1fd94516c10bc08393c6e533 Mon Sep 17 00:00:00 2001 From: Cody Hartsook Date: Wed, 14 Oct 2026 21:32:02 -0700 Subject: [PATCH 39/43] style: group adkconfig egress test imports Signed-off-by: Cody Hartsook --- go/core/internal/translator/adkconfig/egress_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/go/core/internal/translator/adkconfig/egress_test.go b/go/core/internal/translator/adkconfig/egress_test.go index e0ca775c37..a355aabf91 100644 --- a/go/core/internal/translator/adkconfig/egress_test.go +++ b/go/core/internal/translator/adkconfig/egress_test.go @@ -1,11 +1,13 @@ package adkconfig import ( - "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/translator" - "github.com/stretchr/testify/require" "slices" "testing" + + "github.com/stretchr/testify/require" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/translator" ) func TestRetainedADKMetadataDestinations(t *testing.T) { From e24a7e130a95d8302da4eb0b61f903ae7e03aac6 Mon Sep 17 00:00:00 2001 From: Cody Hartsook Date: Wed, 14 Oct 2026 21:46:32 -0700 Subject: [PATCH 40/43] fix: keep Harness literal model keys out of gateway bindings Signed-off-by: Cody Hartsook --- go/core/internal/translator/egress_policy.go | 11 ++--- .../translator/egress_policy_runtime_test.go | 40 +++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/go/core/internal/translator/egress_policy.go b/go/core/internal/translator/egress_policy.go index c4c36b31b6..4575ee11c9 100644 --- a/go/core/internal/translator/egress_policy.go +++ b/go/core/internal/translator/egress_policy.go @@ -216,7 +216,9 @@ func (b *draftBuilder) addModel(model *ResolvedModelConfig, harnessEnv []v1alpha return nil } spec := model.Config.Spec - preserve := modelPreservesAuth(spec, harnessEnv, target.Environment) + // A literal Harness value overrides the model Secret in the rendered environment. + literal := slices.ContainsFunc(harnessEnv, func(variable v1alpha3.RuntimeEnvVar) bool { return variable.Name == target.Environment }) + preserve := literal || modelPreservesAuth(spec) origin, err := b.allowTarget(target.Endpoint, egressext.AuthenticationRequirement{ Header: target.Header, Prefix: target.Prefix, Preserve: preserve, Required: requirements.CredentialRequired && spec.APIKeySecret == "" && !preserve, @@ -224,7 +226,7 @@ func (b *draftBuilder) addModel(model *ResolvedModelConfig, harnessEnv []v1alpha if err != nil { return NewValidationError("invalid model credential destination: %v", err) } - if spec.APIKeySecret == "" || spec.APIKeyPassthrough { + if spec.APIKeySecret == "" || spec.APIKeyPassthrough || literal { return nil } key := modelSecretKey(model) @@ -237,10 +239,9 @@ func (b *draftBuilder) addModel(model *ResolvedModelConfig, harnessEnv []v1alpha // modelPreservesAuth reports explicit model authentication that new bindings must not override. // Keyless Gemini is included until its runtime can use a gateway binding. -func modelPreservesAuth(spec v1alpha3.ModelConfigSpec, harnessEnv []v1alpha3.RuntimeEnvVar, environment string) bool { - literal := slices.ContainsFunc(harnessEnv, func(variable v1alpha3.RuntimeEnvVar) bool { return variable.Name == environment }) +func modelPreservesAuth(spec v1alpha3.ModelConfigSpec) bool { keylessGemini := spec.Provider == v1alpha3.ModelProviderGemini && spec.APIKeySecret == "" - return literal || spec.APIKeyPassthrough || len(spec.DefaultHeaders) != 0 || keylessGemini + return spec.APIKeyPassthrough || len(spec.DefaultHeaders) != 0 || keylessGemini } func (b *draftBuilder) addMCPServer(server *v1alpha3.RemoteMCPServer) error { diff --git a/go/core/internal/translator/egress_policy_runtime_test.go b/go/core/internal/translator/egress_policy_runtime_test.go index 4f7520e3a4..eeabdc2940 100644 --- a/go/core/internal/translator/egress_policy_runtime_test.go +++ b/go/core/internal/translator/egress_policy_runtime_test.go @@ -353,3 +353,43 @@ func TestPolicyResolvesAuxiliaryModelsBeforeHarnessRendering(t *testing.T) { require.Len(t, result.Credentials, 3) require.Equal(t, []string{"https://api.openai.com:443", "https://embeddings.example.com:443", "https://summary.example.com:443"}, result.EgressDestinations) } + +func TestHarnessLiteralKeyOverridesModelSecretBinding(t *testing.T) { + for _, test := range []struct { + name, env string + model v1alpha3.ModelConfigSpec + secret map[string][]byte + }{ + {"bedrock", "AWS_BEARER_TOKEN_BEDROCK", + v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderBedrock, Model: "claude", Bedrock: &v1alpha3.BedrockConfig{Region: "us-east-1"}, APIKeySecret: "model-auth"}, + map[string][]byte{"AWS_BEARER_TOKEN_BEDROCK": []byte("secret")}}, + {"openai", "OPENAI_API_KEY", + v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, Model: "gpt-4o", APIKeySecret: "model-auth", APIKeySecretKey: "token"}, + map[string][]byte{"token": []byte("secret")}}, + } { + t.Run(test.name, func(t *testing.T) { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ + Kagent: &v1alpha3.KagentHarness{}, + Env: []v1alpha3.RuntimeEnvVar{{Name: test.env, Value: "explicit"}}, + Workload: v1alpha3.HarnessWorkload{Image: new("example.com/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + model := modelConfig() + model.Spec = test.model + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "model-auth", Namespace: "test"}, Data: test.secret} + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help", + }} + agent := inlineAgent(harness, template) + collections := mockCollections(t, harness, model, secret, defaultWorkerPool()) + ctx := krt.TestingDummyContext{} + compilers := map[translator.HarnessType]translator.HarnessCompiler{ + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), + } + result, err := translator.NewCompiler(ctx, collections, compilers, nil, nil).CompileAgent(t.Context(), agent) + require.NoError(t, err) + require.Contains(t, result.Environment, corev1.EnvVar{Name: test.env, Value: "explicit"}) + require.Empty(t, result.Credentials, "the gateway must not replace an explicit Harness key") + }) + } +} From d5dafd43b40c937a519afcf1d250dfaf5acfaaf8 Mon Sep 17 00:00:00 2001 From: Cody Hartsook Date: Thu, 15 Oct 2026 12:52:05 -0700 Subject: [PATCH 41/43] fix: preserve explicit auth and unify credential binding rendering Signed-off-by: Cody Hartsook --- docs/architecture/credential-injection.md | 3 +- go/core/internal/translator/egress_policy.go | 20 ++-- .../translator/egress_policy_runtime_test.go | 56 ++++++++++ .../internal/translator/egress_policy_test.go | 100 +++++++++++++++--- 4 files changed, 156 insertions(+), 23 deletions(-) diff --git a/docs/architecture/credential-injection.md b/docs/architecture/credential-injection.md index 2bf9f76bfb..62dc681080 100644 --- a/docs/architecture/credential-injection.md +++ b/docs/architecture/credential-injection.md @@ -93,7 +93,8 @@ The shared compiler builds a draft from model and MCP inputs. EgressPolicy grants are live authorization and stay out of the draft. The extension can only add bindings for eligible destinations in that draft. Core validates the result, preserves explicit authentication, and pins it in the revision. Nil adds no -bindings. +bindings. Secret-backed and extension bindings render the same SDK key and MCP +header placeholders. References use `ate-secret:///` with a lowercase, port-free provider name registered in Substrate. Arbitrary clients must send the bound diff --git a/go/core/internal/translator/egress_policy.go b/go/core/internal/translator/egress_policy.go index 4575ee11c9..dec0a4b326 100644 --- a/go/core/internal/translator/egress_policy.go +++ b/go/core/internal/translator/egress_policy.go @@ -12,6 +12,7 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/pkg/egressext" + "github.com/kagent-dev/kagent/go/core/pkg/env" ) func credentialOrigin(raw string) (string, error) { @@ -106,7 +107,7 @@ func ResolvePolicy(input *HarnessInput, compiler HarnessCompiler, extension egre } result := ResolvedPolicy{ access: access, modelBindings: map[*ResolvedModelConfig]bool{}, secretModels: map[string]bool{}, - secretMCP: secretMCPRefs(input.Root), headers: extensionHeaders(original, access), + secretMCP: secretMCPRefs(input.Root), headers: clientCredentialHeaders(original, access), } for _, model := range models { if err := result.resolveModel(model, requirements[model].CredentialRequired); err != nil { @@ -131,11 +132,15 @@ func secretMCPRefs(root *AgentInput) map[string]bool { return refs } -// extensionHeaders lists, by origin, the headers of bindings the extension added. -func extensionHeaders(original egressext.PolicyDraft, access egress.Access) map[string][]string { +// clientCredentialHeaders excludes origins that preserve explicit authentication. +func clientCredentialHeaders(original egressext.PolicyDraft, access egress.Access) map[string][]string { + preserved := map[string]bool{} + for _, requirement := range original.Requirements { + preserved[requirement.Origin] = preserved[requirement.Origin] || requirement.Preserve + } headers := map[string][]string{} for _, binding := range access.Credentials { - if !slices.Contains(original.Credentials, egressext.Credential(binding)) { + if !preserved[binding.Origin] { headers[binding.Origin] = append(headers[binding.Origin], binding.Header) } } @@ -167,7 +172,7 @@ func (p *ResolvedPolicy) resolveModel(resolved *ResolvedModelConfig, required bo } case required && !bound: return NewValidationError("model %q requires apiKeySecret or a resolved gateway credential for %s", resolved.Config.Name, origin) - // Optional SDK authentication keeps its default mode unless the extension adds a binding. + // Optional SDK authentication uses bindings only at unpreserved origins. case bound && (required || slices.Contains(p.headers[origin], target.Header)): p.modelBindings[resolved] = true p.modelKeys = append(p.modelKeys, target.Environment) @@ -218,7 +223,10 @@ func (b *draftBuilder) addModel(model *ResolvedModelConfig, harnessEnv []v1alpha spec := model.Config.Spec // A literal Harness value overrides the model Secret in the rendered environment. literal := slices.ContainsFunc(harnessEnv, func(variable v1alpha3.RuntimeEnvVar) bool { return variable.Name == target.Environment }) - preserve := literal || modelPreservesAuth(spec) + signing := spec.Provider == v1alpha3.ModelProviderBedrock && slices.ContainsFunc(harnessEnv, func(variable v1alpha3.RuntimeEnvVar) bool { + return variable.Name == env.AWSAccessKeyID.Name() || variable.Name == env.AWSSecretAccessKey.Name() || variable.Name == env.AWSSessionToken.Name() + }) + preserve := literal || signing || modelPreservesAuth(spec) origin, err := b.allowTarget(target.Endpoint, egressext.AuthenticationRequirement{ Header: target.Header, Prefix: target.Prefix, Preserve: preserve, Required: requirements.CredentialRequired && spec.APIKeySecret == "" && !preserve, diff --git a/go/core/internal/translator/egress_policy_runtime_test.go b/go/core/internal/translator/egress_policy_runtime_test.go index eeabdc2940..e65d764864 100644 --- a/go/core/internal/translator/egress_policy_runtime_test.go +++ b/go/core/internal/translator/egress_policy_runtime_test.go @@ -21,6 +21,7 @@ import ( codextranslator "github.com/kagent-dev/kagent/go/core/internal/translator/codex" kagenttranslator "github.com/kagent-dev/kagent/go/core/internal/translator/kagent" "github.com/kagent-dev/kagent/go/core/pkg/egressext" + "github.com/kagent-dev/kagent/go/core/pkg/env" claudeconfig "github.com/kagent-dev/kagent/go/harness/claude/config" codexconfig "github.com/kagent-dev/kagent/go/harness/codex/config" ) @@ -393,3 +394,58 @@ func TestHarnessLiteralKeyOverridesModelSecretBinding(t *testing.T) { }) } } + +func TestHarnessBedrockSigningCredentialsPreserved(t *testing.T) { + for _, kind := range []translator.HarnessType{translator.HarnessTypeKagent, translator.HarnessTypeBYO} { + for _, test := range []struct { + name string + env []v1alpha3.RuntimeEnvVar + bearer bool + }{ + {name: "signing pair", env: []v1alpha3.RuntimeEnvVar{{Name: env.AWSAccessKeyID.Name(), Value: "access"}, {Name: env.AWSSecretAccessKey.Name(), Value: "secret"}}}, + {name: "empty access key", env: []v1alpha3.RuntimeEnvVar{{Name: env.AWSAccessKeyID.Name()}}}, + {name: "empty secret key", env: []v1alpha3.RuntimeEnvVar{{Name: env.AWSSecretAccessKey.Name()}}}, + {name: "empty session token", env: []v1alpha3.RuntimeEnvVar{{Name: env.AWSSessionToken.Name()}}}, + {name: "bearer Secret and signing key", env: []v1alpha3.RuntimeEnvVar{{Name: env.AWSAccessKeyID.Name(), Value: "access"}}, bearer: true}, + } { + t.Run(string(kind)+"/"+test.name, func(t *testing.T) { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ + Env: test.env, Workload: v1alpha3.HarnessWorkload{Image: new("example.com/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + if kind == translator.HarnessTypeKagent { + harness.Spec.Kagent = &v1alpha3.KagentHarness{} + } else { + harness.Spec.BYO = &v1alpha3.BYOHarness{} + } + model := modelConfig() + model.Spec = v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderBedrock, Model: "claude", Bedrock: &v1alpha3.BedrockConfig{Region: "us-east-1"}} + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "model-auth", Namespace: "test"}, Data: map[string][]byte{env.AWSBearerTokenBedrock.Name(): []byte("bearer")}} + if test.bearer { + model.Spec.APIKeySecret = secret.Name + } + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help", + }} + collections := mockCollections(t, harness, model, secret, defaultWorkerPool()) + ctx := krt.TestingDummyContext{} + compilers := map[translator.HarnessType]translator.HarnessCompiler{ + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), + translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), + } + result, err := translator.NewCompiler(ctx, collections, compilers, nil, defaultPolicyTranslator{uri: "ate-secret://provider.example/token"}).CompileAgent(t.Context(), inlineAgent(harness, template)) + require.NoError(t, err) + for _, variable := range test.env { + require.Contains(t, result.Environment, corev1.EnvVar{Name: variable.Name, Value: variable.Value}) + } + if test.bearer { + require.Equal(t, []egress.Credential{{Origin: "https://bedrock-runtime.us-east-1.amazonaws.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/test/model-auth/AWS_BEARER_TOKEN_BEDROCK"}}, result.Credentials) + require.Contains(t, result.Environment, corev1.EnvVar{Name: env.AWSBearerTokenBedrock.Name(), Value: translator.CredentialPlaceholder}) + } else { + require.Empty(t, result.Credentials) + require.False(t, slices.ContainsFunc(result.Environment, func(variable corev1.EnvVar) bool { return variable.Name == env.AWSBearerTokenBedrock.Name() })) + } + }) + } + } +} diff --git a/go/core/internal/translator/egress_policy_test.go b/go/core/internal/translator/egress_policy_test.go index e1ee39b144..861b2d60b7 100644 --- a/go/core/internal/translator/egress_policy_test.go +++ b/go/core/internal/translator/egress_policy_test.go @@ -3,6 +3,7 @@ package translator import ( "context" "errors" + "maps" "slices" "testing" @@ -13,6 +14,7 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/pkg/egressext" + "github.com/kagent-dev/kagent/go/core/pkg/env" ) type policyTranslatorFunc func(types.NamespacedName, egressext.PolicyDraft) (egressext.PolicyDraft, error) @@ -59,6 +61,10 @@ func TestPolicyPreservesExplicitAuthentication(t *testing.T) { {"harness key", func(input *HarnessInput) { input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OPENAI_API_KEY", Value: "literal"}} }}, + {"Bedrock signing key", func(input *HarnessInput) { + input.Root.ResolvedModelConfig.Config.Spec = v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderBedrock, Bedrock: &v1alpha3.BedrockConfig{Region: "us-east-1"}} + input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: env.AWSAccessKeyID.Name()}} + }}, {"literal MCP header", func(input *HarnessInput) { input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.openai.com/mcp", HeadersFrom: []v1alpha3.ValueRef{{Name: "X-Trace", Value: "literal"}}}}}} }}, @@ -69,6 +75,7 @@ func TestPolicyPreservesExplicitAuthentication(t *testing.T) { t.Run(test.name, func(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) test.configure(input) + target := modelCredentialTarget(input.Root.ResolvedModelConfig) _, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { for _, requirement := range draft.Requirements { require.True(t, requirement.Preserve) @@ -77,7 +84,7 @@ func TestPolicyPreservesExplicitAuthentication(t *testing.T) { })) require.NoError(t, err) _, err = ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { - draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://api.openai.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/token"}) + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: target.Endpoint, Header: target.Header, Prefix: target.Prefix, URI: "ate-secret://provider.example/token"}) return draft, nil })) require.ErrorContains(t, err, "preserves explicit authentication") @@ -236,27 +243,88 @@ func TestPolicyModelDestinations(t *testing.T) { } } -func TestPolicyPreservesExistingBindingClientBehavior(t *testing.T) { - input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAnthropic}) - input.Root.MCPTools = []ResolvedMCPTool{ - {Server: &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Namespace: "team"}, Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.anthropic.com/mcp", HeadersFrom: []v1alpha3.ValueRef{ - {Name: "x-api-key", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "auth", Key: "token"}}, - }}}}, - {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.anthropic.com/other"}}}, +func TestPolicyBindingSourcesRenderSameClients(t *testing.T) { + render := func(input *HarnessInput, extension egressext.PolicyTranslator) ([]corev1.EnvVar, map[string]string) { + policy, err := ResolvePolicy(input, nil, extension) + require.NoError(t, err) + environment, err := policy.RenderEnvironment(nil) + require.NoError(t, err) + headers, err := policy.MCPHeaders("https://api.anthropic.com/other", nil) + require.NoError(t, err) + return environment, headers } + other := ResolvedMCPTool{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.anthropic.com/other"}}} + secret := ResolvedMCPTool{Server: &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Namespace: "team"}, Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.anthropic.com/mcp", HeadersFrom: []v1alpha3.ValueRef{ + {Name: "X-Api-Key", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "auth", Key: "token"}}, + }}}} + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAnthropic}) + input.Root.MCPTools = []ResolvedMCPTool{secret, other} + environment, headers := render(input, nil) + require.Equal(t, []corev1.EnvVar{{Name: "ANTHROPIC_API_KEY", Value: CredentialPlaceholder}}, environment) + require.Equal(t, map[string]string{"x-api-key": CredentialPlaceholder}, headers) policy, err := ResolvePolicy(input, nil, nil) require.NoError(t, err) - environment, err := policy.RenderEnvironment(nil) - require.NoError(t, err) - require.Empty(t, environment, "existing bindings do not select SDK authentication") - headers, err := policy.MCPHeaders("https://api.anthropic.com/other", nil) + secretHeaders, err := policy.MCPHeaders("https://api.anthropic.com/mcp", map[string]string{"X-Api-Key": CredentialPlaceholder}) require.NoError(t, err) - require.Empty(t, headers, "existing bindings do not add client headers") - unchanged, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + require.Equal(t, map[string]string{"X-Api-Key": CredentialPlaceholder}, secretHeaders, "do not duplicate a Secret header with different casing") + input.Root.MCPTools = []ResolvedMCPTool{other} + contributedEnvironment, contributedHeaders := render(input, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://api.anthropic.com:443", Header: "x-api-key", URI: "ate-secret://provider.example/token"}) return draft, nil })) - require.NoError(t, err) - require.Equal(t, policy, unchanged) + require.Equal(t, environment, contributedEnvironment, "Secret and extension bindings render the same clients") + require.Equal(t, headers, contributedHeaders) +} + +func TestPolicyExistingBindingsPreserveClientAuthentication(t *testing.T) { + for _, test := range []struct { + name string + provider v1alpha3.ModelProvider + modelHeads map[string]string + env []v1alpha3.RuntimeEnvVar + mcpHeaders []v1alpha3.ValueRef + headers map[string]string + }{ + {name: "keyless Gemini", provider: v1alpha3.ModelProviderGemini}, + {name: "model headers", modelHeads: map[string]string{"Authorization": "literal"}}, + {name: "literal model key", env: []v1alpha3.RuntimeEnvVar{{Name: "ANTHROPIC_API_KEY", Value: "literal"}}}, + {name: "literal MCP header", mcpHeaders: []v1alpha3.ValueRef{{Name: "X-Trace", Value: "literal"}}, headers: map[string]string{"X-Trace": "literal"}}, + {name: "ConfigMap MCP header", mcpHeaders: []v1alpha3.ValueRef{{Name: "X-Trace", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.ConfigMapValueSource, Name: "trace", Key: "value"}}}, headers: map[string]string{"X-Trace": "configured"}}, + } { + t.Run(test.name, func(t *testing.T) { + provider := test.provider + if provider == "" { + provider = v1alpha3.ModelProviderAnthropic + } + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: provider, DefaultHeaders: test.modelHeads}) + input.Harness.Spec.Env = test.env + target := modelCredentialTarget(input.Root.ResolvedModelConfig) + input.Root.MCPTools = []ResolvedMCPTool{ + {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: target.Endpoint + "/mcp", HeadersFrom: test.mcpHeaders}}}, + {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: target.Endpoint + "/other"}}}, + {Server: &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Namespace: "team"}, Spec: v1alpha3.RemoteMCPServerSpec{URL: target.Endpoint + "/secret", HeadersFrom: []v1alpha3.ValueRef{ + {Name: target.Header, ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "auth", Key: "token"}}, + }}}}, + } + policy, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + require.Len(t, policy.access.Credentials, 1, "preservation must retain the Secret binding") + require.False(t, policy.ModelUsesGateway(input.Root.ResolvedModelConfig)) + var environment []corev1.EnvVar + for _, variable := range test.env { + environment = append(environment, corev1.EnvVar{Name: variable.Name, Value: variable.Value}) + } + rendered, err := policy.RenderEnvironment(environment) + require.NoError(t, err) + require.Equal(t, environment, rendered) + headers, err := policy.MCPHeaders(target.Endpoint+"/mcp", maps.Clone(test.headers)) + require.NoError(t, err) + require.Equal(t, test.headers, headers, "do not add auth to explicitly authenticated clients") + otherHeaders, err := policy.MCPHeaders(target.Endpoint+"/other", nil) + require.NoError(t, err) + require.Empty(t, otherHeaders, "preservation applies to every client at the origin") + }) + } } type requiredModelCredentials []v1alpha3.ModelProvider From e6bf128c571f5691f2578cc2a4a6ab0be6bab3fc Mon Sep 17 00:00:00 2001 From: Cody Hartsook Date: Fri, 16 Oct 2026 10:39:19 -0700 Subject: [PATCH 42/43] fix(egress): validate provider references and cover credential composition Signed-off-by: Cody Hartsook --- docs/architecture/credential-injection.md | 28 +++-- go/core/internal/egress/credentials.go | 12 +- go/core/internal/egress/credentials_test.go | 29 +++-- go/core/internal/substrate/policy_test.go | 4 +- .../internal/translator/adkconfig/mcp_test.go | 2 +- .../translator/egress_policy_runtime_test.go | 119 +++++++++++++++--- .../internal/translator/egress_policy_test.go | 36 +++++- 7 files changed, 176 insertions(+), 54 deletions(-) diff --git a/docs/architecture/credential-injection.md b/docs/architecture/credential-injection.md index 62dc681080..7f2fecf207 100644 --- a/docs/architecture/credential-injection.md +++ b/docs/architecture/credential-injection.md @@ -86,19 +86,21 @@ the caller's authentication. ## Controller credential extension -Embedded controllers can set `app.Options.EgressPolicyTranslator` to add gateway -credential references before a Harness renders client configuration. See -[PolicyTranslator](../../go/core/pkg/egressext/policy.go) for the contract. -The shared compiler builds a draft from model and MCP inputs. EgressPolicy -grants are live authorization and stay out of the draft. The extension can only -add bindings for eligible destinations in that draft. Core validates the result, -preserves explicit authentication, and pins it in the revision. Nil adds no -bindings. Secret-backed and extension bindings render the same SDK key and MCP -header placeholders. - -References use `ate-secret:///` with a lowercase, port-free -provider name registered in Substrate. Arbitrary clients must send the bound -header to trigger injection. Keyless Gemini support remains a follow-up. +Embedded controllers can set `app.Options.EgressPolicyTranslator` to add credential +references for eligible model and MCP destinations before client rendering. See +[PolicyTranslator](../../go/core/pkg/egressext/policy.go) for the contract. Core +validates and pins the bindings, then renders client placeholders. + +MCP Secret bindings can coexist with extension bindings on different headers. +Literal or ConfigMap MCP headers prevent additions at their origin. Model +requirements restrict the header and prefix. Conflicting credentials fail +compilation. A later EgressPolicy conflict withholds all explicit grants and +retains the pinned bindings. EgressPolicy-only destinations are outside the draft. + +References require a registered Substrate provider and use +`ate-secret:///`, with a lowercase DNS provider, no port, and at +least two nonempty, unescaped path segments. Clients must send the bound header +to trigger injection. Keyless Gemini support remains a follow-up. ## Deferred API fields diff --git a/go/core/internal/egress/credentials.go b/go/core/internal/egress/credentials.go index 3979ee92d6..f85861c704 100644 --- a/go/core/internal/egress/credentials.go +++ b/go/core/internal/egress/credentials.go @@ -58,7 +58,7 @@ func CanonicalCredentials(bindings []Credential) ([]Credential, error) { } // validCredentialURI accepts ate-secret:///. Provider names are -// lowercase registration keys without ports. k8s.io paths name one Secret key. +// lowercase DNS names without ports. k8s.io paths name one Secret key. func validCredentialURI(raw string) error { u, err := url.Parse(raw) if err != nil || u.Scheme != "ate-secret" || u.Opaque != "" { @@ -67,13 +67,17 @@ func validCredentialURI(raw string) error { if u.User != nil || u.ForceQuery || u.RawQuery != "" || strings.Contains(raw, "#") { return fmt.Errorf("credential URI cannot carry user information, a query or a fragment") } - if u.Host == "" || u.Host != strings.ToLower(u.Hostname()) { - return fmt.Errorf("credential provider must be a lowercase name without a port") + if u.Host != u.Hostname() || len(validation.IsDNS1123Subdomain(u.Host)) != 0 { + return fmt.Errorf("credential provider must be a lowercase DNS name without a port") + } + // Substrate requires at least two nonempty, unescaped path segments. + parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") + if u.EscapedPath() != u.Path || !strings.HasPrefix(u.Path, "/") || len(parts) < 2 || slices.Contains(parts, "") { + return fmt.Errorf("credential URI must have at least two nonempty path segments without percent encoding") } if u.Host != "k8s.io" { return nil } - parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") if len(parts) != 4 || parts[0] != "default" { return fmt.Errorf("credential URI must identify the default locator, namespace, Secret and key") } diff --git a/go/core/internal/egress/credentials_test.go b/go/core/internal/egress/credentials_test.go index 6692f51a72..4686a22d5e 100644 --- a/go/core/internal/egress/credentials_test.go +++ b/go/core/internal/egress/credentials_test.go @@ -25,17 +25,30 @@ func TestCanonicalCredentials(t *testing.T) { {"empty key", func(c *Credential) { c.URI = "ate-secret://k8s.io/default/team/auth/" }}, {"mixed case Kubernetes provider", func(c *Credential) { c.URI = "ate-secret://K8s.io/default/team/auth/token" }}, {"Kubernetes provider port", func(c *Credential) { c.URI = "ate-secret://k8s.io:443/default/team/auth/token" }}, - {"mixed case external provider", func(c *Credential) { c.URI = "ate-secret://Provider.example/token" }}, - {"external provider port", func(c *Credential) { c.URI = "ate-secret://provider.example:443/token" }}, - {"empty provider port", func(c *Credential) { c.URI = "ate-secret://provider.example:/token" }}, - {"IPv6 provider", func(c *Credential) { c.URI = "ate-secret://[2001:db8::1]/token" }}, + {"mixed case external provider", func(c *Credential) { c.URI = "ate-secret://Provider.example/default/token" }}, + {"external provider port", func(c *Credential) { c.URI = "ate-secret://provider.example:443/default/token" }}, + {"empty provider port", func(c *Credential) { c.URI = "ate-secret://provider.example:/default/token" }}, + {"IPv6 provider", func(c *Credential) { c.URI = "ate-secret://[2001:db8::1]/default/token" }}, {"missing provider", func(c *Credential) { c.URI = "ate-secret:///team/auth/token" }}, - {"userinfo", func(c *Credential) { c.URI = "ate-secret://user@provider.example/token" }}, + {"userinfo", func(c *Credential) { c.URI = "ate-secret://user@provider.example/default/token" }}, {"opaque", func(c *Credential) { c.URI = "ate-secret:provider.example/token" }}, {"empty query", func(c *Credential) { c.URI += "?" }}, {"fragment", func(c *Credential) { c.URI += "#token" }}, {"empty fragment", func(c *Credential) { c.URI += "#" }}, {"invalid escape", func(c *Credential) { c.URI = "ate-secret://provider.example/%xx" }}, + {"invalid provider DNS name", func(c *Credential) { c.URI = "ate-secret://provider_name/default/token" }}, + {"provider trailing dot", func(c *Credential) { c.URI = "ate-secret://provider.example./default/token" }}, + {"missing path", func(c *Credential) { c.URI = "ate-secret://provider.example" }}, + {"single path segment", func(c *Credential) { c.URI = "ate-secret://provider.example/token" }}, + {"empty path segment", func(c *Credential) { c.URI = "ate-secret://provider.example/default//token" }}, + {"empty first path segment", func(c *Credential) { c.URI = "ate-secret://provider.example//default/token" }}, + {"trailing slash", func(c *Credential) { c.URI = "ate-secret://provider.example/default/token/" }}, + {"encoded path", func(c *Credential) { + c.URI = "ate-secret://provider.example/default/https%3A%2F%2Fapi.example.com%3A8443" + }}, + {"encoded unreserved character", func(c *Credential) { c.URI = "ate-secret://provider.example/default/%74oken" }}, + {"unescaped space", func(c *Credential) { c.URI = "ate-secret://provider.example/default/session token" }}, + {"encoded Kubernetes key", func(c *Credential) { c.URI = "ate-secret://k8s.io/default/team/auth/%74oken" }}, {"missing locator", func(c *Credential) { c.URI = "ate-secret://k8s.io/team/auth/token" }}, {"unsupported locator", func(c *Credential) { c.URI = "ate-secret://k8s.io/remote/team/auth/token" }}, {"query", func(c *Credential) { c.URI += "?key=other" }}, @@ -55,9 +68,9 @@ func TestCanonicalCredentials(t *testing.T) { func TestCanonicalCredentialsExternalProvider(t *testing.T) { for _, uri := range []string{ - "ate-secret://provider.example/session-token", - "ate-secret://caller-token.kagent.dev/token", - "ate-secret://provider.example/default/https%3A%2F%2Fapi.example.com%3A8443", + "ate-secret://provider.example/default/session-token", + "ate-secret://caller-token.kagent.dev/default/token", + "ate-secret://provider.example/default/api.example.com:8443", } { t.Run(uri, func(t *testing.T) { binding := Credential{Origin: "https://api.example.com:8443", Header: "authorization", Prefix: "Bearer ", URI: uri} diff --git a/go/core/internal/substrate/policy_test.go b/go/core/internal/substrate/policy_test.go index fb2f992ab6..86350c29dd 100644 --- a/go/core/internal/substrate/policy_test.go +++ b/go/core/internal/substrate/policy_test.go @@ -11,7 +11,7 @@ import ( func TestActorPolicyKeepsCredentialsOnExactOrigin(t *testing.T) { policy, err := ActorEgressPolicy("team", []string{"https://*.example.com", "https://api.example.com:443", "https://api.example.com:8443", "http://api.example.com"}, []egress.Credential{ {Origin: "https://api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}, - {Origin: "https://api.example.com:8443", Header: "x-api-key", URI: "ate-secret://provider.example/token"}, + {Origin: "https://api.example.com:8443", Header: "x-api-key", URI: "ate-secret://provider.example/default/token"}, }) require.NoError(t, err) require.Len(t, policy.Rules, 4) @@ -26,5 +26,5 @@ func TestActorPolicyKeepsCredentialsOnExactOrigin(t *testing.T) { } require.Equal(t, []int32{443}, policy.Rules[2].GetHttps().GetPorts().GetNumbers()) require.Equal(t, []int32{8443}, policy.Rules[3].GetHttps().GetPorts().GetNumbers()) - require.Equal(t, "ate-secret://provider.example/token", policy.Rules[3].GetHttps().GetEffects().GetReplaceHeaders()[0].GetCredentialUri()) + require.Equal(t, "ate-secret://provider.example/default/token", policy.Rules[3].GetHttps().GetEffects().GetReplaceHeaders()[0].GetCredentialUri()) } diff --git a/go/core/internal/translator/adkconfig/mcp_test.go b/go/core/internal/translator/adkconfig/mcp_test.go index 05ea91bcb3..da7e633ff7 100644 --- a/go/core/internal/translator/adkconfig/mcp_test.go +++ b/go/core/internal/translator/adkconfig/mcp_test.go @@ -63,7 +63,7 @@ type mcpPolicyTranslator struct{} var _ egressext.PolicyTranslator = mcpPolicyTranslator{} func (mcpPolicyTranslator) Translate(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { - draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://tools.example.com:443", Header: "authorization", URI: "ate-secret://provider.example/token"}) + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://tools.example.com:443", Header: "authorization", URI: "ate-secret://provider.example/default/token"}) return draft, nil } diff --git a/go/core/internal/translator/egress_policy_runtime_test.go b/go/core/internal/translator/egress_policy_runtime_test.go index e65d764864..11d39a4b18 100644 --- a/go/core/internal/translator/egress_policy_runtime_test.go +++ b/go/core/internal/translator/egress_policy_runtime_test.go @@ -3,6 +3,7 @@ package translator_test import ( "encoding/json" "slices" + "strings" "testing" "github.com/stretchr/testify/require" @@ -60,6 +61,15 @@ func (a agentPolicyTranslator) Translate(agent types.NamespacedName, draft egres return defaultPolicyTranslator{uri: a[agent]}.Translate(agent, draft) } +type credentialPolicyTranslator egressext.Credential + +var _ egressext.PolicyTranslator = credentialPolicyTranslator{} + +func (c credentialPolicyTranslator) Translate(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + draft.Credentials = append(draft.Credentials, egressext.Credential(c)) + return draft, nil +} + type recordingPolicyTranslator struct{ draft egressext.PolicyDraft } func (r *recordingPolicyTranslator) Translate(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { @@ -244,35 +254,22 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { } return translator.NewCompiler(ctx, collections, compilers, nil, policyTranslator).CompileAgent(t.Context(), agent) } - result, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/token"}) + result, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/default/token"}) require.NoError(t, err) require.Len(t, result.Credentials, 2, "live EgressPolicy grants are not credential targets") require.Contains(t, result.Environment, corev1.EnvVar{Name: key, Value: translator.CredentialPlaceholder}) for _, variable := range result.Environment { require.Nil(t, variable.ValueFrom) } - switch kind { - case translator.HarnessTypeKagent, translator.HarnessTypeBYO: - var config configv1.ADKConfig - require.NoError(t, runtimeconfig.Unmarshal(result.ConfigJSON, &config)) - require.Equal(t, translator.CredentialPlaceholder, config.Agent.HttpTools[0].Params.Headers["authorization"]) - case translator.HarnessTypeClaude: - var config claudeconfig.Config - require.NoError(t, json.Unmarshal(result.ConfigJSON, &config)) - require.Equal(t, translator.CredentialPlaceholder, config.MCPServers["tools"].Headers["authorization"]) - case translator.HarnessTypeCodex: - var config codexconfig.Config - require.NoError(t, json.Unmarshal(result.ConfigJSON, &config)) - require.Equal(t, translator.CredentialPlaceholder, config.MCPServers["tools"].Headers["authorization"]) - } + require.Equal(t, translator.CredentialPlaceholder, compiledMCPHeaders(t, kind, result)["authorization"]) first, err := result.Digest() require.NoError(t, err) - again, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/token"}) + again, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/default/token"}) require.NoError(t, err) same, err := again.Digest() require.NoError(t, err) require.Equal(t, first, same) - changed, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/next"}) + changed, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/default/next"}) require.NoError(t, err) second, err := changed.Digest() require.NoError(t, err) @@ -321,13 +318,95 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { noopDigest, err := noop.Digest() require.NoError(t, err) require.Equal(t, defaultDigest, noopDigest) - withDefault, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/token"}) + withDefault, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/default/token"}) require.NoError(t, err) require.Contains(t, withDefault.Credentials, baseline.Credentials[0], "default credentials preserve explicit model authentication") + + t.Run("MCP Secret and extension", func(t *testing.T) { + server.Spec.HeadersFrom = []v1alpha3.ValueRef{{Name: "X-API-Key", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "service-auth", Key: "key"}}} + serviceSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "service-auth", Namespace: "test"}, Data: map[string][]byte{"key": []byte("service-secret")}} + collections = mockCollections(t, harness, model, server, policy, defaultWorkerPool(), secret, serviceSecret) + combined, err := compile(credentialPolicyTranslator{ + Origin: "https://tools.example.com:443", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/default/token", + }) + require.NoError(t, err) + require.Len(t, combined.Credentials, 3) + require.Contains(t, combined.Credentials, egress.Credential{ + Origin: "https://tools.example.com:443", Header: "x-api-key", URI: "ate-secret://k8s.io/default/test/service-auth/key", + }) + require.Contains(t, combined.Credentials, egress.Credential{ + Origin: "https://tools.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/default/token", + }) + prefix := "KAGENT_CREDENTIAL_" + if kind == translator.HarnessTypeClaude || kind == translator.HarnessTypeCodex { + prefix = "KAGENT_" + strings.ToUpper(string(kind)) + "_MCP_CREDENTIAL_" + } + var secretHeader string + for _, variable := range combined.Environment { + require.Nil(t, variable.ValueFrom) + if strings.HasPrefix(variable.Name, prefix) { + require.Equal(t, translator.CredentialPlaceholder, variable.Value) + secretHeader = "${" + variable.Name + "}" + if kind == translator.HarnessTypeKagent || kind == translator.HarnessTypeBYO { + secretHeader = "__KAGENT_ENV[" + variable.Name + "]__" + } + } + } + require.NotEmpty(t, secretHeader) + require.Equal(t, map[string]string{ + "X-API-Key": secretHeader, "authorization": translator.CredentialPlaceholder, + }, compiledMCPHeaders(t, kind, combined)) + }) }) } } +func compiledMCPHeaders(t *testing.T, kind translator.HarnessType, result *translator.CompileResult) map[string]string { + t.Helper() + switch kind { + case translator.HarnessTypeKagent, translator.HarnessTypeBYO: + var config configv1.ADKConfig + require.NoError(t, runtimeconfig.Unmarshal(result.ConfigJSON, &config)) + return config.Agent.HttpTools[0].Params.Headers + case translator.HarnessTypeClaude: + var config claudeconfig.Config + require.NoError(t, json.Unmarshal(result.ConfigJSON, &config)) + return config.MCPServers["tools"].Headers + case translator.HarnessTypeCodex: + var config codexconfig.Config + require.NoError(t, json.Unmarshal(result.ConfigJSON, &config)) + return config.MCPServers["tools"].Headers + default: + t.Fatalf("unsupported harness %q", kind) + return nil + } +} + +func TestCompileRejectsEgressPolicyCredentialConflict(t *testing.T) { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ + Kagent: &v1alpha3.KagentHarness{}, Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}}, + }} + model := modelConfig() + server := remoteMCPServer("tools", "https://tools.example.com/mcp") + policy, secret := egressPolicy() + policy.Spec.Destinations[0].Origin = "https://tools.example.com" + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, + Tools: []v1alpha3.ToolBinding{{MCP: &v1alpha3.MCPToolBinding{Server: corev1.TypedLocalObjectReference{Kind: "RemoteMCPServer", Name: server.Name}}}}, + }} + agent := inlineAgent(harness, template) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + collections := mockCollections(t, harness, model, server, policy, secret, defaultWorkerPool()) + extension := credentialPolicyTranslator{ + Origin: "https://tools.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/default/token", + } + compiler := translator.NewCompiler(krt.TestingDummyContext{}, collections, map[translator.HarnessType]translator.HarnessCompiler{ + translator.HarnessTypeKagent: &testHarnessCompiler{}, + }, testHarnessImages, extension) + _, err := compiler.CompileAgent(t.Context(), agent) + require.ErrorContains(t, err, "conflicting credentials for https://tools.example.com:443 header authorization") +} + func TestPolicyResolvesAuxiliaryModelsBeforeHarnessRendering(t *testing.T) { harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ Kagent: &v1alpha3.KagentHarness{ @@ -343,7 +422,7 @@ func TestPolicyResolvesAuxiliaryModelsBeforeHarnessRendering(t *testing.T) { template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ModelConfig: &corev1.LocalObjectReference{Name: root.Name}}} collections := mockCollections(t, harness, root, embedding, summary, defaultWorkerPool()) adapter := &testHarnessCompiler{} - compiler := translator.NewCompiler(krt.TestingDummyContext{}, collections, map[translator.HarnessType]translator.HarnessCompiler{translator.HarnessTypeKagent: adapter}, testHarnessImages, defaultPolicyTranslator{uri: "ate-secret://provider.example/token"}) + compiler := translator.NewCompiler(krt.TestingDummyContext{}, collections, map[translator.HarnessType]translator.HarnessCompiler{translator.HarnessTypeKagent: adapter}, testHarnessImages, defaultPolicyTranslator{uri: "ate-secret://provider.example/default/token"}) result, err := compiler.CompileAgent(t.Context(), inlineAgent(harness, template)) require.NoError(t, err) require.NotNil(t, adapter.input.SummarizerModel) @@ -433,7 +512,7 @@ func TestHarnessBedrockSigningCredentialsPreserved(t *testing.T) { translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), } - result, err := translator.NewCompiler(ctx, collections, compilers, nil, defaultPolicyTranslator{uri: "ate-secret://provider.example/token"}).CompileAgent(t.Context(), inlineAgent(harness, template)) + result, err := translator.NewCompiler(ctx, collections, compilers, nil, defaultPolicyTranslator{uri: "ate-secret://provider.example/default/token"}).CompileAgent(t.Context(), inlineAgent(harness, template)) require.NoError(t, err) for _, variable := range test.env { require.Contains(t, result.Environment, corev1.EnvVar{Name: variable.Name, Value: variable.Value}) diff --git a/go/core/internal/translator/egress_policy_test.go b/go/core/internal/translator/egress_policy_test.go index 861b2d60b7..3403fd727d 100644 --- a/go/core/internal/translator/egress_policy_test.go +++ b/go/core/internal/translator/egress_policy_test.go @@ -37,7 +37,7 @@ func TestPolicyDraftTargets(t *testing.T) { require.Contains(t, draft.Requirements, egressext.AuthenticationRequirement{Origin: "https://api.anthropic.com:443", Header: "x-api-key"}) require.Contains(t, draft.Requirements, egressext.AuthenticationRequirement{Origin: "https://mcp.example.com:443"}) require.Contains(t, draft.Destinations, "http://local.example.com:80") - draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://api.anthropic.com", Header: "x-api-key", URI: "ate-secret://provider.example/token"}) + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://api.anthropic.com", Header: "x-api-key", URI: "ate-secret://provider.example/default/token"}) return draft, nil }) policy, err := ResolvePolicy(input, nil, extension) @@ -84,7 +84,7 @@ func TestPolicyPreservesExplicitAuthentication(t *testing.T) { })) require.NoError(t, err) _, err = ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { - draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: target.Endpoint, Header: target.Header, Prefix: target.Prefix, URI: "ate-secret://provider.example/token"}) + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: target.Endpoint, Header: target.Header, Prefix: target.Prefix, URI: "ate-secret://provider.example/default/token"}) return draft, nil })) require.ErrorContains(t, err, "preserves explicit authentication") @@ -93,7 +93,7 @@ func TestPolicyPreservesExplicitAuthentication(t *testing.T) { } func TestPolicyRejectsInvalidBindings(t *testing.T) { - base := egressext.Credential{Origin: "https://api.openai.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/token"} + base := egressext.Credential{Origin: "https://api.openai.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/default/token"} for _, test := range []struct { name string change func(*egressext.Credential) @@ -139,10 +139,10 @@ func TestPolicyCannotWeakenDraft(t *testing.T) { {"preserve", func(d *egressext.PolicyDraft) { d.Requirements[0].Preserve = true }, "cannot change destinations"}, {"required", func(d *egressext.PolicyDraft) { d.Requirements[0].Required = true }, "cannot change destinations"}, {"remove binding", func(d *egressext.PolicyDraft) { d.Credentials = nil }, "cannot remove or replace"}, - {"replace binding", func(d *egressext.PolicyDraft) { d.Credentials[0].URI = "ate-secret://provider.example/changed" }, "cannot remove or replace"}, + {"replace binding", func(d *egressext.PolicyDraft) { d.Credentials[0].URI = "ate-secret://provider.example/default/changed" }, "cannot remove or replace"}, {"conflict", func(d *egressext.PolicyDraft) { binding := d.Credentials[0] - binding.URI = "ate-secret://provider.example/changed" + binding.URI = "ate-secret://provider.example/default/changed" d.Credentials = append(d.Credentials, binding) }, "conflicting credentials"}, } { @@ -162,6 +162,30 @@ func TestPolicyCannotWeakenDraft(t *testing.T) { } } +func TestPolicyRejectsMCPSecretCredentialConflict(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team"}, + Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://tools.example.com/mcp", HeadersFrom: []v1alpha3.ValueRef{ + {Name: "X-API-Key", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "service-auth", Key: "key"}}, + }}, + }}} + before, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + _, err = ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + draft.Credentials = append(draft.Credentials, egressext.Credential{ + Origin: "https://TOOLS.example.com", Header: "x-api-key", URI: "ate-secret://provider.example/default/token", + }) + return draft, nil + })) + require.ErrorContains(t, err, "conflicting credentials") + after, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + require.Equal(t, before, after) + require.Len(t, after.access.Credentials, 1) + require.Equal(t, "ate-secret://k8s.io/default/team/service-auth/key", after.access.Credentials[0].URI) +} + func TestPolicyAuxiliaryModels(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) input.MemoryModel = &ResolvedModelConfig{Config: &v1alpha3.ModelConfig{ObjectMeta: metav1.ObjectMeta{Name: "embedding", Namespace: "team"}, Spec: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://embeddings.example.com/v1"}}}} @@ -269,7 +293,7 @@ func TestPolicyBindingSourcesRenderSameClients(t *testing.T) { require.Equal(t, map[string]string{"X-Api-Key": CredentialPlaceholder}, secretHeaders, "do not duplicate a Secret header with different casing") input.Root.MCPTools = []ResolvedMCPTool{other} contributedEnvironment, contributedHeaders := render(input, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { - draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://api.anthropic.com:443", Header: "x-api-key", URI: "ate-secret://provider.example/token"}) + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://api.anthropic.com:443", Header: "x-api-key", URI: "ate-secret://provider.example/default/token"}) return draft, nil })) require.Equal(t, environment, contributedEnvironment, "Secret and extension bindings render the same clients") From f0455d64225a1c5395aa3e4eed23743656158d30 Mon Sep 17 00:00:00 2001 From: Cody Hartsook Date: Sat, 17 Oct 2026 12:53:13 -0700 Subject: [PATCH 43/43] test: cover credential extension under live grant admission Signed-off-by: Cody Hartsook --- docs/architecture/credential-injection.md | 9 ++- .../controller/grant_admission_test.go | 57 +++++++++++++++++++ .../internal/controller/preparation_test.go | 12 +++- go/core/internal/egress/live_test.go | 4 ++ .../translator/egress_policy_runtime_test.go | 35 ++---------- 5 files changed, 80 insertions(+), 37 deletions(-) diff --git a/docs/architecture/credential-injection.md b/docs/architecture/credential-injection.md index 7f2fecf207..cc6c35769b 100644 --- a/docs/architecture/credential-injection.md +++ b/docs/architecture/credential-injection.md @@ -93,9 +93,12 @@ validates and pins the bindings, then renders client placeholders. MCP Secret bindings can coexist with extension bindings on different headers. Literal or ConfigMap MCP headers prevent additions at their origin. Model -requirements restrict the header and prefix. Conflicting credentials fail -compilation. A later EgressPolicy conflict withholds all explicit grants and -retains the pinned bindings. EgressPolicy-only destinations are outside the draft. +requirements restrict the header and prefix. An extension binding that conflicts +with a model or MCP binding in the draft fails compilation. A later EgressPolicy +conflict does not block compilation. Pinned integration bindings take precedence, +and admission holds the new grants and reports +[EgressPoliciesApplied](egress-policies.md#preparation-and-lifecycle). +EgressPolicy-only destinations are outside the draft. References require a registered Substrate provider and use `ate-secret:///`, with a lowercase DNS provider, no port, and at diff --git a/go/core/internal/controller/grant_admission_test.go b/go/core/internal/controller/grant_admission_test.go index f3da9908f8..bc467b1f28 100644 --- a/go/core/internal/controller/grant_admission_test.go +++ b/go/core/internal/controller/grant_admission_test.go @@ -1,6 +1,7 @@ package controller import ( + "slices" "strings" "testing" "testing/synctest" @@ -13,12 +14,14 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) func TestGrantAdmissionAcrossSources(t *testing.T) { @@ -142,3 +145,57 @@ func TestEgressConditionSurvivesRuntimeFailureAndBoundsMessages(t *testing.T) { require.LessOrEqual(t, len(condition.Message), 4096) require.False(t, meta.IsStatusConditionTrue(status.Conditions, v1alpha3.AgentConditionReady)) } + +// targetCredential binds its origin once the draft makes that origin a target. +type targetCredential egressext.Credential + +func (c targetCredential) Translate(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + if slices.ContainsFunc(draft.Requirements, func(r egressext.AuthenticationRequirement) bool { return r.Origin == c.Origin }) { + draft.Credentials = append(draft.Credentials, egressext.Credential(c)) + } + return draft, nil +} + +func TestGrantAdmissionHoldsConflictWithExtensionCredential(t *testing.T) { + const tools = "https://tools.example.com:443" + pinned := egress.Credential{Origin: tools, Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/default/token"} + collections, _ := newExtensionTestCollections(t, "gvisor", targetCredential(pinned), + &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "tools"}, Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://tools.example.com/mcp"}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "auth"}, Data: map[string][]byte{"token": []byte("value")}}) + initial := collections.Reconciliations.List()[0] + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(&v1alpha3.EgressPolicy{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, + Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{ + {Origin: "https://x.example.com"}, + {Origin: "https://tools.example.com", Authentication: &v1alpha3.EgressAuthentication{ + Header: "Authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}, Key: "token"}, + }}, + {Origin: "https://new.example.com"}, + }}, + }) + agent := initial.Agent.DeepCopy() + agent.Spec.Template = v1alpha3.AgentTemplateSource{Inline: &v1alpha3.AgentTemplateSpec{Tools: []v1alpha3.ToolBinding{{MCP: &v1alpha3.MCPToolBinding{ + Server: corev1.TypedLocalObjectReference{Kind: "RemoteMCPServer", Name: "tools"}, + }}}}} + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + + var state AgentReconciliation + waitFor(t, func() bool { + current := collections.Reconciliations.GetKey(initial.ResourceName()) + if current == nil || current.Target == nil || current.Target.RevisionID == initial.Target.RevisionID { + return false + } + state = *current + return true + }) + require.Nil(t, state.CompilationFailure) + require.Contains(t, state.Target.Revision.Credentials, pinned, "a live grant conflict must not block compilation") + require.True(t, state.EgressConflict) + condition := meta.FindStatusCondition(statusForAgent(state, agent.Generation, "").Conditions, v1alpha3.AgentConditionEgressPoliciesApplied) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, v1alpha3.EgressPoliciesReasonConflict, condition.Reason) + base, origins := state.Target.egressBase() + previous := egress.Access{Destinations: []string{"https://x.example.com:443"}} + require.Equal(t, previous, retainedEgressGrant(previous, state.EgressGrant, base, origins), "additions stay held") +} diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index 058b38e11d..095fd7f9ee 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -30,6 +30,7 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/dbtest" "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { @@ -539,6 +540,11 @@ func TestInvalidActorTemplateHasNoCompiledTarget(t *testing.T) { } func newPreparationTestCollections(t *testing.T, workerPool string) (Collections, krt.StaticCollection[*kagentv1alpha3.Harness]) { + t.Helper() + return newExtensionTestCollections(t, workerPool, nil) +} + +func newExtensionTestCollections(t *testing.T, workerPool string, extension egressext.PolicyTranslator, objects ...any) (Collections, krt.StaticCollection[*kagentv1alpha3.Harness]) { t.Helper() opts := krt.NewOptionsBuilder(t.Context().Done(), "test-preparation", nil) template := &kagentv1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "assistant", UID: "template-uid"}} @@ -551,11 +557,11 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections WorkerPoolRef: corev1.LocalObjectReference{Name: workerPool}, SnapshotPolicy: kagentv1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}, } harnesses := krt.NewStaticCollection(nil, []*kagentv1alpha3.Harness{runtimeHarness}, opts.WithName("Harnesses")...) - mock := krttest.NewMock(t, []any{ + mock := krttest.NewMock(t, append([]any{ template, &atev1alpha1.WorkerPool{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "gvisor"}, Spec: atev1alpha1.WorkerPoolSpec{SandboxClass: atev1alpha1.SandboxClassGvisor}}, &atev1alpha1.WorkerPool{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "microvm"}, Spec: atev1alpha1.WorkerPoolSpec{SandboxClass: atev1alpha1.SandboxClassMicroVM}}, - }) + }, objects...)) collections := Collections{ EgressPolicies: krt.NewStaticCollection[*kagentv1alpha3.EgressPolicy](nil, nil, opts.WithName("EgressPolicies")...), AgentTemplates: krttest.GetMockCollection[*kagentv1alpha3.AgentTemplate](mock), @@ -573,7 +579,7 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections Harnesses: collections.Harnesses, AgentTemplates: collections.AgentTemplates, ResolvedModelConfigs: collections.ResolvedModelConfigs, RemoteMCPServers: collections.RemoteMCPServers, ConfigMaps: collections.ConfigMaps, Secrets: collections.Secrets, WorkerPools: collections.WorkerPools, EgressPolicies: collections.EgressPolicies, - }, collections.AgentRuntimeObservations, opts, testRuntimeDefaults(), nil, nil) + }, collections.AgentRuntimeObservations, opts, testRuntimeDefaults(), nil, extension) collections.AgentStatuses = newAgentStatuses(collections.Agents, collections.Reconciliations, opts) waitFor(t, func() bool { states := collections.Reconciliations.List() diff --git a/go/core/internal/egress/live_test.go b/go/core/internal/egress/live_test.go index ebe2f2c23d..a9e7e110ab 100644 --- a/go/core/internal/egress/live_test.go +++ b/go/core/internal/egress/live_test.go @@ -39,6 +39,8 @@ func TestFilterConflicts(t *testing.T) { base := canonicalTestAccess(t, []string{binding.Origin}, binding) conflict := binding conflict.URI = "ate-secret://k8s.io/default/team/other/token" + external := binding + external.URI = "ate-secret://provider.example/default/token" const extra = "https://tickets.example.com:443" for _, test := range []struct { name string @@ -49,6 +51,8 @@ func TestFilterConflicts(t *testing.T) { {name: "duplicate", base: base, grants: base, want: base}, {name: "credential conflict keeps network access", base: base, grants: canonicalTestAccess(t, []string{binding.Origin, extra}, conflict), want: canonicalTestAccess(t, []string{binding.Origin, extra}), dropped: []string{binding.Origin}}, + {name: "external credential conflict keeps network access", base: canonicalTestAccess(t, []string{binding.Origin}, external), grants: canonicalTestAccess(t, []string{binding.Origin, extra}, binding), + want: canonicalTestAccess(t, []string{binding.Origin, extra}), dropped: []string{binding.Origin}}, {name: "protocol conflict keeps other grants", base: base, grants: canonicalTestAccess(t, []string{"http://model.example.com:443", extra}), want: canonicalTestAccess(t, []string{extra}), dropped: []string{"http://model.example.com:443"}}, {name: "token passthrough", base: canonicalTestAccess(t, []string{binding.Origin}), tokens: []string{binding.Origin}, diff --git a/go/core/internal/translator/egress_policy_runtime_test.go b/go/core/internal/translator/egress_policy_runtime_test.go index 11d39a4b18..f51ba73731 100644 --- a/go/core/internal/translator/egress_policy_runtime_test.go +++ b/go/core/internal/translator/egress_policy_runtime_test.go @@ -236,13 +236,11 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { key = "AWS_BEARER_TOKEN_BEDROCK" } server := &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Name: "tools", Namespace: "test"}, Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://tools.example.com/mcp"}} - policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Name: "extra", Namespace: "test"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com:8443"}}}} template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help", Tools: []v1alpha3.ToolBinding{{MCP: &v1alpha3.MCPToolBinding{Server: corev1.TypedLocalObjectReference{Kind: "RemoteMCPServer", Name: server.Name}}}}, }} agent := inlineAgent(harness, template) - agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} - collections := mockCollections(t, harness, model, server, policy, defaultWorkerPool()) + collections := mockCollections(t, harness, model, server, defaultWorkerPool()) compile := func(policyTranslator egressext.PolicyTranslator) (*translator.CompileResult, error) { settings := testRuntimeDefaults() ctx := krt.TestingDummyContext{} @@ -256,7 +254,7 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { } result, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/default/token"}) require.NoError(t, err) - require.Len(t, result.Credentials, 2, "live EgressPolicy grants are not credential targets") + require.Len(t, result.Credentials, 2) require.Contains(t, result.Environment, corev1.EnvVar{Name: key, Value: translator.CredentialPlaceholder}) for _, variable := range result.Environment { require.Nil(t, variable.ValueFrom) @@ -301,7 +299,7 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { model.Spec.APIKeySecretKey, secretKey = "", "AWS_BEARER_TOKEN_BEDROCK" } secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "model-auth", Namespace: "test"}, Data: map[string][]byte{secretKey: []byte("model-secret")}} - collections = mockCollections(t, harness, model, server, policy, defaultWorkerPool(), secret) + collections = mockCollections(t, harness, model, server, defaultWorkerPool(), secret) baseline, err := compile(nil) require.NoError(t, err) if test.provider == v1alpha3.ModelProviderBedrock { @@ -325,7 +323,7 @@ func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { t.Run("MCP Secret and extension", func(t *testing.T) { server.Spec.HeadersFrom = []v1alpha3.ValueRef{{Name: "X-API-Key", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "service-auth", Key: "key"}}} serviceSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "service-auth", Namespace: "test"}, Data: map[string][]byte{"key": []byte("service-secret")}} - collections = mockCollections(t, harness, model, server, policy, defaultWorkerPool(), secret, serviceSecret) + collections = mockCollections(t, harness, model, server, defaultWorkerPool(), secret, serviceSecret) combined, err := compile(credentialPolicyTranslator{ Origin: "https://tools.example.com:443", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/default/token", }) @@ -382,31 +380,6 @@ func compiledMCPHeaders(t *testing.T, kind translator.HarnessType, result *trans } } -func TestCompileRejectsEgressPolicyCredentialConflict(t *testing.T) { - harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ - Kagent: &v1alpha3.KagentHarness{}, Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}}, - }} - model := modelConfig() - server := remoteMCPServer("tools", "https://tools.example.com/mcp") - policy, secret := egressPolicy() - policy.Spec.Destinations[0].Origin = "https://tools.example.com" - template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ - ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, - Tools: []v1alpha3.ToolBinding{{MCP: &v1alpha3.MCPToolBinding{Server: corev1.TypedLocalObjectReference{Kind: "RemoteMCPServer", Name: server.Name}}}}, - }} - agent := inlineAgent(harness, template) - agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} - collections := mockCollections(t, harness, model, server, policy, secret, defaultWorkerPool()) - extension := credentialPolicyTranslator{ - Origin: "https://tools.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/default/token", - } - compiler := translator.NewCompiler(krt.TestingDummyContext{}, collections, map[translator.HarnessType]translator.HarnessCompiler{ - translator.HarnessTypeKagent: &testHarnessCompiler{}, - }, testHarnessImages, extension) - _, err := compiler.CompileAgent(t.Context(), agent) - require.ErrorContains(t, err, "conflicting credentials for https://tools.example.com:443 header authorization") -} - func TestPolicyResolvesAuxiliaryModelsBeforeHarnessRendering(t *testing.T) { harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ Kagent: &v1alpha3.KagentHarness{