diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 4502a8a92f..d3f0598df8 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -14,6 +14,7 @@ the agent processes. | `Harness` | Kubernetes (`api.kagent.dev/v1alpha3`) | Runtime implementation, workload, credentials, capacity, and snapshot policy | | `AgentTemplate` | Kubernetes (`api.kagent.dev/v1alpha3`) | Portable agent behavior: model, prompt, tools, skills, and plugins | | `SandboxTemplate` | Kubernetes (`api.kagent.dev/v1alpha3`) | Tools image, environment, and preparation policy for standalone sandboxes | +| `EgressPolicy` | Kubernetes (`api.kagent.dev/v1alpha3`) | Additional HTTP(S) origins and gateway credentials selected by Agents and SandboxTemplates | | prepared revision | PostgreSQL and ate-api | Immutable compiled runtime input and its Substrate ActorTemplate | | `Session` | PostgreSQL, exposed by gRPC | Stable conversation identity and runtime lifecycle | | `Sandbox` | PostgreSQL, exposed by gRPC and MCP | Owner-scoped scratch environment with process/file access and expiration | @@ -95,6 +96,8 @@ cleanup. Session lifecycle workers independently pause/suspend idle Actors. ## Documents +- [Egress policies](egress-policies.md) +- [Runtime credential injection](credential-injection.md) - [Configuration and compilation](configuration-and-compilation.md) - [Runtime and lifecycle](runtime-and-lifecycle.md) - [Client lifecycle retries](../lifecycle-retries.md) diff --git a/docs/architecture/configuration-and-compilation.md b/docs/architecture/configuration-and-compilation.md index 5340355368..b05b685714 100644 --- a/docs/architecture/configuration-and-compilation.md +++ b/docs/architecture/configuration-and-compilation.md @@ -172,9 +172,11 @@ revision. AgentTemplate is shared context with no runtime status. Child template references are resolved under the root Agent's Harness; they do not require separate Agents or matching Harness references. -A failed compile or apply leaves the previous successful revision available. -Sessions pin a prepared revision, so later template edits do not mutate a -running session. +A failed compile or apply retains the previous successful revision, which +remains available for fresh Sessions during pending or failed edits. Initial +creation requires a successful revision. Existing Sessions and creation retries +keep their pinned revision. Explicit egress grants use current policies +independently of runtime preparation. Harness compilers only translate inputs. The controller and Substrate adapter own application and readiness. The central entry points are @@ -391,7 +393,8 @@ no synthetic Kubernetes object is created for an inline template. Create a session with `kagent agent session create --agent assistant -n kagent`. The gRPC create request and ScheduledRun target one `agent` resource reference. -The controller selects that Agent's latest successful revision. Deleting an Agent +The store selects that Agent's latest successful revision, including while an +edit is pending or failed. Deleting an Agent retires its definition; sessions and checkpoints retain their pinned revisions. Recreating the same name creates a new identity and cannot inherit the old Agent's last successful revision. diff --git a/docs/architecture/credential-injection.md b/docs/architecture/credential-injection.md index 3e29ad1f5c..cc6c35769b 100644 --- a/docs/architecture/credential-injection.md +++ b/docs/architecture/credential-injection.md @@ -1,16 +1,21 @@ # Runtime credential injection -Kagent requires Substrate **v0.4.0-alpha1**. The compiler turns ModelConfig API -keys and Secret-backed RemoteMCPServer headers into destination-scoped egress -bindings. Substrate's gateway fetches the referenced Kubernetes Secret and +Kagent requires Substrate **v0.5.0-alpha2**. The runtime compiler turns +ModelConfig API keys and Secret-backed RemoteMCPServer headers into pinned +destination-scoped bindings. Independently, EgressPolicy admission supplies +additional live bindings. Substrate's gateway fetches the referenced Kubernetes Secret and replaces the outgoing HTTP header when the request carries a placeholder. SDKs receive an inert placeholder where they require an API key; real credentials never enter compiled environments, runtime configuration, or revision provenance. -Bindings are persisted with the prepared revision and installed before a -session becomes ready, including retries and checkpoint forks. Secret names, -keys, destinations, and headers affect revision identity. Secret values and -Secret UIDs do not. Rotation is handled by the gateway; its credential cache can +Inferred integration bindings are persisted with the prepared revision; their +Secret names, keys, destinations, and headers affect revision identity. Explicit +EgressPolicy bindings are resolved from current policies and update live actors +without changing that revision. Conflicts hold new grants and report the +independent [EgressPoliciesApplied condition](egress-policies.md#preparation-and-lifecycle); +they do not block runtime compilation. Both are installed before execution, including +creation retries, resume, and checkpoint forks. Secret values and Secret UIDs +do not affect revision identity. Rotation is handled by the gateway; its credential cache can take up to five minutes to refresh, without recompiling or restarting an agent. ## Installation @@ -58,24 +63,48 @@ overrides of these variables are rejected. | RemoteMCPServer Secret-backed header | Configured header; Secret contains its full value | Provider endpoint overrides determine the allowed HTTP(S) origin. Egress rules -match its scheme, DNS name, and port; credential bindings remain scoped to the -hostname and header. Different credentials for the same hostname and header -are rejected, including conflicts between models, memory embeddings, and MCP -servers. Use distinct DNS names for -origins requiring different credentials. IP-address destinations are unsupported -by Substrate egress policies. +and credential bindings both match its scheme, DNS name, and port. Different +credentials for the same origin and header are rejected, including conflicts +between models, memory embeddings, MCP servers, and explicit policies. Distinct +ports on the same DNS name can use different credentials. IP-address destinations +are unsupported by Substrate egress policies. + +The v1alpha3 revision format stores full origins. It assumes a clean installation; +previously persisted hostname-only credential bindings are rejected rather than +reinterpreted across every port on that host. Harness and SandboxTemplate environment entries accept only literal `value` strings, including empty strings. Configure Secret-backed credentials on -ModelConfig or RemoteMCPServer for gateway injection. +ModelConfig, RemoteMCPServer, or [EgressPolicy](egress-policies.md) for gateway injection. AWS IAM signing keys, Google service-account keys, and OAuth client credentials require mechanisms beyond static header injection and are rejected rather than serialized into runtimes. Caller-token passthrough retains its existing behavior. A passthrough model -cannot share a hostname with static gateway credentials, which would override +cannot share an origin with static gateway credentials, which would override the caller's authentication. +## Controller credential extension + +Embedded controllers can set `app.Options.EgressPolicyTranslator` to add credential +references for eligible model and MCP destinations before client rendering. See +[PolicyTranslator](../../go/core/pkg/egressext/policy.go) for the contract. Core +validates and pins the bindings, then renders client placeholders. + +MCP Secret bindings can coexist with extension bindings on different headers. +Literal or ConfigMap MCP headers prevent additions at their origin. Model +requirements restrict the header and prefix. An extension binding that conflicts +with a model or MCP binding in the draft fails compilation. A later EgressPolicy +conflict does not block compilation. Pinned integration bindings take precedence, +and admission holds the new grants and reports +[EgressPoliciesApplied](egress-policies.md#preparation-and-lifecycle). +EgressPolicy-only destinations are outside the draft. + +References require a registered Substrate provider and use +`ate-secret:///`, with a lowercase DNS provider, no port, and at +least two nonempty, unescaped path segments. Clients must send the bound header +to trigger injection. Keyless Gemini support remains a follow-up. + ## Deferred API fields The source retains commented declarations for Harness and SandboxTemplate diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md new file mode 100644 index 0000000000..e2616f9987 --- /dev/null +++ b/docs/architecture/egress-policies.md @@ -0,0 +1,278 @@ +# Egress policies + +`EgressPolicy` grants additional HTTP(S) access to an Agent or standalone +sandbox. An Agent selects policies through `spec.egressPolicyRefs`; a +SandboxTemplate selects them through the same field. References resolve in the +workload's namespace. Policies are managed through the Kubernetes API. + +AgentTemplate owns reusable behavior, and Harness owns how it runs. Neither +selects egress policies. This lets two Agents reuse a template while granting +different access for their tasks. A shared child template contributes its +inferred model, MCP, and artifact destinations, but cannot select additional +policies. The root and all shared children execute in one Actor and share its +complete grant; there is no per-child network isolation. + +## Grant access for a task + +For example, a dependency-installation policy can grant access to registries: + +```yaml +apiVersion: api.kagent.dev/v1alpha3 +kind: EgressPolicy +metadata: + name: node-dependencies + namespace: team-a +spec: + destinations: + - origin: https://registry.npmjs.org + - origin: https://nodejs.org +--- +apiVersion: api.kagent.dev/v1alpha3 +kind: Agent +metadata: + name: code-reviewer + namespace: team-a +spec: + template: + ref: + name: code-review + harnessRef: + name: codex + egressPolicyRefs: + - name: node-dependencies +``` + +An origin is a scheme, DNS name, and optional port. Omitted ports mean 443 for +HTTPS and 80 for HTTP. Every path on that origin is allowed. URL paths, queries, +fragments, user information, IP addresses, and non-HTTP protocols are rejected. +A wildcard replaces exactly one leftmost label: `https://*.example.com` +allows `api.example.com`, but neither `example.com` nor `a.b.example.com`. +`*` and `*.com` are not accepted. Redirects and downloads from other origins +require their own grants. A registry grant is not a guarantee that every +package's install script or release download can reach its dependencies. + +For a standalone sandbox, select the policy on its template: + +```yaml +apiVersion: api.kagent.dev/v1alpha3 +kind: SandboxTemplate +metadata: + name: node-tools + namespace: team-a +spec: + workload: + image: registry.example.com/tools@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + substrate: + workerPoolRef: + name: default + snapshotPolicy: + location: s3://example/snapshots/ + egressPolicyRefs: + - name: node-dependencies +``` + +Each workload can select up to 16 policies, each containing 1–64 destinations. +Grant admission combines them with inferred integration access, removes equivalent +origins, and orders the result deterministically. The final Actor policy is +limited to 256 rules. References are additive; they cannot restrict inferred +model, MCP, artifact, or control-plane access. No references means no additional +access. A standalone sandbox with no references has no allowed egress. + +## Call an API with a credential + +Policies authenticate requests using a header, an optional prefix, and a Secret +in the same namespace. Authentication requires an exact HTTP(S) origin. HTTP +supports in-cluster services whose transport is secured by a service mesh; the gateway injects the +credential without exposing it to the guest: + +```yaml +apiVersion: api.kagent.dev/v1alpha3 +kind: EgressPolicy +metadata: + name: ticket-api + namespace: team-a +spec: + destinations: + - origin: https://tickets.example.com + authentication: + header: Authorization + prefix: "Bearer " + secretKeyRef: + name: ticket-auth + key: token + - origin: https://search.example.com:8443 + authentication: + header: X-API-Key + secretKeyRef: + name: search-auth + key: api-key +``` + +The gateway sets the header value to `prefix + secretValue`. Prefixes are literal: +include the trailing space in `"Bearer "`. Omitting the prefix or setting it to +an empty string uses the Secret value directly, as with an API-key header. + +Create these Secrets separately. Policy resolution validates that each Secret and key +exists, but persists only the reference. Substrate's credential provider fetches +the value and the gateway replaces the request header. Scripts and SDKs must +send that header, using an inert placeholder: + +```sh +curl -H 'Authorization: Bearer kagent-credential-injected' \ + https://tickets.example.com/tickets +curl -H 'X-API-Key: kagent-credential-injected' \ + https://search.example.com:8443/search +``` + +The gateway replaces an existing header; it does not add a missing one. Policy +selection does not automatically configure arbitrary tools or environment +variables. Clients must honor the projected gateway trust bundle. See +[credential injection](credential-injection.md) for provider namespace grants, +CA setup, inferred SDK placeholders, and rotation behavior. + +Bindings use the complete origin and a case-insensitive header name. Different +ports can use different credentials. Identical bindings merge; conflicting +Secret references or prefixes for the same origin/header hold new grants. +Pinned integration bindings take precedence over additional grants. Caller-token passthrough +cannot share its origin with static gateway credentials. Routing, framing, +hop-by-hop, proxy-authentication, and Cookie headers cannot carry credentials. +At most 16 credential headers are supported per origin. + +`RemoteMCPServer.spec.headersFrom` applies the same credential-header rules to +Secret-backed entries during Kubernetes admission. Literal and ConfigMap-backed +headers do not use gateway credential injection. A server may configure up to +64 headers, with names of at most 256 characters. + +An exact credential-bearing rule wins over a matching wildcard rule. Credentials +are never attached to wildcard rules or implicitly copied to other ports or +schemes. HTTP and HTTPS cannot both use the same hostname pattern and port, +because Substrate cannot distinguish that rule selection. + +## Preparation and lifecycle + +KRT watches selected policies and their Secrets independently of runtime +compilation. Each policy resolves separately. A missing or deleting policy, +missing Secret key, or invalid policy contributes no access; grants from the +remaining policies are still considered. Invalid reference names or more than +16 references revoke all additional access. Grant errors never block runtime +compilation or change revision identity. + +Both Agent and SandboxTemplate expose an independent `EgressPoliciesApplied` +condition: + +| Status | Reason | Meaning | +| --- | --- | --- | +| True | `Applied` | All requested grants are admitted. | +| False | `EgressPolicyNotFound` | A referenced policy is missing or deleting. | +| False | `InvalidEgressPolicy` | A policy, Secret reference, or reference list is invalid. | +| False | `EgressGrantConflict` | New grants are held; still-requested, compatible previous grants remain. | + +Messages identify affected policies and conflicts. A conflict takes precedence +when there are also resolution errors. This condition never affects `Ready` or +`ResolvedRefs`; missing policies no longer fail runtime reference resolution. +It reports admission from the controller's current inputs, not acknowledgement +that every actor has received the update. + +Runtime revisions retain the integration access inferred from models and tools, +including credential references and caller-token passthrough constraints. This +pinned access is the revision's base. +The combined additional access from a source's referenced EgressPolicies is its +`EgressGrant`: destinations it may contact and credentials the gateway may inject. +Reconciliation and source definition structs carry it as `EgressGrant egress.Access`. +Revision identity covers compiled runtime inputs and the owning Kubernetes UID. +The full source specification is preserved as provenance for the first preparation +of that revision. Explicit policy grants and policy references do not enter +revision identity. +Changing a grant or its Secret reference does not prepare a new runtime image. +Secret values remain outside compiled inputs; the gateway handles value rotation. + +The elected controller's Agent and SandboxTemplate reconcilers resolve current +grants independently of runtime compilation. A missing Harness, AgentTemplate, +or WorkerPool cannot freeze a revocation. Each preparation queue is the sole +writer of its source definition's grants in PostgreSQL. Definition updates +persist admitted grants and retire replaced Kubernetes UIDs atomically. + +The stored grant set is the last good set: valid on its own and compatible with +the desired revision's base when compilation succeeds. If the complete candidate +conflicts, preparation retains only previously admitted destinations and identical +credential bindings that are still requested, filtering any that conflict with +the desired base. Removals apply immediately; all additions wait until the +conflict is fixed. A new Kubernetes UID inherits no grants. When unrelated +compilation fails, admission checks grants on their own and still applies +revocations. + +A single store transaction records the runtime revision and, when ready, promotes it if +the active source still desires it. Failed runtime preparation preserves +the last successful revision while continuing to update source grants. + +Effective policy is computed when read: the revision's pinned integration access +is merged with grants from its active source UID. Retired or replaced identities +contribute no grants. There is no separately published per-revision policy and +no notification needed when a revision is recorded: every read joins against +the current persisted grants, including for newly recorded revisions. + +Every preparation persists the source definition, then pages its allocated actors +across all retained revisions and Kubernetes UIDs and applies effective policy +with at most 16 actor updates in parallel. Unchanged policies skip the Substrate +write. Failures return to the existing preparation queue, which retries the whole +source with exponential backoff even when grants have not changed. Slow actor +updates therefore delay further work on that preparation queue. Startup replays +every Agent and SandboxTemplate through preparation; no separate actor sweep is +needed. +After its input cache syncs, Agent preparation also enqueues stored definitions +whose Agents are absent from watched namespaces. Retired identities are included +only while they have non-deleted instances, so a restart retries interrupted +revocations without re-enqueueing every historic source. Agent deletion does not +need a finalizer. SandboxTemplate preparation retains its existing +finalizer until retirement and actor updates succeed. + +Running and suspended actors receive updates without replacement or a new +invocation. Missing or invalid policies and Secrets revoke only the affected +policies' grants. Actors on older revisions can have different pinned access: +their effective policy drops only the destinations or credentials that conflict +with that base, caller-token origins, or combined limits. Nonconflicting grants +remain. The pinned base always wins; dropping a credential retains its network +destination if the destination itself is compatible. Missing revisions or malformed persisted inputs return errors and +block execution. + +New Sessions and sandboxes pin the latest successful runtime revision, including +while a configuration edit is unresolved, preparing, or failed. Initial creation +still requires a successful revision. Existing instances and checkpoint forks +retain their runtime configuration. + +Create, creation retries, and resume load and apply the effective policy from +persisted revision inputs and source grants. They do not read or translate +Kubernetes policy. The workflow creates a suspended Actor and installs its +`default` egress policy before it can +run; a fork cannot restore an old policy from a snapshot. Applications read the +Substrate policy version before loading intended policy, then write using its UID +and version preconditions. Conflicts reload both. After applying, they recheck +intended policy to catch updates whose actor discovery raced with creation. +Missing revisions or malformed persisted inputs block execution and are retried. +No database transaction or lock is held across Substrate calls. + +Policy convergence is asynchronous: a Kubernetes write is not a synchronous +revocation barrier. Once the gateway observes the changed policy, subsequent +requests use it. An update does not cancel a request already in flight. Golden +Actor preparation and artifact-startup egress are separate from this API's runtime +Actor access. + +## Authorization and scope + +Policies are namespace-scoped configuration. Administrators can grant separate +RBAC permissions to author EgressPolicy objects; kagent's controller needs only +get/list/watch. Workload authors who can select a policy in that namespace can +use its grant. There is no separate per-policy `use` authorization check. Treat +the namespace as the sharing boundary for selectable policies and credentials. + +The same trust-boundary consideration applies to existing ModelConfig and +RemoteMCPServer Secret references. This resource does not turn the namespace +into a per-workload Secret isolation boundary. Credential injection keeps values +out of compiled inputs; it does not prevent an allowed destination from echoing +credentials back to a client. Do not grant credentials to destinations that +should not receive or disclose them. + +This API intentionally does not expose Substrate rule effects, raw TCP/SSH, +TLS passthrough, OAuth exchange, request signing, cross-namespace references, +or unrestricted public-internet access. Substrate remains the enforcement +backend; kagent exposes task-oriented reusable grants and static authentication. diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 53fb8a1090..f6b10efe95 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -6,7 +6,14 @@ Kubernetes resource. ## Creation and state -Creation selects the latest successful revision for the Agent, creates a deterministic Actor initially suspended, and marks the session +Fresh creation pins the Agent's latest successful runtime revision. Pending or +failed edits leave that revision available; an Agent with no successful revision +cannot create sessions. Creation retries retain the existing session's pinned +revision. Explicit egress grants are applied from the controller's persisted +intended policy before execution. Lifecycle +operations do not read or translate Kubernetes policy. + +The control plane creates a deterministic Actor initially suspended and marks the session ready after Substrate accepts it. Readiness of the image was already established while preparing the ate-api ActorTemplate; Session creation does not resume an Actor merely to probe `/readyz`. diff --git a/docs/architecture/sandboxes.md b/docs/architecture/sandboxes.md index c2aabd6673..3a323cb83c 100644 --- a/docs/architecture/sandboxes.md +++ b/docs/architecture/sandboxes.md @@ -97,8 +97,11 @@ finalizer, including when deletion starts while the controller is offline. The controller advances the latest-successful revision only after the golden snapshot is ready and reports preparation through the template's `Ready` condition and observed generation. Failed preparation preserves the previous -successful revision. Existing Sandboxes retain their revision when a template -changes. Template deletion retires preparation; retained instances continue to +successful revision, which remains available for fresh creation while edits +are unresolved, pending, or failed. Initial creation requires a successful +revision. Existing Sandboxes and +creation retries retain their revision when a template changes. +Template deletion retires preparation; retained instances continue to pin the runtime artifacts they need. Sandbox execution is enabled by default. Lifecycle calls run on the serving API @@ -113,8 +116,14 @@ TTL. Helm supplies these through the controller ConfigMap. Defaults are one CPU, 1 GiB memory, a one-hour TTL, and a 24-hour maximum TTL. Activity does not extend expiration. -Sandboxes have no allowed egress destinations. A follow-up will add destination -configuration to `SandboxTemplate` and pin it with each prepared revision. +Sandboxes have no allowed egress destinations by default. Select same-namespace +[EgressPolicy resources](egress-policies.md) through `SandboxTemplate.spec.egressPolicyRefs` +to grant additional HTTP(S) origins and static gateway credentials. The controller +persists current grants on the source definition; effective policy is merged with +pinned revision access when read. +Policy edits update running and suspended sandboxes without replacing their Actor. +Creation and resume apply that effective policy before execution, without +translating Kubernetes inputs. ## Lifecycle and recovery diff --git a/go/adk/cmd/main.go b/go/adk/cmd/main.go index 1a18a9b0cc..bfa5d6b018 100644 --- a/go/adk/cmd/main.go +++ b/go/adk/cmd/main.go @@ -28,7 +28,6 @@ import ( const ( defaultPluginPackagesRoot = "/plugins" - defaultSkillsRoot = "/skills" defaultPluginDataRoot = "/data/plugins" ) @@ -73,7 +72,7 @@ func run(logLevel, host, port, configDir string) error { logging.IntoContext(context.Background(), logger), agentConfig, config.AgentPluginPaths{ Packages: defaultPluginPackagesRoot, - Skills: defaultSkillsRoot, + Skills: config.DefaultSkillsRoot, Data: defaultPluginDataRoot, }, ); err != nil { diff --git a/go/adk/pkg/config/agentplugins.go b/go/adk/pkg/config/agentplugins.go index db0ab56f01..f4bf1adb19 100644 --- a/go/adk/pkg/config/agentplugins.go +++ b/go/adk/pkg/config/agentplugins.go @@ -9,6 +9,9 @@ import ( "github.com/kagent-dev/kagent/go/core/pkg/agentplugins" ) +// DefaultSkillsRoot is where the ADK installs root-agent skills. +const DefaultSkillsRoot = "/skills" + // AgentPluginPaths contains the ADK runtime destinations for Agent Plugin // packages, selected skills, and mutable MCP server data. type AgentPluginPaths struct { diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/api_client.go b/go/api/clientset/versioned/typed/api/v1alpha3/api_client.go index 97de970b4a..901e7cf393 100644 --- a/go/api/clientset/versioned/typed/api/v1alpha3/api_client.go +++ b/go/api/clientset/versioned/typed/api/v1alpha3/api_client.go @@ -14,6 +14,7 @@ type ApiV1alpha3Interface interface { RESTClient() rest.Interface AgentsGetter AgentTemplatesGetter + EgressPoliciesGetter HarnessesGetter ModelConfigsGetter RemoteMCPServersGetter @@ -33,6 +34,10 @@ func (c *ApiV1alpha3Client) AgentTemplates(namespace string) AgentTemplateInterf return newAgentTemplates(c, namespace) } +func (c *ApiV1alpha3Client) EgressPolicies(namespace string) EgressPolicyInterface { + return newEgressPolicies(c, namespace) +} + func (c *ApiV1alpha3Client) Harnesses(namespace string) HarnessInterface { return newHarnesses(c, namespace) } diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/egresspolicy.go b/go/api/clientset/versioned/typed/api/v1alpha3/egresspolicy.go new file mode 100644 index 0000000000..140295069e --- /dev/null +++ b/go/api/clientset/versioned/typed/api/v1alpha3/egresspolicy.go @@ -0,0 +1,52 @@ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha3 + +import ( + context "context" + + scheme "github.com/kagent-dev/kagent/go/api/clientset/versioned/scheme" + apiv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// EgressPoliciesGetter has a method to return a EgressPolicyInterface. +// A group's client should implement this interface. +type EgressPoliciesGetter interface { + EgressPolicies(namespace string) EgressPolicyInterface +} + +// EgressPolicyInterface has methods to work with EgressPolicy resources. +type EgressPolicyInterface interface { + Create(ctx context.Context, egressPolicy *apiv1alpha3.EgressPolicy, opts v1.CreateOptions) (*apiv1alpha3.EgressPolicy, error) + Update(ctx context.Context, egressPolicy *apiv1alpha3.EgressPolicy, opts v1.UpdateOptions) (*apiv1alpha3.EgressPolicy, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*apiv1alpha3.EgressPolicy, error) + List(ctx context.Context, opts v1.ListOptions) (*apiv1alpha3.EgressPolicyList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *apiv1alpha3.EgressPolicy, err error) + EgressPolicyExpansion +} + +// egressPolicies implements EgressPolicyInterface +type egressPolicies struct { + *gentype.ClientWithList[*apiv1alpha3.EgressPolicy, *apiv1alpha3.EgressPolicyList] +} + +// newEgressPolicies returns a EgressPolicies +func newEgressPolicies(c *ApiV1alpha3Client, namespace string) *egressPolicies { + return &egressPolicies{ + gentype.NewClientWithList[*apiv1alpha3.EgressPolicy, *apiv1alpha3.EgressPolicyList]( + "egresspolicies", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *apiv1alpha3.EgressPolicy { return &apiv1alpha3.EgressPolicy{} }, + func() *apiv1alpha3.EgressPolicyList { return &apiv1alpha3.EgressPolicyList{} }, + ), + } +} diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_api_client.go b/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_api_client.go index 19d7ba79b7..931f3f6f1a 100644 --- a/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_api_client.go +++ b/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_api_client.go @@ -20,6 +20,10 @@ func (c *FakeApiV1alpha3) AgentTemplates(namespace string) v1alpha3.AgentTemplat return newFakeAgentTemplates(c, namespace) } +func (c *FakeApiV1alpha3) EgressPolicies(namespace string) v1alpha3.EgressPolicyInterface { + return newFakeEgressPolicies(c, namespace) +} + func (c *FakeApiV1alpha3) Harnesses(namespace string) v1alpha3.HarnessInterface { return newFakeHarnesses(c, namespace) } diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_egresspolicy.go b/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_egresspolicy.go new file mode 100644 index 0000000000..6375df89c7 --- /dev/null +++ b/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_egresspolicy.go @@ -0,0 +1,36 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + apiv1alpha3 "github.com/kagent-dev/kagent/go/api/clientset/versioned/typed/api/v1alpha3" + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + gentype "k8s.io/client-go/gentype" +) + +// fakeEgressPolicies implements EgressPolicyInterface +type fakeEgressPolicies struct { + *gentype.FakeClientWithList[*v1alpha3.EgressPolicy, *v1alpha3.EgressPolicyList] + Fake *FakeApiV1alpha3 +} + +func newFakeEgressPolicies(fake *FakeApiV1alpha3, namespace string) apiv1alpha3.EgressPolicyInterface { + return &fakeEgressPolicies{ + gentype.NewFakeClientWithList[*v1alpha3.EgressPolicy, *v1alpha3.EgressPolicyList]( + fake.Fake, + namespace, + v1alpha3.SchemeGroupVersion.WithResource("egresspolicies"), + v1alpha3.SchemeGroupVersion.WithKind("EgressPolicy"), + func() *v1alpha3.EgressPolicy { return &v1alpha3.EgressPolicy{} }, + func() *v1alpha3.EgressPolicyList { return &v1alpha3.EgressPolicyList{} }, + func(dst, src *v1alpha3.EgressPolicyList) { dst.ListMeta = src.ListMeta }, + func(list *v1alpha3.EgressPolicyList) []*v1alpha3.EgressPolicy { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1alpha3.EgressPolicyList, items []*v1alpha3.EgressPolicy) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/generated_expansion.go b/go/api/clientset/versioned/typed/api/v1alpha3/generated_expansion.go index bc2146b2a4..da4673fc10 100644 --- a/go/api/clientset/versioned/typed/api/v1alpha3/generated_expansion.go +++ b/go/api/clientset/versioned/typed/api/v1alpha3/generated_expansion.go @@ -6,6 +6,8 @@ type AgentExpansion interface{} type AgentTemplateExpansion interface{} +type EgressPolicyExpansion interface{} + type HarnessExpansion interface{} type ModelConfigExpansion interface{} diff --git a/go/api/v1alpha3/agent_types.go b/go/api/v1alpha3/agent_types.go index 67bae21fba..d44f398b4f 100644 --- a/go/api/v1alpha3/agent_types.go +++ b/go/api/v1alpha3/agent_types.go @@ -52,6 +52,17 @@ type AgentSpec struct { // +kubebuilder:validation:XValidation:rule="has(self.name) && self.name != ''",message="harnessRef.name must not be empty" // +required HarnessRef corev1.LocalObjectReference `json:"harnessRef"` + + // EgressPolicyRefs grants additional access to the whole runtime, including + // shared subagents. Policies resolve in this Agent's namespace. Integration + // access inferred from models and tools is always included. Changes update + // existing Sessions without a new invocation or runtime restart. + // +optional + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:XValidation:rule="self.all(ref, has(ref.name) && ref.name != '')",message="egressPolicyRefs requires non-empty names" + // +listType=map + // +listMapKey=name + EgressPolicyRefs []corev1.LocalObjectReference `json:"egressPolicyRefs,omitempty"` } // AgentTemplateSource selects portable behavior inline or by local reference. @@ -69,10 +80,11 @@ type AgentTemplateSource struct { } const ( - AgentConditionAccepted = "Accepted" - AgentConditionResolvedRefs = "ResolvedRefs" - AgentConditionCompatible = "Compatible" - AgentConditionReady = "Ready" + AgentConditionEgressPoliciesApplied = "EgressPoliciesApplied" + AgentConditionAccepted = "Accepted" + AgentConditionResolvedRefs = "ResolvedRefs" + AgentConditionCompatible = "Compatible" + AgentConditionReady = "Ready" ) // AgentStatus reports compilation and preparation of an Agent. @@ -96,7 +108,11 @@ type AgentStatus struct { // +listType=set // +optional Warnings []string `json:"warnings,omitempty"` - // +kubebuilder:validation:MaxItems=4 + // Conditions report runtime readiness and independent egress grant admission. + // EgressPoliciesApplied is False for EgressPolicyNotFound, InvalidEgressPolicy, + // or EgressGrantConflict; Applied means all grants are admitted. Grant issues + // do not affect Ready. + // +kubebuilder:validation:MaxItems=5 // +listType=map // +listMapKey=type // +optional diff --git a/go/api/v1alpha3/common_types.go b/go/api/v1alpha3/common_types.go index 481189eb0f..80a19120f0 100644 --- a/go/api/v1alpha3/common_types.go +++ b/go/api/v1alpha3/common_types.go @@ -157,6 +157,7 @@ func (s *ValueSource) Resolve(ctx context.Context, client client.Client, namespa // +kubebuilder:validation:ExactlyOneOf=value;valueFrom type ValueRef struct { // +required + // +kubebuilder:validation:MaxLength=256 Name string `json:"name"` // +optional Value string `json:"value,omitempty"` diff --git a/go/api/v1alpha3/configuration_crd_cel_test.go b/go/api/v1alpha3/configuration_crd_cel_test.go index 02d17d4b57..ab236cd4b7 100644 --- a/go/api/v1alpha3/configuration_crd_cel_test.go +++ b/go/api/v1alpha3/configuration_crd_cel_test.go @@ -56,7 +56,7 @@ func TestConfigurationCRDValidation(t *testing.T) { names = append(names, resource.Name) } } - require.ElementsMatch(t, []string{"agents", "agenttemplates", "harnesses", "modelconfigs", "modelproviderconfigs", "remotemcpservers", "sandboxtemplates"}, names) + require.ElementsMatch(t, []string{"agents", "agenttemplates", "egresspolicies", "harnesses", "modelconfigs", "modelproviderconfigs", "remotemcpservers", "sandboxtemplates"}, names) _, err = discoveryClient.ServerResourcesForGroupVersion("kagent.dev/v1alpha3") require.True(t, apierrors.IsNotFound(err), "the new CRDs must not publish the legacy group: %v", err) }) @@ -462,6 +462,9 @@ func TestConfigurationCRDValidation(t *testing.T) { }) } + t.Run("egress policies", func(t *testing.T) { testEgressPolicyValidation(t, cl, namespace) }) + t.Run("MCP credential headers", func(t *testing.T) { testMCPCredentialHeaderValidation(t, cl, namespace) }) + t.Run("literal runtime environment", func(t *testing.T) { harness := validHarness(namespace, "env-harness", HarnessSpec{Kagent: &KagentHarness{}}) for _, resource := range []struct { diff --git a/go/api/v1alpha3/egresspolicy_cel_test.go b/go/api/v1alpha3/egresspolicy_cel_test.go new file mode 100644 index 0000000000..d8f0fa0bd4 --- /dev/null +++ b/go/api/v1alpha3/egresspolicy_cel_test.go @@ -0,0 +1,103 @@ +package v1alpha3 + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func testEgressPolicyValidation(t *testing.T, cl ctrlclient.Client, namespace string) { + secret := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}, Key: "token"} + bearer := &EgressAuthentication{Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: secret} + for i, tc := range []struct { + name string + destination EgressDestination + reject bool + }{ + {"HTTPS", EgressDestination{Origin: "https://api.example.com"}, false}, + {"HTTP", EgressDestination{Origin: "http://registry.example.com:8080"}, false}, + {"wildcard", EgressDestination{Origin: "https://*.example.com"}, false}, + {"bearer", EgressDestination{Origin: "https://api.example.com:8443", Authentication: bearer}, false}, + {"API key", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "X-API-Key", SecretKeyRef: secret}}, false}, + {"empty prefix", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "X-API-Key", Prefix: new(""), SecretKeyRef: secret}}, false}, + {"custom prefix", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "Authorization", Prefix: new("Token "), SecretKeyRef: secret}}, false}, + {"HTTP auth", EgressDestination{Origin: "http://api.example.com", Authentication: bearer}, false}, + {"HTTP wildcard auth", EgressDestination{Origin: "http://*.example.com", Authentication: bearer}, true}, + {"wildcard auth", EgressDestination{Origin: "https://*.example.com", Authentication: bearer}, true}, + {"empty auth", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{}}, true}, + {"missing header", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{SecretKeyRef: secret}}, true}, + {"path", EgressDestination{Origin: "https://api.example.com/v1"}, true}, + {"userinfo", EgressDestination{Origin: "https://user@api.example.com"}, true}, + {"query", EgressDestination{Origin: "https://api.example.com?"}, true}, + {"IP", EgressDestination{Origin: "https://192.0.2.1"}, true}, + {"all hosts", EgressDestination{Origin: "https://*"}, true}, + {"top level wildcard", EgressDestination{Origin: "https://*.com"}, true}, + {"interior wildcard", EgressDestination{Origin: "https://api.*.com"}, true}, + {"port zero", EgressDestination{Origin: "https://api.example.com:0"}, true}, + {"port overflow", EgressDestination{Origin: "https://api.example.com:65536"}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + policy := &EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: fmt.Sprintf("egress-%d", i)}, Spec: EgressPolicySpec{Destinations: []EgressDestination{tc.destination}}} + err := cl.Create(t.Context(), policy) + if tc.reject { + require.Error(t, err) + } else { + require.NoError(t, err) + require.NoError(t, cl.Get(t.Context(), ctrlclient.ObjectKeyFromObject(policy), policy)) + require.Equal(t, tc.destination.Authentication, policy.Spec.Destinations[0].Authentication) + } + }) + } + for i, header := range []string{"Host", "cOnNeCtIoN", "Content-Length", "Transfer-Encoding", "Cookie", "Proxy-Authorization", "X-Key\r\nInjected"} { + policy := &EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: fmt.Sprintf("bad-header-%d", i)}, Spec: EgressPolicySpec{Destinations: []EgressDestination{{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: header, SecretKeyRef: secret}}}}} + require.Error(t, cl.Create(t.Context(), policy), header) + } + for i, prefix := range []string{"Bearer\r", "Bearer\n", "\x00", "\x08", "\x0b", "\x1f", "\x7f", strings.Repeat("x", 257)} { + policy := &EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: fmt.Sprintf("bad-prefix-%d", i)}, Spec: EgressPolicySpec{Destinations: []EgressDestination{{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "Authorization", Prefix: new(prefix), SecretKeyRef: secret}}}}} + require.Error(t, cl.Create(t.Context(), policy), "prefix %q", prefix) + } + for i, ref := range []corev1.SecretKeySelector{{Key: "token"}, {LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}}, {LocalObjectReference: secret.LocalObjectReference, Key: "token", Optional: new(true)}} { + policy := &EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: fmt.Sprintf("bad-secret-%d", i)}, Spec: EgressPolicySpec{Destinations: []EgressDestination{{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "Authorization", SecretKeyRef: ref}}}}} + require.Error(t, cl.Create(t.Context(), policy)) + } + for _, kind := range []string{"Agent", "SandboxTemplate", "AgentTemplate", "Harness"} { + t.Run(kind+" attachment", func(t *testing.T) { + var object ctrlclient.Object + switch kind { + case "Agent": + object = &Agent{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "egress-agent"}, Spec: AgentSpec{Template: AgentTemplateSource{Ref: &corev1.LocalObjectReference{Name: "behavior"}}, HarnessRef: corev1.LocalObjectReference{Name: "runner"}}} + case "SandboxTemplate": + object = sandboxTemplateForValidation(namespace, "egress-sandbox", nil) + case "AgentTemplate": + object = validAgentTemplate(namespace, "egress-template", nil) + case "Harness": + object = validHarness(namespace, "egress-harness", HarnessSpec{Kagent: &KagentHarness{}}) + } + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object) + require.NoError(t, err) + resource := &unstructured.Unstructured{Object: data} + resource.SetAPIVersion(GroupVersion.String()) + resource.SetKind(kind) + require.NoError(t, unstructured.SetNestedSlice(data, []any{map[string]any{"name": "tickets"}}, "spec", "egressPolicyRefs")) + err = cl.Create(t.Context(), resource, &ctrlclient.CreateOptions{FieldValidation: metav1.FieldValidationStrict}) + if kind == "AgentTemplate" || kind == "Harness" { + require.ErrorContains(t, err, "unknown field") + return + } + require.NoError(t, err) + require.NoError(t, cl.Get(t.Context(), ctrlclient.ObjectKeyFromObject(resource), resource)) + refs, _, err := unstructured.NestedSlice(resource.Object, "spec", "egressPolicyRefs") + require.NoError(t, err) + require.Equal(t, []any{map[string]any{"name": "tickets"}}, refs) + require.NoError(t, unstructured.SetNestedSlice(resource.Object, []any{map[string]any{"name": ""}}, "spec", "egressPolicyRefs")) + require.Error(t, cl.Update(t.Context(), resource)) + }) + } +} diff --git a/go/api/v1alpha3/egresspolicy_types.go b/go/api/v1alpha3/egresspolicy_types.go new file mode 100644 index 0000000000..deac70d7f2 --- /dev/null +++ b/go/api/v1alpha3/egresspolicy_types.go @@ -0,0 +1,97 @@ +// Copyright 2026 The kagent Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha3 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// EgressPolicySpec grants additional HTTP(S) access to workloads in its namespace. +type EgressPolicySpec struct { + // Destinations are additional origins the workload may reach. + // +required + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 + // +listType=map + // +listMapKey=origin + Destinations []EgressDestination `json:"destinations"` +} + +// EgressDestination grants access to one HTTP(S) origin, including all its paths. +// +kubebuilder:validation:XValidation:rule="!has(self.authentication) || !self.origin.contains('*')",message="authentication requires an exact HTTP(S) origin" +type EgressDestination struct { + // Origin is a scheme, DNS hostname, and optional port. A wildcard replaces + // exactly one leftmost DNS label, for example https://*.example.com. + // +required + // +kubebuilder:validation:MaxLength=270 + // +kubebuilder:validation:Pattern=`^https?://(\*\.([a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?\.)+|([a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?\.)*)[a-z]([-a-z0-9]{0,61}[a-z0-9])?(:([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?$` + Origin string `json:"origin"` + // Authentication replaces a placeholder header at the gateway. Requests + // must send the header; its credential value is never supplied to the runtime. + // +optional + Authentication *EgressAuthentication `json:"authentication,omitempty"` +} + +// EgressAuthentication replaces a request header with a prefix and Secret value. +type EgressAuthentication struct { + // Header is the request header carrying the credential, for example + // Authorization or X-API-Key. + // +required + // +kubebuilder:validation:MaxLength=256 + // +kubebuilder:validation:Pattern=`^[!#$%&'*+.^_|~0-9A-Za-z-]+$` + // +kubebuilder:validation:XValidation:rule="!(self.lowerAscii() in ['host', 'connection', 'content-length', 'transfer-encoding', 'te', 'trailer', 'upgrade', 'keep-alive', 'proxy-authorization', 'proxy-authenticate', 'cookie'])",message="this header cannot carry an egress credential" + Header string `json:"header"` + // Prefix is prepended verbatim to the Secret value, for example "Bearer ". + // Include any separating space. Omitted or empty means no prefix. + // +optional + // +kubebuilder:validation:MaxLength=256 + // +kubebuilder:validation:Pattern=`^[^\x00-\x08\x0a-\x1f\x7f]*$` + Prefix *string `json:"prefix,omitempty"` + // SecretKeyRef selects a required key in the EgressPolicy's namespace. + // +required + // +kubebuilder:validation:XValidation:rule="has(self.name) && self.name != '' && self.key != ''",message="secretKeyRef requires a name and key" + // +kubebuilder:validation:XValidation:rule="!has(self.optional) || !self.optional",message="egress credentials cannot be optional" + SecretKeyRef corev1.SecretKeySelector `json:"secretKeyRef"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=egresspolicies,singular=egresspolicy,scope=Namespaced,categories=kagent + +// EgressPolicy is a reusable grant of additional network access. Workloads in its +// namespace select it explicitly; changes update their existing runtime Actors. +type EgressPolicy struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + // +required + Spec EgressPolicySpec `json:"spec"` +} + +// +kubebuilder:object:root=true + +// EgressPolicyList contains EgressPolicy resources. +type EgressPolicyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EgressPolicy `json:"items"` +} + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, &EgressPolicy{}, &EgressPolicyList{}) + return nil + }) +} + +// Reasons for the independent EgressPoliciesApplied condition on Agents and +// SandboxTemplates. Grant resolution and admission never block runtime readiness. +const ( + EgressPoliciesReasonApplied = "Applied" + EgressPoliciesReasonNotFound = "EgressPolicyNotFound" + EgressPoliciesReasonInvalidEgressPolicy = "InvalidEgressPolicy" + EgressPoliciesReasonConflict = "EgressGrantConflict" +) diff --git a/go/api/v1alpha3/remotemcpserver_cel_test.go b/go/api/v1alpha3/remotemcpserver_cel_test.go new file mode 100644 index 0000000000..44a0020bfd --- /dev/null +++ b/go/api/v1alpha3/remotemcpserver_cel_test.go @@ -0,0 +1,90 @@ +package v1alpha3 + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func testMCPCredentialHeaderValidation(t *testing.T, cl ctrlclient.Client, namespace string) { + t.Helper() + for _, tc := range []struct { + name string + header string + reject string + }{ + {name: "authorization", header: "Authorization"}, + {name: "api-key", header: "X-API-Key"}, + {name: "token-characters", header: "X-!#$%&'*+-.^_`|~0123456789"}, + {name: "max-length", header: strings.Repeat("x", 256)}, + {name: "empty", reject: "valid HTTP header name"}, + {name: "too-long", header: strings.Repeat("x", 257), reject: "may not be more than 256"}, + {name: "whitespace", header: "X Key", reject: "valid HTTP header name"}, + {name: "newline", header: "X-Key\r\nInjected", reject: "valid HTTP header name"}, + {name: "colon", header: "X:Key", reject: "valid HTTP header name"}, + {name: "backslash", header: "X\\Key", reject: "valid HTTP header name"}, + {name: "unicode", header: "X-名", reject: "valid HTTP header name"}, + {name: "host", header: "Host", reject: "cannot carry an egress credential"}, + {name: "connection", header: "cOnNeCtIoN", reject: "cannot carry an egress credential"}, + {name: "content-length", header: "Content-Length", reject: "cannot carry an egress credential"}, + {name: "transfer-encoding", header: "Transfer-Encoding", reject: "cannot carry an egress credential"}, + {name: "te", header: "TE", reject: "cannot carry an egress credential"}, + {name: "trailer", header: "Trailer", reject: "cannot carry an egress credential"}, + {name: "upgrade", header: "Upgrade", reject: "cannot carry an egress credential"}, + {name: "keep-alive", header: "Keep-Alive", reject: "cannot carry an egress credential"}, + {name: "proxy-authorization", header: "Proxy-Authorization", reject: "cannot carry an egress credential"}, + {name: "proxy-authenticate", header: "Proxy-Authenticate", reject: "cannot carry an egress credential"}, + {name: "cookie", header: "Cookie", reject: "cannot carry an egress credential"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := &RemoteMCPServer{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "mcp-header-" + tc.name}, + Spec: RemoteMCPServerSpec{ + Description: "MCP credential header validation", + URL: "https://mcp.example.com/mcp", + HeadersFrom: []ValueRef{{Name: tc.header, ValueFrom: &ValueSource{Type: SecretValueSource, Name: "auth", Key: "token"}}}, + }, + } + err := cl.Create(t.Context(), server) + if tc.reject != "" { + require.ErrorContains(t, err, tc.reject) + require.ErrorContains(t, err, "spec.headersFrom[0].name") + return + } + require.NoError(t, err) + require.NoError(t, cl.Get(t.Context(), ctrlclient.ObjectKeyFromObject(server), server)) + require.Equal(t, tc.header, server.Spec.HeadersFrom[0].Name) + server.Spec.HeadersFrom[0].Name = "Cookie" + require.ErrorContains(t, cl.Update(t.Context(), server), "cannot carry an egress credential") + }) + } + for _, source := range []ValueRef{ + {Name: "Cookie", Value: "session=example"}, + {Name: "Cookie", ValueFrom: &ValueSource{Type: ConfigMapValueSource, Name: "headers", Key: "cookie"}}, + } { + server := &RemoteMCPServer{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, GenerateName: "mcp-ordinary-header-"}, + Spec: RemoteMCPServerSpec{Description: "Ordinary headers are not gateway credentials", URL: "https://mcp.example.com/mcp", HeadersFrom: []ValueRef{source}}, + } + require.NoError(t, cl.Create(t.Context(), server)) + server.Spec.HeadersFrom[0] = ValueRef{Name: "Cookie", ValueFrom: &ValueSource{Type: SecretValueSource, Name: "auth", Key: "token"}} + require.ErrorContains(t, cl.Update(t.Context(), server), "cannot carry an egress credential") + } + t.Run("header count", func(t *testing.T) { + headers := make([]ValueRef, 64) + for i := range headers { + headers[i] = ValueRef{Name: fmt.Sprintf("X-Header-%d", i), Value: "example"} + } + server := &RemoteMCPServer{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "mcp-header-count"}, + Spec: RemoteMCPServerSpec{Description: "Header count validation", URL: "https://mcp.example.com/mcp", HeadersFrom: headers}, + } + require.NoError(t, cl.Create(t.Context(), server)) + server.Spec.HeadersFrom = append(server.Spec.HeadersFrom, ValueRef{Name: "X-Extra", Value: "example"}) + require.ErrorContains(t, cl.Update(t.Context(), server), "must have at most 64 items") + }) +} diff --git a/go/api/v1alpha3/remotemcpserver_types.go b/go/api/v1alpha3/remotemcpserver_types.go index 41a4c5c885..4fb2b9ce06 100644 --- a/go/api/v1alpha3/remotemcpserver_types.go +++ b/go/api/v1alpha3/remotemcpserver_types.go @@ -48,7 +48,13 @@ type RemoteMCPServerSpec struct { // +kubebuilder:validation:MinLength=1 // +required URL string `json:"url"` + // HeadersFrom supplies request headers. Secret-backed headers are injected by + // the egress gateway and cannot use routing, framing, hop-by-hop, + // proxy-authentication, or Cookie headers. At most 64 headers may be supplied. // +optional + // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:items:XValidation:rule="!has(self.valueFrom) || self.valueFrom.type != 'Secret' || self.name.matches('^[!#$%&\\\\x27*+.^_`|~0-9A-Za-z-]+$')",message="Secret-backed headers require a valid HTTP header name",fieldPath=".name" + // +kubebuilder:validation:items:XValidation:rule="!has(self.valueFrom) || self.valueFrom.type != 'Secret' || !(self.name.lowerAscii() in ['host', 'connection', 'content-length', 'transfer-encoding', 'te', 'trailer', 'upgrade', 'keep-alive', 'proxy-authorization', 'proxy-authenticate', 'cookie'])",message="this header cannot carry an egress credential",fieldPath=".name" HeadersFrom []ValueRef `json:"headersFrom,omitempty"` // +optional // +kubebuilder:default="30s" diff --git a/go/api/v1alpha3/sandboxtemplate_types.go b/go/api/v1alpha3/sandboxtemplate_types.go index 9687b688a9..0b563f7070 100644 --- a/go/api/v1alpha3/sandboxtemplate_types.go +++ b/go/api/v1alpha3/sandboxtemplate_types.go @@ -4,6 +4,7 @@ package v1alpha3 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -37,14 +38,29 @@ type SandboxTemplateSpec struct { // resolved in this template's namespace. // +required Substrate RuntimeSubstratePolicy `json:"substrate"` + + // EgressPolicyRefs grants additional access in this template's namespace. + // Changes update the access of existing sandboxes without restarting them. + // +optional + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:XValidation:rule="self.all(ref, has(ref.name) && ref.name != '')",message="egressPolicyRefs requires non-empty names" + // +listType=map + // +listMapKey=name + EgressPolicyRefs []corev1.LocalObjectReference `json:"egressPolicyRefs,omitempty"` } +// SandboxTemplateConditionEgressPoliciesApplied reports grant admission independently of Ready. +const SandboxTemplateConditionEgressPoliciesApplied = "EgressPoliciesApplied" + // SandboxTemplateStatus reports preparation of the current template inputs. type SandboxTemplateStatus struct { // ObservedGeneration is the generation considered by preparation. // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // Conditions describe whether a prepared revision is available. + // Conditions report runtime readiness and independent egress grant admission. + // EgressPoliciesApplied is False for EgressPolicyNotFound, InvalidEgressPolicy, + // or EgressGrantConflict; Applied means all grants are admitted. Grant issues + // do not affect Ready. // +optional // +listType=map // +listMapKey=type diff --git a/go/api/v1alpha3/zz_generated.deepcopy.go b/go/api/v1alpha3/zz_generated.deepcopy.go index f21a0d5b50..0179022d34 100644 --- a/go/api/v1alpha3/zz_generated.deepcopy.go +++ b/go/api/v1alpha3/zz_generated.deepcopy.go @@ -91,6 +91,11 @@ func (in *AgentSpec) DeepCopyInto(out *AgentSpec) { *out = *in in.Template.DeepCopyInto(&out.Template) out.HarnessRef = in.HarnessRef + if in.EgressPolicyRefs != nil { + in, out := &in.EgressPolicyRefs, &out.EgressPolicyRefs + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentSpec. @@ -577,6 +582,127 @@ func (in *ConfigMapKeyReference) DeepCopy() *ConfigMapKeyReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressAuthentication) DeepCopyInto(out *EgressAuthentication) { + *out = *in + if in.Prefix != nil { + in, out := &in.Prefix, &out.Prefix + *out = new(string) + **out = **in + } + in.SecretKeyRef.DeepCopyInto(&out.SecretKeyRef) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressAuthentication. +func (in *EgressAuthentication) DeepCopy() *EgressAuthentication { + if in == nil { + return nil + } + out := new(EgressAuthentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressDestination) DeepCopyInto(out *EgressDestination) { + *out = *in + if in.Authentication != nil { + in, out := &in.Authentication, &out.Authentication + *out = new(EgressAuthentication) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressDestination. +func (in *EgressDestination) DeepCopy() *EgressDestination { + if in == nil { + return nil + } + out := new(EgressDestination) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressPolicy) DeepCopyInto(out *EgressPolicy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressPolicy. +func (in *EgressPolicy) DeepCopy() *EgressPolicy { + if in == nil { + return nil + } + out := new(EgressPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressPolicy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressPolicyList) DeepCopyInto(out *EgressPolicyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EgressPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressPolicyList. +func (in *EgressPolicyList) DeepCopy() *EgressPolicyList { + if in == nil { + return nil + } + out := new(EgressPolicyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressPolicyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressPolicySpec) DeepCopyInto(out *EgressPolicySpec) { + *out = *in + if in.Destinations != nil { + in, out := &in.Destinations, &out.Destinations + *out = make([]EgressDestination, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressPolicySpec. +func (in *EgressPolicySpec) DeepCopy() *EgressPolicySpec { + if in == nil { + return nil + } + out := new(EgressPolicySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FoundryConfig) DeepCopyInto(out *FoundryConfig) { *out = *in @@ -1649,6 +1775,11 @@ func (in *SandboxTemplateSpec) DeepCopyInto(out *SandboxTemplateSpec) { copy(*out, *in) } out.Substrate = in.Substrate + if in.EgressPolicyRefs != nil { + in, out := &in.EgressPolicyRefs, &out.EgressPolicyRefs + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxTemplateSpec. diff --git a/go/core/internal/controller/apiclient/types.go b/go/core/internal/controller/apiclient/types.go index c396c10f6d..7e5fdc6910 100644 --- a/go/core/internal/controller/apiclient/types.go +++ b/go/core/internal/controller/apiclient/types.go @@ -23,6 +23,19 @@ func RegisterTypes() { } func registerTypes() { + kubeclient.Register( + kagentv1alpha3.GroupVersion.WithResource("egresspolicies"), + kagentv1alpha3.GroupVersion.WithKind("EgressPolicy"), + func(c kubeclient.ClientGetter, namespace string, options metav1.ListOptions) (runtime.Object, error) { + return c.(Client).Kagent().ApiV1alpha3().EgressPolicies(namespace).List(context.Background(), options) + }, + func(c kubeclient.ClientGetter, namespace string, options metav1.ListOptions) (watch.Interface, error) { + return c.(Client).Kagent().ApiV1alpha3().EgressPolicies(namespace).Watch(context.Background(), options) + }, + func(c kubeclient.ClientGetter, namespace string) kubetypes.WriteAPI[*kagentv1alpha3.EgressPolicy] { + return c.(Client).Kagent().ApiV1alpha3().EgressPolicies(namespace) + }, + ) kubeclient.Register( kagentv1alpha3.GroupVersion.WithResource("agents"), kagentv1alpha3.GroupVersion.WithKind("Agent"), diff --git a/go/core/internal/controller/collections.go b/go/core/internal/controller/collections.go index cdd67b9b2e..c15cc91ef5 100644 --- a/go/core/internal/controller/collections.go +++ b/go/core/internal/controller/collections.go @@ -2,6 +2,7 @@ package controller import ( "reflect" + "slices" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -15,10 +16,13 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) // Collections contains shared Kubernetes inputs and compiled Agent state. type Collections struct { + watchNamespaces []string + EgressPolicies krt.Collection[*kagentv1alpha3.EgressPolicy] SandboxTemplates krt.Collection[*kagentv1alpha3.SandboxTemplate] Agents krt.Collection[*kagentv1alpha3.Agent] AgentTemplates krt.Collection[*kagentv1alpha3.AgentTemplate] @@ -62,7 +66,8 @@ func (p AgentRuntimeObservation) Equals(other AgentRuntimeObservation) bool { // NewCollections creates the complete read-only input graph. An empty // watchNamespaces list watches all namespaces. -func NewCollections(client kube.Client, watchNamespaces []string, opts krt.OptionsBuilder, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig) Collections { +func NewCollections(client kube.Client, watchNamespaces []string, opts krt.OptionsBuilder, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig, policyTranslator egressext.PolicyTranslator) Collections { + egressPolicies := typedCollection[*kagentv1alpha3.EgressPolicy](client, watchNamespaces, "EgressPolicies", opts) sandboxTemplates := typedCollection[*kagentv1alpha3.SandboxTemplate](client, watchNamespaces, "SandboxTemplates", opts) agents := typedCollection[*kagentv1alpha3.Agent](client, watchNamespaces, "Agents", opts) agentTemplates := typedCollection[*kagentv1alpha3.AgentTemplate](client, watchNamespaces, "AgentTemplates", opts) @@ -75,13 +80,16 @@ func NewCollections(client kube.Client, watchNamespaces []string, opts krt.Optio agentRuntimeObservations := krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...) modelConfigStatuses, resolvedModelConfigs := newModelConfigReconciliations(modelConfigs, configMaps, secrets, opts) compilerCollections := translator.Collections{ - Harnesses: harnesses, AgentTemplates: agentTemplates, ResolvedModelConfigs: resolvedModelConfigs, RemoteMCPServers: remoteMCPServers, + EgressPolicies: egressPolicies, + Harnesses: harnesses, AgentTemplates: agentTemplates, ResolvedModelConfigs: resolvedModelConfigs, RemoteMCPServers: remoteMCPServers, ConfigMaps: configMaps, Secrets: secrets, WorkerPools: workerPools, } - reconciliations := newAgentReconciliations(agents, compilerCollections, agentRuntimeObservations, opts, settings, images) + reconciliations := newAgentReconciliations(agents, compilerCollections, agentRuntimeObservations, opts, settings, images, policyTranslator) statuses := newAgentStatuses(agents, reconciliations, opts) return Collections{ + watchNamespaces: slices.Clone(watchNamespaces), + EgressPolicies: egressPolicies, SandboxTemplates: sandboxTemplates, Agents: agents, AgentTemplates: agentTemplates, diff --git a/go/core/internal/controller/collections_test.go b/go/core/internal/controller/collections_test.go index 8e8ab3a99d..50b9c928bc 100644 --- a/go/core/internal/controller/collections_test.go +++ b/go/core/internal/controller/collections_test.go @@ -64,7 +64,7 @@ func TestReconciliationCollectionsCompileAndObserveRevision(t *testing.T) { Harnesses: collections.Harnesses, AgentTemplates: collections.AgentTemplates, ResolvedModelConfigs: collections.ResolvedModelConfigs, RemoteMCPServers: collections.RemoteMCPServers, ConfigMaps: collections.ConfigMaps, Secrets: collections.Secrets, WorkerPools: collections.WorkerPools, - }, collections.AgentRuntimeObservations, opts, testRuntimeDefaults(), nil, + }, collections.AgentRuntimeObservations, opts, testRuntimeDefaults(), nil, nil, ) collections.AgentStatuses = newAgentStatuses(collections.Agents, collections.Reconciliations, opts) @@ -160,7 +160,7 @@ func TestReconciliationReportsRuntimeValidationAsUnsupported(t *testing.T) { Harnesses: krttest.GetMockCollection[*kagentv1alpha3.Harness](mock), AgentTemplates: krttest.GetMockCollection[*kagentv1alpha3.AgentTemplate](mock), ResolvedModelConfigs: resolvedModels, ConfigMaps: configMaps, Secrets: secrets, RemoteMCPServers: krttest.GetMockCollection[*kagentv1alpha3.RemoteMCPServer](mock), WorkerPools: krttest.GetMockCollection[*atev1alpha1.WorkerPool](mock), - }, krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("Observations")...), opts, settings, nil, + }, krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("Observations")...), opts, settings, nil, nil, ) waitFor(t, func() bool { return states.GetKey("team-a/assistant") != nil }) state := states.GetKey("team-a/assistant") @@ -230,7 +230,7 @@ func TestReconciliationWorkerPoolSandboxClass(t *testing.T) { Harnesses: krttest.GetMockCollection[*kagentv1alpha3.Harness](mock), AgentTemplates: templates, ResolvedModelConfigs: resolvedModels, RemoteMCPServers: krttest.GetMockCollection[*kagentv1alpha3.RemoteMCPServer](mock), ConfigMaps: configMaps, Secrets: secrets, WorkerPools: workerPools, - }, observations, opts, testRuntimeDefaults(), nil) + }, observations, opts, testRuntimeDefaults(), nil, nil) key := "team-a/assistant" waitFor(t, func() bool { state := reconciliations.GetKey(key) @@ -371,7 +371,7 @@ func TestClaudeReconciliationCompilesActorTemplate(t *testing.T) { RemoteMCPServers: krttest.GetMockCollection[*kagentv1alpha3.RemoteMCPServer](mock), ConfigMaps: configMaps, Secrets: secrets, WorkerPools: krttest.GetMockCollection[*atev1alpha1.WorkerPool](mock), - }, krttest.GetMockCollection[AgentRuntimeObservation](mock), opts, testRuntimeDefaults(), nil, + }, krttest.GetMockCollection[AgentRuntimeObservation](mock), opts, testRuntimeDefaults(), nil, nil, ) waitFor(t, func() bool { states := reconciliations.List() @@ -427,7 +427,7 @@ func TestCodexReconciliationCompilesActorTemplate(t *testing.T) { RemoteMCPServers: krttest.GetMockCollection[*kagentv1alpha3.RemoteMCPServer](mock), ConfigMaps: configMaps, Secrets: secrets, WorkerPools: krttest.GetMockCollection[*atev1alpha1.WorkerPool](mock), - }, krttest.GetMockCollection[AgentRuntimeObservation](mock), opts, testRuntimeDefaults(), nil, + }, krttest.GetMockCollection[AgentRuntimeObservation](mock), opts, testRuntimeDefaults(), nil, nil, ) waitFor(t, func() bool { states := reconciliations.List() @@ -480,7 +480,7 @@ func TestReconciliationTracksSharedAgentTemplate(t *testing.T) { RemoteMCPServers: krttest.GetMockCollection[*kagentv1alpha3.RemoteMCPServer](mock), ConfigMaps: configMaps, Secrets: secrets, WorkerPools: krttest.GetMockCollection[*atev1alpha1.WorkerPool](mock), - }, krttest.GetMockCollection[AgentRuntimeObservation](mock), opts, testRuntimeDefaults(), nil, + }, krttest.GetMockCollection[AgentRuntimeObservation](mock), opts, testRuntimeDefaults(), nil, nil, ) var initial string waitFor(t, func() bool { diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go new file mode 100644 index 0000000000..c5e2016723 --- /dev/null +++ b/go/core/internal/controller/egress.go @@ -0,0 +1,81 @@ +package controller + +import ( + "context" + "fmt" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/hashicorp/go-multierror" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/substrate" +) + +type actorEgressStore interface { + ListRuntimeEgressSourceTargets(context.Context, database.RuntimeEgressSource, string, int) ([]database.RuntimeEgressTarget, error) +} + +type actorEgressClient interface { + GetActor(context.Context, string, string) (*ateapipb.Actor, error) + EnsureActorEgressPolicy(context.Context, string, string, string) (egress.Policy, error) +} + +// applySourceEgress updates every allocated actor, including suspended actors and +// former source UIDs. Preparation retries the source on failure, even when its +// grants are unchanged. Continue past individual failures so one unavailable +// actor cannot prevent revocation on the others. +func applySourceEgress(ctx context.Context, store actorEgressStore, actors actorEgressClient, source database.RuntimeEgressSource) error { + const pageSize = 100 + var failures *multierror.Error + after := "" + for { + targets, err := store.ListRuntimeEgressSourceTargets(ctx, source, after, pageSize) + if err != nil { + return multierror.Append(failures, fmt.Errorf("failed to list %s %s/%s actors: %w", source.Kind, source.Namespace, source.Name, err)).ErrorOrNil() + } + var group errgroup.Group + group.SetLimit(16) + for _, target := range targets { + group.Go(func() error { + if err := applyActorEgress(ctx, actors, target); err != nil { + return fmt.Errorf("failed to apply actor %s egress: %w", target.ID, err) + } + return nil + }) + } + if err := group.Wait(); err != nil { + failures = multierror.Append(failures, err) + } + if len(targets) < pageSize { + return failures.ErrorOrNil() + } + after = targets[len(targets)-1].ID + } +} + +func applyActorEgress(ctx context.Context, actors actorEgressClient, target database.RuntimeEgressTarget) error { + binding := substrate.ActorBinding{Atespace: target.Atespace, Name: substrate.ActorName(target.ID), TemplateAtespace: target.Atespace, TemplateName: target.TemplateName} + actor, err := actors.GetActor(ctx, binding.Atespace, binding.Name) + if status.Code(err) == codes.NotFound { + return nil + } + if err != nil { + return err + } + if !binding.Matches(actor) || (target.ActorUID != "" && target.ActorUID != actor.GetMetadata().GetUid()) { + return nil + } + _, err = actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, target.Revision) + if status.Code(err) == codes.NotFound { + return nil + } + if err != nil { + return err + } + + return nil +} diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go new file mode 100644 index 0000000000..b4519f534f --- /dev/null +++ b/go/core/internal/controller/egress_test.go @@ -0,0 +1,130 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "slices" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +type egressTargetStoreStub struct { + sources []database.RuntimeEgressSource + targets []database.RuntimeEgressTarget + listErr error +} + +var _ actorEgressStore = (*egressTargetStoreStub)(nil) + +func (s *egressTargetStoreStub) ListRuntimeEgressSourceTargets(_ context.Context, source database.RuntimeEgressSource, after string, limit int) ([]database.RuntimeEgressTarget, error) { + s.sources = append(s.sources, source) + if s.listErr != nil { + return nil, s.listErr + } + targets := slices.DeleteFunc(slices.Clone(s.targets), func(target database.RuntimeEgressTarget) bool { return target.ID <= after }) + return targets[:min(len(targets), limit)], nil +} + +type actorEgressStub struct { + getActor func(context.Context, string, string) (*ateapipb.Actor, error) + ensurePolicy func(context.Context, string, string, string) (egress.Policy, error) +} + +var _ actorEgressClient = (*actorEgressStub)(nil) + +func (a *actorEgressStub) GetActor(ctx context.Context, space, name string) (*ateapipb.Actor, error) { + if a.getActor != nil { + return a.getActor(ctx, space, name) + } + return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: "uid"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: "runtime"}}, nil +} +func (a *actorEgressStub) EnsureActorEgressPolicy(ctx context.Context, space, name, revision string) (egress.Policy, error) { + return a.ensurePolicy(ctx, space, name, revision) +} + +func TestSourceEgressAppliesEveryPageWithBoundedConcurrency(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + source := database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team", Name: "source"} + store := &egressTargetStoreStub{} + for i := range 101 { + store.targets = append(store.targets, database.RuntimeEgressTarget{ID: fmt.Sprintf("%03d", i), Revision: "revision", Namespace: "team", Atespace: "team", TemplateName: "runtime"}) + } + var mu sync.Mutex + active, peak := 0, 0 + applied := map[string]int{} + backendErr := errors.New("backend unavailable") + actors := &actorEgressStub{ensurePolicy: func(ctx context.Context, _, name, _ string) (egress.Policy, error) { + mu.Lock() + active++ + peak = max(peak, active) + applied[name]++ + mu.Unlock() + time.Sleep(time.Second) + mu.Lock() + active-- + mu.Unlock() + if name == "session-000" { + return egress.Policy{}, backendErr + } + return egress.Policy{}, ctx.Err() + }} + require.ErrorIs(t, applySourceEgress(t.Context(), store, actors, source), backendErr) + require.Len(t, applied, 101, "a failed actor must not block subsequent actors or pages") + require.Greater(t, peak, 1) + require.LessOrEqual(t, peak, 16) + require.Equal(t, []database.RuntimeEgressSource{source, source}, store.sources) + store.listErr = errors.New("database unavailable") + require.ErrorIs(t, applySourceEgress(t.Context(), store, actors, source), store.listErr) + }) +} + +func TestActorEgressSkipsStaleOrMissingActors(t *testing.T) { + for _, test := range []struct { + name string + uid string + template string + getErr error + applyErr error + wantCalls int + wantErr bool + }{ + {name: "matching", uid: "uid", template: "runtime", wantCalls: 1}, + {name: "replaced template", uid: "uid", template: "replacement"}, + {name: "replaced actor", uid: "replacement", template: "runtime"}, + {name: "missing actor", getErr: status.Error(codes.NotFound, "gone")}, + {name: "deleted during apply", uid: "uid", template: "runtime", applyErr: status.Error(codes.NotFound, "gone"), wantCalls: 1}, + {name: "read failure", getErr: errors.New("unavailable"), wantErr: true}, + {name: "apply failure", uid: "uid", template: "runtime", applyErr: errors.New("unavailable"), wantCalls: 1, wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + calls := 0 + actors := &actorEgressStub{ + getActor: func(_ context.Context, space, name string) (*ateapipb.Actor, error) { + return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: test.uid}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: test.template}}, test.getErr + }, + ensurePolicy: func(context.Context, string, string, string) (egress.Policy, error) { + calls++ + return egress.Policy{}, test.applyErr + }, + } + err := applyActorEgress(t.Context(), actors, database.RuntimeEgressTarget{ID: "actor", Revision: "revision", Atespace: "team", TemplateName: "runtime", ActorUID: "uid"}) + if test.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, test.wantCalls, calls) + }) + } +} diff --git a/go/core/internal/controller/equality_test.go b/go/core/internal/controller/equality_test.go index 6de9fe439f..3c03c03ea8 100644 --- a/go/core/internal/controller/equality_test.go +++ b/go/core/internal/controller/equality_test.go @@ -55,6 +55,9 @@ func TestAgentReconciliationEquality(t *testing.T) { proto.Size(r.ObservedActorTemplate) }, equal: true}, {name: "template source", change: func(r *AgentReconciliation) { r.Agent.Generation++ }}, + {name: "live grants", change: func(r *AgentReconciliation) { + r.EgressGrant.Destinations = []string{"https://tickets.example.com:443"} + }}, {name: "revision identity", change: func(r *AgentReconciliation) { r.Target.RevisionID[0]++ }}, {name: "revision inputs", change: func(r *AgentReconciliation) { r.Target.Revision.SandboxClass = "microvm" }}, {name: "agent card", change: func(r *AgentReconciliation) { r.Target.Revision.AgentCard.Name = "changed" }}, diff --git a/go/core/internal/controller/grant_admission.go b/go/core/internal/controller/grant_admission.go new file mode 100644 index 0000000000..e4a155b896 --- /dev/null +++ b/go/core/internal/controller/grant_admission.go @@ -0,0 +1,75 @@ +package controller + +import ( + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/translator" +) + +// evaluateEgressGrant checks the complete candidate without reading persisted +// grants. On conflict, retain the raw union so preparation can apply removals. +func evaluateEgressGrant(resolution translator.EgressGrantResolution, base egress.Access, tokenOrigins []string) (egress.Access, []translator.EgressGrantIssue, bool) { + candidate, err := egress.CanonicalAccess(resolution.Access.Destinations, resolution.Access.Credentials) + if err == nil { + _, err = egress.MergeAccess(base, candidate, tokenOrigins) + } + if err == nil { + return candidate, resolution.Issues, false + } + issues := append(resolution.Issues, translator.EgressGrantIssue{ + Reason: v1alpha3.EgressPoliciesReasonConflict, Message: err.Error(), + }) + return resolution.Access, issues, true +} + +func retainedEgressGrant(previous, candidate, base egress.Access, tokenOrigins []string) egress.Access { + retained := egress.IntersectAccess(previous, candidate) + filtered, _ := egress.FilterConflicts(base, retained, tokenOrigins) + return filtered +} + +func egressPoliciesCondition(conditionType string, generation int64, issues []translator.EgressGrantIssue, conflict bool) metav1.Condition { + condition := metav1.Condition{ + Type: conditionType, ObservedGeneration: generation, + Status: metav1.ConditionTrue, Reason: v1alpha3.EgressPoliciesReasonApplied, + Message: "All requested egress grants are admitted", + } + if len(issues) == 0 && !conflict { + return condition + } + condition.Status = metav1.ConditionFalse + var messages []string + for _, issue := range issues { + message := issue.Message + if issue.Policy != "" { + message = fmt.Sprintf("%s: %s", issue.Policy, message) + } + messages = append(messages, message) + } + if conflict { + condition.Reason = v1alpha3.EgressPoliciesReasonConflict + condition.Message = "new grants are held, previously applied grants remain in effect; " + } else { + condition.Reason = issues[0].Reason + condition.Message = "" + } + condition.Message += strings.Join(messages, "; ") + // Stay comfortably below the Kubernetes condition message limit. + const maxMessage = 4096 + if len(condition.Message) > maxMessage { + condition.Message = strings.ToValidUTF8(condition.Message[:maxMessage-3], "") + "..." + } + return condition +} + +func (c *compiledTarget) egressBase() (egress.Access, []string) { + if c == nil { + return egress.Access{}, nil + } + return egress.Access{Destinations: c.Revision.EgressDestinations, Credentials: c.Revision.Credentials}, c.Revision.TokenPassthroughOrigins +} diff --git a/go/core/internal/controller/grant_admission_test.go b/go/core/internal/controller/grant_admission_test.go new file mode 100644 index 0000000000..bc467b1f28 --- /dev/null +++ b/go/core/internal/controller/grant_admission_test.go @@ -0,0 +1,201 @@ +package controller + +import ( + "slices" + "strings" + "testing" + "testing/synctest" + + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/substrate" + "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" +) + +func TestGrantAdmissionAcrossSources(t *testing.T) { + for _, kind := range []string{"agent", "sandbox"} { + t.Run(kind, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + opts := krt.NewOptionsBuilder(t.Context().Done(), "admission", nil) + var policies krt.StaticCollection[*v1alpha3.EgressPolicy] + var reconcile func() error + var stored func() egress.Access + var conditions func() []metav1.Condition + switch kind { + case "agent": + collections, _ := newPreparationTestCollections(t, "gvisor") + initial := collections.Reconciliations.List()[0] + agent := initial.Agent.DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "a"}, {Name: "b"}} + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + policies = collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]) + template := proto.CloneOf(initial.Target.ActorTemplate) + template.Metadata.Uid, template.Status = "prepared", sandboxGoldenStatus() + store := &fakeRuntimeRevisionStore{} + r := &Reconciler{collections: collections, templates: &fakeActorTemplates{template: template}, store: store} + reconcile = func() error { return r.reconcileAgent(t.Context(), initial.ResourceName()) } + stored = func() egress.Access { return store.pair.EgressGrant } + conditions = func() []metav1.Condition { + state := collections.Reconciliations.GetKey(initial.ResourceName()) + require.NotNil(t, state.Target, "grant conflicts must not block compilation") + require.Equal(t, initial.Target.RevisionID, state.Target.RevisionID) + return statusForAgent(*state, agent.Generation, "").Conditions + } + case "sandbox": + template := sandboxTestTemplate() + template.Finalizers = []string{sandboxPreparationFinalizer} + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "a"}, {Name: "b"}} + templates := krt.NewStaticCollection(nil, []*v1alpha3.SandboxTemplate{template}, opts.WithName("templates")...) + policies = krt.NewStaticCollection[*v1alpha3.EgressPolicy](nil, nil, opts.WithName("policies")...) + pools := krt.NewStaticCollection(nil, []*atev1alpha1.WorkerPool{{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "default"}}}, opts.WithName("pools")...) + store := &sandboxTestStore{} + actors := &sandboxTestActors{store: store, templates: map[string]*ateapipb.ActorTemplate{}, autoReady: true} + s := &SandboxReconciler{collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools, EgressPolicies: policies}, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, opts), + store: store, actors: actors, client: kagentfake.NewSimpleClientset(template).ApiV1alpha3()} + reconcile = func() error { return s.reconcile(t.Context(), "team-a/scratch") } + stored = func() egress.Access { return store.desired.EgressGrant } + conditions = func() []metav1.Condition { + state := s.collections.states.GetKey("team-a/scratch") + require.NotNil(t, state.DesiredActorTemplate) + return sandboxStatusWithTransitionTimes(*state).Conditions + } + } + policy := func(name string, origins ...string) *v1alpha3.EgressPolicy { + p := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: name}} + for _, origin := range origins { + p.Spec.Destinations = append(p.Spec.Destinations, v1alpha3.EgressDestination{Origin: origin}) + } + return p + } + check := func(reason string, origins ...string) { + t.Helper() + synctest.Wait() + require.NoError(t, reconcile()) + synctest.Wait() + require.Equal(t, origins, stored().Destinations) + observed := conditions() + require.True(t, meta.IsStatusConditionTrue(observed, "Ready"), "%v", observed) + condition := meta.FindStatusCondition(observed, "EgressPoliciesApplied") + require.NotNil(t, condition) + require.Equal(t, reason, condition.Reason) + if reason == v1alpha3.EgressPoliciesReasonApplied { + require.Equal(t, metav1.ConditionTrue, condition.Status) + } else { + require.Equal(t, metav1.ConditionFalse, condition.Status) + } + } + const a = "https://a.example.com:443" + const x = "https://x.example.com:443" + policies.UpdateObject(policy("a", a)) + policies.UpdateObject(policy("b", x)) + check(v1alpha3.EgressPoliciesReasonApplied, a, x) + // The new HTTP rule conflicts with a. An unrelated addition is held + // too, and the removed x is revoked immediately. + policies.UpdateObject(policy("b", "http://a.example.com:443", "https://new.example.com")) + check(v1alpha3.EgressPoliciesReasonConflict, a) + // Repeating preparation cannot admit previously held additions. + check(v1alpha3.EgressPoliciesReasonConflict, a) + policies.DeleteObject("team-a/b") + check(v1alpha3.EgressPoliciesReasonNotFound, a) + policies.UpdateObject(policy("b", "https://new.example.com")) + check(v1alpha3.EgressPoliciesReasonApplied, a, "https://new.example.com:443") + }) + }) + } +} + +func TestGrantAdmissionWithChangedBase(t *testing.T) { + const a = "https://a.example.com:443" + const model = "https://model.example.com:443" + old := egress.Access{Destinations: []string{a, model}} + base := egress.Access{Destinations: []string{"http://model.example.com:443"}} + candidate, issues, conflict := evaluateEgressGrant(translator.EgressGrantResolution{Access: old}, base, nil) + require.True(t, conflict) + require.Len(t, issues, 1) + require.Equal(t, egress.Access{Destinations: []string{a}}, retainedEgressGrant(old, candidate, base, nil)) + // With no compiled target, admission can still revoke grants safely. + require.Equal(t, old, retainedEgressGrant(old, candidate, egress.Access{}, nil)) + require.Empty(t, retainedEgressGrant(egress.Access{}, candidate, base, nil).Destinations, "new identities inherit no grants") +} + +func TestEgressConditionSurvivesRuntimeFailureAndBoundsMessages(t *testing.T) { + issues := []translator.EgressGrantIssue{ + {Policy: "team/missing", Reason: v1alpha3.EgressPoliciesReasonNotFound, Message: "missing"}, + {Reason: v1alpha3.EgressPoliciesReasonConflict, Message: strings.Repeat("conflict", 1000)}, + } + state := AgentReconciliation{EgressIssues: issues, EgressConflict: true, + CompilationFailure: &ReconciliationFailure{Condition: v1alpha3.AgentConditionResolvedRefs, Reason: "ReferenceResolutionFailed", Message: "Harness is missing"}} + status := statusForAgent(state, 1, "") + condition := meta.FindStatusCondition(status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied) + require.Equal(t, v1alpha3.EgressPoliciesReasonConflict, condition.Reason) + require.Contains(t, condition.Message, "new grants are held") + require.Contains(t, condition.Message, "team/missing") + require.LessOrEqual(t, len(condition.Message), 4096) + require.False(t, meta.IsStatusConditionTrue(status.Conditions, v1alpha3.AgentConditionReady)) +} + +// targetCredential binds its origin once the draft makes that origin a target. +type targetCredential egressext.Credential + +func (c targetCredential) Translate(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + if slices.ContainsFunc(draft.Requirements, func(r egressext.AuthenticationRequirement) bool { return r.Origin == c.Origin }) { + draft.Credentials = append(draft.Credentials, egressext.Credential(c)) + } + return draft, nil +} + +func TestGrantAdmissionHoldsConflictWithExtensionCredential(t *testing.T) { + const tools = "https://tools.example.com:443" + pinned := egress.Credential{Origin: tools, Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/default/token"} + collections, _ := newExtensionTestCollections(t, "gvisor", targetCredential(pinned), + &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "tools"}, Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://tools.example.com/mcp"}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "auth"}, Data: map[string][]byte{"token": []byte("value")}}) + initial := collections.Reconciliations.List()[0] + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(&v1alpha3.EgressPolicy{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, + Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{ + {Origin: "https://x.example.com"}, + {Origin: "https://tools.example.com", Authentication: &v1alpha3.EgressAuthentication{ + Header: "Authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}, Key: "token"}, + }}, + {Origin: "https://new.example.com"}, + }}, + }) + agent := initial.Agent.DeepCopy() + agent.Spec.Template = v1alpha3.AgentTemplateSource{Inline: &v1alpha3.AgentTemplateSpec{Tools: []v1alpha3.ToolBinding{{MCP: &v1alpha3.MCPToolBinding{ + Server: corev1.TypedLocalObjectReference{Kind: "RemoteMCPServer", Name: "tools"}, + }}}}} + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + + var state AgentReconciliation + waitFor(t, func() bool { + current := collections.Reconciliations.GetKey(initial.ResourceName()) + if current == nil || current.Target == nil || current.Target.RevisionID == initial.Target.RevisionID { + return false + } + state = *current + return true + }) + require.Nil(t, state.CompilationFailure) + require.Contains(t, state.Target.Revision.Credentials, pinned, "a live grant conflict must not block compilation") + require.True(t, state.EgressConflict) + condition := meta.FindStatusCondition(statusForAgent(state, agent.Generation, "").Conditions, v1alpha3.AgentConditionEgressPoliciesApplied) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, v1alpha3.EgressPoliciesReasonConflict, condition.Reason) + base, origins := state.Target.egressBase() + previous := egress.Access{Destinations: []string{"https://x.example.com:443"}} + require.Equal(t, previous, retainedEgressGrant(previous, state.EgressGrant, base, origins), "additions stay held") +} diff --git a/go/core/internal/controller/grants_test.go b/go/core/internal/controller/grants_test.go new file mode 100644 index 0000000000..1c2f20a557 --- /dev/null +++ b/go/core/internal/controller/grants_test.go @@ -0,0 +1,377 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "testing/synctest" + "time" + + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/substrate" +) + +func TestAgentEgressErrorsDoNotBlockCompilation(t *testing.T) { + for _, test := range []struct { + name string + destinations []v1alpha3.EgressDestination + condition string + reason string + }{ + {name: "missing policy", condition: v1alpha3.AgentConditionEgressPoliciesApplied, reason: v1alpha3.EgressPoliciesReasonNotFound}, + {name: "invalid origin", destinations: []v1alpha3.EgressDestination{{Origin: "invalid"}}, condition: v1alpha3.AgentConditionEgressPoliciesApplied, reason: v1alpha3.EgressPoliciesReasonInvalidEgressPolicy}, + {name: "missing secret", destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com", Authentication: &v1alpha3.EgressAuthentication{Header: "Authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "missing"}, Key: "token"}}}}, condition: v1alpha3.AgentConditionEgressPoliciesApplied, reason: v1alpha3.EgressPoliciesReasonInvalidEgressPolicy}, + } { + t.Run(test.name, func(t *testing.T) { + collections, _ := newPreparationTestCollections(t, "gvisor") + if test.destinations != nil { + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(&v1alpha3.EgressPolicy{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, + Spec: v1alpha3.EgressPolicySpec{Destinations: test.destinations}, + }) + } + agent := collections.Agents.List()[0].DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + waitFor(t, func() bool { + observed := collections.AgentStatuses.GetKey("team-a/assistant") + return observed != nil && meta.IsStatusConditionFalse(observed.Status.Conditions, test.condition) + }) + condition := meta.FindStatusCondition(collections.AgentStatuses.GetKey("team-a/assistant").Status.Conditions, test.condition) + require.Equal(t, test.reason, condition.Reason) + state := collections.Reconciliations.GetKey("team-a/assistant") + require.NotNil(t, state.Target) + require.Nil(t, state.CompilationFailure) + require.Empty(t, state.EgressGrant.Destinations) + require.Empty(t, state.EgressGrant.Credentials) + }) + } +} + +func TestAgentGrantChangesReconcileWithoutRuntimeChange(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + collections, harnesses := newPreparationTestCollections(t, "gvisor") + initial := collections.Reconciliations.List()[0] + template := proto.CloneOf(initial.Target.ActorTemplate) + template.Metadata.Uid = "prepared" + template.Status = sandboxGoldenStatus() + store := &fakeRuntimeRevisionStore{definitionSources: []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}} + backend := &preparationTestBackend{templates: &fakeActorTemplates{template: template}, store: store} + r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3()) + ctx, cancel := context.WithCancel(t.Context()) + var workers sync.WaitGroup + workers.Go(func() { assert.NoError(t, r.Start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + require.Empty(t, store.retired, "startup must preserve a matching live Agent") + store.sources = nil + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(policy) + agent := initial.Agent.DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + synctest.Wait() + backend.inspect(func() { + require.Equal(t, initial.Target.RevisionID.String(), store.pair.DesiredRevision) + require.Equal(t, []string{"https://tickets.example.com:443"}, store.pair.EgressGrant.Destinations) + }) + require.Equal(t, []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}, store.sources) + store.sources = nil + harnesses.DeleteObject("team-a/byo") + synctest.Wait() + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.NotEmpty(t, store.pair.EgressGrant.Destinations) + }) + require.Len(t, store.sources, 1, "every preparation reapplies grants, including unchanged grants") + store.sources = nil + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).DeleteObject("team-a/access") + synctest.Wait() + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.Empty(t, store.pair.EgressGrant.Destinations) + }) + require.Len(t, store.sources, 1, "revocation must reconcile even while the runtime remains unresolvable") + }) +} + +func TestAgentGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { + for _, missing := range []string{"Harness", "AgentTemplate", "WorkerPool"} { + t.Run(missing, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + collections, harnesses := newPreparationTestCollections(t, "gvisor") + agent := collections.Agents.List()[0].DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + switch missing { + case "Harness": + agent.Spec.HarnessRef.Name = "missing" + case "AgentTemplate": + agent.Spec.Template = v1alpha3.AgentTemplateSource{Ref: &corev1.LocalObjectReference{Name: "missing"}} + case "WorkerPool": + harness := harnesses.List()[0].DeepCopy() + harness.Spec.Substrate.WorkerPoolRef.Name = "missing" + harnesses.UpdateObject(harness) + } + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + synctest.Wait() + store := &fakeRuntimeRevisionStore{} + backend := &preparationTestBackend{templates: &fakeActorTemplates{}, store: store} + r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3()) + ctx, cancel := context.WithCancel(t.Context()) + var workers sync.WaitGroup + workers.Go(func() { assert.NoError(t, r.Start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + store.sources = nil + + policies := collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]) + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} + policies.UpdateObject(policy) + synctest.Wait() + state := collections.Reconciliations.GetKey("team-a/assistant") + require.Nil(t, state.Target) + require.NotNil(t, state.CompilationFailure) + require.Contains(t, state.CompilationFailure.Message, missing) + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.Equal(t, []string{"https://tickets.example.com:443"}, store.pair.EgressGrant.Destinations) + }) + wantSource := database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"} + require.Equal(t, []database.RuntimeEgressSource{wantSource}, store.sources) + + policy = policy.DeepCopy() + policy.Spec.Destinations[0].Origin = "https://updated.example.com" + policies.UpdateObject(policy) + synctest.Wait() + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.Equal(t, []string{"https://updated.example.com:443"}, store.pair.EgressGrant.Destinations) + }) + require.Equal(t, []database.RuntimeEgressSource{wantSource, wantSource}, store.sources) + + policies.DeleteObject("team-a/access") + synctest.Wait() + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.Empty(t, store.pair.EgressGrant.Destinations) + require.Zero(t, store.recordCalls, "grant changes must not require a compiled runtime") + }) + require.Equal(t, []database.RuntimeEgressSource{wantSource, wantSource, wantSource}, store.sources, "revocation must apply actors while compilation remains broken") + }) + }) + } +} + +func TestAgentGrantWriteFailureAndRetirementApplyActors(t *testing.T) { + collections, _ := newPreparationTestCollections(t, "gvisor") + state := collections.Reconciliations.List()[0] + store := &fakeRuntimeRevisionStore{pairErr: errors.New("response lost")} + r := &Reconciler{collections: collections, store: store, status: kagentfake.NewSimpleClientset(state.Agent.DeepCopy()).ApiV1alpha3()} + require.ErrorIs(t, r.reconcileAgent(t.Context(), state.ResourceName()), store.pairErr) + require.Empty(t, store.sources, "a failed write retries preparation before applying actors") + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).DeleteObject(state.ResourceName()) + waitFor(t, func() bool { return collections.Reconciliations.GetKey(state.ResourceName()) == nil }) + require.NoError(t, r.reconcileAgent(t.Context(), state.ResourceName())) + require.Equal(t, []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}, store.sources) +} + +func TestAgentStartupRecoversOfflineDeletions(t *testing.T) { + for _, namespaces := range [][]string{nil, {"team-a"}} { + t.Run(fmt.Sprint(namespaces), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + collections, _ := newPreparationTestCollections(t, "gvisor") + collections.watchNamespaces = namespaces + // Delete before registering handlers: no KRT event can recover it. + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).DeleteObject("team-a/assistant") + synctest.Wait() + sources := []database.RuntimeEgressSource{ + {Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, + {Kind: database.RuntimeEgressAgent, Namespace: "team-b", Name: "assistant"}, + } + store := &fakeRuntimeRevisionStore{definitionSources: sources, egressTargetStoreStub: egressTargetStoreStub{ + targets: []database.RuntimeEgressTarget{{ID: "actor", Revision: "old", Atespace: "team-a", TemplateName: "runtime"}}, + }} + attempts := 0 + actors := &fakeActorTemplates{actorEgressStub: actorEgressStub{ensurePolicy: func(context.Context, string, string, string) (egress.Policy, error) { + assert.NotEmpty(t, store.retired, "retire before applying revocation") + attempts++ + if attempts == 1 { + return egress.Policy{}, errors.New("backend unavailable") + } + return egress.Policy{}, nil + }}} + r := newReconciler(collections, actors, store, kagentfake.NewSimpleClientset().ApiV1alpha3()) + var workers sync.WaitGroup + workers.Go(func() { assert.NoError(t, r.Start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + time.Sleep(time.Second) + synctest.Wait() + want := []database.RuntimeEgressSource{sources[0], sources[0]} + if len(namespaces) == 0 { + want = append(want, sources[1]) + } + require.ElementsMatch(t, want, store.sources, "recovery must honor namespace scope and retry revocation") + require.Equal(t, len(want), attempts) + require.Zero(t, store.pairCalls, "missing Agents must not be recreated") + }) + }) + } +} + +func TestAgentStartupRecoveryReadFailure(t *testing.T) { + collections, _ := newPreparationTestCollections(t, "gvisor") + store := &fakeRuntimeRevisionStore{sourcesErr: errors.New("database unavailable")} + r := newReconciler(collections, &fakeActorTemplates{}, store, kagentfake.NewSimpleClientset().ApiV1alpha3()) + require.ErrorIs(t, r.Start(t.Context()), store.sourcesErr) + require.Zero(t, store.pairCalls, "failed recovery must fail startup") +} + +func TestSandboxGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { + opts := krt.NewOptionsBuilder(t.Context().Done(), "test-grants", nil) + template := sandboxTestTemplate() + template.Finalizers = []string{sandboxPreparationFinalizer} + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + templates := krt.NewStaticCollection(nil, []*v1alpha3.SandboxTemplate{template}, opts.WithName("Templates")...) + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} + policies := krt.NewStaticCollection(nil, []*v1alpha3.EgressPolicy{policy}, opts.WithName("Policies")...) + pools := krt.NewStaticCollection[*atev1alpha1.WorkerPool](nil, nil, opts.WithName("Pools")...) + store := &sandboxTestStore{} + s := &SandboxReconciler{collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools, EgressPolicies: policies}, substrate.SandboxPolicy{}, opts), store: store, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3()} + waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch") != nil }) + require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) + require.Nil(t, s.collections.states.GetKey("team-a/scratch").DesiredActorTemplate, "missing WorkerPool prevents runtime compilation") + require.Equal(t, []string{"https://tickets.example.com:443"}, store.desired.EgressGrant.Destinations) + require.Len(t, store.sources, 1) + policies.DeleteObject("team-a/access") + waitFor(t, func() bool { return len(s.collections.states.GetKey("team-a/scratch").EgressIssues) != 0 }) + require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) + require.Empty(t, store.desired.EgressGrant.Destinations) + require.Len(t, store.sources, 2, "revocation must reach existing actors during failed compilation") + store.upsertErr = errors.New("response lost") + require.ErrorIs(t, s.reconcile(t.Context(), "team-a/scratch"), store.upsertErr) + require.Len(t, store.sources, 2, "failed writes retry before actor application") + template = template.DeepCopy() + template.DeletionTimestamp = &metav1.Time{Time: metav1.Now().Time} + templates.UpdateObject(template) + waitFor(t, func() bool { return !s.collections.states.GetKey("team-a/scratch").Template.DeletionTimestamp.IsZero() }) + require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) + require.Len(t, store.sources, 3) +} + +// Registering the KRT handler must recover actors even when their grants already +// match persisted state. A failed push must retry without another graph event. +func TestPreparationReplaysAndRetriesUnchangedGrants(t *testing.T) { + for _, kind := range []string{"agent", "sandbox"} { + t.Run(kind, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + grants := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} + target := database.RuntimeEgressTarget{ID: "actor", Revision: "old", Atespace: "team-a", TemplateName: "runtime"} + var start func(context.Context) error + var persisted func() egress.Access + var actorStub *actorEgressStub + switch kind { + case "agent": + collections, harnesses := newPreparationTestCollections(t, "gvisor") + agent := collections.Agents.List()[0].DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(&v1alpha3.EgressPolicy{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: grants.Destinations[0]}}}, + }) + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + harnesses.DeleteObject("team-a/byo") + synctest.Wait() + store := &fakeRuntimeRevisionStore{pair: &database.AgentDefinition{EgressGrant: grants}, egressTargetStoreStub: egressTargetStoreStub{targets: []database.RuntimeEgressTarget{target}}} + actors := &fakeActorTemplates{} + backend := &preparationTestBackend{templates: actors, store: store} + r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3()) + start = r.Start + persisted = func() egress.Access { backend.mu.Lock(); defer backend.mu.Unlock(); return store.pair.EgressGrant } + actorStub = &actors.actorEgressStub + case "sandbox": + opts := krt.NewOptionsBuilder(ctx.Done(), "retry-grants", nil) + template := sandboxTestTemplate() + template.Finalizers = []string{sandboxPreparationFinalizer} + states := krt.NewStaticCollection(nil, []sandboxReconciliation{{Template: template, EgressGrant: grants}}, opts.WithName("states")...) + store := &sandboxTestStore{desired: database.SandboxTemplateDefinition{EgressGrant: grants}, egressTargetStoreStub: egressTargetStoreStub{targets: []database.RuntimeEgressTarget{target}}} + actors := &sandboxTestActors{} + s := &SandboxReconciler{collections: sandboxCollections{states: states, observations: krt.NewStaticCollection[sandboxRuntimeObservation](nil, nil, opts.WithName("observations")...)}, store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3()} + start = s.Start + persisted = func() egress.Access { store.mu.Lock(); defer store.mu.Unlock(); return store.desired.EgressGrant } + actorStub = &actors.actorEgressStub + } + var mu sync.Mutex + var attempts []time.Time + var applied egress.Access + actorStub.ensurePolicy = func(context.Context, string, string, string) (egress.Policy, error) { + mu.Lock() + defer mu.Unlock() + attempts = append(attempts, time.Now()) + if len(attempts) == 1 { + return egress.Policy{}, errors.New("backend unavailable") + } + applied = persisted() + return egress.Policy{Access: applied}, nil + } + var workers sync.WaitGroup + workers.Go(func() { assert.NoError(t, start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + mu.Lock() + require.Len(t, attempts, 1, "startup must attempt actors with unchanged grants") + mu.Unlock() + time.Sleep(time.Second) + synctest.Wait() + mu.Lock() + defer mu.Unlock() + require.Len(t, attempts, 2, "the preparation queue must retry a failed push without input changes") + require.Equal(t, time.Second, attempts[1].Sub(attempts[0])) + require.Equal(t, grants.Destinations, applied.Destinations) + }) + }) + } +} + +func TestSandboxFinalizerWaitsForActorRevocation(t *testing.T) { + s, templates, _ := newSandboxTestReconciler(t, "") + const key = "team-a/scratch" + require.NoError(t, s.reconcile(t.Context(), key)) + template := syncSandboxTemplate(t, s, templates).DeepCopy() + template.DeletionTimestamp = &metav1.Time{Time: metav1.Now().Time} + templates.UpdateObject(template) + waitFor(t, func() bool { return !s.collections.states.GetKey(key).Template.DeletionTimestamp.IsZero() }) + store := s.store.(*sandboxTestStore) + store.targets = []database.RuntimeEgressTarget{{ID: "actor", Revision: "old", Atespace: "team-a", TemplateName: "runtime"}} + pushErr := errors.New("backend unavailable") + s.actors.(*sandboxTestActors).ensurePolicy = func(context.Context, string, string, string) (egress.Policy, error) { + assert.True(t, store.retired, "retire before applying revocation") + return egress.Policy{}, pushErr + } + require.ErrorIs(t, s.reconcile(t.Context(), key), pushErr) + persisted, err := s.client.SandboxTemplates("team-a").Get(t.Context(), "scratch", metav1.GetOptions{}) + require.NoError(t, err) + require.Contains(t, persisted.Finalizers, sandboxPreparationFinalizer) + pushErr = nil + require.NoError(t, s.reconcile(t.Context(), key)) + persisted, err = s.client.SandboxTemplates("team-a").Get(t.Context(), "scratch", metav1.GetOptions{}) + require.NoError(t, err) + require.NotContains(t, persisted.Finalizers, sandboxPreparationFinalizer) +} diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index 46a07ba93c..095fd7f9ee 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -12,6 +12,7 @@ import ( atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -27,7 +28,9 @@ import ( kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { @@ -97,7 +100,7 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { // A delayed success for the abandoned preparation must not replace A. abandoned, err := store.GetRuntimeRevision(ctx, preparing.Target.RevisionID.String()) require.NoError(t, err) - require.NoError(t, store.RecordRuntimeRevision(ctx, *abandoned, true)) + require.NoError(t, store.RecordRuntimeRevision(ctx, *abandoned, database.RecordRevisionOptions{Ready: true})) unreferenced, err = store.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) require.Len(t, unreferenced, 1) @@ -106,12 +109,12 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { require.NoError(t, err) require.Nil(t, retained, "last-successful A must remain protected without any sessions") - session, _, err := store.CreateSession(ctx, &apiv1.Session{ + fresh, _, err := store.CreateSession(ctx, &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}, }, "last-good-session") - require.NoError(t, err) - require.Equal(t, initial.Target.RevisionID.String(), session.GetPreparedRevision()) + require.NoError(t, err, "unresolved edits must leave the last successful revision available") + require.Equal(t, initial.Target.RevisionID.String(), fresh.PreparedRevision) collector, _ := newTestRuntimeRevisionGC(t, store, templates) require.NoError(t, collector.collect(ctx, abandoned.Revision)) @@ -281,7 +284,7 @@ func TestPreparationQueueAndPollerBackoff(t *testing.T) { r := newReconciler(collections, backend, backend, statusClient) ctx, cancel := context.WithCancel(t.Context()) defer cancel() - go r.Run(ctx.Done()) + go func() { assert.NoError(t, r.Start(ctx)) }() synctest.Wait() backend.inspect(func() { require.Equal(t, 1, test.store.pairCalls, "publishing a failure must not immediately requeue it") @@ -381,22 +384,28 @@ func (p *preparationTestBackend) CreateActorTemplate(ctx context.Context, templa return p.templates.CreateActorTemplate(ctx, template) } +func (p *preparationTestBackend) ListAgentDefinitionSources(ctx context.Context) ([]database.RuntimeEgressSource, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.store.ListAgentDefinitionSources(ctx) +} + func (p *preparationTestBackend) UpsertAgentDefinition(ctx context.Context, definition database.AgentDefinition) error { p.mu.Lock() defer p.mu.Unlock() return p.store.UpsertAgentDefinition(ctx, definition) } -func (p *preparationTestBackend) RecordRuntimeRevision(ctx context.Context, revision database.RuntimeRevision, ready bool) error { +func (p *preparationTestBackend) RecordRuntimeRevision(ctx context.Context, revision database.RuntimeRevision, options database.RecordRevisionOptions) error { p.mu.Lock() defer p.mu.Unlock() - return p.store.RecordRuntimeRevision(ctx, revision, ready) + return p.store.RecordRuntimeRevision(ctx, revision, options) } -func (p *preparationTestBackend) RetireAgentIdentities(ctx context.Context, namespace, name string, except *database.AgentDefinition) error { +func (p *preparationTestBackend) RetireAgentIdentities(ctx context.Context, namespace, name string) error { p.mu.Lock() defer p.mu.Unlock() - return p.store.RetireAgentIdentities(ctx, namespace, name, except) + return p.store.RetireAgentIdentities(ctx, namespace, name) } func TestPreparationDesiredChangesBypassBackoff(t *testing.T) { @@ -408,7 +417,7 @@ func TestPreparationDesiredChangesBypassBackoff(t *testing.T) { r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3()) ctx, cancel := context.WithCancel(t.Context()) defer cancel() - go r.Run(ctx.Done()) + go func() { assert.NoError(t, r.Start(ctx)) }() synctest.Wait() time.Sleep(10 * time.Second) synctest.Wait() @@ -462,7 +471,7 @@ func TestPreparationTerminalFailures(t *testing.T) { r := newReconciler(collections, templates, store, statusClient) ctx, cancel := context.WithCancel(t.Context()) defer cancel() - go r.Run(ctx.Done()) + go func() { assert.NoError(t, r.Start(ctx)) }() synctest.Wait() require.Equal(t, 1, store.pairCalls) state := collections.Reconciliations.GetKey(initial.ResourceName()) @@ -531,6 +540,11 @@ func TestInvalidActorTemplateHasNoCompiledTarget(t *testing.T) { } func newPreparationTestCollections(t *testing.T, workerPool string) (Collections, krt.StaticCollection[*kagentv1alpha3.Harness]) { + t.Helper() + return newExtensionTestCollections(t, workerPool, nil) +} + +func newExtensionTestCollections(t *testing.T, workerPool string, extension egressext.PolicyTranslator, objects ...any) (Collections, krt.StaticCollection[*kagentv1alpha3.Harness]) { t.Helper() opts := krt.NewOptionsBuilder(t.Context().Done(), "test-preparation", nil) template := &kagentv1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "assistant", UID: "template-uid"}} @@ -543,12 +557,13 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections WorkerPoolRef: corev1.LocalObjectReference{Name: workerPool}, SnapshotPolicy: kagentv1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}, } harnesses := krt.NewStaticCollection(nil, []*kagentv1alpha3.Harness{runtimeHarness}, opts.WithName("Harnesses")...) - mock := krttest.NewMock(t, []any{ + mock := krttest.NewMock(t, append([]any{ template, &atev1alpha1.WorkerPool{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "gvisor"}, Spec: atev1alpha1.WorkerPoolSpec{SandboxClass: atev1alpha1.SandboxClassGvisor}}, &atev1alpha1.WorkerPool{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "microvm"}, Spec: atev1alpha1.WorkerPoolSpec{SandboxClass: atev1alpha1.SandboxClassMicroVM}}, - }) + }, objects...)) collections := Collections{ + EgressPolicies: krt.NewStaticCollection[*kagentv1alpha3.EgressPolicy](nil, nil, opts.WithName("EgressPolicies")...), AgentTemplates: krttest.GetMockCollection[*kagentv1alpha3.AgentTemplate](mock), Harnesses: harnesses, ResolvedModelConfigs: krttest.GetMockCollection[translator.ResolvedModelConfig](mock), @@ -563,8 +578,8 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections collections.Reconciliations = newAgentReconciliations(collections.Agents, translator.Collections{ Harnesses: collections.Harnesses, AgentTemplates: collections.AgentTemplates, ResolvedModelConfigs: collections.ResolvedModelConfigs, RemoteMCPServers: collections.RemoteMCPServers, ConfigMaps: collections.ConfigMaps, - Secrets: collections.Secrets, WorkerPools: collections.WorkerPools, - }, collections.AgentRuntimeObservations, opts, testRuntimeDefaults(), nil) + Secrets: collections.Secrets, WorkerPools: collections.WorkerPools, EgressPolicies: collections.EgressPolicies, + }, collections.AgentRuntimeObservations, opts, testRuntimeDefaults(), nil, extension) collections.AgentStatuses = newAgentStatuses(collections.Agents, collections.Reconciliations, opts) waitFor(t, func() bool { states := collections.Reconciliations.List() @@ -572,3 +587,21 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections }) return collections, harnesses } + +func (p *preparationTestBackend) ListRuntimeEgressSourceTargets(ctx context.Context, source database.RuntimeEgressSource, after string, limit int) ([]database.RuntimeEgressTarget, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.store.ListRuntimeEgressSourceTargets(ctx, source, after, limit) +} +func (p *preparationTestBackend) GetActor(ctx context.Context, atespace, name string) (*ateapipb.Actor, error) { + return p.templates.GetActor(ctx, atespace, name) +} +func (p *preparationTestBackend) EnsureActorEgressPolicy(ctx context.Context, atespace, name, revision string) (egress.Policy, error) { + return p.templates.EnsureActorEgressPolicy(ctx, atespace, name, revision) +} + +func (p *preparationTestBackend) GetAgentEgressGrant(ctx context.Context, namespace, uid string) (egress.Access, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.store.GetAgentEgressGrant(ctx, namespace, uid) +} diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index df2c2eda5a..0628e4edec 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "reflect" + "slices" "strings" "time" @@ -26,12 +27,14 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/internal/translator" byotranslator "github.com/kagent-dev/kagent/go/core/internal/translator/byo" claudetranslator "github.com/kagent-dev/kagent/go/core/internal/translator/claude" codextranslator "github.com/kagent-dev/kagent/go/core/internal/translator/codex" kagenttranslator "github.com/kagent-dev/kagent/go/core/internal/translator/kagent" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) // AgentReconciliation is the complete desired and observed state for one @@ -39,6 +42,9 @@ import ( // status instead of disappearing from the graph. type AgentReconciliation struct { Agent *kagentv1alpha3.Agent + EgressGrant egress.Access + EgressIssues []translator.EgressGrantIssue + EgressConflict bool Target *compiledTarget Warnings []string CompilationFailure *ReconciliationFailure @@ -99,29 +105,27 @@ func newAgentReconciliations( opts krt.OptionsBuilder, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig, + policyTranslator egressext.PolicyTranslator, ) krt.Collection[AgentReconciliation] { return krt.NewCollection(agents, func(ctx krt.HandlerContext, agent *kagentv1alpha3.Agent) *AgentReconciliation { state := &AgentReconciliation{Agent: agent} + if !agent.DeletionTimestamp.IsZero() { + return state + } + resolution := translator.CompileEgressPolicies(ctx, collections, agent.Namespace, agent.Spec.EgressPolicyRefs) + // Admission still runs when an unrelated runtime input fails to compile. + defer func() { + base, tokenOrigins := state.Target.egressBase() + state.EgressGrant, state.EgressIssues, state.EgressConflict = evaluateEgressGrant(resolution, base, tokenOrigins) + }() compilation, err := translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings), translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings), translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings), - }, images).CompileAgent(context.Background(), agent) + }, images, policyTranslator).CompileAgent(context.Background(), agent) if err != nil { - condition, reason := kagentv1alpha3.AgentConditionResolvedRefs, "ReferenceResolutionFailed" - var validation *translator.ValidationError - var missingPool *translator.WorkerPoolNotFoundError - switch { - case errors.As(err, &validation): - // TODO(tim): Separate structural Agent validation from resolved - // compatibility failures so invalid sources do not report Accepted=True - // and ResolvedRefs=True. - condition, reason = kagentv1alpha3.AgentConditionCompatible, "UnsupportedConfiguration" - case errors.As(err, &missingPool): - reason = "WorkerPoolNotFound" - } - state.CompilationFailure = &ReconciliationFailure{Condition: condition, Reason: reason, Message: err.Error()} + state.CompilationFailure = compilationFailure(err) return state } revision := &compilation.Revision @@ -161,16 +165,36 @@ func newAgentReconciliations( }, opts.WithName("AgentReconciliations")...) } +func compilationFailure(err error) *ReconciliationFailure { + condition, reason := kagentv1alpha3.AgentConditionResolvedRefs, "ReferenceResolutionFailed" + var validation *translator.ValidationError + var missingPool *translator.WorkerPoolNotFoundError + switch { + case errors.As(err, &validation): + // TODO(tim): Separate structural Agent validation from resolved + // compatibility failures so invalid sources do not report Accepted=True + // and ResolvedRefs=True. + condition, reason = kagentv1alpha3.AgentConditionCompatible, "UnsupportedConfiguration" + case errors.As(err, &missingPool): + reason = "WorkerPoolNotFound" + } + return &ReconciliationFailure{Condition: condition, Reason: reason, Message: err.Error()} +} + // runtimeRevisionStore is the controller's narrow view of the shared database. // Substrate owns ActorTemplates; the database retains revisions while an Agent // or a Session or checkpoint references them. type runtimeRevisionStore interface { + actorEgressStore UpsertAgentDefinition(context.Context, database.AgentDefinition) error - RecordRuntimeRevision(context.Context, database.RuntimeRevision, bool) error - RetireAgentIdentities(ctx context.Context, namespace, name string, except *database.AgentDefinition) error + GetAgentEgressGrant(context.Context, string, string) (egress.Access, error) + RecordRuntimeRevision(context.Context, database.RuntimeRevision, database.RecordRevisionOptions) error + ListAgentDefinitionSources(context.Context) ([]database.RuntimeEgressSource, error) + RetireAgentIdentities(ctx context.Context, namespace, name string) error } type actorTemplateClient interface { + actorEgressClient EnsureAtespace(context.Context, string) error GetActorTemplate(context.Context, string, string) (*ateapipb.ActorTemplate, error) CreateActorTemplate(context.Context, *ateapipb.ActorTemplate) (*ateapipb.ActorTemplate, error) @@ -225,10 +249,13 @@ func newReconciler( }) r.agentHandler = collections.Reconciliations.Register(func(event krt.Event[AgentReconciliation]) { - // Observations and status writes must not bypass failure backoff. Only - // a changed desired runtime or Agent identity needs immediate work. + // Desired inputs and cleanup changes bypass failure backoff; observations + // and status writes do not. Grants must reconcile without a new revision. if event.Old != nil && event.New != nil && event.Old.Agent.UID == event.New.Agent.UID && - event.Old.desiredRevision() == event.New.desiredRevision() { + event.Old.desiredRevision() == event.New.desiredRevision() && + reflect.DeepEqual(event.Old.EgressGrant, event.New.EgressGrant) && + reflect.DeepEqual(event.Old.EgressIssues, event.New.EgressIssues) && + event.Old.EgressConflict == event.New.EgressConflict && event.Old.Agent.DeletionTimestamp.Equal(event.New.Agent.DeletionTimestamp) { return } r.agents.Add(krt.GetKey(event.Latest())) @@ -260,19 +287,44 @@ func newReconciliationQueue(name string, reconcile func(any) error) controllers. ) } -// Run waits for the graph boundary to observe initial state, then processes -// Agent and status writes until stop closes. -func (r *Reconciler) Run(stop <-chan struct{}) { +// Start waits for initial Kubernetes state, recovers offline deletions, then +// processes Agent preparation and status writes until cancellation. +func (r *Reconciler) Start(ctx context.Context) error { + stop := ctx.Done() if !r.agentHandler.WaitUntilSynced(stop) || !r.agentStatusHandler.WaitUntilSynced(stop) || !r.modelConfigStatusHandler.WaitUntilSynced(stop) { r.agents.ShutDownEarly() r.agentStatuses.ShutDownEarly() r.modelConfigStatuses.ShutDownEarly() - return + return nil + } + if err := r.enqueueDeletedAgents(ctx); err != nil { + r.agents.ShutDownEarly() + r.agentStatuses.ShutDownEarly() + r.modelConfigStatuses.ShutDownEarly() + return err } go r.pollPendingTemplates(stop) go r.agentStatuses.Run(stop) go r.modelConfigStatuses.Run(stop) r.agents.Run(stop) + return nil +} + +func (r *Reconciler) enqueueDeletedAgents(ctx context.Context) error { + sources, err := r.store.ListAgentDefinitionSources(ctx) + if err != nil { + return fmt.Errorf("recover deleted Agents: %w", err) + } + for _, source := range sources { + if len(r.collections.watchNamespaces) != 0 && !slices.Contains(r.collections.watchNamespaces, source.Namespace) { + continue + } + key := source.Namespace + "/" + source.Name + if r.collections.Agents.GetKey(key) == nil { + r.agents.Add(key) + } + } + return nil } func (r *Reconciler) pollPendingTemplates(stop <-chan struct{}) { @@ -293,11 +345,6 @@ func (r *Reconciler) pollPendingTemplates(stop <-chan struct{}) { } } -func (r *Reconciler) Start(ctx context.Context) error { - r.Run(ctx.Done()) - return nil -} - func (r *Reconciler) NeedLeaderElection() bool { return true } func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { @@ -306,19 +353,32 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { (state == nil || state.Target == nil || observation.RevisionID != state.Target.RevisionID) { r.collections.AgentRuntimeObservations.DeleteObject(key) } - if state == nil { + if state == nil || !state.Agent.DeletionTimestamp.IsZero() { parts := strings.Split(key, "/") if len(parts) != 2 { return fmt.Errorf("invalid Agent key %q", key) } - if err := r.store.RetireAgentIdentities(ctx, parts[0], parts[1], nil); err != nil { + err := r.store.RetireAgentIdentities(ctx, parts[0], parts[1]) + if err != nil { return fmt.Errorf("retire Agent %s: %w", key, err) } + if err := applySourceEgress(ctx, r.store, r.templates, database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: parts[0], Name: parts[1]}); err != nil { + return err + } return nil } + grants := state.EgressGrant + if state.EgressConflict { + previous, err := r.store.GetAgentEgressGrant(ctx, state.Agent.Namespace, string(state.Agent.UID)) + if err != nil { + return r.observePreparationError(*state, fmt.Errorf("read Agent %s grants: %w", key, err)) + } + base, tokenOrigins := state.Target.egressBase() + grants = retainedEgressGrant(previous, grants, base, tokenOrigins) + } definition := database.AgentDefinition{ Namespace: state.Agent.Namespace, AgentName: state.Agent.Name, - AgentUID: string(state.Agent.UID), DesiredRevision: state.desiredRevision(), + AgentUID: string(state.Agent.UID), DesiredRevision: state.desiredRevision(), EgressGrant: grants, } // Store the desired edge before creating compute so a concurrent collector @@ -328,6 +388,9 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { if err := r.store.UpsertAgentDefinition(ctx, definition); err != nil { return r.observePreparationError(*state, fmt.Errorf("store Agent %s: %w", key, err)) } + if err := applySourceEgress(ctx, r.store, r.templates, database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: definition.Namespace, Name: definition.AgentName}); err != nil { + return r.observePreparationError(*state, err) + } if state.Target == nil { return nil } @@ -368,14 +431,16 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { Revision: target.RevisionID.String(), Namespace: definition.Namespace, AgentName: definition.AgentName, AgentUID: definition.AgentUID, SourceSnapshot: target.Revision.Provenance, AgentCard: target.Revision.AgentCard, - EgressDestinations: target.Revision.EgressDestinations, - Credentials: target.Revision.Credentials, - ActorTemplateAtespace: observed.GetMetadata().GetAtespace(), ActorTemplateName: observed.GetMetadata().GetName(), ActorTemplateUID: observed.GetMetadata().GetUid(), + EgressDestinations: target.Revision.EgressDestinations, + Credentials: target.Revision.Credentials, + TokenPassthroughOrigins: target.Revision.TokenPassthroughOrigins, + ActorTemplateAtespace: observed.GetMetadata().GetAtespace(), ActorTemplateName: observed.GetMetadata().GetName(), ActorTemplateUID: observed.GetMetadata().GetUid(), } ready := observed.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() != nil - if err := r.store.RecordRuntimeRevision(ctx, revision, ready); err != nil { + if err := r.store.RecordRuntimeRevision(ctx, revision, database.RecordRevisionOptions{Ready: ready}); err != nil { return r.observePreparationError(*state, fmt.Errorf("store runtime revision %s: %w", target.RevisionID, err)) } + // This observation drives Kubernetes Ready status on a separate queue. // Publish it only after session creation can select the persisted revision. r.observePreparation(*state, observed, nil) diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index 8548e6adc1..730151ffac 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -11,6 +11,7 @@ import ( a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -26,6 +27,7 @@ import ( kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -183,11 +185,6 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { states.UpdateObject(state) require.NoError(t, reconciler.reconcileAgent(ctx, state.ResourceName())) - // Compile failures preserve the current UID's last successful runtime. - state.Target = nil - states.UpdateObject(state) - require.NoError(t, reconciler.reconcileAgent(ctx, state.ResourceName())) - require.Empty(t, reconciler.collections.AgentRuntimeObservations.List(), "invalid preparation must release its observation before GC") request := &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}, @@ -195,6 +192,17 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { session, _, err := store.CreateSession(ctx, request, "session") require.NoError(t, err) require.Equal(t, id.String(), session.GetPreparedRevision()) + + // Compile failures preserve the last successful runtime and pinned retries. + state.Target = nil + states.UpdateObject(state) + require.NoError(t, reconciler.reconcileAgent(ctx, state.ResourceName())) + require.Empty(t, reconciler.collections.AgentRuntimeObservations.List(), "invalid preparation must release its observation before GC") + retried, created, err := store.CreateSession(ctx, request, "session") + require.NoError(t, err) + require.False(t, created) + require.Equal(t, session.GetId(), retried.GetId()) + require.Equal(t, id.String(), retried.GetPreparedRevision()) operation, err := store.BeginSessionOperation(ctx, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE) require.NoError(t, err) executor := uuid.New() @@ -272,6 +280,7 @@ func (s *failingFinalizationStore) DeleteRuntimeRevision(ctx context.Context, re } type fakeActorTemplates struct { + actorEgressStub template *ateapipb.ActorTemplate ensureErr error getErr error @@ -313,13 +322,21 @@ func (f *fakeActorTemplates) DeleteActorTemplate(context.Context, string, string } type fakeRuntimeRevisionStore struct { - pair *database.AgentDefinition - revision *database.RuntimeRevision - markedSuccessful bool - retired string - revisionErr error - pairErr error - pairCalls int + egressTargetStoreStub + pair *database.AgentDefinition + revision *database.RuntimeRevision + markedSuccessful bool + retired string + revisionErr error + recordCalls int + pairErr error + pairCalls int + definitionSources []database.RuntimeEgressSource + sourcesErr error +} + +func (s *fakeRuntimeRevisionStore) ListAgentDefinitionSources(context.Context) ([]database.RuntimeEgressSource, error) { + return s.definitionSources, s.sourcesErr } func (s *fakeRuntimeRevisionStore) UpsertAgentDefinition(_ context.Context, pair database.AgentDefinition) error { @@ -328,19 +345,20 @@ func (s *fakeRuntimeRevisionStore) UpsertAgentDefinition(_ context.Context, pair return s.pairErr } -func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision, ready bool) error { +func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision, options database.RecordRevisionOptions) error { + s.recordCalls++ if s.revisionErr != nil { return s.revisionErr } s.revision = &revision - s.markedSuccessful = ready + if options.Ready { + s.markedSuccessful = true + } return nil } -func (s *fakeRuntimeRevisionStore) RetireAgentIdentities(_ context.Context, namespace, template string, except *database.AgentDefinition) error { - if except == nil { - s.retired = namespace + "/" + template - } +func (s *fakeRuntimeRevisionStore) RetireAgentIdentities(_ context.Context, namespace, template string) error { + s.retired = namespace + "/" + template return nil } @@ -387,7 +405,7 @@ func TestReconcilerUpdatesModelConfigStatusOnSecretHashChange(t *testing.T) { statusClient, ) - go reconciler.Run(stop) + go func() { assert.NoError(t, reconciler.Start(t.Context())) }() var initialUpdate *kagentv1alpha3.ModelConfig var err error @@ -450,3 +468,10 @@ func TestReconciliationQueueRetriesWithBackoff(t *testing.T) { require.NoError(t, queue.WaitForClose(time.Second)) }) } + +func (s *fakeRuntimeRevisionStore) GetAgentEgressGrant(_ context.Context, namespace, uid string) (egress.Access, error) { + if s.pair != nil && s.pair.Namespace == namespace && s.pair.AgentUID == uid && s.retired == "" { + return s.pair.EgressGrant, nil + } + return egress.Access{}, nil +} diff --git a/go/core/internal/controller/runtime.go b/go/core/internal/controller/runtime.go index 5250425f98..cf9743175b 100644 --- a/go/core/internal/controller/runtime.go +++ b/go/core/internal/controller/runtime.go @@ -10,6 +10,7 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/core/internal/controller/apiclient" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) // Runtime owns the Kubernetes client and common options shared by the KRT @@ -23,13 +24,13 @@ type Runtime struct { // NewRuntime creates the shared KRT client and collection graph. Handlers are // deliberately registered separately at the eventual application boundary. -func NewRuntime(config *rest.Config, watchNamespaces []string, stop <-chan struct{}, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig) (*Runtime, error) { +func NewRuntime(config *rest.Config, watchNamespaces []string, stop <-chan struct{}, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig, policyTranslator egressext.PolicyTranslator) (*Runtime, error) { client, err := apiclient.New(config) if err != nil { return nil, fmt.Errorf("create KRT Kubernetes client: %w", err) } options := krt.NewOptionsBuilder(stop, "kagent", krt.GlobalDebugHandler) - collections := NewCollections(client, watchNamespaces, options, settings, images) + collections := NewCollections(client, watchNamespaces, options, settings, images, policyTranslator) return &Runtime{Client: client, Options: options, Collections: collections}, nil } diff --git a/go/core/internal/controller/sandbox_collections.go b/go/core/internal/controller/sandbox_collections.go index 52712a7eea..ebf6102120 100644 --- a/go/core/internal/controller/sandbox_collections.go +++ b/go/core/internal/controller/sandbox_collections.go @@ -10,7 +10,9 @@ import ( "istio.io/istio/pkg/kube/krt" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" + "github.com/kagent-dev/kagent/go/core/internal/translator" ) // Sandbox preparation uses the same informer graph as Agents. Only observations @@ -24,6 +26,9 @@ type sandboxReconciliation struct { Template *kagentv1alpha3.SandboxTemplate RevisionID string SourceSnapshot json.RawMessage + EgressGrant egress.Access + EgressIssues []translator.EgressGrantIssue + EgressConflict bool DesiredActorTemplate *ateapipb.ActorTemplate ObservedActorTemplate *ateapipb.ActorTemplate CompilationError string @@ -84,6 +89,9 @@ func newSandboxCollections(inputs Collections, policy substrate.SandboxPolicy, o if !template.DeletionTimestamp.IsZero() { return state } + resolution := translator.CompileEgressPolicies(ctx, translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets}, template.Namespace, template.Spec.EgressPolicyRefs) + // Sandboxes have no pinned model or tool access. + state.EgressGrant, state.EgressIssues, state.EgressConflict = evaluateEgressGrant(resolution, egress.Access{}, nil) pool := krt.FetchOne(ctx, inputs.WorkerPools, krt.FilterKey(template.Namespace+"/"+template.Spec.Substrate.WorkerPoolRef.Name)) if pool == nil { state.Failure = &ReconciliationFailure{Reason: "WorkerPoolNotFound", Message: "The referenced sandbox WorkerPool does not exist"} diff --git a/go/core/internal/controller/sandbox_integration_test.go b/go/core/internal/controller/sandbox_integration_test.go index 133fe4b5ee..c9cb9c079c 100644 --- a/go/core/internal/controller/sandbox_integration_test.go +++ b/go/core/internal/controller/sandbox_integration_test.go @@ -42,6 +42,7 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { template := sandboxTestTemplate() template.UID = "" template.Generation = 0 + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "tickets"}} template, err = client.ApiV1alpha3().SandboxTemplates(template.Namespace).Create(t.Context(), template, metav1.CreateOptions{}) require.NoError(t, err) unwatched := template.DeepCopy() @@ -60,6 +61,8 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { options := krt.NewOptionsBuilder(ctx.Done(), "sandbox-integration", nil) pools := krt.NewStaticCollection(nil, []*atev1alpha1.WorkerPool{pool}, options.WithName("WorkerPools")...) runtime := &Runtime{Client: runtimeClient, Options: options, Collections: Collections{ + EgressPolicies: typedCollection[*kagentv1alpha3.EgressPolicy](runtimeClient, []string{"team-a"}, "EgressPolicies", options), + Secrets: typedCollection[*corev1.Secret](runtimeClient, []string{"team-a"}, "Secrets", options), SandboxTemplates: typedCollection[*kagentv1alpha3.SandboxTemplate](runtimeClient, []string{"team-a"}, "SandboxTemplates", options), WorkerPools: pools, }} reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}) @@ -89,6 +92,27 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { } return apimeta.IsStatusConditionTrue(current.Status.Conditions, "Ready") } + waitUnresolved := func(reason, message string) { + t.Helper() + require.Eventually(t, func() bool { + current, err := client.ApiV1alpha3().SandboxTemplates(template.Namespace).Get(t.Context(), template.Name, metav1.GetOptions{}) + if err != nil { + return false + } + condition := apimeta.FindStatusCondition(current.Status.Conditions, kagentv1alpha3.SandboxTemplateConditionEgressPoliciesApplied) + return condition != nil && condition.Status == metav1.ConditionFalse && condition.Reason == reason && strings.Contains(condition.Message, message) + }, 10*time.Second, 10*time.Millisecond) + } + waitUnresolved(kagentv1alpha3.EgressPoliciesReasonNotFound, "tickets") + require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) + grant := &kagentv1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "tickets"}, Spec: kagentv1alpha3.EgressPolicySpec{ + Destinations: []kagentv1alpha3.EgressDestination{{Origin: "https://tickets.example.com", Authentication: &kagentv1alpha3.EgressAuthentication{Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "ticket-auth"}, Key: "token"}}}}, + }} + grant, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Create(t.Context(), grant, metav1.CreateOptions{}) + require.NoError(t, err) + waitUnresolved(kagentv1alpha3.EgressPoliciesReasonInvalidEgressPolicy, "Secret") + secret, err := kube.Kube().CoreV1().Secrets(template.Namespace).Create(t.Context(), &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "ticket-auth"}, Data: map[string][]byte{"token": []byte("original-token")}}, metav1.CreateOptions{}) + require.NoError(t, err) require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) current, err := client.ApiV1alpha3().SandboxTemplates(template.Namespace).Get(t.Context(), template.Name, metav1.GetOptions{}) require.NoError(t, err) @@ -96,12 +120,42 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { store.mu.Lock() revision := store.revision.Revision require.True(t, store.ready) + require.NotContains(t, string(store.revision.SourceSnapshot), "original-token") store.mu.Unlock() other, err := client.ApiV1alpha3().SandboxTemplates(unwatched.Namespace).Get(t.Context(), unwatched.Name, metav1.GetOptions{}) require.NoError(t, err) require.Empty(t, other.Finalizers, "namespace filters must apply to the new informer") require.Empty(t, other.Status.Conditions) + // A missing key revokes its policy without changing runtime readiness. + // Restoring it admits grants without serializing either value. + secret.Data = nil + secret, err = kube.Kube().CoreV1().Secrets(template.Namespace).Update(t.Context(), secret, metav1.UpdateOptions{}) + require.NoError(t, err) + waitUnresolved(kagentv1alpha3.EgressPoliciesReasonInvalidEgressPolicy, "missing key") + require.True(t, ready()) + secret.Data = map[string][]byte{"token": []byte("rotated-token")} + _, err = kube.Kube().CoreV1().Secrets(template.Namespace).Update(t.Context(), secret, metav1.UpdateOptions{}) + require.NoError(t, err) + require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) + store.mu.Lock() + require.Equal(t, revision, store.revision.Revision) + store.mu.Unlock() + grant.Spec.Destinations = append(grant.Spec.Destinations, kagentv1alpha3.EgressDestination{Origin: "https://registry.npmjs.org"}) + grant, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Update(t.Context(), grant, metav1.UpdateOptions{}) + require.NoError(t, err) + require.NoError(t, client.ApiV1alpha3().EgressPolicies(template.Namespace).Delete(t.Context(), grant.Name, metav1.DeleteOptions{})) + waitUnresolved(kagentv1alpha3.EgressPoliciesReasonNotFound, "tickets") + require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) + grant.ResourceVersion, grant.UID = "", "" + _, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Create(t.Context(), grant, metav1.CreateOptions{}) + require.NoError(t, err) + require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) + + store.mu.Lock() + require.Equal(t, revision, store.revision.Revision, "policy changes do not reprepare the runtime") + store.mu.Unlock() + // A dependency update alone must change the persisted revision. changedPool := pool.DeepCopy() changedPool.Spec.SandboxClass = atev1alpha1.SandboxClassMicroVM diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index 884ee3439d..ddec7857c0 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -21,6 +21,7 @@ import ( kagentclient "github.com/kagent-dev/kagent/go/api/clientset/versioned/typed/api/v1alpha3" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" ) @@ -31,9 +32,11 @@ const ( ) type sandboxRevisionStore interface { + actorEgressStore UpsertSandboxTemplateDefinition(context.Context, database.SandboxTemplateDefinition) error + GetSandboxEgressGrant(context.Context, string, string) (egress.Access, error) RetireSandboxTemplateIdentities(context.Context, string, string) error - RecordSandboxRevision(context.Context, database.SandboxRevision, bool) error + RecordSandboxRevision(context.Context, database.SandboxRevision, database.RecordRevisionOptions) error } // SandboxReconciler owns the queued side effects of SandboxTemplate preparation. @@ -112,9 +115,13 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { } if state == nil || !state.Template.DeletionTimestamp.IsZero() { namespace, name, _ := strings.Cut(key, "/") - if err := s.store.RetireSandboxTemplateIdentities(ctx, namespace, name); err != nil { + err := s.store.RetireSandboxTemplateIdentities(ctx, namespace, name) + if err != nil { return fmt.Errorf("retire SandboxTemplate %s: %w", key, err) } + if err := applySourceEgress(ctx, s.store, s.actors, database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: namespace, Name: name}); err != nil { + return err + } if state != nil && slices.Contains(state.Template.Finalizers, sandboxPreparationFinalizer) { updated := state.Template.DeepCopy() updated.Finalizers = slices.DeleteFunc(updated.Finalizers, func(value string) bool { return value == sandboxPreparationFinalizer }) @@ -134,11 +141,23 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { } return nil } - if err := s.store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{ - Namespace: state.Template.Namespace, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), DesiredRevision: state.desiredRevision(), - }); err != nil { + grants := state.EgressGrant + if state.EgressConflict { + previous, err := s.store.GetSandboxEgressGrant(ctx, state.Template.Namespace, string(state.Template.UID)) + if err != nil { + return s.observeError(*state, fmt.Errorf("read SandboxTemplate %s grants: %w", key, err), true) + } + grants = retainedEgressGrant(previous, grants, egress.Access{}, nil) + } + err := s.store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{ + Namespace: state.Template.Namespace, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), DesiredRevision: state.desiredRevision(), EgressGrant: grants, + }) + if err != nil { return s.observeError(*state, fmt.Errorf("store SandboxTemplate %s definition: %w", key, err), true) } + if err := applySourceEgress(ctx, s.store, s.actors, database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: state.Template.Namespace, Name: state.Template.Name}); err != nil { + return s.observeError(*state, err, true) + } if state.DesiredActorTemplate == nil { s.collections.observations.DeleteObject(key) } @@ -175,9 +194,10 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), SourceSnapshot: state.SourceSnapshot, } - if err := s.store.RecordSandboxRevision(ctx, revision, ready); err != nil { + if err := s.store.RecordSandboxRevision(ctx, revision, database.RecordRevisionOptions{Ready: ready}); err != nil { return s.observeError(*state, fmt.Errorf("store sandbox revision %s: %w", state.RevisionID, err), true) } + // Kubernetes readiness can only follow a revision that callers can select // from the store. Failed persistence must never publish a ready observation. s.collections.observations.ConditionalUpdateObject(sandboxRuntimeObservation{Key: key, RevisionID: state.RevisionID, Template: observed}) @@ -221,5 +241,6 @@ func sandboxStatusWithTransitionTimes(state sandboxReconciliation) kagentv1alpha } desired.ObservedGeneration = state.Template.Generation apimeta.SetStatusCondition(&desired.Conditions, condition) + apimeta.SetStatusCondition(&desired.Conditions, egressPoliciesCondition(kagentv1alpha3.SandboxTemplateConditionEgressPoliciesApplied, state.Template.Generation, state.EgressIssues, state.EgressConflict)) return desired } diff --git a/go/core/internal/controller/sandbox_reconciler_test.go b/go/core/internal/controller/sandbox_reconciler_test.go index eee798f795..bd22e0d564 100644 --- a/go/core/internal/controller/sandbox_reconciler_test.go +++ b/go/core/internal/controller/sandbox_reconciler_test.go @@ -25,11 +25,14 @@ import ( kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" ) // These fakes are also used by the informer/queue integration test. type sandboxTestStore struct { + egressTargetStoreStub + upsertErr error mu sync.Mutex desired database.SandboxTemplateDefinition revision database.SandboxRevision @@ -38,21 +41,26 @@ type sandboxTestStore struct { retireAttempts int retireErr error recordErr error + recordCalls int } func (s *sandboxTestStore) UpsertSandboxTemplateDefinition(_ context.Context, value database.SandboxTemplateDefinition) error { s.mu.Lock() defer s.mu.Unlock() s.desired = value - return nil + return s.upsertErr } -func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision, ready bool) error { +func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision, options database.RecordRevisionOptions) error { s.mu.Lock() defer s.mu.Unlock() + s.recordCalls++ if s.recordErr != nil { return s.recordErr } - s.revision, s.ready = value, ready + s.revision = value + if options.Ready { + s.ready = true + } return nil } func (s *sandboxTestStore) RetireSandboxTemplateIdentities(context.Context, string, string) error { @@ -69,6 +77,7 @@ func (s *sandboxTestStore) RetireSandboxTemplateIdentities(context.Context, stri var _ sandboxRevisionStore = (*sandboxTestStore)(nil) type sandboxTestActors struct { + actorEgressStub mu sync.Mutex store *sandboxTestStore templates map[string]*ateapipb.ActorTemplate @@ -295,3 +304,12 @@ func TestSandboxPendingStatusRecoversWithoutGraphEvent(t *testing.T) { return err == nil && apimeta.IsStatusConditionTrue(current.Status.Conditions, "Ready") }, 5*time.Second, 10*time.Millisecond) } + +func (s *sandboxTestStore) GetSandboxEgressGrant(_ context.Context, namespace, uid string) (egress.Access, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.desired.Namespace == namespace && s.desired.SandboxTemplateUID == uid && !s.retired { + return s.desired.EgressGrant, nil + } + return egress.Access{}, nil +} diff --git a/go/core/internal/controller/status.go b/go/core/internal/controller/status.go index 47828ad737..dff6ce40d0 100644 --- a/go/core/internal/controller/status.go +++ b/go/core/internal/controller/status.go @@ -23,6 +23,7 @@ func statusForAgent(state AgentReconciliation, generation int64, latestSuccessfu status := kagentv1alpha3.AgentStatus{ ObservedGeneration: generation, DesiredRevision: state.desiredRevision(), LatestSuccessfulRevision: latestSuccessful, } + status.Conditions = append(status.Conditions, egressPoliciesCondition(kagentv1alpha3.AgentConditionEgressPoliciesApplied, generation, state.EgressIssues, state.EgressConflict)) status.Warnings = append([]string(nil), state.Warnings...) if state.Target != nil { status.WorkloadImage = new(state.Target.Revision.Image) diff --git a/go/core/internal/database/client_session_test.go b/go/core/internal/database/client_session_test.go index 99a49bde77..b06452fc49 100644 --- a/go/core/internal/database/client_session_test.go +++ b/go/core/internal/database/client_session_test.go @@ -377,7 +377,7 @@ func TestForkSessionCopiesBoundedHistory(t *testing.T) { if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision, true); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true}); err != nil { t.Fatal(err) } @@ -536,7 +536,7 @@ func TestSessionCreateAndTransitions(t *testing.T) { if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision, true); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true}); err != nil { t.Fatal(err) } @@ -638,7 +638,7 @@ func sessionFixture(t *testing.T, client *Client, ctx context.Context, namespace if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision, true); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true}); err != nil { t.Fatal(err) } } @@ -952,7 +952,7 @@ func TestDeletedSessionPreservesRequestIdentityAndHidesAccess(t *testing.T) { _, _, err = client.CreateSession(ctx, request, "stable-request") require.ErrorIs(t, err, ErrIdempotencyConflict) // Tombstones release the revision FK and no longer block runtime cleanup. - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) require.Len(t, revisions, 1) diff --git a/go/core/internal/database/models.go b/go/core/internal/database/models.go index 3f66aea9e6..4702c64372 100644 --- a/go/core/internal/database/models.go +++ b/go/core/internal/database/models.go @@ -51,25 +51,27 @@ type AgentMemorySearchResult struct { // AgentDefinition tracks the desired and last successful runtime for an Agent UID. type AgentDefinition struct { - Namespace string - AgentName string - AgentUID string + Namespace string + AgentName string + AgentUID string + EgressGrant egress.Access // DesiredRevision is a compiled runtime digest, or empty while inputs are unresolved. DesiredRevision string } type RuntimeRevision struct { - Revision string - Namespace string - AgentName string - AgentUID string - SourceSnapshot json.RawMessage - AgentCard *a2apb.AgentCard - Credentials []egress.Credential - EgressDestinations []string - ActorTemplateAtespace string - ActorTemplateName string - ActorTemplateUID string + Revision string + Namespace string + AgentName string + AgentUID string + SourceSnapshot json.RawMessage + AgentCard *a2apb.AgentCard + Credentials []egress.Credential + EgressDestinations []string + TokenPassthroughOrigins []string + ActorTemplateAtespace string + ActorTemplateName string + ActorTemplateUID string } // SessionQuery narrows a page of sessions to an optional Agent. diff --git a/go/core/internal/database/protobuf_test.go b/go/core/internal/database/protobuf_test.go index 6e962a5771..ab3104ab59 100644 --- a/go/core/internal/database/protobuf_test.go +++ b/go/core/internal/database/protobuf_test.go @@ -286,11 +286,11 @@ func TestShareAndAgentCardProtobufPersistence(t *testing.T) { require.NoError(t, err) addUnknown(card) revision := RuntimeRevision{Revision: "revision", Namespace: "team-a", AgentName: "assistant", AgentUID: "template", SourceSnapshot: []byte(`{}`), AgentCard: card, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "template"} - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, false)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{})) // Reconciliation can update runtime identity, but the pinned card is immutable. revision.AgentCard = &a2apb.AgentCard{Name: "replacement"} revision.ActorTemplateUID = "new-uid" - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, false)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{})) stored, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) require.True(t, proto.Equal(card, stored.AgentCard)) diff --git a/go/core/internal/database/runtime_creation_test.go b/go/core/internal/database/runtime_creation_test.go new file mode 100644 index 0000000000..e3b83005df --- /dev/null +++ b/go/core/internal/database/runtime_creation_test.go @@ -0,0 +1,83 @@ +package database + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func TestRuntimeCreationUsesLatestSuccessfulRevision(t *testing.T) { + for _, kind := range []string{"session", "sandbox"} { + t.Run(kind, func(t *testing.T) { + pool := setupTestDB(t) + client := NewClient(pool, "public") + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + var create func(*Client, string) (string, bool, error) + var desire func(*Client, string) error + var prepare func(string) + var unresolved string + if kind == "session" { + prepare = func(revision string) { + sessionFixture(t, client, ctx, "team-a", revision, "assistant", "kagent") + } + create = func(c *Client, requestID string) (string, bool, error) { + session, created, err := c.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), requestID) + return session.GetPreparedRevision(), created, err + } + desire = func(c *Client, revision string) error { + err := c.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", DesiredRevision: revision}) + return err + } + } else { + input, options := sandboxFixture(t, client, "revision-1") + prepare = func(revision string) { + _, _ = sandboxFixture(t, client, revision) + } + create = func(c *Client, requestID string) (string, bool, error) { + request := proto.CloneOf(input) + request.Id = uuid.NewString() + instance, created, err := c.CreateSandbox(ctx, request, requestID, options) + return instance.GetPreparedRevision(), created, err + } + desire = func(c *Client, revision string) error { + err := c.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) + return err + } + unresolved = "pending:uid:2" + } + prepare("revision-1") + revision, created, err := create(client, "original") + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "revision-1", revision) + + // Unresolved and pending edits keep the last successful runtime available. + // Retries retain their original revision even after a newer one is ready. + for _, desired := range []string{unresolved, "revision-2"} { + require.NoError(t, desire(client, desired)) + revision, created, err = create(client, "fresh-"+desired) + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "revision-1", revision) + revision, created, err = create(client, "original") + require.NoError(t, err) + require.False(t, created) + require.Equal(t, "revision-1", revision) + } + prepare("revision-2") + revision, created, err = create(client, "fresh") + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "revision-2", revision) + revision, created, err = create(client, "original") + require.NoError(t, err) + require.False(t, created) + require.Equal(t, "revision-1", revision) + }) + } +} diff --git a/go/core/internal/database/runtime_egress.go b/go/core/internal/database/runtime_egress.go new file mode 100644 index 0000000000..2dbc3d4d68 --- /dev/null +++ b/go/core/internal/database/runtime_egress.go @@ -0,0 +1,157 @@ +package database + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +// canonicalStoredGrants validates and normalizes source grants for NOT NULL storage. +func canonicalStoredGrants(grants egress.Access) (egress.Access, error) { + canonical, err := egress.CanonicalAccess(grants.Destinations, grants.Credentials) + if err != nil { + return egress.Access{}, err + } + if canonical.Destinations == nil { + canonical.Destinations = []string{} + } + if canonical.Credentials == nil { + canonical.Credentials = []egress.Credential{} + } + return canonical, nil +} + +// GetRuntimeEgressPolicy merges pinned inputs with the active source UID's grants +// in one database snapshot. Missing/deleting revisions and malformed inputs fail +// closed. Retired or replaced identities contribute no grants. Older revisions +// retain their pinned base and only the grants compatible with it. +func (c *Client) GetRuntimeEgressPolicy(ctx context.Context, revision string) (egress.Policy, error) { + type policyInputs struct { + EgressDestinations []string + Credentials []egress.Credential + TokenPassthroughOrigins []string + GrantDestinations []string + GrantCredentials []egress.Credential + } + row, err := queryOne(ctx, c.db, ` + SELECT r.egress_destinations, r.credentials, r.token_passthrough_origins, + COALESCE(ad.grant_destinations, sd.grant_destinations, '{}') AS grant_destinations, + COALESCE(ad.grant_credentials, sd.grant_credentials, '[]') AS grant_credentials + FROM runtime_revision r + LEFT JOIN agent_revision a ON r.kind = 'agent' AND a.revision = r.revision + LEFT JOIN sandbox_revision s ON r.kind = 'sandbox' AND s.revision = r.revision + LEFT JOIN agent_definition ad ON ad.namespace = r.namespace AND ad.agent_uid = a.agent_uid AND ad.retired_at IS NULL + LEFT JOIN sandbox_template_definition sd ON sd.namespace = r.namespace AND sd.sandbox_template_uid = s.sandbox_template_uid AND sd.retired_at IS NULL + WHERE r.revision = $1 AND r.deleted_at IS NULL + AND (a.revision IS NOT NULL OR s.revision IS NOT NULL) + `, pgx.RowToStructByName[policyInputs], revision) + if err != nil { + return egress.Policy{}, fmt.Errorf("get runtime egress policy %s: %w", revision, notFoundOr(err)) + } + for i, raw := range row.TokenPassthroughOrigins { + origin, err := egress.ParseOrigin(raw) + if err != nil { + return egress.Policy{}, fmt.Errorf("decode runtime token origins %s: %w", revision, err) + } + row.TokenPassthroughOrigins[i] = origin.String() + } + base, err := egress.MergeAccess(egress.Access{Destinations: row.EgressDestinations, Credentials: row.Credentials}, egress.Access{}, row.TokenPassthroughOrigins) + if err != nil { + return egress.Policy{}, fmt.Errorf("decode runtime egress base %s: %w", revision, err) + } + grants, err := egress.CanonicalAccess(row.GrantDestinations, row.GrantCredentials) + if err != nil { + return egress.Policy{}, fmt.Errorf("decode runtime egress grants %s: %w", revision, err) + } + access, err := egress.EffectiveAccess(base, grants, row.TokenPassthroughOrigins) + return egress.Policy{Access: access}, err +} + +var _ egress.PolicyStore = (*Client)(nil) + +// RuntimeEgressSourceKind identifies a Kubernetes owner of runtime instances. +type RuntimeEgressSourceKind string + +const ( + RuntimeEgressAgent RuntimeEgressSourceKind = runtimeKindAgent + RuntimeEgressSandbox RuntimeEgressSourceKind = runtimeKindSandbox +) + +// RuntimeEgressSource includes former UIDs whose grants must be revoked on recreation. +type RuntimeEgressSource struct { + Kind RuntimeEgressSourceKind + Namespace string + Name string +} + +// RuntimeEgressTarget identifies allocated instances, including suspended actors. +type RuntimeEgressTarget struct { + ID string + Revision string + Namespace string + Atespace string + TemplateName string + ActorUID string +} + +// ListRuntimeEgressSourceTargets pages allocated instances across a source's UIDs +// in ID order, excluding deletions and unissued creations. Invalid sources return +// an error; a missing source yields an empty page. +func (c *Client) ListRuntimeEgressSourceTargets(ctx context.Context, source RuntimeEgressSource, afterID string, limit int) ([]RuntimeEgressTarget, error) { + if source.Namespace == "" || source.Name == "" { + return nil, fmt.Errorf("runtime egress source requires namespace and name") + } + switch source.Kind { + case RuntimeEgressAgent: + return queryMany(ctx, c.db, ` + SELECT i.id::text AS id, i.prepared_revision AS revision, r.namespace, + r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, + COALESCE(s.actor_uid, '') AS actor_uid + FROM runtime_instance i + JOIN runtime_revision r ON r.revision = i.prepared_revision + JOIN agent_revision a ON a.revision = r.revision + JOIN session s ON s.id = i.id + WHERE i.kind = 'agent' AND r.kind = 'agent' AND r.namespace = $1 AND a.agent_name = $2 + AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) + AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' + AND i.id > COALESCE(NULLIF($3, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY i.id LIMIT $4 + `, pgx.RowToStructByName[RuntimeEgressTarget], source.Namespace, source.Name, afterID, limit) + case RuntimeEgressSandbox: + return queryMany(ctx, c.db, ` + SELECT i.id::text AS id, i.prepared_revision AS revision, r.namespace, + r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, + '' AS actor_uid + FROM runtime_instance i + JOIN runtime_revision r ON r.revision = i.prepared_revision + JOIN sandbox_revision s ON s.revision = r.revision + WHERE i.kind = 'sandbox' AND r.kind = 'sandbox' AND r.namespace = $1 AND s.sandbox_template_name = $2 + AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) + AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' + AND i.id > COALESCE(NULLIF($3, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY i.id LIMIT $4 + `, pgx.RowToStructByName[RuntimeEgressTarget], source.Namespace, source.Name, afterID, limit) + default: + return nil, fmt.Errorf("unknown runtime egress source kind %q", source.Kind) + } +} + +// decodeEgressGrant treats absent definitions as empty grants and rejects invalid +// durable rules. Callers scope the read to an active Kubernetes source identity. +func decodeEgressGrant(grants egress.Access, err error) (egress.Access, error) { + if errors.Is(err, pgx.ErrNoRows) { + return egress.Access{}, nil + } + if err != nil { + return egress.Access{}, err + } + canonical, err := egress.CanonicalAccess(grants.Destinations, grants.Credentials) + if err != nil { + return egress.Access{}, fmt.Errorf("decode source egress grants: %w", err) + } + return canonical, nil +} diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go new file mode 100644 index 0000000000..fdb8bf2ca5 --- /dev/null +++ b/go/core/internal/database/runtime_egress_test.go @@ -0,0 +1,314 @@ +package database + +import ( + "context" + "fmt" + "slices" + "testing" + + a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "scratch", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", TokenPassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} + upsertErr := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "scratch", AgentUID: "old-uid", DesiredRevision: "agent"}) + require.NoError(t, upsertErr) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) + instance, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "scratch", "kagent", ""), "request") + require.NoError(t, err) + pendingTargets, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "scratch"}, "", 10) + require.NoError(t, err) + require.Empty(t, pendingTargets, "unissued creations cannot have actors") + op, err := client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) + require.NoError(t, err) + executor := uuid.New() + claimed, err := client.ClaimSessionOperation(ctx, instance.Id, op.ID, executor) + require.NoError(t, err) + require.True(t, claimed) + pendingTargets, err = client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "scratch"}, "", 10) + require.NoError(t, err) + require.Len(t, pendingTargets, 1, "a claimed creation may already have allocated an actor") + require.Empty(t, pendingTargets[0].ActorUID, "creation has not yet persisted the actor UID") + _, err = client.FinishSessionOperation(ctx, instance.Id, op.ID, executor, "actor.example.com", "actor-uid", "") + require.NoError(t, err) + op, err = client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND) + require.NoError(t, err) + executor = uuid.New() + claimed, err = client.ClaimSessionOperation(ctx, instance.Id, op.ID, executor) + require.NoError(t, err) + require.True(t, claimed) + _, err = client.FinishSessionOperation(ctx, instance.Id, op.ID, executor, "", "", "") + require.NoError(t, err) + // A recreated Agent must not transfer authorization to an old pinned session. + upsertErr = client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "scratch", AgentUID: "new-uid", DesiredRevision: "pending"}) + require.NoError(t, upsertErr) + sandbox, options := sandboxFixture(t, client, "sandbox") + created, _, err := client.CreateSandbox(ctx, sandbox, "sandbox", options) + require.NoError(t, err) + sandboxOp, err := client.BeginSandboxOperation(ctx, created.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) + require.NoError(t, err) + claimed, err = client.ClaimSandboxOperation(ctx, created.Id, sandboxOp.ID, uuid.New()) + require.NoError(t, err) + require.True(t, claimed) + // Both kinds deliberately share a name; source filtering must preserve kind. + for _, test := range []struct { + source RuntimeEgressSource + want string + }{ + {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "scratch"}, instance.Id}, + {RuntimeEgressSource{Kind: RuntimeEgressSandbox, Namespace: "team-a", Name: "scratch"}, created.Id}, + {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "other", Name: "scratch"}, ""}, + {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "unrelated"}, ""}, + } { + targets, err := client.ListRuntimeEgressSourceTargets(ctx, test.source, "", 10) + require.NoError(t, err) + if test.want == "" { + require.Empty(t, targets) + } else { + require.Len(t, targets, 1) + require.Equal(t, test.want, targets[0].ID) + if test.source.Kind == RuntimeEgressAgent { + require.Equal(t, "actor-uid", targets[0].ActorUID) + } else { + require.Empty(t, targets[0].ActorUID) + } + } + } + for _, kind := range []RuntimeEgressSourceKind{"", "typo"} { + _, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: kind, Namespace: "team-a", Name: "scratch"}, "", 10) + require.Error(t, err) + } + // Page multiple allocated instances of one source without visiting other kinds. + sandbox.Id = uuid.NewString() + secondSandbox, _, err := client.CreateSandbox(ctx, sandbox, "second-sandbox", options) + require.NoError(t, err) + sandboxOp, err = client.BeginSandboxOperation(ctx, secondSandbox.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) + require.NoError(t, err) + claimed, err = client.ClaimSandboxOperation(ctx, secondSandbox.Id, sandboxOp.ID, uuid.New()) + require.NoError(t, err) + require.True(t, claimed) + source := RuntimeEgressSource{Kind: RuntimeEgressSandbox, Namespace: "team-a", Name: "scratch"} + first, err := client.ListRuntimeEgressSourceTargets(ctx, source, "", 1) + require.NoError(t, err) + require.Len(t, first, 1) + second, err := client.ListRuntimeEgressSourceTargets(ctx, source, first[0].ID, 1) + require.NoError(t, err) + require.Len(t, second, 1) + require.ElementsMatch(t, []string{created.Id, secondSandbox.Id}, []string{first[0].ID, second[0].ID}) + last, err := client.ListRuntimeEgressSourceTargets(ctx, source, second[0].ID, 1) + require.NoError(t, err) + require.Empty(t, last) + _, err = client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE) + require.NoError(t, err) + remaining, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "scratch"}, "", 10) + require.NoError(t, err) + require.Empty(t, remaining) +} + +func TestRuntimeEgressTokenOriginsAreNotGrants(t *testing.T) { + many := make([]string, 257) + for i := range many { + many[i] = fmt.Sprintf("https://model-%d.example.com", i) + } + for _, test := range []struct { + name string + origins []string + }{ + {"more origins than policy rules", many}, + {"different protocols on the same port", []string{"http://model.example.com:443", "https://model.example.com:443"}}, + } { + t.Run(test.name, func(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{ + Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", + SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", + TokenPassthroughOrigins: test.origins, + }, RecordRevisionOptions{})) + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + require.Empty(t, policy.Destinations, "token origins constrain credentials without granting network access") + }) + } +} + +func TestRuntimeEgressCanonicalTokenOriginWithholdsCredentials(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", EgressGrant: egress.Access{ + Destinations: []string{"https://model.example.com"}, + Credentials: []egress.Credential{{Origin: "https://model.example.com", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"}}, + }}) + require.NoError(t, err) + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{ + Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", + SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", + TokenPassthroughOrigins: []string{"https://MODEL.example.com."}, + }, RecordRevisionOptions{})) + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + require.Equal(t, []string{"https://model.example.com:443"}, policy.Destinations) + require.Empty(t, policy.Credentials) +} + +func TestSourceGrantsDetermineEffectivePolicy(t *testing.T) { + for _, kind := range []RuntimeEgressSourceKind{RuntimeEgressAgent, RuntimeEgressSandbox} { + t.Run(string(kind), func(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + upsert := func(uid string, grants egress.Access) error { + if kind == RuntimeEgressAgent { + return client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: uid, EgressGrant: grants}) + } + return client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team", SandboxTemplateName: "source", SandboxTemplateUID: uid, EgressGrant: grants}) + } + base := egress.Access{} + if kind == RuntimeEgressAgent { + base = egress.Access{Destinations: []string{"https://model.example.com:443"}, Credentials: []egress.Credential{{Origin: "https://model.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"}}} + } + initial := egress.Access{Destinations: []string{"https://tickets.example.com"}, Credentials: []egress.Credential{{Origin: "https://tickets.example.com", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/tickets/token"}}} + err := upsert("uid", egress.Access{}) + require.NoError(t, err) + err = upsert("uid", initial) + require.NoError(t, err) + canonical, err := egress.CanonicalAccess(initial.Destinations, initial.Credentials) + require.NoError(t, err) + err = upsert("uid", canonical) + require.NoError(t, err) + if kind == RuntimeEgressAgent { + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, EgressDestinations: base.Destinations, Credentials: base.Credentials, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, RecordRevisionOptions{})) + } else { + require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, SandboxTemplateName: "source", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{})) + } + assertPolicy := func(grants egress.Access) { + t.Helper() + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + expected, err := egress.MergeAccess(base, grants, nil) + require.NoError(t, err) + require.True(t, slices.Equal(expected.Destinations, policy.Destinations)) + require.True(t, slices.Equal(expected.Credentials, policy.Credentials)) + } + assertPolicy(canonical) + err = upsert("uid", egress.Access{Destinations: []string{"invalid"}}) + require.Error(t, err) + assertPolicy(canonical) + if kind == RuntimeEgressAgent { + conflict := egress.Access{Destinations: append(slices.Clone(base.Destinations), canonical.Destinations...), Credentials: append(slices.Clone(canonical.Credentials), egress.Credential{Origin: base.Destinations[0], Header: "authorization", URI: "ate-secret://k8s.io/default/team/other/token"})} + err = upsert("uid", conflict) + require.NoError(t, err) + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + expected, err := egress.MergeAccess(base, canonical, nil) + require.NoError(t, err) + require.Equal(t, expected, policy.Access, "older revisions retain nonconflicting grants") + } + if kind == RuntimeEgressAgent { + require.NoError(t, client.RetireAgentIdentities(ctx, "team", "source")) + } else { + require.NoError(t, client.RetireSandboxTemplateIdentities(ctx, "team", "source")) + } + assertPolicy(egress.Access{}) + err = upsert("uid", canonical) + require.NoError(t, err) + assertPolicy(canonical) + err = upsert("replacement", canonical) + require.NoError(t, err) + assertPolicy(egress.Access{}) + err = upsert("replacement", canonical) + require.NoError(t, err) + _, err = client.GetRuntimeEgressPolicy(ctx, "missing") + require.ErrorIs(t, err, ErrNotFound) + }) + } +} + +func TestSourceGrantsRollbackWithInvalidRevision(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + grants := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} + definition := AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", EgressGrant: grants} + err := client.UpsertAgentDefinition(ctx, definition) + require.NoError(t, err) + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "agent", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "agent"}, RecordRevisionOptions{})) + require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "sandbox", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "sandbox"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "sandbox", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{})) + for _, uid := range []string{"uid", "replacement"} { + definition.AgentUID, definition.DesiredRevision, definition.EgressGrant = uid, "sandbox", egress.Access{} + err = client.UpsertAgentDefinition(ctx, definition) + require.ErrorIs(t, err, ErrConflict) + policy, err := client.GetRuntimeEgressPolicy(ctx, "agent") + require.NoError(t, err) + require.Equal(t, grants.Destinations, policy.Destinations, "failed upsert must roll back grants and retirement") + require.Empty(t, policy.Credentials) + } +} + +func TestRuntimeEgressMalformedInputsFailClosed(t *testing.T) { + for _, test := range []struct{ name, statement string }{ + {"grant credentials", `UPDATE agent_definition SET grant_credentials = '[{"hostname":"legacy.example.com"}]'`}, + {"grant origin", `UPDATE agent_definition SET grant_destinations = ARRAY['invalid']`}, + {"pinned credentials", `UPDATE runtime_revision SET credentials = '[{"hostname":"legacy.example.com"}]'`}, + {"token origin", `UPDATE runtime_revision SET token_passthrough_origins = ARRAY['invalid']`}, + {"missing typed revision", `DELETE FROM agent_revision`}, + } { + t.Run(test.name, func(t *testing.T) { + pool := setupTestDB(t) + client := NewClient(pool, "public") + ctx := t.Context() + err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid"}) + require.NoError(t, err) + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, RecordRevisionOptions{})) + _, err = pool.Exec(ctx, test.statement) + require.NoError(t, err) + _, err = client.GetRuntimeEgressPolicy(ctx, "revision") + require.Error(t, err) + }) + } +} + +func TestReadAdmittedGrantsScopesIdentity(t *testing.T) { + for _, kind := range []RuntimeEgressSourceKind{RuntimeEgressAgent, RuntimeEgressSandbox} { + t.Run(string(kind), func(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + var read func(context.Context, string, string) (egress.Access, error) + var retire func(context.Context, string, string) error + upsert := func(uid string, grants egress.Access) error { + if kind == RuntimeEgressAgent { + return client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: uid, EgressGrant: grants}) + } + return client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team", SandboxTemplateName: "source", SandboxTemplateUID: uid, EgressGrant: grants}) + } + if kind == RuntimeEgressAgent { + read, retire = client.GetAgentEgressGrant, client.RetireAgentIdentities + } else { + read, retire = client.GetSandboxEgressGrant, client.RetireSandboxTemplateIdentities + } + grants := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} + require.NoError(t, upsert("uid", grants)) + got, err := read(ctx, "team", "uid") + require.NoError(t, err) + require.Equal(t, grants.Destinations, got.Destinations) + for _, scope := range [][2]string{{"team", "new"}, {"other", "uid"}} { + got, err = read(ctx, scope[0], scope[1]) + require.NoError(t, err) + require.Empty(t, got.Destinations) + } + require.NoError(t, retire(ctx, "team", "source")) + got, err = read(ctx, "team", "uid") + require.NoError(t, err) + require.Empty(t, got.Destinations, "retired grants must not be resurrected") + require.NoError(t, upsert("replacement", grants)) + got, err = read(ctx, "team", "uid") + require.NoError(t, err) + require.Empty(t, got.Destinations, "grants belong to the exact source UID") + }) + } +} diff --git a/go/core/internal/database/runtime_lifecycle_test.go b/go/core/internal/database/runtime_lifecycle_test.go index d6dcddc980..7c06ae747b 100644 --- a/go/core/internal/database/runtime_lifecycle_test.go +++ b/go/core/internal/database/runtime_lifecycle_test.go @@ -69,7 +69,9 @@ func TestAgentWritesRejectSandboxRevisions(t *testing.T) { revision, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) revision.Revision, revision.ActorTemplateUID = "sandbox-revision", "replacement-uid" - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision, true), ErrConflict) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{}), ErrConflict) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{Ready: true}), ErrConflict) + require.ErrorIs(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "scratch"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "scratch-uid", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{Ready: true}), ErrConflict) var uid string require.NoError(t, pool.QueryRow(ctx, "SELECT actor_template_uid FROM runtime_revision WHERE revision = 'sandbox-revision'").Scan(&uid)) require.Equal(t, "original-uid", uid) diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index be54f34260..041cfdcf9f 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -15,29 +15,30 @@ import ( ) // UpsertAgentDefinition records the desired runtime revision for a -// Agent identity. Updating an existing definition revives it if retired and +// Kubernetes Agent identity. Updating an existing definition revives it if retired and // preserves its latest successful revision. It atomically retires older identities -// at the same names and rejects revisions whose deletion has started. +// at the same names and rejects revisions whose deletion has started. Grants +// are canonicalized before writing; errors may include an uncertain commit outcome. func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefinition) error { + grants, err := canonicalStoredGrants(definition.EgressGrant) + if err != nil { + return fmt.Errorf("validate source grants: %w", err) + } return c.withTx(ctx, func(tx pgx.Tx) error { - if err := retireAgentIdentities(ctx, tx, definition.Namespace, definition.AgentName, &definition); err != nil { - return fmt.Errorf("retire replaced Agent: %w", err) + if err := retireAgentIdentities(ctx, tx, definition.Namespace, definition.AgentName, definition.AgentUID); err != nil { + return err } if err := execSQL(ctx, tx, ` - INSERT INTO agent_definition ( - namespace, agent_name, agent_uid, desired_revision, retired_at - ) VALUES ($1, $2, $3, $4, NULL) + INSERT INTO agent_definition (namespace, agent_name, agent_uid, desired_revision, grant_destinations, grant_credentials) + VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (namespace, agent_uid) DO UPDATE SET - agent_name = EXCLUDED.agent_name, - desired_revision = EXCLUDED.desired_revision, - retired_at = NULL, - updated_at = NOW() - `, - definition.Namespace, definition.AgentName, definition.AgentUID, - definition.DesiredRevision, - ); err != nil { + agent_name = EXCLUDED.agent_name, desired_revision = EXCLUDED.desired_revision, + grant_destinations = EXCLUDED.grant_destinations, grant_credentials = EXCLUDED.grant_credentials, + retired_at = NULL, updated_at = NOW() + `, definition.Namespace, definition.AgentName, definition.AgentUID, definition.DesiredRevision, grants.Destinations, grants.Credentials); err != nil { return err } + // Lock Agents before revisions. Validate the resulting definition, including // its retained last-good pointer when reactivating a retired identity. type revisionStatus struct { @@ -67,11 +68,45 @@ func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefi }) } -// RecordRuntimeRevision stores a prepared revision and, when ready, atomically -// promotes it for an active definition that still desires it. Stale reports leave the -// definition unchanged. Existing revisions retain immutable inputs; only their actor -// UID and update time are refreshed. Deleting revisions return ErrObjectDeleting. -func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevision, ready bool) error { +// GetAgentEgressGrant returns the active identity's last admitted grants. +// Missing or retired identities return empty access; malformed stored grants fail. +func (c *Client) GetAgentEgressGrant(ctx context.Context, namespace, uid string) (egress.Access, error) { + grants, err := queryOne(ctx, c.db, ` + SELECT grant_destinations AS destinations, grant_credentials AS credentials + FROM agent_definition WHERE namespace = $1 AND agent_uid = $2 AND retired_at IS NULL + `, pgx.RowToStructByName[egress.Access], namespace, uid) + return decodeEgressGrant(grants, err) +} + +// ListAgentDefinitionSources returns active Agent names and retired identities +// that still have non-deleted instances. Startup recovery includes these retired +// rows because a crash may interrupt actor revocation after retirement commits. +func (c *Client) ListAgentDefinitionSources(ctx context.Context) ([]RuntimeEgressSource, error) { + return queryMany(ctx, c.db, ` + SELECT DISTINCT 'agent' AS kind, d.namespace, d.agent_name AS name + FROM agent_definition d + WHERE d.retired_at IS NULL OR EXISTS ( + SELECT 1 FROM agent_revision a + JOIN runtime_revision r ON r.revision = a.revision + JOIN runtime_instance i ON i.prepared_revision = r.revision + WHERE r.namespace = d.namespace AND a.agent_uid = d.agent_uid + AND i.state <> 'RUNTIME_STATE_DELETED' + ) + ORDER BY namespace, name + `, pgx.RowToStructByName[RuntimeEgressSource]) +} + +// RecordRevisionOptions controls whether a prepared revision becomes available. +type RecordRevisionOptions struct { + // Ready promotes the revision only while its source is active and still desires it. + Ready bool +} + +// RecordRuntimeRevision atomically stores immutable inputs and, when Ready, +// promotes the revision for its active source if still desired. Existing inputs +// are preserved; actor UID and update time refresh. Stale reports cannot replace +// the last successful revision. Deleting revisions return ErrObjectDeleting. +func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevision, options RecordRevisionOptions) error { if revision.AgentCard == nil { return fmt.Errorf("runtime revision %s has no Agent Card", revision.Revision) } @@ -80,24 +115,19 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi return fmt.Errorf("encode runtime revision Agent Card: %w", err) } return c.withTx(ctx, func(tx pgx.Tx) error { - if ready { - // Match GC finalization's lock order: definition before revision. Use the - // stored identity when present, since revision inputs are immutable. - _, err := queryOne(ctx, tx, ` + if options.Ready { + // Match definition preparation and GC: source before revision. + if _, err := queryMany(ctx, tx, ` SELECT 1 FROM agent_definition p - LEFT JOIN agent_runtime_revision r ON r.revision = $1 - WHERE p.namespace = COALESCE(r.namespace, $2) - AND p.agent_uid = COALESCE(r.agent_uid, $3) - FOR UPDATE OF p - `, pgx.RowTo[int], revision.Revision, revision.Namespace, revision.AgentUID) - if err != nil && !errors.Is(err, pgx.ErrNoRows) { + WHERE p.namespace = $1 AND p.agent_uid = $2 FOR UPDATE OF p + `, pgx.RowTo[int], revision.Namespace, revision.AgentUID); err != nil { return err } } if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{ RuntimeArtifact: RuntimeArtifact{Revision: revision.Revision, Kind: runtimeKindAgent, Namespace: revision.Namespace, ActorTemplateAtespace: revision.ActorTemplateAtespace, ActorTemplateName: revision.ActorTemplateName, ActorTemplateUID: revision.ActorTemplateUID}, - SourceSnapshot: revision.SourceSnapshot, EgressDestinations: revision.EgressDestinations, Credentials: revision.Credentials, + SourceSnapshot: revision.SourceSnapshot, EgressDestinations: revision.EgressDestinations, Credentials: revision.Credentials, TokenPassthroughOrigins: revision.TokenPassthroughOrigins, }); err != nil { return err } @@ -107,19 +137,20 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi `, revision.Revision, revision.AgentName, revision.AgentUID, card); err != nil { return err } - if !ready { + if !options.Ready { return nil } return execSQL(ctx, tx, ` UPDATE agent_definition p SET latest_successful_revision = r.revision, updated_at = NOW() FROM agent_runtime_revision r - WHERE r.revision = $1 + WHERE r.revision = $1 AND p.namespace = $2 AND p.agent_uid = $3 AND p.namespace = r.namespace AND p.agent_uid = r.agent_uid AND p.desired_revision = r.revision AND p.retired_at IS NULL - `, revision.Revision) + AND p.latest_successful_revision IS DISTINCT FROM r.revision + `, revision.Revision, revision.Namespace, revision.AgentUID) }) } @@ -128,7 +159,7 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi func (c *Client) GetRuntimeRevision(ctx context.Context, revision string) (*RuntimeRevision, error) { row, err := queryOne(ctx, c.db, ` SELECT revision, namespace, agent_name, agent_uid, - source_snapshot, egress_destinations, credentials, actor_template_atespace, actor_template_name, actor_template_uid, + source_snapshot, egress_destinations, token_passthrough_origins, credentials, actor_template_atespace, actor_template_name, actor_template_uid, agent_card, deleted_at FROM agent_runtime_revision WHERE revision = $1 `, pgx.RowToStructByName[runtimeRevisionRow], revision) if err != nil { @@ -152,33 +183,31 @@ func toRuntimeRevision(row runtimeRevisionRow) (*RuntimeRevision, error) { Revision: row.Revision, Namespace: row.Namespace, AgentName: row.AgentName, AgentUID: row.AgentUID, SourceSnapshot: row.SourceSnapshot, AgentCard: card, - EgressDestinations: row.EgressDestinations, - Credentials: credentials, - ActorTemplateAtespace: row.ActorTemplateAtespace, ActorTemplateName: row.ActorTemplateName, + EgressDestinations: row.EgressDestinations, + TokenPassthroughOrigins: row.TokenPassthroughOrigins, + Credentials: credentials, + ActorTemplateAtespace: row.ActorTemplateAtespace, ActorTemplateName: row.ActorTemplateName, ActorTemplateUID: row.ActorTemplateUID, }, nil } -// RetireAgentIdentities retires identities at the given namespace/Agent -// names, except the supplied UID when non-nil. Existing sessions retain -// their pinned revisions. Missing and already-retired identities are a no-op. -func (c *Client) RetireAgentIdentities(ctx context.Context, namespace, name string, except *AgentDefinition) error { - return retireAgentIdentities(ctx, c.db, namespace, name, except) +// RetireAgentIdentities retires identities at the given namespace/Agent name. +// Existing sessions retain their pinned revisions. Missing and already-retired +// identities are a no-op. +func (c *Client) RetireAgentIdentities(ctx context.Context, namespace, name string) error { + return retireAgentIdentities(ctx, c.db, namespace, name, "") } // retireAgentIdentities uses the caller's executor so retirement can commit -// atomically with definition preparation. A nil exception retires every matching identity. -func retireAgentIdentities(ctx context.Context, db dbExecutor, namespace, name string, except *AgentDefinition) error { - var uid *string - if except != nil { - uid = &except.AgentUID - } +// atomically with definition preparation. An empty +// retainedUID retires every active identity at this source name. +func retireAgentIdentities(ctx context.Context, db dbExecutor, namespace, name, retainedUID string) error { return execSQL(ctx, db, ` UPDATE agent_definition SET retired_at = NOW(), updated_at = NOW() WHERE namespace = $1 AND agent_name = $2 - AND retired_at IS NULL AND agent_uid IS DISTINCT FROM $3::text - `, namespace, name, uid) + AND retired_at IS NULL AND agent_uid IS DISTINCT FROM NULLIF($3, '') + `, namespace, name, retainedUID) } // ListUnreferencedRuntimeRevisions lists revisions unused by active Agents, @@ -204,7 +233,7 @@ func getAvailableRuntimeRevisionForUpdate(ctx context.Context, tx pgx.Tx, revisi } return queryOne(ctx, tx, ` SELECT revision, namespace, agent_name, agent_uid, - source_snapshot, egress_destinations, credentials, actor_template_atespace, actor_template_name, actor_template_uid, + source_snapshot, egress_destinations, token_passthrough_origins, credentials, actor_template_atespace, actor_template_name, actor_template_uid, agent_card, deleted_at FROM agent_runtime_revision WHERE revision = $1 `, pgx.RowToStructByName[runtimeRevisionRow], revision) } @@ -227,14 +256,14 @@ func lockRuntimeRevisionForReference(ctx context.Context, tx pgx.Tx, revision, k type runtimeRevisionRecord struct { RuntimeArtifact - SourceSnapshot json.RawMessage - EgressDestinations []string - Credentials []egress.Credential + SourceSnapshot json.RawMessage + EgressDestinations []string + TokenPassthroughOrigins []string + Credentials []egress.Credential } -// recordRuntimeRevision writes immutable common inputs with mutable artifact -// status. The caller writes its typed extension and promotion pointer in this -// transaction, holding preparation locks before taking the revision lock. +// recordRuntimeRevision writes immutable common inputs and mutable artifact status. +// The caller writes its typed extension in the same transaction. func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevisionRecord) error { if revision.Credentials == nil { revision.Credentials = []egress.Credential{} @@ -242,14 +271,17 @@ func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevis if revision.EgressDestinations == nil { revision.EgressDestinations = []string{} } + if revision.TokenPassthroughOrigins == nil { + revision.TokenPassthroughOrigins = []string{} + } result, err := tx.Exec(ctx, ` INSERT INTO runtime_revision (revision, kind, namespace, source_snapshot, egress_destinations, - actor_template_atespace, actor_template_name, actor_template_uid, credentials) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + actor_template_atespace, actor_template_name, actor_template_uid, credentials, token_passthrough_origins) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (revision) DO UPDATE SET actor_template_uid = EXCLUDED.actor_template_uid, updated_at = NOW() WHERE runtime_revision.deleted_at IS NULL AND runtime_revision.kind = EXCLUDED.kind `, revision.Revision, revision.Kind, revision.Namespace, revision.SourceSnapshot, revision.EgressDestinations, - revision.ActorTemplateAtespace, revision.ActorTemplateName, revision.ActorTemplateUID, revision.Credentials) + revision.ActorTemplateAtespace, revision.ActorTemplateName, revision.ActorTemplateUID, revision.Credentials, revision.TokenPassthroughOrigins) if err != nil { return fmt.Errorf("record runtime revision %s: %w", revision.Revision, err) } @@ -337,16 +369,17 @@ func (c *Client) DeleteRuntimeRevision(ctx context.Context, revision, actorTempl } type runtimeRevisionRow struct { - Revision string - Namespace string - AgentName string - AgentUID string - SourceSnapshot []byte - EgressDestinations []string - Credentials []egress.Credential - ActorTemplateAtespace string - ActorTemplateName string - ActorTemplateUID string - AgentCard []byte - DeletedAt *time.Time + Revision string + Namespace string + AgentName string + AgentUID string + SourceSnapshot []byte + EgressDestinations []string + TokenPassthroughOrigins []string + Credentials []egress.Credential + ActorTemplateAtespace string + ActorTemplateName string + ActorTemplateUID string + AgentCard []byte + DeletedAt *time.Time } diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index b623c32060..7cc451a3a4 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -20,38 +20,48 @@ import ( ) func TestRetireAgentIdentities(t *testing.T) { - for _, test := range []struct { - name string - except *AgentDefinition - keep bool - }{ - {name: "all"}, - {name: "except current", except: &AgentDefinition{AgentUID: "assistant-uid"}, keep: true}, - {name: "replacement template", except: &AgentDefinition{AgentUID: "replacement-uid"}}, + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") + sessionFixture(t, client, ctx, "team-b", "other-namespace", "assistant", "kagent") + sessionFixture(t, client, ctx, "team-a", "other-template", "other", "kagent") + sessionFixture(t, client, ctx, "team-a", "other-agent", "another", "other") + for range 2 { + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) + } + revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) + require.NoError(t, err) + require.Len(t, revisions, 1, "retirement must stay within the requested namespace and name") + require.Equal(t, "revision", revisions[0].Revision) +} + +func TestListAgentDefinitionSourcesBoundsRetiredIdentities(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") + session, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "request") + require.NoError(t, err) + _, err = finishSessionOperation(ctx, client, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, "actor.example.com") + require.NoError(t, err) + for _, definition := range []AgentDefinition{ + {Namespace: "team-a", AgentName: "assistant", AgentUID: "replacement"}, + {Namespace: "team-a", AgentName: "unused", AgentUID: "unused"}, + {Namespace: "team-b", AgentName: "assistant", AgentUID: "other"}, } { - t.Run(test.name, func(t *testing.T) { - client := NewClient(setupTestDB(t), "public") - ctx := t.Context() - sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") - sessionFixture(t, client, ctx, "team-b", "other-namespace", "assistant", "kagent") - sessionFixture(t, client, ctx, "team-a", "other-template", "other", "kagent") - sessionFixture(t, client, ctx, "team-a", "other-agent", "another", "other") - for range 2 { - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", test.except)) - } - revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) - require.NoError(t, err) - if test.keep { - require.Empty(t, revisions) - session, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "session") - require.NoError(t, err) - require.Equal(t, "revision", session.GetPreparedRevision(), "the exception retains its last-good revision") - } else { - require.Len(t, revisions, 1, "retirement must stay within the requested namespace and names") - require.Equal(t, "revision", revisions[0].Revision) - } - }) + require.NoError(t, client.UpsertAgentDefinition(ctx, definition)) } + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "unused")) + sources, err := client.ListAgentDefinitionSources(ctx) + require.NoError(t, err) + require.Equal(t, []RuntimeEgressSource{ + {Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, + {Kind: RuntimeEgressAgent, Namespace: "team-b", Name: "assistant"}, + }, sources, "include only active sources and retired identities that still have actors") + require.NoError(t, deleteSession(ctx, client, session.Id)) + sources, err = client.ListAgentDefinitionSources(ctx) + require.NoError(t, err) + require.Equal(t, []RuntimeEgressSource{{Kind: RuntimeEgressAgent, Namespace: "team-b", Name: "assistant"}}, sources) } func TestRuntimeRevisionCollectionAfterPairRetirement(t *testing.T) { @@ -67,7 +77,7 @@ func TestRuntimeRevisionCollectionAfterPairRetirement(t *testing.T) { _, err = client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) - err = client.RetireAgentIdentities(ctx, "team-a", "assistant", nil) + err = client.RetireAgentIdentities(ctx, "team-a", "assistant") require.NoError(t, err) revisions, err = client.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) @@ -90,7 +100,7 @@ func TestRuntimeRevisionCollectionPreservesSessionAndCheckpoint(t *testing.T) { require.NoError(t, err) _, err = markSessionReady(ctx, client, session.GetId(), "runtime") require.NoError(t, err) - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) assertRetained := func() { t.Helper() @@ -151,24 +161,35 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { revision.Revision = "new" revision.AgentUID = "replacement-uid" revision.ActorTemplateName = "new-actor-template" - require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, false)) + require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{})) pair := AgentDefinition{ Namespace: "team-a", AgentName: "assistant", AgentUID: revision.AgentUID, DesiredRevision: "new", } - require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) + upsertErr := client.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) request := newSessionRequest(uuid.NewString(), "assistant", "kagent", "") _, _, err = client.CreateSession(ctx, request, "session") require.ErrorIs(t, err, ErrNotFound, "a replacement must not select the previous template UID") - require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, true)) + require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{Ready: true})) + session, _, err := client.CreateSession(ctx, request, "session") + require.NoError(t, err) + require.Equal(t, "new", session.GetPreparedRevision()) // Reconciliation of the same identity must preserve its last success while // a new desired revision is still preparing. pair.DesiredRevision = "pending" - require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) - session, _, err := client.CreateSession(ctx, request, "session") + upsertErr = client.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) + retried, created, err := client.CreateSession(ctx, request, "session") require.NoError(t, err) - require.Equal(t, "new", session.GetPreparedRevision()) + require.False(t, created) + require.Equal(t, session.GetId(), retried.GetId()) + require.Equal(t, "new", retried.GetPreparedRevision()) + fresh, created, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "pending") + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "new", fresh.PreparedRevision) revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) require.Len(t, revisions, 1) @@ -215,10 +236,10 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") - query := "FROM runtime_revision WHERE revision = $1 FOR UPDATE" + query := "FROM agent_definition p" if source != "session" { query = "FOR UPDATE OF r" - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) } barrier := &runtimeReferenceBarrier{query: query, afterQuery: referenceFirst, reached: make(chan struct{}), resume: make(chan struct{})} var resume sync.Once @@ -244,7 +265,8 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) pair.AgentUID = "replacement-uid" pair.DesiredRevision = "revision" } - created <- creating.UpsertAgentDefinition(ctx, pair) + upsertErr := creating.UpsertAgentDefinition(ctx, pair) + created <- upsertErr }() select { case <-barrier.reached: @@ -252,7 +274,18 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) t.Fatal(ctx.Err()) } if source == "session" { - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + if referenceFirst { + // Creation holds the definition lock until its reference commits, + // so retirement must follow creation before GC can claim it. + resume.Do(func() { close(barrier.resume) }) + require.NoError(t, <-created) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) + claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "revision") + require.NoError(t, err) + require.Nil(t, claimed) + return + } + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) } type claimResult struct { revision *RuntimeArtifact @@ -281,7 +314,11 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) require.NoError(t, result.err) require.NotNil(t, result.revision) resume.Do(func() { close(barrier.resume) }) - require.ErrorIs(t, <-created, ErrObjectDeleting) + if source == "session" { + require.ErrorIs(t, <-created, ErrNotFound) + } else { + require.ErrorIs(t, <-created, ErrObjectDeleting) + } } }) } @@ -295,17 +332,18 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") pair := AgentDefinition{ Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", - DesiredRevision: "pending", + DesiredRevision: "revision", } - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) // A skipped finalization must leave last-good intact for reactivation. require.NoError(t, client.DeleteRuntimeRevision(ctx, "revision", "revision-actor-uid")) - require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) + upsertErr := client.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) session, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "session") require.NoError(t, err) require.Equal(t, "revision", session.GetPreparedRevision()) require.NoError(t, deleteSession(ctx, client, session.GetId())) - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) original, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "revision") @@ -313,12 +351,14 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.NotNil(t, claimed) // Reactivating a retired success pointer and adding a new desired edge are // both forbidden, as is overwriting a runtime while deletion is in flight. - require.ErrorIs(t, client.UpsertAgentDefinition(ctx, pair), ErrObjectDeleting) + upsertErr = client.UpsertAgentDefinition(ctx, pair) + require.ErrorIs(t, upsertErr, ErrObjectDeleting) pair.AgentUID = "replacement-uid" pair.DesiredRevision = "revision" - require.ErrorIs(t, client.UpsertAgentDefinition(ctx, pair), ErrObjectDeleting) - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, false), ErrObjectDeleting) - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, true), ErrObjectDeleting) + upsertErr = client.UpsertAgentDefinition(ctx, pair) + require.ErrorIs(t, upsertErr, ErrObjectDeleting) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, RecordRevisionOptions{}), ErrObjectDeleting) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, RecordRevisionOptions{Ready: true}), ErrObjectDeleting) // A new client rediscovers and retries the committed claim after a crash. restarted := NewClient(pool, "public") revisions, err := restarted.ListUnreferencedRuntimeRevisions(ctx) @@ -334,7 +374,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.ErrorIs(t, err, ErrNotFound) // The same digest can be prepared again once cleanup has completed. original.ActorTemplateUID = "recreated-actor-uid" - require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, false)) + require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, RecordRevisionOptions{})) newClaim, err := restarted.BeginRuntimeRevisionDeletion(ctx, "revision") require.NoError(t, err) require.NotNil(t, newClaim) @@ -342,12 +382,13 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) _, err = restarted.GetRuntimeRevision(ctx, "revision") require.NoError(t, err, "a delayed collector must not finalize the new runtime") require.NoError(t, restarted.DeleteRuntimeRevision(ctx, "revision", "recreated-actor-uid")) - require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, false)) - require.NoError(t, restarted.UpsertAgentDefinition(ctx, pair)) + require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, RecordRevisionOptions{})) + upsertErr = restarted.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) } func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { - for _, operation := range []string{"reactivate", "record"} { + for _, operation := range []string{"reactivate", "promote"} { for _, finalizeFirst := range []bool{false, true} { t.Run(fmt.Sprintf("%s/finalize_first=%t", operation, finalizeFirst), func(t *testing.T) { pool := setupTestDB(t) @@ -355,9 +396,9 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) - original, err := client.GetRuntimeRevision(ctx, "revision") + revision, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "revision") require.NoError(t, err) require.NotNil(t, claimed) @@ -365,14 +406,14 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { barrier := &runtimeReferenceBarrier{ query: "FOR UPDATE OF r", reached: make(chan struct{}), resume: make(chan struct{}), } - if operation == "record" { - barrier.query = "INSERT INTO runtime_revision" + if operation == "promote" { + barrier.query = "FROM runtime_revision WHERE revision = $1 FOR UPDATE" } waitingQuery := "ORDER BY namespace, agent_uid" if finalizeFirst { barrier.query, barrier.afterQuery = "FROM runtime_revision WHERE revision = $1 FOR UPDATE", true waitingQuery = "INSERT INTO agent_definition" - if operation == "record" { + if operation == "promote" { waitingQuery = "FOR UPDATE OF p" } } @@ -393,11 +434,12 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { } created, deleted := make(chan error, 1), make(chan error, 1) create := func() { - if operation == "record" { - created <- creating.RecordRuntimeRevision(ctx, *original, true) + if operation == "promote" { + created <- creating.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{Ready: true}) return } - created <- creating.UpsertAgentDefinition(ctx, pair) + upsertErr := creating.UpsertAgentDefinition(ctx, pair) + created <- upsertErr } finalize := func() { deleted <- deleting.DeleteRuntimeRevision(ctx, "revision", claimed.ActorTemplateUID) } if finalizeFirst { @@ -427,13 +469,14 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { require.ErrorIs(t, <-created, ErrObjectDeleting) } require.NoError(t, <-deleted, "finalization and pair writes must not deadlock") - require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) + upsertErr := client.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) }) } } } -func TestRecordRuntimeRevisionPromotesOnlyCurrentActivePair(t *testing.T) { +func TestRecordReadyRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { pool := setupTestDB(t) c := NewClient(pool, "public") ctx := t.Context() @@ -461,57 +504,137 @@ func TestRecordRuntimeRevisionPromotesOnlyCurrentActivePair(t *testing.T) { require.NoError(t, err) require.Equal(t, want, session.PreparedRevision) } - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, false)) + upsertErr := c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{})) assertAvailableRevision("") stored, err := c.GetRuntimeRevision(ctx, revision.Revision) require.NoError(t, err) require.Equal(t, "original", stored.AgentCard.Name) - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) assertAvailableRevision("first") pair.DesiredRevision = "second" - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) + upsertErr = c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) revision.Revision, revision.ActorTemplateName = "second", "second" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{})) + assertAvailableRevision("first") + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) assertAvailableRevision("second") // A delayed report for the old revision must not replace the newer ready revision. revision.Revision, revision.ActorTemplateName = "first", "first" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) assertAvailableRevision("second") pair.DesiredRevision = "third" - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) - require.NoError(t, c.RetireAgentIdentities(ctx, pair.Namespace, pair.AgentName, nil)) + upsertErr = c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) + require.NoError(t, c.RetireAgentIdentities(ctx, pair.Namespace, pair.AgentName)) revision.Revision, revision.ActorTemplateName = "third", "third" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) assertAvailableRevision("") // Reviving the pair must retain the last success, not a report made while retired. - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) + upsertErr = c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) + assertAvailableRevision("second") + pair.DesiredRevision = "second" + upsertErr = c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) assertAvailableRevision("second") } func TestRuntimeRevisionPersistsCredentialBindings(t *testing.T) { - client := NewClient(setupTestDB(t), "public") - revision := RuntimeRevision{Revision: "credential-revision", Namespace: "team", AgentName: "agent", AgentUID: "agent", SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{"api.example.com"}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", Credentials: []egress.Credential{{Hostname: "api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}}} - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, false)) - got, err := client.GetRuntimeRevision(t.Context(), revision.Revision) - require.NoError(t, err) - require.Equal(t, revision.Credentials, got.Credentials) - revision.Credentials[0].URI = "ate-secret://k8s.io/default/team/other/token" - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, false)) - unchanged, err := client.GetRuntimeRevision(t.Context(), revision.Revision) - require.NoError(t, err) - require.Equal(t, got.Credentials, unchanged.Credentials, "revision credentials are immutable") + for _, provider := range []string{"k8s.io", "provider.example"} { + t.Run(provider, func(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + revision := RuntimeRevision{Revision: "credential-revision-" + provider, Namespace: "team", AgentName: "agent", AgentUID: "agent", SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{"https://api.example.com:443"}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", Credentials: []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://" + provider + "/default/team/auth/token"}}} + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, RecordRevisionOptions{})) + got, err := client.GetRuntimeRevision(t.Context(), revision.Revision) + require.NoError(t, err) + require.Equal(t, revision.Credentials, got.Credentials) + revision.Credentials[0].URI = "ate-secret://k8s.io/default/team/other/token" + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, RecordRevisionOptions{})) + unchanged, err := client.GetRuntimeRevision(t.Context(), revision.Revision) + require.NoError(t, err) + require.Equal(t, got.Credentials, unchanged.Credentials, "revision credentials are immutable") + }) + } } func TestRuntimeRevisionRejectsMalformedStoredCredentials(t *testing.T) { revision, err := toRuntimeRevision(runtimeRevisionRow{ Revision: "bad-revision", Credentials: []egress.Credential{{ - Hostname: "*", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token", + Origin: "*", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token", }}, }) require.ErrorContains(t, err, "decode runtime revision bad-revision credentials") - require.ErrorContains(t, err, "exact DNS hostname") + require.ErrorContains(t, err, "exact HTTP(S) DNS origin") require.Nil(t, revision) } + +func TestRecordReadyRevisionRollsBackOnPromotionFailure(t *testing.T) { + for _, kind := range []string{"agent", "sandbox"} { + for _, existing := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/existing=%t", kind, existing), func(t *testing.T) { + pool := setupTestDB(t) + client := NewClient(pool, "public") + ctx := t.Context() + var record func(string, RecordRevisionOptions) error + var storedUID func() (string, error) + switch kind { + case "agent": + require.NoError(t, client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", DesiredRevision: "revision"})) + record = func(uid string, options RecordRevisionOptions) error { + return client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", AgentCard: &a2apb.AgentCard{}, SourceSnapshot: []byte(`{}`), ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: uid}, options) + } + storedUID = func() (string, error) { + revision, err := client.GetRuntimeRevision(ctx, "revision") + if err != nil { + return "", err + } + return revision.ActorTemplateUID, nil + } + // Force the final write to fail after the revision has been recorded. + _, err := pool.Exec(ctx, `ALTER TABLE agent_definition ADD CONSTRAINT reject_promotion CHECK (latest_successful_revision IS NULL)`) + require.NoError(t, err) + t.Cleanup(func() { + if _, err := pool.Exec(context.Background(), `ALTER TABLE agent_definition DROP CONSTRAINT reject_promotion`); err != nil { + t.Errorf("remove test constraint: %v", err) + } + }) + case "sandbox": + require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team", SandboxTemplateName: "source", SandboxTemplateUID: "uid", DesiredRevision: "revision"})) + record = func(uid string, options RecordRevisionOptions) error { + return client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: uid}, SandboxTemplateName: "source", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, options) + } + storedUID = func() (string, error) { + revision, err := client.GetSandboxRevision(ctx, "revision") + if err != nil { + return "", err + } + return revision.ActorTemplateUID, nil + } + _, err := pool.Exec(ctx, `ALTER TABLE sandbox_template_definition ADD CONSTRAINT reject_promotion CHECK (latest_successful_revision IS NULL)`) + require.NoError(t, err) + t.Cleanup(func() { + if _, err := pool.Exec(context.Background(), `ALTER TABLE sandbox_template_definition DROP CONSTRAINT reject_promotion`); err != nil { + t.Errorf("remove test constraint: %v", err) + } + }) + } + if existing { + require.NoError(t, record("original", RecordRevisionOptions{})) + } + require.ErrorContains(t, record("updated", RecordRevisionOptions{Ready: true}), "reject_promotion") + uid, err := storedUID() + if existing { + require.NoError(t, err) + require.Equal(t, "original", uid, "a failed promotion must roll back artifact updates") + } else { + require.ErrorIs(t, err, ErrNotFound, "a failed promotion must not leave a partially recorded revision") + } + }) + } + } +} diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index 5bee645014..6efbd1faee 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -6,6 +6,8 @@ import ( "fmt" "github.com/jackc/pgx/v5" + + "github.com/kagent-dev/kagent/go/core/internal/egress" ) type SandboxRevision struct { @@ -20,27 +22,33 @@ type SandboxTemplateDefinition struct { Namespace string SandboxTemplateName string SandboxTemplateUID string + EgressGrant egress.Access DesiredRevision string } // UpsertSandboxTemplateDefinition pins desired inputs before any backend mutation. A new // Kubernetes UID retires the former identity without inheriting its last success. +// Grants are canonicalized before writing; errors may include an uncertain commit outcome. func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition SandboxTemplateDefinition) error { + grants, err := canonicalStoredGrants(definition.EgressGrant) + if err != nil { + return fmt.Errorf("validate source grants: %w", err) + } return c.withTx(ctx, func(tx pgx.Tx) error { - if err := execSQL(ctx, tx, ` - UPDATE sandbox_template_definition SET retired_at = NOW(), updated_at = NOW() - WHERE namespace = $1 AND sandbox_template_name = $2 AND sandbox_template_uid <> $3 AND retired_at IS NULL - `, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID); err != nil { + if err := retireSandboxTemplateIdentities(ctx, tx, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID); err != nil { return err } if err := execSQL(ctx, tx, ` - INSERT INTO sandbox_template_definition (namespace, sandbox_template_name, sandbox_template_uid, desired_revision) - VALUES ($1, $2, $3, $4) + INSERT INTO sandbox_template_definition (namespace, sandbox_template_name, sandbox_template_uid, desired_revision, grant_destinations, grant_credentials) + VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (namespace, sandbox_template_uid) DO UPDATE SET - desired_revision = EXCLUDED.desired_revision, retired_at = NULL, updated_at = NOW() - `, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID, definition.DesiredRevision); err != nil { + sandbox_template_name = EXCLUDED.sandbox_template_name, desired_revision = EXCLUDED.desired_revision, + grant_destinations = EXCLUDED.grant_destinations, grant_credentials = EXCLUDED.grant_credentials, + retired_at = NULL, updated_at = NOW() + `, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID, definition.DesiredRevision, grants.Destinations, grants.Credentials); err != nil { return err } + revisions, err := queryMany(ctx, tx, ` SELECT r.revision, r.kind, r.namespace, r.actor_template_atespace, r.actor_template_name, r.actor_template_uid, r.deleted_at FROM runtime_revision r JOIN sandbox_template_definition p @@ -62,21 +70,46 @@ func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition }) } +// GetSandboxEgressGrant returns the active identity's last admitted grants. +// Missing or retired identities return empty access; malformed stored grants fail. +func (c *Client) GetSandboxEgressGrant(ctx context.Context, namespace, uid string) (egress.Access, error) { + grants, err := queryOne(ctx, c.db, ` + SELECT grant_destinations AS destinations, grant_credentials AS credentials + FROM sandbox_template_definition WHERE namespace = $1 AND sandbox_template_uid = $2 AND retired_at IS NULL + `, pgx.RowToStructByName[egress.Access], namespace, uid) + return decodeEgressGrant(grants, err) +} + +// RetireSandboxTemplateIdentities retires active identities at this source name. +// Missing/already-retired identities are a no-op; retained actors lose live grants. func (c *Client) RetireSandboxTemplateIdentities(ctx context.Context, namespace, name string) error { - return execSQL(ctx, c.db, ` + return retireSandboxTemplateIdentities(ctx, c.db, namespace, name, "") +} + +// retireSandboxTemplateIdentities shares retirement with definition preparation +// in the caller's transaction. An empty retainedUID retires all active identities. +func retireSandboxTemplateIdentities(ctx context.Context, db dbExecutor, namespace, name, retainedUID string) error { + return execSQL(ctx, db, ` UPDATE sandbox_template_definition SET retired_at = NOW(), updated_at = NOW() WHERE namespace = $1 AND sandbox_template_name = $2 AND retired_at IS NULL - `, namespace, name) + AND sandbox_template_uid IS DISTINCT FROM NULLIF($3, '') + `, namespace, name, retainedUID) } -// RecordSandboxRevision promotes only the active identity still desiring this -// revision. A late preparation response cannot replace a newer successful input. -func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevision, ready bool) error { +// RecordSandboxRevision atomically stores immutable inputs and, when Ready, +// promotes the revision for its active source if still desired. Existing inputs +// are preserved; actor UID and update time refresh. Stale reports cannot replace +// the last successful revision. Deleting revisions return ErrObjectDeleting. +func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevision, options RecordRevisionOptions) error { return c.withTx(ctx, func(tx pgx.Tx) error { - if _, err := queryMany(ctx, tx, ` - SELECT 1 FROM sandbox_template_definition WHERE namespace = $1 AND sandbox_template_uid = $2 FOR UPDATE - `, pgx.RowTo[int], revision.Namespace, revision.SandboxTemplateUID); err != nil { - return err + if options.Ready { + // Match definition preparation and GC: source before revision. + if _, err := queryMany(ctx, tx, ` + SELECT 1 FROM sandbox_template_definition p + WHERE p.namespace = $1 AND p.sandbox_template_uid = $2 FOR UPDATE OF p + `, pgx.RowTo[int], revision.Namespace, revision.SandboxTemplateUID); err != nil { + return err + } } revision.Kind = runtimeKindSandbox if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{RuntimeArtifact: revision.RuntimeArtifact, SourceSnapshot: revision.SourceSnapshot}); err != nil { @@ -88,18 +121,21 @@ func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevi `, revision.Revision, revision.SandboxTemplateName, revision.SandboxTemplateUID); err != nil { return err } - if !ready { + if !options.Ready { return nil } return execSQL(ctx, tx, ` UPDATE sandbox_template_definition p SET latest_successful_revision = r.revision, updated_at = NOW() FROM runtime_revision r JOIN sandbox_revision s USING (revision) - WHERE r.revision = $1 AND p.namespace = r.namespace AND p.sandbox_template_uid = s.sandbox_template_uid + WHERE r.revision = $1 AND p.namespace = $2 AND p.sandbox_template_uid = $3 + AND p.namespace = r.namespace AND p.sandbox_template_uid = s.sandbox_template_uid AND p.desired_revision = r.revision AND p.retired_at IS NULL - `, revision.Revision) + AND p.latest_successful_revision IS DISTINCT FROM r.revision + `, revision.Revision, revision.Namespace, revision.SandboxTemplateUID) }) } +// GetSandboxRevision reads immutable sandbox inputs; missing revisions return ErrNotFound. func (c *Client) GetSandboxRevision(ctx context.Context, revision string) (*SandboxRevision, error) { row, err := queryOne(ctx, c.db, ` SELECT r.revision, r.kind, r.namespace, r.actor_template_atespace, r.actor_template_name, r.actor_template_uid, diff --git a/go/core/internal/database/sandboxes.go b/go/core/internal/database/sandboxes.go index d758492026..412e21cc40 100644 --- a/go/core/internal/database/sandboxes.go +++ b/go/core/internal/database/sandboxes.go @@ -63,8 +63,9 @@ type SandboxCreateOptions struct { TTL time.Duration } -// CreateSandbox atomically reserves a sandbox and its prepared revision. A -// duplicate creator/requestID returns the original resource when inputs match. +// CreateSandbox atomically reserves a sandbox and its latest successful revision, +// or returns ErrNotFound if none is available. A duplicate creator/requestID +// returns the original resource when inputs match. // The boolean reports whether this call created the sandbox. func (c *Client) CreateSandbox(ctx context.Context, request *apiv1.Sandbox, requestID string, options SandboxCreateOptions) (*apiv1.Sandbox, bool, error) { existing, err := readSandboxRequest(ctx, c.db, request.GetCreator(), requestID) @@ -113,7 +114,8 @@ func readSandboxRequest(ctx context.Context, db dbExecutor, userID, requestID st } // insertSandbox reserves a new sandbox in CREATING/CREATE and pins the active -// template's latest successful revision. A duplicate creator/requestID returns +// template's latest successful revision. A missing prepared +// target returns ErrNotFound. A duplicate creator/requestID returns // pgx.ErrNoRows so the caller can roll back and read the winning transaction. func insertSandbox(ctx context.Context, tx pgx.Tx, request *apiv1.Sandbox, requestID string, options SandboxCreateOptions) (sandboxRow, error) { if options.TTL < time.Second || options.TTL > 24*time.Hour { @@ -124,7 +126,8 @@ func insertSandbox(ctx context.Context, tx pgx.Tx, request *apiv1.Sandbox, reque err := tx.QueryRow(ctx, ` SELECT latest_successful_revision, clock_timestamp() FROM sandbox_template_definition WHERE namespace = $1 AND sandbox_template_name = $2 AND sandbox_template_uid = $3 - AND retired_at IS NULL AND latest_successful_revision IS NOT NULL FOR UPDATE + AND retired_at IS NULL AND latest_successful_revision IS NOT NULL + FOR UPDATE `, request.GetSandboxTemplate().GetNamespace(), request.GetSandboxTemplate().GetName(), options.TemplateUID).Scan(&revision, &createdAt) if err != nil { return sandboxRow{}, notFoundOr(err) diff --git a/go/core/internal/database/sandboxes_test.go b/go/core/internal/database/sandboxes_test.go index 766118b448..40bed380c9 100644 --- a/go/core/internal/database/sandboxes_test.go +++ b/go/core/internal/database/sandboxes_test.go @@ -16,14 +16,58 @@ import ( func sandboxFixture(t *testing.T, client *Client, revision string) (*apiv1.Sandbox, SandboxCreateOptions) { t.Helper() ctx := t.Context() - require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision})) + upsertErr := client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) + require.NoError(t, upsertErr) require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: revision, Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: revision, ActorTemplateUID: "actor-uid"}, - SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, true)) + SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{Ready: true})) hash := sha256.Sum256([]byte("request")) return &apiv1.Sandbox{Id: uuid.NewString(), Creator: "alice", SandboxTemplate: &apiv1.ResourceReference{Namespace: "team-a", Name: "scratch"}, Name: "experiment"}, SandboxCreateOptions{RequestHash: hash[:], TemplateUID: "uid", TTL: time.Hour} } +func TestRecordReadySandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + request, options := sandboxFixture(t, client, "first") + definition := SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: "second"} + upsertErr := client.UpsertSandboxTemplateDefinition(ctx, definition) + require.NoError(t, upsertErr) + revision, err := client.GetSandboxRevision(ctx, "first") + require.NoError(t, err) + revision.Revision, revision.ActorTemplateName = "second", "second" + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{})) + assertAvailable := func(want string) { + t.Helper() + input := proto.CloneOf(request) + input.Id = uuid.NewString() + instance, _, err := client.CreateSandbox(ctx, input, uuid.NewString(), options) + if want == "" { + require.ErrorIs(t, err, ErrNotFound) + return + } + require.NoError(t, err) + require.Equal(t, want, instance.PreparedRevision) + } + assertAvailable("first") + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{Ready: true})) + assertAvailable("second") + revision.Revision, revision.ActorTemplateName = "first", "first" + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{Ready: true})) + assertAvailable("second") + require.NoError(t, client.RetireSandboxTemplateIdentities(ctx, "team-a", "scratch")) + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{Ready: true})) + assertAvailable("") + upsertErr = client.UpsertSandboxTemplateDefinition(ctx, definition) + require.NoError(t, upsertErr) + assertAvailable("second") + revision.Revision, revision.ActorTemplateName = "unused", "unused" + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{})) + claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "unused") + require.NoError(t, err) + require.NotNil(t, claimed) + require.ErrorIs(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{Ready: true}), ErrObjectDeleting) +} + func finishSandbox(t *testing.T, client *Client, id string, kind apiv1.RuntimeOperation) *apiv1.Sandbox { t.Helper() op, err := client.BeginSandboxOperation(t.Context(), id, kind) diff --git a/go/core/internal/database/scheduled_runs_test.go b/go/core/internal/database/scheduled_runs_test.go index 80aa0e82a9..ca835dc53d 100644 --- a/go/core/internal/database/scheduled_runs_test.go +++ b/go/core/internal/database/scheduled_runs_test.go @@ -393,7 +393,7 @@ func TestScheduledExecutionWaitsForPreparedRevision(t *testing.T) { db := setupTestDB(t) c := NewClient(db, "public") schedule, _ := createTestSchedule(t, c) - require.NoError(t, c.RetireAgentIdentities(t.Context(), "team-a", "report", nil)) + require.NoError(t, c.RetireAgentIdentities(t.Context(), "team-a", "report")) execution, err := c.TriggerScheduledRun(t.Context(), uuid.MustParse(schedule.Id), "alice", "manual") require.NoError(t, err) _, err = c.ReserveScheduledRunExecutionSession(t.Context(), uuid.MustParse(execution.Id), "alice") diff --git a/go/core/internal/database/sessions.go b/go/core/internal/database/sessions.go index dd0e4c496d..096f5ac661 100644 --- a/go/core/internal/database/sessions.go +++ b/go/core/internal/database/sessions.go @@ -53,11 +53,11 @@ func sameSessionRequest(session, request *apiv1.Session) bool { } // CreateSession atomically reserves a session, its conversation history, and the -// latest successful runtime revision. A repeated creator/requestID returns the existing -// session when the Agent matches, or ErrIdempotencyConflict otherwise. A -// deleted session returns ErrFailedPrecondition and keeps its request ID reserved. The -// boolean reports whether this call created the session; runtime provisioning belongs to -// the caller. +// latest successful revision, or returns ErrNotFound if none is available. A repeated +// creator/requestID returns the existing session when the Agent matches, or +// ErrIdempotencyConflict otherwise. A deleted session returns ErrFailedPrecondition +// and keeps its request ID reserved. The boolean reports whether this call created +// the session; runtime provisioning belongs to the caller. func (c *Client) CreateSession(ctx context.Context, request *apiv1.Session, requestID string) (*apiv1.Session, bool, error) { existing, err := readSessionRequest(ctx, c.db, request.GetCreator(), requestID) if err == nil { @@ -101,22 +101,23 @@ func (c *Client) CreateSession(ctx context.Context, request *apiv1.Session, requ } // insertSession reserves a new conversation in CREATING/CREATE and pins the active -// Agent's latest successful revision. Callers must supply a transaction so -// the history and session commit together. A missing prepared target returns ErrNotFound; -// a duplicate creator/requestID returns pgx.ErrNoRows for the caller to resolve. +// Agent's latest successful revision, locking the revision against cleanup. Callers must supply +// a transaction so the history and session commit together. A missing prepared target +// returns ErrNotFound; a duplicate creator/requestID returns pgx.ErrNoRows for the caller +// to resolve. func insertSession(ctx context.Context, db pgx.Tx, request *apiv1.Session, requestID string) (sessionRow, error) { type preparedRevision struct { Revision string DBTime time.Time } revision, err := queryOne(ctx, db, ` - SELECT r.revision, clock_timestamp() AS db_time + SELECT p.latest_successful_revision AS revision, clock_timestamp() AS db_time FROM agent_definition p - JOIN agent_runtime_revision r ON r.revision = p.latest_successful_revision - WHERE p.namespace = $1 AND p.agent_name = $2 AND p.retired_at IS NULL - `, pgx.RowToStructByName[preparedRevision], request.GetAgent().GetNamespace(), request.GetAgent().GetName()) + WHERE p.namespace = $1 AND p.agent_name = $2 AND p.retired_at IS NULL + AND p.latest_successful_revision IS NOT NULL + `, pgx.RowToStructByName[preparedRevision], request.GetAgent().GetNamespace(), request.GetAgent().GetName()) if err != nil { - return sessionRow{}, fmt.Errorf("get latest successful runtime revision: %w", notFoundOr(err)) + return sessionRow{}, fmt.Errorf("get latest prepared runtime revision: %w", notFoundOr(err)) } if _, err := getAvailableRuntimeRevisionForUpdate(ctx, db, revision.Revision); err != nil { return sessionRow{}, err diff --git a/go/core/internal/egress/credentials.go b/go/core/internal/egress/credentials.go index e0136cdfee..f85861c704 100644 --- a/go/core/internal/egress/credentials.go +++ b/go/core/internal/egress/credentials.go @@ -2,7 +2,6 @@ package egress import ( "fmt" - "net/netip" "net/url" "slices" "strings" @@ -11,55 +10,92 @@ import ( "k8s.io/apimachinery/pkg/util/validation" ) -// Credential binds a Secret reference to an exact destination and HTTP header. +// Credential binds a provider reference to an exact destination and HTTP header. // Values are fetched by the gateway and never enter a runtime revision. type Credential struct { - Hostname string `json:"hostname"` - Header string `json:"header"` - Prefix string `json:"prefix,omitempty"` - URI string `json:"uri"` + Origin string `json:"origin"` + Header string `json:"header"` + Prefix string `json:"prefix,omitempty"` + URI string `json:"uri"` } -// CanonicalCredentials validates and orders bindings. Substrate matches only -// hostnames, so two credentials for the same host and header cannot coexist. +// CanonicalCredentials validates and orders bindings by complete origin and header. func CanonicalCredentials(bindings []Credential) ([]Credential, error) { result := slices.Clone(bindings) for i := range result { c := &result[i] - c.Hostname = strings.TrimSuffix(strings.ToLower(c.Hostname), ".") - c.Header = strings.ToLower(c.Header) - _, ipErr := netip.ParseAddr(c.Hostname) - if ipErr == nil || len(validation.IsDNS1123Subdomain(c.Hostname)) != 0 { - return nil, fmt.Errorf("credential injection requires an exact DNS hostname, got %q", c.Hostname) + u, err := ParseOrigin(c.Origin) + if err != nil || strings.Contains(c.Origin, "*") { + return nil, fmt.Errorf("credential injection requires an exact HTTP(S) DNS origin, got %q", c.Origin) } - if !httpguts.ValidHeaderFieldName(c.Header) || !httpguts.ValidHeaderFieldValue(c.Prefix) { + c.Origin = u.String() + c.Header = strings.ToLower(c.Header) + if !validCredentialHeader(c.Header) || !httpguts.ValidHeaderFieldValue(c.Prefix) { return nil, fmt.Errorf("invalid credential injection header %q", c.Header) } - u, err := url.Parse(c.URI) - if err != nil || u.Scheme != "ate-secret" || u.Host != "k8s.io" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { - return nil, fmt.Errorf("invalid Kubernetes credential URI") - } - parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") - if len(parts) != 4 || parts[0] != "default" || len(validation.IsDNS1123Label(parts[1])) != 0 || len(validation.IsDNS1123Subdomain(parts[2])) != 0 || parts[3] == "" || len(validation.IsConfigMapKey(parts[3])) != 0 { - return nil, fmt.Errorf("credential URI must identify the default locator, namespace, Secret and key") + if err := validCredentialURI(c.URI); err != nil { + return nil, err } } slices.SortFunc(result, func(a, b Credential) int { - return strings.Compare(a.Hostname+"\x00"+a.Header, b.Hostname+"\x00"+b.Header) + return strings.Compare(a.Origin+"\x00"+a.Header, b.Origin+"\x00"+b.Header) }) for i := 1; i < len(result); i++ { a, b := result[i-1], result[i] - if a.Hostname == b.Hostname && a.Header == b.Header && a != b { - return nil, fmt.Errorf("conflicting credentials for %s header %s", a.Hostname, a.Header) + if a.Origin == b.Origin && a.Header == b.Header && a != b { + return nil, fmt.Errorf("conflicting credentials for %s header %s", a.Origin, a.Header) } } result = slices.Compact(result) counts := map[string]int{} for _, c := range result { - counts[c.Hostname]++ - if counts[c.Hostname] > 16 { - return nil, fmt.Errorf("credential injection supports at most 16 headers per hostname") + counts[c.Origin]++ + if counts[c.Origin] > 16 { + return nil, fmt.Errorf("credential injection supports at most 16 headers per origin") } } return result, nil } + +// validCredentialURI accepts ate-secret:///. Provider names are +// lowercase DNS names without ports. k8s.io paths name one Secret key. +func validCredentialURI(raw string) error { + u, err := url.Parse(raw) + if err != nil || u.Scheme != "ate-secret" || u.Opaque != "" { + return fmt.Errorf("invalid credential URI") + } + if u.User != nil || u.ForceQuery || u.RawQuery != "" || strings.Contains(raw, "#") { + return fmt.Errorf("credential URI cannot carry user information, a query or a fragment") + } + if u.Host != u.Hostname() || len(validation.IsDNS1123Subdomain(u.Host)) != 0 { + return fmt.Errorf("credential provider must be a lowercase DNS name without a port") + } + // Substrate requires at least two nonempty, unescaped path segments. + parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") + if u.EscapedPath() != u.Path || !strings.HasPrefix(u.Path, "/") || len(parts) < 2 || slices.Contains(parts, "") { + return fmt.Errorf("credential URI must have at least two nonempty path segments without percent encoding") + } + if u.Host != "k8s.io" { + return nil + } + if len(parts) != 4 || parts[0] != "default" { + return fmt.Errorf("credential URI must identify the default locator, namespace, Secret and key") + } + namespace, secret, key := parts[1], parts[2], parts[3] + if len(validation.IsDNS1123Label(namespace)) != 0 || len(validation.IsDNS1123Subdomain(secret)) != 0 || len(validation.IsConfigMapKey(key)) != 0 { + return fmt.Errorf("credential URI must identify the default locator, namespace, Secret and key") + } + return nil +} + +// validCredentialHeader excludes routing, framing, hop-by-hop and cookie headers. +func validCredentialHeader(header string) bool { + if len(header) > 256 || !httpguts.ValidHeaderFieldName(header) { + return false + } + switch strings.ToLower(header) { + case "host", "connection", "content-length", "transfer-encoding", "te", "trailer", "upgrade", "keep-alive", "proxy-authorization", "proxy-authenticate", "cookie": + return false + } + return true +} diff --git a/go/core/internal/egress/credentials_test.go b/go/core/internal/egress/credentials_test.go index e24254caaa..4686a22d5e 100644 --- a/go/core/internal/egress/credentials_test.go +++ b/go/core/internal/egress/credentials_test.go @@ -7,22 +7,48 @@ import ( ) func TestCanonicalCredentials(t *testing.T) { - base := Credential{Hostname: "API.Example.com.", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"} + base := Credential{Origin: "https://API.Example.com.", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"} got, err := CanonicalCredentials([]Credential{base, base}) require.NoError(t, err) - require.Equal(t, []Credential{{Hostname: "api.example.com", Header: "authorization", Prefix: "Bearer ", URI: base.URI}}, got) - require.Equal(t, "API.Example.com.", base.Hostname) + require.Equal(t, []Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: base.URI}}, got) + require.Equal(t, "https://API.Example.com.", base.Origin) for _, test := range []struct { name string change func(*Credential) }{ - {"wildcard", func(c *Credential) { c.Hostname = "*.example.com" }}, - {"IP", func(c *Credential) { c.Hostname = "192.0.2.1" }}, + {"wildcard", func(c *Credential) { c.Origin = "https://*.example.com" }}, + {"IP", func(c *Credential) { c.Origin = "https://192.0.2.1" }}, {"header", func(c *Credential) { c.Header = "Authorization\r\nInjected" }}, {"prefix", func(c *Credential) { c.Prefix = "Bearer\n" }}, {"provider", func(c *Credential) { c.URI = "https://example.com/secret" }}, {"missing key", func(c *Credential) { c.URI = "ate-secret://k8s.io/default/team/auth" }}, - {"old provider", func(c *Credential) { c.URI = "ate-secret://kubernetes.io/team/auth/token" }}, + {"empty key", func(c *Credential) { c.URI = "ate-secret://k8s.io/default/team/auth/" }}, + {"mixed case Kubernetes provider", func(c *Credential) { c.URI = "ate-secret://K8s.io/default/team/auth/token" }}, + {"Kubernetes provider port", func(c *Credential) { c.URI = "ate-secret://k8s.io:443/default/team/auth/token" }}, + {"mixed case external provider", func(c *Credential) { c.URI = "ate-secret://Provider.example/default/token" }}, + {"external provider port", func(c *Credential) { c.URI = "ate-secret://provider.example:443/default/token" }}, + {"empty provider port", func(c *Credential) { c.URI = "ate-secret://provider.example:/default/token" }}, + {"IPv6 provider", func(c *Credential) { c.URI = "ate-secret://[2001:db8::1]/default/token" }}, + {"missing provider", func(c *Credential) { c.URI = "ate-secret:///team/auth/token" }}, + {"userinfo", func(c *Credential) { c.URI = "ate-secret://user@provider.example/default/token" }}, + {"opaque", func(c *Credential) { c.URI = "ate-secret:provider.example/token" }}, + {"empty query", func(c *Credential) { c.URI += "?" }}, + {"fragment", func(c *Credential) { c.URI += "#token" }}, + {"empty fragment", func(c *Credential) { c.URI += "#" }}, + {"invalid escape", func(c *Credential) { c.URI = "ate-secret://provider.example/%xx" }}, + {"invalid provider DNS name", func(c *Credential) { c.URI = "ate-secret://provider_name/default/token" }}, + {"provider trailing dot", func(c *Credential) { c.URI = "ate-secret://provider.example./default/token" }}, + {"missing path", func(c *Credential) { c.URI = "ate-secret://provider.example" }}, + {"single path segment", func(c *Credential) { c.URI = "ate-secret://provider.example/token" }}, + {"empty path segment", func(c *Credential) { c.URI = "ate-secret://provider.example/default//token" }}, + {"empty first path segment", func(c *Credential) { c.URI = "ate-secret://provider.example//default/token" }}, + {"trailing slash", func(c *Credential) { c.URI = "ate-secret://provider.example/default/token/" }}, + {"encoded path", func(c *Credential) { + c.URI = "ate-secret://provider.example/default/https%3A%2F%2Fapi.example.com%3A8443" + }}, + {"encoded unreserved character", func(c *Credential) { c.URI = "ate-secret://provider.example/default/%74oken" }}, + {"unescaped space", func(c *Credential) { c.URI = "ate-secret://provider.example/default/session token" }}, + {"encoded Kubernetes key", func(c *Credential) { c.URI = "ate-secret://k8s.io/default/team/auth/%74oken" }}, {"missing locator", func(c *Credential) { c.URI = "ate-secret://k8s.io/team/auth/token" }}, {"unsupported locator", func(c *Credential) { c.URI = "ate-secret://k8s.io/remote/team/auth/token" }}, {"query", func(c *Credential) { c.URI += "?key=other" }}, @@ -39,3 +65,18 @@ func TestCanonicalCredentials(t *testing.T) { _, err = CanonicalCredentials([]Credential{base, other}) require.ErrorContains(t, err, "conflicting credentials") } + +func TestCanonicalCredentialsExternalProvider(t *testing.T) { + for _, uri := range []string{ + "ate-secret://provider.example/default/session-token", + "ate-secret://caller-token.kagent.dev/default/token", + "ate-secret://provider.example/default/api.example.com:8443", + } { + t.Run(uri, func(t *testing.T) { + binding := Credential{Origin: "https://api.example.com:8443", Header: "authorization", Prefix: "Bearer ", URI: uri} + got, err := CanonicalCredentials([]Credential{binding}) + require.NoError(t, err) + require.Equal(t, []Credential{binding}, got) + }) + } +} diff --git a/go/core/internal/egress/destination.go b/go/core/internal/egress/destination.go index ade6f203ed..398a3022dd 100644 --- a/go/core/internal/egress/destination.go +++ b/go/core/internal/egress/destination.go @@ -1,9 +1,15 @@ package egress import ( + "fmt" "net" + "net/netip" "net/url" + "slices" + "strconv" "strings" + + "k8s.io/apimachinery/pkg/util/validation" ) // Origin retains the protocol and destination port needed by Substrate's @@ -24,3 +30,75 @@ func Origin(u *url.URL) string { } return (&url.URL{Scheme: u.Scheme, Host: host}).String() } + +// ParseOrigin validates an HTTP(S) DNS origin and makes its port explicit. +// Wildcards replace one leftmost label and cannot cover an entire top-level domain. +func ParseOrigin(raw string) (*url.URL, error) { + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.Hostname() == "" || u.Path != "" || u.ForceQuery || u.RawQuery != "" || strings.Contains(raw, "#") || strings.HasSuffix(u.Host, ":") { + return nil, fmt.Errorf("invalid egress origin %q: expected scheme, DNS hostname and optional port", raw) + } + host := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") + dns := strings.TrimPrefix(host, "*.") + _, ipErr := netip.ParseAddr(dns) + if ipErr == nil || len(validation.IsDNS1123Subdomain(dns)) != 0 || (strings.HasPrefix(host, "*.") && !strings.Contains(dns, ".")) { + return nil, fmt.Errorf("invalid egress origin %q: expected a DNS name or leftmost-label wildcard", raw) + } + portText := u.Port() + if portText == "" { + portText = "80" + if u.Scheme == "https" { + portText = "443" + } + } + port, err := strconv.ParseUint(portText, 10, 16) + if err != nil || port == 0 { + return nil, fmt.Errorf("invalid egress port in %q", raw) + } + u.Host = net.JoinHostPort(host, strconv.FormatUint(port, 10)) + return u, nil +} + +// Access is a compiled network grant. It contains references, +// never Secret values. +type Access struct { + Destinations []string `json:"destinations,omitempty"` + Credentials []Credential `json:"credentials,omitempty"` +} + +// CanonicalAccess merges equivalent origins and validates the complete grant +// before a revision can become ready or an Actor can be resumed. +func CanonicalAccess(destinations []string, credentials []Credential) (Access, error) { + result := Access{} + protocols := map[string]string{} + allowed := map[string]bool{} + for _, raw := range destinations { + u, err := ParseOrigin(raw) + if err != nil { + return Access{}, err + } + if scheme, exists := protocols[u.Host]; exists && scheme != u.Scheme { + return Access{}, fmt.Errorf("egress destination %q uses both HTTP and HTTPS on the same port", u.Host) + } + protocols[u.Host] = u.Scheme + allowed[u.String()] = true + } + for origin := range allowed { + result.Destinations = append(result.Destinations, origin) + } + slices.Sort(result.Destinations) + if len(result.Destinations) > 256 { + return Access{}, fmt.Errorf("egress policy exceeds 256 rules") + } + var err error + result.Credentials, err = CanonicalCredentials(credentials) + if err != nil { + return Access{}, err + } + for _, binding := range result.Credentials { + if !allowed[binding.Origin] { + return Access{}, fmt.Errorf("credential destination %q is not explicitly allowed", binding.Origin) + } + } + return result, nil +} diff --git a/go/core/internal/egress/destination_test.go b/go/core/internal/egress/destination_test.go index dc9fa6d812..69da32d12f 100644 --- a/go/core/internal/egress/destination_test.go +++ b/go/core/internal/egress/destination_test.go @@ -1,12 +1,54 @@ package egress import ( + "fmt" "net/url" "testing" "github.com/stretchr/testify/require" ) +func TestCanonicalAccess(t *testing.T) { + credential := Credential{Origin: "https://API.example.com", Header: "Authorization", URI: "ate-secret://k8s.io/default/team/auth/token"} + access, err := CanonicalAccess([]string{"https://*.example.com", "https://api.example.com:0443", "https://API.example.com.", "https://api.example.com:8443", "http://api.example.com"}, []Credential{credential, credential}) + require.NoError(t, err) + require.Equal(t, []string{"http://api.example.com:80", "https://*.example.com:443", "https://api.example.com:443", "https://api.example.com:8443"}, access.Destinations) + require.Len(t, access.Credentials, 1) + require.Equal(t, "https://api.example.com:443", access.Credentials[0].Origin) + for _, raw := range []string{"https://*", "https://*.com", "https://foo.*.com", "https://192.0.2.1", "https://[::1]", "https://user@api.example.com", "https://api.example.com/", "https://api.example.com?", "https://api.example.com#", "https://api.example.com:", "https://api.example.com:0", "https://api.example.com:65536", "ftp://api.example.com"} { + t.Run(raw, func(t *testing.T) { + _, err := ParseOrigin(raw) + require.Error(t, err) + }) + } + for _, allowed := range []string{"https://*.example.com", "http://api.example.com", "https://api.example.com:8443"} { + _, err := CanonicalAccess([]string{allowed}, []Credential{credential}) + require.ErrorContains(t, err, "not explicitly allowed", "credential must require its exact origin") + } + _, err = CanonicalAccess([]string{"http://api.example.com:443", "https://api.example.com"}, nil) + require.ErrorContains(t, err, "both HTTP and HTTPS") + var origins []string + for i := range 257 { + origins = append(origins, fmt.Sprintf("https://api%d.example.com", i)) + } + _, err = CanonicalAccess(origins, nil) + require.ErrorContains(t, err, "256 rules") +} + +func TestCredentialsIsolatePortsAndRejectProtectedHeaders(t *testing.T) { + one := Credential{Origin: "https://api.example.com", Header: "Authorization", URI: "ate-secret://k8s.io/default/team/auth/token"} + two := one + two.Origin, two.URI = "https://api.example.com:8443", "ate-secret://k8s.io/default/team/other/token" + bindings, err := CanonicalCredentials([]Credential{one, two}) + require.NoError(t, err) + require.Len(t, bindings, 2) + for _, header := range []string{"hOsT", "Content-Length", "Connection", "Transfer-Encoding", "Cookie", "Proxy-Authorization"} { + one.Header = header + _, err := CanonicalCredentials([]Credential{one}) + require.Error(t, err) + } +} + func TestOrigin(t *testing.T) { for _, tt := range []struct{ raw, want string }{ {"https://API.Example.com./v1?key=private#fragment", "https://api.example.com:443"}, diff --git a/go/core/internal/egress/live.go b/go/core/internal/egress/live.go new file mode 100644 index 0000000000..e07b66df87 --- /dev/null +++ b/go/core/internal/egress/live.go @@ -0,0 +1,95 @@ +package egress + +import ( + "context" + "fmt" + "slices" +) + +// Policy is the effective access for one runtime revision. +type Policy struct { + Access +} + +// PolicyStore reads effective policy from persisted revision inputs and source grants. +// A missing or malformed policy is an error, never permission to retain old access. +type PolicyStore interface { + GetRuntimeEgressPolicy(context.Context, string) (Policy, error) +} + +// EffectiveAccess retains the base and all grants compatible with this revision. +// Stored grants can conflict with an older actor's pinned base. +func EffectiveAccess(base, grants Access, tokenPassthroughOrigins []string) (Access, error) { + filtered, _ := FilterConflicts(base, grants, tokenPassthroughOrigins) + return MergeAccess(base, filtered, tokenPassthroughOrigins) +} + +// MergeAccess validates live grants against the pinned runtime's credentials and +// caller-token origins. Inputs are never modified. +func MergeAccess(base, grants Access, tokenPassthroughOrigins []string) (Access, error) { + access, err := CanonicalAccess(append(slices.Clone(base.Destinations), grants.Destinations...), append(slices.Clone(base.Credentials), grants.Credentials...)) + if err != nil { + return Access{}, err + } + for _, binding := range access.Credentials { + if slices.Contains(tokenPassthroughOrigins, binding.Origin) { + return Access{}, fmt.Errorf("destination %q cannot combine caller-token passthrough with gateway credentials", binding.Origin) + } + } + return access, nil +} + +// IntersectAccess retains only destinations and complete credential bindings +// present in both inputs. Inputs contain canonical rules; b may be an invalid +// union of otherwise valid policies. Taking a subset of valid a remains valid. +func IntersectAccess(a, b Access) Access { + var result Access + for _, origin := range a.Destinations { + if slices.Contains(b.Destinations, origin) { + result.Destinations = append(result.Destinations, origin) + } + } + for _, credential := range a.Credentials { + if slices.Contains(result.Destinations, credential.Origin) && slices.Contains(b.Credentials, credential) { + result.Credentials = append(result.Credentials, credential) + } + } + return result +} + +// FilterConflicts retains grants compatible with a valid base and caller-token +// origins. Inputs must be individually canonical and valid. The base wins; +// sorted grant order determines which rules fit when the combined limits are +// exceeded. A conflicting credential does not remove its network destination. +// Neither input is modified; dropped origins are sorted and deduplicated. +func FilterConflicts(base, grants Access, tokenOrigins []string) (Access, []string) { + if _, err := MergeAccess(base, grants, tokenOrigins); err == nil { + return grants, nil + } + var result Access + var dropped []string + for _, origin := range grants.Destinations { + candidate := Access{Destinations: append(slices.Clone(result.Destinations), origin)} + if _, err := MergeAccess(base, candidate, tokenOrigins); err != nil { + dropped = append(dropped, origin) + } else { + result = candidate + } + } + for _, credential := range grants.Credentials { + if !slices.Contains(result.Destinations, credential.Origin) { + continue + } + candidate := Access{ + Destinations: result.Destinations, + Credentials: append(slices.Clone(result.Credentials), credential), + } + if _, err := MergeAccess(base, candidate, tokenOrigins); err != nil { + dropped = append(dropped, credential.Origin) + } else { + result = candidate + } + } + slices.Sort(dropped) + return result, slices.Compact(dropped) +} diff --git a/go/core/internal/egress/live_test.go b/go/core/internal/egress/live_test.go new file mode 100644 index 0000000000..a9e7e110ab --- /dev/null +++ b/go/core/internal/egress/live_test.go @@ -0,0 +1,103 @@ +package egress + +import ( + "fmt" + "slices" + "testing" + + "github.com/stretchr/testify/require" +) + +func canonicalTestAccess(t *testing.T, destinations []string, credentials ...Credential) Access { + t.Helper() + access, err := CanonicalAccess(destinations, credentials) + require.NoError(t, err) + return access +} + +func cloneAccess(a Access) Access { + return Access{Destinations: slices.Clone(a.Destinations), Credentials: slices.Clone(a.Credentials)} +} + +func TestIntersectAccess(t *testing.T) { + binding := Credential{Origin: "https://a.example.com", Header: "authorization", URI: "ate-secret://k8s.io/default/team/key/token"} + a := canonicalTestAccess(t, []string{"https://a.example.com:443", "https://removed.example.com"}, binding) + changed := a.Credentials[0] + changed.Prefix = "Bearer " + b := Access{Destinations: []string{"https://a.example.com:443", "https://new.example.com:443"}, Credentials: []Credential{changed, a.Credentials[0]}} + beforeA, beforeB := cloneAccess(a), cloneAccess(b) + require.Equal(t, Access{Destinations: []string{a.Destinations[0]}, Credentials: a.Credentials}, IntersectAccess(a, b)) + b.Credentials = []Credential{changed} + require.Empty(t, IntersectAccess(a, b).Credentials, "changed credential values are additions") + b.Credentials = beforeB.Credentials + require.Equal(t, beforeA, a) + require.Equal(t, beforeB, b) +} + +func TestFilterConflicts(t *testing.T) { + binding := Credential{Origin: "https://model.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"} + base := canonicalTestAccess(t, []string{binding.Origin}, binding) + conflict := binding + conflict.URI = "ate-secret://k8s.io/default/team/other/token" + external := binding + external.URI = "ate-secret://provider.example/default/token" + const extra = "https://tickets.example.com:443" + for _, test := range []struct { + name string + base, grants, want Access + tokens, dropped []string + }{ + {name: "additive", base: base, grants: canonicalTestAccess(t, []string{extra}), want: canonicalTestAccess(t, []string{extra})}, + {name: "duplicate", base: base, grants: base, want: base}, + {name: "credential conflict keeps network access", base: base, grants: canonicalTestAccess(t, []string{binding.Origin, extra}, conflict), + want: canonicalTestAccess(t, []string{binding.Origin, extra}), dropped: []string{binding.Origin}}, + {name: "external credential conflict keeps network access", base: canonicalTestAccess(t, []string{binding.Origin}, external), grants: canonicalTestAccess(t, []string{binding.Origin, extra}, binding), + want: canonicalTestAccess(t, []string{binding.Origin, extra}), dropped: []string{binding.Origin}}, + {name: "protocol conflict keeps other grants", base: base, grants: canonicalTestAccess(t, []string{"http://model.example.com:443", extra}), + want: canonicalTestAccess(t, []string{extra}), dropped: []string{"http://model.example.com:443"}}, + {name: "token passthrough", base: canonicalTestAccess(t, []string{binding.Origin}), tokens: []string{binding.Origin}, + grants: canonicalTestAccess(t, []string{binding.Origin, extra}, binding), want: canonicalTestAccess(t, []string{binding.Origin, extra}), dropped: []string{binding.Origin}}, + } { + t.Run(test.name, func(t *testing.T) { + beforeBase, beforeGrants := cloneAccess(test.base), cloneAccess(test.grants) + filtered, dropped := FilterConflicts(test.base, test.grants, test.tokens) + require.Equal(t, test.want, filtered) + require.Equal(t, test.dropped, dropped) + effective, err := EffectiveAccess(test.base, test.grants, test.tokens) + require.NoError(t, err) + want, err := MergeAccess(test.base, test.want, test.tokens) + require.NoError(t, err) + require.Equal(t, want, effective) + require.Equal(t, beforeBase, test.base) + require.Equal(t, beforeGrants, test.grants) + }) + } +} + +func TestFilterConflictsLimits(t *testing.T) { + var destinations []string + for i := range 255 { + destinations = append(destinations, fmt.Sprintf("https://host%03d.example.com:443", i)) + } + base := canonicalTestAccess(t, destinations) + for _, input := range [][]string{ + {"https://z.example.com", "https://a.example.com"}, + {"https://a.example.com", "https://z.example.com"}, + } { + filtered, dropped := FilterConflicts(base, canonicalTestAccess(t, input), nil) + require.Equal(t, []string{"https://a.example.com:443"}, filtered.Destinations) + require.Equal(t, []string{"https://z.example.com:443"}, dropped) + } + var credentials []Credential + for i := range 16 { + credentials = append(credentials, Credential{Origin: "https://model.example.com:443", Header: fmt.Sprintf("x-header-%02d", i), URI: "ate-secret://k8s.io/default/team/auth/token"}) + } + base = canonicalTestAccess(t, []string{credentials[0].Origin}, credentials...) + extra := credentials[0] + extra.Header = "x-new" + grants := canonicalTestAccess(t, base.Destinations, extra) + filtered, dropped := FilterConflicts(base, grants, nil) + require.Empty(t, filtered.Credentials) + require.Equal(t, base.Destinations, filtered.Destinations) + require.Equal(t, base.Destinations, dropped) +} diff --git a/go/core/internal/grpcserver/scheduledrun_test.go b/go/core/internal/grpcserver/scheduledrun_test.go index 4fc98efd31..453442b661 100644 --- a/go/core/internal/grpcserver/scheduledrun_test.go +++ b/go/core/internal/grpcserver/scheduledrun_test.go @@ -180,12 +180,13 @@ func scheduledRunTestServer(t *testing.T) (*database.Client, apiv1.ScheduledRunS t.Cleanup(db.Close) store := database.NewClient(db, "public") pair := database.AgentDefinition{Namespace: "team", AgentName: "report", AgentUID: "template-uid", DesiredRevision: "scheduled-revision"} - require.NoError(t, store.UpsertAgentDefinition(t.Context(), pair)) + upsertErr := store.UpsertAgentDefinition(t.Context(), pair) + require.NoError(t, upsertErr) require.NoError(t, store.RecordRuntimeRevision(t.Context(), database.RuntimeRevision{ Revision: pair.DesiredRevision, Namespace: pair.Namespace, AgentName: pair.AgentName, AgentUID: pair.AgentUID, SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: "runtime-uid", - }, true)) + }, database.RecordRevisionOptions{Ready: true})) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) agent := &v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "report"}, Spec: v1alpha3.AgentSpec{ diff --git a/go/core/internal/grpcserver/task_store_test.go b/go/core/internal/grpcserver/task_store_test.go index 5f1367897e..7ae3ad4cdb 100644 --- a/go/core/internal/grpcserver/task_store_test.go +++ b/go/core/internal/grpcserver/task_store_test.go @@ -637,11 +637,12 @@ func createTaskStoreSession(t *testing.T, store *database.Client) *apiv1.Session AgentCard: &a2apb.AgentCard{Name: "assistant"}, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "assistant-kagent-revision", ActorTemplateUID: "actor-template-uid", } - require.NoError(t, store.UpsertAgentDefinition(t.Context(), database.AgentDefinition{ + upsertErr := store.UpsertAgentDefinition(t.Context(), database.AgentDefinition{ Namespace: revision.Namespace, AgentName: revision.AgentName, AgentUID: revision.AgentUID, DesiredRevision: revision.Revision, - })) - require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision, true)) + }) + require.NoError(t, upsertErr) + require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision, database.RecordRevisionOptions{Ready: true})) session, _, err := store.CreateSession(t.Context(), &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", diff --git a/go/core/internal/service/sandbox/guest_test.go b/go/core/internal/service/sandbox/guest_test.go index c4c2bd2daf..a0c67cf652 100644 --- a/go/core/internal/service/sandbox/guest_test.go +++ b/go/core/internal/service/sandbox/guest_test.go @@ -28,6 +28,7 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" sandboxservice "github.com/kagent-dev/kagent/go/core/internal/service/sandbox" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/pkg/auth" @@ -82,11 +83,11 @@ func (a *testActors) CreateActor(_ context.Context, space, name, templateSpace, Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}} return proto.CloneOf(a.actor), a.mutationErr } -func (a *testActors) EnsureActorEgressPolicy(context.Context, string, string, *ateapipb.EgressPolicy) error { +func (a *testActors) EnsureActorEgressPolicy(context.Context, string, string, string) (egress.Policy, error) { a.mu.Lock() defer a.mu.Unlock() a.calls = append(a.calls, "policy") - return a.mutationErr + return egress.Policy{}, a.mutationErr } func (a *testActors) ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) { a.mu.Lock() @@ -129,11 +130,12 @@ func guestFixture(t *testing.T) (*sandboxservice.Service, context.Context, *test require.NoError(t, err) t.Cleanup(pool.Close) store := database.NewClient(pool, "public") - require.NoError(t, store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"})) + upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"}) + require.NoError(t, upsertErr) require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - }, true)) + }, database.RecordRevisionOptions{Ready: true})) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&v1alpha3.SandboxTemplate{ diff --git a/go/core/internal/service/sandbox/service_test.go b/go/core/internal/service/sandbox/service_test.go index f3de53197d..77b67bd89e 100644 --- a/go/core/internal/service/sandbox/service_test.go +++ b/go/core/internal/service/sandbox/service_test.go @@ -22,6 +22,7 @@ import ( "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/pkg/auth" ) @@ -45,7 +46,7 @@ type testActors struct { substrate.LifecycleClient mu sync.Mutex actor *ateapipb.Actor - egressPolicy *ateapipb.EgressPolicy + egressRevision string calls []string readErr error mutationErr error @@ -73,12 +74,12 @@ func (a *testActors) CreateActor(_ context.Context, space, name, templateSpace, Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}} return proto.CloneOf(a.actor), a.mutationErr } -func (a *testActors) EnsureActorEgressPolicy(_ context.Context, _, _ string, policy *ateapipb.EgressPolicy) error { +func (a *testActors) EnsureActorEgressPolicy(_ context.Context, _, _ string, revision string) (egress.Policy, error) { a.mu.Lock() defer a.mu.Unlock() a.calls = append(a.calls, "policy") - a.egressPolicy = proto.CloneOf(policy) - return a.mutationErr + a.egressRevision = revision + return egress.Policy{}, a.mutationErr } func (a *testActors) ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) { a.mu.Lock() @@ -134,11 +135,12 @@ func serviceFixture(t *testing.T) (*Service, context.Context, *testActors) { require.NoError(t, err) t.Cleanup(pool.Close) store := database.NewClient(pool, "public") - require.NoError(t, store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"})) + upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"}) + require.NoError(t, upsertErr) require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - }, true)) + }, database.RecordRevisionOptions{Ready: true})) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&v1alpha3.SandboxTemplate{ @@ -215,8 +217,7 @@ func TestSandboxLifecycleClientRetriesAfterRestart(t *testing.T) { } } require.Equal(t, 1, count, "retry must preserve the original Actor and files") - require.NotNil(t, actors.egressPolicy) - require.Empty(t, actors.egressPolicy.Rules) + require.NotEmpty(t, actors.egressRevision) }) } } @@ -285,5 +286,24 @@ func TestSandboxConflictingRequestCanRetryAfterActiveAttempt(t *testing.T) { require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, current.State) require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, current.Operation) - require.Equal(t, []string{"create", "policy", "resume", "suspend", "resume"}, actors.observedCalls()) + require.Equal(t, []string{"create", "policy", "resume", "suspend", "policy", "resume"}, actors.observedCalls()) +} + +func TestSandboxResolvesCurrentGrantsBeforeResume(t *testing.T) { + service, ctx, actors := serviceFixture(t) + revision := database.SandboxRevision{ + RuntimeArtifact: database.RuntimeArtifact{Revision: "policy-revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "policy-runtime", ActorTemplateUID: "policy-runtime-uid"}, + SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), + } + store := service.config.Store + upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: revision.Revision}) + require.NoError(t, upsertErr) + require.NoError(t, store.RecordSandboxRevision(ctx, revision, database.RecordRevisionOptions{Ready: true})) + instance, err := service.Create(ctx, createRequest()) + require.NoError(t, err) + require.Equal(t, revision.Revision, instance.PreparedRevision) + require.Equal(t, revision.Revision, actors.egressRevision) + calls := actors.observedCalls() + require.Less(t, slices.Index(calls, "policy"), slices.Index(calls, "resume")) + require.Contains(t, calls, "policy") } diff --git a/go/core/internal/service/sandbox/transports_test.go b/go/core/internal/service/sandbox/transports_test.go index eca00727d0..0874a3f9f0 100644 --- a/go/core/internal/service/sandbox/transports_test.go +++ b/go/core/internal/service/sandbox/transports_test.go @@ -250,5 +250,5 @@ func TestSandboxTransports(t *testing.T) { require.NoError(t, err) _, err = client.DeleteSandbox(ctx, &apiv1.DeleteSandboxRequest{SandboxId: id}) require.NoError(t, err) - require.Equal(t, []string{"create", "policy", "resume", "suspend", "suspend", "resume", "suspend", "delete"}, actors.observedCalls()) + require.Equal(t, []string{"create", "policy", "resume", "suspend", "suspend", "policy", "resume", "suspend", "delete"}, actors.observedCalls()) } diff --git a/go/core/internal/service/sandbox/workflow.go b/go/core/internal/service/sandbox/workflow.go index e285492684..fc1df12f15 100644 --- a/go/core/internal/service/sandbox/workflow.go +++ b/go/core/internal/service/sandbox/workflow.go @@ -39,15 +39,11 @@ func (s *Service) run(ctx context.Context, id string, kind apiv1.RuntimeOperatio if err != nil { return nil, sandboxError(err) } - binding := substrate.ActorBinding{Atespace: revision.ActorTemplateAtespace, Name: substrate.ActorName(id), + binding := substrate.ActorBinding{Revision: revision.Revision, Atespace: revision.ActorTemplateAtespace, Name: substrate.ActorName(id), TemplateAtespace: revision.ActorTemplateAtespace, TemplateName: revision.ActorTemplateName} var creation *substrate.ActorCreation if kind == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { - policy, err := substrate.ActorEgressPolicy(binding.Atespace, nil, nil) - if err != nil { - return nil, sandboxError(err) - } - creation = &substrate.ActorCreation{EgressPolicy: policy, Resume: true} + creation = &substrate.ActorCreation{Resume: true} } prepare := substrate.PrepareActorTransition if op.ExecutorID != uuid.Nil { diff --git a/go/core/internal/service/session/workflow.go b/go/core/internal/service/session/workflow.go index 63a0517ead..805e073bb4 100644 --- a/go/core/internal/service/session/workflow.go +++ b/go/core/internal/service/session/workflow.go @@ -158,15 +158,11 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session tagName = "checkpoint-" + operation.SourceCheckpointID.String() } - binding := substrate.ActorBinding{Atespace: revision.ActorTemplateAtespace, Name: substrate.ActorName(session.Id), + binding := substrate.ActorBinding{Revision: revision.Revision, Atespace: revision.ActorTemplateAtespace, Name: substrate.ActorName(session.Id), TemplateAtespace: revision.ActorTemplateAtespace, TemplateName: revision.ActorTemplateName} var creation *substrate.ActorCreation if kind == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { - policy, err := substrate.ActorEgressPolicy(binding.Atespace, revision.EgressDestinations, revision.Credentials) - if err != nil { - return w.failPreparation(ctx, operation, err) - } - creation = &substrate.ActorCreation{EgressPolicy: policy} + creation = &substrate.ActorCreation{} if snapshot != nil { creation.Snapshot = &substrate.ActorSnapshot{Tag: &ateapipb.ObjectRef{Atespace: snapshot.Atespace, Name: tagName}, URI: snapshot.URI, ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA} } diff --git a/go/core/internal/service/session/workflow_test.go b/go/core/internal/service/session/workflow_test.go index ff9439c3a4..8ce3df2d6b 100644 --- a/go/core/internal/service/session/workflow_test.go +++ b/go/core/internal/service/session/workflow_test.go @@ -148,8 +148,9 @@ func lifecycleFixture(t *testing.T) (*lifecycleTestStore, *apiv1.Session) { AgentCard: &a2apb.AgentCard{Name: "assistant"}, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "assistant-kagent-revision", ActorTemplateUID: "actor-template-uid", } - require.NoError(t, client.UpsertAgentDefinition(t.Context(), database.AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision})) - require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision, true)) + upsertErr := client.UpsertAgentDefinition(t.Context(), database.AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision}) + require.NoError(t, upsertErr) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision, database.RecordRevisionOptions{Ready: true})) session, _, err := client.CreateSession(t.Context(), &apiv1.Session{Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}}, uuid.NewString()) require.NoError(t, err) return &lifecycleTestStore{Client: client, revision: revision}, session @@ -165,12 +166,12 @@ func (s *lifecycleTestStore) GetRuntimeRevision(context.Context, string) (*datab } type lifecycleTestActors struct { - mu sync.Mutex - actors map[string]*ateapipb.Actor - policyErr error - policy *ateapipb.EgressPolicy - policyActor string - policyCalls int + mu sync.Mutex + actors map[string]*ateapipb.Actor + policyErr error + policyRevision string + policyActor string + policyCalls int } func actorKey(atespace, name string) string { return atespace + "/" + name } @@ -284,13 +285,13 @@ func lifecycleForkFixture(t *testing.T, store *lifecycleTestStore, actors *lifec return fork, checkpoint.Id } -func (a *lifecycleTestActors) EnsureActorEgressPolicy(_ context.Context, atespace, name string, policy *ateapipb.EgressPolicy) error { +func (a *lifecycleTestActors) EnsureActorEgressPolicy(_ context.Context, atespace, name string, revision string) (egress.Policy, error) { a.mu.Lock() defer a.mu.Unlock() a.policyActor = actorKey(atespace, name) - a.policy = proto.CloneOf(policy) + a.policyRevision = revision a.policyCalls++ - return a.policyErr + return egress.Policy{}, a.policyErr } func TestActorCreationRetainsEgressPolicyFailure(t *testing.T) { @@ -304,27 +305,15 @@ func TestActorCreationRetainsEgressPolicyFailure(t *testing.T) { actors := &retryTestActors{lifecycleTestActors: base} workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second) callsBefore := base.policyCalls - store.revision.EgressDestinations = []string{"*"} - _, err := workflow.Create(t.Context(), session) - require.ErrorContains(t, err, "invalid egress destination") - require.Zero(t, actors.mutations.Load(), "validate the allowlist before issuing Actor creation") - require.Equal(t, callsBefore, base.policyCalls) - - store.revision.EgressDestinations = []string{"https://api.example.com:443", "https://other.example.com:8443"} - store.revision.Credentials = []egress.Credential{{Hostname: "api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/auth/token"}} base.policyErr = context.DeadlineExceeded - _, err = workflow.Create(t.Context(), session) + _, err := workflow.Create(t.Context(), session) require.ErrorIs(t, err, context.DeadlineExceeded) current, err := store.GetSessionByID(t.Context(), session.Id) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_CREATING, current.State) require.Empty(t, current.A2AAuthority) require.Equal(t, actorKey("team-a", substrate.ActorName(session.Id)), base.policyActor) - require.Equal(t, &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "default"}, base.policy.Metadata) - require.Len(t, base.policy.Rules, 2) - require.Equal(t, &ateapipb.CredentialHeader{Header: "authorization", Prefix: "Bearer ", CredentialUri: "ate-secret://k8s.io/default/team-a/auth/token"}, base.policy.Rules[0].GetHttps().GetEffects().GetReplaceHeaders()[0]) - require.Equal(t, []string{"api.example.com"}, base.policy.Rules[0].GetHttps().GetHostnames()) - require.Equal(t, []string{"other.example.com"}, base.policy.Rules[1].GetHttps().GetHostnames()) + require.Equal(t, session.PreparedRevision, base.policyRevision) // Retry completes policy setup for the existing Actor before readiness. base.policyErr = nil @@ -349,7 +338,7 @@ func TestActorEgressPolicy(t *testing.T) { policy, err = substrate.ActorEgressPolicy("team-a", nil, nil) require.NoError(t, err) require.Empty(t, policy.Rules, "no destinations must deny all egress") - for _, destination := range []string{"", "*", "api.example.com", "https://api.example.com/path", "http://api.example.com:0", "http://api.example.com:65536", "http://user@api.example.com", "http://api.example.com?key=value", "http://*.example.com"} { + for _, destination := range []string{"", "*", "api.example.com", "https://api.example.com/path", "http://api.example.com:0", "http://api.example.com:65536", "http://user@api.example.com", "http://api.example.com?key=value"} { t.Run(destination, func(t *testing.T) { _, err := substrate.ActorEgressPolicy("team-a", []string{destination}, nil) require.Error(t, err) @@ -357,7 +346,7 @@ func TestActorEgressPolicy(t *testing.T) { } for _, destination := range []string{"http://192.0.2.1", "http://[2001:db8::1]", "http://[::ffff:192.0.2.1]"} { _, err := substrate.ActorEgressPolicy("team-a", []string{destination}, nil) - require.ErrorContains(t, err, "requires a DNS name") + require.ErrorContains(t, err, "expected a DNS name") } _, err = substrate.ActorEgressPolicy("team-a", []string{"http://api.example.com:8443", "https://api.example.com:8443"}, nil) require.ErrorContains(t, err, "both HTTP and HTTPS on the same port") @@ -365,15 +354,15 @@ func TestActorEgressPolicy(t *testing.T) { func TestActorEgressCredentialsRequireAllowedDestination(t *testing.T) { bindings := []egress.Credential{ - {Hostname: "api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}, - {Hostname: "api.example.com", Header: "x-api-key", URI: "ate-secret://k8s.io/default/team/auth/key"}, + {Origin: "https://api.example.com:8443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}, + {Origin: "https://api.example.com:8443", Header: "x-api-key", URI: "ate-secret://k8s.io/default/team/auth/key"}, } _, err := substrate.ActorEgressPolicy("team", []string{"https://other.example.com"}, bindings) - require.ErrorContains(t, err, "is not allowed") + require.ErrorContains(t, err, "is not explicitly allowed") policy, err := substrate.ActorEgressPolicy("team", []string{"http://api.example.com:8080", "https://api.example.com:8443", "https://other.example.com"}, bindings) require.NoError(t, err) require.Len(t, policy.Rules, 3) - require.Len(t, policy.Rules[0].GetHttp().GetEffects().GetReplaceHeaders(), 2, "configured HTTP endpoints also require credential replacement") + require.Empty(t, policy.Rules[0].GetHttp().GetEffects().GetReplaceHeaders(), "credentials must not cross schemes or ports") require.Equal(t, []string{"api.example.com"}, policy.Rules[1].GetHttps().GetHostnames()) require.Equal(t, []int32{8443}, policy.Rules[1].GetHttps().GetPorts().GetNumbers()) require.Len(t, policy.Rules[1].GetHttps().GetEffects().GetReplaceHeaders(), 2) diff --git a/go/core/internal/substrate/config.go b/go/core/internal/substrate/config.go index 55939730fb..8699c200c3 100644 --- a/go/core/internal/substrate/config.go +++ b/go/core/internal/substrate/config.go @@ -1,9 +1,14 @@ package substrate -import "time" +import ( + "time" + + "github.com/kagent-dev/kagent/go/core/internal/egress" +) // Config holds connection settings for Agent Substrate ate-api. type Config struct { + EgressPolicies egress.PolicyStore // AteAPIEndpoint is a gRPC target (e.g. dns:///api.ate-system.svc:443). AteAPIEndpoint string // CAFile contains the CA certificates used to verify ate-api. diff --git a/go/core/internal/substrate/egress.go b/go/core/internal/substrate/egress.go index dd7d0be647..67faa1b7b9 100644 --- a/go/core/internal/substrate/egress.go +++ b/go/core/internal/substrate/egress.go @@ -3,32 +3,65 @@ package substrate import ( "context" "fmt" + "slices" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + + "github.com/kagent-dev/kagent/go/core/internal/egress" ) -// EnsureActorEgressPolicy can be retried after a lost create response. A prepared -// revision is immutable, so an existing policy must match; never accept or -// overwrite a different allowlist. Substrate deletes the policy with its Actor. -func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name string, policy *ateapipb.EgressPolicy) error { +// EnsureActorEgressPolicy installs current authorization without restarting the +// actor and returns the effective policy it applied. Read the backend version +// before the controller's persisted intended +// policy so concurrent applications cannot overwrite a newer policy with an old +// one. Conflicts reload both; lifecycle paths never translate Kubernetes. +// Substrate deletes the policy with its Actor. +func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name, revision string) (egress.Policy, error) { ctx, cancel := c.callCtx(ctx) defer cancel() - actor := actorRef(atespace, name) - _, err := c.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{ - Actor: actor, EgressPolicy: policy, - }) - if status.Code(err) != codes.AlreadyExists { - return err + if c.cfg.EgressPolicies == nil { + return egress.Policy{}, fmt.Errorf("actor egress policy store is required") } - existing, err := c.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) - if err != nil { - return err - } - if !proto.Equal(&ateapipb.EgressPolicy{Rules: existing.GetRules()}, &ateapipb.EgressPolicy{Rules: policy.Rules}) { - return fmt.Errorf("existing Actor egress policy does not match the prepared revision") + actor := actorRef(atespace, name) + for range 5 { + existing, err := c.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) + if err != nil && status.Code(err) != codes.NotFound { + return egress.Policy{}, err + } + access, err := c.cfg.EgressPolicies.GetRuntimeEgressPolicy(ctx, revision) + if err != nil { + return egress.Policy{}, fmt.Errorf("load intended actor egress: %w", err) + } + policy, err := ActorEgressPolicy(atespace, access.Destinations, access.Credentials) + if err != nil { + return egress.Policy{}, err + } + if existing == nil { + _, err = c.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{Actor: actor, EgressPolicy: policy}) + } else { + if !proto.Equal(&ateapipb.EgressPolicy{Rules: existing.GetRules()}, &ateapipb.EgressPolicy{Rules: policy.GetRules()}) { + policy.Metadata = proto.CloneOf(existing.GetMetadata()) + _, err = c.UpdateActorEgressPolicy(ctx, &ateapipb.UpdateActorEgressPolicyRequest{Actor: actor, EgressPolicy: policy}) + } + } + if status.Code(err) == codes.AlreadyExists || status.Code(err) == codes.Aborted { + continue + } + if err != nil { + return egress.Policy{}, err + } + // A source update may have discovered this instance before its Actor + // existed. Recheck after applying so creation cannot miss that update. + latest, err := c.cfg.EgressPolicies.GetRuntimeEgressPolicy(ctx, revision) + if err != nil { + return egress.Policy{}, fmt.Errorf("reload intended actor egress: %w", err) + } + if slices.Equal(access.Destinations, latest.Destinations) && slices.Equal(access.Credentials, latest.Credentials) { + return latest, nil + } } - return nil + return egress.Policy{}, status.Error(codes.Aborted, "actor egress policy changed during reconciliation") } diff --git a/go/core/internal/substrate/egress_test.go b/go/core/internal/substrate/egress_test.go index 79f4e2484b..71a7b4acd5 100644 --- a/go/core/internal/substrate/egress_test.go +++ b/go/core/internal/substrate/egress_test.go @@ -2,6 +2,7 @@ package substrate import ( "context" + "errors" "testing" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -10,28 +11,71 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + + "github.com/kagent-dev/kagent/go/core/internal/egress" ) -func TestEnsureActorEgressPolicyRetriesLostResponse(t *testing.T) { +type egressPolicyStoreFunc func(context.Context, string) (egress.Policy, error) + +func (f egressPolicyStoreFunc) GetRuntimeEgressPolicy(ctx context.Context, revision string) (egress.Policy, error) { + return f(ctx, revision) +} + +var _ egress.PolicyStore = egressPolicyStoreFunc(nil) + +func TestEnsureActorEgressPolicyUpdatesCurrentGrants(t *testing.T) { fake := &egressPolicyFake{createErr: context.DeadlineExceeded} - client := &Client{ControlClient: fake} - policy := &ateapipb.EgressPolicy{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "default"}, - Rules: []*ateapipb.EgressRule{ - {Http: &ateapipb.HTTPRule{Hostnames: []string{"api.example.com"}}}, - {Https: &ateapipb.HTTPSRule{Hostnames: []string{"api.example.com"}}}, - }, - } - require.ErrorIs(t, client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", policy), context.DeadlineExceeded) + access := egress.Access{Destinations: []string{"https://api.example.com:443"}} + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(_ context.Context, revision string) (egress.Policy, error) { + require.Equal(t, "pinned-revision", revision) + return egress.Policy{Access: access}, nil + })}} + reconcile := func() error { + _, err := client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "pinned-revision") + return err + } + require.ErrorIs(t, reconcile(), context.DeadlineExceeded) require.NotNil(t, fake.policy, "the server committed before the response was lost") fake.createErr = nil - require.NoError(t, client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", policy)) - require.NoError(t, client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", policy)) + require.NoError(t, reconcile()) + require.NoError(t, reconcile()) require.Equal(t, 1, fake.created) + require.Zero(t, fake.updated) require.Equal(t, actorRef("team-a", "actor"), fake.actor) - require.ErrorContains(t, client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", &ateapipb.EgressPolicy{}), "does not match") + + access = egress.Access{} + require.NoError(t, reconcile()) + require.Empty(t, fake.policy.Rules, "removing the grant revokes the existing actor") + require.Equal(t, 1, fake.updated) + access = egress.Access{Destinations: []string{"https://api.example.com:443"}, Credentials: []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team-a/auth/token", Prefix: "Bearer "}}} + require.NoError(t, reconcile()) + require.Equal(t, "Bearer ", fake.policy.Rules[0].GetHttps().GetEffects().GetReplaceHeaders()[0].Prefix) + access.Credentials[0].URI = "ate-secret://k8s.io/default/team-a/rotated/token" + require.NoError(t, reconcile()) + require.Equal(t, access.Credentials[0].URI, fake.policy.Rules[0].GetHttps().GetEffects().GetReplaceHeaders()[0].CredentialUri) fake.getErr = status.Error(codes.Unavailable, "unavailable") - require.Equal(t, codes.Unavailable, status.Code(client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", policy))) + require.Equal(t, codes.Unavailable, status.Code(reconcile())) +} + +func TestEnsureActorEgressPolicyReloadsAfterConflict(t *testing.T) { + fake := &egressPolicyFake{} + access := egress.Access{} + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Policy, error) { return egress.Policy{Access: access}, nil })}} + reconcile := func() error { + _, err := client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "revision") + return err + } + require.NoError(t, reconcile()) + access.Destinations = []string{"https://revoked.example.com:443"} + fake.beforeUpdate = func() { + // Another reconciliation committed revocation after this attempt read the + // backend version. Retrying must resolve current grants, not replay the allow. + access = egress.Access{} + fake.policy.Metadata.Version++ + } + require.NoError(t, reconcile()) + require.Empty(t, fake.policy.Rules) + require.Zero(t, fake.updated) } type egressPolicyFake struct { @@ -39,7 +83,9 @@ type egressPolicyFake struct { policy *ateapipb.EgressPolicy actor *ateapipb.ObjectRef createErr, getErr error - created int + created, updated int + beforeUpdate func() + afterCreate func() } func (f *egressPolicyFake) CreateActorEgressPolicy(_ context.Context, req *ateapipb.CreateActorEgressPolicyRequest, _ ...grpc.CallOption) (*ateapipb.EgressPolicy, error) { @@ -51,10 +97,60 @@ func (f *egressPolicyFake) CreateActorEgressPolicy(_ context.Context, req *ateap f.policy.Metadata.Uid = "policy-uid" f.policy.Metadata.Version = 1 f.created++ - return f.policy, f.createErr + if f.afterCreate != nil { + f.afterCreate() + } + return proto.CloneOf(f.policy), f.createErr } func (f *egressPolicyFake) GetActorEgressPolicy(_ context.Context, req *ateapipb.GetActorEgressPolicyRequest, _ ...grpc.CallOption) (*ateapipb.EgressPolicy, error) { f.actor = req.Actor - return f.policy, f.getErr + if f.getErr != nil { + return nil, f.getErr + } + if f.policy == nil { + return nil, status.Error(codes.NotFound, "missing") + } + return proto.CloneOf(f.policy), nil +} + +func (f *egressPolicyFake) UpdateActorEgressPolicy(_ context.Context, req *ateapipb.UpdateActorEgressPolicyRequest, _ ...grpc.CallOption) (*ateapipb.EgressPolicy, error) { + if f.beforeUpdate != nil { + f.beforeUpdate() + f.beforeUpdate = nil + } + if req.EgressPolicy.Metadata.Version != f.policy.Metadata.Version || req.EgressPolicy.Metadata.Uid != f.policy.Metadata.Uid { + return nil, status.Error(codes.Aborted, "version conflict") + } + f.policy = proto.CloneOf(req.EgressPolicy) + f.policy.Metadata.Version++ + f.updated++ + return proto.CloneOf(f.policy), nil +} + +func TestEnsureActorEgressPolicyCatchesUpdateDuringCreation(t *testing.T) { + fake := &egressPolicyFake{} + access := egress.Access{Destinations: []string{"https://revoked.example.com:443"}} + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Policy, error) { return egress.Policy{Access: access}, nil })}} + fake.afterCreate = func() { + // The controller published revocation, but its actor discovery ran before + // creation. There is no concurrent Substrate write to cause a CAS conflict. + access = egress.Access{} + } + applied, err := client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision") + require.NoError(t, err) + require.Equal(t, access, applied.Access, "the returned policy must reflect the post-creation revocation") + require.Empty(t, fake.policy.Rules) + require.Equal(t, 1, fake.created) + require.Equal(t, 1, fake.updated) +} + +func TestEnsureActorEgressPolicyRequiresPersistedPolicy(t *testing.T) { + fake := &egressPolicyFake{} + unavailable := errors.New("intended policy unavailable") + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Policy, error) { return egress.Policy{}, unavailable })}} + _, err := client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision") + require.ErrorIs(t, err, unavailable) + require.Zero(t, fake.created) + require.Zero(t, fake.updated) } diff --git a/go/core/internal/substrate/lifecycle.go b/go/core/internal/substrate/lifecycle.go index 169647eded..cb8f8800ca 100644 --- a/go/core/internal/substrate/lifecycle.go +++ b/go/core/internal/substrate/lifecycle.go @@ -10,13 +10,14 @@ import ( "google.golang.org/protobuf/proto" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) type LifecycleClient interface { GetActor(context.Context, string, string) (*ateapipb.Actor, error) CreateActor(context.Context, string, string, string, string) (*ateapipb.Actor, error) CreateActorFromTag(context.Context, string, string, string, string, string, string) (*ateapipb.Actor, error) - EnsureActorEgressPolicy(context.Context, string, string, *ateapipb.EgressPolicy) error + EnsureActorEgressPolicy(context.Context, string, string, string) (egress.Policy, error) ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) SuspendActor(context.Context, string, string) (*ateapipb.Actor, error) DeleteActor(context.Context, string, string) error @@ -25,6 +26,7 @@ type LifecycleClient interface { var _ LifecycleClient = (*Client)(nil) type ActorBinding struct { + Revision string Atespace string Name string TemplateAtespace string @@ -40,9 +42,8 @@ type ActorSnapshot struct { // ActorCreation describes runtime startup, independent of the owning API. // Agent creation stays suspended; a standalone guest starts serving immediately. type ActorCreation struct { - EgressPolicy *ateapipb.EgressPolicy - Snapshot *ActorSnapshot - Resume bool + Snapshot *ActorSnapshot + Resume bool } type ActorTransition struct { @@ -75,8 +76,8 @@ func prepareActorTransition(ctx context.Context, actors LifecycleClient, binding return nil, fmt.Errorf("invalid actor lifecycle operation %s", operation) } if operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { - if creation == nil || creation.EgressPolicy == nil { - return nil, fmt.Errorf("actor creation requires an egress policy") + if creation == nil || binding.Revision == "" { + return nil, fmt.Errorf("actor creation requires a prepared revision") } if snapshot := creation.Snapshot; snapshot != nil && (snapshot.Tag.GetAtespace() == "" || snapshot.Tag.GetName() == "" || snapshot.URI == "" || snapshot.ContentScope != ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA) { return nil, fmt.Errorf("actor restoration requires a retained DATA snapshot") @@ -96,7 +97,7 @@ func prepareActorTransition(ctx context.Context, actors LifecycleClient, binding if operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE && !reconcile { return nil, fmt.Errorf("refuse to adopt existing Actor %s/%s while creation is still pending", binding.Atespace, binding.Name) } - if !binding.matches(actor) { + if !binding.Matches(actor) { return nil, fmt.Errorf("actor %s/%s identity or template changed", binding.Atespace, binding.Name) } if operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME || operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND { @@ -144,7 +145,7 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio } } if err == nil { - err = actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, creation.EgressPolicy) + _, err = actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, binding.Revision) } if err == nil && creation.Resume && actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_RUNNING { actor, err = actors.ResumeActor(ctx, binding.Atespace, binding.Name) @@ -153,6 +154,9 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio } } case apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME: + if _, err := actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, binding.Revision); err != nil { + return err + } actor, err = actors.ResumeActor(ctx, binding.Atespace, binding.Name) if err == nil { err = transition.checkResult(actor, ateapipb.ActorState_ACTOR_STATE_RUNNING) @@ -192,13 +196,14 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio return nil } -func (b ActorBinding) matches(actor *ateapipb.Actor) bool { +// Matches checks that the actor belongs to the expected runtime and template. +func (b ActorBinding) Matches(actor *ateapipb.Actor) bool { return actor.GetMetadata().GetName() == b.Name && actor.GetMetadata().GetAtespace() == b.Atespace && actor.GetMetadata().GetUid() != "" && actor.GetActorTemplate().GetAtespace() == b.TemplateAtespace && actor.GetActorTemplate().GetName() == b.TemplateName } func (t *ActorTransition) checkResult(actor *ateapipb.Actor, state ateapipb.ActorState) error { - if !t.binding.matches(actor) || actor.GetStatus().GetState() != state || (t.actor != nil && actor.GetMetadata().GetUid() != t.actor.GetMetadata().GetUid()) { + if !t.binding.Matches(actor) || actor.GetStatus().GetState() != state || (t.actor != nil && actor.GetMetadata().GetUid() != t.actor.GetMetadata().GetUid()) { return fmt.Errorf("actor returned unexpected identity or state") } return nil diff --git a/go/core/internal/substrate/lifecycle_test.go b/go/core/internal/substrate/lifecycle_test.go index 6abfa64346..a3a620d24b 100644 --- a/go/core/internal/substrate/lifecycle_test.go +++ b/go/core/internal/substrate/lifecycle_test.go @@ -12,6 +12,7 @@ import ( "google.golang.org/protobuf/proto" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) // Unexpected calls use the embedded nil interface and fail the test. @@ -41,9 +42,9 @@ func (a *transitionActors) CreateActor(_ context.Context, space, name, templateS return proto.CloneOf(a.actor), nil } -func (a *transitionActors) EnsureActorEgressPolicy(context.Context, string, string, *ateapipb.EgressPolicy) error { +func (a *transitionActors) EnsureActorEgressPolicy(context.Context, string, string, string) (egress.Policy, error) { a.calls = append(a.calls, "policy") - return a.policyErr + return egress.Policy{}, a.policyErr } func (a *transitionActors) ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) { @@ -88,9 +89,9 @@ func TestActorCreationStartupPolicy(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { actors := &transitionActors{policyErr: test.policyErr, resumeUID: test.resumeUID} - binding := ActorBinding{Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} + binding := ActorBinding{Revision: "revision", Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} transition, err := PrepareActorTransition(t.Context(), actors, binding, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, - &ActorCreation{EgressPolicy: &ateapipb.EgressPolicy{}, Resume: test.resume}) + &ActorCreation{Resume: test.resume}) require.NoError(t, err) require.Empty(t, actors.calls, "preparation must not mutate the backend") err = ApplyActorTransition(t.Context(), actors, transition) @@ -118,8 +119,8 @@ func TestActorRetryContinuesExistingCreation(t *testing.T) { Status: &ateapipb.ActorStatus{State: state}, }} transition, err := PrepareActorRetry(t.Context(), actors, - ActorBinding{Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"}, - apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, &ActorCreation{EgressPolicy: &ateapipb.EgressPolicy{}, Resume: true}) + ActorBinding{Revision: "revision", Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"}, + apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, &ActorCreation{Resume: true}) require.NoError(t, err) require.NoError(t, ApplyActorTransition(t.Context(), actors, transition)) require.Equal(t, "original", actors.actor.Metadata.Uid) @@ -152,8 +153,8 @@ func TestActorTransitionAdmission(t *testing.T) { if test.mutate != nil { test.mutate(actors.actor) } - binding := ActorBinding{Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} - _, err = PrepareActorTransition(t.Context(), actors, binding, test.operation, &ActorCreation{EgressPolicy: &ateapipb.EgressPolicy{}}) + binding := ActorBinding{Revision: "revision", Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} + _, err = PrepareActorTransition(t.Context(), actors, binding, test.operation, &ActorCreation{}) require.Error(t, err) require.Empty(t, actors.calls) }) @@ -166,7 +167,7 @@ func TestActorDeleteStopsRunningCompute(t *testing.T) { require.NoError(t, err) actors.actor.Status.State = ateapipb.ActorState_ACTOR_STATE_RUNNING actors.calls = nil - binding := ActorBinding{Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} + binding := ActorBinding{Revision: "revision", Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} transition, err := PrepareActorTransition(t.Context(), actors, binding, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE, nil) require.NoError(t, err) require.NoError(t, ApplyActorTransition(t.Context(), actors, transition)) @@ -191,7 +192,7 @@ func TestActorSuspendRequiresDataSnapshot(t *testing.T) { actors := &transitionActors{suspendSnapshot: test.snapshot} _, err := actors.CreateActor(t.Context(), "team", "instance", "team", "revision") require.NoError(t, err) - transition, err := PrepareActorTransition(t.Context(), actors, ActorBinding{Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"}, apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND, nil) + transition, err := PrepareActorTransition(t.Context(), actors, ActorBinding{Revision: "revision", Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"}, apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND, nil) require.NoError(t, err) require.ErrorContains(t, ApplyActorTransition(t.Context(), actors, transition), "external DATA snapshot") }) diff --git a/go/core/internal/substrate/policy.go b/go/core/internal/substrate/policy.go index 3fe905fb6e..f373572a00 100644 --- a/go/core/internal/substrate/policy.go +++ b/go/core/internal/substrate/policy.go @@ -1,16 +1,9 @@ package substrate import ( - "fmt" - "net" - "net/netip" - "net/url" - "slices" "strconv" - "strings" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "k8s.io/apimachinery/pkg/util/validation" "github.com/kagent-dev/kagent/go/core/internal/egress" ) @@ -18,71 +11,29 @@ import ( // ActorEgressPolicy compiles HTTP(S) origins into an actor's default allowlist. // Credential bindings are already canonicalized by the store. func ActorEgressPolicy(atespace string, destinations []string, credentials []egress.Credential) (*ateapipb.EgressPolicy, error) { - type originTarget struct { - scheme, hostname string - port int32 - } - origins := map[string]originTarget{} - protocols := map[string]string{} - hosts := map[string]bool{} - for _, destination := range destinations { - u, err := url.Parse(destination) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.Hostname() == "" || u.ForceQuery || u.RawQuery != "" || u.Fragment != "" || u.Path != "" { - return nil, fmt.Errorf("invalid egress destination %q: expected an HTTP(S) origin", destination) - } - host := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") - if _, err := netip.ParseAddr(host); err == nil { - return nil, fmt.Errorf("egress destination %q requires a DNS name: Substrate does not support IP allowlists", destination) - } - if len(validation.IsDNS1123Subdomain(host)) != 0 { - return nil, fmt.Errorf("invalid egress destination %q", destination) - } - origin := egress.Origin(u) - u, err = url.Parse(origin) - if err != nil { - return nil, fmt.Errorf("invalid egress destination %q: %w", destination, err) - } - port, err := strconv.ParseUint(u.Port(), 10, 16) - if err != nil || port == 0 { - return nil, fmt.Errorf("invalid egress port in %q", destination) - } - u.Host = net.JoinHostPort(host, strconv.FormatUint(port, 10)) - if protocol, found := protocols[u.Host]; found && protocol != u.Scheme { - return nil, fmt.Errorf("egress destination %q uses both HTTP and HTTPS on the same port", u.Host) - } - protocols[u.Host] = u.Scheme - hosts[host] = true - origins[u.String()] = originTarget{scheme: u.Scheme, hostname: host, port: int32(port)} + access, err := egress.CanonicalAccess(destinations, credentials) + if err != nil { + return nil, err } effects := map[string]*ateapipb.HttpRuleEffects{} - for _, binding := range credentials { - if !hosts[binding.Hostname] { - return nil, fmt.Errorf("credential destination %q is not allowed", binding.Hostname) - } - if effects[binding.Hostname] == nil { - effects[binding.Hostname] = &ateapipb.HttpRuleEffects{} + for _, binding := range access.Credentials { + if effects[binding.Origin] == nil { + effects[binding.Origin] = &ateapipb.HttpRuleEffects{} } - effects[binding.Hostname].ReplaceHeaders = append(effects[binding.Hostname].ReplaceHeaders, &ateapipb.CredentialHeader{Header: binding.Header, Prefix: binding.Prefix, CredentialUri: binding.URI}) + effects[binding.Origin].ReplaceHeaders = append(effects[binding.Origin].ReplaceHeaders, &ateapipb.CredentialHeader{Header: binding.Header, Prefix: binding.Prefix, CredentialUri: binding.URI}) } - names := make([]string, 0, len(origins)) - for origin := range origins { - names = append(names, origin) - } - slices.Sort(names) policy := &ateapipb.EgressPolicy{Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: "default"}} - for _, origin := range names { - target := origins[origin] - ports := &ateapipb.Ports{Numbers: []int32{target.port}} - // Keep exactly one rule per host and port so an overlapping rule cannot - // bypass credential replacement. - if target.scheme == "https" { - policy.Rules = append(policy.Rules, &ateapipb.EgressRule{Https: &ateapipb.HTTPSRule{Hostnames: []string{target.hostname}, Ports: ports, Effects: effects[target.hostname]}}) + for _, origin := range access.Destinations { + target, _ := egress.ParseOrigin(origin) // validated by CanonicalAccess + port, _ := strconv.ParseInt(target.Port(), 10, 32) + ports := &ateapipb.Ports{Numbers: []int32{int32(port)}} + // One rule per origin: exact matches win over wildcard rules, keeping + // credentials on their exact origin even when a wildcard also grants access. + if target.Scheme == "https" { + policy.Rules = append(policy.Rules, &ateapipb.EgressRule{Https: &ateapipb.HTTPSRule{Hostnames: []string{target.Hostname()}, Ports: ports, Effects: effects[origin]}}) } else { - policy.Rules = append(policy.Rules, &ateapipb.EgressRule{Http: &ateapipb.HTTPRule{Hostnames: []string{target.hostname}, Ports: ports, Effects: effects[target.hostname]}}) + policy.Rules = append(policy.Rules, &ateapipb.EgressRule{Http: &ateapipb.HTTPRule{Hostnames: []string{target.Hostname()}, Ports: ports, Effects: effects[origin]}}) } } - if len(policy.Rules) > 256 { - return nil, fmt.Errorf("egress policy exceeds 256 rules") - } return policy, nil } diff --git a/go/core/internal/substrate/policy_test.go b/go/core/internal/substrate/policy_test.go new file mode 100644 index 0000000000..86350c29dd --- /dev/null +++ b/go/core/internal/substrate/policy_test.go @@ -0,0 +1,30 @@ +package substrate + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +func TestActorPolicyKeepsCredentialsOnExactOrigin(t *testing.T) { + policy, err := ActorEgressPolicy("team", []string{"https://*.example.com", "https://api.example.com:443", "https://api.example.com:8443", "http://api.example.com"}, []egress.Credential{ + {Origin: "https://api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}, + {Origin: "https://api.example.com:8443", Header: "x-api-key", URI: "ate-secret://provider.example/default/token"}, + }) + require.NoError(t, err) + require.Len(t, policy.Rules, 4) + require.Empty(t, policy.Rules[0].GetHttp().GetEffects().GetReplaceHeaders()) + require.Equal(t, []string{"*.example.com"}, policy.Rules[1].GetHttps().GetHostnames()) + require.Empty(t, policy.Rules[1].GetHttps().GetEffects().GetReplaceHeaders()) + for i, header := range []string{"authorization", "x-api-key"} { + rule := policy.Rules[i+2].GetHttps() + require.Equal(t, []string{"api.example.com"}, rule.GetHostnames()) + require.Len(t, rule.GetEffects().GetReplaceHeaders(), 1) + require.Equal(t, header, rule.GetEffects().GetReplaceHeaders()[0].Header) + } + require.Equal(t, []int32{443}, policy.Rules[2].GetHttps().GetPorts().GetNumbers()) + require.Equal(t, []int32{8443}, policy.Rules[3].GetHttps().GetPorts().GetNumbers()) + require.Equal(t, "ate-secret://provider.example/default/token", policy.Rules[3].GetHttps().GetEffects().GetReplaceHeaders()[0].GetCredentialUri()) +} diff --git a/go/core/internal/substrate/sandbox.go b/go/core/internal/substrate/sandbox.go index 5282acf36d..cfe27e2e28 100644 --- a/go/core/internal/substrate/sandbox.go +++ b/go/core/internal/substrate/sandbox.go @@ -9,6 +9,7 @@ import ( atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" @@ -25,7 +26,6 @@ type SandboxPolicy struct { type sandboxInputs struct { Kind string - Version int Namespace string Name string UID string @@ -67,17 +67,13 @@ func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1. if len(environment) > 32 { return nil, "", nil, fmt.Errorf("sandbox exceeds Substrate's 32 environment variables") } - // Namespace-scoped placement changes immutable template contents, so it - // needs a new revision even when the operator's inputs stay the same. - inputs := sandboxInputs{Kind: "sandbox", Version: 2, Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), Spec: template.Spec, Class: class, Policy: policy} + inputs := sandboxInputs{Kind: "sandbox", Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), Spec: template.Spec, Class: class, Policy: policy} snapshot, err := json.Marshal(inputs) if err != nil { return nil, "", nil, fmt.Errorf("encode sandbox inputs: %w", err) } - digest := sha256.Sum256(snapshot) - revision := hex.EncodeToString(digest[:]) result := &ateapipb.ActorTemplate{ - Metadata: &ateapipb.ResourceMetadata{Atespace: template.Namespace, Name: "sandbox-" + revision[:40]}, + Metadata: &ateapipb.ResourceMetadata{Atespace: template.Namespace}, SandboxConfig: config, WorkerSelector: workerSelectorForPool(types.NamespacedName{Namespace: template.Namespace, Name: template.Spec.Substrate.WorkerPoolRef.Name}), Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{{Name: "cpu", Quantity: policy.CPU}, {Name: "memory", Quantity: policy.Memory}}}, @@ -99,5 +95,22 @@ func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1. OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, }, } + // Hash the compiled template and source identity. Live policy references + // remain in provenance but do not affect immutable runtime contents. + compiled, err := proto.MarshalOptions{Deterministic: true}.Marshal(result) + if err != nil { + return nil, "", nil, fmt.Errorf("encode sandbox template: %w", err) + } + identity, err := json.Marshal(struct { + Name string + UID string + Template []byte + }{template.Name, string(template.UID), compiled}) + if err != nil { + return nil, "", nil, fmt.Errorf("encode sandbox revision: %w", err) + } + digest := sha256.Sum256(identity) + revision := hex.EncodeToString(digest[:]) + result.Metadata.Name = "sandbox-" + revision[:40] return result, revision, snapshot, nil } diff --git a/go/core/internal/substrate/sandbox_test.go b/go/core/internal/substrate/sandbox_test.go index 88edb45c88..4c2f30ff64 100644 --- a/go/core/internal/substrate/sandbox_test.go +++ b/go/core/internal/substrate/sandbox_test.go @@ -1,6 +1,7 @@ package substrate import ( + "encoding/json" "strings" "testing" @@ -35,6 +36,13 @@ func TestSandboxPreparationPinsCompleteInputs(t *testing.T) { _, same, _, err := SandboxActorTemplate(template, atev1alpha1.SandboxClassGvisor, policy) require.NoError(t, err) require.Equal(t, digest, same) + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "live"}} + _, unchanged, source, err := SandboxActorTemplate(template, "", policy) + require.NoError(t, err) + require.Equal(t, digest, unchanged, "policy references are live authorization") + var recorded sandboxInputs + require.NoError(t, json.Unmarshal(source, &recorded)) + require.Equal(t, template.Spec, recorded.Spec, "provenance retains the complete source spec") for _, change := range []struct { name string mutate func(*v1alpha3.SandboxTemplate, *SandboxPolicy) diff --git a/go/core/internal/translator/adkconfig/builder.go b/go/core/internal/translator/adkconfig/builder.go index a7e5e54ed0..acea3d8dbc 100644 --- a/go/core/internal/translator/adkconfig/builder.go +++ b/go/core/internal/translator/adkconfig/builder.go @@ -5,20 +5,16 @@ import ( "crypto/sha256" "encoding/json" "fmt" - "net/url" "slices" "strings" - "google.golang.org/protobuf/encoding/protojson" "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" runtimeconfig "github.com/kagent-dev/kagent/go/adk/pkg/config" - "github.com/kagent-dev/kagent/go/adk/pkg/models" configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -71,7 +67,7 @@ func (c *Builder) Build(ctx context.Context, input *translator.HarnessInput) (*R return nil, err } } - result, err := c.compileAgent(ctx, input.Root) + result, err := c.compileAgent(ctx, input.Root, input.Policy) if err != nil { return nil, err } @@ -79,10 +75,10 @@ func (c *Builder) Build(ctx context.Context, input *translator.HarnessInput) (*R if err := applyOutputSchema(result.Config, input.OutputSchema); err != nil { return nil, err } - if err := c.applyCompaction(result, input.Harness, input.Root.Template); err != nil { + if err := applyCompaction(result, input.Harness, input.Root.Template, input.SummarizerModel); err != nil { return nil, err } - if err := c.applyMemory(result, input.Harness); err != nil { + if err := applyMemory(result, input.Harness, input.MemoryModel); err != nil { return nil, err } } @@ -92,7 +88,7 @@ func (c *Builder) Build(ctx context.Context, input *translator.HarnessInput) (*R return result, nil } -func (c *Builder) compileAgent(ctx context.Context, input *translator.AgentInput) (*Result, error) { +func (c *Builder) compileAgent(ctx context.Context, input *translator.AgentInput, policy translator.ResolvedPolicy) (*Result, error) { modelRuntime := &modelRuntime{} var modelConfig *v1alpha3.ModelConfig if input.ResolvedModelConfig != nil { @@ -123,6 +119,10 @@ func (c *Builder) compileAgent(ctx context.Context, input *translator.AgentInput if err != nil { return nil, fmt.Errorf("resolve %s %q: %w", tool.Binding.Server.Kind, tool.Binding.Server.Name, err) } + headers, err = policy.MCPHeaders(tool.Server.Spec.URL, headers) + if err != nil { + return nil, err + } server := tool.Server.DeepCopy() server.Spec.HeadersFrom = nil if err := c.addRemoteMCPServer(cfg, server, tool.Binding.Tools, tool.Binding.RequireApproval, headers); err != nil { @@ -133,14 +133,14 @@ func (c *Builder) compileAgent(ctx context.Context, input *translator.AgentInput result := &Result{ Config: cfg, Templates: []*translator.TemplateConfiguration{input.Template}, Environment: modelRuntime.Environment, - Egress: append(agentConfigDestinations(cfg, modelConfig, modelRuntime.Model), plugins.Egress...), + Egress: plugins.Egress, SkillImages: plugins.Images, } if modelConfig != nil { result.Models = []*translator.ResolvedModelConfig{input.ResolvedModelConfig} } for _, binding := range input.Shared { - child, err := c.compileAgent(ctx, binding.Agent) + child, err := c.compileAgent(ctx, binding.Agent, policy) if err != nil { return nil, err } @@ -303,109 +303,3 @@ func DedupeEnv(values []corev1.EnvVar) []corev1.EnvVar { } return result } - -// agentConfigDestinations extracts the network allowlist required by the -// resolved model and MCP configuration. Provider defaults are included when -// no explicit endpoint appears in the serialized model. -func agentConfigDestinations(cfg *configv1.ADKAgentConfig, modelConfig *v1alpha3.ModelConfig, model *configv1.ADKModelConfig) []string { - destinations := make([]string, 0, len(cfg.HttpTools)+len(cfg.SseTools)+1) - for _, tool := range cfg.HttpTools { - destinations = appendURLOrigin(destinations, tool.Params.Url) - } - for _, tool := range cfg.SseTools { - destinations = appendURLOrigin(destinations, tool.Params.Url) - } - modelJSON, _ := protojson.Marshal(model) - var values any - if json.Unmarshal(modelJSON, &values) == nil { - destinations = appendURLValues(destinations, values) - } - if modelConfig == nil { - slices.Sort(destinations) - return slices.Compact(destinations) - } - switch modelConfig.Spec.Provider { - case v1alpha3.ModelProviderOpenAI: - destinations = append(destinations, "https://api.openai.com:443") - case v1alpha3.ModelProviderAnthropic: - destinations = append(destinations, "https://api.anthropic.com:443") - case v1alpha3.ModelProviderGemini: - destinations = append(destinations, "https://generativelanguage.googleapis.com:443") - case v1alpha3.ModelProviderMistral: - if mistral := modelConfig.Spec.Mistral; mistral == nil || mistral.BaseURL == nil || *mistral.BaseURL == "" { - destinations = append(destinations, "https://api.mistral.ai:443") - } - case v1alpha3.ModelProviderOllama: - // Ollama's endpoint is the provider's own field and is not part of the - // serialized model, so the walk above never sees it. Unlike the - // providers above there is no default to fall back on: the host is the - // operator's, so it has to be read from the spec. - if ollama := modelConfig.Spec.Ollama; ollama != nil { - if ollama.Host != "" { - destinations = appendURLOrigin(destinations, withDefaultScheme(ollama.Host)) - } - // A cloud model with a key and no explicit host reaches - // api.ollama.com, so the agent needs that host allowed or the call - // is denied by egress policy. The condition is shared with the key - // reference and credential binding (models.OllamaReachesCloud). - // - // This used to add the host whenever a cloud model had any key, - // ignoring the operator's host. That over-allowed egress: with a - // host set the request goes to that host, so api.ollama.com never - // needs to be reachable. - hasCredential := modelConfig.Spec.APIKeySecret != "" || modelConfig.Spec.APIKeyPassthrough - if models.OllamaReachesCloud(modelConfig.Spec.Model, ollama.Host, hasCredential) { - destinations = append(destinations, "https://api.ollama.com:443") - } - } - } - slices.Sort(destinations) - return slices.Compact(destinations) -} - -// withDefaultScheme makes a bare host:port an absolute URL. The Ollama host -// field accepts either form, but net/url reads the bare one as a scheme, so a -// caller that wants a hostname from it has to normalize first. -// -// A bare host defaults to http, which is right for a daemon (host:port on a -// private address). It is wrong for ollama.com's API, which serves HTTPS only: -// Cloudflare answers the http form with a redirect, and Go turns a 301 into a -// GET, so the POST /api/chat that should carry the request comes back 405 -// Method Not Allowed. The cloud endpoint therefore has to be normalized to -// https here, which is what the runtime's own copy of this does — the two -// disagreed, and the actor was pinned to http://api.ollama.com. -func withDefaultScheme(host string) string { - if strings.HasPrefix(host, "http://") || strings.HasPrefix(host, "https://") { - return host - } - if models.IsOllamaCloudEndpoint(host) { - return "https://" + host - } - return "http://" + host -} - -// appendURLValues walks serialized provider config because endpoint fields are -// provider-specific but all URLs reduce to HTTP(S) origins. -func appendURLValues(destinations []string, value any) []string { - switch value := value.(type) { - case string: - return appendURLOrigin(destinations, value) - case []any: - for _, item := range value { - destinations = appendURLValues(destinations, item) - } - case map[string]any: - for _, item := range value { - destinations = appendURLValues(destinations, item) - } - } - return destinations -} - -func appendURLOrigin(destinations []string, raw string) []string { - parsed, err := url.Parse(raw) - if err == nil && parsed.Hostname() != "" { - return append(destinations, egress.Origin(parsed)) - } - return destinations -} diff --git a/go/core/internal/translator/adkconfig/builder_test.go b/go/core/internal/translator/adkconfig/builder_test.go index 2babf5a0cb..c8ec6c290b 100644 --- a/go/core/internal/translator/adkconfig/builder_test.go +++ b/go/core/internal/translator/adkconfig/builder_test.go @@ -85,6 +85,12 @@ func TestBuildCompaction(t *testing.T) { builder := NewBuilder(krt.TestingDummyContext{}, collections) return builder.Build(context.Background(), &translator.HarnessInput{ Harness: harness, Root: &translator.AgentInput{Template: template, ResolvedModelConfig: agentModel}, + SummarizerModel: func() *translator.ResolvedModelConfig { + if c := harness.Spec.Kagent.Compaction; c != nil && c.Summarizer != nil && c.Summarizer.ModelConfigRef != nil && c.Summarizer.ModelConfigRef.Name == "summarizer" { + return resolvedModel(t, collections, "summarizer") + } + return nil + }(), }) } @@ -123,7 +129,7 @@ func TestBuildCompaction(t *testing.T) { require.Equal(t, "https://summarizer.example.com/v1", summarizer.BaseUrl) require.Len(t, result.Models, 2) require.Equal(t, "summarizer", result.Models[1].Config.Name) - require.Contains(t, result.Egress, "https://summarizer.example.com:443") + var credentials []string for _, variable := range result.Environment { if variable.ValueFrom != nil && variable.ValueFrom.SecretKeyRef != nil { @@ -153,131 +159,6 @@ func openAIModel(name, baseURL string) *v1alpha3.ModelConfig { } } -func TestMistralEgressDestination(t *testing.T) { - for _, test := range []struct { - name string - mistral *v1alpha3.MistralConfig - want string - }{ - {name: "default endpoint", want: "https://api.mistral.ai:443"}, - {name: "base URL override", mistral: &v1alpha3.MistralConfig{BaseURL: new("https://mistral.example.com/v1")}, want: "https://mistral.example.com:443"}, - } { - t.Run(test.name, func(t *testing.T) { - model := &v1alpha3.ModelConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "mistral", Namespace: "test"}, - Spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderMistral, Model: "mistral-large-latest", Mistral: test.mistral, - }, - } - collections := contextTestCollections(t, model) - result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(context.Background(), - &translator.HarnessInput{ - Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}}}, - Root: &translator.AgentInput{ - Template: &translator.TemplateConfiguration{Name: "pi", Namespace: "test"}, - ResolvedModelConfig: resolvedModel(t, collections, "mistral"), - }}) - require.NoError(t, err) - require.Equal(t, []string{test.want}, result.Egress) - }) - } -} - -// TestOllamaEgressDestination covers the one provider whose endpoint is not -// reachable through the serialized model: it lives on the ModelConfig's own -// Ollama field, so the generic walk over the model never sees it. Without an -// explicit case the allowlist came back empty, which compiles into a deny-all -// Actor egress policy and fails the model call with a 403. -func TestOllamaEgressDestination(t *testing.T) { - for _, host := range []string{"http://host.docker.internal:11434", "host.docker.internal:11434"} { - t.Run(host, func(t *testing.T) { - model := &v1alpha3.ModelConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "ollama", Namespace: "test"}, - Spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "llama3.2", - Ollama: &v1alpha3.OllamaConfig{Host: host}, - }, - } - collections := contextTestCollections(t, model) - result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(context.Background(), - &translator.HarnessInput{ - Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}}}, - Root: &translator.AgentInput{ - Template: &translator.TemplateConfiguration{Name: "pi", Namespace: "test"}, - ResolvedModelConfig: resolvedModel(t, collections, "ollama"), - }}) - require.NoError(t, err) - require.Equal(t, []string{"http://host.docker.internal:11434"}, result.Egress) - }) - } - - t.Run("no host configured", func(t *testing.T) { - model := &v1alpha3.ModelConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "ollama", Namespace: "test"}, - Spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "llama3.2", - Ollama: &v1alpha3.OllamaConfig{}, - }, - } - collections := contextTestCollections(t, model) - result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(context.Background(), - &translator.HarnessInput{ - Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}}}, - Root: &translator.AgentInput{ - Template: &translator.TemplateConfiguration{Name: "pi", Namespace: "test"}, - ResolvedModelConfig: resolvedModel(t, collections, "ollama"), - }}) - require.NoError(t, err) - require.Empty(t, result.Egress) - }) - - // A cloud-tagged model with a key bypasses the operator's host, so the cloud - // host has to be allowed as well or egress policy denies the call. - t.Run("cloud model with a secret allows api.ollama.com", func(t *testing.T) { - model := &v1alpha3.ModelConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "ollama", Namespace: "test"}, - Spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "deepseek-v4-flash:0731-cloud", - APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", - Ollama: &v1alpha3.OllamaConfig{}, - }, - } - collections := contextTestCollections(t, model) - result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(context.Background(), - &translator.HarnessInput{ - Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}}}, - Root: &translator.AgentInput{ - Template: &translator.TemplateConfiguration{Name: "pi", Namespace: "test"}, - ResolvedModelConfig: resolvedModel(t, collections, "ollama"), - }}) - require.NoError(t, err) - require.Contains(t, result.Egress, "https://api.ollama.com:443") - }) - - // A local model must never pick up the cloud host, even with a key present: - // that is the rerouting regression the routing rules exist to prevent. - t.Run("local model with a key does not allow the cloud host", func(t *testing.T) { - model := &v1alpha3.ModelConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "ollama", Namespace: "test"}, - Spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "llama3.2", - APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", - Ollama: &v1alpha3.OllamaConfig{Host: "host.docker.internal:11434"}, - }, - } - collections := contextTestCollections(t, model) - result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(context.Background(), - &translator.HarnessInput{ - Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{Kagent: &v1alpha3.KagentHarness{}}}, - Root: &translator.AgentInput{ - Template: &translator.TemplateConfiguration{Name: "pi", Namespace: "test"}, - ResolvedModelConfig: resolvedModel(t, collections, "ollama"), - }}) - require.NoError(t, err) - require.Equal(t, []string{"http://host.docker.internal:11434"}, result.Egress) - }) -} - func contextTestCollections(t *testing.T, models ...*v1alpha3.ModelConfig) translator.Collections { t.Helper() objects := make([]any, 0, 2*len(models)) @@ -339,7 +220,12 @@ func TestBuildMemory(t *testing.T) { ModelConfigRef: corev1.LocalObjectReference{Name: tc.modelName}, TTLDays: 7, }}}, } - result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(t.Context(), &translator.HarnessInput{Harness: harness, Root: root}) + result, err := NewBuilder(krt.TestingDummyContext{}, collections).Build(t.Context(), &translator.HarnessInput{Harness: harness, Root: root, MemoryModel: func() *translator.ResolvedModelConfig { + if tc.modelName == "embedding" { + return resolvedModel(t, collections, "embedding") + } + return nil + }()}) if tc.wantErr != "" { require.ErrorContains(t, err, tc.wantErr) require.Nil(t, result) @@ -350,7 +236,7 @@ func TestBuildMemory(t *testing.T) { require.Equal(t, "https://embedding.example.com/v1", result.Config.Memory.Embedding.BaseUrl) require.Len(t, result.Models, 2) require.Equal(t, "embedding", result.Models[1].Config.Name) - require.Contains(t, result.Egress, "https://embedding.example.com:443") + require.Len(t, result.Environment, 2) require.Equal(t, "embedding-auth", result.Environment[1].ValueFrom.SecretKeyRef.Name) }) diff --git a/go/core/internal/translator/adkconfig/egress.go b/go/core/internal/translator/adkconfig/egress.go new file mode 100644 index 0000000000..d5274f2df0 --- /dev/null +++ b/go/core/internal/translator/adkconfig/egress.go @@ -0,0 +1,72 @@ +package adkconfig + +import ( + "net/url" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/translator" +) + +// AdditionalModelDestinations preserves ADK grants beyond the effective model endpoint. +// Retained defaults and metadata origins do not create credential targets. +func AdditionalModelDestinations(resolved *translator.ResolvedModelConfig) []string { + spec := resolved.Config.Spec + var destinations []string + switch spec.Provider { + case v1alpha3.ModelProviderOpenAI: + destinations = append(destinations, "https://api.openai.com:443") + case v1alpha3.ModelProviderAnthropic: + destinations = append(destinations, "https://api.anthropic.com:443") + } + // The ADK config walk also allowed URL-valued metadata. Preserve those grants + // without rendering a client or scanning inactive provider configurations. + modelName := spec.Model + if spec.Provider == v1alpha3.ModelProviderAzureOpenAI && spec.AzureOpenAI != nil { + modelName = spec.AzureOpenAI.DeploymentName + } + destinations = appendModelMetadataOrigin(destinations, modelName) + for _, value := range spec.DefaultHeaders { + destinations = appendModelMetadataOrigin(destinations, value) + } + var values []string + switch spec.Provider { + case v1alpha3.ModelProviderAzureOpenAI: + if spec.AzureOpenAI != nil { + values = []string{spec.AzureOpenAI.APIVersion} + } + case v1alpha3.ModelProviderOllama: + if spec.Ollama != nil { + for _, value := range spec.Ollama.Options { + values = append(values, value) + } + } + case v1alpha3.ModelProviderBedrock: + if spec.Bedrock != nil { + values = []string{spec.Bedrock.Region, spec.Bedrock.CacheTTL} + if guardrail := spec.Bedrock.Guardrail; guardrail != nil { + values = append(values, guardrail.Identifier, guardrail.Version, guardrail.Trace) + } + } + case v1alpha3.ModelProviderSAPAICore: + if spec.SAPAICore != nil { + values = []string{spec.SAPAICore.ResourceGroup} + } + case v1alpha3.ModelProviderFoundry: + if spec.Foundry != nil { + values = []string{spec.Foundry.Deployment, spec.Foundry.APIVersion} + } + } + for _, value := range values { + destinations = appendModelMetadataOrigin(destinations, value) + } + return destinations +} + +func appendModelMetadataOrigin(destinations []string, value string) []string { + parsed, err := url.Parse(value) + if err == nil && parsed.Hostname() != "" { + return append(destinations, egress.Origin(parsed)) + } + return destinations +} diff --git a/go/core/internal/translator/adkconfig/egress_test.go b/go/core/internal/translator/adkconfig/egress_test.go new file mode 100644 index 0000000000..a355aabf91 --- /dev/null +++ b/go/core/internal/translator/adkconfig/egress_test.go @@ -0,0 +1,33 @@ +package adkconfig + +import ( + "slices" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/translator" +) + +func TestRetainedADKMetadataDestinations(t *testing.T) { + for _, test := range []struct { + name string + spec v1alpha3.ModelConfigSpec + want []string + }{ + {"model and headers", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, Model: "https://model.example.com/name", DefaultHeaders: map[string]string{"HTTP-Referer": "https://app.example.com/page"}, Ollama: &v1alpha3.OllamaConfig{Options: map[string]string{"inactive": "https://unused.example.com"}}}, []string{"https://api.openai.com:443", "https://app.example.com:443", "https://model.example.com:443"}}, + {"Azure deployment replaces model name", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAzureOpenAI, Model: "https://unused.example.com", AzureOpenAI: &v1alpha3.AzureOpenAIConfig{DeploymentName: "https://deployment.example.com", APIVersion: "https://version.example.com"}}, []string{"https://deployment.example.com:443", "https://version.example.com:443"}}, + {"Ollama options", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Ollama: &v1alpha3.OllamaConfig{Options: map[string]string{"url": "https://options.example.com/path"}}}, []string{"https://options.example.com:443"}}, + {"Bedrock guardrail", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderBedrock, Bedrock: &v1alpha3.BedrockConfig{Region: "us-east-1", Guardrail: &v1alpha3.BedrockGuardrailConfig{Identifier: "https://guardrail.example.com/id", Version: "DRAFT"}}}, []string{"https://guardrail.example.com:443"}}, + {"SAP resource group", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderSAPAICore, SAPAICore: &v1alpha3.SAPAICoreConfig{ResourceGroup: "https://group.example.com"}}, []string{"https://group.example.com:443"}}, + {"Foundry metadata", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderFoundry, Foundry: &v1alpha3.FoundryConfig{Deployment: "https://deployment.example.com", APIVersion: "https://version.example.com"}}, []string{"https://deployment.example.com:443", "https://version.example.com:443"}}, + } { + t.Run(test.name, func(t *testing.T) { + model := &translator.ResolvedModelConfig{Config: &v1alpha3.ModelConfig{Spec: test.spec}} + got := AdditionalModelDestinations(model) + slices.Sort(got) + require.Equal(t, test.want, got) + }) + } +} diff --git a/go/core/internal/translator/adkconfig/mcp_test.go b/go/core/internal/translator/adkconfig/mcp_test.go index b7839cce56..da7e633ff7 100644 --- a/go/core/internal/translator/adkconfig/mcp_test.go +++ b/go/core/internal/translator/adkconfig/mcp_test.go @@ -4,9 +4,13 @@ import ( "testing" "github.com/stretchr/testify/require" + "istio.io/istio/pkg/kube/krt" + "k8s.io/apimachinery/pkg/types" configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/translator" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) func TestAddRemoteMCPServerPreservesBindingApproval(t *testing.T) { @@ -52,3 +56,32 @@ func TestAddRemoteMCPServerPreservesBindingApproval(t *testing.T) { }) } } + +// The extension runs before Build, including shared-agent and SSE clients. +type mcpPolicyTranslator struct{} + +var _ egressext.PolicyTranslator = mcpPolicyTranslator{} + +func (mcpPolicyTranslator) Translate(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://tools.example.com:443", Header: "authorization", URI: "ate-secret://provider.example/default/token"}) + return draft, nil +} + +func TestBuildResolvedCredentialHeaders(t *testing.T) { + server := func(path string, protocol v1alpha3.RemoteMCPServerProtocol) translator.ResolvedMCPTool { + return translator.ResolvedMCPTool{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://tools.example.com/" + path, Protocol: protocol}}} + } + input := &translator.HarnessInput{ + Harness: &translator.HarnessConfiguration{Spec: v1alpha3.HarnessSpec{BYO: &v1alpha3.BYOHarness{}}}, + Root: &translator.AgentInput{Template: &translator.TemplateConfiguration{}, MCPTools: []translator.ResolvedMCPTool{server("http", v1alpha3.RemoteMCPServerProtocolStreamableHttp)}, Shared: []translator.AgentInputBinding{ + {Name: "child", Agent: &translator.AgentInput{Template: &translator.TemplateConfiguration{}, MCPTools: []translator.ResolvedMCPTool{server("sse", v1alpha3.RemoteMCPServerProtocolSse)}}}, + }}, + } + var err error + input.Policy, err = translator.ResolvePolicy(input, nil, mcpPolicyTranslator{}) + require.NoError(t, err) + result, err := NewBuilder(krt.TestingDummyContext{}, translator.Collections{}).Build(t.Context(), input) + require.NoError(t, err) + require.Equal(t, translator.CredentialPlaceholder, result.Config.HttpTools[0].Params.Headers["authorization"]) + require.Equal(t, translator.CredentialPlaceholder, result.Config.SubAgents[0].SseTools[0].Params.Headers["authorization"]) +} diff --git a/go/core/internal/translator/adkconfig/model.go b/go/core/internal/translator/adkconfig/model.go index 1ede7b2b68..137ba54f2e 100644 --- a/go/core/internal/translator/adkconfig/model.go +++ b/go/core/internal/translator/adkconfig/model.go @@ -326,7 +326,7 @@ func translateModel(resolved *translator.ResolvedModelConfig) (*configv1.ADKMode if model.Spec.Ollama.Host != "" { modelDeploymentData.EnvVars = append(modelDeploymentData.EnvVars, corev1.EnvVar{ Name: env.OllamaAPIBase.Name(), - Value: withDefaultScheme(model.Spec.Ollama.Host), + Value: translator.OllamaURL(model.Spec.Ollama.Host), }) } // Bind the Secret only when the model reaches api.ollama.com; the diff --git a/go/core/internal/translator/adkconfig/model_test.go b/go/core/internal/translator/adkconfig/model_test.go index 41049692c7..29e2c6541e 100644 --- a/go/core/internal/translator/adkconfig/model_test.go +++ b/go/core/internal/translator/adkconfig/model_test.go @@ -139,7 +139,7 @@ func TestTranslateOllamaEnvironment(t *testing.T) { // The chart ships a default host, so this is the ordinary case for // an operator who follows the values.yaml comment introducing // apiKeySecretRef. The request goes to that host, so no credential - // binding exists, and mounting the key made CompileCredentials + // binding exists, and mounting the key made credential rendering // reject the whole revision ("cannot use gateway header injection") // instead of simply using the daemon. name: "an explicit host does not mount the key even for a cloud model", @@ -232,7 +232,7 @@ func TestWithDefaultScheme(t *testing.T) { } for _, tt := range tests { t.Run(tt.host, func(t *testing.T) { - require.Equal(t, tt.want, withDefaultScheme(tt.host)) + require.Equal(t, tt.want, translator.OllamaURL(tt.host)) }) } } diff --git a/go/core/internal/translator/adkconfig/runtime.go b/go/core/internal/translator/adkconfig/runtime.go index 82a9f133e4..975b1b4e83 100644 --- a/go/core/internal/translator/adkconfig/runtime.go +++ b/go/core/internal/translator/adkconfig/runtime.go @@ -4,9 +4,7 @@ import ( "encoding/json" "fmt" - "istio.io/istio/pkg/kube/krt" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" runtimeconfig "github.com/kagent-dev/kagent/go/adk/pkg/config" adkoutputschema "github.com/kagent-dev/kagent/go/adk/pkg/outputschema" @@ -19,20 +17,12 @@ type modelResult struct { Resolved *translator.ResolvedModelConfig Model *configv1.ADKModelConfig Environment []corev1.EnvVar - Egress []string } // buildModel translates a standalone ModelConfig without building an agent. -func (c *Builder) buildModel(namespace, name string) (*modelResult, error) { - resolved := krt.FetchOne(c.ctx, c.collections.ResolvedModelConfigs, krt.FilterObjectName(types.NamespacedName{Namespace: namespace, Name: name})) +func buildModel(resolved *translator.ResolvedModelConfig) (*modelResult, error) { if resolved == nil { - return nil, fmt.Errorf("model config %q not found", name) - } - if failures := resolved.SemanticFailures; len(failures) > 0 { - return nil, translator.NewValidationError("ModelConfig %q: %s", name, failures[0].Message) - } - if failures := resolved.ReferenceFailures; len(failures) > 0 { - return nil, fmt.Errorf("ModelConfig %q: %s", name, failures[0].Message) + return nil, fmt.Errorf("auxiliary model must be resolved before rendering") } runtime, err := resolveModel(resolved) if err != nil { @@ -43,7 +33,6 @@ func (c *Builder) buildModel(namespace, name string) (*modelResult, error) { } return &modelResult{ Resolved: resolved, Model: runtime.Model, Environment: runtime.Environment, - Egress: agentConfigDestinations(&configv1.ADKAgentConfig{}, resolved.Config, runtime.Model), }, nil } @@ -57,7 +46,7 @@ func (c *Builder) buildModel(namespace, name string) (*modelResult, error) { // and egress join the revision, and it joins the provenance so a change to it // compiles a new revision. The agent's own model is left out because the // runtime already summarizes with it by default. -func (c *Builder) applyCompaction(result *Result, harness *translator.HarnessConfiguration, template *translator.TemplateConfiguration) error { +func applyCompaction(result *Result, harness *translator.HarnessConfiguration, template *translator.TemplateConfiguration, resolved *translator.ResolvedModelConfig) error { spec := harness.Spec.Kagent.Compaction if spec == nil { return nil @@ -71,14 +60,13 @@ func (c *Builder) applyCompaction(result *Result, harness *translator.HarnessCon if summarizer := spec.Summarizer; summarizer != nil { compaction.PromptTemplate = summarizer.PromptTemplate if ref := summarizer.ModelConfigRef; ref != nil && !isAgentModel(template, ref.Name) { - model, err := c.buildModel(harness.Namespace, ref.Name) + model, err := buildModel(resolved) if err != nil { return fmt.Errorf("resolve summarizer ModelConfig %q: %w", ref.Name, err) } compaction.SummarizerModel = model.Model result.Models = append(result.Models, model.Resolved) result.Environment = append(result.Environment, model.Environment...) - result.Egress = append(result.Egress, model.Egress...) } } result.Config.ContextConfig = &configv1.ADKContextConfig{Compaction: compaction} @@ -119,19 +107,18 @@ func requireModels(input *translator.AgentInput) error { return nil } -func (c *Builder) applyMemory(result *Result, harness *translator.HarnessConfiguration) error { +func applyMemory(result *Result, harness *translator.HarnessConfiguration, resolved *translator.ResolvedModelConfig) error { memory := harness.Spec.Kagent.Memory if memory == nil { return nil } name := memory.ModelConfigRef.Name - model, err := c.buildModel(harness.Namespace, name) + model, err := buildModel(resolved) if err != nil { return fmt.Errorf("resolve memory ModelConfig %q: %w", name, err) } result.Config.Memory = &configv1.ADKMemoryConfig{TtlDays: int64(memory.TTLDays), Embedding: runtimeconfig.ModelToEmbeddingConfig(model.Model)} result.Models = append(result.Models, model.Resolved) result.Environment = append(result.Environment, model.Environment...) - result.Egress = append(result.Egress, model.Egress...) return nil } diff --git a/go/core/internal/translator/byo/compiler.go b/go/core/internal/translator/byo/compiler.go index c8b173811f..46e73dad89 100644 --- a/go/core/internal/translator/byo/compiler.go +++ b/go/core/internal/translator/byo/compiler.go @@ -38,17 +38,6 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) return nil, err } template, harness := input.Root.Template, input.Harness - configJSON, err := runtimeconfig.Marshal(&configv1.ADKConfig{ - Runtime: translator.RuntimeConfig(c.settings, input.AgentName, template.Namespace), - Agent: compiled.Config, - }) - if err != nil { - var invalid *protovalidate.ValidationError - if errors.As(err, &invalid) { - return nil, translator.NewValidationError("invalid agent config: %w", err) - } - return nil, fmt.Errorf("marshal agent config: %w", err) - } card, err := pbconv.ToProtoAgentCard(agentTemplateCard(input.AgentName, template)) if err != nil { return nil, fmt.Errorf("convert agent card: %w", err) @@ -70,10 +59,22 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build revision provenance: %w", err) } - environment, credentials, err := translator.CompileCredentials(input, compiled.Models, environment) + environment, err = input.Policy.RenderEnvironment(environment) if err != nil { return nil, err } + + configJSON, err := runtimeconfig.Marshal(&configv1.ADKConfig{ + Runtime: translator.RuntimeConfig(c.settings, input.AgentName, template.Namespace), + Agent: compiled.Config, + }) + if err != nil { + var invalid *protovalidate.ValidationError + if errors.As(err, &invalid) { + return nil, translator.NewValidationError("invalid agent config: %w", err) + } + return nil, fmt.Errorf("marshal agent config: %w", err) + } compiled.Egress = append(compiled.Egress, telemetryConfig.Destinations(environment)...) compiled.Egress = append(compiled.Egress, c.settings.GetControlPlaneUrl()) slices.Sort(compiled.Egress) @@ -84,7 +85,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) Environment: environment, ConfigJSON: configJSON, AgentCard: card, SkillImages: compiled.SkillImages, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - Credentials: credentials, Provenance: provenance, EgressDestinations: slices.Compact(compiled.Egress), + Provenance: provenance, EgressDestinations: slices.Compact(compiled.Egress), }}, nil } @@ -96,3 +97,8 @@ func agentTemplateCard(agentName string, template *translator.TemplateConfigurat DefaultInputModes: []string{"text"}, DefaultOutputModes: []string{"text"}, } } + +// ModelRequirements declares authentication and grants before client rendering. +func (*Compiler) ModelRequirements(model *translator.ResolvedModelConfig) translator.ModelRequirements { + return translator.ModelRequirements{Destinations: adkconfig.AdditionalModelDestinations(model)} +} diff --git a/go/core/internal/translator/claude/compiler.go b/go/core/internal/translator/claude/compiler.go index 59714932a7..eb60aa5312 100644 --- a/go/core/internal/translator/claude/compiler.go +++ b/go/core/internal/translator/claude/compiler.go @@ -19,7 +19,6 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" claudeconfig "github.com/kagent-dev/kagent/go/harness/claude/config" "github.com/kagent-dev/kagent/go/pkg/tracing" @@ -56,7 +55,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) runtimeTelemetry := telemetryConfig.RuntimeTelemetry( tracing.RuntimeClaude, input.AgentName, template.Namespace, model.Spec) - providerEnvironment, egress, err := c.provider(ctx, model) + providerEnvironment, err := c.provider(ctx, model, input.Policy.ModelUsesGateway(input.Root.ResolvedModelConfig)) if err != nil { return nil, err } @@ -64,7 +63,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, err } - mcp, err := c.compileMCP(ctx, input.Root.Template.Namespace, input.Root.MCPTools) + mcp, err := c.compileMCP(ctx, input.Root.Template.Namespace, input.Root.MCPTools, input.Policy) if err != nil { return nil, err } @@ -95,13 +94,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) config.SkillResources = &skills.Resources } config.MCPServers = mcp.servers - if err := config.Validate(); err != nil { - return nil, translator.NewValidationError("invalid compiled Claude configuration: %v", err) - } - configJSON, err := json.Marshal(config) - if err != nil { - return nil, fmt.Errorf("marshal Claude config: %w", err) - } + card, err := pbconv.ToProtoAgentCard(translator.ManagedAgentCard(input.AgentName, input.Root.Template)) if err != nil { return nil, fmt.Errorf("convert Claude agent card: %w", err) @@ -110,13 +103,20 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build Claude revision provenance: %w", err) } - environment, credentials, err := translator.CompileCredentials(input, nil, environment) + environment, err = input.Policy.RenderEnvironment(environment) if err != nil { return nil, err } + if err := config.Validate(); err != nil { + return nil, translator.NewValidationError("invalid compiled Claude configuration: %v", err) + } - egress = append(egress, skills.Egress...) - egress = append(egress, mcp.egress...) + configJSON, err := json.Marshal(config) + if err != nil { + return nil, fmt.Errorf("marshal Claude config: %w", err) + } + + egress := skills.Egress egress = append(egress, telemetryConfig.Destinations(environment)...) egress = append(egress, c.settings.GetControlPlaneUrl()) slices.Sort(egress) @@ -129,7 +129,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) ConfigJSON: configJSON, AgentCard: card, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - Credentials: credentials, Provenance: provenance, EgressDestinations: egress, + Provenance: provenance, EgressDestinations: egress, }, Warnings: mcp.warnings, }, nil @@ -175,7 +175,7 @@ func sameProviderConfiguration(root, child v1alpha3.ModelConfigSpec) bool { return reflect.DeepEqual(root, child) } -func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig) ([]corev1.EnvVar, []string, error) { +func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig, gateway bool) ([]corev1.EnvVar, error) { switch model.Spec.Provider { case v1alpha3.ModelProviderAnthropic: var baseURL string @@ -188,27 +188,28 @@ func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig) ([ options.CacheTTL = "" } if !reflect.DeepEqual(options, v1alpha3.AnthropicConfig{}) { - return nil, nil, translator.NewValidationError("Claude does not support Anthropic provider options beyond baseUrl yet") + return nil, translator.NewValidationError("Claude does not support Anthropic provider options beyond baseUrl yet") } } - if err := c.requireSecretKey(ctx, model, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey, false); err != nil { - return nil, nil, err + environment := []corev1.EnvVar{{Name: claudeconfig.AnthropicAPIKeyEnvName, Value: translator.CredentialPlaceholder}} + if model.Spec.APIKeySecret != "" || !gateway { + if err := c.requireSecretKey(ctx, model, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey, false); err != nil { + return nil, err + } + environment[0] = secretEnvironment(claudeconfig.AnthropicAPIKeyEnvName, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey) } - environment := []corev1.EnvVar{secretEnvironment(claudeconfig.AnthropicAPIKeyEnvName, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey)} - egress := []string{"https://api.anthropic.com:443"} if baseURL != "" { - hostname, err := anthropicBaseURLOrigin(baseURL) + err := validateAnthropicBaseURL(baseURL) if err != nil { - return nil, nil, err + return nil, err } environment = append(environment, corev1.EnvVar{Name: claudeconfig.AnthropicBaseURLEnvName, Value: baseURL}) - egress = []string{hostname} } - return environment, egress, nil + return environment, nil case v1alpha3.ModelProviderBedrock: if model.Spec.Bedrock == nil || strings.TrimSpace(model.Spec.Bedrock.Region) == "" { - return nil, nil, translator.NewValidationError("Claude Bedrock requires bedrock.region") + return nil, translator.NewValidationError("Claude Bedrock requires bedrock.region") } options := *model.Spec.Bedrock options.Region = "" @@ -217,17 +218,24 @@ func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig) ([ options.CacheTTL = "" } if !reflect.DeepEqual(options, v1alpha3.BedrockConfig{}) { - return nil, nil, translator.NewValidationError("Claude does not support Bedrock provider options beyond region yet") + return nil, translator.NewValidationError("Claude does not support Bedrock provider options beyond region yet") + } + if model.Spec.APIKeySecret == "" && gateway { + return []corev1.EnvVar{ + {Name: claudeconfig.UseBedrockEnvName, Value: "1"}, + {Name: claudeconfig.AWSRegionEnvName, Value: model.Spec.Bedrock.Region}, + {Name: claudeconfig.AWSBedrockTokenEnvName, Value: translator.CredentialPlaceholder}, + }, nil } if model.Spec.APIKeySecret == "" { - return nil, nil, translator.NewValidationError("Claude Bedrock requires apiKeySecret with AWS credentials") + return nil, translator.NewValidationError("Claude Bedrock requires apiKeySecret with AWS credentials") } if model.Spec.APIKeySecretKey != "" { - return nil, nil, translator.NewValidationError("Claude Bedrock reads standard AWS keys from apiKeySecret; apiKeySecretKey must be empty") + return nil, translator.NewValidationError("Claude Bedrock reads standard AWS keys from apiKeySecret; apiKeySecretKey must be empty") } secret, err := c.secret(ctx, model.Namespace, model.Spec.APIKeySecret) if err != nil { - return nil, nil, err + return nil, err } environment := []corev1.EnvVar{{Name: claudeconfig.UseBedrockEnvName, Value: "1"}, {Name: claudeconfig.AWSRegionEnvName, Value: model.Spec.Bedrock.Region}} if value := secret.Data[claudeconfig.AWSBedrockTokenEnvName]; len(value) != 0 { @@ -235,7 +243,7 @@ func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig) ([ } else { for _, key := range []string{claudeconfig.AWSAccessKeyEnvName, claudeconfig.AWSSecretKeyEnvName} { if len(secret.Data[key]) == 0 { - return nil, nil, translator.NewValidationError("Claude Bedrock Secret %q requires %s and %s, or %s", secret.Name, claudeconfig.AWSAccessKeyEnvName, claudeconfig.AWSSecretKeyEnvName, claudeconfig.AWSBedrockTokenEnvName) + return nil, translator.NewValidationError("Claude Bedrock Secret %q requires %s and %s, or %s", secret.Name, claudeconfig.AWSAccessKeyEnvName, claudeconfig.AWSSecretKeyEnvName, claudeconfig.AWSBedrockTokenEnvName) } environment = append(environment, secretEnvironment(key, secret.Name, key)) } @@ -243,36 +251,36 @@ func (c *Compiler) provider(ctx context.Context, model *v1alpha3.ModelConfig) ([ environment = append(environment, secretEnvironment(claudeconfig.AWSSessionTokenEnvName, secret.Name, claudeconfig.AWSSessionTokenEnvName)) } } - return environment, []string{"https://bedrock-runtime." + model.Spec.Bedrock.Region + ".amazonaws.com:443"}, nil + return environment, nil case v1alpha3.ModelProviderAnthropicVertexAI: if model.Spec.AnthropicVertexAI == nil || strings.TrimSpace(model.Spec.AnthropicVertexAI.ProjectID) == "" || strings.TrimSpace(model.Spec.AnthropicVertexAI.Location) == "" { - return nil, nil, translator.NewValidationError("Claude Vertex requires anthropicVertexAI.projectID and location") + return nil, translator.NewValidationError("Claude Vertex requires anthropicVertexAI.projectID and location") } options := *model.Spec.AnthropicVertexAI options.ProjectID, options.Location = "", "" if !reflect.DeepEqual(options, v1alpha3.AnthropicVertexAIConfig{}) { - return nil, nil, translator.NewValidationError("Claude does not support AnthropicVertexAI provider options beyond projectID and location yet") + return nil, translator.NewValidationError("Claude does not support AnthropicVertexAI provider options beyond projectID and location yet") } if err := c.requireGoogleCredentials(ctx, model); err != nil { - return nil, nil, err + return nil, err } cfg := model.Spec.AnthropicVertexAI return []corev1.EnvVar{ {Name: claudeconfig.UseVertexEnvName, Value: "1"}, {Name: claudeconfig.VertexProjectEnvName, Value: cfg.ProjectID}, {Name: claudeconfig.VertexRegionEnvName, Value: cfg.Location}, secretEnvironment(claudeconfig.GoogleCredentialsJSONEnvName, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey), - }, []string{"https://" + vertexHostname(cfg.Location) + ":443", "https://oauth2.googleapis.com:443"}, nil + }, nil default: - return nil, nil, translator.NewValidationError("Claude does not support ModelConfig provider %q", model.Spec.Provider) + return nil, translator.NewValidationError("Claude does not support ModelConfig provider %q", model.Spec.Provider) } } -func anthropicBaseURLOrigin(raw string) (string, error) { +func validateAnthropicBaseURL(raw string) error { parsed, err := url.Parse(raw) if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { - return "", translator.NewValidationError("Claude Anthropic baseUrl must be an absolute HTTP(S) URL without credentials, query, or fragment") + return translator.NewValidationError("Claude Anthropic baseUrl must be an absolute HTTP(S) URL without credentials, query, or fragment") } - return egress.Origin(parsed), nil + return nil } func (c *Compiler) requireSecretKey(ctx context.Context, model *v1alpha3.ModelConfig, name, key string, requireJSON bool) error { @@ -338,17 +346,6 @@ func secretEnvironment(environmentName, secretName, key string) corev1.EnvVar { }}} } -func vertexHostname(location string) string { - switch location { - case "global": - return "aiplatform.googleapis.com" - case "us", "eu": - return "aiplatform." + location + ".rep.googleapis.com" - default: - return location + "-aiplatform.googleapis.com" - } -} - type provenanceEntry struct { APIVersion string `json:"apiVersion"` Kind string `json:"kind"` @@ -441,3 +438,13 @@ func objectProvenance(apiVersion, kind, name string, uid types.UID, generation i } var _ translator.HarnessCompiler = (*Compiler)(nil) + +// ModelRequirements declares authentication and grants before client rendering. +func (*Compiler) ModelRequirements(model *translator.ResolvedModelConfig) translator.ModelRequirements { + switch model.Config.Spec.Provider { + case v1alpha3.ModelProviderAnthropic, v1alpha3.ModelProviderBedrock: + return translator.ModelRequirements{CredentialRequired: true} + default: + return translator.ModelRequirements{} + } +} diff --git a/go/core/internal/translator/claude/compiler_test.go b/go/core/internal/translator/claude/compiler_test.go index d67f2604e0..6a9989c026 100644 --- a/go/core/internal/translator/claude/compiler_test.go +++ b/go/core/internal/translator/claude/compiler_test.go @@ -32,7 +32,6 @@ func TestCompileProviderCredentials(t *testing.T) { model v1alpha3.ModelConfigSpec secretData map[string][]byte wantEnv map[string]string - wantEgress []string wantErr string }{ { @@ -41,7 +40,6 @@ func TestCompileProviderCredentials(t *testing.T) { APIKeySecret: "model-auth", APIKeySecretKey: "api-key"}, secretData: map[string][]byte{"api-key": []byte(credentialValue)}, wantEnv: map[string]string{claudeconfig.AnthropicAPIKeyEnvName: translator.CredentialPlaceholder}, - wantEgress: []string{"http://kagent-controller.kagent:8083", "https://api.anthropic.com:443"}, }, { name: "Anthropic gateway", @@ -52,7 +50,6 @@ func TestCompileProviderCredentials(t *testing.T) { secretData: map[string][]byte{"api-key": []byte(credentialValue)}, wantEnv: map[string]string{claudeconfig.AnthropicAPIKeyEnvName: translator.CredentialPlaceholder, claudeconfig.AnthropicBaseURLEnvName: "http://host.docker.internal:8090/anthropic"}, - wantEgress: []string{"http://host.docker.internal:8090", "http://kagent-controller.kagent:8083"}, }, { name: "Bedrock IAM", @@ -67,7 +64,6 @@ func TestCompileProviderCredentials(t *testing.T) { APIKeySecret: "model-auth", Bedrock: &v1alpha3.BedrockConfig{Region: "us-west-2"}}, secretData: map[string][]byte{claudeconfig.AWSBedrockTokenEnvName: []byte(credentialValue)}, wantEnv: map[string]string{claudeconfig.UseBedrockEnvName: "1", claudeconfig.AWSRegionEnvName: "us-west-2", claudeconfig.AWSBedrockTokenEnvName: translator.CredentialPlaceholder}, - wantEgress: []string{"http://kagent-controller.kagent:8083", "https://bedrock-runtime.us-west-2.amazonaws.com:443"}, }, { name: "Anthropic Vertex AI", @@ -82,7 +78,7 @@ func TestCompileProviderCredentials(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { input, reader := testInput(t, tt.model, tt.secretData) - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if tt.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tt.wantErr) { t.Fatalf("expected unsupported credential error, got %v", err) @@ -114,7 +110,7 @@ func TestCompileProviderCredentials(t *testing.T) { if gotEnvironment[claudeconfig.SandboxEnvName] != "1" { t.Errorf("environment[%s] = %q, want %q", claudeconfig.SandboxEnvName, gotEnvironment[claudeconfig.SandboxEnvName], "1") } - if !reflect.DeepEqual(revision.EgressDestinations, tt.wantEgress) { + if !reflect.DeepEqual(revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083"}) { t.Errorf("egress = %v", revision.EgressDestinations) } if bytes.Contains(revision.ConfigJSON, []byte(credentialValue)) || bytes.Contains(revision.Provenance, []byte(credentialValue)) { @@ -124,7 +120,7 @@ func TestCompileProviderCredentials(t *testing.T) { t.Fatalf("provenance contains gateway-managed Secret: %s", revision.Provenance) } - again, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + again, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil || !reflect.DeepEqual(revision, again) { t.Fatalf("compilation is not deterministic: %v", err) } @@ -151,7 +147,7 @@ func TestCompileRejectsUnsupportedConfiguration(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { input, reader := testInput(t, tt.model, map[string][]byte{"api-key": []byte("secret"), claudeconfig.AWSAccessKeyEnvName: []byte("access"), claudeconfig.AWSSecretKeyEnvName: []byte("secret"), "credentials.json": []byte(`{"type":"service_account"}`)}) - _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) var validation *translator.ValidationError if !errors.As(err, &validation) { t.Fatalf("Compile() error = %v, want validation error", err) @@ -168,7 +164,7 @@ func TestCompileRejectsProviderOwnedHarnessEnvironment(t *testing.T) { input, reader := testInput(t, model, map[string][]byte{"api-key": []byte("secret")}) value := "http://mock.example.com" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: claudeconfig.AnthropicBaseURLEnvName, Value: value}} - _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) var validation *translator.ValidationError if !errors.As(err, &validation) { t.Fatalf("Compile() error = %v, want validation error", err) @@ -184,7 +180,7 @@ func TestCompileAllowsRuntimeOTELEndpoint(t *testing.T) { value := "http://other-collector:4317" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: value}} - result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + result, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -202,7 +198,7 @@ func TestCompileAllowsRuntimeTelemetryOverrides(t *testing.T) { value := "department=engineering" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OTEL_RESOURCE_ATTRIBUTES", Value: value}} - result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + result, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -227,7 +223,7 @@ func TestCompileRootSkillsAndPluginSelections(t *testing.T) { Skills: []string{"deploy"}, }} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -239,13 +235,13 @@ func TestCompileRootSkillsAndPluginSelections(t *testing.T) { len(cfg.SkillResources.Plugins) != 1 || !reflect.DeepEqual(cfg.SkillResources.Plugins[0].Skills, []string{"deploy"}) { t.Fatalf("compiled skills = %#v", cfg.SkillResources) } - wantEgress := []string{"http://kagent-controller.kagent:8083", "https://api.anthropic.com:443", "https://git.example.com:443"} + wantEgress := []string{"http://kagent-controller.kagent:8083", "https://git.example.com:443"} if !reflect.DeepEqual(revision.EgressDestinations, wantEgress) { t.Fatalf("egress = %v, want %v", revision.EgressDestinations, wantEgress) } input.Root.Template.Spec.Plugins[0].Skills = []string{"review"} - if _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input); err == nil || !strings.Contains(err.Error(), "duplicate skill name") { + if _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input); err == nil || !strings.Contains(err.Error(), "duplicate skill name") { t.Fatalf("duplicate skill Compile() error = %v", err) } } @@ -279,7 +275,7 @@ func TestCompileDirectWholeServerMCP(t *testing.T) { }}} input.Root.MCPTools = []translator.ResolvedMCPTool{{Binding: *input.Root.Template.Spec.Tools[0].MCP.DeepCopy(), Server: server}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -307,7 +303,7 @@ func TestCompileDirectWholeServerMCP(t *testing.T) { if !foundSecret { t.Fatalf("MCP credential environment missing: %#v", revision.Environment) } - if !reflect.DeepEqual(revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083", "https://api.anthropic.com:443", "https://mcp.example.com:443"}) { + if !reflect.DeepEqual(revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083"}) { t.Fatalf("egress = %v", revision.EgressDestinations) } if !bytes.Contains(revision.Provenance, []byte(`"kind":"RemoteMCPServer"`)) { @@ -330,7 +326,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { } binding := v1alpha3.MCPToolBinding{Server: corev1.TypedLocalObjectReference{Kind: "RemoteMCPServer", Name: server.Name}} input.Root.MCPTools = []translator.ResolvedMCPTool{{Binding: binding, Server: server}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatalf("omitted selection Compile() error = %v", err) } @@ -339,7 +335,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { } input.Root.MCPTools[0].Binding.Tools = []string{"one"} - revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err = compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatalf("partial selection Compile() error = %v", err) } @@ -348,7 +344,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { } server.Status.ObservedGeneration = 0 - revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err = compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatalf("stale discovery Compile() error = %v", err) } @@ -362,7 +358,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { server.Spec.TLS = &v1alpha3.TLSConfig{DisableVerify: true} server.Spec.Timeout = &metav1.Duration{Duration: time.Minute} server.Spec.TerminateOnClose = &terminateOnClose - revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err = compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatalf("unsupported MCP options Compile() error = %v", err) } @@ -376,7 +372,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { } server.Spec.Protocol = v1alpha3.RemoteMCPServerProtocol("STDIO") - if _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input); err == nil || !strings.Contains(err.Error(), "unsupported protocol") { + if _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input); err == nil || !strings.Contains(err.Error(), "unsupported protocol") { t.Fatalf("unsupported MCP protocol Compile() error = %v", err) } @@ -386,7 +382,7 @@ func TestCompileWholeServerMCPSelectionWarnings(t *testing.T) { server.Spec.TerminateOnClose = nil input.Root.MCPTools[0].Binding.Tools = []string{"one"} input.Root.MCPTools[0].Binding.RequireApproval = true - revision, err = NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err = compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatalf("approval-required MCP binding Compile() error = %v", err) } @@ -433,7 +429,7 @@ func TestCompileLocalSharedAgent(t *testing.T) { Name: "specialist", Description: "Handles specialist requests", Agent: child, }} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -466,7 +462,7 @@ func TestCompileRejectsUnsupportedLocalAgentConfiguration(t *testing.T) { }{ {name: "provider configuration", mutate: func(binding *translator.AgentInputBinding) { binding.Agent.ResolvedModelConfig.Config.Spec.APIKeySecret = "different-auth" - }, want: "root agent's provider"}, + }, want: "conflicting credentials"}, {name: "nested tools", mutate: func(binding *translator.AgentInputBinding) { binding.Agent.Template.Spec.Tools = []v1alpha3.ToolBinding{{MCP: &v1alpha3.MCPToolBinding{}}} }, want: "cannot contain MCP or nested agent tools"}, @@ -495,7 +491,7 @@ func TestCompileRejectsUnsupportedLocalAgentConfiguration(t *testing.T) { } tt.mutate(&binding) input.Root.Shared = []translator.AgentInputBinding{binding} - _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err == nil || !strings.Contains(err.Error(), tt.want) { t.Fatalf("Compile() error = %v, want containing %q", err, tt.want) } @@ -531,7 +527,7 @@ func TestCompileRuntimeTelemetry(t *testing.T) { input.Harness.Name = "fast" settings := testRuntimeDefaults() - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, settings).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, settings), input) if err != nil { t.Fatal(err) } @@ -552,7 +548,7 @@ func TestCompileRuntimeTelemetry(t *testing.T) { } settings.Telemetry = &configv1.TelemetryConfig{CaptureContent: true, MaxCaptureBytes: new(int32(4096))} - captured, err := NewCompiler(krt.TestingDummyContext{}, reader, settings).Compile(context.Background(), input) + captured, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, settings), input) if err != nil { t.Fatal(err) } @@ -604,7 +600,7 @@ func TestCompiledTelemetryFitsTheActorEnvironmentBudget(t *testing.T) { }}} input.Root.MCPTools = []translator.ResolvedMCPTool{{Binding: *input.Root.Template.Spec.Tools[0].MCP.DeepCopy(), Server: server}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, settings).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, settings), input) if err != nil { t.Fatal(err) } @@ -622,3 +618,31 @@ func TestCompiledTelemetryFitsTheActorEnvironmentBudget(t *testing.T) { func testRuntimeDefaults() *configv1.RuntimeDefaultsConfig { return &configv1.RuntimeDefaultsConfig{ControlPlaneUrl: new("http://kagent-controller.kagent:8083"), GatewayUrl: new("http://kagent-controller.kagent:8083"), Logging: &configv1.LoggingConfig{Level: new("info")}} } + +func compileTestInput(ctx context.Context, compiler *Compiler, input *translator.HarnessInput) (*translator.CompileResult, error) { + var resolve func(*translator.AgentInput) error + resolve = func(agent *translator.AgentInput) error { + if agent.ResolvedModelConfig != nil { + model, err := translator.ResolveModelConfig(compiler.ctx, compiler.collections, agent.ResolvedModelConfig.Config) + if err != nil { + return err + } + agent.ResolvedModelConfig = model + } + for _, child := range agent.Shared { + if err := resolve(child.Agent); err != nil { + return err + } + } + return nil + } + if err := resolve(input.Root); err != nil { + return nil, err + } + policy, err := translator.ResolvePolicy(input, compiler, nil) + if err != nil { + return nil, err + } + input.Policy = policy + return compiler.Compile(ctx, input) +} diff --git a/go/core/internal/translator/claude/mcp.go b/go/core/internal/translator/claude/mcp.go index 15d2f2ad16..e6f85e6fbc 100644 --- a/go/core/internal/translator/claude/mcp.go +++ b/go/core/internal/translator/claude/mcp.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "fmt" - "net/url" "slices" "strings" "time" @@ -14,7 +13,6 @@ import ( "k8s.io/apimachinery/pkg/types" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" claudeconfig "github.com/kagent-dev/kagent/go/harness/claude/config" ) @@ -22,7 +20,6 @@ import ( type mcpCompilation struct { servers map[string]claudeconfig.MCPServer environment []corev1.EnvVar - egress []string warnings []string } @@ -30,6 +27,7 @@ func (c *Compiler) compileMCP( ctx context.Context, namespace string, tools []translator.ResolvedMCPTool, + policy translator.ResolvedPolicy, ) (mcpCompilation, error) { if len(tools) == 0 { return mcpCompilation{}, nil @@ -60,20 +58,19 @@ func (c *Compiler) compileMCP( if compatibilityWarning != "" { result.warnings = append(result.warnings, compatibilityWarning) } - hostname, err := mcpOrigin(server.Spec.URL) - if err != nil { - return mcpCompilation{}, err - } headers, headerEnvironment, err := c.compileMCPHeaders(ctx, namespace, server.Spec.HeadersFrom) if err != nil { return mcpCompilation{}, fmt.Errorf("compile RemoteMCPServer %q headers: %w", server.Name, err) } + headers, err = policy.MCPHeaders(server.Spec.URL, headers) + if err != nil { + return mcpCompilation{}, err + } result.servers[name] = claudeconfig.MCPServer{ Type: transport, URL: server.Spec.URL, Headers: headers, RequireApproval: tool.Binding.RequireApproval, } result.environment = append(result.environment, headerEnvironment...) - result.egress = append(result.egress, hostname) } return result, nil } @@ -144,14 +141,6 @@ func claudeMCPTransport(server *v1alpha3.RemoteMCPServer) (string, string, error } } -func mcpOrigin(raw string) (string, error) { - parsed, err := url.Parse(raw) - if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" || parsed.User != nil || parsed.Fragment != "" { - return "", translator.NewValidationError("Claude RemoteMCPServer URL must be absolute HTTP(S) without credentials or fragment") - } - return egress.Origin(parsed), nil -} - func (c *Compiler) compileMCPHeaders(ctx context.Context, namespace string, refs []v1alpha3.ValueRef) (map[string]string, []corev1.EnvVar, error) { if len(refs) == 0 { return nil, nil, nil diff --git a/go/core/internal/translator/codex/compiler.go b/go/core/internal/translator/codex/compiler.go index 9323d220fe..183c3ed364 100644 --- a/go/core/internal/translator/codex/compiler.go +++ b/go/core/internal/translator/codex/compiler.go @@ -6,7 +6,6 @@ import ( "crypto/sha256" "encoding/json" "fmt" - "net/url" "reflect" "slices" "strings" @@ -18,7 +17,6 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" codexconfig "github.com/kagent-dev/kagent/go/harness/codex/config" "github.com/kagent-dev/kagent/go/pkg/tracing" @@ -72,7 +70,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) runtimeTelemetry := telemetryConfig.RuntimeTelemetry( tracing.RuntimeCodex, input.AgentName, template.Namespace, model.Spec) - provider, providerEnvironment, egress, err := c.compileProvider(ctx, model) + provider, providerEnvironment, err := c.compileProvider(ctx, model, input.Policy.ModelUsesGateway(input.Root.ResolvedModelConfig)) if err != nil { return nil, err } @@ -80,7 +78,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, err } - mcp, err := c.compileMCP(ctx, input.Root.Template.Namespace, input.Root.MCPTools) + mcp, err := c.compileMCP(ctx, input.Root.Template.Namespace, input.Root.MCPTools, input.Policy) if err != nil { return nil, err } @@ -115,13 +113,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if len(skills.Resources.Skills) != 0 || len(skills.Resources.Plugins) != 0 { cfg.SkillResources = &skills.Resources } - if err := cfg.Validate(); err != nil { - return nil, translator.NewValidationError("invalid compiled Codex configuration: %v", err) - } - configJSON, err := json.Marshal(cfg) - if err != nil { - return nil, fmt.Errorf("marshal Codex config: %w", err) - } + card, err := pbconv.ToProtoAgentCard(translator.ManagedAgentCard(input.AgentName, input.Root.Template)) if err != nil { return nil, fmt.Errorf("convert Codex agent card: %w", err) @@ -130,12 +122,19 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build Codex revision provenance: %w", err) } - environment, credentials, err := translator.CompileCredentials(input, nil, environment) + environment, err = input.Policy.RenderEnvironment(environment) if err != nil { return nil, err } - egress = append(egress, skills.Egress...) - egress = append(egress, mcp.egress...) + if err := cfg.Validate(); err != nil { + return nil, translator.NewValidationError("invalid compiled Codex configuration: %v", err) + } + + configJSON, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("marshal Codex config: %w", err) + } + egress := skills.Egress egress = append(egress, telemetryConfig.Destinations(environment)...) egress = append(egress, c.settings.GetControlPlaneUrl()) slices.Sort(egress) @@ -146,59 +145,61 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) SkillImages: skills.Images, Environment: environment, ConfigJSON: configJSON, AgentCard: card, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - Credentials: credentials, Provenance: provenance, EgressDestinations: egress, + Provenance: provenance, EgressDestinations: egress, }, Warnings: mcp.warnings, }, nil } -func (c *Compiler) compileProvider(ctx context.Context, model *v1alpha3.ModelConfig) (codexconfig.Provider, []corev1.EnvVar, []string, error) { +func (c *Compiler) compileProvider(ctx context.Context, model *v1alpha3.ModelConfig, gateway bool) (codexconfig.Provider, []corev1.EnvVar, error) { switch model.Spec.Provider { case v1alpha3.ModelProviderOpenAI: if model.Spec.OpenAI == nil || model.Spec.OpenAI.APIFormat == nil || *model.Spec.OpenAI.APIFormat != v1alpha3.OpenAIAPIFormatResponses { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex OpenAI requires openAI.apiFormat responses") + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex OpenAI requires openAI.apiFormat responses") } options := *model.Spec.OpenAI baseURL := strings.TrimSpace(options.BaseURL) options.BaseURL, options.APIFormat = "", nil if !reflect.DeepEqual(options, v1alpha3.OpenAIConfig{}) { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex does not support OpenAI provider options beyond baseUrl and apiFormat responses") + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex does not support OpenAI provider options beyond baseUrl and apiFormat responses") } - if err := c.requireSecretKey(ctx, model.Namespace, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey); err != nil { - return codexconfig.Provider{}, nil, nil, err - } - provider := codexconfig.Provider{Name: "openai", BaseURL: baseURL} - egress := []string{"https://api.openai.com:443"} - if baseURL != "" { - host, err := absoluteHTTPOrigin(baseURL) - if err != nil { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex OpenAI baseUrl %v", err) + environment := []corev1.EnvVar{{Name: openAIAPIKeyEnv, Value: translator.CredentialPlaceholder}} + if model.Spec.APIKeySecret != "" || !gateway { + if err := c.requireSecretKey(ctx, model.Namespace, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey); err != nil { + return codexconfig.Provider{}, nil, err } - egress = []string{host} + environment[0] = secretEnvironment(openAIAPIKeyEnv, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey) } - return provider, []corev1.EnvVar{secretEnvironment(openAIAPIKeyEnv, model.Spec.APIKeySecret, model.Spec.APIKeySecretKey)}, egress, nil + provider := codexconfig.Provider{Name: "openai", BaseURL: baseURL} + return provider, environment, nil case v1alpha3.ModelProviderBedrock: if model.Spec.Bedrock == nil || strings.TrimSpace(model.Spec.Bedrock.Region) == "" { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex Bedrock requires bedrock.region") + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex Bedrock requires bedrock.region") } options := *model.Spec.Bedrock region := strings.TrimSpace(options.Region) if !strings.HasPrefix(model.Spec.Model, "gpt-") { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex Bedrock supports only OpenAI gpt-* model IDs") + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex Bedrock supports only OpenAI gpt-* model IDs") } options.Region = "" if options.CacheTTL == "5m" { options.CacheTTL = "" } if !reflect.DeepEqual(options, v1alpha3.BedrockConfig{}) { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex does not support Bedrock provider options beyond region") + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex does not support Bedrock provider options beyond region") + } + if model.Spec.APIKeySecret == "" && gateway { + return codexconfig.Provider{Name: "amazon-bedrock"}, []corev1.EnvVar{ + {Name: awsRegionEnv, Value: region}, + {Name: awsBedrockTokenEnv, Value: translator.CredentialPlaceholder}, + }, nil } if model.Spec.APIKeySecret == "" || model.Spec.APIKeySecretKey != "" { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex Bedrock requires apiKeySecret and an empty apiKeySecretKey") + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex Bedrock requires apiKeySecret and an empty apiKeySecretKey") } secret, err := c.secret(ctx, model.Namespace, model.Spec.APIKeySecret) if err != nil { - return codexconfig.Provider{}, nil, nil, err + return codexconfig.Provider{}, nil, err } environment := []corev1.EnvVar{{Name: awsRegionEnv, Value: region}} if len(secret.Data[awsBedrockTokenEnv]) != 0 { @@ -206,7 +207,7 @@ func (c *Compiler) compileProvider(ctx context.Context, model *v1alpha3.ModelCon } else { for _, key := range []string{awsAccessKeyEnv, awsSecretKeyEnv} { if len(secret.Data[key]) == 0 { - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex Bedrock Secret %q requires %s and %s, or %s", secret.Name, awsAccessKeyEnv, awsSecretKeyEnv, awsBedrockTokenEnv) + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex Bedrock Secret %q requires %s and %s, or %s", secret.Name, awsAccessKeyEnv, awsSecretKeyEnv, awsBedrockTokenEnv) } environment = append(environment, secretEnvironment(key, secret.Name, key)) } @@ -214,9 +215,9 @@ func (c *Compiler) compileProvider(ctx context.Context, model *v1alpha3.ModelCon environment = append(environment, secretEnvironment(awsSessionTokenEnv, secret.Name, awsSessionTokenEnv)) } } - return codexconfig.Provider{Name: "amazon-bedrock"}, environment, []string{"https://bedrock-runtime." + region + ".amazonaws.com:443"}, nil + return codexconfig.Provider{Name: "amazon-bedrock"}, environment, nil default: - return codexconfig.Provider{}, nil, nil, translator.NewValidationError("Codex does not support ModelConfig provider %q", model.Spec.Provider) + return codexconfig.Provider{}, nil, translator.NewValidationError("Codex does not support ModelConfig provider %q", model.Spec.Provider) } } @@ -277,14 +278,6 @@ func secretEnvironment(environmentName, secretName, key string) corev1.EnvVar { }}} } -func absoluteHTTPOrigin(raw string) (string, error) { - parsed, err := url.Parse(raw) - if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" || parsed.User != nil || parsed.Fragment != "" { - return "", fmt.Errorf("must be an absolute HTTP(S) URL without credentials or fragment") - } - return egress.Origin(parsed), nil -} - type provenanceEntry struct { APIVersion string `json:"apiVersion"` Kind string `json:"kind"` @@ -383,3 +376,13 @@ func objectProvenance(apiVersion, kind, name string, uid types.UID, generation i } var _ translator.HarnessCompiler = (*Compiler)(nil) + +// ModelRequirements declares authentication and grants before client rendering. +func (*Compiler) ModelRequirements(model *translator.ResolvedModelConfig) translator.ModelRequirements { + switch model.Config.Spec.Provider { + case v1alpha3.ModelProviderOpenAI, v1alpha3.ModelProviderBedrock: + return translator.ModelRequirements{CredentialRequired: true} + default: + return translator.ModelRequirements{} + } +} diff --git a/go/core/internal/translator/codex/compiler_test.go b/go/core/internal/translator/codex/compiler_test.go index 2cc9eab92a..a9a3dcb359 100644 --- a/go/core/internal/translator/codex/compiler_test.go +++ b/go/core/internal/translator/codex/compiler_test.go @@ -34,20 +34,19 @@ func TestCompileProviderCredentials(t *testing.T) { secret map[string][]byte provider string environment map[string]string - egress []string wantErr string }{ { name: "OpenAI", model: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, Model: "gpt-5.2-codex", APIKeySecret: "model-auth", APIKeySecretKey: "api-key", OpenAI: &v1alpha3.OpenAIConfig{APIFormat: &responses}}, - secret: map[string][]byte{"api-key": []byte(credentialValue)}, provider: "openai", environment: map[string]string{openAIAPIKeyEnv: translator.CredentialPlaceholder}, egress: []string{"http://kagent-controller.kagent:8083", "https://api.openai.com:443"}, + secret: map[string][]byte{"api-key": []byte(credentialValue)}, provider: "openai", environment: map[string]string{openAIAPIKeyEnv: translator.CredentialPlaceholder}, }, { name: "OpenAI gateway", model: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, Model: "gpt", APIKeySecret: "model-auth", APIKeySecretKey: "api-key", OpenAI: &v1alpha3.OpenAIConfig{APIFormat: &responses, BaseURL: "https://gateway.example.com/v1"}}, - secret: map[string][]byte{"api-key": []byte(credentialValue)}, provider: "openai", environment: map[string]string{openAIAPIKeyEnv: translator.CredentialPlaceholder}, egress: []string{"http://kagent-controller.kagent:8083", "https://gateway.example.com:443"}, + secret: map[string][]byte{"api-key": []byte(credentialValue)}, provider: "openai", environment: map[string]string{openAIAPIKeyEnv: translator.CredentialPlaceholder}, }, { name: "Bedrock API key", model: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderBedrock, Model: "gpt-5.2", APIKeySecret: "model-auth", Bedrock: &v1alpha3.BedrockConfig{Region: "us-east-1", CacheTTL: "5m"}}, - secret: map[string][]byte{awsBedrockTokenEnv: []byte(credentialValue)}, provider: "amazon-bedrock", environment: map[string]string{awsRegionEnv: "us-east-1", awsBedrockTokenEnv: translator.CredentialPlaceholder}, egress: []string{"http://kagent-controller.kagent:8083", "https://bedrock-runtime.us-east-1.amazonaws.com:443"}, + secret: map[string][]byte{awsBedrockTokenEnv: []byte(credentialValue)}, provider: "amazon-bedrock", environment: map[string]string{awsRegionEnv: "us-east-1", awsBedrockTokenEnv: translator.CredentialPlaceholder}, }, { name: "Bedrock IAM", model: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderBedrock, Model: "gpt-5.2", APIKeySecret: "model-auth", Bedrock: &v1alpha3.BedrockConfig{Region: "us-west-2"}}, @@ -57,7 +56,7 @@ func TestCompileProviderCredentials(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { input, reader := testInput(t, test.model, test.secret) - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if test.wantErr != "" { if err == nil || !strings.Contains(err.Error(), test.wantErr) { t.Fatalf("expected unsupported credential error, got %v", err) @@ -67,6 +66,9 @@ func TestCompileProviderCredentials(t *testing.T) { if err != nil { t.Fatal(err) } + if len(revision.Credentials) != 0 { + t.Fatal("renderer must leave final credentials to the shared compiler") + } var cfg codexconfig.Config if err := json.Unmarshal(revision.ConfigJSON, &cfg); err != nil { t.Fatal(err) @@ -86,13 +88,13 @@ func TestCompileProviderCredentials(t *testing.T) { t.Errorf("environment[%s] = %q, want %q", name, gotEnvironment[name], value) } } - if !reflect.DeepEqual(revision.EgressDestinations, test.egress) { - t.Errorf("egress = %v, want %v", revision.EgressDestinations, test.egress) + if !reflect.DeepEqual(revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083"}) { + t.Errorf("egress = %v, want %v", revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083"}) } if bytes.Contains(revision.ConfigJSON, []byte(credentialValue)) || bytes.Contains(revision.Provenance, []byte(credentialValue)) { t.Fatal("credential leaked into immutable revision") } - again, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + again, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil || !reflect.DeepEqual(revision, again) { t.Fatalf("compilation is not deterministic: %v", err) } @@ -111,7 +113,7 @@ func TestCompileAllowsRuntimeOTELEndpoint(t *testing.T) { value := "http://other-collector:4317" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: value}} - result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + result, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -131,7 +133,7 @@ func TestCompileAllowsRuntimeTelemetryOverrides(t *testing.T) { value := "x-tenant=team-a" input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OTEL_EXPORTER_OTLP_HEADERS", Value: value}} - result, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + result, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -150,7 +152,7 @@ func TestCompileRejectsUnsupportedProviderConfiguration(t *testing.T) { } for _, model := range tests { input, reader := testInput(t, model, map[string][]byte{"api-key": []byte("secret"), awsAccessKeyEnv: []byte("access"), awsSecretKeyEnv: []byte("secret")}) - _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) var validation *translator.ValidationError if !errors.As(err, &validation) { t.Errorf("Compile(%s) error = %v, want validation", model.Provider, err) @@ -175,7 +177,7 @@ func TestCompileMCPAndSharedAgent(t *testing.T) { }, Instruction: "Review carefully", }}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatal(err) } @@ -192,7 +194,7 @@ func TestCompileMCPAndSharedAgent(t *testing.T) { if !strings.HasPrefix(cfg.MCPServers["tools"].Headers["Authorization"], "${"+mcpCredentialPrefix) { t.Fatalf("MCP headers = %#v", cfg.MCPServers["tools"].Headers) } - if !reflect.DeepEqual(revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083", "https://api.openai.com:443", "https://mcp.example.com:443"}) { + if !reflect.DeepEqual(revision.EgressDestinations, []string{"http://kagent-controller.kagent:8083"}) { t.Fatalf("egress = %v", revision.EgressDestinations) } } @@ -218,7 +220,7 @@ func TestCompileMCPCompatibilityWarnings(t *testing.T) { } input.Root.MCPTools = []translator.ResolvedMCPTool{{Binding: v1alpha3.MCPToolBinding{RequireApproval: true}, Server: server}} - compilation, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input) + compilation, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input) if err != nil { t.Fatalf("Compile() error = %v", err) } @@ -239,7 +241,7 @@ func TestCompileMCPCompatibilityWarnings(t *testing.T) { } server.Spec.Protocol = v1alpha3.RemoteMCPServerProtocolSse - if _, err := NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()).Compile(context.Background(), input); err == nil || !strings.Contains(err.Error(), "requires Streamable HTTP") { + if _, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, testRuntimeDefaults()), input); err == nil || !strings.Contains(err.Error(), "requires Streamable HTTP") { t.Fatalf("unsupported MCP protocol Compile() error = %v", err) } } @@ -275,7 +277,7 @@ func TestCompileRuntimeTelemetry(t *testing.T) { settings := testRuntimeDefaults() settings.Telemetry = &configv1.TelemetryConfig{Traces: &configv1.OTLPConfig{Endpoint: "http://collector:4317", Protocol: new("grpc")}} - revision, err := NewCompiler(krt.TestingDummyContext{}, reader, settings).Compile(context.Background(), input) + revision, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, settings), input) if err != nil { t.Fatal(err) } @@ -297,7 +299,7 @@ func TestCompileRuntimeTelemetry(t *testing.T) { } settings.Telemetry.CaptureContent = true - captured, err := NewCompiler(krt.TestingDummyContext{}, reader, settings).Compile(context.Background(), input) + captured, err := compileTestInput(context.Background(), NewCompiler(krt.TestingDummyContext{}, reader, settings), input) if err != nil { t.Fatal(err) } @@ -334,3 +336,31 @@ func TestCompileRuntimeTelemetry(t *testing.T) { func testRuntimeDefaults() *configv1.RuntimeDefaultsConfig { return &configv1.RuntimeDefaultsConfig{ControlPlaneUrl: new("http://kagent-controller.kagent:8083"), GatewayUrl: new("http://kagent-controller.kagent:8083"), Logging: &configv1.LoggingConfig{Level: new("info")}} } + +func compileTestInput(ctx context.Context, compiler *Compiler, input *translator.HarnessInput) (*translator.CompileResult, error) { + var resolve func(*translator.AgentInput) error + resolve = func(agent *translator.AgentInput) error { + if agent.ResolvedModelConfig != nil { + model, err := translator.ResolveModelConfig(compiler.ctx, compiler.collections, agent.ResolvedModelConfig.Config) + if err != nil { + return err + } + agent.ResolvedModelConfig = model + } + for _, child := range agent.Shared { + if err := resolve(child.Agent); err != nil { + return err + } + } + return nil + } + if err := resolve(input.Root); err != nil { + return nil, err + } + policy, err := translator.ResolvePolicy(input, compiler, nil) + if err != nil { + return nil, err + } + input.Policy = policy + return compiler.Compile(ctx, input) +} diff --git a/go/core/internal/translator/codex/mcp.go b/go/core/internal/translator/codex/mcp.go index dc4726f781..0f50ec8809 100644 --- a/go/core/internal/translator/codex/mcp.go +++ b/go/core/internal/translator/codex/mcp.go @@ -20,11 +20,10 @@ import ( type mcpCompilation struct { servers map[string]codexconfig.MCPServer environment []corev1.EnvVar - egress []string warnings []string } -func (c *Compiler) compileMCP(ctx context.Context, namespace string, tools []translator.ResolvedMCPTool) (mcpCompilation, error) { +func (c *Compiler) compileMCP(ctx context.Context, namespace string, tools []translator.ResolvedMCPTool, policy translator.ResolvedPolicy) (mcpCompilation, error) { if len(tools) == 0 { return mcpCompilation{}, nil } @@ -49,14 +48,14 @@ func (c *Compiler) compileMCP(ctx context.Context, namespace string, tools []tra if warning := codexMCPCompatibilityWarning(server); warning != "" { result.warnings = append(result.warnings, warning) } - host, err := absoluteHTTPOrigin(server.Spec.URL) - if err != nil { - return mcpCompilation{}, translator.NewValidationError("Codex RemoteMCPServer %q URL %v", server.Name, err) - } headers, environment, err := c.compileMCPHeaders(ctx, namespace, server.Spec.HeadersFrom) if err != nil { return mcpCompilation{}, fmt.Errorf("compile RemoteMCPServer %q headers: %w", server.Name, err) } + headers, err = policy.MCPHeaders(server.Spec.URL, headers) + if err != nil { + return mcpCompilation{}, err + } selected := append([]string(nil), tool.Binding.Tools...) slices.Sort(selected) selected = slices.Compact(selected) @@ -65,7 +64,6 @@ func (c *Compiler) compileMCP(ctx context.Context, namespace string, tools []tra RequireApproval: tool.Binding.RequireApproval, } result.environment = append(result.environment, environment...) - result.egress = append(result.egress, host) } return result, nil } diff --git a/go/core/internal/translator/collections.go b/go/core/internal/translator/collections.go index 02d07639fc..46563043f6 100644 --- a/go/core/internal/translator/collections.go +++ b/go/core/internal/translator/collections.go @@ -11,6 +11,7 @@ import ( // Collections contains every typed input used while compiling a revision. // Production supplies informer-backed collections; tests supply KRT mocks. type Collections struct { + EgressPolicies krt.Collection[*v1alpha3.EgressPolicy] Harnesses krt.Collection[*v1alpha3.Harness] AgentTemplates krt.Collection[*v1alpha3.AgentTemplate] ResolvedModelConfigs krt.Collection[ResolvedModelConfig] diff --git a/go/core/internal/translator/compiler.go b/go/core/internal/translator/compiler.go index 9a1770d9f1..3e08a12197 100644 --- a/go/core/internal/translator/compiler.go +++ b/go/core/internal/translator/compiler.go @@ -12,11 +12,14 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" ) // Compiler resolves public API objects into a complete, immutable runtime // revision. It owns the translation boundary from API objects to runtime inputs. type Compiler struct { + policyTranslator egressext.PolicyTranslator ctx krt.HandlerContext collections Collections harnessCompilers map[HarnessType]HarnessCompiler @@ -38,6 +41,13 @@ const ( // revision and its user-facing diagnostics. type HarnessCompiler interface { Compile(context.Context, *HarnessInput) (*CompileResult, error) + ModelRequirements(*ResolvedModelConfig) ModelRequirements +} + +// ModelRequirements declares integration-specific grants and authentication before rendering. +type ModelRequirements struct { + Destinations []string + CredentialRequired bool } // resolvedTemplateTree is the validated AgentTemplate topology for one Harness. @@ -61,6 +71,9 @@ type resolvedTemplateBinding struct { // HarnessInput contains the Kubernetes inputs needed by a harness compiler. type HarnessInput struct { + Policy ResolvedPolicy + SummarizerModel *ResolvedModelConfig + MemoryModel *ResolvedModelConfig // AgentName is the runnable identity, independent of inline or referenced inputs. AgentName string Harness *HarnessConfiguration @@ -91,8 +104,8 @@ type AgentInputBinding struct { } // NewCompiler constructs the runtime compiler. -func NewCompiler(ctx krt.HandlerContext, collections Collections, harnessCompilers map[HarnessType]HarnessCompiler, images *configv1.HarnessImagesConfig) *Compiler { - return &Compiler{ctx: ctx, collections: collections, harnessCompilers: maps.Clone(harnessCompilers), images: images} +func NewCompiler(ctx krt.HandlerContext, collections Collections, harnessCompilers map[HarnessType]HarnessCompiler, images *configv1.HarnessImagesConfig, policyTranslator egressext.PolicyTranslator) *Compiler { + return &Compiler{ctx: ctx, collections: collections, harnessCompilers: maps.Clone(harnessCompilers), images: images, policyTranslator: policyTranslator} } // CompileAgent resolves inline or referenced behavior with its referenced @@ -124,7 +137,7 @@ func (c *Compiler) CompileAgent(ctx context.Context, agent *v1alpha3.Agent) (*Co return nil, fmt.Errorf("resolve Harness %q: not found", harnessKey) } harness := harnessConfiguration(*foundHarness) - result, err := c.compileConfiguration(ctx, agent.Name, harness, template) + result, err := c.compileConfiguration(ctx, agent, harness, template) if err != nil { return nil, err } @@ -141,9 +154,9 @@ func (c *Compiler) CompileAgent(ctx context.Context, agent *v1alpha3.Agent) (*Co return result, nil } -// compileConfiguration compiles resolved configuration for the named Agent. +// compileConfiguration resolves inputs and compiles configuration for the Agent. // The Agent name owns runtime identity; template and Harness names are provenance. -func (c *Compiler) compileConfiguration(ctx context.Context, agentName string, harness *HarnessConfiguration, template *TemplateConfiguration) (*CompileResult, error) { +func (c *Compiler) compileConfiguration(ctx context.Context, agent *v1alpha3.Agent, harness *HarnessConfiguration, template *TemplateConfiguration) (*CompileResult, error) { harnessCompiler := c.harnessCompilers[harnessType(harness)] if harnessCompiler == nil { return nil, NewValidationError("Harness runtime is not supported by any compiler") @@ -156,7 +169,14 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agentName string, h if err != nil { return nil, err } - input.AgentName = agentName + input.AgentName = agent.Name + if err := c.resolveAuxiliaryModels(input); err != nil { + return nil, err + } + input.Policy, err = ResolvePolicy(input, harnessCompiler, c.policyTranslator) + if err != nil { + return nil, err + } image, err := harnessImage(harness, c.images) if err != nil { return nil, err @@ -165,12 +185,18 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agentName string, h if err != nil { return nil, err } + // Pin integration access. + pinned, err := egress.CanonicalAccess(append(result.EgressDestinations, input.Policy.access.Destinations...), input.Policy.access.Credentials) + if err != nil { + return nil, NewValidationError("%v", err) + } + result.EgressDestinations, result.Credentials, result.TokenPassthroughOrigins = pinned.Destinations, pinned.Credentials, input.Policy.tokenPassthroughOrigins workerKey := types.NamespacedName{Namespace: harness.Namespace, Name: harness.Spec.Substrate.WorkerPoolRef.Name} workerPool := krt.FetchOne(c.ctx, c.collections.WorkerPools, krt.FilterObjectName(workerKey)) if workerPool == nil { return nil, &WorkerPoolNotFoundError{WorkerPool: workerKey} } - result.AgentName = agentName + result.AgentName = agent.Name result.Image = image result.SandboxClass = (*workerPool).Spec.SandboxClass // Shared agents and plugin selections can contribute the same image. @@ -331,3 +357,40 @@ func (c *Compiler) buildInputs(ctx context.Context, tree *resolvedTemplateTree) } return &HarnessInput{Harness: tree.Harness, Root: root, OutputSchema: outputSchema}, nil } + +// resolveAuxiliaryModels tracks every ADK model before policy resolution and rendering. +func (c *Compiler) resolveAuxiliaryModels(input *HarnessInput) error { + if input.Harness.Spec.Kagent == nil { + return nil + } + resolve := func(name string) (*ResolvedModelConfig, error) { + model := krt.FetchOne(c.ctx, c.collections.ResolvedModelConfigs, krt.FilterObjectName(types.NamespacedName{Namespace: input.Harness.Namespace, Name: name})) + if model == nil { + return nil, fmt.Errorf("model config %q not found", name) + } + if failure := model.Failure(); failure != nil { + if len(model.SemanticFailures) != 0 { + return nil, NewValidationError("ModelConfig %q: %s", name, failure.Message) + } + return nil, fmt.Errorf("ModelConfig %q: %s", name, failure.Message) + } + return model, nil + } + var err error + if compaction := input.Harness.Spec.Kagent.Compaction; compaction != nil && compaction.Summarizer != nil && compaction.Summarizer.ModelConfigRef != nil { + name := compaction.Summarizer.ModelConfigRef.Name + if input.Root.Template.Spec.ModelConfig == nil || input.Root.Template.Spec.ModelConfig.Name != name { + input.SummarizerModel, err = resolve(name) + if err != nil { + return err + } + } + } + if memory := input.Harness.Spec.Kagent.Memory; memory != nil { + input.MemoryModel, err = resolve(memory.ModelConfigRef.Name) + if err != nil { + return err + } + } + return nil +} diff --git a/go/core/internal/translator/compiler_test.go b/go/core/internal/translator/compiler_test.go index 3b712f52f4..cbb3fac792 100644 --- a/go/core/internal/translator/compiler_test.go +++ b/go/core/internal/translator/compiler_test.go @@ -216,7 +216,7 @@ func compilerWithImages(t *testing.T, images *configv1.HarnessImagesConfig, obje translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), - }, images) + }, images, nil) } func defaultWorkerPool() *atev1alpha1.WorkerPool { @@ -227,6 +227,7 @@ func mockCollections(t *testing.T, objects ...any) translator.Collections { t.Helper() mock := krttest.NewMock(t, objects) collections := translator.Collections{ + EgressPolicies: krttest.GetMockCollection[*v1alpha3.EgressPolicy](mock), AgentTemplates: krttest.GetMockCollection[*v1alpha3.AgentTemplate](mock), Harnesses: krttest.GetMockCollection[*v1alpha3.Harness](mock), RemoteMCPServers: krttest.GetMockCollection[*v1alpha3.RemoteMCPServer](mock), @@ -405,6 +406,14 @@ func TestCompileAgentStructuredOutput(t *testing.T) { require.Empty(t, config.Agent.SubAgents[0].SessionDbUrl) require.Contains(t, string(revision.Provenance), `"kind":"ConfigMap"`) require.Equal(t, []string{"application/json"}, revision.AgentCard.DefaultOutputModes) + original, err := revision.Digest() + require.NoError(t, err) + configMap.Data["answer.json"] = `{"type":"object","properties":{"changed":{"type":"string"}}}` + changed, err := compiler(t, modelConfig(), configMap, child, harness).CompileAgent(t.Context(), inlineAgent(harness, root)) + require.NoError(t, err) + updated, err := changed.Digest() + require.NoError(t, err) + require.NotEqual(t, original, updated, "resolved ConfigMap contents must still affect compiled runtime identity") } func TestResolveModelConfigFoundryEndpoint(t *testing.T) { @@ -460,8 +469,13 @@ func TestResolveModelConfigFoundryEndpoint(t *testing.T) { type testHarnessCompiler struct{ input *translator.HarnessInput } +func (*testHarnessCompiler) ModelRequirements(*translator.ResolvedModelConfig) translator.ModelRequirements { + return translator.ModelRequirements{} +} + func (c *testHarnessCompiler) Compile(_ context.Context, input *translator.HarnessInput) (*translator.CompileResult, error) { - c.input = input + snapshot := *input + c.input = &snapshot return &translator.CompileResult{Revision: translator.Revision{AgentName: input.AgentName}}, nil } @@ -479,7 +493,7 @@ func TestCompilerAcceptsExternalHarnessCompiler(t *testing.T) { revision, err := translator.NewCompiler(krt.TestingDummyContext{}, collections, map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeCodex: adapter, - }, testHarnessImages).CompileAgent(context.Background(), inlineAgent(harness, template)) + }, testHarnessImages, nil).CompileAgent(context.Background(), inlineAgent(harness, template)) require.NoError(t, err) require.Equal(t, "runnable-agent", revision.AgentName) require.Equal(t, testHarnessImages.GetCodex(), revision.Image) @@ -505,7 +519,7 @@ func TestCompilerRejectsStructuredOutputForUnsupportedHarness(t *testing.T) { _, err := translator.NewCompiler(krt.TestingDummyContext{}, collections, map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeCodex: adapter, - }, testHarnessImages).CompileAgent(context.Background(), inlineAgent(harness, template)) + }, testHarnessImages, nil).CompileAgent(context.Background(), inlineAgent(harness, template)) require.ErrorContains(t, err, `Harness runtime "codex" does not support structured output`) require.Nil(t, adapter.input) } @@ -524,7 +538,7 @@ func TestCompilerRejectsUnusableModelConfigBeforeHarnessCompiler(t *testing.T) { _, err := translator.NewCompiler(krt.TestingDummyContext{}, collections, map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeCodex: adapter, - }, testHarnessImages).CompileAgent(context.Background(), inlineAgent(harness, template)) + }, testHarnessImages, nil).CompileAgent(context.Background(), inlineAgent(harness, template)) require.ErrorContains(t, err, `resolve ModelConfig "default-model": secret missing not found`) require.Nil(t, adapter.input) } @@ -540,7 +554,7 @@ func TestCompilerPermitsBYOWithoutModelConfig(t *testing.T) { _, err := translator.NewCompiler(krt.TestingDummyContext{}, mockCollections(t, defaultWorkerPool(), harness), map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeBYO: adapter, - }, testHarnessImages).CompileAgent(context.Background(), inlineAgent(harness, template)) + }, testHarnessImages, nil).CompileAgent(context.Background(), inlineAgent(harness, template)) require.NoError(t, err) require.Nil(t, adapter.input.Root.ResolvedModelConfig) } @@ -627,7 +641,7 @@ func TestCompileAgentInjectsCredentialsAtGateway(t *testing.T) { rotated.Data["token"] = []byte("rotated-token") rotatedCompiler := translator.NewCompiler(krt.TestingDummyContext{}, mockCollections(t, modelConfig(), server, secondServer, rotated, secondSecret, defaultWorkerPool(), harness), map[translator.HarnessType]translator.HarnessCompiler{ translator.HarnessTypeKagent: kagenttranslator.NewCompiler(krt.TestingDummyContext{}, mockCollections(t, modelConfig(), server, secondServer, rotated, secondSecret), testRuntimeDefaults()), - }, testHarnessImages) + }, testHarnessImages, nil) next, err := rotatedCompiler.CompileAgent(t.Context(), inlineAgent(harness, template)) require.NoError(t, err) nextDigest, err := next.Digest() @@ -682,7 +696,10 @@ func TestCompileAgentSharedADKConfig(t *testing.T) { runtimeHarness.Spec.BYO = &v1alpha3.BYOHarness{} runtimeHarness.Spec.Workload.Command = []string{"/app"} } - revision, err := compiler(t, modelConfig(), child, runtimeHarness, remoteMCPServer("search", "https://search.example.com/mcp")).CompileAgent(context.Background(), inlineAgent(runtimeHarness, root)) + policy, secret := egressPolicy() + agent := inlineAgent(runtimeHarness, root) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + revision, err := compiler(t, modelConfig(), child, runtimeHarness, policy, secret, remoteMCPServer("search", "https://search.example.com/mcp")).CompileAgent(context.Background(), agent) require.NoError(t, err) var config configv1.ADKConfig require.NoError(t, runtimeconfig.Unmarshal(revision.ConfigJSON, &config)) @@ -692,6 +709,7 @@ func TestCompileAgentSharedADKConfig(t *testing.T) { require.Equal(t, "research carefully", config.Agent.SubAgents[0].Instruction) require.Equal(t, []string{"lookup"}, config.Agent.SubAgents[0].HttpTools[0].Tools) require.Equal(t, "review", config.Agent.SubAgents[0].AgentPlugins.Skills[0].Name) + require.NotContains(t, revision.EgressDestinations, "https://tickets.example.com:443") require.Contains(t, revision.EgressDestinations, "https://search.example.com:443") require.NotContains(t, revision.EgressDestinations, "https://ghcr.io:443") require.Equal(t, []translator.SkillImage{translator.SkillImage(child.Spec.Skills[0].Source.OCI)}, revision.SkillImages) diff --git a/go/core/internal/translator/credentials.go b/go/core/internal/translator/credentials.go deleted file mode 100644 index 29ac153c2e..0000000000 --- a/go/core/internal/translator/credentials.go +++ /dev/null @@ -1,182 +0,0 @@ -package translator - -import ( - "fmt" - "net/url" - "strings" - - corev1 "k8s.io/api/core/v1" - - "github.com/kagent-dev/kagent/go/adk/pkg/models" - "github.com/kagent-dev/kagent/go/api/v1alpha3" - "github.com/kagent-dev/kagent/go/core/internal/egress" - "github.com/kagent-dev/kagent/go/core/pkg/env" -) - -// CredentialPlaceholder satisfies SDKs that require a configured API key. -// The gateway overwrites the corresponding header with the current Secret. -const CredentialPlaceholder = "kagent-credential-injected" - -// CompileCredentials replaces secret-backed environment values with inert SDK -// placeholders and compiles their destination-scoped gateway bindings. Models -// outside the agent tree (such as memory embeddings) are supplied separately. -func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, environment []corev1.EnvVar) ([]corev1.EnvVar, []egress.Credential, error) { - var bindings []egress.Credential - boundModels := map[string]bool{} - boundMCP := map[string]bool{} - bind := func(rawURL, header, prefix, namespace, name, key string) error { - u, err := url.Parse(strings.TrimSpace(rawURL)) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.Fragment != "" { - return NewValidationError("credential destination must be an absolute HTTP(S) URL without user information or fragment") - } - uri := "ate-secret://k8s.io/default/" + namespace + "/" + name + "/" + key - bindings = append(bindings, egress.Credential{Hostname: u.Hostname(), Header: header, Prefix: prefix, URI: uri}) - return nil - } - models := append([]*ResolvedModelConfig(nil), extraModels...) - var visit func(*AgentInput) error - visit = func(agent *AgentInput) error { - if len(extraModels) == 0 && agent.ResolvedModelConfig != nil { - models = append(models, agent.ResolvedModelConfig) - } - for _, tool := range agent.MCPTools { - for _, ref := range tool.Server.Spec.HeadersFrom { - if ref.ValueFrom != nil && ref.ValueFrom.Type == v1alpha3.SecretValueSource { - boundMCP[ref.ValueFrom.Name+"\x00"+ref.ValueFrom.Key] = true - if err := bind(tool.Server.Spec.URL, ref.Name, "", tool.Server.Namespace, ref.ValueFrom.Name, ref.ValueFrom.Key); err != nil { - return err - } - } - } - } - for _, child := range agent.Shared { - if err := visit(child.Agent); err != nil { - return err - } - } - return nil - } - if err := visit(input.Root); err != nil { - return nil, nil, err - } - for _, resolved := range models { - model := resolved.Config - if model.Spec.APIKeyPassthrough || model.Spec.APIKeySecret == "" { - continue - } - name, endpoint, header, prefix := modelCredentialTarget(resolved) - if name == "" { - continue - } - key := model.Spec.APIKeySecretKey - if model.Spec.Provider == v1alpha3.ModelProviderBedrock { - key = env.AWSBearerTokenBedrock.Name() - bearer := false - for _, variable := range environment { - if variable.Name == name && variable.ValueFrom != nil && variable.ValueFrom.SecretKeyRef != nil && variable.ValueFrom.SecretKeyRef.Name == model.Spec.APIKeySecret && variable.ValueFrom.SecretKeyRef.Key == key { - bearer = true - } - } - if !bearer { - continue - } - } - if err := bind(endpoint, header, prefix, model.Namespace, model.Spec.APIKeySecret, key); err != nil { - return nil, nil, err - } - boundModels[name+"\x00"+model.Spec.APIKeySecret+"\x00"+key] = true - } - bindings, err := egress.CanonicalCredentials(bindings) - if err != nil { - return nil, nil, NewValidationError("%v", err) - } - for _, resolved := range models { - if !resolved.Config.Spec.APIKeyPassthrough { - continue - } - _, endpoint, _, _ := modelCredentialTarget(resolved) - u, err := url.Parse(endpoint) - if err != nil { - return nil, nil, NewValidationError("invalid passthrough credential destination") - } - hostname := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") - for _, binding := range bindings { - if binding.Hostname == hostname { - return nil, nil, NewValidationError("destination %q cannot combine caller-token passthrough with gateway credentials", hostname) - } - } - } - result := append([]corev1.EnvVar(nil), environment...) - for i, variable := range result { - if variable.ValueFrom == nil { - continue - } - ref := variable.ValueFrom.SecretKeyRef - isMCP := strings.HasPrefix(variable.Name, "KAGENT_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CODEX_MCP_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CLAUDE_MCP_CREDENTIAL_") - if ref == nil || (!boundModels[variable.Name+"\x00"+ref.Name+"\x00"+ref.Key] && (!isMCP || !boundMCP[ref.Name+"\x00"+ref.Key])) { - return nil, nil, NewValidationError("environment credential %q cannot use gateway header injection; local signing and arbitrary secret environment variables are unsupported", variable.Name) - } - result[i].Value, result[i].ValueFrom = CredentialPlaceholder, nil - } - return result, bindings, nil -} - -func modelCredentialTarget(resolved *ResolvedModelConfig) (name, endpoint, header, prefix string) { - spec := resolved.Config.Spec - switch spec.Provider { - case v1alpha3.ModelProviderOpenAI: - name, endpoint, header, prefix = env.OpenAIAPIKey.Name(), "https://api.openai.com", "authorization", "Bearer " - if spec.OpenAI != nil && spec.OpenAI.BaseURL != "" { - endpoint = spec.OpenAI.BaseURL - } - case v1alpha3.ModelProviderAnthropic: - name, endpoint, header = env.AnthropicAPIKey.Name(), "https://api.anthropic.com", "x-api-key" - if spec.Anthropic != nil && spec.Anthropic.BaseURL != "" { - endpoint = spec.Anthropic.BaseURL - } - case v1alpha3.ModelProviderAzureOpenAI: - name, header = env.AzureOpenAIAPIKey.Name(), "api-key" - if spec.AzureOpenAI != nil { - endpoint = spec.AzureOpenAI.Endpoint - } - case v1alpha3.ModelProviderGemini: - name, endpoint, header = env.GoogleAPIKey.Name(), "https://generativelanguage.googleapis.com", "x-goog-api-key" - case v1alpha3.ModelProviderMistral: - name, endpoint, header, prefix = env.MistralAPIKey.Name(), "https://api.mistral.ai", "authorization", "Bearer " - if spec.Mistral != nil && spec.Mistral.BaseURL != nil && *spec.Mistral.BaseURL != "" { - endpoint = *spec.Mistral.BaseURL - } - case v1alpha3.ModelProviderBedrock: - name, header, prefix = env.AWSBearerTokenBedrock.Name(), "authorization", "Bearer " - if spec.Bedrock != nil { - endpoint = fmt.Sprintf("https://bedrock-runtime.%s.amazonaws.com", spec.Bedrock.Region) - } - case v1alpha3.ModelProviderOllama: - // Ollama Cloud is the only keyed Ollama endpoint, so a binding is - // declared exactly when the model reaches it. That condition lives in - // models.OllamaReachesCloud, which the compiler's Secret reference and - // the egress list also use: a cloud model picked from the catalog, an - // explicit host, and a missing key were each judged differently in each - // place, so a valid configuration could compile with an env var that had - // no binding to match, or with an egress list that omitted the host it - // was about to call. - // - // Without a binding this leaves the name empty, and the env var is - // treated as a plain secret reference, which is correct — no request - // leaves for api.ollama.com. - if spec.Ollama == nil { - break - } - hasCredential := spec.APIKeySecret != "" || spec.APIKeyPassthrough - if !models.OllamaReachesCloud(spec.Model, spec.Ollama.Host, hasCredential) { - break - } - name, endpoint, header, prefix = env.OllamaAPIKey.Name(), "https://api.ollama.com", "authorization", "Bearer " - case v1alpha3.ModelProviderFoundry: - name, endpoint, header = env.FoundryAPIKey.Name(), resolved.FoundryEndpoint, "api-key" - if spec.Foundry != nil && spec.Foundry.APIFormat == v1alpha3.FoundryAPIFormatAnthropic { - header = "x-api-key" - } - } - return name, endpoint, header, prefix -} diff --git a/go/core/internal/translator/egress.go b/go/core/internal/translator/egress.go new file mode 100644 index 0000000000..346d291e98 --- /dev/null +++ b/go/core/internal/translator/egress.go @@ -0,0 +1,136 @@ +package translator + +import ( + "fmt" + "slices" + "strings" + + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation" + + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +// EgressGrantIssue describes one policy resolution error or admission conflict. +type EgressGrantIssue struct { + Policy string + Reason string + Message string +} + +// EgressGrantResolution contains the union of individually valid policies. +// Admission checks conflicts across this union and the desired revision's base. +type EgressGrantResolution struct { + Access egress.Access + Issues []EgressGrantIssue +} + +// CompileEgressPolicies resolves each same-namespace policy independently. +// KRT tracks policy and Secret dependencies. An unresolved policy contributes +// no grants, without removing grants supplied by the remaining policies. +func CompileEgressPolicies(ctx krt.HandlerContext, collections Collections, namespace string, refs []corev1.LocalObjectReference) EgressGrantResolution { + return compileEgressPolicies(namespace, refs, func(name string) (*v1alpha3.EgressPolicy, error) { + policy := krt.FetchOne(ctx, collections.EgressPolicies, krt.FilterKey(namespace+"/"+name)) + if policy == nil { + return nil, apierrors.NewNotFound(schema.GroupResource{Group: "api.kagent.dev", Resource: "egresspolicies"}, name) + } + return *policy, nil + }, func(name string) (*corev1.Secret, error) { + secret := krt.FetchOne(ctx, collections.Secrets, krt.FilterKey(namespace+"/"+name)) + if secret == nil { + return nil, apierrors.NewNotFound(corev1.Resource("secrets"), name) + } + return *secret, nil + }) +} + +func compileEgressPolicies(namespace string, refs []corev1.LocalObjectReference, getPolicy func(string) (*v1alpha3.EgressPolicy, error), getSecret func(string) (*corev1.Secret, error)) EgressGrantResolution { + var result EgressGrantResolution + var invalidRefs string + if len(refs) > 16 { + invalidRefs = "at most 16 egressPolicyRefs are supported" + } + for _, ref := range refs { + if len(validation.IsDNS1123Subdomain(ref.Name)) != 0 { + invalidRefs = fmt.Sprintf("invalid EgressPolicy reference %q", ref.Name) + break + } + } + for _, ref := range refs { + issue := EgressGrantIssue{Policy: namespace + "/" + ref.Name, Reason: v1alpha3.EgressPoliciesReasonInvalidEgressPolicy} + if invalidRefs != "" { + issue.Message = invalidRefs + result.Issues = append(result.Issues, issue) + continue + } + policy, err := getPolicy(ref.Name) + if err != nil || !policy.DeletionTimestamp.IsZero() { + if err == nil || apierrors.IsNotFound(err) { + issue.Reason = v1alpha3.EgressPoliciesReasonNotFound + } + issue.Message = "EgressPolicy is being deleted" + if err != nil { + issue.Message = err.Error() + } + result.Issues = append(result.Issues, issue) + continue + } + access, err := compileEgressPolicy(namespace, policy, getSecret) + if err != nil { + issue.Message = err.Error() + result.Issues = append(result.Issues, issue) + continue + } + result.Access.Destinations = append(result.Access.Destinations, access.Destinations...) + result.Access.Credentials = append(result.Access.Credentials, access.Credentials...) + } + slices.SortFunc(result.Issues, func(a, b EgressGrantIssue) int { return strings.Compare(a.Policy, b.Policy) }) + return result +} + +func compileEgressPolicy(namespace string, policy *v1alpha3.EgressPolicy, getSecret func(string) (*corev1.Secret, error)) (egress.Access, error) { + if len(policy.Spec.Destinations) == 0 || len(policy.Spec.Destinations) > 64 { + return egress.Access{}, NewValidationError("requires 1 to 64 destinations") + } + var destinations []string + var credentials []egress.Credential + for _, destination := range policy.Spec.Destinations { + origin, err := egress.ParseOrigin(destination.Origin) + if err != nil { + return egress.Access{}, err + } + destinations = append(destinations, origin.String()) + auth := destination.Authentication + if auth == nil { + continue + } + if strings.Contains(origin.Host, "*") { + return egress.Access{}, NewValidationError("authentication requires an exact HTTP(S) origin") + } + secret := auth.SecretKeyRef + binding := egress.Credential{Origin: origin.String(), Header: auth.Header} + if auth.Prefix != nil { + binding.Prefix = *auth.Prefix + } + if secret.Name == "" || secret.Key == "" || (secret.Optional != nil && *secret.Optional) { + return egress.Access{}, NewValidationError("requires a non-optional Secret name and key") + } + binding.URI = "ate-secret://k8s.io/default/" + namespace + "/" + secret.Name + "/" + secret.Key + if _, err := egress.CanonicalCredentials([]egress.Credential{binding}); err != nil { + return egress.Access{}, err + } + found, err := getSecret(secret.Name) + if err != nil { + return egress.Access{}, fmt.Errorf("resolve Secret %q: %w", secret.Name, err) + } + if _, ok := found.Data[secret.Key]; !ok { + return egress.Access{}, NewValidationError("Secret %q is missing key %q", secret.Name, secret.Key) + } + credentials = append(credentials, binding) + } + return egress.CanonicalAccess(destinations, credentials) +} diff --git a/go/core/internal/translator/egress_policy.go b/go/core/internal/translator/egress_policy.go new file mode 100644 index 0000000000..dec0a4b326 --- /dev/null +++ b/go/core/internal/translator/egress_policy.go @@ -0,0 +1,379 @@ +package translator + +import ( + "cmp" + "fmt" + "net/url" + "slices" + "strings" + + "k8s.io/apimachinery/pkg/types" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" + "github.com/kagent-dev/kagent/go/core/pkg/env" +) + +func credentialOrigin(raw string) (string, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.User != nil || u.Fragment != "" { + return "", fmt.Errorf("invalid credential destination") + } + origin, err := egress.ParseOrigin(egress.Origin(u)) + if err != nil { + return "", err + } + return origin.String(), nil +} + +func secretURI(namespace, name, key string) string { + return "ate-secret://k8s.io/default/" + namespace + "/" + name + "/" + key +} + +func isSecretHeader(ref v1alpha3.ValueRef) bool { + return ref.ValueFrom != nil && ref.ValueFrom.Type == v1alpha3.SecretValueSource +} + +func bindsTarget(origin, header, prefix string) func(egress.Credential) bool { + return func(binding egress.Credential) bool { + return binding.Origin == origin && binding.Header == header && binding.Prefix == prefix + } +} + +func publicCredentials(bindings []egress.Credential) []egressext.Credential { + result := make([]egressext.Credential, len(bindings)) + for i, binding := range bindings { + result[i] = egressext.Credential(binding) + } + return result +} + +func internalCredentials(bindings []egressext.Credential) []egress.Credential { + result := make([]egress.Credential, len(bindings)) + for i, binding := range bindings { + result[i] = egress.Credential(binding) + } + return result +} + +// policyAgents returns the root and its shared agents in depth-first order. +func policyAgents(root *AgentInput) []*AgentInput { + agents := []*AgentInput{root} + for _, child := range root.Shared { + agents = append(agents, policyAgents(child.Agent)...) + } + return agents +} + +func policyModels(input *HarnessInput) []*ResolvedModelConfig { + var models []*ResolvedModelConfig + for _, agent := range policyAgents(input.Root) { + if agent.ResolvedModelConfig != nil { + models = append(models, agent.ResolvedModelConfig) + } + } + for _, model := range []*ResolvedModelConfig{input.SummarizerModel, input.MemoryModel} { + if model != nil { + models = append(models, model) + } + } + return models +} + +// ResolvePolicy resolves authentication before any harness renders client configuration. +func ResolvePolicy(input *HarnessInput, compiler HarnessCompiler, extension egressext.PolicyTranslator) (ResolvedPolicy, error) { + models := policyModels(input) + requirements := make(map[*ResolvedModelConfig]ModelRequirements, len(models)) + if compiler != nil { + for _, model := range models { + requirements[model] = compiler.ModelRequirements(model) + } + } + draft, err := policyDraft(input, models, requirements) + if err != nil { + return ResolvedPolicy{}, err + } + original := draft + if extension != nil { + draft, err = extension.Translate(types.NamespacedName{Namespace: input.Harness.Namespace, Name: input.AgentName}, clonePolicyDraft(draft)) + if err != nil { + return ResolvedPolicy{}, fmt.Errorf("failed to translate egress policy: %w", err) + } + } + access, err := validatePolicyDraft(original, draft) + if err != nil { + return ResolvedPolicy{}, err + } + result := ResolvedPolicy{ + access: access, modelBindings: map[*ResolvedModelConfig]bool{}, secretModels: map[string]bool{}, + secretMCP: secretMCPRefs(input.Root), headers: clientCredentialHeaders(original, access), + } + for _, model := range models { + if err := result.resolveModel(model, requirements[model].CredentialRequired); err != nil { + return ResolvedPolicy{}, err + } + } + return result, nil +} + +// secretMCPRefs lists the Secret keys that MCP headers may reference. +func secretMCPRefs(root *AgentInput) map[string]bool { + refs := map[string]bool{} + for _, agent := range policyAgents(root) { + for _, tool := range agent.MCPTools { + for _, ref := range tool.Server.Spec.HeadersFrom { + if isSecretHeader(ref) { + refs[ref.ValueFrom.Name+"\x00"+ref.ValueFrom.Key] = true + } + } + } + } + return refs +} + +// clientCredentialHeaders excludes origins that preserve explicit authentication. +func clientCredentialHeaders(original egressext.PolicyDraft, access egress.Access) map[string][]string { + preserved := map[string]bool{} + for _, requirement := range original.Requirements { + preserved[requirement.Origin] = preserved[requirement.Origin] || requirement.Preserve + } + headers := map[string][]string{} + for _, binding := range access.Credentials { + if !preserved[binding.Origin] { + headers[binding.Origin] = append(headers[binding.Origin], binding.Header) + } + } + return headers +} + +// resolveModel records whether a model client authenticates through the gateway. +func (p *ResolvedPolicy) resolveModel(resolved *ResolvedModelConfig, required bool) error { + spec := resolved.Config.Spec + target := modelCredentialTarget(resolved) + if target.Environment == "" { + return nil + } + origin, err := credentialOrigin(target.Endpoint) + if err != nil { + return NewValidationError("invalid credential destination: %v", err) + } + bound := slices.ContainsFunc(p.access.Credentials, bindsTarget(origin, target.Header, target.Prefix)) + switch { + case spec.APIKeyPassthrough: + if slices.ContainsFunc(p.access.Credentials, func(binding egress.Credential) bool { return binding.Origin == origin }) { + return NewValidationError("destination %q cannot combine caller-token passthrough with gateway credentials", origin) + } + p.tokenPassthroughOrigins = append(p.tokenPassthroughOrigins, origin) + case spec.APIKeySecret != "": + if bound { + p.modelBindings[resolved] = true + p.secretModels[target.Environment+"\x00"+spec.APIKeySecret+"\x00"+modelSecretKey(resolved)] = true + } + case required && !bound: + return NewValidationError("model %q requires apiKeySecret or a resolved gateway credential for %s", resolved.Config.Name, origin) + // Optional SDK authentication uses bindings only at unpreserved origins. + case bound && (required || slices.Contains(p.headers[origin], target.Header)): + p.modelBindings[resolved] = true + p.modelKeys = append(p.modelKeys, target.Environment) + } + return nil +} + +// draftBuilder collects application destinations, bindings and authentication requirements. +type draftBuilder struct { + destinations []string + bindings []egress.Credential + requirements []egressext.AuthenticationRequirement +} + +func (b *draftBuilder) allow(raw string) (string, error) { + origin, err := credentialOrigin(raw) + if err != nil { + return "", err + } + b.destinations = append(b.destinations, origin) + return origin, nil +} + +// allowTarget allows an origin that can carry credentials. Only exact HTTPS origins get a requirement. +func (b *draftBuilder) allowTarget(raw string, requirement egressext.AuthenticationRequirement) (string, error) { + origin, err := b.allow(raw) + if err != nil { + return "", err + } + if strings.HasPrefix(origin, "https://") && !strings.Contains(origin, "*") { + requirement.Origin = origin + b.requirements = append(b.requirements, requirement) + } + return origin, nil +} + +func (b *draftBuilder) addModel(model *ResolvedModelConfig, harnessEnv []v1alpha3.RuntimeEnvVar, requirements ModelRequirements) error { + for _, endpoint := range modelEndpoints(model) { + if _, err := b.allow(endpoint); err != nil { + return NewValidationError("invalid model destination: %v", err) + } + } + b.destinations = append(b.destinations, requirements.Destinations...) + target := modelCredentialTarget(model) + if target.Environment == "" { + return nil + } + spec := model.Config.Spec + // A literal Harness value overrides the model Secret in the rendered environment. + literal := slices.ContainsFunc(harnessEnv, func(variable v1alpha3.RuntimeEnvVar) bool { return variable.Name == target.Environment }) + signing := spec.Provider == v1alpha3.ModelProviderBedrock && slices.ContainsFunc(harnessEnv, func(variable v1alpha3.RuntimeEnvVar) bool { + return variable.Name == env.AWSAccessKeyID.Name() || variable.Name == env.AWSSecretAccessKey.Name() || variable.Name == env.AWSSessionToken.Name() + }) + preserve := literal || signing || modelPreservesAuth(spec) + origin, err := b.allowTarget(target.Endpoint, egressext.AuthenticationRequirement{ + Header: target.Header, Prefix: target.Prefix, Preserve: preserve, + Required: requirements.CredentialRequired && spec.APIKeySecret == "" && !preserve, + }) + if err != nil { + return NewValidationError("invalid model credential destination: %v", err) + } + if spec.APIKeySecret == "" || spec.APIKeyPassthrough || literal { + return nil + } + key := modelSecretKey(model) + if spec.Provider == v1alpha3.ModelProviderBedrock && key == "" { + return nil + } + b.bindings = append(b.bindings, egress.Credential{Origin: origin, Header: target.Header, Prefix: target.Prefix, URI: secretURI(model.Config.Namespace, spec.APIKeySecret, key)}) + return nil +} + +// modelPreservesAuth reports explicit model authentication that new bindings must not override. +// Keyless Gemini is included until its runtime can use a gateway binding. +func modelPreservesAuth(spec v1alpha3.ModelConfigSpec) bool { + keylessGemini := spec.Provider == v1alpha3.ModelProviderGemini && spec.APIKeySecret == "" + return spec.APIKeyPassthrough || len(spec.DefaultHeaders) != 0 || keylessGemini +} + +func (b *draftBuilder) addMCPServer(server *v1alpha3.RemoteMCPServer) error { + // Literal and ConfigMap headers can carry authentication. + preserve := slices.ContainsFunc(server.Spec.HeadersFrom, func(ref v1alpha3.ValueRef) bool { return !isSecretHeader(ref) }) + origin, err := b.allowTarget(server.Spec.URL, egressext.AuthenticationRequirement{Preserve: preserve}) + if err != nil { + return NewValidationError("invalid MCP credential destination: %v", err) + } + for _, ref := range server.Spec.HeadersFrom { + if isSecretHeader(ref) { + b.bindings = append(b.bindings, egress.Credential{Origin: origin, Header: ref.Name, URI: secretURI(server.Namespace, ref.ValueFrom.Name, ref.ValueFrom.Key)}) + } + } + return nil +} + +func policyDraft(input *HarnessInput, models []*ResolvedModelConfig, requirementsByModel map[*ResolvedModelConfig]ModelRequirements) (egressext.PolicyDraft, error) { + var b draftBuilder + for _, model := range models { + if err := b.addModel(model, input.Harness.Spec.Env, requirementsByModel[model]); err != nil { + return egressext.PolicyDraft{}, err + } + } + for _, agent := range policyAgents(input.Root) { + for _, tool := range agent.MCPTools { + if err := b.addMCPServer(tool.Server); err != nil { + return egressext.PolicyDraft{}, err + } + } + } + access, err := egress.CanonicalAccess(b.destinations, b.bindings) + if err != nil { + return egressext.PolicyDraft{}, NewValidationError("%v", err) + } + return egressext.PolicyDraft{Destinations: access.Destinations, Requirements: mergeRequirements(b.requirements), Credentials: publicCredentials(access.Credentials)}, nil +} + +// mergeRequirements applies preservation to a whole origin, including overlapping +// resource references, and merges requirements that share a header format. +func mergeRequirements(requirements []egressext.AuthenticationRequirement) []egressext.AuthenticationRequirement { + merged := slices.Clone(requirements) + for i := range merged { + for _, other := range requirements { + if other.Origin != merged[i].Origin { + continue + } + merged[i].Preserve = merged[i].Preserve || other.Preserve + if other.Header == merged[i].Header && other.Prefix == merged[i].Prefix { + merged[i].Required = merged[i].Required || other.Required + } + } + } + slices.SortFunc(merged, func(a, b egressext.AuthenticationRequirement) int { + return cmp.Or(strings.Compare(a.Origin, b.Origin), strings.Compare(a.Header, b.Header), strings.Compare(a.Prefix, b.Prefix)) + }) + return slices.Compact(merged) +} + +func clonePolicyDraft(draft egressext.PolicyDraft) egressext.PolicyDraft { + return egressext.PolicyDraft{Destinations: slices.Clone(draft.Destinations), Requirements: slices.Clone(draft.Requirements), Credentials: slices.Clone(draft.Credentials)} +} + +func validatePolicyDraft(original, translated egressext.PolicyDraft) (egress.Access, error) { + if !slices.Equal(original.Destinations, translated.Destinations) || !slices.Equal(original.Requirements, translated.Requirements) { + return egress.Access{}, NewValidationError("policy extension cannot change destinations or authentication requirements") + } + access, err := egress.CanonicalAccess(translated.Destinations, internalCredentials(translated.Credentials)) + if err != nil { + return egress.Access{}, NewValidationError("%v", err) + } + for _, binding := range original.Credentials { + if !slices.Contains(access.Credentials, egress.Credential(binding)) { + return egress.Access{}, NewValidationError("policy extension cannot remove or replace existing credentials") + } + } + for _, binding := range access.Credentials { + if slices.Contains(original.Credentials, egressext.Credential(binding)) { + continue + } + if err := validateAddedCredential(binding, original.Requirements); err != nil { + return egress.Access{}, err + } + } + if err := requireCredentials(original.Requirements, access.Credentials); err != nil { + return egress.Access{}, err + } + return access, nil +} + +// validateAddedCredential checks an extension binding against the requirements at its origin. +func validateAddedCredential(binding egress.Credential, requirements []egressext.AuthenticationRequirement) error { + allowed, known, matches := false, false, false + for _, requirement := range requirements { + if requirement.Origin != binding.Origin { + continue + } + if requirement.Preserve { + return NewValidationError("credential destination %q preserves explicit authentication", binding.Origin) + } + allowed = true + if requirement.Header != "" { + known = true + matches = matches || bindsTarget(requirement.Origin, requirement.Header, requirement.Prefix)(binding) + } + } + if !allowed { + return NewValidationError("credential destination %q is not an eligible application target", binding.Origin) + } + if known && !matches { + return NewValidationError("credential for %q must match a model header and prefix", binding.Origin) + } + return nil +} + +// requireCredentials checks that every required, unpreserved requirement has a binding. +func requireCredentials(requirements []egressext.AuthenticationRequirement, credentials []egress.Credential) error { + for _, requirement := range requirements { + if !requirement.Required || requirement.Preserve { + continue + } + if !slices.ContainsFunc(credentials, bindsTarget(requirement.Origin, requirement.Header, requirement.Prefix)) { + return NewValidationError("model requires a gateway credential for %s header %s; configure apiKeySecret", requirement.Origin, requirement.Header) + } + } + return nil +} diff --git a/go/core/internal/translator/egress_policy_runtime_test.go b/go/core/internal/translator/egress_policy_runtime_test.go new file mode 100644 index 0000000000..f51ba73731 --- /dev/null +++ b/go/core/internal/translator/egress_policy_runtime_test.go @@ -0,0 +1,503 @@ +package translator_test + +import ( + "encoding/json" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + runtimeconfig "github.com/kagent-dev/kagent/go/adk/pkg/config" + configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/translator" + byotranslator "github.com/kagent-dev/kagent/go/core/internal/translator/byo" + claudetranslator "github.com/kagent-dev/kagent/go/core/internal/translator/claude" + codextranslator "github.com/kagent-dev/kagent/go/core/internal/translator/codex" + kagenttranslator "github.com/kagent-dev/kagent/go/core/internal/translator/kagent" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" + "github.com/kagent-dev/kagent/go/core/pkg/env" + claudeconfig "github.com/kagent-dev/kagent/go/harness/claude/config" + codexconfig "github.com/kagent-dev/kagent/go/harness/codex/config" +) + +// This consumer uses only the public extension contract. +type defaultPolicyTranslator struct{ uri string } + +var _ egressext.PolicyTranslator = defaultPolicyTranslator{} + +func (d defaultPolicyTranslator) Translate(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + if d.uri == "" { + return draft, nil + } + for _, target := range draft.Requirements { + if target.Preserve || slices.ContainsFunc(draft.Credentials, func(c egressext.Credential) bool { return c.Origin == target.Origin }) { + continue + } + header, prefix := target.Header, target.Prefix + // A generic requirement must not override the model format at a shared origin. + if header == "" { + if slices.ContainsFunc(draft.Requirements, func(r egressext.AuthenticationRequirement) bool { return r.Origin == target.Origin && r.Header != "" }) { + continue + } + header, prefix = "authorization", "Bearer " + } + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: target.Origin, Header: header, Prefix: prefix, URI: d.uri}) + } + return draft, nil +} + +type agentPolicyTranslator map[types.NamespacedName]string + +var _ egressext.PolicyTranslator = agentPolicyTranslator(nil) + +func (a agentPolicyTranslator) Translate(agent types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + return defaultPolicyTranslator{uri: a[agent]}.Translate(agent, draft) +} + +type credentialPolicyTranslator egressext.Credential + +var _ egressext.PolicyTranslator = credentialPolicyTranslator{} + +func (c credentialPolicyTranslator) Translate(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + draft.Credentials = append(draft.Credentials, egressext.Credential(c)) + return draft, nil +} + +type recordingPolicyTranslator struct{ draft egressext.PolicyDraft } + +func (r *recordingPolicyTranslator) Translate(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + r.draft = draft + return draft, nil +} + +func TestPolicyDraftMatchesPersistedApplicationGrant(t *testing.T) { + for _, test := range []struct { + kind translator.HarnessType + provider v1alpha3.ModelProvider + retainedDefault string + }{ + {translator.HarnessTypeKagent, v1alpha3.ModelProviderOpenAI, "https://api.openai.com:443"}, + {translator.HarnessTypeBYO, v1alpha3.ModelProviderOpenAI, "https://api.openai.com:443"}, + {translator.HarnessTypeKagent, v1alpha3.ModelProviderAnthropic, "https://api.anthropic.com:443"}, + {translator.HarnessTypeBYO, v1alpha3.ModelProviderAnthropic, "https://api.anthropic.com:443"}, + {translator.HarnessTypeClaude, v1alpha3.ModelProviderAnthropic, ""}, + {translator.HarnessTypeCodex, v1alpha3.ModelProviderOpenAI, ""}, + } { + t.Run(string(test.kind)+"/"+string(test.provider), func(t *testing.T) { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ + Workload: v1alpha3.HarnessWorkload{Image: new("example.com/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + switch test.kind { + case translator.HarnessTypeKagent: + harness.Spec.Kagent = &v1alpha3.KagentHarness{} + case translator.HarnessTypeBYO: + harness.Spec.BYO = &v1alpha3.BYOHarness{} + case translator.HarnessTypeClaude: + harness.Spec.Claude = &v1alpha3.ClaudeHarness{} + case translator.HarnessTypeCodex: + harness.Spec.Codex = &v1alpha3.CodexHarness{} + } + model := modelConfig() + model.Spec.Provider = test.provider + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = "model-auth", "token" + if test.provider == v1alpha3.ModelProviderOpenAI { + model.Spec.OpenAI = &v1alpha3.OpenAIConfig{BaseURL: "https://models.example.com/v1", APIFormat: new(v1alpha3.OpenAIAPIFormatResponses)} + } else { + model.Spec.Anthropic = &v1alpha3.AnthropicConfig{BaseURL: "https://models.example.com/v1"} + } + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "model-auth", Namespace: "test"}, Data: map[string][]byte{"token": []byte("secret")}} + server := remoteMCPServer("tools", "https://tools.example.com/mcp") + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help", + Tools: []v1alpha3.ToolBinding{{MCP: &v1alpha3.MCPToolBinding{Server: corev1.TypedLocalObjectReference{Kind: "RemoteMCPServer", Name: server.Name}}}}, + }} + collections := mockCollections(t, harness, model, server, secret, defaultWorkerPool()) + ctx := krt.TestingDummyContext{} + settings := testRuntimeDefaults() + adapters := map[translator.HarnessType]translator.HarnessCompiler{ + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings), + } + recorder := &recordingPolicyTranslator{} + result, err := translator.NewCompiler(ctx, collections, adapters, nil, recorder).CompileAgent(t.Context(), inlineAgent(harness, template)) + require.NoError(t, err) + want := []string{"https://models.example.com:443", "https://tools.example.com:443"} + if test.retainedDefault != "" { + want = append(want, test.retainedDefault) + } + slices.Sort(want) + require.Equal(t, want, recorder.draft.Destinations) + for _, requirement := range recorder.draft.Requirements { + require.NotEqual(t, test.retainedDefault, requirement.Origin, "retaining a destination must not make it a credential target") + } + want = append(want, settings.GetControlPlaneUrl()) + slices.Sort(want) + require.Equal(t, want, result.EgressDestinations) + require.Len(t, result.Credentials, 1) + require.Equal(t, recorder.draft.Credentials[0].URI, result.Credentials[0].URI) + }) + } +} + +func TestPolicyUsesRunnableAgentIdentity(t *testing.T) { + policyTranslator := agentPolicyTranslator{ + {Namespace: "team-a", Name: "researcher"}: "ate-secret://provider.example/team-a/researcher", + {Namespace: "team-a", Name: "reviewer"}: "ate-secret://provider.example/team-a/reviewer", + {Namespace: "team-b", Name: "researcher"}: "ate-secret://provider.example/team-b/researcher", + {Namespace: "team-b", Name: "reviewer"}: "ate-secret://provider.example/team-b/reviewer", + } + for _, namespace := range []string{"team-a", "team-b"} { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: namespace}, Spec: v1alpha3.HarnessSpec{ + Kagent: &v1alpha3.KagentHarness{}, Workload: v1alpha3.HarnessWorkload{Image: new("example.com/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + model := modelConfig() + model.Namespace = namespace + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "shared-behavior", Namespace: namespace}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help", + }} + pool := defaultWorkerPool() + pool.Namespace = namespace + collections := mockCollections(t, harness, model, template, pool) + settings := testRuntimeDefaults() + ctx := krt.TestingDummyContext{} + compiler := translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), + }, nil, policyTranslator) + for _, source := range []string{"inline", "reference"} { + for _, name := range []string{"researcher", "reviewer"} { + t.Run(namespace+"/"+source+"/"+name, func(t *testing.T) { + agent := inlineAgent(harness, template) + agent.Name = name + if source == "reference" { + agent.Spec.Template = v1alpha3.AgentTemplateSource{Ref: &corev1.LocalObjectReference{Name: template.Name}} + } + result, err := compiler.CompileAgent(t.Context(), agent) + require.NoError(t, err) + require.Len(t, result.Credentials, 1) + require.Equal(t, policyTranslator[types.NamespacedName{Namespace: namespace, Name: name}], result.Credentials[0].URI) + }) + } + } + } +} + +func TestCompileContributedCredentialsAcrossHarnesses(t *testing.T) { + for _, test := range []struct { + kind translator.HarnessType + provider v1alpha3.ModelProvider + }{ + {translator.HarnessTypeKagent, v1alpha3.ModelProviderOpenAI}, + {translator.HarnessTypeBYO, v1alpha3.ModelProviderOpenAI}, + {translator.HarnessTypeClaude, v1alpha3.ModelProviderAnthropic}, + {translator.HarnessTypeCodex, v1alpha3.ModelProviderOpenAI}, + {translator.HarnessTypeKagent, v1alpha3.ModelProviderBedrock}, + {translator.HarnessTypeBYO, v1alpha3.ModelProviderBedrock}, + {translator.HarnessTypeClaude, v1alpha3.ModelProviderBedrock}, + {translator.HarnessTypeCodex, v1alpha3.ModelProviderBedrock}, + } { + kind := test.kind + t.Run(string(kind)+"/"+string(test.provider), func(t *testing.T) { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ + Workload: v1alpha3.HarnessWorkload{Image: new("example.com/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + model := modelConfig() + key := "OPENAI_API_KEY" + switch kind { + case translator.HarnessTypeKagent: + harness.Spec.Kagent = &v1alpha3.KagentHarness{} + case translator.HarnessTypeBYO: + harness.Spec.BYO = &v1alpha3.BYOHarness{} + case translator.HarnessTypeClaude: + harness.Spec.Claude = &v1alpha3.ClaudeHarness{} + model.Spec.Provider = v1alpha3.ModelProviderAnthropic + key = "ANTHROPIC_API_KEY" + case translator.HarnessTypeCodex: + harness.Spec.Codex = &v1alpha3.CodexHarness{} + model.Spec.OpenAI = &v1alpha3.OpenAIConfig{APIFormat: new(v1alpha3.OpenAIAPIFormatResponses)} + } + model.Spec.Provider = test.provider + switch test.provider { + case v1alpha3.ModelProviderBedrock: + model.Spec.Model = "gpt-5.2" + model.Spec.OpenAI = nil + model.Spec.Bedrock = &v1alpha3.BedrockConfig{Region: "us-east-1"} + key = "AWS_BEARER_TOKEN_BEDROCK" + } + server := &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Name: "tools", Namespace: "test"}, Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://tools.example.com/mcp"}} + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help", Tools: []v1alpha3.ToolBinding{{MCP: &v1alpha3.MCPToolBinding{Server: corev1.TypedLocalObjectReference{Kind: "RemoteMCPServer", Name: server.Name}}}}, + }} + agent := inlineAgent(harness, template) + collections := mockCollections(t, harness, model, server, defaultWorkerPool()) + compile := func(policyTranslator egressext.PolicyTranslator) (*translator.CompileResult, error) { + settings := testRuntimeDefaults() + ctx := krt.TestingDummyContext{} + compilers := map[translator.HarnessType]translator.HarnessCompiler{ + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings), + } + return translator.NewCompiler(ctx, collections, compilers, nil, policyTranslator).CompileAgent(t.Context(), agent) + } + result, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/default/token"}) + require.NoError(t, err) + require.Len(t, result.Credentials, 2) + require.Contains(t, result.Environment, corev1.EnvVar{Name: key, Value: translator.CredentialPlaceholder}) + for _, variable := range result.Environment { + require.Nil(t, variable.ValueFrom) + } + require.Equal(t, translator.CredentialPlaceholder, compiledMCPHeaders(t, kind, result)["authorization"]) + first, err := result.Digest() + require.NoError(t, err) + again, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/default/token"}) + require.NoError(t, err) + same, err := again.Digest() + require.NoError(t, err) + require.Equal(t, first, same) + changed, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/default/next"}) + require.NoError(t, err) + second, err := changed.Digest() + require.NoError(t, err) + require.NotEqual(t, first, second) + require.Equal(t, result.EgressDestinations, changed.EgressDestinations) + if kind == translator.HarnessTypeKagent || kind == translator.HarnessTypeBYO { + baseline, err := compile(nil) + require.NoError(t, err) + require.Equal(t, baseline.EgressDestinations, result.EgressDestinations) + require.Empty(t, baseline.Credentials) + noop, err := compile(defaultPolicyTranslator{}) + require.NoError(t, err) + require.Equal(t, baseline.Revision, noop.Revision) + nilDigest, err := baseline.Digest() + require.NoError(t, err) + noopDigest, err := noop.Digest() + require.NoError(t, err) + require.Equal(t, nilDigest, noopDigest) + require.NotContains(t, string(baseline.ConfigJSON), translator.CredentialPlaceholder) + } else { + _, err := compile(nil) + require.ErrorContains(t, err, "configure apiKeySecret") + _, err = compile(defaultPolicyTranslator{}) + require.ErrorContains(t, err, "requires a gateway credential") + } + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = "model-auth", "token" + secretKey := "token" + if test.provider == v1alpha3.ModelProviderBedrock { + model.Spec.APIKeySecretKey, secretKey = "", "AWS_BEARER_TOKEN_BEDROCK" + } + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "model-auth", Namespace: "test"}, Data: map[string][]byte{secretKey: []byte("model-secret")}} + collections = mockCollections(t, harness, model, server, defaultWorkerPool(), secret) + baseline, err := compile(nil) + require.NoError(t, err) + if test.provider == v1alpha3.ModelProviderBedrock { + require.Equal(t, []egress.Credential{{ + Origin: "https://bedrock-runtime.us-east-1.amazonaws.com:443", Header: "authorization", Prefix: "Bearer ", + URI: "ate-secret://k8s.io/default/test/model-auth/AWS_BEARER_TOKEN_BEDROCK", + }}, baseline.Credentials) + } + noop, err := compile(defaultPolicyTranslator{}) + require.NoError(t, err) + require.Equal(t, baseline.Revision, noop.Revision) + defaultDigest, err := baseline.Digest() + require.NoError(t, err) + noopDigest, err := noop.Digest() + require.NoError(t, err) + require.Equal(t, defaultDigest, noopDigest) + withDefault, err := compile(defaultPolicyTranslator{uri: "ate-secret://provider.example/default/token"}) + require.NoError(t, err) + require.Contains(t, withDefault.Credentials, baseline.Credentials[0], "default credentials preserve explicit model authentication") + + t.Run("MCP Secret and extension", func(t *testing.T) { + server.Spec.HeadersFrom = []v1alpha3.ValueRef{{Name: "X-API-Key", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "service-auth", Key: "key"}}} + serviceSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "service-auth", Namespace: "test"}, Data: map[string][]byte{"key": []byte("service-secret")}} + collections = mockCollections(t, harness, model, server, defaultWorkerPool(), secret, serviceSecret) + combined, err := compile(credentialPolicyTranslator{ + Origin: "https://tools.example.com:443", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/default/token", + }) + require.NoError(t, err) + require.Len(t, combined.Credentials, 3) + require.Contains(t, combined.Credentials, egress.Credential{ + Origin: "https://tools.example.com:443", Header: "x-api-key", URI: "ate-secret://k8s.io/default/test/service-auth/key", + }) + require.Contains(t, combined.Credentials, egress.Credential{ + Origin: "https://tools.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/default/token", + }) + prefix := "KAGENT_CREDENTIAL_" + if kind == translator.HarnessTypeClaude || kind == translator.HarnessTypeCodex { + prefix = "KAGENT_" + strings.ToUpper(string(kind)) + "_MCP_CREDENTIAL_" + } + var secretHeader string + for _, variable := range combined.Environment { + require.Nil(t, variable.ValueFrom) + if strings.HasPrefix(variable.Name, prefix) { + require.Equal(t, translator.CredentialPlaceholder, variable.Value) + secretHeader = "${" + variable.Name + "}" + if kind == translator.HarnessTypeKagent || kind == translator.HarnessTypeBYO { + secretHeader = "__KAGENT_ENV[" + variable.Name + "]__" + } + } + } + require.NotEmpty(t, secretHeader) + require.Equal(t, map[string]string{ + "X-API-Key": secretHeader, "authorization": translator.CredentialPlaceholder, + }, compiledMCPHeaders(t, kind, combined)) + }) + }) + } +} + +func compiledMCPHeaders(t *testing.T, kind translator.HarnessType, result *translator.CompileResult) map[string]string { + t.Helper() + switch kind { + case translator.HarnessTypeKagent, translator.HarnessTypeBYO: + var config configv1.ADKConfig + require.NoError(t, runtimeconfig.Unmarshal(result.ConfigJSON, &config)) + return config.Agent.HttpTools[0].Params.Headers + case translator.HarnessTypeClaude: + var config claudeconfig.Config + require.NoError(t, json.Unmarshal(result.ConfigJSON, &config)) + return config.MCPServers["tools"].Headers + case translator.HarnessTypeCodex: + var config codexconfig.Config + require.NoError(t, json.Unmarshal(result.ConfigJSON, &config)) + return config.MCPServers["tools"].Headers + default: + t.Fatalf("unsupported harness %q", kind) + return nil + } +} + +func TestPolicyResolvesAuxiliaryModelsBeforeHarnessRendering(t *testing.T) { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ + Kagent: &v1alpha3.KagentHarness{ + Memory: &v1alpha3.KagentHarnessMemory{ModelConfigRef: corev1.LocalObjectReference{Name: "embedding"}}, + Compaction: &v1alpha3.KagentHarnessCompaction{Summarizer: &v1alpha3.KagentHarnessSummarizer{ModelConfigRef: &corev1.LocalObjectReference{Name: "summary"}}}, + }, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}}, + }} + root, embedding, summary := modelConfig(), modelConfig(), modelConfig() + embedding.Name, summary.Name = "embedding", "summary" + embedding.Spec.OpenAI = &v1alpha3.OpenAIConfig{BaseURL: "https://embeddings.example.com/v1"} + summary.Spec.OpenAI = &v1alpha3.OpenAIConfig{BaseURL: "https://summary.example.com/v1"} + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ModelConfig: &corev1.LocalObjectReference{Name: root.Name}}} + collections := mockCollections(t, harness, root, embedding, summary, defaultWorkerPool()) + adapter := &testHarnessCompiler{} + compiler := translator.NewCompiler(krt.TestingDummyContext{}, collections, map[translator.HarnessType]translator.HarnessCompiler{translator.HarnessTypeKagent: adapter}, testHarnessImages, defaultPolicyTranslator{uri: "ate-secret://provider.example/default/token"}) + result, err := compiler.CompileAgent(t.Context(), inlineAgent(harness, template)) + require.NoError(t, err) + require.NotNil(t, adapter.input.SummarizerModel) + require.NotNil(t, adapter.input.MemoryModel) + require.True(t, adapter.input.Policy.ModelUsesGateway(adapter.input.Root.ResolvedModelConfig), "authentication must be resolved when rendering starts") + require.True(t, adapter.input.Policy.ModelUsesGateway(adapter.input.SummarizerModel)) + require.True(t, adapter.input.Policy.ModelUsesGateway(adapter.input.MemoryModel)) + require.Len(t, result.Credentials, 3) + require.Equal(t, []string{"https://api.openai.com:443", "https://embeddings.example.com:443", "https://summary.example.com:443"}, result.EgressDestinations) +} + +func TestHarnessLiteralKeyOverridesModelSecretBinding(t *testing.T) { + for _, test := range []struct { + name, env string + model v1alpha3.ModelConfigSpec + secret map[string][]byte + }{ + {"bedrock", "AWS_BEARER_TOKEN_BEDROCK", + v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderBedrock, Model: "claude", Bedrock: &v1alpha3.BedrockConfig{Region: "us-east-1"}, APIKeySecret: "model-auth"}, + map[string][]byte{"AWS_BEARER_TOKEN_BEDROCK": []byte("secret")}}, + {"openai", "OPENAI_API_KEY", + v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, Model: "gpt-4o", APIKeySecret: "model-auth", APIKeySecretKey: "token"}, + map[string][]byte{"token": []byte("secret")}}, + } { + t.Run(test.name, func(t *testing.T) { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ + Kagent: &v1alpha3.KagentHarness{}, + Env: []v1alpha3.RuntimeEnvVar{{Name: test.env, Value: "explicit"}}, + Workload: v1alpha3.HarnessWorkload{Image: new("example.com/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + model := modelConfig() + model.Spec = test.model + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "model-auth", Namespace: "test"}, Data: test.secret} + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help", + }} + agent := inlineAgent(harness, template) + collections := mockCollections(t, harness, model, secret, defaultWorkerPool()) + ctx := krt.TestingDummyContext{} + compilers := map[translator.HarnessType]translator.HarnessCompiler{ + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), + } + result, err := translator.NewCompiler(ctx, collections, compilers, nil, nil).CompileAgent(t.Context(), agent) + require.NoError(t, err) + require.Contains(t, result.Environment, corev1.EnvVar{Name: test.env, Value: "explicit"}) + require.Empty(t, result.Credentials, "the gateway must not replace an explicit Harness key") + }) + } +} + +func TestHarnessBedrockSigningCredentialsPreserved(t *testing.T) { + for _, kind := range []translator.HarnessType{translator.HarnessTypeKagent, translator.HarnessTypeBYO} { + for _, test := range []struct { + name string + env []v1alpha3.RuntimeEnvVar + bearer bool + }{ + {name: "signing pair", env: []v1alpha3.RuntimeEnvVar{{Name: env.AWSAccessKeyID.Name(), Value: "access"}, {Name: env.AWSSecretAccessKey.Name(), Value: "secret"}}}, + {name: "empty access key", env: []v1alpha3.RuntimeEnvVar{{Name: env.AWSAccessKeyID.Name()}}}, + {name: "empty secret key", env: []v1alpha3.RuntimeEnvVar{{Name: env.AWSSecretAccessKey.Name()}}}, + {name: "empty session token", env: []v1alpha3.RuntimeEnvVar{{Name: env.AWSSessionToken.Name()}}}, + {name: "bearer Secret and signing key", env: []v1alpha3.RuntimeEnvVar{{Name: env.AWSAccessKeyID.Name(), Value: "access"}}, bearer: true}, + } { + t.Run(string(kind)+"/"+test.name, func(t *testing.T) { + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: "test"}, Spec: v1alpha3.HarnessSpec{ + Env: test.env, Workload: v1alpha3.HarnessWorkload{Image: new("example.com/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + if kind == translator.HarnessTypeKagent { + harness.Spec.Kagent = &v1alpha3.KagentHarness{} + } else { + harness.Spec.BYO = &v1alpha3.BYOHarness{} + } + model := modelConfig() + model.Spec = v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderBedrock, Model: "claude", Bedrock: &v1alpha3.BedrockConfig{Region: "us-east-1"}} + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "model-auth", Namespace: "test"}, Data: map[string][]byte{env.AWSBearerTokenBedrock.Name(): []byte("bearer")}} + if test.bearer { + model.Spec.APIKeySecret = secret.Name + } + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Name: "assistant", Namespace: "test"}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help", + }} + collections := mockCollections(t, harness, model, secret, defaultWorkerPool()) + ctx := krt.TestingDummyContext{} + compilers := map[translator.HarnessType]translator.HarnessCompiler{ + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), + translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, testRuntimeDefaults()), + } + result, err := translator.NewCompiler(ctx, collections, compilers, nil, defaultPolicyTranslator{uri: "ate-secret://provider.example/default/token"}).CompileAgent(t.Context(), inlineAgent(harness, template)) + require.NoError(t, err) + for _, variable := range test.env { + require.Contains(t, result.Environment, corev1.EnvVar{Name: variable.Name, Value: variable.Value}) + } + if test.bearer { + require.Equal(t, []egress.Credential{{Origin: "https://bedrock-runtime.us-east-1.amazonaws.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/test/model-auth/AWS_BEARER_TOKEN_BEDROCK"}}, result.Credentials) + require.Contains(t, result.Environment, corev1.EnvVar{Name: env.AWSBearerTokenBedrock.Name(), Value: translator.CredentialPlaceholder}) + } else { + require.Empty(t, result.Credentials) + require.False(t, slices.ContainsFunc(result.Environment, func(variable corev1.EnvVar) bool { return variable.Name == env.AWSBearerTokenBedrock.Name() })) + } + }) + } + } +} diff --git a/go/core/internal/translator/egress_policy_test.go b/go/core/internal/translator/egress_policy_test.go new file mode 100644 index 0000000000..3403fd727d --- /dev/null +++ b/go/core/internal/translator/egress_policy_test.go @@ -0,0 +1,378 @@ +package translator + +import ( + "context" + "errors" + "maps" + "slices" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" + "github.com/kagent-dev/kagent/go/core/pkg/env" +) + +type policyTranslatorFunc func(types.NamespacedName, egressext.PolicyDraft) (egressext.PolicyDraft, error) + +func (f policyTranslatorFunc) Translate(agent types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + return f(agent, draft) +} + +var _ egressext.PolicyTranslator = policyTranslatorFunc(nil) + +func TestPolicyDraftTargets(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "model-key", APIKeySecretKey: "token"}) + input.Root.MCPTools = []ResolvedMCPTool{ + {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://MCP.Example.com./tools"}}}, + {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "http://local.example.com/tools"}}}, + } + input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAnthropic}).Root}} + extension := policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + require.Len(t, draft.Credentials, 1) + require.Contains(t, draft.Requirements, egressext.AuthenticationRequirement{Origin: "https://api.anthropic.com:443", Header: "x-api-key"}) + require.Contains(t, draft.Requirements, egressext.AuthenticationRequirement{Origin: "https://mcp.example.com:443"}) + require.Contains(t, draft.Destinations, "http://local.example.com:80") + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://api.anthropic.com", Header: "x-api-key", URI: "ate-secret://provider.example/default/token"}) + return draft, nil + }) + policy, err := ResolvePolicy(input, nil, extension) + require.NoError(t, err) + require.Len(t, policy.access.Credentials, 2) + rendered, err := policy.RenderEnvironment([]corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "model-key", "token")}) + require.NoError(t, err) + require.Contains(t, rendered, corev1.EnvVar{Name: "ANTHROPIC_API_KEY", Value: CredentialPlaceholder}) + require.Equal(t, "ate-secret://k8s.io/default/team/model-key/token", policy.access.Credentials[1].URI) +} + +func TestPolicyPreservesExplicitAuthentication(t *testing.T) { + for _, test := range []struct { + name string + configure func(*HarnessInput) + }{ + {"passthrough", func(input *HarnessInput) { input.Root.ResolvedModelConfig.Config.Spec.APIKeyPassthrough = true }}, + {"model headers", func(input *HarnessInput) { + input.Root.ResolvedModelConfig.Config.Spec.DefaultHeaders = map[string]string{"Authorization": "literal"} + }}, + {"harness key", func(input *HarnessInput) { + input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OPENAI_API_KEY", Value: "literal"}} + }}, + {"Bedrock signing key", func(input *HarnessInput) { + input.Root.ResolvedModelConfig.Config.Spec = v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderBedrock, Bedrock: &v1alpha3.BedrockConfig{Region: "us-east-1"}} + input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: env.AWSAccessKeyID.Name()}} + }}, + {"literal MCP header", func(input *HarnessInput) { + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.openai.com/mcp", HeadersFrom: []v1alpha3.ValueRef{{Name: "X-Trace", Value: "literal"}}}}}} + }}, + {"ConfigMap MCP header", func(input *HarnessInput) { + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.openai.com/mcp", HeadersFrom: []v1alpha3.ValueRef{{Name: "Authorization", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.ConfigMapValueSource, Name: "auth", Key: "token"}}}}}}} + }}, + } { + t.Run(test.name, func(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + test.configure(input) + target := modelCredentialTarget(input.Root.ResolvedModelConfig) + _, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + for _, requirement := range draft.Requirements { + require.True(t, requirement.Preserve) + } + return draft, nil + })) + require.NoError(t, err) + _, err = ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: target.Endpoint, Header: target.Header, Prefix: target.Prefix, URI: "ate-secret://provider.example/default/token"}) + return draft, nil + })) + require.ErrorContains(t, err, "preserves explicit authentication") + }) + } +} + +func TestPolicyRejectsInvalidBindings(t *testing.T) { + base := egressext.Credential{Origin: "https://api.openai.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://provider.example/default/token"} + for _, test := range []struct { + name string + change func(*egressext.Credential) + want string + }{ + {"unknown origin", func(c *egressext.Credential) { c.Origin = "https://unrelated.example.com" }, "not explicitly allowed"}, + {"different port", func(c *egressext.Credential) { c.Origin = "https://api.openai.com:8443" }, "not explicitly allowed"}, + {"HTTP", func(c *egressext.Credential) { c.Origin = "http://api.openai.com" }, "not explicitly allowed"}, + {"wildcard", func(c *egressext.Credential) { c.Origin = "https://*.openai.com" }, "exact HTTP(S) DNS origin"}, + {"routing header", func(c *egressext.Credential) { c.Header = "Host" }, "invalid credential injection header"}, + {"invalid prefix", func(c *egressext.Credential) { c.Prefix = "Bearer\n" }, "invalid credential injection header"}, + {"invalid URI", func(c *egressext.Credential) { c.URI = "https://provider.example/token" }, "invalid credential URI"}, + {"model header mismatch", func(c *egressext.Credential) { c.Header = "x-api-key" }, "must match a model header and prefix"}, + {"model prefix mismatch", func(c *egressext.Credential) { c.Prefix = "Basic " }, "must match a model header and prefix"}, + } { + t.Run(test.name, func(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.openai.com/mcp"}}}} + binding := base + test.change(&binding) + _, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + draft.Credentials = append(draft.Credentials, binding) + return draft, nil + })) + require.ErrorContains(t, err, test.want) + }) + } + failure := errors.New("policy unavailable") + _, err := ResolvePolicy(credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}), nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + return draft, failure + })) + require.ErrorIs(t, err, failure) +} + +func TestPolicyCannotWeakenDraft(t *testing.T) { + for _, test := range []struct { + name string + change func(*egressext.PolicyDraft) + want string + }{ + {"destinations", func(d *egressext.PolicyDraft) { d.Destinations[0] = "https://other.example.com:443" }, "cannot change destinations"}, + {"header", func(d *egressext.PolicyDraft) { d.Requirements[0].Header = "x-other" }, "cannot change destinations"}, + {"preserve", func(d *egressext.PolicyDraft) { d.Requirements[0].Preserve = true }, "cannot change destinations"}, + {"required", func(d *egressext.PolicyDraft) { d.Requirements[0].Required = true }, "cannot change destinations"}, + {"remove binding", func(d *egressext.PolicyDraft) { d.Credentials = nil }, "cannot remove or replace"}, + {"replace binding", func(d *egressext.PolicyDraft) { d.Credentials[0].URI = "ate-secret://provider.example/default/changed" }, "cannot remove or replace"}, + {"conflict", func(d *egressext.PolicyDraft) { + binding := d.Credentials[0] + binding.URI = "ate-secret://provider.example/default/changed" + d.Credentials = append(d.Credentials, binding) + }, "conflicting credentials"}, + } { + t.Run(test.name, func(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "auth", APIKeySecretKey: "token"}) + before, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + _, err = ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + test.change(&draft) + return draft, nil + })) + require.ErrorContains(t, err, test.want) + after, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + require.Equal(t, before, after, "extension mutation must not affect subsequent compilation") + }) + } +} + +func TestPolicyRejectsMCPSecretCredentialConflict(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team"}, + Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://tools.example.com/mcp", HeadersFrom: []v1alpha3.ValueRef{ + {Name: "X-API-Key", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "service-auth", Key: "key"}}, + }}, + }}} + before, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + _, err = ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + draft.Credentials = append(draft.Credentials, egressext.Credential{ + Origin: "https://TOOLS.example.com", Header: "x-api-key", URI: "ate-secret://provider.example/default/token", + }) + return draft, nil + })) + require.ErrorContains(t, err, "conflicting credentials") + after, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + require.Equal(t, before, after) + require.Len(t, after.access.Credentials, 1) + require.Equal(t, "ate-secret://k8s.io/default/team/service-auth/key", after.access.Credentials[0].URI) +} + +func TestPolicyAuxiliaryModels(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + input.MemoryModel = &ResolvedModelConfig{Config: &v1alpha3.ModelConfig{ObjectMeta: metav1.ObjectMeta{Name: "embedding", Namespace: "team"}, Spec: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://embeddings.example.com/v1"}}}} + input.SummarizerModel = &ResolvedModelConfig{Config: &v1alpha3.ModelConfig{ObjectMeta: metav1.ObjectMeta{Name: "summary", Namespace: "team"}, Spec: v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAnthropic}}} + _, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + require.Len(t, draft.Requirements, 3) + require.Contains(t, draft.Destinations, "https://embeddings.example.com:443") + require.Contains(t, draft.Destinations, "https://api.anthropic.com:443") + return draft, nil + })) + require.NoError(t, err) +} + +func TestPolicyGeminiRequirements(t *testing.T) { + for _, secret := range []string{"", "model-key"} { + t.Run("secret="+secret, func(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderGemini, APIKeySecret: secret, APIKeySecretKey: "token"}) + draft, err := policyDraft(input, policyModels(input), nil) + require.NoError(t, err) + require.NotEmpty(t, draft.Requirements) + for _, r := range draft.Requirements { + require.Equal(t, secret == "", r.Preserve) + } + }) + } +} + +func TestPolicyRequiredAuthenticationIsNotAPlaceholderSentinel(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI}) + input.Harness.Spec.Env = []v1alpha3.RuntimeEnvVar{{Name: "OPENAI_API_KEY", Value: CredentialPlaceholder}} + policy, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + _, err = policy.RenderEnvironment([]corev1.EnvVar{{Name: "OPENAI_API_KEY", Value: CredentialPlaceholder}}) + require.NoError(t, err, "a literal value must not create an authentication requirement") + input.Harness.Spec.Env = nil + input.Harness.Spec.Codex = &v1alpha3.CodexHarness{} + _, err = ResolvePolicy(input, requiredModelCredentials{v1alpha3.ModelProviderOpenAI}, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + require.True(t, slices.ContainsFunc(draft.Requirements, func(r egressext.AuthenticationRequirement) bool { return r.Required })) + return draft, nil + })) + require.ErrorContains(t, err, "requires a gateway credential") +} + +func TestPolicyRetainsDestinationsWithoutCredentialRequirements(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Ollama: &v1alpha3.OllamaConfig{Host: "daemon.example.com:11434"}}) + policy, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + require.Equal(t, []string{"http://daemon.example.com:11434"}, draft.Destinations) + require.Empty(t, draft.Requirements) + return draft, nil + })) + require.NoError(t, err) + require.Equal(t, []string{"http://daemon.example.com:11434"}, policy.access.Destinations) +} + +func TestPolicyModelDestinations(t *testing.T) { + for _, test := range []struct { + name string + spec v1alpha3.ModelConfigSpec + want []string + }{ + {"Mistral default", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderMistral}, []string{"https://api.mistral.ai:443"}}, + {"Mistral override", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderMistral, Mistral: &v1alpha3.MistralConfig{BaseURL: new("https://mistral.example.com/v1")}}, []string{"https://mistral.example.com:443"}}, + {"Ollama bare host", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Ollama: &v1alpha3.OllamaConfig{Host: "daemon.example.com:11434"}}, []string{"http://daemon.example.com:11434"}}, + {"Ollama explicit scheme", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Ollama: &v1alpha3.OllamaConfig{Host: "http://daemon.example.com:11434"}}, []string{"http://daemon.example.com:11434"}}, + {"Ollama no host", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Ollama: &v1alpha3.OllamaConfig{}}, nil}, + {"Ollama cloud", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", APIKeySecret: "auth", APIKeySecretKey: "token", Ollama: &v1alpha3.OllamaConfig{}}, []string{"https://api.ollama.com:443"}}, + {"Ollama local with key", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOllama, Model: "llama3.2", APIKeySecret: "auth", APIKeySecretKey: "token", Ollama: &v1alpha3.OllamaConfig{Host: "daemon.example.com:11434"}}, []string{"http://daemon.example.com:11434"}}, + {"SAP endpoints", v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderSAPAICore, SAPAICore: &v1alpha3.SAPAICoreConfig{BaseURL: "https://sap.example.com/v1", AuthURL: "https://auth.example.com/token"}}, []string{"https://auth.example.com:443", "https://sap.example.com:443"}}, + } { + t.Run(test.name, func(t *testing.T) { + input := credentialInput(test.spec) + policy, err := ResolvePolicy(input, nil, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + require.Equal(t, test.want, draft.Destinations) + return draft, nil + })) + require.NoError(t, err) + require.Equal(t, test.want, policy.access.Destinations) + }) + } +} + +func TestPolicyBindingSourcesRenderSameClients(t *testing.T) { + render := func(input *HarnessInput, extension egressext.PolicyTranslator) ([]corev1.EnvVar, map[string]string) { + policy, err := ResolvePolicy(input, nil, extension) + require.NoError(t, err) + environment, err := policy.RenderEnvironment(nil) + require.NoError(t, err) + headers, err := policy.MCPHeaders("https://api.anthropic.com/other", nil) + require.NoError(t, err) + return environment, headers + } + other := ResolvedMCPTool{Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.anthropic.com/other"}}} + secret := ResolvedMCPTool{Server: &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Namespace: "team"}, Spec: v1alpha3.RemoteMCPServerSpec{URL: "https://api.anthropic.com/mcp", HeadersFrom: []v1alpha3.ValueRef{ + {Name: "X-Api-Key", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "auth", Key: "token"}}, + }}}} + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderAnthropic}) + input.Root.MCPTools = []ResolvedMCPTool{secret, other} + environment, headers := render(input, nil) + require.Equal(t, []corev1.EnvVar{{Name: "ANTHROPIC_API_KEY", Value: CredentialPlaceholder}}, environment) + require.Equal(t, map[string]string{"x-api-key": CredentialPlaceholder}, headers) + policy, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + secretHeaders, err := policy.MCPHeaders("https://api.anthropic.com/mcp", map[string]string{"X-Api-Key": CredentialPlaceholder}) + require.NoError(t, err) + require.Equal(t, map[string]string{"X-Api-Key": CredentialPlaceholder}, secretHeaders, "do not duplicate a Secret header with different casing") + input.Root.MCPTools = []ResolvedMCPTool{other} + contributedEnvironment, contributedHeaders := render(input, policyTranslatorFunc(func(_ types.NamespacedName, draft egressext.PolicyDraft) (egressext.PolicyDraft, error) { + draft.Credentials = append(draft.Credentials, egressext.Credential{Origin: "https://api.anthropic.com:443", Header: "x-api-key", URI: "ate-secret://provider.example/default/token"}) + return draft, nil + })) + require.Equal(t, environment, contributedEnvironment, "Secret and extension bindings render the same clients") + require.Equal(t, headers, contributedHeaders) +} + +func TestPolicyExistingBindingsPreserveClientAuthentication(t *testing.T) { + for _, test := range []struct { + name string + provider v1alpha3.ModelProvider + modelHeads map[string]string + env []v1alpha3.RuntimeEnvVar + mcpHeaders []v1alpha3.ValueRef + headers map[string]string + }{ + {name: "keyless Gemini", provider: v1alpha3.ModelProviderGemini}, + {name: "model headers", modelHeads: map[string]string{"Authorization": "literal"}}, + {name: "literal model key", env: []v1alpha3.RuntimeEnvVar{{Name: "ANTHROPIC_API_KEY", Value: "literal"}}}, + {name: "literal MCP header", mcpHeaders: []v1alpha3.ValueRef{{Name: "X-Trace", Value: "literal"}}, headers: map[string]string{"X-Trace": "literal"}}, + {name: "ConfigMap MCP header", mcpHeaders: []v1alpha3.ValueRef{{Name: "X-Trace", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.ConfigMapValueSource, Name: "trace", Key: "value"}}}, headers: map[string]string{"X-Trace": "configured"}}, + } { + t.Run(test.name, func(t *testing.T) { + provider := test.provider + if provider == "" { + provider = v1alpha3.ModelProviderAnthropic + } + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: provider, DefaultHeaders: test.modelHeads}) + input.Harness.Spec.Env = test.env + target := modelCredentialTarget(input.Root.ResolvedModelConfig) + input.Root.MCPTools = []ResolvedMCPTool{ + {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: target.Endpoint + "/mcp", HeadersFrom: test.mcpHeaders}}}, + {Server: &v1alpha3.RemoteMCPServer{Spec: v1alpha3.RemoteMCPServerSpec{URL: target.Endpoint + "/other"}}}, + {Server: &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Namespace: "team"}, Spec: v1alpha3.RemoteMCPServerSpec{URL: target.Endpoint + "/secret", HeadersFrom: []v1alpha3.ValueRef{ + {Name: target.Header, ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "auth", Key: "token"}}, + }}}}, + } + policy, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err) + require.Len(t, policy.access.Credentials, 1, "preservation must retain the Secret binding") + require.False(t, policy.ModelUsesGateway(input.Root.ResolvedModelConfig)) + var environment []corev1.EnvVar + for _, variable := range test.env { + environment = append(environment, corev1.EnvVar{Name: variable.Name, Value: variable.Value}) + } + rendered, err := policy.RenderEnvironment(environment) + require.NoError(t, err) + require.Equal(t, environment, rendered) + headers, err := policy.MCPHeaders(target.Endpoint+"/mcp", maps.Clone(test.headers)) + require.NoError(t, err) + require.Equal(t, test.headers, headers, "do not add auth to explicitly authenticated clients") + otherHeaders, err := policy.MCPHeaders(target.Endpoint+"/other", nil) + require.NoError(t, err) + require.Empty(t, otherHeaders, "preservation applies to every client at the origin") + }) + } +} + +type requiredModelCredentials []v1alpha3.ModelProvider + +func (r requiredModelCredentials) ModelRequirements(model *ResolvedModelConfig) ModelRequirements { + return ModelRequirements{CredentialRequired: slices.Contains(r, model.Config.Spec.Provider)} +} +func (requiredModelCredentials) Compile(context.Context, *HarnessInput) (*CompileResult, error) { + panic("policy resolution must not render a client") +} + +func TestMergeRequirements(t *testing.T) { + const origin, other = "https://models.example.com:443", "https://tools.example.com:443" + merged := mergeRequirements([]egressext.AuthenticationRequirement{ + {Origin: other}, + {Origin: origin, Header: "authorization", Prefix: "Bearer ", Required: true}, + {Origin: origin, Preserve: true}, + {Origin: origin, Header: "authorization", Prefix: "Bearer "}, + {Origin: origin, Header: "x-api-key"}, + }) + require.Equal(t, []egressext.AuthenticationRequirement{ + {Origin: origin, Preserve: true}, + {Origin: origin, Header: "authorization", Prefix: "Bearer ", Preserve: true, Required: true}, + {Origin: origin, Header: "x-api-key", Preserve: true}, + {Origin: other}, + }, merged) +} diff --git a/go/core/internal/translator/egress_test.go b/go/core/internal/translator/egress_test.go new file mode 100644 index 0000000000..37c70e3741 --- /dev/null +++ b/go/core/internal/translator/egress_test.go @@ -0,0 +1,267 @@ +package translator_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/translator" +) + +func egressPolicy() (*v1alpha3.EgressPolicy, *corev1.Secret) { + return &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "tickets"}, Spec: v1alpha3.EgressPolicySpec{ + Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com", Authentication: &v1alpha3.EgressAuthentication{ + Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "ticket-auth"}, Key: "token"}, + }}}, + }}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "ticket-auth"}, Data: map[string][]byte{"token": []byte("private-ticket-token")}} +} + +func TestEgressPolicyCredentialPrefixes(t *testing.T) { + for _, tc := range []struct { + name string + header string + prefix *string + want string + }{ + {name: "bearer token", header: "Authorization", prefix: new("Bearer "), want: "Bearer "}, + {name: "API key", header: "X-API-Key"}, + {name: "empty prefix", header: "X-API-Key", prefix: new("")}, + {name: "custom prefix", header: "Authorization", prefix: new("Token "), want: "Token "}, + {name: "literal prefix", header: "Authorization", prefix: new("Token"), want: "Token"}, + } { + t.Run(tc.name, func(t *testing.T) { + policy, secret := egressPolicy() + auth := policy.Spec.Destinations[0].Authentication + auth.Header, auth.Prefix = tc.header, tc.prefix + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", []corev1.LocalObjectReference{{Name: policy.Name}}) + require.Empty(t, resolution.Issues) + access := resolution.Access + wantHeader := "authorization" + if tc.header == "X-API-Key" { + wantHeader = "x-api-key" + } + require.Equal(t, []egress.Credential{{Origin: "https://tickets.example.com:443", Header: wantHeader, Prefix: tc.want, URI: "ate-secret://k8s.io/default/test/ticket-auth/token"}}, access.Credentials) + }) + } +} + +func TestEgressPolicyHTTPCredentials(t *testing.T) { + policy, secret := egressPolicy() + policy.Spec.Destinations[0].Origin = "http://tickets.example.com:8080" + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", []corev1.LocalObjectReference{{Name: policy.Name}}) + require.Empty(t, resolution.Issues) + access := resolution.Access + require.Equal(t, []string{"http://tickets.example.com:8080"}, access.Destinations) + require.Equal(t, []egress.Credential{{Origin: "http://tickets.example.com:8080", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/test/ticket-auth/token"}}, access.Credentials) +} + +func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { + for _, kind := range []string{"kagent", "codex", "claude", "byo"} { + t.Run(kind, func(t *testing.T) { + policy, secret := egressPolicy() + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "runner"}, Spec: v1alpha3.HarnessSpec{ + Workload: v1alpha3.HarnessWorkload{Image: new("runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + switch kind { + case "kagent": + harness.Spec.Kagent = &v1alpha3.KagentHarness{} + case "codex": + harness.Spec.Codex = &v1alpha3.CodexHarness{} + case "claude": + harness.Spec.Claude = &v1alpha3.ClaudeHarness{} + case "byo": + harness.Spec.BYO = &v1alpha3.BYOHarness{} + } + model := modelConfig() + if kind == "codex" { + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = secret.Name, "token" + model.Spec.OpenAI = &v1alpha3.OpenAIConfig{APIFormat: new(v1alpha3.OpenAIAPIFormatResponses)} + } + if kind == "claude" { + model.Spec.Provider, model.Spec.Model = v1alpha3.ModelProviderAnthropic, "claude-sonnet-4-5" + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = secret.Name, "token" + } + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "helper"}, Spec: v1alpha3.AgentTemplateSpec{ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help"}} + agent := inlineAgent(harness, template) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + compile := func() *translator.CompileResult { + result, err := compiler(t, model, harness, policy, secret).CompileAgent(t.Context(), agent) + require.NoError(t, err) + return result + } + result := compile() + var recorded struct{ Spec v1alpha3.AgentSpec } + require.NoError(t, json.Unmarshal(result.Provenance, &recorded)) + require.Equal(t, agent.Spec, recorded.Spec, "record the full source spec, including live policy references") + require.NotContains(t, result.EgressDestinations, "https://tickets.example.com:443") + require.NotEmpty(t, result.EgressDestinations, "preserve inferred integration access") + for _, binding := range result.Credentials { + require.NotEqual(t, "https://tickets.example.com:443", binding.Origin) + } + encoded, err := json.Marshal(result) + require.NoError(t, err) + require.NotContains(t, string(encoded), string(secret.Data["token"])) + original, err := result.Digest() + require.NoError(t, err) + secret.Data["token"], secret.UID = []byte("rotated-token"), "replacement" + rotated, err := compile().Digest() + require.NoError(t, err) + require.Equal(t, original, rotated) + policy.Spec.Destinations = append(policy.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://registry.npmjs.org"}) + changed := compile() + updated, err := changed.Digest() + require.NoError(t, err) + require.Equal(t, original, updated) + agent.Spec.EgressPolicyRefs = nil + unreferenced, err := compile().Digest() + require.NoError(t, err) + require.Equal(t, original, unreferenced) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + require.NotContains(t, result.EgressDestinations, "https://registry.npmjs.org:443", "previous revision remains pinned") + // Conflicting and missing policies are independent of runtime compilation. + policy.Spec.Destinations[0].Origin = "https://api.openai.com" + auth := policy.Spec.Destinations[0].Authentication + if kind == "claude" { + policy.Spec.Destinations[0].Origin = "https://api.anthropic.com" + auth.Header, auth.Prefix = "x-api-key", nil + } + auth.SecretKeyRef.Key = "alternate" + secret.Data["alternate"] = []byte("different-token") + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = secret.Name, "token" + _, err = compiler(t, model, harness, policy, secret).CompileAgent(t.Context(), agent) + require.NoError(t, err) + withConflict, err := compile().Digest() + require.NoError(t, err) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "missing"}} + missing, err := compile().Digest() + require.NoError(t, err) + require.Equal(t, withConflict, missing) + // Codex and Claude reject caller-token passthrough independently. + if kind == "claude" || kind == "codex" { + return + } + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = "", "" + model.Spec.APIKeyPassthrough = true + _, err = compiler(t, model, harness, policy, secret).CompileAgent(t.Context(), agent) + require.NoError(t, err) + }) + } +} + +func TestEgressPolicyResolutionFailsClosed(t *testing.T) { + for _, name := range []string{"missing policy", "other namespace", "missing secret", "missing key", "optional secret", "wildcard auth", "empty auth", "missing header", "protected header", "invalid prefix", "conflicting auth", "conflicting prefix"} { + t.Run(name, func(t *testing.T) { + policy, secret := egressPolicy() + objects := []any{policy, secret} + refs := []corev1.LocalObjectReference{{Name: policy.Name}} + switch name { + case "missing policy": + objects = []any{secret} + case "other namespace": + policy.Namespace = "other" + case "missing secret": + objects = []any{policy} + case "missing key": + secret.Data = nil + case "optional secret": + policy.Spec.Destinations[0].Authentication.SecretKeyRef.Optional = new(true) + case "wildcard auth": + policy.Spec.Destinations[0].Origin = "https://*.example.com" + case "empty auth": + policy.Spec.Destinations[0].Authentication = &v1alpha3.EgressAuthentication{} + case "missing header": + policy.Spec.Destinations[0].Authentication.Header = "" + case "protected header": + policy.Spec.Destinations[0].Authentication.Header = "Host" + case "invalid prefix": + policy.Spec.Destinations[0].Authentication.Prefix = new("Bearer\r\nInjected: ") + case "conflicting auth", "conflicting prefix": + other := policy.DeepCopy() + other.Name = "other" + if name == "conflicting prefix" { + other.Spec.Destinations[0].Authentication.Prefix = new("Token ") + } else { + other.Spec.Destinations[0].Authentication.SecretKeyRef.Key = "other" + secret.Data["other"] = []byte("different") + } + objects = append(objects, other) + refs = append(refs, corev1.LocalObjectReference{Name: other.Name}) + } + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, objects...), "test", refs) + if name == "conflicting auth" || name == "conflicting prefix" { + require.Empty(t, resolution.Issues, "individually valid policies reach admission") + _, err := egress.CanonicalAccess(resolution.Access.Destinations, resolution.Access.Credentials) + require.Error(t, err) + } else { + require.Len(t, resolution.Issues, 1) + require.Empty(t, resolution.Access.Destinations) + } + }) + } +} + +func TestEgressPoliciesMergeCredentialsAndOrigins(t *testing.T) { + policy, secret := egressPolicy() + other := policy.DeepCopy() + other.Name = "registries" + other.Spec.Destinations = append(other.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://*.npmjs.org"}, v1alpha3.EgressDestination{ + Origin: "https://tickets.example.com:8443", Authentication: &v1alpha3.EgressAuthentication{Header: "X-API-Key", SecretKeyRef: policy.Spec.Destinations[0].Authentication.SecretKeyRef}, + }) + collections := mockCollections(t, policy, other, secret) + refs := []corev1.LocalObjectReference{{Name: policy.Name}, {Name: other.Name}} + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, collections, "test", refs) + require.Empty(t, resolution.Issues) + access, err := egress.CanonicalAccess(resolution.Access.Destinations, resolution.Access.Credentials) + require.NoError(t, err) + require.Len(t, access.Destinations, 3) + require.Len(t, access.Credentials, 2) + require.Equal(t, "x-api-key", access.Credentials[1].Header) + refs[0], refs[1] = refs[1], refs[0] + reversed := translator.CompileEgressPolicies(krt.TestingDummyContext{}, collections, "test", refs) + require.Empty(t, reversed.Issues) + canonical, err := egress.CanonicalAccess(reversed.Access.Destinations, reversed.Access.Credentials) + require.NoError(t, err) + require.Equal(t, access, canonical) +} + +func TestEgressPoliciesResolveIndependently(t *testing.T) { + policy, secret := egressPolicy() + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", + []corev1.LocalObjectReference{{Name: "z-missing"}, {Name: policy.Name}, {Name: "a-missing"}}) + require.Equal(t, []string{"https://tickets.example.com:443"}, resolution.Access.Destinations) + require.Len(t, resolution.Access.Credentials, 1) + require.Len(t, resolution.Issues, 2) + require.Equal(t, "test/a-missing", resolution.Issues[0].Policy) + require.Equal(t, v1alpha3.EgressPoliciesReasonNotFound, resolution.Issues[0].Reason) + require.Equal(t, "test/z-missing", resolution.Issues[1].Policy) +} + +func TestInvalidEgressReferencesRevokeAll(t *testing.T) { + for _, name := range []string{"too many", "invalid name"} { + t.Run(name, func(t *testing.T) { + policy, secret := egressPolicy() + refs := []corev1.LocalObjectReference{{Name: policy.Name}} + if name == "invalid name" { + refs = append(refs, corev1.LocalObjectReference{Name: "Invalid!"}) + } else { + for range 16 { + refs = append(refs, corev1.LocalObjectReference{Name: policy.Name}) + } + } + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", refs) + require.Empty(t, resolution.Access.Destinations) + require.Empty(t, resolution.Access.Credentials) + require.Len(t, resolution.Issues, len(refs)) + for _, issue := range resolution.Issues { + require.Equal(t, v1alpha3.EgressPoliciesReasonInvalidEgressPolicy, issue.Reason) + } + }) + } +} diff --git a/go/core/internal/translator/foundry_test.go b/go/core/internal/translator/foundry_test.go index eeb1491ce3..b2198fc2ed 100644 --- a/go/core/internal/translator/foundry_test.go +++ b/go/core/internal/translator/foundry_test.go @@ -90,7 +90,7 @@ func TestCompileFoundryEndpoint(t *testing.T) { } require.Contains(t, revision.Environment, corev1.EnvVar{Name: "FOUNDRY_ENDPOINT", Value: endpoint}) require.Len(t, revision.Credentials, 1) - require.Equal(t, host, revision.Credentials[0].Hostname) + require.Equal(t, endpoint+":443", revision.Credentials[0].Origin) require.Equal(t, "api-key", revision.Credentials[0].Header) require.Equal(t, "ate-secret://k8s.io/default/test/foundry-auth/token", revision.Credentials[0].URI) require.Contains(t, revision.EgressDestinations, "https://"+host+":443") diff --git a/go/core/internal/translator/kagent/compiler.go b/go/core/internal/translator/kagent/compiler.go index 3bcb0b7401..9e8c513d0b 100644 --- a/go/core/internal/translator/kagent/compiler.go +++ b/go/core/internal/translator/kagent/compiler.go @@ -38,17 +38,6 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) return nil, err } template, harness := input.Root.Template, input.Harness - configJSON, err := runtimeconfig.Marshal(&configv1.ADKConfig{ - Runtime: translator.RuntimeConfig(c.settings, input.AgentName, template.Namespace), - Agent: compiled.Config, - }) - if err != nil { - var invalid *protovalidate.ValidationError - if errors.As(err, &invalid) { - return nil, translator.NewValidationError("invalid agent config: %w", err) - } - return nil, fmt.Errorf("marshal agent config: %w", err) - } card, err := pbconv.ToProtoAgentCard(translator.ManagedAgentCard(input.AgentName, template)) if err != nil { return nil, fmt.Errorf("convert agent card: %w", err) @@ -61,10 +50,22 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build revision provenance: %w", err) } - environment, credentials, err := translator.CompileCredentials(input, compiled.Models, environment) + environment, err = input.Policy.RenderEnvironment(environment) if err != nil { return nil, err } + + configJSON, err := runtimeconfig.Marshal(&configv1.ADKConfig{ + Runtime: translator.RuntimeConfig(c.settings, input.AgentName, template.Namespace), + Agent: compiled.Config, + }) + if err != nil { + var invalid *protovalidate.ValidationError + if errors.As(err, &invalid) { + return nil, translator.NewValidationError("invalid agent config: %w", err) + } + return nil, fmt.Errorf("marshal agent config: %w", err) + } compiled.Egress = append(compiled.Egress, telemetryConfig.Destinations(environment)...) compiled.Egress = append(compiled.Egress, c.settings.GetControlPlaneUrl()) slices.Sort(compiled.Egress) @@ -75,6 +76,11 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) Environment: environment, ConfigJSON: configJSON, AgentCard: card, SkillImages: compiled.SkillImages, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - Credentials: credentials, Provenance: provenance, EgressDestinations: compiled.Egress, + Provenance: provenance, EgressDestinations: compiled.Egress, }}, nil } + +// ModelRequirements declares authentication and grants before client rendering. +func (*Compiler) ModelRequirements(model *translator.ResolvedModelConfig) translator.ModelRequirements { + return translator.ModelRequirements{Destinations: adkconfig.AdditionalModelDestinations(model)} +} diff --git a/go/core/internal/translator/model_egress.go b/go/core/internal/translator/model_egress.go new file mode 100644 index 0000000000..ba4e5abbeb --- /dev/null +++ b/go/core/internal/translator/model_egress.go @@ -0,0 +1,128 @@ +package translator + +import ( + "fmt" + "slices" + "strings" + + "k8s.io/apimachinery/pkg/types" + + "github.com/kagent-dev/kagent/go/adk/pkg/models" + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/pkg/env" +) + +func modelSecretKey(model *ResolvedModelConfig) string { + if model.Config.Spec.Provider != v1alpha3.ModelProviderBedrock { + return model.Config.Spec.APIKeySecretKey + } + secret := types.NamespacedName{Namespace: model.Config.Namespace, Name: model.Config.Spec.APIKeySecret} + if slices.ContainsFunc(model.References, func(ref ModelConfigReference) bool { + return ref.Kind == "Secret" && ref.NamespacedName == secret && ref.Key == env.AWSBearerTokenBedrock.Name() + }) { + return env.AWSBearerTokenBedrock.Name() + } + return "" +} + +type modelCredential struct { + Environment string + Endpoint string + Header string + Prefix string +} + +func modelCredentialTarget(resolved *ResolvedModelConfig) modelCredential { + var name, endpoint, header, prefix string + spec := resolved.Config.Spec + switch spec.Provider { + case v1alpha3.ModelProviderOpenAI: + name, endpoint, header, prefix = env.OpenAIAPIKey.Name(), "https://api.openai.com", "authorization", "Bearer " + if spec.OpenAI != nil && spec.OpenAI.BaseURL != "" { + endpoint = spec.OpenAI.BaseURL + } + case v1alpha3.ModelProviderAnthropic: + name, endpoint, header = env.AnthropicAPIKey.Name(), "https://api.anthropic.com", "x-api-key" + if spec.Anthropic != nil && spec.Anthropic.BaseURL != "" { + endpoint = spec.Anthropic.BaseURL + } + case v1alpha3.ModelProviderAzureOpenAI: + name, header = env.AzureOpenAIAPIKey.Name(), "api-key" + if spec.AzureOpenAI != nil { + endpoint = spec.AzureOpenAI.Endpoint + } + case v1alpha3.ModelProviderGemini: + name, endpoint, header = env.GoogleAPIKey.Name(), "https://generativelanguage.googleapis.com", "x-goog-api-key" + case v1alpha3.ModelProviderMistral: + name, endpoint, header, prefix = env.MistralAPIKey.Name(), "https://api.mistral.ai", "authorization", "Bearer " + if spec.Mistral != nil && spec.Mistral.BaseURL != nil && *spec.Mistral.BaseURL != "" { + endpoint = *spec.Mistral.BaseURL + } + case v1alpha3.ModelProviderBedrock: + name, header, prefix = env.AWSBearerTokenBedrock.Name(), "authorization", "Bearer " + if spec.Bedrock != nil { + endpoint = fmt.Sprintf("https://bedrock-runtime.%s.amazonaws.com", spec.Bedrock.Region) + } + case v1alpha3.ModelProviderOllama: + // Ollama Cloud is the only keyed Ollama endpoint, so a binding is + // declared exactly when the model reaches it. That condition lives in + // models.OllamaReachesCloud, which the compiler's Secret reference and + // the egress list also use: a cloud model picked from the catalog, an + // explicit host, and a missing key were each judged differently in each + // place, so a valid configuration could compile with an env var that had + // no binding to match, or with an egress list that omitted the host it + // was about to call. + // + // Without a binding this leaves the name empty, and the env var is + // treated as a plain secret reference, which is correct — no request + // leaves for api.ollama.com. + if spec.Ollama == nil { + break + } + hasCredential := spec.APIKeySecret != "" || spec.APIKeyPassthrough + if !models.OllamaReachesCloud(spec.Model, spec.Ollama.Host, hasCredential) { + break + } + name, endpoint, header, prefix = env.OllamaAPIKey.Name(), "https://api.ollama.com", "authorization", "Bearer " + case v1alpha3.ModelProviderFoundry: + name, endpoint, header = env.FoundryAPIKey.Name(), resolved.FoundryEndpoint, "api-key" + if spec.Foundry != nil && spec.Foundry.APIFormat == v1alpha3.FoundryAPIFormatAnthropic { + header = "x-api-key" + } + } + return modelCredential{Environment: name, Endpoint: endpoint, Header: header, Prefix: prefix} +} + +// modelEndpoints returns application endpoints before client configuration is rendered. +func modelEndpoints(resolved *ResolvedModelConfig) []string { + if resolved == nil { + return nil + } + spec := resolved.Config.Spec + endpoint := modelCredentialTarget(resolved).Endpoint + var endpoints []string + if endpoint != "" { + endpoints = append(endpoints, endpoint) + } + if spec.Provider == v1alpha3.ModelProviderOllama && spec.Ollama != nil && spec.Ollama.Host != "" { + endpoints = append(endpoints, OllamaURL(spec.Ollama.Host)) + } + if spec.Provider == v1alpha3.ModelProviderSAPAICore && spec.SAPAICore != nil { + endpoints = append(endpoints, spec.SAPAICore.BaseURL) + if spec.SAPAICore.AuthURL != "" { + endpoints = append(endpoints, spec.SAPAICore.AuthURL) + } + } + return endpoints +} + +// OllamaURL uses HTTPS for the cloud endpoint and HTTP for bare daemon addresses. +func OllamaURL(host string) string { + if strings.HasPrefix(host, "http://") || strings.HasPrefix(host, "https://") { + return host + } + if models.IsOllamaCloudEndpoint(host) { + return "https://" + host + } + return "http://" + host +} diff --git a/go/core/internal/translator/model_egress_test.go b/go/core/internal/translator/model_egress_test.go new file mode 100644 index 0000000000..15ee43ee67 --- /dev/null +++ b/go/core/internal/translator/model_egress_test.go @@ -0,0 +1,86 @@ +package translator + +import ( + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" +) + +// A cloud-tagged Ollama model reaches api.ollama.com with the key, which is a +// gateway-injected credential like any other — without this binding the +// translator rejects OLLAMA_API_KEY as an unsupported local secret and the +// AgentTemplate goes Compatible=False. +func TestModelCredentialTargetOllamaCloud(t *testing.T) { + secret := v1alpha3.ModelConfigSpec{ + Provider: v1alpha3.ModelProviderOllama, + Model: "deepseek-v4-flash:0731-cloud", + APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", + Ollama: &v1alpha3.OllamaConfig{}, + } + + tests := []struct { + name string + spec v1alpha3.ModelConfigSpec + wantName string + wantEndpoint string + }{ + { + name: "cloud model with a secret binds the cloud", + spec: secret, + wantName: "OLLAMA_API_KEY", + wantEndpoint: "https://api.ollama.com", + }, + { + name: "cloud model with passthrough binds the cloud", + spec: v1alpha3.ModelConfigSpec{ + Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", + APIKeyPassthrough: true, Ollama: &v1alpha3.OllamaConfig{}, + }, + wantName: "OLLAMA_API_KEY", + wantEndpoint: "https://api.ollama.com", + }, + { + // A local model must declare no binding: no request leaves for the + // cloud, so there is nothing to inject a header into. + name: "local model declares no binding", + spec: v1alpha3.ModelConfigSpec{ + Provider: v1alpha3.ModelProviderOllama, Model: "llama3.2", + APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", + Ollama: &v1alpha3.OllamaConfig{}, + }, + }, + { + // An explicit host wins over the cloud route, so the key stays a + // plain secret reference rather than a gateway binding. + name: "explicit host declares no binding", + spec: v1alpha3.ModelConfigSpec{ + Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", + APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", + Ollama: &v1alpha3.OllamaConfig{Host: "host.docker.internal:11434"}, + }, + }, + { + name: "cloud model without any key declares no binding", + spec: v1alpha3.ModelConfigSpec{ + Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", + Ollama: &v1alpha3.OllamaConfig{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resolved := &ResolvedModelConfig{ + Config: &v1alpha3.ModelConfig{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test"}, + Spec: tt.spec, + }, + } + target := modelCredentialTarget(resolved) + require.Equal(t, tt.wantName, target.Environment) + require.Equal(t, tt.wantEndpoint, target.Endpoint) + }) + } +} diff --git a/go/core/internal/translator/resolved_policy.go b/go/core/internal/translator/resolved_policy.go new file mode 100644 index 0000000000..571f8b8cbb --- /dev/null +++ b/go/core/internal/translator/resolved_policy.go @@ -0,0 +1,77 @@ +package translator + +import ( + "slices" + "strings" + + corev1 "k8s.io/api/core/v1" + + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +// CredentialPlaceholder satisfies SDKs that require a configured API key. +const CredentialPlaceholder = "kagent-credential-injected" + +// ResolvedPolicy contains pinned integration access and the authentication used to render clients. +// Its private fields prevent renderers from changing authentication decisions. +type ResolvedPolicy struct { + access egress.Access + modelKeys []string + modelBindings map[*ResolvedModelConfig]bool + secretModels map[string]bool + secretMCP map[string]bool + headers map[string][]string + // tokenPassthroughOrigins cannot receive gateway credentials, including live grants. + tokenPassthroughOrigins []string +} + +// ModelUsesGateway reports the resolved authentication mode of a model. +func (p ResolvedPolicy) ModelUsesGateway(model *ResolvedModelConfig) bool { + return p.modelBindings[model] +} + +// RenderEnvironment replaces supported Secret references and emits resolved SDK keys. +func (p ResolvedPolicy) RenderEnvironment(environment []corev1.EnvVar) ([]corev1.EnvVar, error) { + result := slices.Clone(environment) + for i, variable := range result { + if variable.ValueFrom == nil { + continue + } + ref := variable.ValueFrom.SecretKeyRef + isMCP := strings.HasPrefix(variable.Name, "KAGENT_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CODEX_MCP_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CLAUDE_MCP_CREDENTIAL_") + if ref == nil || (!p.secretModels[variable.Name+"\x00"+ref.Name+"\x00"+ref.Key] && (!isMCP || !p.secretMCP[ref.Name+"\x00"+ref.Key])) { + return nil, NewValidationError("environment credential %q cannot use gateway header injection; local signing and arbitrary secret environment variables are unsupported", variable.Name) + } + result[i].Value, result[i].ValueFrom = CredentialPlaceholder, nil + } + for _, name := range p.modelKeys { + if !slices.ContainsFunc(result, func(variable corev1.EnvVar) bool { return variable.Name == name }) { + result = append(result, corev1.EnvVar{Name: name, Value: CredentialPlaceholder}) + } + } + return result, nil +} + +// MCPHeaders renders missing headers from the resolved client authentication. +func (p ResolvedPolicy) MCPHeaders(rawURL string, headers map[string]string) (map[string]string, error) { + if len(p.headers) == 0 { + return headers, nil + } + origin, err := credentialOrigin(rawURL) + if err != nil { + return nil, NewValidationError("invalid MCP credential destination: %v", err) + } + for _, header := range p.headers[origin] { + present := false + for name := range headers { + present = present || strings.EqualFold(name, header) + } + if !present { + if headers == nil { + headers = map[string]string{} + } + headers[header] = CredentialPlaceholder + } + } + return headers, nil +} diff --git a/go/core/internal/translator/credentials_test.go b/go/core/internal/translator/resolved_policy_test.go similarity index 59% rename from go/core/internal/translator/credentials_test.go rename to go/core/internal/translator/resolved_policy_test.go index ba106e67b3..37fc282cf6 100644 --- a/go/core/internal/translator/credentials_test.go +++ b/go/core/internal/translator/resolved_policy_test.go @@ -6,11 +6,13 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" ) -func TestCompileCredentialDestinations(t *testing.T) { +func TestResolvedPolicyCredentialDestinations(t *testing.T) { for _, test := range []struct { name string spec v1alpha3.ModelConfigSpec @@ -34,13 +36,13 @@ func TestCompileCredentialDestinations(t *testing.T) { } input := credentialInput(test.spec) environment := []corev1.EnvVar{credentialEnv(test.env, "auth", test.spec.APIKeySecretKey)} - got, bindings, err := CompileCredentials(input, nil, environment) + got, bindings, err := renderTestCredentials(input, environment) require.NoError(t, err) require.Equal(t, CredentialPlaceholder, got[0].Value) require.Nil(t, got[0].ValueFrom) require.NotNil(t, environment[0].ValueFrom, "compilation must not mutate inputs") require.Len(t, bindings, 1) - require.Equal(t, test.host, bindings[0].Hostname) + require.Equal(t, "https://"+test.host+":443", bindings[0].Origin) require.Equal(t, test.header, bindings[0].Header) require.Equal(t, test.prefix, bindings[0].Prefix) require.Equal(t, "ate-secret://k8s.io/default/team/auth/"+test.spec.APIKeySecretKey, bindings[0].URI) @@ -48,40 +50,46 @@ func TestCompileCredentialDestinations(t *testing.T) { } } -func TestCompileCredentialsRejectsConflictingSharedModels(t *testing.T) { +func TestResolvedPolicyRejectsConflictingSharedModels(t *testing.T) { spec := v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "root", APIKeySecretKey: "token"} input := credentialInput(spec) spec.APIKeySecret = "child" input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(spec).Root}} // The ADK environment is deduplicated, but every model still needs its own binding. - _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) + environment, _, err := renderTestCredentials(input, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) require.ErrorContains(t, err, "conflicting credentials") + require.Nil(t, environment) input.Root.Shared[0].Agent.ResolvedModelConfig.Config.Spec.OpenAI = &v1alpha3.OpenAIConfig{BaseURL: "https://child.example.com/v1"} - _, bindings, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) + _, bindings, err := renderTestCredentials(input, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) require.NoError(t, err) require.Len(t, bindings, 2) } -func TestCompileCredentialsRejectsLocalSecrets(t *testing.T) { +func TestResolvedPolicyRejectsLocalSecrets(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{}) - _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("AWS_SECRET_ACCESS_KEY", "auth", "token")}) + environment, _, err := renderTestCredentials(input, []corev1.EnvVar{credentialEnv("AWS_SECRET_ACCESS_KEY", "auth", "token")}) require.ErrorContains(t, err, "cannot use gateway header injection") + require.Nil(t, environment) } -func TestCompileCredentialsPreservesPassthrough(t *testing.T) { +func TestResolvedPolicyPreservesPassthrough(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: true}) - _, bindings, err := CompileCredentials(input, nil, nil) + _, bindings, err := renderTestCredentials(input, nil) require.NoError(t, err) require.Empty(t, bindings) input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(v1alpha3.ModelConfigSpec{ Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "auth", APIKeySecretKey: "token", }).Root}} - _, _, err = CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}) + environment, _, err := renderTestCredentials(input, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}) require.ErrorContains(t, err, "cannot combine caller-token passthrough") + require.Nil(t, environment) } func credentialInput(spec v1alpha3.ModelConfigSpec) *HarnessInput { resolved := &ResolvedModelConfig{Config: &v1alpha3.ModelConfig{ObjectMeta: metav1.ObjectMeta{Namespace: "team"}, Spec: spec}} + if spec.Provider == v1alpha3.ModelProviderBedrock && spec.APIKeySecretKey == "AWS_BEARER_TOKEN_BEDROCK" { + resolved.References = []ModelConfigReference{{Kind: "Secret", NamespacedName: types.NamespacedName{Namespace: "team", Name: spec.APIKeySecret}, Key: spec.APIKeySecretKey}} + } if spec.Foundry != nil { resolved.FoundryEndpoint = spec.Foundry.Endpoint } @@ -95,78 +103,50 @@ func credentialEnv(name, secret, key string) corev1.EnvVar { return corev1.EnvVar{Name: name, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secret}, Key: key}}} } -// A cloud-tagged Ollama model reaches api.ollama.com with the key, which is a -// gateway-injected credential like any other — without this binding the -// translator rejects OLLAMA_API_KEY as an unsupported local secret and the -// AgentTemplate goes Compatible=False. -func TestModelCredentialTargetOllamaCloud(t *testing.T) { - secret := v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, - Model: "deepseek-v4-flash:0731-cloud", - APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", - Ollama: &v1alpha3.OllamaConfig{}, - } +func TestPassthroughCredentialConflictUsesCanonicalOrigin(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: true, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://API.OPENAI.COM.:0443/v1"}}) + input.Root.MCPTools = []ResolvedMCPTool{{Server: &v1alpha3.RemoteMCPServer{ObjectMeta: metav1.ObjectMeta{Namespace: "team"}, Spec: v1alpha3.RemoteMCPServerSpec{ + URL: "https://api.openai.com/mcp", HeadersFrom: []v1alpha3.ValueRef{{Name: "authorization", ValueFrom: &v1alpha3.ValueSource{Type: v1alpha3.SecretValueSource, Name: "auth", Key: "token"}}}, + }}}} + environment, _, err := renderTestCredentials(input, nil) + require.ErrorContains(t, err, "cannot combine caller-token passthrough") + require.Nil(t, environment) + input.Root.ResolvedModelConfig.Config.Spec.OpenAI.BaseURL = "https://api.openai.com:8443/v1" + policy, err := ResolvePolicy(input, nil, nil) + require.NoError(t, err, "a different port is a distinct credential scope") + require.Equal(t, []string{"https://api.openai.com:8443"}, policy.tokenPassthroughOrigins) +} - tests := []struct { - name string - spec v1alpha3.ModelConfigSpec - wantName string - wantEndpoint string - }{ - { - name: "cloud model with a secret binds the cloud", - spec: secret, - wantName: "OLLAMA_API_KEY", - wantEndpoint: "https://api.ollama.com", - }, - { - name: "cloud model with passthrough binds the cloud", - spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", - APIKeyPassthrough: true, Ollama: &v1alpha3.OllamaConfig{}, - }, - wantName: "OLLAMA_API_KEY", - wantEndpoint: "https://api.ollama.com", - }, - { - // A local model must declare no binding: no request leaves for the - // cloud, so there is nothing to inject a header into. - name: "local model declares no binding", - spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "llama3.2", - APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", - Ollama: &v1alpha3.OllamaConfig{}, - }, - }, - { - // An explicit host wins over the cloud route, so the key stays a - // plain secret reference rather than a gateway binding. - name: "explicit host declares no binding", - spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", - APIKeySecret: "ollama-cloud", APIKeySecretKey: "OLLAMA_API_KEY", - Ollama: &v1alpha3.OllamaConfig{Host: "host.docker.internal:11434"}, - }, - }, - { - name: "cloud model without any key declares no binding", - spec: v1alpha3.ModelConfigSpec{ - Provider: v1alpha3.ModelProviderOllama, Model: "minimax-m3:cloud", - Ollama: &v1alpha3.OllamaConfig{}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - resolved := &ResolvedModelConfig{ - Config: &v1alpha3.ModelConfig{ - ObjectMeta: metav1.ObjectMeta{Namespace: "test"}, - Spec: tt.spec, - }, +func TestResolvedPolicyRejectsInvalidOrigins(t *testing.T) { + for _, endpoint := range []string{ + "ftp://api.example.com", "https://192.0.2.1", + "https://user@api.example.com", "https://api.example.com/#fragment", + } { + t.Run(endpoint, func(t *testing.T) { + for _, passthrough := range []bool{false, true} { + spec := v1alpha3.ModelConfigSpec{ + Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: passthrough, + OpenAI: &v1alpha3.OpenAIConfig{BaseURL: endpoint}, + } + if !passthrough { + spec.APIKeySecret, spec.APIKeySecretKey = "auth", "token" + } + _, bindings, err := renderTestCredentials(credentialInput(spec), nil) + require.Error(t, err) + require.Empty(t, bindings) } - name, endpoint, _, _ := modelCredentialTarget(resolved) - require.Equal(t, tt.wantName, name) - require.Equal(t, tt.wantEndpoint, endpoint) }) } } + +func renderTestCredentials(input *HarnessInput, environment []corev1.EnvVar) ([]corev1.EnvVar, []egress.Credential, error) { + policy, err := ResolvePolicy(input, nil, nil) + if err != nil { + return nil, nil, err + } + rendered, err := policy.RenderEnvironment(environment) + if err != nil { + return nil, nil, err + } + return rendered, policy.access.Credentials, nil +} diff --git a/go/core/internal/translator/revision.go b/go/core/internal/translator/revision.go index 4772cb3bf9..37863518dc 100644 --- a/go/core/internal/translator/revision.go +++ b/go/core/internal/translator/revision.go @@ -30,9 +30,8 @@ func (id RevisionID) Short() string { return hex.EncodeToString(id[:shortRevisio // IsZero reports whether compilation has not produced an identity. func (id RevisionID) IsZero() bool { return id == RevisionID{} } -// CompileResult contains one immutable runtime revision and the non-blocking -// diagnostics produced while compiling it. Diagnostics are deliberately kept -// outside Revision because they do not describe runtime behavior. +// CompileResult contains immutable runtime inputs and non-blocking diagnostics. +// Diagnostics do not enter the revision's identity. type CompileResult struct { Revision Warnings []string @@ -61,11 +60,13 @@ type Revision struct { SandboxClass atev1alpha1.SandboxClass SnapshotLocation string - // Provenance identifies non-secret Kubernetes inputs. Gateway-fetched - // credential values are deliberately excluded from revision identity. + // Provenance records non-secret source configuration for diagnostics. Revision + // identity comes from compiled runtime inputs, not this source snapshot. Provenance json.RawMessage // Credentials contains references resolved by the egress gateway. Credentials []egress.Credential + // TokenPassthroughOrigins cannot receive gateway credentials, including live grants. + TokenPassthroughOrigins []string // EgressDestinations is the HTTP(S) origin allowlist required by this revision. EgressDestinations []string } @@ -92,21 +93,21 @@ func (r *Revision) Digest() (RevisionID, error) { return RevisionID{}, fmt.Errorf("unsupported sandbox class %q", sandboxClass) } raw, err := json.Marshal(struct { - AgentName string `json:"agentName"` - AgentUID string `json:"agentUID"` - Namespace string `json:"namespace"` - Image string `json:"image"` - Command []string `json:"command,omitempty"` - Args []string `json:"args,omitempty"` - Environment []corev1.EnvVar `json:"environment"` - ConfigJSON json.RawMessage `json:"config"` - SkillImages []SkillImage `json:"skillImages,omitempty"` - WorkerPoolName string `json:"workerPoolName"` - SnapshotLocation string `json:"snapshotLocation"` - Provenance json.RawMessage `json:"provenance"` - Credentials []egress.Credential `json:"credentials,omitempty"` - EgressDestinations []string `json:"egressDestinations"` - SandboxClass atev1alpha1.SandboxClass `json:"sandboxClass"` + AgentName string `json:"agentName"` + AgentUID string `json:"agentUID"` + Namespace string `json:"namespace"` + Image string `json:"image"` + Command []string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Environment []corev1.EnvVar `json:"environment"` + ConfigJSON json.RawMessage `json:"config"` + SkillImages []SkillImage `json:"skillImages,omitempty"` + WorkerPoolName string `json:"workerPoolName"` + SnapshotLocation string `json:"snapshotLocation"` + Credentials []egress.Credential `json:"credentials,omitempty"` + TokenPassthroughOrigins []string `json:"tokenPassthroughOrigins,omitempty"` + EgressDestinations []string `json:"egressDestinations"` + SandboxClass atev1alpha1.SandboxClass `json:"sandboxClass"` // Placement now matches the pool's namespace as well as its name. // Hash it explicitly so existing immutable templates do not conflict after upgrades. @@ -115,8 +116,8 @@ func (r *Revision) Digest() (RevisionID, error) { AgentName: r.AgentName, AgentUID: r.AgentUID, Namespace: r.Namespace, Image: r.Image, Command: r.Command, Args: r.Args, Environment: r.Environment, ConfigJSON: r.ConfigJSON, SkillImages: r.SkillImages, - WorkerPoolName: r.WorkerPoolName, SnapshotLocation: r.SnapshotLocation, Provenance: r.Provenance, - Credentials: r.Credentials, EgressDestinations: r.EgressDestinations, + WorkerPoolName: r.WorkerPoolName, SnapshotLocation: r.SnapshotLocation, + TokenPassthroughOrigins: r.TokenPassthroughOrigins, Credentials: r.Credentials, EgressDestinations: r.EgressDestinations, SandboxClass: sandboxClass, WorkerPoolNamespace: r.Namespace, }) if err != nil { diff --git a/go/core/internal/translator/revision_test.go b/go/core/internal/translator/revision_test.go index ecde2c53c5..e817061e39 100644 --- a/go/core/internal/translator/revision_test.go +++ b/go/core/internal/translator/revision_test.go @@ -49,7 +49,7 @@ func TestRevisionDigestReplacesNameOnlyWorkerPoolTemplate(t *testing.T) { require.Equal(t, id, repeated) } -func TestRevisionDigestIncludesProvenance(t *testing.T) { +func TestRevisionDigestExcludesProvenance(t *testing.T) { revision := &Revision{Namespace: "agents", AgentName: "helper", Provenance: []byte(`[{"kind":"ConfigMap","hash":"first"}]`)} first, err := revision.Digest() if err != nil { @@ -60,8 +60,8 @@ func TestRevisionDigestIncludesProvenance(t *testing.T) { if err != nil { t.Fatal(err) } - if first == second { - t.Fatal("configuration change did not change runtime revision") + if first != second { + t.Fatal("source-only change altered runtime revision") } if len(first.Short()) != 12 || !strings.HasPrefix(first.String(), first.Short()) { t.Fatalf("short revision %q is not a prefix of %q", first.Short(), first.String()) diff --git a/go/core/pkg/app/app.go b/go/core/pkg/app/app.go index 13bfd5fe13..4f03d81f68 100644 --- a/go/core/pkg/app/app.go +++ b/go/core/pkg/app/app.go @@ -68,6 +68,7 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/version" "github.com/kagent-dev/kagent/go/core/pkg/auth" "github.com/kagent-dev/kagent/go/core/pkg/config" + "github.com/kagent-dev/kagent/go/core/pkg/egressext" "github.com/kagent-dev/kagent/go/core/pkg/migrations" "github.com/kagent-dev/kagent/go/core/pkg/postgres" "github.com/kagent-dev/kagent/go/pkg/logging" @@ -83,6 +84,9 @@ import ( // every service that takes one — so a library consumer cannot narrow where // its own policy applies. type Options struct { + // EgressPolicyTranslator resolves additional outbound credentials before client rendering. + // Nil preserves the built-in authentication rules. + EgressPolicyTranslator egressext.PolicyTranslator // Authenticator identifies the caller. Nil selects InsecureAuthenticator, // which admits every request. Authenticator auth.AuthProvider @@ -272,11 +276,12 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er if err != nil { return fmt.Errorf("create controller manager: %w", err) } - runtime, err := controller.NewRuntime(kubeConfig, watchNamespaces, ctx.Done(), cfg.GetRuntimeDefaults(), cfg.GetHarnessImages()) + runtime, err := controller.NewRuntime(kubeConfig, watchNamespaces, ctx.Done(), cfg.GetRuntimeDefaults(), cfg.GetHarnessImages(), opts.EgressPolicyTranslator) if err != nil { return err } actors, err := substrate.Dial(ctx, substrate.Config{ + EgressPolicies: store, AteAPIEndpoint: cfg.GetSubstrate().GetApiEndpoint(), CAFile: cfg.GetSubstrate().GetCaFile(), ClientCertFile: cfg.GetSubstrate().GetClientCertFile(), diff --git a/go/core/pkg/egressext/policy.go b/go/core/pkg/egressext/policy.go new file mode 100644 index 0000000000..f17c6d8dc1 --- /dev/null +++ b/go/core/pkg/egressext/policy.go @@ -0,0 +1,40 @@ +// Package egressext defines outbound policy extensions. +package egressext + +import "k8s.io/apimachinery/pkg/types" + +// AuthenticationRequirement describes an application's authentication at an origin. +// A nonempty Header requires that header and prefix for new bindings. +type AuthenticationRequirement struct { + Origin string + Header string + Prefix string + // Preserve prevents additional bindings at this origin. + Preserve bool + // Required requires a gateway binding before the client can be rendered. + Required bool +} + +// Credential binds a provider reference to an outbound origin and header. +type Credential struct { + Origin string + Header string + Prefix string + URI string +} + +// PolicyDraft contains application destinations, authentication requirements and credential references. +// It contains no credential values. The extension can only add Credentials. +type PolicyDraft struct { + Destinations []string + Requirements []AuthenticationRequirement + Credentials []Credential +} + +// PolicyTranslator resolves policy before harness configuration is rendered. +// Implementations must be deterministic and safe for concurrent calls, with no I/O. +// Core rejects changed destinations, requirements, and existing bindings. +type PolicyTranslator interface { + // agent identifies the runnable Agent, not its referenced AgentTemplate. + Translate(agent types.NamespacedName, draft PolicyDraft) (PolicyDraft, error) +} diff --git a/go/core/pkg/migrations/core/000001_initial.sql b/go/core/pkg/migrations/core/000001_initial.sql index d04ebba522..27b0c5e819 100644 --- a/go/core/pkg/migrations/core/000001_initial.sql +++ b/go/core/pkg/migrations/core/000001_initial.sql @@ -34,6 +34,7 @@ CREATE TABLE runtime_revision ( source_snapshot JSONB NOT NULL, egress_destinations TEXT[] NOT NULL DEFAULT '{}', credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(credentials) = 'array'), + token_passthrough_origins TEXT[] NOT NULL DEFAULT '{}', actor_template_atespace TEXT CONSTRAINT runtime_revision_actor_template_namespace_not_null NOT NULL, actor_template_name TEXT NOT NULL, actor_template_uid TEXT NOT NULL DEFAULT '', @@ -52,6 +53,7 @@ CREATE TABLE agent_revision ( agent_card BYTEA NOT NULL, FOREIGN KEY (revision) REFERENCES runtime_revision(revision) ON DELETE CASCADE ); +CREATE INDEX agent_revision_source_idx ON agent_revision (agent_name, revision); CREATE TABLE sandbox_revision ( revision TEXT PRIMARY KEY, @@ -59,11 +61,14 @@ CREATE TABLE sandbox_revision ( sandbox_template_uid TEXT NOT NULL, FOREIGN KEY (revision) REFERENCES runtime_revision(revision) ON DELETE CASCADE ); +CREATE INDEX sandbox_revision_source_idx ON sandbox_revision (sandbox_template_name, revision); CREATE TABLE agent_definition ( namespace TEXT NOT NULL, agent_name TEXT NOT NULL, agent_uid TEXT NOT NULL, + grant_destinations TEXT[] NOT NULL DEFAULT '{}', + grant_credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(grant_credentials) = 'array'), desired_revision TEXT NOT NULL, latest_successful_revision TEXT REFERENCES agent_revision(revision) ON DELETE RESTRICT, retired_at TIMESTAMPTZ, @@ -78,6 +83,8 @@ CREATE TABLE sandbox_template_definition ( namespace TEXT NOT NULL, sandbox_template_name TEXT NOT NULL, sandbox_template_uid TEXT NOT NULL, + grant_destinations TEXT[] NOT NULL DEFAULT '{}', + grant_credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(grant_credentials) = 'array'), desired_revision TEXT NOT NULL, latest_successful_revision TEXT REFERENCES sandbox_revision(revision) ON DELETE RESTRICT, retired_at TIMESTAMPTZ, @@ -135,6 +142,8 @@ CREATE TABLE runtime_instance ( ); CREATE INDEX runtime_instance_owner_idx ON runtime_instance (kind, user_id, id) WHERE state <> 'RUNTIME_STATE_DELETED'; +CREATE INDEX runtime_instance_revision_idx ON runtime_instance (prepared_revision, id) + WHERE state <> 'RUNTIME_STATE_DELETED' AND operation <> 'RUNTIME_OPERATION_DELETE'; CREATE TABLE session ( id UUID PRIMARY KEY, diff --git a/go/core/pkg/migrations/cross_track_test.go b/go/core/pkg/migrations/cross_track_test.go index ab4401efc4..13f8585722 100644 --- a/go/core/pkg/migrations/cross_track_test.go +++ b/go/core/pkg/migrations/cross_track_test.go @@ -184,7 +184,7 @@ var schemaQualifiedChecks = []sqlCheck{ {"search_path", regexp.MustCompile(`(?i)\bsearch_path\b`)}, {"SET SCHEMA", regexp.MustCompile(`(?i)\bSET\s+SCHEMA\b`)}, {"schema-qualified table", regexp.MustCompile(`(?i)\b(?:CREATE|ALTER|DROP)\s+TABLE\s+(?:IF\s+(?:NOT\s+)?EXISTS\s+)?\w+\.\w+`)}, - {"schema-qualified index target", regexp.MustCompile(`(?i)\bON\s+\w+\.\w+`)}, + {"schema-qualified index target", regexp.MustCompile(`(?i)\bCREATE\s+(?:UNIQUE\s+)?INDEX\b[^;]*?\bON\s+(?:ONLY\s+)?\w+\.\w+`)}, {"schema-qualified reference", regexp.MustCompile(`(?i)\bREFERENCES\s+\w+\.\w+`)}, } @@ -240,12 +240,15 @@ func TestSchemaViolations(t *testing.T) { }{ {"unqualified table", `CREATE TABLE IF NOT EXISTS eval_set (id TEXT);`, nil}, {"unqualified index", `CREATE INDEX IF NOT EXISTS i ON eval_set(id);`, nil}, + {"join condition", `CREATE VIEW live AS SELECT a.id FROM actors a JOIN revisions r ON a.revision = r.id;`, nil}, + {"index followed by join", `CREATE INDEX i ON actors(id); CREATE VIEW live AS SELECT a.id FROM actors a JOIN revisions r ON a.revision = r.id;`, nil}, {"schema in comment", `-- create table myschema.foo here` + "\n" + `CREATE TABLE IF NOT EXISTS foo (id TEXT);`, nil}, {"qualified table", `CREATE TABLE myschema.eval_set (id TEXT);`, []string{"schema-qualified table"}}, {"create schema", `CREATE SCHEMA IF NOT EXISTS myschema;`, []string{"CREATE SCHEMA"}}, {"set search_path", `SET search_path TO myschema;`, []string{"search_path"}}, {"set schema", `ALTER TABLE foo SET SCHEMA myschema;`, []string{"SET SCHEMA"}}, {"qualified index target", `CREATE INDEX i ON myschema.foo(id);`, []string{"schema-qualified index target"}}, + {"qualified concurrent index target", `CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS i ON ONLY myschema.foo(id);`, []string{"schema-qualified index target"}}, {"qualified reference", `ALTER TABLE foo ADD CONSTRAINT fk FOREIGN KEY (b) REFERENCES myschema.bar(id);`, []string{"schema-qualified reference"}}, } for _, tt := range tests { diff --git a/go/core/pkg/migrations/runner_test.go b/go/core/pkg/migrations/runner_test.go index 6348ccea19..019b448a56 100644 --- a/go/core/pkg/migrations/runner_test.go +++ b/go/core/pkg/migrations/runner_test.go @@ -323,6 +323,14 @@ func TestBuiltinMigrationsRoundTrip(t *testing.T) { t.Fatalf("%s versions = %v, want %v", source.Name, versions, expected) } } + if !testColumnExists(t, dsn, "runtime_revision", "token_passthrough_origins") || !testColumnExists(t, dsn, "agent_runtime_revision", "token_passthrough_origins") { + t.Fatal("runtime and agent revisions must expose passthrough origins") + } + for _, table := range []string{"agent_definition", "sandbox_template_definition"} { + if !testColumnExists(t, dsn, table, "grant_destinations") || !testColumnExists(t, dsn, table, "grant_credentials") { + t.Fatalf("%s source grants are missing", table) + } + } if !testColumnExists(t, dsn, "session", "deletion_reason") { t.Fatal("session deletion reason is missing") } diff --git a/go/core/test/e2e/README.md b/go/core/test/e2e/README.md index f59a3e04e4..cca55676bf 100644 --- a/go/core/test/e2e/README.md +++ b/go/core/test/e2e/README.md @@ -238,6 +238,24 @@ selection quality. The registry must accept pushes from the test runner and be reachable by Substrate workers through their existing registry rewrite. Private-registry credentials are not required. +`TestSessionEgressPolicyUpdatesLiveActors` loads the same OCI skill and runs +the same HTTP probe script through each harness's shell tool. During a single +active invocation, it adds a policy reference and revokes the destination, +checking actual script output and destination requests without replacing the +actor. It also checks that deleting a policy while suspended revokes access +after resume and in a checkpoint fork. Runtime configuration stays on the same +prepared revision. New sessions continue using that revision with current grants +even when a policy cannot resolve. `TestSandboxEgressPolicyUpdatesLiveActor` exercises live +grant, revocation, and resume through the standalone sandbox process API. + +`TestEgressPolicyCredentialDelivery` runs the same skill on every harness against +an HTTPS server that rejects requests unless both Authorization (with a Bearer +prefix) and X-API-Key (without a prefix) match the Secret. The script has only +inert placeholders. This test runs sequentially because it temporarily installs +a test origin CA in the Substrate gateway, rolls it out, and restores the gateway +configuration afterwards. Actor identity, policy checks, and credential injection +use the normal Substrate path. + `TestSessionHTTPInteraction` discovers an Agent's Agent Card, invokes the advertised JSON-RPC interface, streams a second turn, and checks task persistence across HTTP and gRPC. `TestSessionHTTPResubscribeAndCancel` diff --git a/go/core/test/e2e/a2a_http_test.go b/go/core/test/e2e/a2a_http_test.go index fb386fdc48..eeae1f04f6 100644 --- a/go/core/test/e2e/a2a_http_test.go +++ b/go/core/test/e2e/a2a_http_test.go @@ -246,8 +246,6 @@ func TestSessionHTTPPushFromNonleader(t *testing.T) { func preparePushLeaderHandoff(t *testing.T) (string, func()) { t.Helper() kube := interactionKubeClient(t) - require.NoError(t, appsv1.AddToScheme(kube.Scheme())) - require.NoError(t, coordinationv1.AddToScheme(kube.Scheme())) deployments := &appsv1.DeploymentList{} require.NoError(t, kube.List(t.Context(), deployments, ctrlclient.InNamespace("kagent"), ctrlclient.MatchingLabels{"app.kubernetes.io/component": "controller"})) require.Len(t, deployments.Items, 1) diff --git a/go/core/test/e2e/egress_credentials_test.go b/go/core/test/e2e/egress_credentials_test.go new file mode 100644 index 0000000000..5573ed6a2e --- /dev/null +++ b/go/core/test/e2e/egress_credentials_test.go @@ -0,0 +1,97 @@ +package e2e_test + +import ( + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" +) + +func TestEgressPolicyCredentialDelivery(t *testing.T) { + t.Parallel() + target := interactionTarget(t) + kube := interactionKubeClient(t) + const token = "explicit-policy-test-token" + destination := startAuthenticatedEgressDestination(t, token) + image := publishSkillImage(t) + forEachHarness(t, func(t *testing.T, harness testHarness) { + t.Parallel() + modelURL, model, runs := startEgressSkillMock(t, harness.skillsRoot(), destination, "credentials") + config := harness.createModel(t, kube, modelURL, nil) + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-credentials-", Namespace: "kagent", Labels: harness.labels()}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: config.Name}, SystemPrompt: "Load the egress skill and run its probe.", + Skills: []v1alpha3.AgentTemplateSkill{{Name: "egress", Source: v1alpha3.ArtifactSource{OCI: image, Path: "egress"}}}, + }} + createAndWaitInteractionTemplate(t, harness, kube, template) + fixture := newInteractionFixtureForTemplate(t, harness, target, template.Name) + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-header-", Namespace: "kagent"}, StringData: map[string]string{"token": token}} + require.NoError(t, kube.Create(t.Context(), secret)) + t.Cleanup(func() { assert.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), secret))) }) + ref := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secret.Name}, Key: "token"} + var refs []corev1.LocalObjectReference + for _, authentication := range []*v1alpha3.EgressAuthentication{ + {Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: ref}, + {Header: "X-API-Key", SecretKeyRef: ref}, + } { + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-header-", Namespace: "kagent"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: destination, Authentication: authentication}}}} + require.NoError(t, kube.Create(t.Context(), policy)) + t.Cleanup(func() { assert.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), policy))) }) + refs = append(refs, corev1.LocalObjectReference{Name: policy.Name}) + } + run := runs("credentials") + finish := runEgressSkill(t, fixture, model, run, "initial_DENIED", "authenticated_ALLOWED") + run.expect(t, "initial", "DENIED") + agent := &v1alpha3.Agent{} + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKey{Namespace: "kagent", Name: template.Name}, agent)) + agent.Spec.EgressPolicyRefs = refs + require.NoError(t, kube.Update(t.Context(), agent)) + run.setPhase("authenticated") + run.expect(t, "authenticated", "ALLOWED") + // The HTTP server returns success only when both placeholders have been + // replaced from the Secret, with and without a prefix respectively. + finish() + }) +} + +func startAuthenticatedEgressDestination(t *testing.T, token string) string { + t.Helper() + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer "+token || r.Header.Get("X-API-Key") != token { + http.Error(w, "credential injection required", http.StatusForbidden) + return + } + _, _ = io.WriteString(w, "EGRESS_DESTINATION_AUTHENTICATED\n") + })) + require.NoError(t, server.Listener.Close()) + listener, err := net.Listen("tcp", "0.0.0.0:0") + require.NoError(t, err) + server.Listener = listener + origin := reachableServerURL(t, "http://"+listener.Addr().String(), "") + parsed, err := url.Parse(origin) + require.NoError(t, err) + server.Start() + t.Cleanup(server.Close) + // Prove a successful fetch cannot be produced by a missing/unchanged header. + for _, value := range []string{"", "kagent-credential-injected"} { + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://"+net.JoinHostPort("127.0.0.1", parsed.Port()), nil) + require.NoError(t, err) + request.Header.Set("Authorization", "Bearer "+value) + request.Header.Set("X-API-Key", value) + response, err := server.Client().Do(request) + require.NoError(t, err) + require.NoError(t, response.Body.Close()) + require.Equal(t, http.StatusForbidden, response.StatusCode) + } + return origin +} diff --git a/go/core/test/e2e/egress_test.go b/go/core/test/e2e/egress_test.go index 0088ccea35..075ecf00f5 100644 --- a/go/core/test/e2e/egress_test.go +++ b/go/core/test/e2e/egress_test.go @@ -2,74 +2,511 @@ package e2e_test import ( "context" + "encoding/json" + "fmt" + "io" "net" "net/http" "net/http/httptest" + "net/http/httputil" "net/url" + "strings" + "sync" "sync/atomic" "testing" + "time" a2atype "github.com/a2aproject/a2a-go/v2/a2a" + a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" + "github.com/a2aproject/a2a-go/v2/a2apb/v1/pbconv" + guestpb "github.com/agent-substrate/env/proto/ateenv/v1alpha" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/google/uuid" + "github.com/kagent-dev/mockllm" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/substrate" ) -func TestSessionEgressDeniesUnconfiguredDestination(t *testing.T) { +// Every harness executes this same foreground skill through its normal shell +// tool. The test changes policy while the invocation is waiting for the probe. +const egressProbeScript = `#!/bin/sh +control=$1 +target=$2 +run=$3 +echo "EGRESS_PROBE_STARTED_$run" +for i in $(seq 1 900); do + phase=$(wget -T 5 -qO- "$control/phase") || exit 1 + if [ "$phase" = stop ]; then + echo "EGRESS_PROBE_FINISHED_$run" + wget -T 5 -qO- "$control/stopped" >/dev/null + exit + fi + if wget -T 3 -qO- --header 'Authorization: Bearer kagent-credential-injected' --header 'X-API-Key: kagent-credential-injected' "$target/?phase=$phase"; then + result=ALLOWED + else + result=DENIED + fi + echo "EGRESS_RESULT_${phase}_$result" + wget -T 5 -qO- "$control/result?phase=$phase&result=$result" >/dev/null || exit 1 + sleep 0.2 +done +exit 1 +` + +func TestSessionEgressPolicyUpdatesLiveActors(t *testing.T) { t.Parallel() + target := interactionTarget(t) + image := publishSkillImage(t) forEachHarness(t, func(t *testing.T, harness testHarness) { t.Parallel() - target := interactionTarget(t) - var reachedDenied atomic.Bool - origin := startModelRecorder(t, startMockLLMServer(t, interactionMocks, "mocks/invoke_agent.json"), func([]byte) error { - reachedDenied.Store(true) - return nil - }) - denied, err := url.Parse(reachableModelURL(t, origin.URL)) - require.NoError(t, err) - // Give the destination a distinct hostname even when host mocks use - // host.docker.internal. The gateway allowlist is keyed by hostname. kube := interactionKubeClient(t) - alias := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{GenerateName: "denied-origin-", Namespace: "kagent"}, - Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeExternalName, ExternalName: denied.Hostname()}, + destination, requests := startEgressDestination(t) + modelURL, model, runs := startEgressSkillMock(t, harness.skillsRoot(), destination, "live", "checkpoint", "fresh", "resumed", "fork") + config := harness.createModel(t, kube, modelURL, nil) + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{GenerateName: "live-egress-", Namespace: "kagent", Labels: harness.labels()}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: config.Name}, SystemPrompt: "Load the egress skill and run its probe.", + Skills: []v1alpha3.AgentTemplateSkill{{Name: "egress", Source: v1alpha3.ArtifactSource{OCI: image, Path: "egress"}}}, + }} + createAndWaitInteractionTemplate(t, harness, kube, template) + fixture := newInteractionFixtureForTemplate(t, harness, target, template.Name) + actor, err := findSubstrateActor(fixture.ctx, fixture.system, "kagent", substrate.ActorName(fixture.sessionID)) + require.NoError(t, err) + actorUID := actor.GetMetadata().GetUid() + agent := &v1alpha3.Agent{} + key := ctrlclient.ObjectKey{Namespace: "kagent", Name: template.Name} + require.NoError(t, kube.Get(t.Context(), key, agent)) + revision := agent.Status.LatestSuccessfulRevision + grant := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "live-access-", Namespace: "kagent"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: destination}}}} + require.NoError(t, kube.Create(t.Context(), grant)) + t.Cleanup(func() { require.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), grant))) }) + setGrant := func(origin string) { + t.Helper() + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKeyFromObject(grant), grant)) + grant.Spec.Destinations[0].Origin = origin + require.NoError(t, kube.Update(t.Context(), grant)) } - require.NoError(t, kube.Create(t.Context(), alias)) - t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), alias)) }) - denied.Host = net.JoinHostPort(alias.Name+".kagent.svc.cluster.local", denied.Port()) - // First prove the destination is reachable and speaks this harness's - // model protocol when explicitly configured. - control := newInteractionFixture(t, harness, target, denied.String()) - _, _, task := control.send(t, "What is 2+2?") - require.Equal(t, a2atype.TaskStateCompleted, task.Status.State) - require.Contains(t, taskText(task), "The answer is 4.") - require.True(t, reachedDenied.Load()) - reachedDenied.Store(false) + run := runs("live") + finish := runEgressSkill(t, fixture, model, run, "initial_DENIED", "allowed_ALLOWED", "conflict_ALLOWED", "fixed_ALLOWED", "revoked_DENIED") + run.expect(t, "initial", "DENIED") + require.Zero(t, requests("live_initial"), "ungranted requests reached the destination") + require.NoError(t, kube.Get(t.Context(), key, agent)) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: grant.Name}} + require.NoError(t, kube.Update(t.Context(), agent)) + run.setPhase("allowed") + run.expect(t, "allowed", "ALLOWED") + require.Positive(t, requests("live_allowed")) + waitEgressCondition := func(reason string) { + t.Helper() + require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, agent); err != nil { + return false, err + } + condition := apimeta.FindStatusCondition(agent.Status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied) + return condition != nil && condition.Reason == reason && + condition.ObservedGeneration == agent.Generation && + apimeta.IsStatusConditionTrue(agent.Status.Conditions, v1alpha3.AgentConditionReady), nil + })) + require.Equal(t, revision, agent.Status.LatestSuccessfulRevision) + } + waitEgressCondition(v1alpha3.EgressPoliciesReasonApplied) + // The opposite protocol on the model's exact host/port conflicts with + // pinned access. Existing grants must keep working in this invocation. + modelOrigin, err := url.Parse(modelURL) + require.NoError(t, err) + require.Equal(t, "http", modelOrigin.Scheme) + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKeyFromObject(grant), grant)) + grant.Spec.Destinations = append(grant.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://" + modelOrigin.Host}) + require.NoError(t, kube.Update(t.Context(), grant)) + waitEgressCondition(v1alpha3.EgressPoliciesReasonConflict) + require.True(t, apimeta.IsStatusConditionFalse(agent.Status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied)) + run.setPhase("conflict") + run.expect(t, "conflict", "ALLOWED") + require.Positive(t, requests("live_conflict")) + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKeyFromObject(grant), grant)) + grant.Spec.Destinations = grant.Spec.Destinations[:1] + require.NoError(t, kube.Update(t.Context(), grant)) + waitEgressCondition(v1alpha3.EgressPoliciesReasonApplied) + require.True(t, apimeta.IsStatusConditionTrue(agent.Status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied)) + run.setPhase("fixed") + run.expect(t, "fixed", "ALLOWED") - var reachedAllowed atomic.Bool - server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - reachedAllowed.Store(true) - // The model host is allowed, but its redirect must not expand the - // Actor's policy to another reachable, unconfigured destination. - redirect := *denied - redirect.Path, redirect.RawQuery = r.URL.Path, r.URL.RawQuery - http.Redirect(w, r, redirect.String(), http.StatusTemporaryRedirect) + setGrant("https://unused.example.com") + run.setPhase("revoked") + run.expect(t, "revoked", "DENIED") + revokedCount := requests("live_revoked") + require.Never(t, func() bool { return requests("live_revoked") != revokedCount }, time.Second, 100*time.Millisecond) + actor, err = findSubstrateActor(fixture.ctx, fixture.system, "kagent", substrate.ActorName(fixture.sessionID)) + require.NoError(t, err) + require.Equal(t, actorUID, actor.GetMetadata().GetUid(), "live updates must not replace the actor") + require.Equal(t, ateapipb.ActorState_ACTOR_STATE_RUNNING, actor.GetStatus().GetState()) + finish() + assertActorSuspended(t, fixture) + + // Save a checkpoint while the source is granted access, then revoke it while + // suspended. Neither resume nor a DATA fork may resurrect that old grant. + setGrant(destination) + run = runs("checkpoint") + finish = runEgressSkill(t, fixture, model, run, "initial_ALLOWED") + run.expect(t, "initial", "ALLOWED") + task := finish() + assertActorSuspended(t, fixture) + checkpoint := createCheckpoint(t, fixture.ctx, fixture.checkpoints, &apiv1.CreateCheckpointRequest{SessionId: fixture.sessionID, RequestId: uuid.NewString(), ExpectedHeadTaskId: string(task.ID)}).GetCheckpoint() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(fixture.ctx), time.Minute) + defer cancel() + _, err := fixture.checkpoints.DeleteCheckpoint(ctx, &apiv1.DeleteCheckpointRequest{CheckpointId: checkpoint.Id}) + require.NoError(t, err) + }) + require.NoError(t, kube.Delete(t.Context(), grant)) + require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, agent); err != nil { + return false, err + } + condition := apimeta.FindStatusCondition(agent.Status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied) + return condition != nil && condition.Reason == v1alpha3.EgressPoliciesReasonNotFound && apimeta.IsStatusConditionTrue(agent.Status.Conditions, v1alpha3.AgentConditionReady), nil })) - _ = server.Listener.Close() - listener, err := net.Listen("tcp", "0.0.0.0:0") - if err != nil { - t.Fatal(err) - } - server.Listener = listener - server.Start() - t.Cleanup(server.Close) - fixture := newInteractionFixture(t, harness, target, reachableModelURL(t, server.URL)) - _, _, task = fixture.send(t, "What is 2+2?") - if !reachedAllowed.Load() { - t.Fatal("the configured model destination was not reached") + require.Equal(t, revision, agent.Status.LatestSuccessfulRevision, "policy edits must not replace the runtime revision") + created, err := fixture.sessions.CreateSession(fixture.ctx, &apiv1.CreateSessionRequest{Agent: &apiv1.ResourceReference{Namespace: key.Namespace, Name: key.Name}, RequestId: uuid.NewString()}) + require.NoError(t, err, "unresolved policy must not block the last successful runtime") + require.Equal(t, revision, created.Session.PreparedRevision) + fresh := *fixture + fresh.sessionID, fresh.contextID = created.Session.Id, created.Session.ContextId + t.Cleanup(func() { + require.NoError(t, deleteIdleSession(context.WithoutCancel(fresh.ctx), fresh.sessions, fresh.sessionID)) + }) + run = runs("fresh") + finish = runEgressSkill(t, &fresh, model, run, "initial_DENIED") + run.expect(t, "initial", "DENIED") + require.Zero(t, requests("fresh_initial")) + finish() + run = runs("resumed") + finish = runEgressSkill(t, fixture, model, run, "initial_DENIED") + run.expect(t, "initial", "DENIED") + require.Zero(t, requests("resumed_initial")) + finish() + assertActorSuspended(t, fixture) + actor, err = findSubstrateActor(fixture.ctx, fixture.system, "kagent", substrate.ActorName(fixture.sessionID)) + require.NoError(t, err) + require.Equal(t, actorUID, actor.GetMetadata().GetUid()) + forked, err := fixture.checkpoints.ForkSession(fixture.ctx, &apiv1.ForkSessionRequest{CheckpointId: checkpoint.Id, RequestId: uuid.NewString()}) + require.NoError(t, err) + fork := *fixture + fork.sessionID = forked.Session.Id + fork.contextID = forked.Session.ContextId + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(fork.ctx), time.Minute) + defer cancel() + require.NoError(t, deleteIdleSession(ctx, fork.sessions, fork.sessionID)) + }) + run = runs("fork") + finish = runEgressSkill(t, &fork, model, run, "initial_DENIED") + run.expect(t, "initial", "DENIED") + require.Zero(t, requests("fork_initial")) + finish() + }) +} + +type egressProbeRun struct { + name string + proxy *httputil.ReverseProxy + mu sync.Mutex + phase string + lastPhase, lastResult string + consecutive int + stopped chan struct{} + stopOnce sync.Once +} + +func (e *egressProbeRun) setPhase(phase string) { + e.mu.Lock() + defer e.mu.Unlock() + e.phase = e.name + "_" + phase +} +func (e *egressProbeRun) expect(t *testing.T, phase, result string) { + t.Helper() + require.Eventually(t, func() bool { + e.mu.Lock() + defer e.mu.Unlock() + return e.lastPhase == e.name+"_"+phase && e.lastResult == result && e.consecutive >= 3 + }, 90*time.Second, 100*time.Millisecond, "%s %s did not become %s", e.name, phase, result) +} +func (e *egressProbeRun) finish(ctx context.Context) error { + e.mu.Lock() + e.phase = "stop" + e.mu.Unlock() + select { + case <-e.stopped: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func runEgressSkill(t *testing.T, fixture *interactionFixture, model *modelRecorder, run *egressProbeRun, expectedResults ...string) func() *a2atype.Task { + t.Helper() + start := len(model.Requests("", "")) + _, request := newMessageRequest(t, "Run egress skill "+run.name+".") + request.Message.ContextId = fixture.sessionID + request.Tenant = fixture.tenant + type response struct { + message *a2apb.SendMessageResponse + err error + } + done := make(chan response, 1) + ctx, cancel := context.WithCancel(fixture.ctx) + t.Cleanup(func() { + cancel() + if t.Failed() { + for _, request := range model.Requests("", "")[start:] { + last, err := lastEgressModelInput(request.Body) + if err == nil && len(last) != 0 { + t.Logf("egress %s model input: %s", run.name, last) + } + } } - require.Equal(t, a2atype.TaskStateFailed, task.Status.State, "redirect task text = %q", taskText(task)) - require.False(t, reachedDenied.Load(), "redirect reached an unconfigured destination") }) + go func() { r, err := sendMessageWithRetry(ctx, fixture.client, request); done <- response{r, err} }() + return func() *a2atype.Task { + t.Helper() + stopCtx, stop := context.WithTimeout(ctx, 15*time.Second) + defer stop() + require.NoError(t, run.finish(stopCtx)) + var result response + select { + case result = <-done: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + require.NoError(t, result.err) + parsed, err := pbconv.FromProtoSendMessageResponse(result.message) + require.NoError(t, err) + task, ok := parsed.(*a2atype.Task) + require.True(t, ok) + require.Equal(t, a2atype.TaskStateCompleted, task.Status.State, taskText(task)) + require.Contains(t, taskText(task), "EGRESS_"+run.name+"_DONE") + // Check actual tool output in fresh model inputs, not the model's fixed text + // or conversation history copied into a resumed session or checkpoint fork. + var outputs []byte + for _, request := range model.Requests("", "")[start:] { + last, err := lastEgressModelInput(request.Body) + require.NoError(t, err) + outputs = append(outputs, last...) + } + for _, marker := range []string{"SKILL_BODY_EGRESS", "EGRESS_PROBE_STARTED_" + run.name, "EGRESS_PROBE_FINISHED_" + run.name} { + require.Contains(t, string(outputs), marker) + } + for _, result := range expectedResults { + require.Contains(t, string(outputs), "EGRESS_RESULT_"+run.name+"_"+result) + } + return task + } +} + +func lastEgressModelInput(raw []byte) (json.RawMessage, error) { + if len(raw) == 0 { + return nil, nil + } + var body struct { + Messages []json.RawMessage `json:"messages"` + Input []json.RawMessage `json:"input"` + } + if err := json.Unmarshal(raw, &body); err != nil { + return nil, err + } + messages := append(body.Messages, body.Input...) + if len(messages) == 0 { + return nil, nil + } + return messages[len(messages)-1], nil +} + +func startEgressDestination(t *testing.T) (string, func(string) int) { + t.Helper() + var mu sync.Mutex + requests := map[string]int{} + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requests[r.URL.Query().Get("phase")]++ + mu.Unlock() + _, _ = io.WriteString(w, "EGRESS_DESTINATION_REACHED\n") + })) + _ = server.Listener.Close() + listener, err := net.Listen("tcp", "0.0.0.0:0") + require.NoError(t, err) + server.Listener = listener + server.Start() + t.Cleanup(server.Close) + destination, err := url.Parse(reachableServerURL(t, server.URL, "")) + require.NoError(t, err) + // Keep a distinct origin even on Docker Desktop, where host mocks otherwise + // share host.docker.internal with the implicitly allowed model endpoint. + kube := interactionKubeClient(t) + alias := &corev1.Service{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-origin-", Namespace: "kagent"}, Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeExternalName, ExternalName: destination.Hostname()}} + require.NoError(t, kube.Create(t.Context(), alias)) + t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), alias)) }) + destination.Host = net.JoinHostPort(alias.Name+".kagent.svc.cluster.local", destination.Port()) + return destination.String(), func(phase string) int { mu.Lock(); defer mu.Unlock(); return requests[phase] } +} + +func startEgressSkillMock(t *testing.T, root, destination string, runNames ...string) (string, *modelRecorder, func(string) *egressProbeRun) { + t.Helper() + require.NotEmpty(t, runNames) + var active atomic.Pointer[egressProbeRun] + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + run := active.Load() + if run == nil { + http.Error(w, "no active probe", http.StatusServiceUnavailable) + return + } + if strings.HasPrefix(r.URL.Path, "/egress/") { + if !strings.HasPrefix(r.URL.Path, "/egress/"+run.name+"/") { + http.Error(w, "old probe", http.StatusGone) + return + } + switch { + case strings.HasSuffix(r.URL.Path, "/phase"): + run.mu.Lock() + phase := run.phase + run.mu.Unlock() + _, _ = io.WriteString(w, phase) + case strings.HasSuffix(r.URL.Path, "/result"): + phase, result := r.URL.Query().Get("phase"), r.URL.Query().Get("result") + run.mu.Lock() + if phase == run.lastPhase && result == run.lastResult { + run.consecutive++ + } else { + run.lastPhase, run.lastResult, run.consecutive = phase, result, 1 + } + run.mu.Unlock() + case strings.HasSuffix(r.URL.Path, "/stopped"): + run.stopOnce.Do(func() { close(run.stopped) }) + default: + http.NotFound(w, r) + } + return + } + run.proxy.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + recorder := startModelRecorder(t, server.URL, nil) + modelURL := reachableModelURL(t, recorder.URL) + origin := strings.TrimSuffix(modelURL, "/v1") + raw, err := interactionMocks.ReadFile("mocks/invoke_egress_skill.json") + require.NoError(t, err) + runs := map[string]*egressProbeRun{} + for _, phase := range runNames { + command := fmt.Sprintf("sh %s/egress/probe.sh '%s/egress/%s' '%s' %s", root, origin, phase, destination, phase) + replacements := []string{"PHASE", phase, "ROOT", root} + args, err := json.Marshal(struct { + Command string `json:"command"` + }{command}) + require.NoError(t, err) + encodedArgs, err := json.Marshal(string(args)) + require.NoError(t, err) + codexArgs, err := json.Marshal(struct { + Command string `json:"cmd"` + YieldTimeMS int `json:"yield_time_ms"` + }{command, 30000}) + require.NoError(t, err) + encodedCodexArgs, err := json.Marshal(string(codexArgs)) + require.NoError(t, err) + encodedCommand, err := json.Marshal(command) + require.NoError(t, err) + replacements = append(replacements, `"COMMAND_ARGUMENTS"`, string(encodedArgs), `"CODEX_COMMAND_ARGUMENTS"`, string(encodedCodexArgs), `"COMMAND"`, string(encodedCommand)) + var config mockllm.Config + require.NoError(t, json.Unmarshal([]byte(strings.NewReplacer(replacements...).Replace(string(raw))), &config)) + upstream, err := url.Parse(startMockLLMConfig(t, config)) + require.NoError(t, err) + run := &egressProbeRun{name: phase, proxy: httputil.NewSingleHostReverseProxy(upstream), stopped: make(chan struct{})} + run.setPhase("initial") + runs[phase] = run + } + active.Store(runs[runNames[0]]) + return modelURL, recorder, func(name string) *egressProbeRun { + t.Helper() + run := runs[name] + require.NotNil(t, run, "unregistered egress probe %q", name) + active.Store(run) + return run + } +} + +func TestSandboxEgressPolicyUpdatesLiveActor(t *testing.T) { + t.Parallel() + f := newSandboxFixture(t) + kube := interactionKubeClient(t) + destination, requests := startEgressDestination(t) + instance := f.create(t, 5*time.Minute) + actor, err := findSubstrateActor(f.ctx, f.system, f.template.Namespace, substrate.ActorName(instance.Id)) + require.NoError(t, err) + actorUID := actor.GetMetadata().GetUid() + probe := func(phase string) string { + t.Helper() + command := fmt.Sprintf("if wget -T 3 -qO- '%s/?phase=%s'; then printf ALLOWED; else printf DENIED; fi", destination, phase) + process, err := f.processes.StartProcess(f.guestContext(instance.Id), &guestpb.StartProcessRequest{Command: []string{"sh", "-c", command}}) + require.NoError(t, err) + outputs, err := f.processes.StreamProcessOutputs(f.guestContext(instance.Id), &guestpb.StreamProcessOutputsRequest{ProcessId: process.ProcessId, Follow: true}) + require.NoError(t, err) + var result strings.Builder + for { + chunk, err := outputs.Recv() + if err == io.EOF { + break + } + require.NoError(t, err) + if chunk.Source == guestpb.OutputSource_OUTPUT_SOURCE_STDOUT { + result.Write(chunk.Data) + } + } + return result.String() + } + require.Contains(t, probe("initial"), "DENIED") + require.Zero(t, requests("initial")) + grant := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "sandbox-access-", Namespace: f.template.Namespace}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: destination}}}} + require.NoError(t, kube.Create(f.ctx, grant)) + t.Cleanup(func() { require.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), grant))) }) + template := &v1alpha3.SandboxTemplate{} + key := ctrlclient.ObjectKey{Namespace: f.template.Namespace, Name: f.template.Name} + require.NoError(t, kube.Get(f.ctx, key, template)) + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: grant.Name}} + require.NoError(t, kube.Update(f.ctx, template)) + require.Eventually(t, func() bool { return strings.Contains(probe("allowed"), "ALLOWED") }, time.Minute, time.Second) + require.Positive(t, requests("allowed")) + require.NoError(t, kube.Delete(f.ctx, grant)) + require.Eventually(t, func() bool { return strings.Contains(probe("revoked"), "DENIED") }, time.Minute, time.Second) + require.NoError(t, wait.PollUntilContextTimeout(f.ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, template); err != nil { + return false, err + } + condition := apimeta.FindStatusCondition(template.Status.Conditions, v1alpha3.SandboxTemplateConditionEgressPoliciesApplied) + return condition != nil && condition.Reason == v1alpha3.EgressPoliciesReasonNotFound && apimeta.IsStatusConditionTrue(template.Status.Conditions, "Ready"), nil + })) + original := instance + instance = f.create(t, 5*time.Minute) + require.Equal(t, original.PreparedRevision, instance.PreparedRevision) + require.Contains(t, probe("fresh"), "DENIED") + require.Zero(t, requests("fresh")) + instance = original + _, err = f.client.SuspendSandbox(f.ctx, &apiv1.SuspendSandboxRequest{SandboxId: instance.Id}) + require.NoError(t, err) + _, err = f.client.ResumeSandbox(f.ctx, &apiv1.ResumeSandboxRequest{SandboxId: instance.Id}) + require.NoError(t, err) + require.Contains(t, probe("resumed"), "DENIED") + require.Zero(t, requests("resumed")) + current, err := f.client.GetSandbox(f.ctx, &apiv1.GetSandboxRequest{SandboxId: instance.Id}) + require.NoError(t, err) + require.Equal(t, instance.PreparedRevision, current.Sandbox.PreparedRevision) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, current.Sandbox.State) + actor, err = findSubstrateActor(f.ctx, f.system, f.template.Namespace, substrate.ActorName(instance.Id)) + require.NoError(t, err) + require.Equal(t, actorUID, actor.GetMetadata().GetUid()) } diff --git a/go/core/test/e2e/harness_test.go b/go/core/test/e2e/harness_test.go index fe86992f59..cb45e61592 100644 --- a/go/core/test/e2e/harness_test.go +++ b/go/core/test/e2e/harness_test.go @@ -6,7 +6,10 @@ import ( ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + adkconfig "github.com/kagent-dev/kagent/go/adk/pkg/config" "github.com/kagent-dev/kagent/go/api/v1alpha3" + claudeconfig "github.com/kagent-dev/kagent/go/harness/claude/config" + codexconfig "github.com/kagent-dev/kagent/go/harness/codex/config" ) type testHarness struct { @@ -56,3 +59,14 @@ func (h testHarness) createModel(t *testing.T, kube ctrlclient.Client, modelURL return nil } } + +func (h testHarness) skillsRoot() string { + switch h.name { + case codexE2EHarness: + return codexconfig.SkillsRoot("/data") + case claudeE2EHarness: + return claudeconfig.SkillsRoot("/data") + default: + return adkconfig.DefaultSkillsRoot + } +} diff --git a/go/core/test/e2e/interaction_test.go b/go/core/test/e2e/interaction_test.go index 1fefbeef47..e0169dd08d 100644 --- a/go/core/test/e2e/interaction_test.go +++ b/go/core/test/e2e/interaction_test.go @@ -37,7 +37,6 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -50,7 +49,7 @@ import ( ) //go:embed mocks/invoke_agent.json mocks/invoke_golang_hitl_ask_user.json mocks/invoke_mcp_agent.json mocks/invoke_shared_agent.json mocks/invoke_structured_output.json mocks/invoke_native_ask_user.json -//go:embed mocks/invoke_mounted_skills.json +//go:embed mocks/invoke_mounted_skills.json mocks/invoke_egress_skill.json var interactionMocks embed.FS const structuredOutputSchema = `{"type":"object","properties":{"answer":{"type":"integer"},"explanation":{"type":"string"}},"required":["answer","explanation"],"additionalProperties":false}` @@ -1243,17 +1242,7 @@ func interactionKubeClient(t *testing.T) ctrlclient.Client { if err != nil { t.Fatalf("load Kubernetes config: %v", err) } - clientScheme := k8sruntime.NewScheme() - if err := corev1.AddToScheme(clientScheme); err != nil { - t.Fatalf("register Kubernetes core API: %v", err) - } - if err := discoveryv1.AddToScheme(clientScheme); err != nil { - t.Fatalf("register discovery API: %v", err) - } - if err := v1alpha3.AddToScheme(clientScheme); err != nil { - t.Fatalf("register kagent API: %v", err) - } - kube, err := ctrlclient.New(cfg, ctrlclient.Options{Scheme: clientScheme}) + kube, err := ctrlclient.New(cfg, ctrlclient.Options{Scheme: suiteKubeScheme}) if err != nil { t.Fatalf("create Kubernetes client: %v", err) } diff --git a/go/core/test/e2e/main_test.go b/go/core/test/e2e/main_test.go index 252d6bfa74..945816a0a0 100644 --- a/go/core/test/e2e/main_test.go +++ b/go/core/test/e2e/main_test.go @@ -8,16 +8,28 @@ import ( "os" "testing" + appsv1 "k8s.io/api/apps/v1" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" + "github.com/kagent-dev/kagent/go/api/v1alpha3" kagentenv "github.com/kagent-dev/kagent/go/core/pkg/env" "github.com/kagent-dev/kagent/go/pkg/logging" ) var suiteTraceReceiver *otlpTraceReceiver +var suiteKubeScheme = runtime.NewScheme() func TestMain(m *testing.M) { ctrl.SetLogger(logging.AsLogr(slog.Default())) + schemeBuilder := runtime.SchemeBuilder{corev1.AddToScheme, discoveryv1.AddToScheme, appsv1.AddToScheme, coordinationv1.AddToScheme, v1alpha3.AddToScheme} + if err := schemeBuilder.AddToScheme(suiteKubeScheme); err != nil { + fmt.Fprintf(os.Stderr, "register Kubernetes APIs: %v\n", err) + os.Exit(1) + } address := kagentenv.E2EOTLPListenAddress.Get() if address == "" { os.Exit(m.Run()) diff --git a/go/core/test/e2e/mocks/invoke_egress_skill.json b/go/core/test/e2e/mocks/invoke_egress_skill.json new file mode 100644 index 0000000000..3042e44924 --- /dev/null +++ b/go/core/test/e2e/mocks/invoke_egress_skill.json @@ -0,0 +1,306 @@ +{ + "openai": [ + { + "name": "skill_PHASE_discover", + "match": { + "match_type": "contains", + "message": { + "role": "user", + "content": "Run egress skill PHASE." + } + }, + "response": { + "id": "skill_PHASE_discover", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-4.1-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_load_egress_PHASE", + "type": "function", + "function": { + "name": "load_skill", + "arguments": "{\"name\": \"egress\"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ] + } + }, + { + "name": "skill_PHASE_execute", + "match": { + "match_type": "contains", + "message": { + "role": "tool", + "content": "SKILL_BODY_EGRESS" + } + }, + "response": { + "id": "skill_PHASE_execute", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-4.1-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_skill_PHASE", + "type": "function", + "function": { + "name": "bash", + "arguments": "COMMAND_ARGUMENTS" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ] + } + }, + { + "name": "skill_PHASE_result", + "match": { + "match_type": "contains", + "message": { + "role": "tool", + "content": "EGRESS_PROBE_FINISHED_PHASE", + "tool_call_id": "call_skill_PHASE" + } + }, + "response": { + "id": "skill_PHASE_result", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-4.1-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "EGRESS_PHASE_DONE" + }, + "finish_reason": "stop" + } + ] + } + } + ], + "openai_response": [ + { + "name": "skill_PHASE_discover", + "match": { + "match_type": "contains", + "input": "Run egress skill PHASE." + }, + "response": { + "id": "skill_PHASE_discover", + "object": "response", + "created_at": 1677652290, + "model": "gpt-5.2-codex", + "output": [ + { + "id": "fc_load_egress_PHASE", + "type": "function_call", + "status": "completed", + "call_id": "call_load_egress_PHASE", + "name": "exec_command", + "arguments": "{\"cmd\": \"cat ROOT/egress/SKILL.md\"}" + } + ], + "parallel_tool_calls": false, + "tools": [], + "usage": { + "input_tokens": 10, + "output_tokens": 8, + "total_tokens": 18 + } + } + }, + { + "name": "skill_PHASE_execute", + "match": { + "match_type": "contains", + "input": "SKILL_BODY_EGRESS" + }, + "response": { + "id": "skill_PHASE_execute", + "object": "response", + "created_at": 1677652290, + "model": "gpt-5.2-codex", + "output": [ + { + "id": "fc_skill_PHASE", + "type": "function_call", + "status": "completed", + "call_id": "call_skill_PHASE", + "name": "exec_command", + "arguments": "CODEX_COMMAND_ARGUMENTS" + } + ], + "parallel_tool_calls": false, + "tools": [], + "usage": { + "input_tokens": 10, + "output_tokens": 8, + "total_tokens": 18 + } + } + }, + { + "name": "skill_PHASE_result", + "match": { + "match_type": "contains", + "input": "EGRESS_PROBE_FINISHED_PHASE" + }, + "response": { + "id": "skill_PHASE_result", + "object": "response", + "created_at": 1677652291, + "model": "gpt-5.2-codex", + "output": [ + { + "id": "msg_skill_PHASE_done", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "EGRESS_PHASE_DONE", + "annotations": [] + } + ] + } + ], + "parallel_tool_calls": false, + "tools": [], + "usage": { + "input_tokens": 14, + "output_tokens": 5, + "total_tokens": 19 + } + } + } + ], + "anthropic": [ + { + "name": "skill_PHASE_discover", + "match": { + "match_type": "contains", + "message": { + "role": "user", + "content": [ + { + "type": "text", + "text": "Run egress skill PHASE." + } + ] + } + }, + "response": { + "id": "skill_PHASE_discover", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_load_egress_PHASE", + "name": "Skill", + "input": { + "skill": "egress" + } + } + ], + "model": "claude-sonnet-4-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 10, + "output_tokens": 4 + } + } + }, + { + "name": "skill_PHASE_execute", + "match": { + "match_type": "contains", + "message": { + "role": "user", + "content": [ + { + "type": "text", + "text": "SKILL_BODY_EGRESS" + } + ] + } + }, + "response": { + "id": "skill_PHASE_execute", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_skill_PHASE", + "name": "Bash", + "input": { + "command": "COMMAND" + } + } + ], + "model": "claude-sonnet-4-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 10, + "output_tokens": 4 + } + } + }, + { + "name": "skill_PHASE_result", + "match": { + "match_type": "contains", + "message": { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_skill_PHASE" + } + ] + } + }, + "response": { + "id": "skill_PHASE_result", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "EGRESS_PHASE_DONE" + } + ], + "model": "claude-sonnet-4-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 14, + "output_tokens": 4 + } + } + } + ] +} diff --git a/go/core/test/e2e/runtime_revision_test.go b/go/core/test/e2e/runtime_revision_test.go index 888ad5ca2b..268d0fc860 100644 --- a/go/core/test/e2e/runtime_revision_test.go +++ b/go/core/test/e2e/runtime_revision_test.go @@ -68,11 +68,12 @@ func TestRuntimeRevisionLifecycle(t *testing.T) { } // No FailedPrecondition retry: Ready must already be backed by a persisted // successful revision when the first create request arrives. - created, err := sessions.CreateSession(ctx, request(templateName)) + createRequest := request(templateName) + created, err := sessions.CreateSession(ctx, createRequest) require.NoError(t, err) source := created.GetSession() t.Cleanup(func() { deleteSession(source.GetId()) }) - sourceTaskID := send(source.GetId()) + send(source.GetId()) // Observe the actual runtime through the public inventory API before deleting // references, so an empty response cannot falsely prove cleanup later. @@ -111,11 +112,18 @@ func TestRuntimeRevisionLifecycle(t *testing.T) { return false, nil })) fallback, err := sessions.CreateSession(ctx, request(templateName)) - require.NoError(t, err, "an invalid edit must preserve the current UID's last-good runtime") - t.Cleanup(func() { deleteSession(fallback.GetSession().GetId()) }) + require.NoError(t, err, "an invalid edit must preserve the last successful runtime for fresh sessions") + fallbackID := fallback.GetSession().GetId() + t.Cleanup(func() { deleteSession(fallbackID) }) + require.NotEqual(t, source.GetId(), fallbackID) require.Equal(t, source.GetPreparedRevision(), fallback.GetSession().GetPreparedRevision()) - send(fallback.GetSession().GetId()) - deleteSession(fallback.GetSession().GetId()) + send(fallbackID) + deleteSession(fallbackID) + retried, err := sessions.CreateSession(ctx, createRequest) + require.NoError(t, err, "creation retries must retain the existing session's pinned runtime") + require.Equal(t, source.GetId(), retried.GetSession().GetId()) + require.Equal(t, source.GetPreparedRevision(), retried.GetSession().GetPreparedRevision()) + sourceTaskID := send(source.GetId()) checkpointResponse := createCheckpoint(t, ctx, checkpoints, &apiv1.CreateCheckpointRequest{SessionId: source.GetId(), RequestId: uuid.NewString(), ExpectedHeadTaskId: sourceTaskID}) checkpointID := checkpointResponse.GetCheckpoint().GetId() @@ -132,19 +140,21 @@ func TestRuntimeRevisionLifecycle(t *testing.T) { agent := &v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: template.Name}} require.NoError(t, kube.Get(ctx, ctrlclient.ObjectKeyFromObject(agent), agent)) require.NoError(t, kube.Delete(ctx, agent)) - // Wait for the controller to retire the pair, using the same public create - // path as a client. Dispose of any session created before retirement wins. + // Kubernetes deletion reaches the database through the controller. Wait for + // retirement, releasing any sessions created before that event is observed + // so they cannot keep the runtime alive during the GC assertion below. require.NoError(t, wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - response, err := sessions.CreateSession(ctx, request(templateName)) + created, err := sessions.CreateSession(ctx, request(templateName)) if status.Code(err) == codes.FailedPrecondition { return true, nil } if err != nil { return false, err } - deleteSession(response.GetSession().GetId()) - return false, nil - })) + id := created.GetSession().GetId() + t.Cleanup(func() { deleteSession(id) }) + return false, deleteIdleSession(ctx, sessions, id) + }), "Agent deletion must eventually block fresh sessions") forked, err := checkpoints.ForkSession(ctx, &apiv1.ForkSessionRequest{CheckpointId: checkpointID, RequestId: uuid.NewString()}) require.NoError(t, err, "a checkpoint must retain runnable inputs after its source and template are deleted") forkID := forked.GetSession().GetId() diff --git a/go/core/test/e2e/session_expiration_test.go b/go/core/test/e2e/session_expiration_test.go index a045e2b468..9ab841c561 100644 --- a/go/core/test/e2e/session_expiration_test.go +++ b/go/core/test/e2e/session_expiration_test.go @@ -92,7 +92,6 @@ func TestSessionIdleExpiration(t *testing.T) { func setSessionExpirationPolicy(t *testing.T, target, ttl, pollInterval string) { t.Helper() kube := interactionKubeClient(t) - require.NoError(t, appsv1.AddToScheme(kube.Scheme())) deployments := &appsv1.DeploymentList{} require.NoError(t, kube.List(t.Context(), deployments, ctrlclient.InNamespace("kagent"), ctrlclient.MatchingLabels{"app.kubernetes.io/component": "controller"})) require.Len(t, deployments.Items, 1) diff --git a/go/core/test/e2e/skills_test.go b/go/core/test/e2e/skills_test.go index de8ca4d81d..d88920d4bc 100644 --- a/go/core/test/e2e/skills_test.go +++ b/go/core/test/e2e/skills_test.go @@ -36,13 +36,7 @@ func TestOCISkillsSurviveSuspension(t *testing.T) { image := publishSkillImage(t) forEachHarness(t, func(t *testing.T, harness testHarness) { t.Parallel() - root := "/skills" - switch harness.name { - case codexE2EHarness: - root = "/data/codex/skills" - case claudeE2EHarness: - root = "/data/generated/claude/.claude/skills" - } + root := harness.skillsRoot() model, setPhase := startMountedSkillsMock(t, root) kube := interactionKubeClient(t) config := harness.createModel(t, kube, reachableModelURL(t, model.URL), nil) @@ -112,6 +106,8 @@ func publishSkillImage(t *testing.T) string { body string mode int64 }{ + {"egress/SKILL.md", "---\nname: egress\ndescription: Probe live HTTP egress permissions.\n---\nSKILL_BODY_EGRESS: Run probe.sh against the requested control and destination URLs.\n", 0o644}, + {"egress/probe.sh", egressProbeScript, 0o755}, {"standalone/SKILL.md", "---\nname: standalone\ndescription: Execute the standalone mounted skill.\n---\nSKILL_BODY_STANDALONE: Run run.sh with the requested phase.\n", 0o644}, {"standalone/run.sh", "#!/bin/sh\nprintf 'mounted-standalone-%s:' \"$1\"\n", 0o755}, {"plugin/plugin.json", `{"$schema":"https://agent-plugins.org/schemas/1.0.0/plugin.schema.json","name":"mounted-test"}`, 0o644}, diff --git a/go/harness/claude/config/paths.go b/go/harness/claude/config/paths.go new file mode 100644 index 0000000000..2b1953ba29 --- /dev/null +++ b/go/harness/claude/config/paths.go @@ -0,0 +1,8 @@ +package config + +import "path/filepath" + +// SkillsRoot is the directory of selected skills in the generated Claude plugin. +func SkillsRoot(dataDir string) string { + return filepath.Join(dataDir, "generated", "claude", ".claude", "skills") +} diff --git a/go/harness/claude/internal/adapter/adapter.go b/go/harness/claude/internal/adapter/adapter.go index b6067d9803..1945cee735 100644 --- a/go/harness/claude/internal/adapter/adapter.go +++ b/go/harness/claude/internal/adapter/adapter.go @@ -59,7 +59,7 @@ func New(ctx context.Context, input Input) (*driver.ProcessDriver, error) { } var pluginDirs []string if cfg.SkillResources != nil { - skillsDir := filepath.Join(skillRoot, ".claude", "skills") + skillsDir := config.SkillsRoot(input.DurableDir) if err := utils.EnsurePrivateDir(skillsDir); err != nil { return nil, fmt.Errorf("prepare generated Claude skills directory: %w", err) } diff --git a/go/harness/codex/config/paths.go b/go/harness/codex/config/paths.go new file mode 100644 index 0000000000..0b76ba467a --- /dev/null +++ b/go/harness/codex/config/paths.go @@ -0,0 +1,8 @@ +package config + +import "path/filepath" + +// SkillsRoot is the directory where Codex discovers installed skills. +func SkillsRoot(dataDir string) string { + return filepath.Join(dataDir, "codex", "skills") +} diff --git a/go/harness/codex/internal/adapter/adapter.go b/go/harness/codex/internal/adapter/adapter.go index 1552695aa5..d92bb6ca02 100644 --- a/go/harness/codex/internal/adapter/adapter.go +++ b/go/harness/codex/internal/adapter/adapter.go @@ -41,7 +41,8 @@ func New(ctx context.Context, input Input) (*driver.ProcessDriver, error) { return nil, fmt.Errorf("workspace and durable directories must be absolute paths") } codexHome := filepath.Join(input.DurableDir, "codex") - for _, directory := range []string{input.Workspace, codexHome, filepath.Join(codexHome, "agents"), filepath.Join(codexHome, "skills")} { + skillsRoot := config.SkillsRoot(input.DurableDir) + for _, directory := range []string{input.Workspace, codexHome, filepath.Join(codexHome, "agents"), skillsRoot} { if err := utils.EnsurePrivateDir(directory); err != nil { return nil, fmt.Errorf("prepare Codex directory %q: %w", directory, err) } @@ -49,13 +50,13 @@ func New(ctx context.Context, input Input) (*driver.ProcessDriver, error) { if err := reconcileGeneratedDir(filepath.Join(codexHome, "agents"), agentFileNames(cfg.Agents)); err != nil { return nil, fmt.Errorf("reconcile Codex agents: %w", err) } - if err := reconcileGeneratedDir(filepath.Join(codexHome, "skills"), nil); err != nil { + if err := reconcileGeneratedDir(skillsRoot, nil); err != nil { return nil, fmt.Errorf("reconcile Codex skills: %w", err) } if cfg.SkillResources != nil { materialized, err := agentplugins.Materialize(ctx, *cfg.SkillResources, agentplugins.Paths{ Packages: filepath.Join(codexHome, "packages"), - Skills: filepath.Join(codexHome, "skills"), + Skills: skillsRoot, }) if err != nil { return nil, fmt.Errorf("materialize Codex skills: %w", err) diff --git a/helm/kagent-crds/templates/api.kagent.dev_agents.yaml b/helm/kagent-crds/templates/api.kagent.dev_agents.yaml index bdc7bbab3e..6ba119ed9f 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_agents.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_agents.yaml @@ -52,6 +52,36 @@ spec: are local to the Agent's namespace, including references nested in inline template specs. properties: + egressPolicyRefs: + description: |- + EgressPolicyRefs grants additional access to the whole runtime, including + shared subagents. Policies resolve in this Agent's namespace. Integration + access inferred from models and tools is always included. Changes update + existing Sessions without a new invocation or runtime restart. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: egressPolicyRefs requires non-empty names + rule: self.all(ref, has(ref.name) && ref.name != '') harnessRef: description: |- HarnessRef selects the platform-managed runtime configuration that executes @@ -487,6 +517,11 @@ spec: description: AgentStatus reports compilation and preparation of an Agent. properties: conditions: + description: |- + Conditions report runtime readiness and independent egress grant admission. + EgressPoliciesApplied is False for EgressPolicyNotFound, InvalidEgressPolicy, + or EgressGrantConflict; Applied means all grants are admitted. Grant issues + do not affect Ready. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -541,7 +576,7 @@ spec: - status - type type: object - maxItems: 4 + maxItems: 5 type: array x-kubernetes-list-map-keys: - type diff --git a/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml b/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml new file mode 100644 index 0000000000..3945fc758b --- /dev/null +++ b/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml @@ -0,0 +1,140 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.22.0 + name: egresspolicies.api.kagent.dev +spec: + group: api.kagent.dev + names: + categories: + - kagent + kind: EgressPolicy + listKind: EgressPolicyList + plural: egresspolicies + singular: egresspolicy + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: |- + EgressPolicy is a reusable grant of additional network access. Workloads in its + namespace select it explicitly; changes update their existing runtime Actors. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EgressPolicySpec grants additional HTTP(S) access to workloads + in its namespace. + properties: + destinations: + description: Destinations are additional origins the workload may + reach. + items: + description: EgressDestination grants access to one HTTP(S) origin, + including all its paths. + properties: + authentication: + description: |- + Authentication replaces a placeholder header at the gateway. Requests + must send the header; its credential value is never supplied to the runtime. + properties: + header: + description: |- + Header is the request header carrying the credential, for example + Authorization or X-API-Key. + maxLength: 256 + pattern: ^[!#$%&'*+.^_|~0-9A-Za-z-]+$ + type: string + x-kubernetes-validations: + - message: this header cannot carry an egress credential + rule: '!(self.lowerAscii() in [''host'', ''connection'', + ''content-length'', ''transfer-encoding'', ''te'', ''trailer'', + ''upgrade'', ''keep-alive'', ''proxy-authorization'', + ''proxy-authenticate'', ''cookie''])' + prefix: + description: |- + Prefix is prepended verbatim to the Secret value, for example "Bearer ". + Include any separating space. Omitted or empty means no prefix. + maxLength: 256 + pattern: ^[^\x00-\x08\x0a-\x1f\x7f]*$ + type: string + secretKeyRef: + description: SecretKeyRef selects a required key in the + EgressPolicy's namespace. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: secretKeyRef requires a name and key + rule: has(self.name) && self.name != '' && self.key != + '' + - message: egress credentials cannot be optional + rule: '!has(self.optional) || !self.optional' + required: + - header + - secretKeyRef + type: object + origin: + description: |- + Origin is a scheme, DNS hostname, and optional port. A wildcard replaces + exactly one leftmost DNS label, for example https://*.example.com. + maxLength: 270 + pattern: ^https?://(\*\.([a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?\.)+|([a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?\.)*)[a-z]([-a-z0-9]{0,61}[a-z0-9])?(:([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?$ + type: string + required: + - origin + type: object + x-kubernetes-validations: + - message: authentication requires an exact HTTP(S) origin + rule: '!has(self.authentication) || !self.origin.contains(''*'')' + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - origin + x-kubernetes-list-type: map + required: + - destinations + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/helm/kagent-crds/templates/api.kagent.dev_remotemcpservers.yaml b/helm/kagent-crds/templates/api.kagent.dev_remotemcpservers.yaml index 0ade50f33d..5f78ca86b9 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_remotemcpservers.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_remotemcpservers.yaml @@ -128,10 +128,15 @@ spec: description: type: string headersFrom: + description: |- + HeadersFrom supplies request headers. Secret-backed headers are injected by + the egress gateway and cannot use routing, framing, hop-by-hop, + proxy-authentication, or Cookie headers. At most 64 headers may be supplied. items: description: ValueRef represents a configuration value properties: name: + maxLength: 256 type: string value: type: string @@ -161,9 +166,20 @@ spec: - name type: object x-kubernetes-validations: + - fieldPath: .name + message: Secret-backed headers require a valid HTTP header name + rule: '!has(self.valueFrom) || self.valueFrom.type != ''Secret'' + || self.name.matches(''^[!#$%&\\x27*+.^_`|~0-9A-Za-z-]+$'')' + - fieldPath: .name + message: this header cannot carry an egress credential + rule: '!has(self.valueFrom) || self.valueFrom.type != ''Secret'' + || !(self.name.lowerAscii() in [''host'', ''connection'', ''content-length'', + ''transfer-encoding'', ''te'', ''trailer'', ''upgrade'', ''keep-alive'', + ''proxy-authorization'', ''proxy-authenticate'', ''cookie''])' - message: exactly one of the fields in [value valueFrom] must be set rule: (has(self.value)?1:0)+(has(self.valueFrom)?1:0) == 1 + maxItems: 64 type: array protocol: default: STREAMABLE_HTTP diff --git a/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml b/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml index 4749ca0147..bf2fd9509d 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml @@ -51,6 +51,34 @@ spec: description: SandboxTemplateSpec defines standalone sandbox environment configuration. properties: + egressPolicyRefs: + description: |- + EgressPolicyRefs grants additional access in this template's namespace. + Changes update the access of existing sandboxes without restarting them. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: egressPolicyRefs requires non-empty names + rule: self.all(ref, has(ref.name) && ref.name != '') env: description: |- Env supplies runtime environment defaults. @@ -132,7 +160,11 @@ spec: template inputs. properties: conditions: - description: Conditions describe whether a prepared revision is available. + description: |- + Conditions report runtime readiness and independent egress grant admission. + EgressPoliciesApplied is False for EgressPolicyNotFound, InvalidEgressPolicy, + or EgressGrantConflict; Applied means all grants are admitted. Grant issues + do not affect Ready. items: description: Condition contains details for one aspect of the current state of this API Resource. diff --git a/helm/kagent/templates/rbac/getter-role.yaml b/helm/kagent/templates/rbac/getter-role.yaml index c136b629ec..007b3fbab0 100644 --- a/helm/kagent/templates/rbac/getter-role.yaml +++ b/helm/kagent/templates/rbac/getter-role.yaml @@ -4,6 +4,7 @@ resources: - harnesses - agents + - egresspolicies - sandboxtemplates - agenttemplates - modelconfigs diff --git a/helm/kagent/tests/rbac_test.yaml b/helm/kagent/tests/rbac_test.yaml index ef6d6debb2..4e5a82d805 100644 --- a/helm/kagent/tests/rbac_test.yaml +++ b/helm/kagent/tests/rbac_test.yaml @@ -140,9 +140,18 @@ tests: - equal: path: rules[0].apiGroups value: [api.kagent.dev] + - equal: + path: rules[0].resources + value: [harnesses, agents, egresspolicies, sandboxtemplates, agenttemplates, modelconfigs, modelproviderconfigs, remotemcpservers] + template: rbac/getter-role.yaml + - equal: + path: rules[0].verbs + value: [get, list, watch] + template: rbac/getter-role.yaml - equal: path: rules[0].resources value: [harnesses, agents, sandboxtemplates, agenttemplates, modelconfigs, modelproviderconfigs, remotemcpservers] + template: rbac/writer-role.yaml - it: should grant namespaced agent access in the new group set: @@ -154,9 +163,18 @@ tests: - equal: path: rules[0].apiGroups value: [api.kagent.dev] + - equal: + path: rules[0].resources + value: [harnesses, agents, egresspolicies, sandboxtemplates, agenttemplates, modelconfigs, modelproviderconfigs, remotemcpservers] + template: rbac/getter-role.yaml + - equal: + path: rules[0].verbs + value: [get, list, watch] + template: rbac/getter-role.yaml - equal: path: rules[0].resources value: [harnesses, agents, sandboxtemplates, agenttemplates, modelconfigs, modelproviderconfigs, remotemcpservers] + template: rbac/writer-role.yaml - it: should have correct labels on all resources templates: diff --git a/ui/src/api/domain/agents.ts b/ui/src/api/domain/agents.ts index 2ee45c74a6..e818b80df4 100644 --- a/ui/src/api/domain/agents.ts +++ b/ui/src/api/domain/agents.ts @@ -8,6 +8,7 @@ export type AgentTemplateSource = export interface AgentSpec { template: AgentTemplateSource; harnessRef: { name: string }; + egressPolicyRefs?: { name: string }[]; } /** One condition the controller recorded for an Agent. */ export interface AgentCondition { diff --git a/ui/src/pages/agents/AgentFormPage.tsx b/ui/src/pages/agents/AgentFormPage.tsx index 17eb636b01..44bc160331 100644 --- a/ui/src/pages/agents/AgentFormPage.tsx +++ b/ui/src/pages/agents/AgentFormPage.tsx @@ -117,6 +117,7 @@ function AgentForm({ agent }: { agent?: Agent }) { setSaving(true); setFailure(undefined); const agentSpec = { + ...(spec?.egressPolicyRefs ? { egressPolicyRefs: spec.egressPolicyRefs } : {}), template: templateSource === "reference" ? { ref: { name: templateRef! } } : { inline: specFromDraft(inlineTemplate, spec?.template.inline) },