diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 01ba154179..4ea4f4c487 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -14,6 +14,7 @@ the agent processes. | `Harness` | Kubernetes (`api.kagent.dev/v1alpha3`) | Runtime implementation, workload, credentials, capacity, and snapshot policy | | `AgentTemplate` | Kubernetes (`api.kagent.dev/v1alpha3`) | Portable agent behavior: model, prompt, tools, skills, and plugins | | `SandboxTemplate` | Kubernetes (`api.kagent.dev/v1alpha3`) | Tools image, environment, and preparation policy for standalone sandboxes | +| `EgressPolicy` | Kubernetes (`api.kagent.dev/v1alpha3`) | Additional HTTP(S) origins and gateway credentials selected by Agents and SandboxTemplates | | prepared revision | PostgreSQL and ate-api | Immutable compiled runtime input and its Substrate ActorTemplate | | `Session` | PostgreSQL, exposed by gRPC | Stable conversation identity and runtime lifecycle | | `Sandbox` | PostgreSQL, exposed by gRPC and MCP | Owner-scoped scratch environment with process/file access and expiration | @@ -95,6 +96,8 @@ cleanup. Session lifecycle workers independently pause/suspend idle Actors. ## Documents +- [Egress policies](egress-policies.md) +- [Runtime credential injection](credential-injection.md) - [Configuration and compilation](configuration-and-compilation.md) - [Runtime and lifecycle](runtime-and-lifecycle.md) - [Client lifecycle retries](../lifecycle-retries.md) diff --git a/docs/architecture/configuration-and-compilation.md b/docs/architecture/configuration-and-compilation.md index 5340355368..b05b685714 100644 --- a/docs/architecture/configuration-and-compilation.md +++ b/docs/architecture/configuration-and-compilation.md @@ -172,9 +172,11 @@ revision. AgentTemplate is shared context with no runtime status. Child template references are resolved under the root Agent's Harness; they do not require separate Agents or matching Harness references. -A failed compile or apply leaves the previous successful revision available. -Sessions pin a prepared revision, so later template edits do not mutate a -running session. +A failed compile or apply retains the previous successful revision, which +remains available for fresh Sessions during pending or failed edits. Initial +creation requires a successful revision. Existing Sessions and creation retries +keep their pinned revision. Explicit egress grants use current policies +independently of runtime preparation. Harness compilers only translate inputs. The controller and Substrate adapter own application and readiness. The central entry points are @@ -391,7 +393,8 @@ no synthetic Kubernetes object is created for an inline template. Create a session with `kagent agent session create --agent assistant -n kagent`. The gRPC create request and ScheduledRun target one `agent` resource reference. -The controller selects that Agent's latest successful revision. Deleting an Agent +The store selects that Agent's latest successful revision, including while an +edit is pending or failed. Deleting an Agent retires its definition; sessions and checkpoints retain their pinned revisions. Recreating the same name creates a new identity and cannot inherit the old Agent's last successful revision. diff --git a/docs/architecture/credential-injection.md b/docs/architecture/credential-injection.md index 3e29ad1f5c..78def5dc58 100644 --- a/docs/architecture/credential-injection.md +++ b/docs/architecture/credential-injection.md @@ -1,16 +1,21 @@ # Runtime credential injection -Kagent requires Substrate **v0.4.0-alpha1**. The compiler turns ModelConfig API -keys and Secret-backed RemoteMCPServer headers into destination-scoped egress -bindings. Substrate's gateway fetches the referenced Kubernetes Secret and +Kagent requires Substrate **v0.5.0-alpha2**. The runtime compiler turns +ModelConfig API keys and Secret-backed RemoteMCPServer headers into pinned +destination-scoped bindings. Independently, EgressPolicy admission supplies +additional live bindings. Substrate's gateway fetches the referenced Kubernetes Secret and replaces the outgoing HTTP header when the request carries a placeholder. SDKs receive an inert placeholder where they require an API key; real credentials never enter compiled environments, runtime configuration, or revision provenance. -Bindings are persisted with the prepared revision and installed before a -session becomes ready, including retries and checkpoint forks. Secret names, -keys, destinations, and headers affect revision identity. Secret values and -Secret UIDs do not. Rotation is handled by the gateway; its credential cache can +Inferred integration bindings are persisted with the prepared revision; their +Secret names, keys, destinations, and headers affect revision identity. Explicit +EgressPolicy bindings are resolved from current policies and update live actors +without changing that revision. Conflicts hold new grants and report the +independent [EgressPoliciesApplied condition](egress-policies.md#preparation-and-lifecycle); +they do not block runtime compilation. Both are installed before execution, including +creation retries, resume, and checkpoint forks. Secret values and Secret UIDs +do not affect revision identity. Rotation is handled by the gateway; its credential cache can take up to five minutes to refresh, without recompiling or restarting an agent. ## Installation @@ -58,22 +63,25 @@ overrides of these variables are rejected. | RemoteMCPServer Secret-backed header | Configured header; Secret contains its full value | Provider endpoint overrides determine the allowed HTTP(S) origin. Egress rules -match its scheme, DNS name, and port; credential bindings remain scoped to the -hostname and header. Different credentials for the same hostname and header -are rejected, including conflicts between models, memory embeddings, and MCP -servers. Use distinct DNS names for -origins requiring different credentials. IP-address destinations are unsupported -by Substrate egress policies. +and credential bindings both match its scheme, DNS name, and port. Different +credentials for the same origin and header are rejected, including conflicts +between models, memory embeddings, MCP servers, and explicit policies. Distinct +ports on the same DNS name can use different credentials. IP-address destinations +are unsupported by Substrate egress policies. + +The v1alpha3 revision format stores full origins. It assumes a clean installation; +previously persisted hostname-only credential bindings are rejected rather than +reinterpreted across every port on that host. Harness and SandboxTemplate environment entries accept only literal `value` strings, including empty strings. Configure Secret-backed credentials on -ModelConfig or RemoteMCPServer for gateway injection. +ModelConfig, RemoteMCPServer, or [EgressPolicy](egress-policies.md) for gateway injection. AWS IAM signing keys, Google service-account keys, and OAuth client credentials require mechanisms beyond static header injection and are rejected rather than serialized into runtimes. Caller-token passthrough retains its existing behavior. A passthrough model -cannot share a hostname with static gateway credentials, which would override +cannot share an origin with static gateway credentials, which would override the caller's authentication. ## Deferred API fields diff --git a/docs/architecture/egress-policies.md b/docs/architecture/egress-policies.md new file mode 100644 index 0000000000..e2616f9987 --- /dev/null +++ b/docs/architecture/egress-policies.md @@ -0,0 +1,278 @@ +# Egress policies + +`EgressPolicy` grants additional HTTP(S) access to an Agent or standalone +sandbox. An Agent selects policies through `spec.egressPolicyRefs`; a +SandboxTemplate selects them through the same field. References resolve in the +workload's namespace. Policies are managed through the Kubernetes API. + +AgentTemplate owns reusable behavior, and Harness owns how it runs. Neither +selects egress policies. This lets two Agents reuse a template while granting +different access for their tasks. A shared child template contributes its +inferred model, MCP, and artifact destinations, but cannot select additional +policies. The root and all shared children execute in one Actor and share its +complete grant; there is no per-child network isolation. + +## Grant access for a task + +For example, a dependency-installation policy can grant access to registries: + +```yaml +apiVersion: api.kagent.dev/v1alpha3 +kind: EgressPolicy +metadata: + name: node-dependencies + namespace: team-a +spec: + destinations: + - origin: https://registry.npmjs.org + - origin: https://nodejs.org +--- +apiVersion: api.kagent.dev/v1alpha3 +kind: Agent +metadata: + name: code-reviewer + namespace: team-a +spec: + template: + ref: + name: code-review + harnessRef: + name: codex + egressPolicyRefs: + - name: node-dependencies +``` + +An origin is a scheme, DNS name, and optional port. Omitted ports mean 443 for +HTTPS and 80 for HTTP. Every path on that origin is allowed. URL paths, queries, +fragments, user information, IP addresses, and non-HTTP protocols are rejected. +A wildcard replaces exactly one leftmost label: `https://*.example.com` +allows `api.example.com`, but neither `example.com` nor `a.b.example.com`. +`*` and `*.com` are not accepted. Redirects and downloads from other origins +require their own grants. A registry grant is not a guarantee that every +package's install script or release download can reach its dependencies. + +For a standalone sandbox, select the policy on its template: + +```yaml +apiVersion: api.kagent.dev/v1alpha3 +kind: SandboxTemplate +metadata: + name: node-tools + namespace: team-a +spec: + workload: + image: registry.example.com/tools@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + substrate: + workerPoolRef: + name: default + snapshotPolicy: + location: s3://example/snapshots/ + egressPolicyRefs: + - name: node-dependencies +``` + +Each workload can select up to 16 policies, each containing 1–64 destinations. +Grant admission combines them with inferred integration access, removes equivalent +origins, and orders the result deterministically. The final Actor policy is +limited to 256 rules. References are additive; they cannot restrict inferred +model, MCP, artifact, or control-plane access. No references means no additional +access. A standalone sandbox with no references has no allowed egress. + +## Call an API with a credential + +Policies authenticate requests using a header, an optional prefix, and a Secret +in the same namespace. Authentication requires an exact HTTP(S) origin. HTTP +supports in-cluster services whose transport is secured by a service mesh; the gateway injects the +credential without exposing it to the guest: + +```yaml +apiVersion: api.kagent.dev/v1alpha3 +kind: EgressPolicy +metadata: + name: ticket-api + namespace: team-a +spec: + destinations: + - origin: https://tickets.example.com + authentication: + header: Authorization + prefix: "Bearer " + secretKeyRef: + name: ticket-auth + key: token + - origin: https://search.example.com:8443 + authentication: + header: X-API-Key + secretKeyRef: + name: search-auth + key: api-key +``` + +The gateway sets the header value to `prefix + secretValue`. Prefixes are literal: +include the trailing space in `"Bearer "`. Omitting the prefix or setting it to +an empty string uses the Secret value directly, as with an API-key header. + +Create these Secrets separately. Policy resolution validates that each Secret and key +exists, but persists only the reference. Substrate's credential provider fetches +the value and the gateway replaces the request header. Scripts and SDKs must +send that header, using an inert placeholder: + +```sh +curl -H 'Authorization: Bearer kagent-credential-injected' \ + https://tickets.example.com/tickets +curl -H 'X-API-Key: kagent-credential-injected' \ + https://search.example.com:8443/search +``` + +The gateway replaces an existing header; it does not add a missing one. Policy +selection does not automatically configure arbitrary tools or environment +variables. Clients must honor the projected gateway trust bundle. See +[credential injection](credential-injection.md) for provider namespace grants, +CA setup, inferred SDK placeholders, and rotation behavior. + +Bindings use the complete origin and a case-insensitive header name. Different +ports can use different credentials. Identical bindings merge; conflicting +Secret references or prefixes for the same origin/header hold new grants. +Pinned integration bindings take precedence over additional grants. Caller-token passthrough +cannot share its origin with static gateway credentials. Routing, framing, +hop-by-hop, proxy-authentication, and Cookie headers cannot carry credentials. +At most 16 credential headers are supported per origin. + +`RemoteMCPServer.spec.headersFrom` applies the same credential-header rules to +Secret-backed entries during Kubernetes admission. Literal and ConfigMap-backed +headers do not use gateway credential injection. A server may configure up to +64 headers, with names of at most 256 characters. + +An exact credential-bearing rule wins over a matching wildcard rule. Credentials +are never attached to wildcard rules or implicitly copied to other ports or +schemes. HTTP and HTTPS cannot both use the same hostname pattern and port, +because Substrate cannot distinguish that rule selection. + +## Preparation and lifecycle + +KRT watches selected policies and their Secrets independently of runtime +compilation. Each policy resolves separately. A missing or deleting policy, +missing Secret key, or invalid policy contributes no access; grants from the +remaining policies are still considered. Invalid reference names or more than +16 references revoke all additional access. Grant errors never block runtime +compilation or change revision identity. + +Both Agent and SandboxTemplate expose an independent `EgressPoliciesApplied` +condition: + +| Status | Reason | Meaning | +| --- | --- | --- | +| True | `Applied` | All requested grants are admitted. | +| False | `EgressPolicyNotFound` | A referenced policy is missing or deleting. | +| False | `InvalidEgressPolicy` | A policy, Secret reference, or reference list is invalid. | +| False | `EgressGrantConflict` | New grants are held; still-requested, compatible previous grants remain. | + +Messages identify affected policies and conflicts. A conflict takes precedence +when there are also resolution errors. This condition never affects `Ready` or +`ResolvedRefs`; missing policies no longer fail runtime reference resolution. +It reports admission from the controller's current inputs, not acknowledgement +that every actor has received the update. + +Runtime revisions retain the integration access inferred from models and tools, +including credential references and caller-token passthrough constraints. This +pinned access is the revision's base. +The combined additional access from a source's referenced EgressPolicies is its +`EgressGrant`: destinations it may contact and credentials the gateway may inject. +Reconciliation and source definition structs carry it as `EgressGrant egress.Access`. +Revision identity covers compiled runtime inputs and the owning Kubernetes UID. +The full source specification is preserved as provenance for the first preparation +of that revision. Explicit policy grants and policy references do not enter +revision identity. +Changing a grant or its Secret reference does not prepare a new runtime image. +Secret values remain outside compiled inputs; the gateway handles value rotation. + +The elected controller's Agent and SandboxTemplate reconcilers resolve current +grants independently of runtime compilation. A missing Harness, AgentTemplate, +or WorkerPool cannot freeze a revocation. Each preparation queue is the sole +writer of its source definition's grants in PostgreSQL. Definition updates +persist admitted grants and retire replaced Kubernetes UIDs atomically. + +The stored grant set is the last good set: valid on its own and compatible with +the desired revision's base when compilation succeeds. If the complete candidate +conflicts, preparation retains only previously admitted destinations and identical +credential bindings that are still requested, filtering any that conflict with +the desired base. Removals apply immediately; all additions wait until the +conflict is fixed. A new Kubernetes UID inherits no grants. When unrelated +compilation fails, admission checks grants on their own and still applies +revocations. + +A single store transaction records the runtime revision and, when ready, promotes it if +the active source still desires it. Failed runtime preparation preserves +the last successful revision while continuing to update source grants. + +Effective policy is computed when read: the revision's pinned integration access +is merged with grants from its active source UID. Retired or replaced identities +contribute no grants. There is no separately published per-revision policy and +no notification needed when a revision is recorded: every read joins against +the current persisted grants, including for newly recorded revisions. + +Every preparation persists the source definition, then pages its allocated actors +across all retained revisions and Kubernetes UIDs and applies effective policy +with at most 16 actor updates in parallel. Unchanged policies skip the Substrate +write. Failures return to the existing preparation queue, which retries the whole +source with exponential backoff even when grants have not changed. Slow actor +updates therefore delay further work on that preparation queue. Startup replays +every Agent and SandboxTemplate through preparation; no separate actor sweep is +needed. +After its input cache syncs, Agent preparation also enqueues stored definitions +whose Agents are absent from watched namespaces. Retired identities are included +only while they have non-deleted instances, so a restart retries interrupted +revocations without re-enqueueing every historic source. Agent deletion does not +need a finalizer. SandboxTemplate preparation retains its existing +finalizer until retirement and actor updates succeed. + +Running and suspended actors receive updates without replacement or a new +invocation. Missing or invalid policies and Secrets revoke only the affected +policies' grants. Actors on older revisions can have different pinned access: +their effective policy drops only the destinations or credentials that conflict +with that base, caller-token origins, or combined limits. Nonconflicting grants +remain. The pinned base always wins; dropping a credential retains its network +destination if the destination itself is compatible. Missing revisions or malformed persisted inputs return errors and +block execution. + +New Sessions and sandboxes pin the latest successful runtime revision, including +while a configuration edit is unresolved, preparing, or failed. Initial creation +still requires a successful revision. Existing instances and checkpoint forks +retain their runtime configuration. + +Create, creation retries, and resume load and apply the effective policy from +persisted revision inputs and source grants. They do not read or translate +Kubernetes policy. The workflow creates a suspended Actor and installs its +`default` egress policy before it can +run; a fork cannot restore an old policy from a snapshot. Applications read the +Substrate policy version before loading intended policy, then write using its UID +and version preconditions. Conflicts reload both. After applying, they recheck +intended policy to catch updates whose actor discovery raced with creation. +Missing revisions or malformed persisted inputs block execution and are retried. +No database transaction or lock is held across Substrate calls. + +Policy convergence is asynchronous: a Kubernetes write is not a synchronous +revocation barrier. Once the gateway observes the changed policy, subsequent +requests use it. An update does not cancel a request already in flight. Golden +Actor preparation and artifact-startup egress are separate from this API's runtime +Actor access. + +## Authorization and scope + +Policies are namespace-scoped configuration. Administrators can grant separate +RBAC permissions to author EgressPolicy objects; kagent's controller needs only +get/list/watch. Workload authors who can select a policy in that namespace can +use its grant. There is no separate per-policy `use` authorization check. Treat +the namespace as the sharing boundary for selectable policies and credentials. + +The same trust-boundary consideration applies to existing ModelConfig and +RemoteMCPServer Secret references. This resource does not turn the namespace +into a per-workload Secret isolation boundary. Credential injection keeps values +out of compiled inputs; it does not prevent an allowed destination from echoing +credentials back to a client. Do not grant credentials to destinations that +should not receive or disclose them. + +This API intentionally does not expose Substrate rule effects, raw TCP/SSH, +TLS passthrough, OAuth exchange, request signing, cross-namespace references, +or unrestricted public-internet access. Substrate remains the enforcement +backend; kagent exposes task-oriented reusable grants and static authentication. diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 53fb8a1090..f6b10efe95 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -6,7 +6,14 @@ Kubernetes resource. ## Creation and state -Creation selects the latest successful revision for the Agent, creates a deterministic Actor initially suspended, and marks the session +Fresh creation pins the Agent's latest successful runtime revision. Pending or +failed edits leave that revision available; an Agent with no successful revision +cannot create sessions. Creation retries retain the existing session's pinned +revision. Explicit egress grants are applied from the controller's persisted +intended policy before execution. Lifecycle +operations do not read or translate Kubernetes policy. + +The control plane creates a deterministic Actor initially suspended and marks the session ready after Substrate accepts it. Readiness of the image was already established while preparing the ate-api ActorTemplate; Session creation does not resume an Actor merely to probe `/readyz`. diff --git a/docs/architecture/sandboxes.md b/docs/architecture/sandboxes.md index c2aabd6673..3a323cb83c 100644 --- a/docs/architecture/sandboxes.md +++ b/docs/architecture/sandboxes.md @@ -97,8 +97,11 @@ finalizer, including when deletion starts while the controller is offline. The controller advances the latest-successful revision only after the golden snapshot is ready and reports preparation through the template's `Ready` condition and observed generation. Failed preparation preserves the previous -successful revision. Existing Sandboxes retain their revision when a template -changes. Template deletion retires preparation; retained instances continue to +successful revision, which remains available for fresh creation while edits +are unresolved, pending, or failed. Initial creation requires a successful +revision. Existing Sandboxes and +creation retries retain their revision when a template changes. +Template deletion retires preparation; retained instances continue to pin the runtime artifacts they need. Sandbox execution is enabled by default. Lifecycle calls run on the serving API @@ -113,8 +116,14 @@ TTL. Helm supplies these through the controller ConfigMap. Defaults are one CPU, 1 GiB memory, a one-hour TTL, and a 24-hour maximum TTL. Activity does not extend expiration. -Sandboxes have no allowed egress destinations. A follow-up will add destination -configuration to `SandboxTemplate` and pin it with each prepared revision. +Sandboxes have no allowed egress destinations by default. Select same-namespace +[EgressPolicy resources](egress-policies.md) through `SandboxTemplate.spec.egressPolicyRefs` +to grant additional HTTP(S) origins and static gateway credentials. The controller +persists current grants on the source definition; effective policy is merged with +pinned revision access when read. +Policy edits update running and suspended sandboxes without replacing their Actor. +Creation and resume apply that effective policy before execution, without +translating Kubernetes inputs. ## Lifecycle and recovery diff --git a/go/adk/cmd/main.go b/go/adk/cmd/main.go index 1a18a9b0cc..bfa5d6b018 100644 --- a/go/adk/cmd/main.go +++ b/go/adk/cmd/main.go @@ -28,7 +28,6 @@ import ( const ( defaultPluginPackagesRoot = "/plugins" - defaultSkillsRoot = "/skills" defaultPluginDataRoot = "/data/plugins" ) @@ -73,7 +72,7 @@ func run(logLevel, host, port, configDir string) error { logging.IntoContext(context.Background(), logger), agentConfig, config.AgentPluginPaths{ Packages: defaultPluginPackagesRoot, - Skills: defaultSkillsRoot, + Skills: config.DefaultSkillsRoot, Data: defaultPluginDataRoot, }, ); err != nil { diff --git a/go/adk/pkg/config/agentplugins.go b/go/adk/pkg/config/agentplugins.go index db0ab56f01..f4bf1adb19 100644 --- a/go/adk/pkg/config/agentplugins.go +++ b/go/adk/pkg/config/agentplugins.go @@ -9,6 +9,9 @@ import ( "github.com/kagent-dev/kagent/go/core/pkg/agentplugins" ) +// DefaultSkillsRoot is where the ADK installs root-agent skills. +const DefaultSkillsRoot = "/skills" + // AgentPluginPaths contains the ADK runtime destinations for Agent Plugin // packages, selected skills, and mutable MCP server data. type AgentPluginPaths struct { diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/api_client.go b/go/api/clientset/versioned/typed/api/v1alpha3/api_client.go index 97de970b4a..901e7cf393 100644 --- a/go/api/clientset/versioned/typed/api/v1alpha3/api_client.go +++ b/go/api/clientset/versioned/typed/api/v1alpha3/api_client.go @@ -14,6 +14,7 @@ type ApiV1alpha3Interface interface { RESTClient() rest.Interface AgentsGetter AgentTemplatesGetter + EgressPoliciesGetter HarnessesGetter ModelConfigsGetter RemoteMCPServersGetter @@ -33,6 +34,10 @@ func (c *ApiV1alpha3Client) AgentTemplates(namespace string) AgentTemplateInterf return newAgentTemplates(c, namespace) } +func (c *ApiV1alpha3Client) EgressPolicies(namespace string) EgressPolicyInterface { + return newEgressPolicies(c, namespace) +} + func (c *ApiV1alpha3Client) Harnesses(namespace string) HarnessInterface { return newHarnesses(c, namespace) } diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/egresspolicy.go b/go/api/clientset/versioned/typed/api/v1alpha3/egresspolicy.go new file mode 100644 index 0000000000..140295069e --- /dev/null +++ b/go/api/clientset/versioned/typed/api/v1alpha3/egresspolicy.go @@ -0,0 +1,52 @@ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha3 + +import ( + context "context" + + scheme "github.com/kagent-dev/kagent/go/api/clientset/versioned/scheme" + apiv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// EgressPoliciesGetter has a method to return a EgressPolicyInterface. +// A group's client should implement this interface. +type EgressPoliciesGetter interface { + EgressPolicies(namespace string) EgressPolicyInterface +} + +// EgressPolicyInterface has methods to work with EgressPolicy resources. +type EgressPolicyInterface interface { + Create(ctx context.Context, egressPolicy *apiv1alpha3.EgressPolicy, opts v1.CreateOptions) (*apiv1alpha3.EgressPolicy, error) + Update(ctx context.Context, egressPolicy *apiv1alpha3.EgressPolicy, opts v1.UpdateOptions) (*apiv1alpha3.EgressPolicy, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*apiv1alpha3.EgressPolicy, error) + List(ctx context.Context, opts v1.ListOptions) (*apiv1alpha3.EgressPolicyList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *apiv1alpha3.EgressPolicy, err error) + EgressPolicyExpansion +} + +// egressPolicies implements EgressPolicyInterface +type egressPolicies struct { + *gentype.ClientWithList[*apiv1alpha3.EgressPolicy, *apiv1alpha3.EgressPolicyList] +} + +// newEgressPolicies returns a EgressPolicies +func newEgressPolicies(c *ApiV1alpha3Client, namespace string) *egressPolicies { + return &egressPolicies{ + gentype.NewClientWithList[*apiv1alpha3.EgressPolicy, *apiv1alpha3.EgressPolicyList]( + "egresspolicies", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *apiv1alpha3.EgressPolicy { return &apiv1alpha3.EgressPolicy{} }, + func() *apiv1alpha3.EgressPolicyList { return &apiv1alpha3.EgressPolicyList{} }, + ), + } +} diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_api_client.go b/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_api_client.go index 19d7ba79b7..931f3f6f1a 100644 --- a/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_api_client.go +++ b/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_api_client.go @@ -20,6 +20,10 @@ func (c *FakeApiV1alpha3) AgentTemplates(namespace string) v1alpha3.AgentTemplat return newFakeAgentTemplates(c, namespace) } +func (c *FakeApiV1alpha3) EgressPolicies(namespace string) v1alpha3.EgressPolicyInterface { + return newFakeEgressPolicies(c, namespace) +} + func (c *FakeApiV1alpha3) Harnesses(namespace string) v1alpha3.HarnessInterface { return newFakeHarnesses(c, namespace) } diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_egresspolicy.go b/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_egresspolicy.go new file mode 100644 index 0000000000..6375df89c7 --- /dev/null +++ b/go/api/clientset/versioned/typed/api/v1alpha3/fake/fake_egresspolicy.go @@ -0,0 +1,36 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + apiv1alpha3 "github.com/kagent-dev/kagent/go/api/clientset/versioned/typed/api/v1alpha3" + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + gentype "k8s.io/client-go/gentype" +) + +// fakeEgressPolicies implements EgressPolicyInterface +type fakeEgressPolicies struct { + *gentype.FakeClientWithList[*v1alpha3.EgressPolicy, *v1alpha3.EgressPolicyList] + Fake *FakeApiV1alpha3 +} + +func newFakeEgressPolicies(fake *FakeApiV1alpha3, namespace string) apiv1alpha3.EgressPolicyInterface { + return &fakeEgressPolicies{ + gentype.NewFakeClientWithList[*v1alpha3.EgressPolicy, *v1alpha3.EgressPolicyList]( + fake.Fake, + namespace, + v1alpha3.SchemeGroupVersion.WithResource("egresspolicies"), + v1alpha3.SchemeGroupVersion.WithKind("EgressPolicy"), + func() *v1alpha3.EgressPolicy { return &v1alpha3.EgressPolicy{} }, + func() *v1alpha3.EgressPolicyList { return &v1alpha3.EgressPolicyList{} }, + func(dst, src *v1alpha3.EgressPolicyList) { dst.ListMeta = src.ListMeta }, + func(list *v1alpha3.EgressPolicyList) []*v1alpha3.EgressPolicy { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1alpha3.EgressPolicyList, items []*v1alpha3.EgressPolicy) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/go/api/clientset/versioned/typed/api/v1alpha3/generated_expansion.go b/go/api/clientset/versioned/typed/api/v1alpha3/generated_expansion.go index bc2146b2a4..da4673fc10 100644 --- a/go/api/clientset/versioned/typed/api/v1alpha3/generated_expansion.go +++ b/go/api/clientset/versioned/typed/api/v1alpha3/generated_expansion.go @@ -6,6 +6,8 @@ type AgentExpansion interface{} type AgentTemplateExpansion interface{} +type EgressPolicyExpansion interface{} + type HarnessExpansion interface{} type ModelConfigExpansion interface{} diff --git a/go/api/v1alpha3/agent_types.go b/go/api/v1alpha3/agent_types.go index 67bae21fba..76ffc37846 100644 --- a/go/api/v1alpha3/agent_types.go +++ b/go/api/v1alpha3/agent_types.go @@ -52,6 +52,17 @@ type AgentSpec struct { // +kubebuilder:validation:XValidation:rule="has(self.name) && self.name != ''",message="harnessRef.name must not be empty" // +required HarnessRef corev1.LocalObjectReference `json:"harnessRef"` + + // EgressPolicyRefs grants additional access to the whole runtime, including + // shared subagents. Policies resolve in this Agent's namespace. Integration + // access inferred from models and tools is always included. Changes update + // existing Sessions without a new invocation or runtime restart. + // +optional + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:XValidation:rule="self.all(ref, has(ref.name) && ref.name != '')",message="egressPolicyRefs requires non-empty names" + // +listType=map + // +listMapKey=name + EgressPolicyRefs []corev1.LocalObjectReference `json:"egressPolicyRefs,omitempty"` } // AgentTemplateSource selects portable behavior inline or by local reference. @@ -69,10 +80,11 @@ type AgentTemplateSource struct { } const ( - AgentConditionAccepted = "Accepted" - AgentConditionResolvedRefs = "ResolvedRefs" - AgentConditionCompatible = "Compatible" - AgentConditionReady = "Ready" + AgentConditionAccepted = "Accepted" + AgentConditionEgressPoliciesApplied = "EgressPoliciesApplied" + AgentConditionResolvedRefs = "ResolvedRefs" + AgentConditionCompatible = "Compatible" + AgentConditionReady = "Ready" ) // AgentStatus reports compilation and preparation of an Agent. @@ -96,7 +108,11 @@ type AgentStatus struct { // +listType=set // +optional Warnings []string `json:"warnings,omitempty"` - // +kubebuilder:validation:MaxItems=4 + // Conditions report runtime readiness and independent egress grant admission. + // EgressPoliciesApplied is False for EgressPolicyNotFound, InvalidEgressPolicy, + // or EgressGrantConflict; Applied means all grants are admitted. Grant issues + // do not affect Ready. + // +kubebuilder:validation:MaxItems=5 // +listType=map // +listMapKey=type // +optional diff --git a/go/api/v1alpha3/common_types.go b/go/api/v1alpha3/common_types.go index 481189eb0f..80a19120f0 100644 --- a/go/api/v1alpha3/common_types.go +++ b/go/api/v1alpha3/common_types.go @@ -157,6 +157,7 @@ func (s *ValueSource) Resolve(ctx context.Context, client client.Client, namespa // +kubebuilder:validation:ExactlyOneOf=value;valueFrom type ValueRef struct { // +required + // +kubebuilder:validation:MaxLength=256 Name string `json:"name"` // +optional Value string `json:"value,omitempty"` diff --git a/go/api/v1alpha3/configuration_crd_cel_test.go b/go/api/v1alpha3/configuration_crd_cel_test.go index 02d17d4b57..ab236cd4b7 100644 --- a/go/api/v1alpha3/configuration_crd_cel_test.go +++ b/go/api/v1alpha3/configuration_crd_cel_test.go @@ -56,7 +56,7 @@ func TestConfigurationCRDValidation(t *testing.T) { names = append(names, resource.Name) } } - require.ElementsMatch(t, []string{"agents", "agenttemplates", "harnesses", "modelconfigs", "modelproviderconfigs", "remotemcpservers", "sandboxtemplates"}, names) + require.ElementsMatch(t, []string{"agents", "agenttemplates", "egresspolicies", "harnesses", "modelconfigs", "modelproviderconfigs", "remotemcpservers", "sandboxtemplates"}, names) _, err = discoveryClient.ServerResourcesForGroupVersion("kagent.dev/v1alpha3") require.True(t, apierrors.IsNotFound(err), "the new CRDs must not publish the legacy group: %v", err) }) @@ -462,6 +462,9 @@ func TestConfigurationCRDValidation(t *testing.T) { }) } + t.Run("egress policies", func(t *testing.T) { testEgressPolicyValidation(t, cl, namespace) }) + t.Run("MCP credential headers", func(t *testing.T) { testMCPCredentialHeaderValidation(t, cl, namespace) }) + t.Run("literal runtime environment", func(t *testing.T) { harness := validHarness(namespace, "env-harness", HarnessSpec{Kagent: &KagentHarness{}}) for _, resource := range []struct { diff --git a/go/api/v1alpha3/egresspolicy_cel_test.go b/go/api/v1alpha3/egresspolicy_cel_test.go new file mode 100644 index 0000000000..d8f0fa0bd4 --- /dev/null +++ b/go/api/v1alpha3/egresspolicy_cel_test.go @@ -0,0 +1,103 @@ +package v1alpha3 + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func testEgressPolicyValidation(t *testing.T, cl ctrlclient.Client, namespace string) { + secret := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}, Key: "token"} + bearer := &EgressAuthentication{Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: secret} + for i, tc := range []struct { + name string + destination EgressDestination + reject bool + }{ + {"HTTPS", EgressDestination{Origin: "https://api.example.com"}, false}, + {"HTTP", EgressDestination{Origin: "http://registry.example.com:8080"}, false}, + {"wildcard", EgressDestination{Origin: "https://*.example.com"}, false}, + {"bearer", EgressDestination{Origin: "https://api.example.com:8443", Authentication: bearer}, false}, + {"API key", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "X-API-Key", SecretKeyRef: secret}}, false}, + {"empty prefix", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "X-API-Key", Prefix: new(""), SecretKeyRef: secret}}, false}, + {"custom prefix", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "Authorization", Prefix: new("Token "), SecretKeyRef: secret}}, false}, + {"HTTP auth", EgressDestination{Origin: "http://api.example.com", Authentication: bearer}, false}, + {"HTTP wildcard auth", EgressDestination{Origin: "http://*.example.com", Authentication: bearer}, true}, + {"wildcard auth", EgressDestination{Origin: "https://*.example.com", Authentication: bearer}, true}, + {"empty auth", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{}}, true}, + {"missing header", EgressDestination{Origin: "https://api.example.com", Authentication: &EgressAuthentication{SecretKeyRef: secret}}, true}, + {"path", EgressDestination{Origin: "https://api.example.com/v1"}, true}, + {"userinfo", EgressDestination{Origin: "https://user@api.example.com"}, true}, + {"query", EgressDestination{Origin: "https://api.example.com?"}, true}, + {"IP", EgressDestination{Origin: "https://192.0.2.1"}, true}, + {"all hosts", EgressDestination{Origin: "https://*"}, true}, + {"top level wildcard", EgressDestination{Origin: "https://*.com"}, true}, + {"interior wildcard", EgressDestination{Origin: "https://api.*.com"}, true}, + {"port zero", EgressDestination{Origin: "https://api.example.com:0"}, true}, + {"port overflow", EgressDestination{Origin: "https://api.example.com:65536"}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + policy := &EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: fmt.Sprintf("egress-%d", i)}, Spec: EgressPolicySpec{Destinations: []EgressDestination{tc.destination}}} + err := cl.Create(t.Context(), policy) + if tc.reject { + require.Error(t, err) + } else { + require.NoError(t, err) + require.NoError(t, cl.Get(t.Context(), ctrlclient.ObjectKeyFromObject(policy), policy)) + require.Equal(t, tc.destination.Authentication, policy.Spec.Destinations[0].Authentication) + } + }) + } + for i, header := range []string{"Host", "cOnNeCtIoN", "Content-Length", "Transfer-Encoding", "Cookie", "Proxy-Authorization", "X-Key\r\nInjected"} { + policy := &EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: fmt.Sprintf("bad-header-%d", i)}, Spec: EgressPolicySpec{Destinations: []EgressDestination{{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: header, SecretKeyRef: secret}}}}} + require.Error(t, cl.Create(t.Context(), policy), header) + } + for i, prefix := range []string{"Bearer\r", "Bearer\n", "\x00", "\x08", "\x0b", "\x1f", "\x7f", strings.Repeat("x", 257)} { + policy := &EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: fmt.Sprintf("bad-prefix-%d", i)}, Spec: EgressPolicySpec{Destinations: []EgressDestination{{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "Authorization", Prefix: new(prefix), SecretKeyRef: secret}}}}} + require.Error(t, cl.Create(t.Context(), policy), "prefix %q", prefix) + } + for i, ref := range []corev1.SecretKeySelector{{Key: "token"}, {LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}}, {LocalObjectReference: secret.LocalObjectReference, Key: "token", Optional: new(true)}} { + policy := &EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: fmt.Sprintf("bad-secret-%d", i)}, Spec: EgressPolicySpec{Destinations: []EgressDestination{{Origin: "https://api.example.com", Authentication: &EgressAuthentication{Header: "Authorization", SecretKeyRef: ref}}}}} + require.Error(t, cl.Create(t.Context(), policy)) + } + for _, kind := range []string{"Agent", "SandboxTemplate", "AgentTemplate", "Harness"} { + t.Run(kind+" attachment", func(t *testing.T) { + var object ctrlclient.Object + switch kind { + case "Agent": + object = &Agent{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "egress-agent"}, Spec: AgentSpec{Template: AgentTemplateSource{Ref: &corev1.LocalObjectReference{Name: "behavior"}}, HarnessRef: corev1.LocalObjectReference{Name: "runner"}}} + case "SandboxTemplate": + object = sandboxTemplateForValidation(namespace, "egress-sandbox", nil) + case "AgentTemplate": + object = validAgentTemplate(namespace, "egress-template", nil) + case "Harness": + object = validHarness(namespace, "egress-harness", HarnessSpec{Kagent: &KagentHarness{}}) + } + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object) + require.NoError(t, err) + resource := &unstructured.Unstructured{Object: data} + resource.SetAPIVersion(GroupVersion.String()) + resource.SetKind(kind) + require.NoError(t, unstructured.SetNestedSlice(data, []any{map[string]any{"name": "tickets"}}, "spec", "egressPolicyRefs")) + err = cl.Create(t.Context(), resource, &ctrlclient.CreateOptions{FieldValidation: metav1.FieldValidationStrict}) + if kind == "AgentTemplate" || kind == "Harness" { + require.ErrorContains(t, err, "unknown field") + return + } + require.NoError(t, err) + require.NoError(t, cl.Get(t.Context(), ctrlclient.ObjectKeyFromObject(resource), resource)) + refs, _, err := unstructured.NestedSlice(resource.Object, "spec", "egressPolicyRefs") + require.NoError(t, err) + require.Equal(t, []any{map[string]any{"name": "tickets"}}, refs) + require.NoError(t, unstructured.SetNestedSlice(resource.Object, []any{map[string]any{"name": ""}}, "spec", "egressPolicyRefs")) + require.Error(t, cl.Update(t.Context(), resource)) + }) + } +} diff --git a/go/api/v1alpha3/egresspolicy_types.go b/go/api/v1alpha3/egresspolicy_types.go new file mode 100644 index 0000000000..deac70d7f2 --- /dev/null +++ b/go/api/v1alpha3/egresspolicy_types.go @@ -0,0 +1,97 @@ +// Copyright 2026 The kagent Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha3 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// EgressPolicySpec grants additional HTTP(S) access to workloads in its namespace. +type EgressPolicySpec struct { + // Destinations are additional origins the workload may reach. + // +required + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 + // +listType=map + // +listMapKey=origin + Destinations []EgressDestination `json:"destinations"` +} + +// EgressDestination grants access to one HTTP(S) origin, including all its paths. +// +kubebuilder:validation:XValidation:rule="!has(self.authentication) || !self.origin.contains('*')",message="authentication requires an exact HTTP(S) origin" +type EgressDestination struct { + // Origin is a scheme, DNS hostname, and optional port. A wildcard replaces + // exactly one leftmost DNS label, for example https://*.example.com. + // +required + // +kubebuilder:validation:MaxLength=270 + // +kubebuilder:validation:Pattern=`^https?://(\*\.([a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?\.)+|([a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?\.)*)[a-z]([-a-z0-9]{0,61}[a-z0-9])?(:([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?$` + Origin string `json:"origin"` + // Authentication replaces a placeholder header at the gateway. Requests + // must send the header; its credential value is never supplied to the runtime. + // +optional + Authentication *EgressAuthentication `json:"authentication,omitempty"` +} + +// EgressAuthentication replaces a request header with a prefix and Secret value. +type EgressAuthentication struct { + // Header is the request header carrying the credential, for example + // Authorization or X-API-Key. + // +required + // +kubebuilder:validation:MaxLength=256 + // +kubebuilder:validation:Pattern=`^[!#$%&'*+.^_|~0-9A-Za-z-]+$` + // +kubebuilder:validation:XValidation:rule="!(self.lowerAscii() in ['host', 'connection', 'content-length', 'transfer-encoding', 'te', 'trailer', 'upgrade', 'keep-alive', 'proxy-authorization', 'proxy-authenticate', 'cookie'])",message="this header cannot carry an egress credential" + Header string `json:"header"` + // Prefix is prepended verbatim to the Secret value, for example "Bearer ". + // Include any separating space. Omitted or empty means no prefix. + // +optional + // +kubebuilder:validation:MaxLength=256 + // +kubebuilder:validation:Pattern=`^[^\x00-\x08\x0a-\x1f\x7f]*$` + Prefix *string `json:"prefix,omitempty"` + // SecretKeyRef selects a required key in the EgressPolicy's namespace. + // +required + // +kubebuilder:validation:XValidation:rule="has(self.name) && self.name != '' && self.key != ''",message="secretKeyRef requires a name and key" + // +kubebuilder:validation:XValidation:rule="!has(self.optional) || !self.optional",message="egress credentials cannot be optional" + SecretKeyRef corev1.SecretKeySelector `json:"secretKeyRef"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=egresspolicies,singular=egresspolicy,scope=Namespaced,categories=kagent + +// EgressPolicy is a reusable grant of additional network access. Workloads in its +// namespace select it explicitly; changes update their existing runtime Actors. +type EgressPolicy struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + // +required + Spec EgressPolicySpec `json:"spec"` +} + +// +kubebuilder:object:root=true + +// EgressPolicyList contains EgressPolicy resources. +type EgressPolicyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EgressPolicy `json:"items"` +} + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, &EgressPolicy{}, &EgressPolicyList{}) + return nil + }) +} + +// Reasons for the independent EgressPoliciesApplied condition on Agents and +// SandboxTemplates. Grant resolution and admission never block runtime readiness. +const ( + EgressPoliciesReasonApplied = "Applied" + EgressPoliciesReasonNotFound = "EgressPolicyNotFound" + EgressPoliciesReasonInvalidEgressPolicy = "InvalidEgressPolicy" + EgressPoliciesReasonConflict = "EgressGrantConflict" +) diff --git a/go/api/v1alpha3/remotemcpserver_cel_test.go b/go/api/v1alpha3/remotemcpserver_cel_test.go new file mode 100644 index 0000000000..44a0020bfd --- /dev/null +++ b/go/api/v1alpha3/remotemcpserver_cel_test.go @@ -0,0 +1,90 @@ +package v1alpha3 + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func testMCPCredentialHeaderValidation(t *testing.T, cl ctrlclient.Client, namespace string) { + t.Helper() + for _, tc := range []struct { + name string + header string + reject string + }{ + {name: "authorization", header: "Authorization"}, + {name: "api-key", header: "X-API-Key"}, + {name: "token-characters", header: "X-!#$%&'*+-.^_`|~0123456789"}, + {name: "max-length", header: strings.Repeat("x", 256)}, + {name: "empty", reject: "valid HTTP header name"}, + {name: "too-long", header: strings.Repeat("x", 257), reject: "may not be more than 256"}, + {name: "whitespace", header: "X Key", reject: "valid HTTP header name"}, + {name: "newline", header: "X-Key\r\nInjected", reject: "valid HTTP header name"}, + {name: "colon", header: "X:Key", reject: "valid HTTP header name"}, + {name: "backslash", header: "X\\Key", reject: "valid HTTP header name"}, + {name: "unicode", header: "X-名", reject: "valid HTTP header name"}, + {name: "host", header: "Host", reject: "cannot carry an egress credential"}, + {name: "connection", header: "cOnNeCtIoN", reject: "cannot carry an egress credential"}, + {name: "content-length", header: "Content-Length", reject: "cannot carry an egress credential"}, + {name: "transfer-encoding", header: "Transfer-Encoding", reject: "cannot carry an egress credential"}, + {name: "te", header: "TE", reject: "cannot carry an egress credential"}, + {name: "trailer", header: "Trailer", reject: "cannot carry an egress credential"}, + {name: "upgrade", header: "Upgrade", reject: "cannot carry an egress credential"}, + {name: "keep-alive", header: "Keep-Alive", reject: "cannot carry an egress credential"}, + {name: "proxy-authorization", header: "Proxy-Authorization", reject: "cannot carry an egress credential"}, + {name: "proxy-authenticate", header: "Proxy-Authenticate", reject: "cannot carry an egress credential"}, + {name: "cookie", header: "Cookie", reject: "cannot carry an egress credential"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := &RemoteMCPServer{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "mcp-header-" + tc.name}, + Spec: RemoteMCPServerSpec{ + Description: "MCP credential header validation", + URL: "https://mcp.example.com/mcp", + HeadersFrom: []ValueRef{{Name: tc.header, ValueFrom: &ValueSource{Type: SecretValueSource, Name: "auth", Key: "token"}}}, + }, + } + err := cl.Create(t.Context(), server) + if tc.reject != "" { + require.ErrorContains(t, err, tc.reject) + require.ErrorContains(t, err, "spec.headersFrom[0].name") + return + } + require.NoError(t, err) + require.NoError(t, cl.Get(t.Context(), ctrlclient.ObjectKeyFromObject(server), server)) + require.Equal(t, tc.header, server.Spec.HeadersFrom[0].Name) + server.Spec.HeadersFrom[0].Name = "Cookie" + require.ErrorContains(t, cl.Update(t.Context(), server), "cannot carry an egress credential") + }) + } + for _, source := range []ValueRef{ + {Name: "Cookie", Value: "session=example"}, + {Name: "Cookie", ValueFrom: &ValueSource{Type: ConfigMapValueSource, Name: "headers", Key: "cookie"}}, + } { + server := &RemoteMCPServer{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, GenerateName: "mcp-ordinary-header-"}, + Spec: RemoteMCPServerSpec{Description: "Ordinary headers are not gateway credentials", URL: "https://mcp.example.com/mcp", HeadersFrom: []ValueRef{source}}, + } + require.NoError(t, cl.Create(t.Context(), server)) + server.Spec.HeadersFrom[0] = ValueRef{Name: "Cookie", ValueFrom: &ValueSource{Type: SecretValueSource, Name: "auth", Key: "token"}} + require.ErrorContains(t, cl.Update(t.Context(), server), "cannot carry an egress credential") + } + t.Run("header count", func(t *testing.T) { + headers := make([]ValueRef, 64) + for i := range headers { + headers[i] = ValueRef{Name: fmt.Sprintf("X-Header-%d", i), Value: "example"} + } + server := &RemoteMCPServer{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "mcp-header-count"}, + Spec: RemoteMCPServerSpec{Description: "Header count validation", URL: "https://mcp.example.com/mcp", HeadersFrom: headers}, + } + require.NoError(t, cl.Create(t.Context(), server)) + server.Spec.HeadersFrom = append(server.Spec.HeadersFrom, ValueRef{Name: "X-Extra", Value: "example"}) + require.ErrorContains(t, cl.Update(t.Context(), server), "must have at most 64 items") + }) +} diff --git a/go/api/v1alpha3/remotemcpserver_types.go b/go/api/v1alpha3/remotemcpserver_types.go index 41a4c5c885..4fb2b9ce06 100644 --- a/go/api/v1alpha3/remotemcpserver_types.go +++ b/go/api/v1alpha3/remotemcpserver_types.go @@ -48,7 +48,13 @@ type RemoteMCPServerSpec struct { // +kubebuilder:validation:MinLength=1 // +required URL string `json:"url"` + // HeadersFrom supplies request headers. Secret-backed headers are injected by + // the egress gateway and cannot use routing, framing, hop-by-hop, + // proxy-authentication, or Cookie headers. At most 64 headers may be supplied. // +optional + // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:items:XValidation:rule="!has(self.valueFrom) || self.valueFrom.type != 'Secret' || self.name.matches('^[!#$%&\\\\x27*+.^_`|~0-9A-Za-z-]+$')",message="Secret-backed headers require a valid HTTP header name",fieldPath=".name" + // +kubebuilder:validation:items:XValidation:rule="!has(self.valueFrom) || self.valueFrom.type != 'Secret' || !(self.name.lowerAscii() in ['host', 'connection', 'content-length', 'transfer-encoding', 'te', 'trailer', 'upgrade', 'keep-alive', 'proxy-authorization', 'proxy-authenticate', 'cookie'])",message="this header cannot carry an egress credential",fieldPath=".name" HeadersFrom []ValueRef `json:"headersFrom,omitempty"` // +optional // +kubebuilder:default="30s" diff --git a/go/api/v1alpha3/sandboxtemplate_types.go b/go/api/v1alpha3/sandboxtemplate_types.go index 9687b688a9..0b563f7070 100644 --- a/go/api/v1alpha3/sandboxtemplate_types.go +++ b/go/api/v1alpha3/sandboxtemplate_types.go @@ -4,6 +4,7 @@ package v1alpha3 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -37,14 +38,29 @@ type SandboxTemplateSpec struct { // resolved in this template's namespace. // +required Substrate RuntimeSubstratePolicy `json:"substrate"` + + // EgressPolicyRefs grants additional access in this template's namespace. + // Changes update the access of existing sandboxes without restarting them. + // +optional + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:XValidation:rule="self.all(ref, has(ref.name) && ref.name != '')",message="egressPolicyRefs requires non-empty names" + // +listType=map + // +listMapKey=name + EgressPolicyRefs []corev1.LocalObjectReference `json:"egressPolicyRefs,omitempty"` } +// SandboxTemplateConditionEgressPoliciesApplied reports grant admission independently of Ready. +const SandboxTemplateConditionEgressPoliciesApplied = "EgressPoliciesApplied" + // SandboxTemplateStatus reports preparation of the current template inputs. type SandboxTemplateStatus struct { // ObservedGeneration is the generation considered by preparation. // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // Conditions describe whether a prepared revision is available. + // Conditions report runtime readiness and independent egress grant admission. + // EgressPoliciesApplied is False for EgressPolicyNotFound, InvalidEgressPolicy, + // or EgressGrantConflict; Applied means all grants are admitted. Grant issues + // do not affect Ready. // +optional // +listType=map // +listMapKey=type diff --git a/go/api/v1alpha3/zz_generated.deepcopy.go b/go/api/v1alpha3/zz_generated.deepcopy.go index f21a0d5b50..0179022d34 100644 --- a/go/api/v1alpha3/zz_generated.deepcopy.go +++ b/go/api/v1alpha3/zz_generated.deepcopy.go @@ -91,6 +91,11 @@ func (in *AgentSpec) DeepCopyInto(out *AgentSpec) { *out = *in in.Template.DeepCopyInto(&out.Template) out.HarnessRef = in.HarnessRef + if in.EgressPolicyRefs != nil { + in, out := &in.EgressPolicyRefs, &out.EgressPolicyRefs + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentSpec. @@ -577,6 +582,127 @@ func (in *ConfigMapKeyReference) DeepCopy() *ConfigMapKeyReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressAuthentication) DeepCopyInto(out *EgressAuthentication) { + *out = *in + if in.Prefix != nil { + in, out := &in.Prefix, &out.Prefix + *out = new(string) + **out = **in + } + in.SecretKeyRef.DeepCopyInto(&out.SecretKeyRef) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressAuthentication. +func (in *EgressAuthentication) DeepCopy() *EgressAuthentication { + if in == nil { + return nil + } + out := new(EgressAuthentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressDestination) DeepCopyInto(out *EgressDestination) { + *out = *in + if in.Authentication != nil { + in, out := &in.Authentication, &out.Authentication + *out = new(EgressAuthentication) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressDestination. +func (in *EgressDestination) DeepCopy() *EgressDestination { + if in == nil { + return nil + } + out := new(EgressDestination) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressPolicy) DeepCopyInto(out *EgressPolicy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressPolicy. +func (in *EgressPolicy) DeepCopy() *EgressPolicy { + if in == nil { + return nil + } + out := new(EgressPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressPolicy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressPolicyList) DeepCopyInto(out *EgressPolicyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EgressPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressPolicyList. +func (in *EgressPolicyList) DeepCopy() *EgressPolicyList { + if in == nil { + return nil + } + out := new(EgressPolicyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressPolicyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressPolicySpec) DeepCopyInto(out *EgressPolicySpec) { + *out = *in + if in.Destinations != nil { + in, out := &in.Destinations, &out.Destinations + *out = make([]EgressDestination, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressPolicySpec. +func (in *EgressPolicySpec) DeepCopy() *EgressPolicySpec { + if in == nil { + return nil + } + out := new(EgressPolicySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FoundryConfig) DeepCopyInto(out *FoundryConfig) { *out = *in @@ -1649,6 +1775,11 @@ func (in *SandboxTemplateSpec) DeepCopyInto(out *SandboxTemplateSpec) { copy(*out, *in) } out.Substrate = in.Substrate + if in.EgressPolicyRefs != nil { + in, out := &in.EgressPolicyRefs, &out.EgressPolicyRefs + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxTemplateSpec. diff --git a/go/core/internal/controller/apiclient/types.go b/go/core/internal/controller/apiclient/types.go index c396c10f6d..7e5fdc6910 100644 --- a/go/core/internal/controller/apiclient/types.go +++ b/go/core/internal/controller/apiclient/types.go @@ -23,6 +23,19 @@ func RegisterTypes() { } func registerTypes() { + kubeclient.Register( + kagentv1alpha3.GroupVersion.WithResource("egresspolicies"), + kagentv1alpha3.GroupVersion.WithKind("EgressPolicy"), + func(c kubeclient.ClientGetter, namespace string, options metav1.ListOptions) (runtime.Object, error) { + return c.(Client).Kagent().ApiV1alpha3().EgressPolicies(namespace).List(context.Background(), options) + }, + func(c kubeclient.ClientGetter, namespace string, options metav1.ListOptions) (watch.Interface, error) { + return c.(Client).Kagent().ApiV1alpha3().EgressPolicies(namespace).Watch(context.Background(), options) + }, + func(c kubeclient.ClientGetter, namespace string) kubetypes.WriteAPI[*kagentv1alpha3.EgressPolicy] { + return c.(Client).Kagent().ApiV1alpha3().EgressPolicies(namespace) + }, + ) kubeclient.Register( kagentv1alpha3.GroupVersion.WithResource("agents"), kagentv1alpha3.GroupVersion.WithKind("Agent"), diff --git a/go/core/internal/controller/collections.go b/go/core/internal/controller/collections.go index cdd67b9b2e..b7c9dbe33c 100644 --- a/go/core/internal/controller/collections.go +++ b/go/core/internal/controller/collections.go @@ -2,6 +2,7 @@ package controller import ( "reflect" + "slices" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -19,6 +20,8 @@ import ( // Collections contains shared Kubernetes inputs and compiled Agent state. type Collections struct { + watchNamespaces []string + EgressPolicies krt.Collection[*kagentv1alpha3.EgressPolicy] SandboxTemplates krt.Collection[*kagentv1alpha3.SandboxTemplate] Agents krt.Collection[*kagentv1alpha3.Agent] AgentTemplates krt.Collection[*kagentv1alpha3.AgentTemplate] @@ -63,6 +66,7 @@ func (p AgentRuntimeObservation) Equals(other AgentRuntimeObservation) bool { // NewCollections creates the complete read-only input graph. An empty // watchNamespaces list watches all namespaces. func NewCollections(client kube.Client, watchNamespaces []string, opts krt.OptionsBuilder, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig) Collections { + egressPolicies := typedCollection[*kagentv1alpha3.EgressPolicy](client, watchNamespaces, "EgressPolicies", opts) sandboxTemplates := typedCollection[*kagentv1alpha3.SandboxTemplate](client, watchNamespaces, "SandboxTemplates", opts) agents := typedCollection[*kagentv1alpha3.Agent](client, watchNamespaces, "Agents", opts) agentTemplates := typedCollection[*kagentv1alpha3.AgentTemplate](client, watchNamespaces, "AgentTemplates", opts) @@ -75,13 +79,16 @@ func NewCollections(client kube.Client, watchNamespaces []string, opts krt.Optio agentRuntimeObservations := krt.NewStaticCollection[AgentRuntimeObservation](nil, nil, opts.WithName("AgentRuntimeObservations")...) modelConfigStatuses, resolvedModelConfigs := newModelConfigReconciliations(modelConfigs, configMaps, secrets, opts) compilerCollections := translator.Collections{ - Harnesses: harnesses, AgentTemplates: agentTemplates, ResolvedModelConfigs: resolvedModelConfigs, RemoteMCPServers: remoteMCPServers, + EgressPolicies: egressPolicies, + Harnesses: harnesses, AgentTemplates: agentTemplates, ResolvedModelConfigs: resolvedModelConfigs, RemoteMCPServers: remoteMCPServers, ConfigMaps: configMaps, Secrets: secrets, WorkerPools: workerPools, } reconciliations := newAgentReconciliations(agents, compilerCollections, agentRuntimeObservations, opts, settings, images) statuses := newAgentStatuses(agents, reconciliations, opts) return Collections{ + watchNamespaces: slices.Clone(watchNamespaces), + EgressPolicies: egressPolicies, SandboxTemplates: sandboxTemplates, Agents: agents, AgentTemplates: agentTemplates, diff --git a/go/core/internal/controller/egress.go b/go/core/internal/controller/egress.go new file mode 100644 index 0000000000..fdbcf8e3e7 --- /dev/null +++ b/go/core/internal/controller/egress.go @@ -0,0 +1,76 @@ +package controller + +import ( + "context" + "fmt" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/hashicorp/go-multierror" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/substrate" +) + +type actorEgressStore interface { + ListRuntimeEgressSourceTargets(context.Context, database.RuntimeEgressSource, string, int) ([]database.RuntimeEgressTarget, error) +} + +type actorEgressClient interface { + GetActor(context.Context, string, string) (*ateapipb.Actor, error) + EnsureActorEgressPolicy(context.Context, string, string, string) error +} + +// applySourceEgress updates every allocated actor, including suspended actors and +// former source UIDs. Preparation retries the source on failure, even when its +// grants are unchanged. Continue past individual failures so one unavailable +// actor cannot prevent revocation on the others. +func applySourceEgress(ctx context.Context, store actorEgressStore, actors actorEgressClient, source database.RuntimeEgressSource) error { + const pageSize = 100 + var failures *multierror.Error + after := "" + for { + targets, err := store.ListRuntimeEgressSourceTargets(ctx, source, after, pageSize) + if err != nil { + return multierror.Append(failures, fmt.Errorf("failed to list %s %s/%s actors: %w", source.Kind, source.Namespace, source.Name, err)).ErrorOrNil() + } + var group errgroup.Group + group.SetLimit(16) + for _, target := range targets { + group.Go(func() error { + if err := applyActorEgress(ctx, actors, target); err != nil { + return fmt.Errorf("failed to apply actor %s egress: %w", target.ID, err) + } + return nil + }) + } + if err := group.Wait(); err != nil { + failures = multierror.Append(failures, err) + } + if len(targets) < pageSize { + return failures.ErrorOrNil() + } + after = targets[len(targets)-1].ID + } +} + +func applyActorEgress(ctx context.Context, actors actorEgressClient, target database.RuntimeEgressTarget) error { + binding := substrate.ActorBinding{Atespace: target.Atespace, Name: substrate.ActorName(target.ID), TemplateAtespace: target.Atespace, TemplateName: target.TemplateName} + actor, err := actors.GetActor(ctx, binding.Atespace, binding.Name) + if status.Code(err) == codes.NotFound { + return nil + } + if err != nil { + return err + } + if !binding.Matches(actor) || (target.ActorUID != "" && target.ActorUID != actor.GetMetadata().GetUid()) { + return nil + } + err = actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, target.Revision) + if status.Code(err) == codes.NotFound { + return nil + } + return err +} diff --git a/go/core/internal/controller/egress_test.go b/go/core/internal/controller/egress_test.go new file mode 100644 index 0000000000..1842b207b7 --- /dev/null +++ b/go/core/internal/controller/egress_test.go @@ -0,0 +1,129 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "slices" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/kagent-dev/kagent/go/core/internal/database" +) + +type egressTargetStoreStub struct { + sources []database.RuntimeEgressSource + targets []database.RuntimeEgressTarget + listErr error +} + +var _ actorEgressStore = (*egressTargetStoreStub)(nil) + +func (s *egressTargetStoreStub) ListRuntimeEgressSourceTargets(_ context.Context, source database.RuntimeEgressSource, after string, limit int) ([]database.RuntimeEgressTarget, error) { + s.sources = append(s.sources, source) + if s.listErr != nil { + return nil, s.listErr + } + targets := slices.DeleteFunc(slices.Clone(s.targets), func(target database.RuntimeEgressTarget) bool { return target.ID <= after }) + return targets[:min(len(targets), limit)], nil +} + +type actorEgressStub struct { + getActor func(context.Context, string, string) (*ateapipb.Actor, error) + ensurePolicy func(context.Context, string, string, string) error +} + +var _ actorEgressClient = (*actorEgressStub)(nil) + +func (a *actorEgressStub) GetActor(ctx context.Context, space, name string) (*ateapipb.Actor, error) { + if a.getActor != nil { + return a.getActor(ctx, space, name) + } + return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: "uid"}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: "runtime"}}, nil +} +func (a *actorEgressStub) EnsureActorEgressPolicy(ctx context.Context, space, name, revision string) error { + return a.ensurePolicy(ctx, space, name, revision) +} + +func TestSourceEgressAppliesEveryPageWithBoundedConcurrency(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + source := database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team", Name: "source"} + store := &egressTargetStoreStub{} + for i := range 101 { + store.targets = append(store.targets, database.RuntimeEgressTarget{ID: fmt.Sprintf("%03d", i), Revision: "revision", Namespace: "team", Atespace: "team", TemplateName: "runtime"}) + } + var mu sync.Mutex + active, peak := 0, 0 + applied := map[string]int{} + backendErr := errors.New("backend unavailable") + actors := &actorEgressStub{ensurePolicy: func(ctx context.Context, _, name, _ string) error { + mu.Lock() + active++ + peak = max(peak, active) + applied[name]++ + mu.Unlock() + time.Sleep(time.Second) + mu.Lock() + active-- + mu.Unlock() + if name == "session-000" { + return backendErr + } + return ctx.Err() + }} + require.ErrorIs(t, applySourceEgress(t.Context(), store, actors, source), backendErr) + require.Len(t, applied, 101, "a failed actor must not block subsequent actors or pages") + require.Greater(t, peak, 1) + require.LessOrEqual(t, peak, 16) + require.Equal(t, []database.RuntimeEgressSource{source, source}, store.sources) + store.listErr = errors.New("database unavailable") + require.ErrorIs(t, applySourceEgress(t.Context(), store, actors, source), store.listErr) + }) +} + +func TestActorEgressSkipsStaleOrMissingActors(t *testing.T) { + for _, test := range []struct { + name string + uid string + template string + getErr error + applyErr error + wantCalls int + wantErr bool + }{ + {name: "matching", uid: "uid", template: "runtime", wantCalls: 1}, + {name: "replaced template", uid: "uid", template: "replacement"}, + {name: "replaced actor", uid: "replacement", template: "runtime"}, + {name: "missing actor", getErr: status.Error(codes.NotFound, "gone")}, + {name: "deleted during apply", uid: "uid", template: "runtime", applyErr: status.Error(codes.NotFound, "gone"), wantCalls: 1}, + {name: "read failure", getErr: errors.New("unavailable"), wantErr: true}, + {name: "apply failure", uid: "uid", template: "runtime", applyErr: errors.New("unavailable"), wantCalls: 1, wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + calls := 0 + actors := &actorEgressStub{ + getActor: func(_ context.Context, space, name string) (*ateapipb.Actor, error) { + return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: space, Name: name, Uid: test.uid}, ActorTemplate: &ateapipb.ObjectRef{Atespace: space, Name: test.template}}, test.getErr + }, + ensurePolicy: func(context.Context, string, string, string) error { + calls++ + return test.applyErr + }, + } + err := applyActorEgress(t.Context(), actors, database.RuntimeEgressTarget{ID: "actor", Revision: "revision", Atespace: "team", TemplateName: "runtime", ActorUID: "uid"}) + if test.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, test.wantCalls, calls) + }) + } +} diff --git a/go/core/internal/controller/equality_test.go b/go/core/internal/controller/equality_test.go index 6de9fe439f..3c03c03ea8 100644 --- a/go/core/internal/controller/equality_test.go +++ b/go/core/internal/controller/equality_test.go @@ -55,6 +55,9 @@ func TestAgentReconciliationEquality(t *testing.T) { proto.Size(r.ObservedActorTemplate) }, equal: true}, {name: "template source", change: func(r *AgentReconciliation) { r.Agent.Generation++ }}, + {name: "live grants", change: func(r *AgentReconciliation) { + r.EgressGrant.Destinations = []string{"https://tickets.example.com:443"} + }}, {name: "revision identity", change: func(r *AgentReconciliation) { r.Target.RevisionID[0]++ }}, {name: "revision inputs", change: func(r *AgentReconciliation) { r.Target.Revision.SandboxClass = "microvm" }}, {name: "agent card", change: func(r *AgentReconciliation) { r.Target.Revision.AgentCard.Name = "changed" }}, diff --git a/go/core/internal/controller/grant_admission_test.go b/go/core/internal/controller/grant_admission_test.go new file mode 100644 index 0000000000..aff81efc9d --- /dev/null +++ b/go/core/internal/controller/grant_admission_test.go @@ -0,0 +1,172 @@ +package controller + +import ( + "strings" + "testing" + "testing/synctest" + + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/substrate" +) + +func TestGrantAdmissionAcrossSources(t *testing.T) { + for _, kind := range []string{"agent", "sandbox"} { + t.Run(kind, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + opts := krt.NewOptionsBuilder(t.Context().Done(), "admission", nil) + var policies krt.StaticCollection[*v1alpha3.EgressPolicy] + var reconcile func() error + var stored func() egress.Access + var conditions func() []metav1.Condition + switch kind { + case "agent": + collections, _ := newPreparationTestCollections(t, "gvisor") + initial := collections.Reconciliations.List()[0] + agent := initial.Agent.DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "a"}, {Name: "b"}} + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + policies = collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]) + template := proto.CloneOf(initial.Target.ActorTemplate) + template.Metadata.Uid, template.Status = "prepared", sandboxGoldenStatus() + store := &fakeRuntimeRevisionStore{} + r := &Reconciler{collections: collections, templates: &fakeActorTemplates{template: template}, store: store} + reconcile = func() error { return r.reconcileAgent(t.Context(), initial.ResourceName()) } + stored = func() egress.Access { return store.pair.EgressGrant } + conditions = func() []metav1.Condition { + state := collections.Reconciliations.GetKey(initial.ResourceName()) + require.NotNil(t, state.Target, "grant conflicts must not block compilation") + require.Equal(t, initial.Target.RevisionID, state.Target.RevisionID) + return statusForAgent(*state, agent.Generation, "").Conditions + } + case "sandbox": + template := sandboxTestTemplate() + template.Finalizers = []string{sandboxPreparationFinalizer} + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "a"}, {Name: "b"}} + templates := krt.NewStaticCollection(nil, []*v1alpha3.SandboxTemplate{template}, opts.WithName("templates")...) + policies = krt.NewStaticCollection[*v1alpha3.EgressPolicy](nil, nil, opts.WithName("policies")...) + pools := krt.NewStaticCollection(nil, []*atev1alpha1.WorkerPool{{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "default"}}}, opts.WithName("pools")...) + store := &sandboxTestStore{} + actors := &sandboxTestActors{store: store, templates: map[string]*ateapipb.ActorTemplate{}, autoReady: true} + s := &SandboxReconciler{collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools, EgressPolicies: policies}, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}, opts), + store: store, actors: actors, client: kagentfake.NewSimpleClientset(template).ApiV1alpha3()} + reconcile = func() error { return s.reconcile(t.Context(), "team-a/scratch") } + stored = func() egress.Access { return store.desired.EgressGrant } + conditions = func() []metav1.Condition { + state := s.collections.states.GetKey("team-a/scratch") + require.NotNil(t, state.DesiredActorTemplate) + return sandboxStatusWithTransitionTimes(*state).Conditions + } + } + policy := func(name string, origins ...string) *v1alpha3.EgressPolicy { + p := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: name}} + for _, origin := range origins { + p.Spec.Destinations = append(p.Spec.Destinations, v1alpha3.EgressDestination{Origin: origin}) + } + return p + } + check := func(reason string, origins ...string) { + t.Helper() + synctest.Wait() + require.NoError(t, reconcile()) + synctest.Wait() + require.Equal(t, origins, stored().Destinations) + observed := conditions() + require.True(t, meta.IsStatusConditionTrue(observed, "Ready"), "%v", observed) + condition := meta.FindStatusCondition(observed, "EgressPoliciesApplied") + require.NotNil(t, condition) + require.Equal(t, reason, condition.Reason) + if reason == v1alpha3.EgressPoliciesReasonConflict { + require.Contains(t, condition.Message, "new grants are held until the conflict is resolved; first conflict:") + } + if reason == v1alpha3.EgressPoliciesReasonApplied { + require.Equal(t, metav1.ConditionTrue, condition.Status) + } else { + require.Equal(t, metav1.ConditionFalse, condition.Status) + } + } + const a = "https://a.example.com:443" + const x = "https://x.example.com:443" + policies.UpdateObject(policy("a", a)) + policies.UpdateObject(policy("b", "http://a.example.com:443")) + check(v1alpha3.EgressPoliciesReasonConflict) + policies.UpdateObject(policy("b", x)) + check(v1alpha3.EgressPoliciesReasonApplied, a, x) + // The new HTTP rule conflicts with a. An unrelated addition is held + // too, and the removed x is revoked immediately. + policies.UpdateObject(policy("b", "http://a.example.com:443", "https://new.example.com")) + check(v1alpha3.EgressPoliciesReasonConflict, a) + // Repeating preparation cannot admit previously held additions. + check(v1alpha3.EgressPoliciesReasonConflict, a) + policies.DeleteObject("team-a/b") + check(v1alpha3.EgressPoliciesReasonNotFound, a) + policies.UpdateObject(policy("b", "https://new.example.com")) + check(v1alpha3.EgressPoliciesReasonApplied, a, "https://new.example.com:443") + }) + }) + } +} + +func TestAgentGrantAdmissionWithAndWithoutCompiledBase(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + collections, harnesses := newPreparationTestCollections(t, "gvisor") + initial := collections.Reconciliations.List()[0] + template := proto.CloneOf(initial.Target.ActorTemplate) + template.Metadata.Uid, template.Status = "prepared", sandboxGoldenStatus() + store := &fakeRuntimeRevisionStore{} + r := &Reconciler{collections: collections, templates: &fakeActorTemplates{template: template}, store: store} + policies := collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]) + agents := collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]) + agent := initial.Agent.DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "a"}, {Name: "b"}} + agents.UpdateObject(agent) + harness := harnesses.List()[0] + harnesses.DeleteObject("team-a/byo") + + const tickets = "https://tickets.example.com:443" + const controller = "https://kagent-controller.kagent:8083" + policy := func(name, origin string) *v1alpha3.EgressPolicy { + return &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: name}, + Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: origin}}}} + } + policies.UpdateObject(policy("a", tickets)) + policies.UpdateObject(policy("b", controller)) + check := func(compiled, conflict bool, want ...string) { + t.Helper() + synctest.Wait() + state := collections.Reconciliations.GetKey(initial.ResourceName()) + require.Equal(t, compiled, state.Target != nil) + require.Equal(t, conflict, state.EgressConflict != "") + require.NoError(t, r.reconcileAgent(t.Context(), initial.ResourceName())) + require.ElementsMatch(t, want, store.pair.EgressGrant.Destinations) + } + // With no target, the requested grants are validated on their own. + check(false, false, tickets, controller) + // Restoring compilation introduces an HTTP base that conflicts with the + // previously admitted HTTPS grant on the same host and port. + harnesses.UpdateObject(harness) + check(true, true, tickets) + + harnesses.DeleteObject("team-a/byo") + conflicting := policy("b", "http://tickets.example.com:443") + conflicting.Spec.Destinations = append(conflicting.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://new.example.com"}) + policies.UpdateObject(conflicting) + // Conflicts between policies still hold additions without a target. + check(false, true, tickets) + // A replacement source cannot retain grants belonging to the old UID. + agent = agent.DeepCopy() + agent.UID = "replacement" + agents.UpdateObject(agent) + check(false, true) + }) +} diff --git a/go/core/internal/controller/grants_test.go b/go/core/internal/controller/grants_test.go new file mode 100644 index 0000000000..d331bb9774 --- /dev/null +++ b/go/core/internal/controller/grants_test.go @@ -0,0 +1,377 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "testing/synctest" + "time" + + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/substrate" +) + +func TestAgentEgressErrorsDoNotBlockCompilation(t *testing.T) { + for _, test := range []struct { + name string + destinations []v1alpha3.EgressDestination + condition string + reason string + }{ + {name: "missing policy", condition: v1alpha3.AgentConditionEgressPoliciesApplied, reason: v1alpha3.EgressPoliciesReasonNotFound}, + {name: "invalid origin", destinations: []v1alpha3.EgressDestination{{Origin: "invalid"}}, condition: v1alpha3.AgentConditionEgressPoliciesApplied, reason: v1alpha3.EgressPoliciesReasonInvalidEgressPolicy}, + {name: "missing secret", destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com", Authentication: &v1alpha3.EgressAuthentication{Header: "Authorization", SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "missing"}, Key: "token"}}}}, condition: v1alpha3.AgentConditionEgressPoliciesApplied, reason: v1alpha3.EgressPoliciesReasonInvalidEgressPolicy}, + } { + t.Run(test.name, func(t *testing.T) { + collections, _ := newPreparationTestCollections(t, "gvisor") + if test.destinations != nil { + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(&v1alpha3.EgressPolicy{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, + Spec: v1alpha3.EgressPolicySpec{Destinations: test.destinations}, + }) + } + agent := collections.Agents.List()[0].DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + waitFor(t, func() bool { + observed := collections.AgentStatuses.GetKey("team-a/assistant") + return observed != nil && meta.IsStatusConditionFalse(observed.Status.Conditions, test.condition) + }) + condition := meta.FindStatusCondition(collections.AgentStatuses.GetKey("team-a/assistant").Status.Conditions, test.condition) + require.Equal(t, test.reason, condition.Reason) + state := collections.Reconciliations.GetKey("team-a/assistant") + require.NotNil(t, state.Target) + require.Nil(t, state.CompilationFailure) + require.Empty(t, state.EgressGrant.Destinations) + require.Empty(t, state.EgressGrant.Credentials) + }) + } +} + +func TestAgentGrantChangesReconcileWithoutRuntimeChange(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + collections, harnesses := newPreparationTestCollections(t, "gvisor") + initial := collections.Reconciliations.List()[0] + template := proto.CloneOf(initial.Target.ActorTemplate) + template.Metadata.Uid = "prepared" + template.Status = sandboxGoldenStatus() + store := &fakeRuntimeRevisionStore{definitionSources: []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}} + backend := &preparationTestBackend{templates: &fakeActorTemplates{template: template}, store: store} + r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3()) + ctx, cancel := context.WithCancel(t.Context()) + var workers sync.WaitGroup + workers.Go(func() { assert.NoError(t, r.Start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + require.Empty(t, store.retired, "startup must preserve a matching live Agent") + store.sources = nil + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(policy) + agent := initial.Agent.DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + synctest.Wait() + backend.inspect(func() { + require.Equal(t, initial.Target.RevisionID.String(), store.pair.DesiredRevision) + require.Equal(t, []string{"https://tickets.example.com:443"}, store.pair.EgressGrant.Destinations) + }) + require.Equal(t, []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}, store.sources) + store.sources = nil + harnesses.DeleteObject("team-a/byo") + synctest.Wait() + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.NotEmpty(t, store.pair.EgressGrant.Destinations) + }) + require.Len(t, store.sources, 1, "every preparation reapplies grants, including unchanged grants") + store.sources = nil + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).DeleteObject("team-a/access") + synctest.Wait() + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.Empty(t, store.pair.EgressGrant.Destinations) + }) + require.Len(t, store.sources, 1, "revocation must reconcile even while the runtime remains unresolvable") + }) +} + +func TestAgentGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { + for _, missing := range []string{"Harness", "AgentTemplate", "WorkerPool"} { + t.Run(missing, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + collections, harnesses := newPreparationTestCollections(t, "gvisor") + agent := collections.Agents.List()[0].DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + switch missing { + case "Harness": + agent.Spec.HarnessRef.Name = "missing" + case "AgentTemplate": + agent.Spec.Template = v1alpha3.AgentTemplateSource{Ref: &corev1.LocalObjectReference{Name: "missing"}} + case "WorkerPool": + harness := harnesses.List()[0].DeepCopy() + harness.Spec.Substrate.WorkerPoolRef.Name = "missing" + harnesses.UpdateObject(harness) + } + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + synctest.Wait() + store := &fakeRuntimeRevisionStore{} + backend := &preparationTestBackend{templates: &fakeActorTemplates{}, store: store} + r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3()) + ctx, cancel := context.WithCancel(t.Context()) + var workers sync.WaitGroup + workers.Go(func() { assert.NoError(t, r.Start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + store.sources = nil + + policies := collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]) + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} + policies.UpdateObject(policy) + synctest.Wait() + state := collections.Reconciliations.GetKey("team-a/assistant") + require.Nil(t, state.Target) + require.NotNil(t, state.CompilationFailure) + require.Contains(t, state.CompilationFailure.Message, missing) + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.Equal(t, []string{"https://tickets.example.com:443"}, store.pair.EgressGrant.Destinations) + }) + wantSource := database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"} + require.Equal(t, []database.RuntimeEgressSource{wantSource}, store.sources) + + policy = policy.DeepCopy() + policy.Spec.Destinations[0].Origin = "https://updated.example.com" + policies.UpdateObject(policy) + synctest.Wait() + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.Equal(t, []string{"https://updated.example.com:443"}, store.pair.EgressGrant.Destinations) + }) + require.Equal(t, []database.RuntimeEgressSource{wantSource, wantSource}, store.sources) + + policies.DeleteObject("team-a/access") + synctest.Wait() + backend.inspect(func() { + require.Empty(t, store.pair.DesiredRevision) + require.Empty(t, store.pair.EgressGrant.Destinations) + require.Zero(t, store.recordCalls, "grant changes must not require a compiled runtime") + }) + require.Equal(t, []database.RuntimeEgressSource{wantSource, wantSource, wantSource}, store.sources, "revocation must apply actors while compilation remains broken") + }) + }) + } +} + +func TestAgentGrantWriteFailureAndRetirementApplyActors(t *testing.T) { + collections, _ := newPreparationTestCollections(t, "gvisor") + state := collections.Reconciliations.List()[0] + store := &fakeRuntimeRevisionStore{pairErr: errors.New("response lost")} + r := &Reconciler{collections: collections, store: store, status: kagentfake.NewSimpleClientset(state.Agent.DeepCopy()).ApiV1alpha3()} + require.ErrorIs(t, r.reconcileAgent(t.Context(), state.ResourceName()), store.pairErr) + require.Empty(t, store.sources, "a failed write retries preparation before applying actors") + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).DeleteObject(state.ResourceName()) + waitFor(t, func() bool { return collections.Reconciliations.GetKey(state.ResourceName()) == nil }) + require.NoError(t, r.reconcileAgent(t.Context(), state.ResourceName())) + require.Equal(t, []database.RuntimeEgressSource{{Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}}, store.sources) +} + +func TestAgentStartupRecoversOfflineDeletions(t *testing.T) { + for _, namespaces := range [][]string{nil, {"team-a"}} { + t.Run(fmt.Sprint(namespaces), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + collections, _ := newPreparationTestCollections(t, "gvisor") + collections.watchNamespaces = namespaces + // Delete before registering handlers: no KRT event can recover it. + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).DeleteObject("team-a/assistant") + synctest.Wait() + sources := []database.RuntimeEgressSource{ + {Kind: database.RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, + {Kind: database.RuntimeEgressAgent, Namespace: "team-b", Name: "assistant"}, + } + store := &fakeRuntimeRevisionStore{definitionSources: sources, egressTargetStoreStub: egressTargetStoreStub{ + targets: []database.RuntimeEgressTarget{{ID: "actor", Revision: "old", Atespace: "team-a", TemplateName: "runtime"}}, + }} + attempts := 0 + actors := &fakeActorTemplates{actorEgressStub: actorEgressStub{ensurePolicy: func(context.Context, string, string, string) error { + assert.NotEmpty(t, store.retired, "retire before applying revocation") + attempts++ + if attempts == 1 { + return errors.New("backend unavailable") + } + return nil + }}} + r := newReconciler(collections, actors, store, kagentfake.NewSimpleClientset().ApiV1alpha3()) + var workers sync.WaitGroup + workers.Go(func() { assert.NoError(t, r.Start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + time.Sleep(time.Second) + synctest.Wait() + want := []database.RuntimeEgressSource{sources[0], sources[0]} + if len(namespaces) == 0 { + want = append(want, sources[1]) + } + require.ElementsMatch(t, want, store.sources, "recovery must honor namespace scope and retry revocation") + require.Equal(t, len(want), attempts) + require.Zero(t, store.pairCalls, "missing Agents must not be recreated") + }) + }) + } +} + +func TestAgentStartupRecoveryReadFailure(t *testing.T) { + collections, _ := newPreparationTestCollections(t, "gvisor") + store := &fakeRuntimeRevisionStore{sourcesErr: errors.New("database unavailable")} + r := newReconciler(collections, &fakeActorTemplates{}, store, kagentfake.NewSimpleClientset().ApiV1alpha3()) + require.ErrorIs(t, r.Start(t.Context()), store.sourcesErr) + require.Zero(t, store.pairCalls, "failed recovery must fail startup") +} + +func TestSandboxGrantsSurviveUnrelatedCompilationFailure(t *testing.T) { + opts := krt.NewOptionsBuilder(t.Context().Done(), "test-grants", nil) + template := sandboxTestTemplate() + template.Finalizers = []string{sandboxPreparationFinalizer} + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + templates := krt.NewStaticCollection(nil, []*v1alpha3.SandboxTemplate{template}, opts.WithName("Templates")...) + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com"}}}} + policies := krt.NewStaticCollection(nil, []*v1alpha3.EgressPolicy{policy}, opts.WithName("Policies")...) + pools := krt.NewStaticCollection[*atev1alpha1.WorkerPool](nil, nil, opts.WithName("Pools")...) + store := &sandboxTestStore{} + s := &SandboxReconciler{collections: newSandboxCollections(Collections{SandboxTemplates: templates, WorkerPools: pools, EgressPolicies: policies}, substrate.SandboxPolicy{}, opts), store: store, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3()} + waitFor(t, func() bool { return s.collections.states.GetKey("team-a/scratch") != nil }) + require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) + require.Nil(t, s.collections.states.GetKey("team-a/scratch").DesiredActorTemplate, "missing WorkerPool prevents runtime compilation") + require.Equal(t, []string{"https://tickets.example.com:443"}, store.desired.EgressGrant.Destinations) + require.Len(t, store.sources, 1) + policies.DeleteObject("team-a/access") + waitFor(t, func() bool { return len(s.collections.states.GetKey("team-a/scratch").EgressIssues) != 0 }) + require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) + require.Empty(t, store.desired.EgressGrant.Destinations) + require.Len(t, store.sources, 2, "revocation must reach existing actors during failed compilation") + store.upsertErr = errors.New("response lost") + require.ErrorIs(t, s.reconcile(t.Context(), "team-a/scratch"), store.upsertErr) + require.Len(t, store.sources, 2, "failed writes retry before actor application") + template = template.DeepCopy() + template.DeletionTimestamp = &metav1.Time{Time: metav1.Now().Time} + templates.UpdateObject(template) + waitFor(t, func() bool { return !s.collections.states.GetKey("team-a/scratch").Template.DeletionTimestamp.IsZero() }) + require.NoError(t, s.reconcile(t.Context(), "team-a/scratch")) + require.Len(t, store.sources, 3) +} + +// Registering the KRT handler must recover actors even when their grants already +// match persisted state. A failed push must retry without another graph event. +func TestPreparationReplaysAndRetriesUnchangedGrants(t *testing.T) { + for _, kind := range []string{"agent", "sandbox"} { + t.Run(kind, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + grants := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} + target := database.RuntimeEgressTarget{ID: "actor", Revision: "old", Atespace: "team-a", TemplateName: "runtime"} + var start func(context.Context) error + var persisted func() egress.Access + var actorStub *actorEgressStub + switch kind { + case "agent": + collections, harnesses := newPreparationTestCollections(t, "gvisor") + agent := collections.Agents.List()[0].DeepCopy() + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "access"}} + collections.EgressPolicies.(krt.StaticCollection[*v1alpha3.EgressPolicy]).UpdateObject(&v1alpha3.EgressPolicy{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "access"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: grants.Destinations[0]}}}, + }) + collections.Agents.(krt.StaticCollection[*v1alpha3.Agent]).UpdateObject(agent) + harnesses.DeleteObject("team-a/byo") + synctest.Wait() + store := &fakeRuntimeRevisionStore{pair: &database.AgentDefinition{EgressGrant: grants}, egressTargetStoreStub: egressTargetStoreStub{targets: []database.RuntimeEgressTarget{target}}} + actors := &fakeActorTemplates{} + backend := &preparationTestBackend{templates: actors, store: store} + r := newReconciler(collections, backend, backend, kagentfake.NewSimpleClientset(agent.DeepCopy()).ApiV1alpha3()) + start = r.Start + persisted = func() egress.Access { backend.mu.Lock(); defer backend.mu.Unlock(); return store.pair.EgressGrant } + actorStub = &actors.actorEgressStub + case "sandbox": + opts := krt.NewOptionsBuilder(ctx.Done(), "retry-grants", nil) + template := sandboxTestTemplate() + template.Finalizers = []string{sandboxPreparationFinalizer} + states := krt.NewStaticCollection(nil, []sandboxReconciliation{{Template: template, EgressGrant: grants}}, opts.WithName("states")...) + store := &sandboxTestStore{desired: database.SandboxTemplateDefinition{EgressGrant: grants}, egressTargetStoreStub: egressTargetStoreStub{targets: []database.RuntimeEgressTarget{target}}} + actors := &sandboxTestActors{} + s := &SandboxReconciler{collections: sandboxCollections{states: states, observations: krt.NewStaticCollection[sandboxRuntimeObservation](nil, nil, opts.WithName("observations")...)}, store: store, actors: actors, client: kagentfake.NewSimpleClientset(template.DeepCopy()).ApiV1alpha3()} + start = s.Start + persisted = func() egress.Access { store.mu.Lock(); defer store.mu.Unlock(); return store.desired.EgressGrant } + actorStub = &actors.actorEgressStub + } + var mu sync.Mutex + var attempts []time.Time + var applied egress.Access + actorStub.ensurePolicy = func(context.Context, string, string, string) error { + mu.Lock() + defer mu.Unlock() + attempts = append(attempts, time.Now()) + if len(attempts) == 1 { + return errors.New("backend unavailable") + } + applied = persisted() + return nil + } + var workers sync.WaitGroup + workers.Go(func() { assert.NoError(t, start(ctx)) }) + defer func() { cancel(); workers.Wait() }() + synctest.Wait() + mu.Lock() + require.Len(t, attempts, 1, "startup must attempt actors with unchanged grants") + mu.Unlock() + time.Sleep(time.Second) + synctest.Wait() + mu.Lock() + defer mu.Unlock() + require.Len(t, attempts, 2, "the preparation queue must retry a failed push without input changes") + require.Equal(t, time.Second, attempts[1].Sub(attempts[0])) + require.Equal(t, grants.Destinations, applied.Destinations) + }) + }) + } +} + +func TestSandboxFinalizerWaitsForActorRevocation(t *testing.T) { + s, templates, _ := newSandboxTestReconciler(t, "") + const key = "team-a/scratch" + require.NoError(t, s.reconcile(t.Context(), key)) + template := syncSandboxTemplate(t, s, templates).DeepCopy() + template.DeletionTimestamp = &metav1.Time{Time: metav1.Now().Time} + templates.UpdateObject(template) + waitFor(t, func() bool { return !s.collections.states.GetKey(key).Template.DeletionTimestamp.IsZero() }) + store := s.store.(*sandboxTestStore) + store.targets = []database.RuntimeEgressTarget{{ID: "actor", Revision: "old", Atespace: "team-a", TemplateName: "runtime"}} + pushErr := errors.New("backend unavailable") + s.actors.(*sandboxTestActors).ensurePolicy = func(context.Context, string, string, string) error { + assert.True(t, store.retired, "retire before applying revocation") + return pushErr + } + require.ErrorIs(t, s.reconcile(t.Context(), key), pushErr) + persisted, err := s.client.SandboxTemplates("team-a").Get(t.Context(), "scratch", metav1.GetOptions{}) + require.NoError(t, err) + require.Contains(t, persisted.Finalizers, sandboxPreparationFinalizer) + pushErr = nil + require.NoError(t, s.reconcile(t.Context(), key)) + persisted, err = s.client.SandboxTemplates("team-a").Get(t.Context(), "scratch", metav1.GetOptions{}) + require.NoError(t, err) + require.NotContains(t, persisted.Finalizers, sandboxPreparationFinalizer) +} diff --git a/go/core/internal/controller/preparation_test.go b/go/core/internal/controller/preparation_test.go index 46a07ba93c..08684d2560 100644 --- a/go/core/internal/controller/preparation_test.go +++ b/go/core/internal/controller/preparation_test.go @@ -12,6 +12,7 @@ import ( atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -27,6 +28,7 @@ import ( kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -97,7 +99,7 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { // A delayed success for the abandoned preparation must not replace A. abandoned, err := store.GetRuntimeRevision(ctx, preparing.Target.RevisionID.String()) require.NoError(t, err) - require.NoError(t, store.RecordRuntimeRevision(ctx, *abandoned, true)) + require.NoError(t, store.RecordRuntimeRevision(ctx, *abandoned, database.RecordRevisionOptions{Ready: true})) unreferenced, err = store.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) require.Len(t, unreferenced, 1) @@ -106,12 +108,12 @@ func TestUnresolvedPoolReleasesAbandonedRevision(t *testing.T) { require.NoError(t, err) require.Nil(t, retained, "last-successful A must remain protected without any sessions") - session, _, err := store.CreateSession(ctx, &apiv1.Session{ + fresh, _, err := store.CreateSession(ctx, &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}, }, "last-good-session") - require.NoError(t, err) - require.Equal(t, initial.Target.RevisionID.String(), session.GetPreparedRevision()) + require.NoError(t, err, "unresolved edits must leave the last successful revision available") + require.Equal(t, initial.Target.RevisionID.String(), fresh.PreparedRevision) collector, _ := newTestRuntimeRevisionGC(t, store, templates) require.NoError(t, collector.collect(ctx, abandoned.Revision)) @@ -281,7 +283,7 @@ func TestPreparationQueueAndPollerBackoff(t *testing.T) { r := newReconciler(collections, backend, backend, statusClient) ctx, cancel := context.WithCancel(t.Context()) defer cancel() - go r.Run(ctx.Done()) + go func() { assert.NoError(t, r.Start(ctx)) }() synctest.Wait() backend.inspect(func() { require.Equal(t, 1, test.store.pairCalls, "publishing a failure must not immediately requeue it") @@ -381,22 +383,28 @@ func (p *preparationTestBackend) CreateActorTemplate(ctx context.Context, templa return p.templates.CreateActorTemplate(ctx, template) } +func (p *preparationTestBackend) ListAgentEgressSources(ctx context.Context) ([]database.RuntimeEgressSource, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.store.ListAgentEgressSources(ctx) +} + func (p *preparationTestBackend) UpsertAgentDefinition(ctx context.Context, definition database.AgentDefinition) error { p.mu.Lock() defer p.mu.Unlock() return p.store.UpsertAgentDefinition(ctx, definition) } -func (p *preparationTestBackend) RecordRuntimeRevision(ctx context.Context, revision database.RuntimeRevision, ready bool) error { +func (p *preparationTestBackend) RecordRuntimeRevision(ctx context.Context, revision database.RuntimeRevision, options database.RecordRevisionOptions) error { p.mu.Lock() defer p.mu.Unlock() - return p.store.RecordRuntimeRevision(ctx, revision, ready) + return p.store.RecordRuntimeRevision(ctx, revision, options) } -func (p *preparationTestBackend) RetireAgentIdentities(ctx context.Context, namespace, name string, except *database.AgentDefinition) error { +func (p *preparationTestBackend) RetireAgentIdentities(ctx context.Context, namespace, name string) error { p.mu.Lock() defer p.mu.Unlock() - return p.store.RetireAgentIdentities(ctx, namespace, name, except) + return p.store.RetireAgentIdentities(ctx, namespace, name) } func TestPreparationDesiredChangesBypassBackoff(t *testing.T) { @@ -408,7 +416,7 @@ func TestPreparationDesiredChangesBypassBackoff(t *testing.T) { r := newReconciler(collections, templates, store, kagentfake.NewSimpleClientset(initial.Agent.DeepCopy()).ApiV1alpha3()) ctx, cancel := context.WithCancel(t.Context()) defer cancel() - go r.Run(ctx.Done()) + go func() { assert.NoError(t, r.Start(ctx)) }() synctest.Wait() time.Sleep(10 * time.Second) synctest.Wait() @@ -462,7 +470,7 @@ func TestPreparationTerminalFailures(t *testing.T) { r := newReconciler(collections, templates, store, statusClient) ctx, cancel := context.WithCancel(t.Context()) defer cancel() - go r.Run(ctx.Done()) + go func() { assert.NoError(t, r.Start(ctx)) }() synctest.Wait() require.Equal(t, 1, store.pairCalls) state := collections.Reconciliations.GetKey(initial.ResourceName()) @@ -549,6 +557,7 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections &atev1alpha1.WorkerPool{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "microvm"}, Spec: atev1alpha1.WorkerPoolSpec{SandboxClass: atev1alpha1.SandboxClassMicroVM}}, }) collections := Collections{ + EgressPolicies: krt.NewStaticCollection[*kagentv1alpha3.EgressPolicy](nil, nil, opts.WithName("EgressPolicies")...), AgentTemplates: krttest.GetMockCollection[*kagentv1alpha3.AgentTemplate](mock), Harnesses: harnesses, ResolvedModelConfigs: krttest.GetMockCollection[translator.ResolvedModelConfig](mock), @@ -563,7 +572,7 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections collections.Reconciliations = newAgentReconciliations(collections.Agents, translator.Collections{ Harnesses: collections.Harnesses, AgentTemplates: collections.AgentTemplates, ResolvedModelConfigs: collections.ResolvedModelConfigs, RemoteMCPServers: collections.RemoteMCPServers, ConfigMaps: collections.ConfigMaps, - Secrets: collections.Secrets, WorkerPools: collections.WorkerPools, + Secrets: collections.Secrets, WorkerPools: collections.WorkerPools, EgressPolicies: collections.EgressPolicies, }, collections.AgentRuntimeObservations, opts, testRuntimeDefaults(), nil) collections.AgentStatuses = newAgentStatuses(collections.Agents, collections.Reconciliations, opts) waitFor(t, func() bool { @@ -572,3 +581,21 @@ func newPreparationTestCollections(t *testing.T, workerPool string) (Collections }) return collections, harnesses } + +func (p *preparationTestBackend) ListRuntimeEgressSourceTargets(ctx context.Context, source database.RuntimeEgressSource, after string, limit int) ([]database.RuntimeEgressTarget, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.store.ListRuntimeEgressSourceTargets(ctx, source, after, limit) +} +func (p *preparationTestBackend) GetActor(ctx context.Context, atespace, name string) (*ateapipb.Actor, error) { + return p.templates.GetActor(ctx, atespace, name) +} +func (p *preparationTestBackend) EnsureActorEgressPolicy(ctx context.Context, atespace, name, revision string) error { + return p.templates.EnsureActorEgressPolicy(ctx, atespace, name, revision) +} + +func (p *preparationTestBackend) GetAgentEgressGrant(ctx context.Context, namespace, uid string) (egress.Access, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.store.GetAgentEgressGrant(ctx, namespace, uid) +} diff --git a/go/core/internal/controller/reconciler.go b/go/core/internal/controller/reconciler.go index df2c2eda5a..23ecc054db 100644 --- a/go/core/internal/controller/reconciler.go +++ b/go/core/internal/controller/reconciler.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "reflect" + "slices" "strings" "time" @@ -26,6 +27,7 @@ import ( configv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/config/v1" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" "github.com/kagent-dev/kagent/go/core/internal/translator" byotranslator "github.com/kagent-dev/kagent/go/core/internal/translator/byo" @@ -39,6 +41,9 @@ import ( // status instead of disappearing from the graph. type AgentReconciliation struct { Agent *kagentv1alpha3.Agent + EgressGrant egress.Access // Requested union, including conflicting grants. + EgressIssues []translator.EgressGrantIssue + EgressConflict string Target *compiledTarget Warnings []string CompilationFailure *ReconciliationFailure @@ -102,42 +107,27 @@ func newAgentReconciliations( ) krt.Collection[AgentReconciliation] { return krt.NewCollection(agents, func(ctx krt.HandlerContext, agent *kagentv1alpha3.Agent) *AgentReconciliation { state := &AgentReconciliation{Agent: agent} - compilation, err := translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ - translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), - translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings), - translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings), - translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings), - }, images).CompileAgent(context.Background(), agent) - if err != nil { - condition, reason := kagentv1alpha3.AgentConditionResolvedRefs, "ReferenceResolutionFailed" - var validation *translator.ValidationError - var missingPool *translator.WorkerPoolNotFoundError - switch { - case errors.As(err, &validation): - // TODO(tim): Separate structural Agent validation from resolved - // compatibility failures so invalid sources do not report Accepted=True - // and ResolvedRefs=True. - condition, reason = kagentv1alpha3.AgentConditionCompatible, "UnsupportedConfiguration" - case errors.As(err, &missingPool): - reason = "WorkerPoolNotFound" - } - state.CompilationFailure = &ReconciliationFailure{Condition: condition, Reason: reason, Message: err.Error()} + if !agent.DeletionTimestamp.IsZero() { return state } - revision := &compilation.Revision - state.Warnings = append([]string(nil), compilation.Warnings...) - revisionID, err := revision.Digest() - if err != nil { - state.CompilationFailure = &ReconciliationFailure{Condition: kagentv1alpha3.AgentConditionCompatible, Reason: "RevisionInvalid", Message: err.Error()} - return state + resolution := translator.CompileEgressPolicies(ctx, collections, agent.Namespace, agent.Spec.EgressPolicyRefs) + state.EgressGrant, state.EgressIssues = resolution.Access, resolution.Issues + state.Target, state.Warnings, state.CompilationFailure = compileAgentTarget(ctx, collections, agent, settings, images) + var err error + if state.Target == nil { + // Unrelated compilation failures must not freeze grant updates. + _, err = egress.CanonicalAccess(state.EgressGrant.Destinations, state.EgressGrant.Credentials, nil) + } else { + revision := &state.Target.Revision + base := egress.Access{Destinations: revision.EgressDestinations, Credentials: revision.Credentials} + _, err = egress.MergeAccess(base, state.EgressGrant, revision.TokenPassthroughOrigins) } - - actorTemplate, err := substrate.ActorTemplateForRevision(revision, revisionID) if err != nil { - state.CompilationFailure = &ReconciliationFailure{Condition: kagentv1alpha3.AgentConditionCompatible, Reason: "ActorTemplateInvalid", Message: err.Error()} + state.EgressConflict = err.Error() + } + if state.CompilationFailure != nil { return state } - state.Target = &compiledTarget{Revision: *revision, RevisionID: revisionID, ActorTemplate: actorTemplate} observed := krt.FetchOne(ctx, agentRuntimeObservations, krt.FilterKey(state.ResourceName())) if observed == nil || observed.RevisionID != state.Target.RevisionID { @@ -161,16 +151,59 @@ func newAgentReconciliations( }, opts.WithName("AgentReconciliations")...) } +func compileAgentTarget(ctx krt.HandlerContext, collections translator.Collections, agent *kagentv1alpha3.Agent, settings *configv1.RuntimeDefaultsConfig, images *configv1.HarnessImagesConfig) (*compiledTarget, []string, *ReconciliationFailure) { + compilation, err := translator.NewCompiler(ctx, collections, map[translator.HarnessType]translator.HarnessCompiler{ + translator.HarnessTypeKagent: kagenttranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeCodex: codextranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeClaude: claudetranslator.NewCompiler(ctx, collections, settings), + translator.HarnessTypeBYO: byotranslator.NewCompiler(ctx, collections, settings), + }, images).CompileAgent(context.Background(), agent) + if err != nil { + return nil, nil, compilationFailure(err) + } + revision := &compilation.Revision + warnings := slices.Clone(compilation.Warnings) + revisionID, err := revision.Digest() + if err != nil { + return nil, warnings, &ReconciliationFailure{Condition: kagentv1alpha3.AgentConditionCompatible, Reason: "RevisionInvalid", Message: err.Error()} + } + actorTemplate, err := substrate.ActorTemplateForRevision(revision, revisionID) + if err != nil { + return nil, warnings, &ReconciliationFailure{Condition: kagentv1alpha3.AgentConditionCompatible, Reason: "ActorTemplateInvalid", Message: err.Error()} + } + return &compiledTarget{Revision: *revision, RevisionID: revisionID, ActorTemplate: actorTemplate}, warnings, nil +} + +func compilationFailure(err error) *ReconciliationFailure { + condition, reason := kagentv1alpha3.AgentConditionResolvedRefs, "ReferenceResolutionFailed" + var validation *translator.ValidationError + var missingPool *translator.WorkerPoolNotFoundError + switch { + case errors.As(err, &validation): + // TODO(tim): Separate structural Agent validation from resolved + // compatibility failures so invalid sources do not report Accepted=True + // and ResolvedRefs=True. + condition, reason = kagentv1alpha3.AgentConditionCompatible, "UnsupportedConfiguration" + case errors.As(err, &missingPool): + reason = "WorkerPoolNotFound" + } + return &ReconciliationFailure{Condition: condition, Reason: reason, Message: err.Error()} +} + // runtimeRevisionStore is the controller's narrow view of the shared database. // Substrate owns ActorTemplates; the database retains revisions while an Agent // or a Session or checkpoint references them. type runtimeRevisionStore interface { + actorEgressStore UpsertAgentDefinition(context.Context, database.AgentDefinition) error - RecordRuntimeRevision(context.Context, database.RuntimeRevision, bool) error - RetireAgentIdentities(ctx context.Context, namespace, name string, except *database.AgentDefinition) error + GetAgentEgressGrant(context.Context, string, string) (egress.Access, error) + RecordRuntimeRevision(context.Context, database.RuntimeRevision, database.RecordRevisionOptions) error + ListAgentEgressSources(context.Context) ([]database.RuntimeEgressSource, error) + RetireAgentIdentities(ctx context.Context, namespace, name string) error } type actorTemplateClient interface { + actorEgressClient EnsureAtespace(context.Context, string) error GetActorTemplate(context.Context, string, string) (*ateapipb.ActorTemplate, error) CreateActorTemplate(context.Context, *ateapipb.ActorTemplate) (*ateapipb.ActorTemplate, error) @@ -225,10 +258,13 @@ func newReconciler( }) r.agentHandler = collections.Reconciliations.Register(func(event krt.Event[AgentReconciliation]) { - // Observations and status writes must not bypass failure backoff. Only - // a changed desired runtime or Agent identity needs immediate work. + // Desired inputs and cleanup changes bypass failure backoff; observations + // and status writes do not. Grants must reconcile without a new revision. if event.Old != nil && event.New != nil && event.Old.Agent.UID == event.New.Agent.UID && - event.Old.desiredRevision() == event.New.desiredRevision() { + event.Old.desiredRevision() == event.New.desiredRevision() && + reflect.DeepEqual(event.Old.EgressGrant, event.New.EgressGrant) && + reflect.DeepEqual(event.Old.EgressIssues, event.New.EgressIssues) && + event.Old.EgressConflict == event.New.EgressConflict && event.Old.Agent.DeletionTimestamp.Equal(event.New.Agent.DeletionTimestamp) { return } r.agents.Add(krt.GetKey(event.Latest())) @@ -260,19 +296,44 @@ func newReconciliationQueue(name string, reconcile func(any) error) controllers. ) } -// Run waits for the graph boundary to observe initial state, then processes -// Agent and status writes until stop closes. -func (r *Reconciler) Run(stop <-chan struct{}) { +// Start waits for initial Kubernetes state, recovers offline deletions, then +// processes Agent preparation and status writes until cancellation. +func (r *Reconciler) Start(ctx context.Context) error { + stop := ctx.Done() if !r.agentHandler.WaitUntilSynced(stop) || !r.agentStatusHandler.WaitUntilSynced(stop) || !r.modelConfigStatusHandler.WaitUntilSynced(stop) { r.agents.ShutDownEarly() r.agentStatuses.ShutDownEarly() r.modelConfigStatuses.ShutDownEarly() - return + return nil + } + if err := r.enqueueDeletedAgents(ctx); err != nil { + r.agents.ShutDownEarly() + r.agentStatuses.ShutDownEarly() + r.modelConfigStatuses.ShutDownEarly() + return err } go r.pollPendingTemplates(stop) go r.agentStatuses.Run(stop) go r.modelConfigStatuses.Run(stop) r.agents.Run(stop) + return nil +} + +func (r *Reconciler) enqueueDeletedAgents(ctx context.Context) error { + sources, err := r.store.ListAgentEgressSources(ctx) + if err != nil { + return fmt.Errorf("recover deleted Agents: %w", err) + } + for _, source := range sources { + if len(r.collections.watchNamespaces) != 0 && !slices.Contains(r.collections.watchNamespaces, source.Namespace) { + continue + } + key := source.Namespace + "/" + source.Name + if r.collections.Agents.GetKey(key) == nil { + r.agents.Add(key) + } + } + return nil } func (r *Reconciler) pollPendingTemplates(stop <-chan struct{}) { @@ -293,11 +354,6 @@ func (r *Reconciler) pollPendingTemplates(stop <-chan struct{}) { } } -func (r *Reconciler) Start(ctx context.Context) error { - r.Run(ctx.Done()) - return nil -} - func (r *Reconciler) NeedLeaderElection() bool { return true } func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { @@ -306,19 +362,36 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { (state == nil || state.Target == nil || observation.RevisionID != state.Target.RevisionID) { r.collections.AgentRuntimeObservations.DeleteObject(key) } - if state == nil { + if state == nil || !state.Agent.DeletionTimestamp.IsZero() { parts := strings.Split(key, "/") if len(parts) != 2 { return fmt.Errorf("invalid Agent key %q", key) } - if err := r.store.RetireAgentIdentities(ctx, parts[0], parts[1], nil); err != nil { + err := r.store.RetireAgentIdentities(ctx, parts[0], parts[1]) + if err != nil { return fmt.Errorf("retire Agent %s: %w", key, err) } + if err := applySourceEgress(ctx, r.store, r.templates, database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: parts[0], Name: parts[1]}); err != nil { + return err + } return nil } + grants := state.EgressGrant + if state.EgressConflict != "" { + previous, err := r.store.GetAgentEgressGrant(ctx, state.Agent.Namespace, string(state.Agent.UID)) + if err != nil { + return r.observePreparationError(*state, fmt.Errorf("read Agent %s grants: %w", key, err)) + } + grants = egress.IntersectAccess(previous, grants) + if state.Target != nil { + revision := &state.Target.Revision + base := egress.Access{Destinations: revision.EgressDestinations, Credentials: revision.Credentials} + grants, _ = egress.FilterConflicts(base, grants, revision.TokenPassthroughOrigins) + } + } definition := database.AgentDefinition{ Namespace: state.Agent.Namespace, AgentName: state.Agent.Name, - AgentUID: string(state.Agent.UID), DesiredRevision: state.desiredRevision(), + AgentUID: string(state.Agent.UID), DesiredRevision: state.desiredRevision(), EgressGrant: grants, } // Store the desired edge before creating compute so a concurrent collector @@ -328,6 +401,9 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { if err := r.store.UpsertAgentDefinition(ctx, definition); err != nil { return r.observePreparationError(*state, fmt.Errorf("store Agent %s: %w", key, err)) } + if err := applySourceEgress(ctx, r.store, r.templates, database.RuntimeEgressSource{Kind: database.RuntimeEgressAgent, Namespace: definition.Namespace, Name: definition.AgentName}); err != nil { + return r.observePreparationError(*state, err) + } if state.Target == nil { return nil } @@ -368,14 +444,16 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, key string) error { Revision: target.RevisionID.String(), Namespace: definition.Namespace, AgentName: definition.AgentName, AgentUID: definition.AgentUID, SourceSnapshot: target.Revision.Provenance, AgentCard: target.Revision.AgentCard, - EgressDestinations: target.Revision.EgressDestinations, - Credentials: target.Revision.Credentials, - ActorTemplateAtespace: observed.GetMetadata().GetAtespace(), ActorTemplateName: observed.GetMetadata().GetName(), ActorTemplateUID: observed.GetMetadata().GetUid(), + EgressDestinations: target.Revision.EgressDestinations, + Credentials: target.Revision.Credentials, + TokenPassthroughOrigins: target.Revision.TokenPassthroughOrigins, + ActorTemplateAtespace: observed.GetMetadata().GetAtespace(), ActorTemplateName: observed.GetMetadata().GetName(), ActorTemplateUID: observed.GetMetadata().GetUid(), } ready := observed.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() != nil - if err := r.store.RecordRuntimeRevision(ctx, revision, ready); err != nil { + if err := r.store.RecordRuntimeRevision(ctx, revision, database.RecordRevisionOptions{Ready: ready}); err != nil { return r.observePreparationError(*state, fmt.Errorf("store runtime revision %s: %w", target.RevisionID, err)) } + // This observation drives Kubernetes Ready status on a separate queue. // Publish it only after session creation can select the persisted revision. r.observePreparation(*state, observed, nil) diff --git a/go/core/internal/controller/reconciler_test.go b/go/core/internal/controller/reconciler_test.go index 8548e6adc1..39dc1136f7 100644 --- a/go/core/internal/controller/reconciler_test.go +++ b/go/core/internal/controller/reconciler_test.go @@ -11,6 +11,7 @@ import ( a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -26,6 +27,7 @@ import ( kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/dbtest" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/translator" ) @@ -183,11 +185,6 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { states.UpdateObject(state) require.NoError(t, reconciler.reconcileAgent(ctx, state.ResourceName())) - // Compile failures preserve the current UID's last successful runtime. - state.Target = nil - states.UpdateObject(state) - require.NoError(t, reconciler.reconcileAgent(ctx, state.ResourceName())) - require.Empty(t, reconciler.collections.AgentRuntimeObservations.List(), "invalid preparation must release its observation before GC") request := &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}, @@ -195,6 +192,17 @@ func TestRuntimeRevisionGCCollectsRetiredRevisions(t *testing.T) { session, _, err := store.CreateSession(ctx, request, "session") require.NoError(t, err) require.Equal(t, id.String(), session.GetPreparedRevision()) + + // Compile failures preserve the last successful runtime and pinned retries. + state.Target = nil + states.UpdateObject(state) + require.NoError(t, reconciler.reconcileAgent(ctx, state.ResourceName())) + require.Empty(t, reconciler.collections.AgentRuntimeObservations.List(), "invalid preparation must release its observation before GC") + retried, created, err := store.CreateSession(ctx, request, "session") + require.NoError(t, err) + require.False(t, created) + require.Equal(t, session.GetId(), retried.GetId()) + require.Equal(t, id.String(), retried.GetPreparedRevision()) operation, err := store.BeginSessionOperation(ctx, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE) require.NoError(t, err) executor := uuid.New() @@ -272,6 +280,7 @@ func (s *failingFinalizationStore) DeleteRuntimeRevision(ctx context.Context, re } type fakeActorTemplates struct { + actorEgressStub template *ateapipb.ActorTemplate ensureErr error getErr error @@ -313,13 +322,21 @@ func (f *fakeActorTemplates) DeleteActorTemplate(context.Context, string, string } type fakeRuntimeRevisionStore struct { - pair *database.AgentDefinition - revision *database.RuntimeRevision - markedSuccessful bool - retired string - revisionErr error - pairErr error - pairCalls int + egressTargetStoreStub + pair *database.AgentDefinition + revision *database.RuntimeRevision + markedSuccessful bool + retired string + revisionErr error + recordCalls int + pairErr error + pairCalls int + definitionSources []database.RuntimeEgressSource + sourcesErr error +} + +func (s *fakeRuntimeRevisionStore) ListAgentEgressSources(context.Context) ([]database.RuntimeEgressSource, error) { + return s.definitionSources, s.sourcesErr } func (s *fakeRuntimeRevisionStore) UpsertAgentDefinition(_ context.Context, pair database.AgentDefinition) error { @@ -328,19 +345,20 @@ func (s *fakeRuntimeRevisionStore) UpsertAgentDefinition(_ context.Context, pair return s.pairErr } -func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision, ready bool) error { +func (s *fakeRuntimeRevisionStore) RecordRuntimeRevision(_ context.Context, revision database.RuntimeRevision, options database.RecordRevisionOptions) error { + s.recordCalls++ if s.revisionErr != nil { return s.revisionErr } s.revision = &revision - s.markedSuccessful = ready + if options.Ready { + s.markedSuccessful = true + } return nil } -func (s *fakeRuntimeRevisionStore) RetireAgentIdentities(_ context.Context, namespace, template string, except *database.AgentDefinition) error { - if except == nil { - s.retired = namespace + "/" + template - } +func (s *fakeRuntimeRevisionStore) RetireAgentIdentities(_ context.Context, namespace, template string) error { + s.retired = namespace + "/" + template return nil } @@ -387,7 +405,7 @@ func TestReconcilerUpdatesModelConfigStatusOnSecretHashChange(t *testing.T) { statusClient, ) - go reconciler.Run(stop) + go func() { assert.NoError(t, reconciler.Start(t.Context())) }() var initialUpdate *kagentv1alpha3.ModelConfig var err error @@ -450,3 +468,10 @@ func TestReconciliationQueueRetriesWithBackoff(t *testing.T) { require.NoError(t, queue.WaitForClose(time.Second)) }) } + +func (s *fakeRuntimeRevisionStore) GetAgentEgressGrant(_ context.Context, namespace, uid string) (egress.Access, error) { + if s.pair != nil && s.pair.Namespace == namespace && s.pair.AgentUID == uid && s.retired == "" { + return s.pair.EgressGrant, nil + } + return egress.Access{}, nil +} diff --git a/go/core/internal/controller/sandbox_collections.go b/go/core/internal/controller/sandbox_collections.go index 52712a7eea..f02fa30fb6 100644 --- a/go/core/internal/controller/sandbox_collections.go +++ b/go/core/internal/controller/sandbox_collections.go @@ -10,7 +10,9 @@ import ( "istio.io/istio/pkg/kube/krt" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" + "github.com/kagent-dev/kagent/go/core/internal/translator" ) // Sandbox preparation uses the same informer graph as Agents. Only observations @@ -24,6 +26,9 @@ type sandboxReconciliation struct { Template *kagentv1alpha3.SandboxTemplate RevisionID string SourceSnapshot json.RawMessage + EgressGrant egress.Access // Requested union, including conflicting grants. + EgressIssues []translator.EgressGrantIssue + EgressConflict string DesiredActorTemplate *ateapipb.ActorTemplate ObservedActorTemplate *ateapipb.ActorTemplate CompilationError string @@ -84,6 +89,12 @@ func newSandboxCollections(inputs Collections, policy substrate.SandboxPolicy, o if !template.DeletionTimestamp.IsZero() { return state } + resolution := translator.CompileEgressPolicies(ctx, translator.Collections{EgressPolicies: inputs.EgressPolicies, Secrets: inputs.Secrets}, template.Namespace, template.Spec.EgressPolicyRefs) + state.EgressGrant, state.EgressIssues = resolution.Access, resolution.Issues + // Sandboxes have no pinned model or tool access. + if _, err := egress.CanonicalAccess(state.EgressGrant.Destinations, state.EgressGrant.Credentials, nil); err != nil { + state.EgressConflict = err.Error() + } pool := krt.FetchOne(ctx, inputs.WorkerPools, krt.FilterKey(template.Namespace+"/"+template.Spec.Substrate.WorkerPoolRef.Name)) if pool == nil { state.Failure = &ReconciliationFailure{Reason: "WorkerPoolNotFound", Message: "The referenced sandbox WorkerPool does not exist"} diff --git a/go/core/internal/controller/sandbox_integration_test.go b/go/core/internal/controller/sandbox_integration_test.go index 133fe4b5ee..c9cb9c079c 100644 --- a/go/core/internal/controller/sandbox_integration_test.go +++ b/go/core/internal/controller/sandbox_integration_test.go @@ -42,6 +42,7 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { template := sandboxTestTemplate() template.UID = "" template.Generation = 0 + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "tickets"}} template, err = client.ApiV1alpha3().SandboxTemplates(template.Namespace).Create(t.Context(), template, metav1.CreateOptions{}) require.NoError(t, err) unwatched := template.DeepCopy() @@ -60,6 +61,8 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { options := krt.NewOptionsBuilder(ctx.Done(), "sandbox-integration", nil) pools := krt.NewStaticCollection(nil, []*atev1alpha1.WorkerPool{pool}, options.WithName("WorkerPools")...) runtime := &Runtime{Client: runtimeClient, Options: options, Collections: Collections{ + EgressPolicies: typedCollection[*kagentv1alpha3.EgressPolicy](runtimeClient, []string{"team-a"}, "EgressPolicies", options), + Secrets: typedCollection[*corev1.Secret](runtimeClient, []string{"team-a"}, "Secrets", options), SandboxTemplates: typedCollection[*kagentv1alpha3.SandboxTemplate](runtimeClient, []string{"team-a"}, "SandboxTemplates", options), WorkerPools: pools, }} reconciler, err := NewSandboxReconciler(config, runtime, store, actors, substrate.SandboxPolicy{GuestImage: "guest@sha256:" + strings.Repeat("b", 64), CPU: "1", Memory: "1Gi"}) @@ -89,6 +92,27 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { } return apimeta.IsStatusConditionTrue(current.Status.Conditions, "Ready") } + waitUnresolved := func(reason, message string) { + t.Helper() + require.Eventually(t, func() bool { + current, err := client.ApiV1alpha3().SandboxTemplates(template.Namespace).Get(t.Context(), template.Name, metav1.GetOptions{}) + if err != nil { + return false + } + condition := apimeta.FindStatusCondition(current.Status.Conditions, kagentv1alpha3.SandboxTemplateConditionEgressPoliciesApplied) + return condition != nil && condition.Status == metav1.ConditionFalse && condition.Reason == reason && strings.Contains(condition.Message, message) + }, 10*time.Second, 10*time.Millisecond) + } + waitUnresolved(kagentv1alpha3.EgressPoliciesReasonNotFound, "tickets") + require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) + grant := &kagentv1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "tickets"}, Spec: kagentv1alpha3.EgressPolicySpec{ + Destinations: []kagentv1alpha3.EgressDestination{{Origin: "https://tickets.example.com", Authentication: &kagentv1alpha3.EgressAuthentication{Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "ticket-auth"}, Key: "token"}}}}, + }} + grant, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Create(t.Context(), grant, metav1.CreateOptions{}) + require.NoError(t, err) + waitUnresolved(kagentv1alpha3.EgressPoliciesReasonInvalidEgressPolicy, "Secret") + secret, err := kube.Kube().CoreV1().Secrets(template.Namespace).Create(t.Context(), &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: "ticket-auth"}, Data: map[string][]byte{"token": []byte("original-token")}}, metav1.CreateOptions{}) + require.NoError(t, err) require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) current, err := client.ApiV1alpha3().SandboxTemplates(template.Namespace).Get(t.Context(), template.Name, metav1.GetOptions{}) require.NoError(t, err) @@ -96,12 +120,42 @@ func TestSandboxKRTInformerQueueAndRestartCleanup(t *testing.T) { store.mu.Lock() revision := store.revision.Revision require.True(t, store.ready) + require.NotContains(t, string(store.revision.SourceSnapshot), "original-token") store.mu.Unlock() other, err := client.ApiV1alpha3().SandboxTemplates(unwatched.Namespace).Get(t.Context(), unwatched.Name, metav1.GetOptions{}) require.NoError(t, err) require.Empty(t, other.Finalizers, "namespace filters must apply to the new informer") require.Empty(t, other.Status.Conditions) + // A missing key revokes its policy without changing runtime readiness. + // Restoring it admits grants without serializing either value. + secret.Data = nil + secret, err = kube.Kube().CoreV1().Secrets(template.Namespace).Update(t.Context(), secret, metav1.UpdateOptions{}) + require.NoError(t, err) + waitUnresolved(kagentv1alpha3.EgressPoliciesReasonInvalidEgressPolicy, "missing key") + require.True(t, ready()) + secret.Data = map[string][]byte{"token": []byte("rotated-token")} + _, err = kube.Kube().CoreV1().Secrets(template.Namespace).Update(t.Context(), secret, metav1.UpdateOptions{}) + require.NoError(t, err) + require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) + store.mu.Lock() + require.Equal(t, revision, store.revision.Revision) + store.mu.Unlock() + grant.Spec.Destinations = append(grant.Spec.Destinations, kagentv1alpha3.EgressDestination{Origin: "https://registry.npmjs.org"}) + grant, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Update(t.Context(), grant, metav1.UpdateOptions{}) + require.NoError(t, err) + require.NoError(t, client.ApiV1alpha3().EgressPolicies(template.Namespace).Delete(t.Context(), grant.Name, metav1.DeleteOptions{})) + waitUnresolved(kagentv1alpha3.EgressPoliciesReasonNotFound, "tickets") + require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) + grant.ResourceVersion, grant.UID = "", "" + _, err = client.ApiV1alpha3().EgressPolicies(template.Namespace).Create(t.Context(), grant, metav1.CreateOptions{}) + require.NoError(t, err) + require.Eventually(t, ready, 10*time.Second, 10*time.Millisecond) + + store.mu.Lock() + require.Equal(t, revision, store.revision.Revision, "policy changes do not reprepare the runtime") + store.mu.Unlock() + // A dependency update alone must change the persisted revision. changedPool := pool.DeepCopy() changedPool.Spec.SandboxClass = atev1alpha1.SandboxClassMicroVM diff --git a/go/core/internal/controller/sandbox_reconciler.go b/go/core/internal/controller/sandbox_reconciler.go index 884ee3439d..83d5183caf 100644 --- a/go/core/internal/controller/sandbox_reconciler.go +++ b/go/core/internal/controller/sandbox_reconciler.go @@ -21,6 +21,7 @@ import ( kagentclient "github.com/kagent-dev/kagent/go/api/clientset/versioned/typed/api/v1alpha3" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" ) @@ -31,9 +32,11 @@ const ( ) type sandboxRevisionStore interface { + actorEgressStore UpsertSandboxTemplateDefinition(context.Context, database.SandboxTemplateDefinition) error + GetSandboxEgressGrant(context.Context, string, string) (egress.Access, error) RetireSandboxTemplateIdentities(context.Context, string, string) error - RecordSandboxRevision(context.Context, database.SandboxRevision, bool) error + RecordSandboxRevision(context.Context, database.SandboxRevision, database.RecordRevisionOptions) error } // SandboxReconciler owns the queued side effects of SandboxTemplate preparation. @@ -112,9 +115,13 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { } if state == nil || !state.Template.DeletionTimestamp.IsZero() { namespace, name, _ := strings.Cut(key, "/") - if err := s.store.RetireSandboxTemplateIdentities(ctx, namespace, name); err != nil { + err := s.store.RetireSandboxTemplateIdentities(ctx, namespace, name) + if err != nil { return fmt.Errorf("retire SandboxTemplate %s: %w", key, err) } + if err := applySourceEgress(ctx, s.store, s.actors, database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: namespace, Name: name}); err != nil { + return err + } if state != nil && slices.Contains(state.Template.Finalizers, sandboxPreparationFinalizer) { updated := state.Template.DeepCopy() updated.Finalizers = slices.DeleteFunc(updated.Finalizers, func(value string) bool { return value == sandboxPreparationFinalizer }) @@ -134,11 +141,23 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { } return nil } - if err := s.store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{ - Namespace: state.Template.Namespace, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), DesiredRevision: state.desiredRevision(), - }); err != nil { + grants := state.EgressGrant + if state.EgressConflict != "" { + previous, err := s.store.GetSandboxEgressGrant(ctx, state.Template.Namespace, string(state.Template.UID)) + if err != nil { + return s.observeError(*state, fmt.Errorf("read SandboxTemplate %s grants: %w", key, err), true) + } + grants = egress.IntersectAccess(previous, grants) + } + err := s.store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{ + Namespace: state.Template.Namespace, SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), DesiredRevision: state.desiredRevision(), EgressGrant: grants, + }) + if err != nil { return s.observeError(*state, fmt.Errorf("store SandboxTemplate %s definition: %w", key, err), true) } + if err := applySourceEgress(ctx, s.store, s.actors, database.RuntimeEgressSource{Kind: database.RuntimeEgressSandbox, Namespace: state.Template.Namespace, Name: state.Template.Name}); err != nil { + return s.observeError(*state, err, true) + } if state.DesiredActorTemplate == nil { s.collections.observations.DeleteObject(key) } @@ -175,9 +194,10 @@ func (s *SandboxReconciler) reconcile(ctx context.Context, key string) error { SandboxTemplateName: state.Template.Name, SandboxTemplateUID: string(state.Template.UID), SourceSnapshot: state.SourceSnapshot, } - if err := s.store.RecordSandboxRevision(ctx, revision, ready); err != nil { + if err := s.store.RecordSandboxRevision(ctx, revision, database.RecordRevisionOptions{Ready: ready}); err != nil { return s.observeError(*state, fmt.Errorf("store sandbox revision %s: %w", state.RevisionID, err), true) } + // Kubernetes readiness can only follow a revision that callers can select // from the store. Failed persistence must never publish a ready observation. s.collections.observations.ConditionalUpdateObject(sandboxRuntimeObservation{Key: key, RevisionID: state.RevisionID, Template: observed}) @@ -221,5 +241,6 @@ func sandboxStatusWithTransitionTimes(state sandboxReconciliation) kagentv1alpha } desired.ObservedGeneration = state.Template.Generation apimeta.SetStatusCondition(&desired.Conditions, condition) + apimeta.SetStatusCondition(&desired.Conditions, egressPoliciesCondition(kagentv1alpha3.SandboxTemplateConditionEgressPoliciesApplied, state.Template.Generation, state.EgressIssues, state.EgressConflict)) return desired } diff --git a/go/core/internal/controller/sandbox_reconciler_test.go b/go/core/internal/controller/sandbox_reconciler_test.go index eee798f795..bd22e0d564 100644 --- a/go/core/internal/controller/sandbox_reconciler_test.go +++ b/go/core/internal/controller/sandbox_reconciler_test.go @@ -25,11 +25,14 @@ import ( kagentfake "github.com/kagent-dev/kagent/go/api/clientset/versioned/fake" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/egress" "github.com/kagent-dev/kagent/go/core/internal/substrate" ) // These fakes are also used by the informer/queue integration test. type sandboxTestStore struct { + egressTargetStoreStub + upsertErr error mu sync.Mutex desired database.SandboxTemplateDefinition revision database.SandboxRevision @@ -38,21 +41,26 @@ type sandboxTestStore struct { retireAttempts int retireErr error recordErr error + recordCalls int } func (s *sandboxTestStore) UpsertSandboxTemplateDefinition(_ context.Context, value database.SandboxTemplateDefinition) error { s.mu.Lock() defer s.mu.Unlock() s.desired = value - return nil + return s.upsertErr } -func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision, ready bool) error { +func (s *sandboxTestStore) RecordSandboxRevision(_ context.Context, value database.SandboxRevision, options database.RecordRevisionOptions) error { s.mu.Lock() defer s.mu.Unlock() + s.recordCalls++ if s.recordErr != nil { return s.recordErr } - s.revision, s.ready = value, ready + s.revision = value + if options.Ready { + s.ready = true + } return nil } func (s *sandboxTestStore) RetireSandboxTemplateIdentities(context.Context, string, string) error { @@ -69,6 +77,7 @@ func (s *sandboxTestStore) RetireSandboxTemplateIdentities(context.Context, stri var _ sandboxRevisionStore = (*sandboxTestStore)(nil) type sandboxTestActors struct { + actorEgressStub mu sync.Mutex store *sandboxTestStore templates map[string]*ateapipb.ActorTemplate @@ -295,3 +304,12 @@ func TestSandboxPendingStatusRecoversWithoutGraphEvent(t *testing.T) { return err == nil && apimeta.IsStatusConditionTrue(current.Status.Conditions, "Ready") }, 5*time.Second, 10*time.Millisecond) } + +func (s *sandboxTestStore) GetSandboxEgressGrant(_ context.Context, namespace, uid string) (egress.Access, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.desired.Namespace == namespace && s.desired.SandboxTemplateUID == uid && !s.retired { + return s.desired.EgressGrant, nil + } + return egress.Access{}, nil +} diff --git a/go/core/internal/controller/status.go b/go/core/internal/controller/status.go index 47828ad737..4ed4883f13 100644 --- a/go/core/internal/controller/status.go +++ b/go/core/internal/controller/status.go @@ -1,10 +1,14 @@ package controller import ( + "fmt" + "strings" + "istio.io/istio/pkg/kube/krt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/translator" ) func newAgentStatuses(agents krt.Collection[*kagentv1alpha3.Agent], states krt.Collection[AgentReconciliation], opts krt.OptionsBuilder) krt.StatusCollection[*kagentv1alpha3.Agent, kagentv1alpha3.AgentStatus] { @@ -28,6 +32,7 @@ func statusForAgent(state AgentReconciliation, generation int64, latestSuccessfu status.WorkloadImage = new(state.Target.Revision.Image) } setAgentCondition(&status, generation, kagentv1alpha3.AgentConditionAccepted, metav1.ConditionTrue, "Accepted", "Agent explicitly selects its template and harness") + status.Conditions = append(status.Conditions, egressPoliciesCondition(kagentv1alpha3.AgentConditionEgressPoliciesApplied, generation, state.EgressIssues, state.EgressConflict)) failure := state.CompilationFailure if failure == nil { failure = state.PreparationFailure @@ -73,3 +78,38 @@ func setAgentCondition(status *kagentv1alpha3.AgentStatus, generation int64, con Type: conditionType, Status: conditionStatus, Reason: reason, Message: message, ObservedGeneration: generation, }) } + +func egressPoliciesCondition(conditionType string, generation int64, issues []translator.EgressGrantIssue, conflict string) metav1.Condition { + condition := metav1.Condition{ + Type: conditionType, ObservedGeneration: generation, + Status: metav1.ConditionTrue, Reason: kagentv1alpha3.EgressPoliciesReasonApplied, + Message: "All requested egress grants are admitted", + } + if len(issues) == 0 && conflict == "" { + return condition + } + condition.Status = metav1.ConditionFalse + var messages []string + for _, issue := range issues { + message := issue.Message + if issue.Policy != "" { + message = fmt.Sprintf("%s: %s", issue.Policy, message) + } + messages = append(messages, message) + } + if conflict != "" { + condition.Reason = kagentv1alpha3.EgressPoliciesReasonConflict + condition.Message = "new grants are held until the conflict is resolved; " + messages = append(messages, "first conflict: "+conflict) + } else { + condition.Reason = issues[0].Reason + condition.Message = "" + } + condition.Message += strings.Join(messages, "; ") + // Stay comfortably below the Kubernetes condition message limit. + const maxMessage = 4096 + if len(condition.Message) > maxMessage { + condition.Message = strings.ToValidUTF8(condition.Message[:maxMessage-3], "") + "..." + } + return condition +} diff --git a/go/core/internal/controller/status_test.go b/go/core/internal/controller/status_test.go index a7d447c734..dd79fdcb3a 100644 --- a/go/core/internal/controller/status_test.go +++ b/go/core/internal/controller/status_test.go @@ -1,9 +1,11 @@ package controller import ( + "strings" "testing" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/meta" kagentv1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" "github.com/kagent-dev/kagent/go/core/internal/translator" @@ -38,3 +40,18 @@ func TestStatusReportsDesiredWorkloadImage(t *testing.T) { require.Nil(t, status.WorkloadImage) require.Equal(t, "previous-revision", status.LatestSuccessfulRevision) } + +func TestEgressConditionSurvivesRuntimeFailureAndBoundsMessages(t *testing.T) { + issues := []translator.EgressGrantIssue{ + {Policy: "team/missing", Reason: kagentv1alpha3.EgressPoliciesReasonNotFound, Message: "missing"}, + } + state := AgentReconciliation{EgressIssues: issues, EgressConflict: strings.Repeat("conflict", 1000), + CompilationFailure: &ReconciliationFailure{Condition: kagentv1alpha3.AgentConditionResolvedRefs, Reason: "ReferenceResolutionFailed", Message: "Harness is missing"}} + status := statusForAgent(state, 1, "") + condition := meta.FindStatusCondition(status.Conditions, kagentv1alpha3.AgentConditionEgressPoliciesApplied) + require.Equal(t, kagentv1alpha3.EgressPoliciesReasonConflict, condition.Reason) + require.Contains(t, condition.Message, "new grants are held until the conflict is resolved") + require.Contains(t, condition.Message, "team/missing") + require.LessOrEqual(t, len(condition.Message), 4096) + require.False(t, meta.IsStatusConditionTrue(status.Conditions, kagentv1alpha3.AgentConditionReady)) +} diff --git a/go/core/internal/database/client_session_test.go b/go/core/internal/database/client_session_test.go index 99a49bde77..b06452fc49 100644 --- a/go/core/internal/database/client_session_test.go +++ b/go/core/internal/database/client_session_test.go @@ -377,7 +377,7 @@ func TestForkSessionCopiesBoundedHistory(t *testing.T) { if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision, true); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true}); err != nil { t.Fatal(err) } @@ -536,7 +536,7 @@ func TestSessionCreateAndTransitions(t *testing.T) { if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision, true); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true}); err != nil { t.Fatal(err) } @@ -638,7 +638,7 @@ func sessionFixture(t *testing.T, client *Client, ctx context.Context, namespace if err := client.UpsertAgentDefinition(ctx, pair); err != nil { t.Fatal(err) } - if err := client.RecordRuntimeRevision(ctx, revision, true); err != nil { + if err := client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true}); err != nil { t.Fatal(err) } } @@ -952,7 +952,7 @@ func TestDeletedSessionPreservesRequestIdentityAndHidesAccess(t *testing.T) { _, _, err = client.CreateSession(ctx, request, "stable-request") require.ErrorIs(t, err, ErrIdempotencyConflict) // Tombstones release the revision FK and no longer block runtime cleanup. - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) require.Len(t, revisions, 1) diff --git a/go/core/internal/database/models.go b/go/core/internal/database/models.go index 3f66aea9e6..4702c64372 100644 --- a/go/core/internal/database/models.go +++ b/go/core/internal/database/models.go @@ -51,25 +51,27 @@ type AgentMemorySearchResult struct { // AgentDefinition tracks the desired and last successful runtime for an Agent UID. type AgentDefinition struct { - Namespace string - AgentName string - AgentUID string + Namespace string + AgentName string + AgentUID string + EgressGrant egress.Access // DesiredRevision is a compiled runtime digest, or empty while inputs are unresolved. DesiredRevision string } type RuntimeRevision struct { - Revision string - Namespace string - AgentName string - AgentUID string - SourceSnapshot json.RawMessage - AgentCard *a2apb.AgentCard - Credentials []egress.Credential - EgressDestinations []string - ActorTemplateAtespace string - ActorTemplateName string - ActorTemplateUID string + Revision string + Namespace string + AgentName string + AgentUID string + SourceSnapshot json.RawMessage + AgentCard *a2apb.AgentCard + Credentials []egress.Credential + EgressDestinations []string + TokenPassthroughOrigins []string + ActorTemplateAtespace string + ActorTemplateName string + ActorTemplateUID string } // SessionQuery narrows a page of sessions to an optional Agent. diff --git a/go/core/internal/database/protobuf_test.go b/go/core/internal/database/protobuf_test.go index 6e962a5771..ab3104ab59 100644 --- a/go/core/internal/database/protobuf_test.go +++ b/go/core/internal/database/protobuf_test.go @@ -286,11 +286,11 @@ func TestShareAndAgentCardProtobufPersistence(t *testing.T) { require.NoError(t, err) addUnknown(card) revision := RuntimeRevision{Revision: "revision", Namespace: "team-a", AgentName: "assistant", AgentUID: "template", SourceSnapshot: []byte(`{}`), AgentCard: card, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "template"} - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, false)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{})) // Reconciliation can update runtime identity, but the pinned card is immutable. revision.AgentCard = &a2apb.AgentCard{Name: "replacement"} revision.ActorTemplateUID = "new-uid" - require.NoError(t, client.RecordRuntimeRevision(ctx, revision, false)) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{})) stored, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) require.True(t, proto.Equal(card, stored.AgentCard)) diff --git a/go/core/internal/database/runtime_creation_test.go b/go/core/internal/database/runtime_creation_test.go new file mode 100644 index 0000000000..e3b83005df --- /dev/null +++ b/go/core/internal/database/runtime_creation_test.go @@ -0,0 +1,83 @@ +package database + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func TestRuntimeCreationUsesLatestSuccessfulRevision(t *testing.T) { + for _, kind := range []string{"session", "sandbox"} { + t.Run(kind, func(t *testing.T) { + pool := setupTestDB(t) + client := NewClient(pool, "public") + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + var create func(*Client, string) (string, bool, error) + var desire func(*Client, string) error + var prepare func(string) + var unresolved string + if kind == "session" { + prepare = func(revision string) { + sessionFixture(t, client, ctx, "team-a", revision, "assistant", "kagent") + } + create = func(c *Client, requestID string) (string, bool, error) { + session, created, err := c.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), requestID) + return session.GetPreparedRevision(), created, err + } + desire = func(c *Client, revision string) error { + err := c.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", DesiredRevision: revision}) + return err + } + } else { + input, options := sandboxFixture(t, client, "revision-1") + prepare = func(revision string) { + _, _ = sandboxFixture(t, client, revision) + } + create = func(c *Client, requestID string) (string, bool, error) { + request := proto.CloneOf(input) + request.Id = uuid.NewString() + instance, created, err := c.CreateSandbox(ctx, request, requestID, options) + return instance.GetPreparedRevision(), created, err + } + desire = func(c *Client, revision string) error { + err := c.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) + return err + } + unresolved = "pending:uid:2" + } + prepare("revision-1") + revision, created, err := create(client, "original") + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "revision-1", revision) + + // Unresolved and pending edits keep the last successful runtime available. + // Retries retain their original revision even after a newer one is ready. + for _, desired := range []string{unresolved, "revision-2"} { + require.NoError(t, desire(client, desired)) + revision, created, err = create(client, "fresh-"+desired) + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "revision-1", revision) + revision, created, err = create(client, "original") + require.NoError(t, err) + require.False(t, created) + require.Equal(t, "revision-1", revision) + } + prepare("revision-2") + revision, created, err = create(client, "fresh") + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "revision-2", revision) + revision, created, err = create(client, "original") + require.NoError(t, err) + require.False(t, created) + require.Equal(t, "revision-1", revision) + }) + } +} diff --git a/go/core/internal/database/runtime_egress.go b/go/core/internal/database/runtime_egress.go new file mode 100644 index 0000000000..91eaf0e12b --- /dev/null +++ b/go/core/internal/database/runtime_egress.go @@ -0,0 +1,156 @@ +package database + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +// canonicalStoredGrants validates and normalizes source grants for NOT NULL storage. +func canonicalStoredGrants(grants egress.Access) (egress.Access, error) { + canonical, err := egress.CanonicalAccess(grants.Destinations, grants.Credentials, nil) + if err != nil { + return egress.Access{}, err + } + if canonical.Destinations == nil { + canonical.Destinations = []string{} + } + if canonical.Credentials == nil { + canonical.Credentials = []egress.Credential{} + } + return canonical, nil +} + +// GetRuntimeEgressPolicy merges pinned inputs with the active source UID's grants +// in one database snapshot. Missing/deleting revisions and malformed inputs fail +// closed. Retired or replaced identities contribute no grants. Older revisions +// retain their pinned base and only the grants compatible with it. +func (c *Client) GetRuntimeEgressPolicy(ctx context.Context, revision string) (egress.Access, error) { + type policyInputs struct { + EgressDestinations []string + Credentials []egress.Credential + TokenPassthroughOrigins []string + GrantDestinations []string + GrantCredentials []egress.Credential + } + row, err := queryOne(ctx, c.db, ` + SELECT r.egress_destinations, r.credentials, r.token_passthrough_origins, + COALESCE(ad.grant_destinations, sd.grant_destinations, '{}') AS grant_destinations, + COALESCE(ad.grant_credentials, sd.grant_credentials, '[]') AS grant_credentials + FROM runtime_revision r + LEFT JOIN agent_revision a ON r.kind = 'agent' AND a.revision = r.revision + LEFT JOIN sandbox_revision s ON r.kind = 'sandbox' AND s.revision = r.revision + LEFT JOIN agent_definition ad ON ad.namespace = r.namespace AND ad.agent_uid = a.agent_uid AND ad.retired_at IS NULL + LEFT JOIN sandbox_template_definition sd ON sd.namespace = r.namespace AND sd.sandbox_template_uid = s.sandbox_template_uid AND sd.retired_at IS NULL + WHERE r.revision = $1 AND r.deleted_at IS NULL + AND (a.revision IS NOT NULL OR s.revision IS NOT NULL) + `, pgx.RowToStructByName[policyInputs], revision) + if err != nil { + return egress.Access{}, fmt.Errorf("get runtime egress policy %s: %w", revision, notFoundOr(err)) + } + for i, raw := range row.TokenPassthroughOrigins { + origin, err := egress.ParseOrigin(raw) + if err != nil { + return egress.Access{}, fmt.Errorf("decode runtime token origins %s: %w", revision, err) + } + row.TokenPassthroughOrigins[i] = origin.String() + } + base, err := egress.CanonicalAccess(row.EgressDestinations, row.Credentials, row.TokenPassthroughOrigins) + if err != nil { + return egress.Access{}, fmt.Errorf("decode runtime egress base %s: %w", revision, err) + } + grants, err := egress.CanonicalAccess(row.GrantDestinations, row.GrantCredentials, nil) + if err != nil { + return egress.Access{}, fmt.Errorf("decode runtime egress grants %s: %w", revision, err) + } + return egress.EffectiveAccess(base, grants, row.TokenPassthroughOrigins) +} + +var _ egress.PolicyStore = (*Client)(nil) + +// RuntimeEgressSourceKind identifies a Kubernetes owner of runtime instances. +type RuntimeEgressSourceKind string + +const ( + RuntimeEgressAgent RuntimeEgressSourceKind = runtimeKindAgent + RuntimeEgressSandbox RuntimeEgressSourceKind = runtimeKindSandbox +) + +// RuntimeEgressSource includes former UIDs whose grants must be revoked on recreation. +type RuntimeEgressSource struct { + Kind RuntimeEgressSourceKind + Namespace string + Name string +} + +// RuntimeEgressTarget identifies allocated instances, including suspended actors. +type RuntimeEgressTarget struct { + ID string + Revision string + Namespace string + Atespace string + TemplateName string + ActorUID string +} + +// ListRuntimeEgressSourceTargets pages allocated instances across a source's UIDs +// in ID order, excluding deletions and unissued creations. Invalid sources return +// an error; a missing source yields an empty page. +func (c *Client) ListRuntimeEgressSourceTargets(ctx context.Context, source RuntimeEgressSource, afterID string, limit int) ([]RuntimeEgressTarget, error) { + if source.Namespace == "" || source.Name == "" { + return nil, fmt.Errorf("runtime egress source requires namespace and name") + } + switch source.Kind { + case RuntimeEgressAgent: + return queryMany(ctx, c.db, ` + SELECT i.id::text AS id, i.prepared_revision AS revision, r.namespace, + r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, + COALESCE(s.actor_uid, '') AS actor_uid + FROM runtime_instance i + JOIN runtime_revision r ON r.revision = i.prepared_revision + JOIN agent_revision a ON a.revision = r.revision + JOIN session s ON s.id = i.id + WHERE i.kind = 'agent' AND r.kind = 'agent' AND r.namespace = $1 AND a.agent_name = $2 + AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) + AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' + AND i.id > COALESCE(NULLIF($3, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY i.id LIMIT $4 + `, pgx.RowToStructByName[RuntimeEgressTarget], source.Namespace, source.Name, afterID, limit) + case RuntimeEgressSandbox: + return queryMany(ctx, c.db, ` + SELECT i.id::text AS id, i.prepared_revision AS revision, r.namespace, + r.actor_template_atespace AS atespace, r.actor_template_name AS template_name, + '' AS actor_uid + FROM runtime_instance i + JOIN runtime_revision r ON r.revision = i.prepared_revision + JOIN sandbox_revision s ON s.revision = r.revision + WHERE i.kind = 'sandbox' AND r.kind = 'sandbox' AND r.namespace = $1 AND s.sandbox_template_name = $2 + AND (i.operation <> 'RUNTIME_OPERATION_CREATE' OR i.executor_id IS NOT NULL) + AND i.state <> 'RUNTIME_STATE_DELETED' AND i.operation <> 'RUNTIME_OPERATION_DELETE' + AND i.id > COALESCE(NULLIF($3, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY i.id LIMIT $4 + `, pgx.RowToStructByName[RuntimeEgressTarget], source.Namespace, source.Name, afterID, limit) + default: + return nil, fmt.Errorf("unknown runtime egress source kind %q", source.Kind) + } +} + +// decodeEgressGrant treats absent definitions as empty grants and rejects invalid +// durable rules. Callers scope the read to an active Kubernetes source identity. +func decodeEgressGrant(grants egress.Access, err error) (egress.Access, error) { + if errors.Is(err, pgx.ErrNoRows) { + return egress.Access{}, nil + } + if err != nil { + return egress.Access{}, err + } + canonical, err := egress.CanonicalAccess(grants.Destinations, grants.Credentials, nil) + if err != nil { + return egress.Access{}, fmt.Errorf("decode source egress grants: %w", err) + } + return canonical, nil +} diff --git a/go/core/internal/database/runtime_egress_test.go b/go/core/internal/database/runtime_egress_test.go new file mode 100644 index 0000000000..c53d23093f --- /dev/null +++ b/go/core/internal/database/runtime_egress_test.go @@ -0,0 +1,314 @@ +package database + +import ( + "context" + "fmt" + "slices" + "testing" + + a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +func TestRuntimeEgressRetainsSourceAndListsSuspendedActors(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + revision := RuntimeRevision{Revision: "agent", Namespace: "team-a", AgentName: "scratch", AgentUID: "old-uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "runtime", TokenPassthroughOrigins: []string{"https://model.example.com:443"}, EgressDestinations: []string{"https://model.example.com:443"}} + upsertErr := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "scratch", AgentUID: "old-uid", DesiredRevision: "agent"}) + require.NoError(t, upsertErr) + require.NoError(t, client.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) + instance, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "scratch", "kagent", ""), "request") + require.NoError(t, err) + pendingTargets, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "scratch"}, "", 10) + require.NoError(t, err) + require.Empty(t, pendingTargets, "unissued creations cannot have actors") + op, err := client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) + require.NoError(t, err) + executor := uuid.New() + claimed, err := client.ClaimSessionOperation(ctx, instance.Id, op.ID, executor) + require.NoError(t, err) + require.True(t, claimed) + pendingTargets, err = client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "scratch"}, "", 10) + require.NoError(t, err) + require.Len(t, pendingTargets, 1, "a claimed creation may already have allocated an actor") + require.Empty(t, pendingTargets[0].ActorUID, "creation has not yet persisted the actor UID") + _, err = client.FinishSessionOperation(ctx, instance.Id, op.ID, executor, "actor.example.com", "actor-uid", "") + require.NoError(t, err) + op, err = client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND) + require.NoError(t, err) + executor = uuid.New() + claimed, err = client.ClaimSessionOperation(ctx, instance.Id, op.ID, executor) + require.NoError(t, err) + require.True(t, claimed) + _, err = client.FinishSessionOperation(ctx, instance.Id, op.ID, executor, "", "", "") + require.NoError(t, err) + // A recreated Agent must not transfer authorization to an old pinned session. + upsertErr = client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team-a", AgentName: "scratch", AgentUID: "new-uid", DesiredRevision: "pending"}) + require.NoError(t, upsertErr) + sandbox, options := sandboxFixture(t, client, "sandbox") + created, _, err := client.CreateSandbox(ctx, sandbox, "sandbox", options) + require.NoError(t, err) + sandboxOp, err := client.BeginSandboxOperation(ctx, created.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) + require.NoError(t, err) + claimed, err = client.ClaimSandboxOperation(ctx, created.Id, sandboxOp.ID, uuid.New()) + require.NoError(t, err) + require.True(t, claimed) + // Both kinds deliberately share a name; source filtering must preserve kind. + for _, test := range []struct { + source RuntimeEgressSource + want string + }{ + {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "scratch"}, instance.Id}, + {RuntimeEgressSource{Kind: RuntimeEgressSandbox, Namespace: "team-a", Name: "scratch"}, created.Id}, + {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "other", Name: "scratch"}, ""}, + {RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "unrelated"}, ""}, + } { + targets, err := client.ListRuntimeEgressSourceTargets(ctx, test.source, "", 10) + require.NoError(t, err) + if test.want == "" { + require.Empty(t, targets) + } else { + require.Len(t, targets, 1) + require.Equal(t, test.want, targets[0].ID) + if test.source.Kind == RuntimeEgressAgent { + require.Equal(t, "actor-uid", targets[0].ActorUID) + } else { + require.Empty(t, targets[0].ActorUID) + } + } + } + for _, kind := range []RuntimeEgressSourceKind{"", "typo"} { + _, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: kind, Namespace: "team-a", Name: "scratch"}, "", 10) + require.Error(t, err) + } + // Page multiple allocated instances of one source without visiting other kinds. + sandbox.Id = uuid.NewString() + secondSandbox, _, err := client.CreateSandbox(ctx, sandbox, "second-sandbox", options) + require.NoError(t, err) + sandboxOp, err = client.BeginSandboxOperation(ctx, secondSandbox.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE) + require.NoError(t, err) + claimed, err = client.ClaimSandboxOperation(ctx, secondSandbox.Id, sandboxOp.ID, uuid.New()) + require.NoError(t, err) + require.True(t, claimed) + source := RuntimeEgressSource{Kind: RuntimeEgressSandbox, Namespace: "team-a", Name: "scratch"} + first, err := client.ListRuntimeEgressSourceTargets(ctx, source, "", 1) + require.NoError(t, err) + require.Len(t, first, 1) + second, err := client.ListRuntimeEgressSourceTargets(ctx, source, first[0].ID, 1) + require.NoError(t, err) + require.Len(t, second, 1) + require.ElementsMatch(t, []string{created.Id, secondSandbox.Id}, []string{first[0].ID, second[0].ID}) + last, err := client.ListRuntimeEgressSourceTargets(ctx, source, second[0].ID, 1) + require.NoError(t, err) + require.Empty(t, last) + _, err = client.BeginSessionOperation(ctx, instance.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE) + require.NoError(t, err) + remaining, err := client.ListRuntimeEgressSourceTargets(ctx, RuntimeEgressSource{Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "scratch"}, "", 10) + require.NoError(t, err) + require.Empty(t, remaining) +} + +func TestRuntimeEgressTokenOriginsAreNotGrants(t *testing.T) { + many := make([]string, 257) + for i := range many { + many[i] = fmt.Sprintf("https://model-%d.example.com", i) + } + for _, test := range []struct { + name string + origins []string + }{ + {"more origins than policy rules", many}, + {"different protocols on the same port", []string{"http://model.example.com:443", "https://model.example.com:443"}}, + } { + t.Run(test.name, func(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{ + Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", + SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", + TokenPassthroughOrigins: test.origins, + }, RecordRevisionOptions{})) + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + require.Empty(t, policy.Destinations, "token origins constrain credentials without granting network access") + }) + } +} + +func TestRuntimeEgressCanonicalTokenOriginWithholdsCredentials(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", EgressGrant: egress.Access{ + Destinations: []string{"https://model.example.com"}, + Credentials: []egress.Credential{{Origin: "https://model.example.com", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"}}, + }}) + require.NoError(t, err) + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{ + Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", + SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", + TokenPassthroughOrigins: []string{"https://MODEL.example.com."}, + }, RecordRevisionOptions{})) + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + require.Equal(t, []string{"https://model.example.com:443"}, policy.Destinations) + require.Empty(t, policy.Credentials) +} + +func TestSourceGrantsDetermineEffectivePolicy(t *testing.T) { + for _, kind := range []RuntimeEgressSourceKind{RuntimeEgressAgent, RuntimeEgressSandbox} { + t.Run(string(kind), func(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + upsert := func(uid string, grants egress.Access) error { + if kind == RuntimeEgressAgent { + return client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: uid, EgressGrant: grants}) + } + return client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team", SandboxTemplateName: "source", SandboxTemplateUID: uid, EgressGrant: grants}) + } + base := egress.Access{} + if kind == RuntimeEgressAgent { + base = egress.Access{Destinations: []string{"https://model.example.com:443"}, Credentials: []egress.Credential{{Origin: "https://model.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"}}} + } + initial := egress.Access{Destinations: []string{"https://tickets.example.com"}, Credentials: []egress.Credential{{Origin: "https://tickets.example.com", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/tickets/token"}}} + err := upsert("uid", egress.Access{}) + require.NoError(t, err) + err = upsert("uid", initial) + require.NoError(t, err) + canonical, err := egress.CanonicalAccess(initial.Destinations, initial.Credentials, nil) + require.NoError(t, err) + err = upsert("uid", canonical) + require.NoError(t, err) + if kind == RuntimeEgressAgent { + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, EgressDestinations: base.Destinations, Credentials: base.Credentials, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, RecordRevisionOptions{})) + } else { + require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, SandboxTemplateName: "source", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{})) + } + assertPolicy := func(grants egress.Access) { + t.Helper() + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + expected, err := egress.MergeAccess(base, grants, nil) + require.NoError(t, err) + require.True(t, slices.Equal(expected.Destinations, policy.Destinations)) + require.True(t, slices.Equal(expected.Credentials, policy.Credentials)) + } + assertPolicy(canonical) + err = upsert("uid", egress.Access{Destinations: []string{"invalid"}}) + require.Error(t, err) + assertPolicy(canonical) + if kind == RuntimeEgressAgent { + conflict := egress.Access{Destinations: append(slices.Clone(base.Destinations), canonical.Destinations...), Credentials: append(slices.Clone(canonical.Credentials), egress.Credential{Origin: base.Destinations[0], Header: "authorization", URI: "ate-secret://k8s.io/default/team/other/token"})} + err = upsert("uid", conflict) + require.NoError(t, err) + policy, err := client.GetRuntimeEgressPolicy(ctx, "revision") + require.NoError(t, err) + expected, err := egress.MergeAccess(base, canonical, nil) + require.NoError(t, err) + require.Equal(t, expected, policy, "older revisions retain nonconflicting grants") + } + if kind == RuntimeEgressAgent { + require.NoError(t, client.RetireAgentIdentities(ctx, "team", "source")) + } else { + require.NoError(t, client.RetireSandboxTemplateIdentities(ctx, "team", "source")) + } + assertPolicy(egress.Access{}) + err = upsert("uid", canonical) + require.NoError(t, err) + assertPolicy(canonical) + err = upsert("replacement", canonical) + require.NoError(t, err) + assertPolicy(egress.Access{}) + err = upsert("replacement", canonical) + require.NoError(t, err) + _, err = client.GetRuntimeEgressPolicy(ctx, "missing") + require.ErrorIs(t, err, ErrNotFound) + }) + } +} + +func TestSourceGrantsRollbackWithInvalidRevision(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + grants := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} + definition := AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", EgressGrant: grants} + err := client.UpsertAgentDefinition(ctx, definition) + require.NoError(t, err) + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "agent", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "agent"}, RecordRevisionOptions{})) + require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "sandbox", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "sandbox"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "sandbox", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{})) + for _, uid := range []string{"uid", "replacement"} { + definition.AgentUID, definition.DesiredRevision, definition.EgressGrant = uid, "sandbox", egress.Access{} + err = client.UpsertAgentDefinition(ctx, definition) + require.ErrorIs(t, err, ErrConflict) + policy, err := client.GetRuntimeEgressPolicy(ctx, "agent") + require.NoError(t, err) + require.Equal(t, grants.Destinations, policy.Destinations, "failed upsert must roll back grants and retirement") + require.Empty(t, policy.Credentials) + } +} + +func TestRuntimeEgressMalformedInputsFailClosed(t *testing.T) { + for _, test := range []struct{ name, statement string }{ + {"grant credentials", `UPDATE agent_definition SET grant_credentials = '[{"hostname":"legacy.example.com"}]'`}, + {"grant origin", `UPDATE agent_definition SET grant_destinations = ARRAY['invalid']`}, + {"pinned credentials", `UPDATE runtime_revision SET credentials = '[{"hostname":"legacy.example.com"}]'`}, + {"token origin", `UPDATE runtime_revision SET token_passthrough_origins = ARRAY['invalid']`}, + {"missing typed revision", `DELETE FROM agent_revision`}, + } { + t.Run(test.name, func(t *testing.T) { + pool := setupTestDB(t) + client := NewClient(pool, "public") + ctx := t.Context() + err := client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid"}) + require.NoError(t, err) + require.NoError(t, client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", SourceSnapshot: []byte(`{}`), AgentCard: &a2apb.AgentCard{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime"}, RecordRevisionOptions{})) + _, err = pool.Exec(ctx, test.statement) + require.NoError(t, err) + _, err = client.GetRuntimeEgressPolicy(ctx, "revision") + require.Error(t, err) + }) + } +} + +func TestReadAdmittedGrantsScopesIdentity(t *testing.T) { + for _, kind := range []RuntimeEgressSourceKind{RuntimeEgressAgent, RuntimeEgressSandbox} { + t.Run(string(kind), func(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + var read func(context.Context, string, string) (egress.Access, error) + var retire func(context.Context, string, string) error + upsert := func(uid string, grants egress.Access) error { + if kind == RuntimeEgressAgent { + return client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: uid, EgressGrant: grants}) + } + return client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team", SandboxTemplateName: "source", SandboxTemplateUID: uid, EgressGrant: grants}) + } + if kind == RuntimeEgressAgent { + read, retire = client.GetAgentEgressGrant, client.RetireAgentIdentities + } else { + read, retire = client.GetSandboxEgressGrant, client.RetireSandboxTemplateIdentities + } + grants := egress.Access{Destinations: []string{"https://tickets.example.com:443"}} + require.NoError(t, upsert("uid", grants)) + got, err := read(ctx, "team", "uid") + require.NoError(t, err) + require.Equal(t, grants.Destinations, got.Destinations) + for _, scope := range [][2]string{{"team", "new"}, {"other", "uid"}} { + got, err = read(ctx, scope[0], scope[1]) + require.NoError(t, err) + require.Empty(t, got.Destinations) + } + require.NoError(t, retire(ctx, "team", "source")) + got, err = read(ctx, "team", "uid") + require.NoError(t, err) + require.Empty(t, got.Destinations, "retired grants must not be resurrected") + require.NoError(t, upsert("replacement", grants)) + got, err = read(ctx, "team", "uid") + require.NoError(t, err) + require.Empty(t, got.Destinations, "grants belong to the exact source UID") + }) + } +} diff --git a/go/core/internal/database/runtime_lifecycle_test.go b/go/core/internal/database/runtime_lifecycle_test.go index d6dcddc980..7c06ae747b 100644 --- a/go/core/internal/database/runtime_lifecycle_test.go +++ b/go/core/internal/database/runtime_lifecycle_test.go @@ -69,7 +69,9 @@ func TestAgentWritesRejectSandboxRevisions(t *testing.T) { revision, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) revision.Revision, revision.ActorTemplateUID = "sandbox-revision", "replacement-uid" - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision, true), ErrConflict) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{}), ErrConflict) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{Ready: true}), ErrConflict) + require.ErrorIs(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "scratch"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "scratch-uid", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{Ready: true}), ErrConflict) var uid string require.NoError(t, pool.QueryRow(ctx, "SELECT actor_template_uid FROM runtime_revision WHERE revision = 'sandbox-revision'").Scan(&uid)) require.Equal(t, "original-uid", uid) diff --git a/go/core/internal/database/runtime_revisions.go b/go/core/internal/database/runtime_revisions.go index be54f34260..a577d59af0 100644 --- a/go/core/internal/database/runtime_revisions.go +++ b/go/core/internal/database/runtime_revisions.go @@ -15,29 +15,30 @@ import ( ) // UpsertAgentDefinition records the desired runtime revision for a -// Agent identity. Updating an existing definition revives it if retired and +// Kubernetes Agent identity. Updating an existing definition revives it if retired and // preserves its latest successful revision. It atomically retires older identities -// at the same names and rejects revisions whose deletion has started. +// at the same names and rejects revisions whose deletion has started. Grants +// are canonicalized before writing; errors may include an uncertain commit outcome. func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefinition) error { + grants, err := canonicalStoredGrants(definition.EgressGrant) + if err != nil { + return fmt.Errorf("validate source grants: %w", err) + } return c.withTx(ctx, func(tx pgx.Tx) error { - if err := retireAgentIdentities(ctx, tx, definition.Namespace, definition.AgentName, &definition); err != nil { - return fmt.Errorf("retire replaced Agent: %w", err) + if err := retireAgentIdentities(ctx, tx, definition.Namespace, definition.AgentName, definition.AgentUID); err != nil { + return err } if err := execSQL(ctx, tx, ` - INSERT INTO agent_definition ( - namespace, agent_name, agent_uid, desired_revision, retired_at - ) VALUES ($1, $2, $3, $4, NULL) + INSERT INTO agent_definition (namespace, agent_name, agent_uid, desired_revision, grant_destinations, grant_credentials) + VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (namespace, agent_uid) DO UPDATE SET - agent_name = EXCLUDED.agent_name, - desired_revision = EXCLUDED.desired_revision, - retired_at = NULL, - updated_at = NOW() - `, - definition.Namespace, definition.AgentName, definition.AgentUID, - definition.DesiredRevision, - ); err != nil { + agent_name = EXCLUDED.agent_name, desired_revision = EXCLUDED.desired_revision, + grant_destinations = EXCLUDED.grant_destinations, grant_credentials = EXCLUDED.grant_credentials, + retired_at = NULL, updated_at = NOW() + `, definition.Namespace, definition.AgentName, definition.AgentUID, definition.DesiredRevision, grants.Destinations, grants.Credentials); err != nil { return err } + // Lock Agents before revisions. Validate the resulting definition, including // its retained last-good pointer when reactivating a retired identity. type revisionStatus struct { @@ -67,11 +68,45 @@ func (c *Client) UpsertAgentDefinition(ctx context.Context, definition AgentDefi }) } -// RecordRuntimeRevision stores a prepared revision and, when ready, atomically -// promotes it for an active definition that still desires it. Stale reports leave the -// definition unchanged. Existing revisions retain immutable inputs; only their actor -// UID and update time are refreshed. Deleting revisions return ErrObjectDeleting. -func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevision, ready bool) error { +// GetAgentEgressGrant returns the active identity's last admitted grants. +// Missing or retired identities return empty access; malformed stored grants fail. +func (c *Client) GetAgentEgressGrant(ctx context.Context, namespace, uid string) (egress.Access, error) { + grants, err := queryOne(ctx, c.db, ` + SELECT grant_destinations AS destinations, grant_credentials AS credentials + FROM agent_definition WHERE namespace = $1 AND agent_uid = $2 AND retired_at IS NULL + `, pgx.RowToStructByName[egress.Access], namespace, uid) + return decodeEgressGrant(grants, err) +} + +// ListAgentEgressSources returns names for active Agent definitions and retired +// identities that still have non-deleted instances. Startup recovery includes +// retired identities because a crash may interrupt revocation after retirement. +func (c *Client) ListAgentEgressSources(ctx context.Context) ([]RuntimeEgressSource, error) { + return queryMany(ctx, c.db, ` + SELECT DISTINCT 'agent' AS kind, d.namespace, d.agent_name AS name + FROM agent_definition d + WHERE d.retired_at IS NULL OR EXISTS ( + SELECT 1 FROM agent_revision a + JOIN runtime_revision r ON r.revision = a.revision + JOIN runtime_instance i ON i.prepared_revision = r.revision + WHERE r.namespace = d.namespace AND a.agent_uid = d.agent_uid + AND i.state <> 'RUNTIME_STATE_DELETED' + ) + ORDER BY namespace, name + `, pgx.RowToStructByName[RuntimeEgressSource]) +} + +// RecordRevisionOptions controls whether a prepared revision becomes available. +type RecordRevisionOptions struct { + // Ready promotes the revision only while its source is active and still desires it. + Ready bool +} + +// RecordRuntimeRevision atomically stores immutable inputs and, when Ready, +// promotes the revision for its active source if still desired. Existing inputs +// are preserved; actor UID and update time refresh. Stale reports cannot replace +// the last successful revision. Deleting revisions return ErrObjectDeleting. +func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevision, options RecordRevisionOptions) error { if revision.AgentCard == nil { return fmt.Errorf("runtime revision %s has no Agent Card", revision.Revision) } @@ -80,24 +115,19 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi return fmt.Errorf("encode runtime revision Agent Card: %w", err) } return c.withTx(ctx, func(tx pgx.Tx) error { - if ready { - // Match GC finalization's lock order: definition before revision. Use the - // stored identity when present, since revision inputs are immutable. - _, err := queryOne(ctx, tx, ` + if options.Ready { + // Match definition preparation and GC: source before revision. + if _, err := queryMany(ctx, tx, ` SELECT 1 FROM agent_definition p - LEFT JOIN agent_runtime_revision r ON r.revision = $1 - WHERE p.namespace = COALESCE(r.namespace, $2) - AND p.agent_uid = COALESCE(r.agent_uid, $3) - FOR UPDATE OF p - `, pgx.RowTo[int], revision.Revision, revision.Namespace, revision.AgentUID) - if err != nil && !errors.Is(err, pgx.ErrNoRows) { + WHERE p.namespace = $1 AND p.agent_uid = $2 FOR UPDATE OF p + `, pgx.RowTo[int], revision.Namespace, revision.AgentUID); err != nil { return err } } if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{ RuntimeArtifact: RuntimeArtifact{Revision: revision.Revision, Kind: runtimeKindAgent, Namespace: revision.Namespace, ActorTemplateAtespace: revision.ActorTemplateAtespace, ActorTemplateName: revision.ActorTemplateName, ActorTemplateUID: revision.ActorTemplateUID}, - SourceSnapshot: revision.SourceSnapshot, EgressDestinations: revision.EgressDestinations, Credentials: revision.Credentials, + SourceSnapshot: revision.SourceSnapshot, EgressDestinations: revision.EgressDestinations, Credentials: revision.Credentials, TokenPassthroughOrigins: revision.TokenPassthroughOrigins, }); err != nil { return err } @@ -107,19 +137,20 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi `, revision.Revision, revision.AgentName, revision.AgentUID, card); err != nil { return err } - if !ready { + if !options.Ready { return nil } return execSQL(ctx, tx, ` UPDATE agent_definition p SET latest_successful_revision = r.revision, updated_at = NOW() FROM agent_runtime_revision r - WHERE r.revision = $1 + WHERE r.revision = $1 AND p.namespace = $2 AND p.agent_uid = $3 AND p.namespace = r.namespace AND p.agent_uid = r.agent_uid AND p.desired_revision = r.revision AND p.retired_at IS NULL - `, revision.Revision) + AND p.latest_successful_revision IS DISTINCT FROM r.revision + `, revision.Revision, revision.Namespace, revision.AgentUID) }) } @@ -128,7 +159,7 @@ func (c *Client) RecordRuntimeRevision(ctx context.Context, revision RuntimeRevi func (c *Client) GetRuntimeRevision(ctx context.Context, revision string) (*RuntimeRevision, error) { row, err := queryOne(ctx, c.db, ` SELECT revision, namespace, agent_name, agent_uid, - source_snapshot, egress_destinations, credentials, actor_template_atespace, actor_template_name, actor_template_uid, + source_snapshot, egress_destinations, token_passthrough_origins, credentials, actor_template_atespace, actor_template_name, actor_template_uid, agent_card, deleted_at FROM agent_runtime_revision WHERE revision = $1 `, pgx.RowToStructByName[runtimeRevisionRow], revision) if err != nil { @@ -152,33 +183,31 @@ func toRuntimeRevision(row runtimeRevisionRow) (*RuntimeRevision, error) { Revision: row.Revision, Namespace: row.Namespace, AgentName: row.AgentName, AgentUID: row.AgentUID, SourceSnapshot: row.SourceSnapshot, AgentCard: card, - EgressDestinations: row.EgressDestinations, - Credentials: credentials, - ActorTemplateAtespace: row.ActorTemplateAtespace, ActorTemplateName: row.ActorTemplateName, + EgressDestinations: row.EgressDestinations, + TokenPassthroughOrigins: row.TokenPassthroughOrigins, + Credentials: credentials, + ActorTemplateAtespace: row.ActorTemplateAtespace, ActorTemplateName: row.ActorTemplateName, ActorTemplateUID: row.ActorTemplateUID, }, nil } -// RetireAgentIdentities retires identities at the given namespace/Agent -// names, except the supplied UID when non-nil. Existing sessions retain -// their pinned revisions. Missing and already-retired identities are a no-op. -func (c *Client) RetireAgentIdentities(ctx context.Context, namespace, name string, except *AgentDefinition) error { - return retireAgentIdentities(ctx, c.db, namespace, name, except) +// RetireAgentIdentities retires identities at the given namespace/Agent name. +// Existing sessions retain their pinned revisions. Missing and already-retired +// identities are a no-op. +func (c *Client) RetireAgentIdentities(ctx context.Context, namespace, name string) error { + return retireAgentIdentities(ctx, c.db, namespace, name, "") } // retireAgentIdentities uses the caller's executor so retirement can commit -// atomically with definition preparation. A nil exception retires every matching identity. -func retireAgentIdentities(ctx context.Context, db dbExecutor, namespace, name string, except *AgentDefinition) error { - var uid *string - if except != nil { - uid = &except.AgentUID - } +// atomically with definition preparation. An empty +// retainedUID retires every active identity at this source name. +func retireAgentIdentities(ctx context.Context, db dbExecutor, namespace, name, retainedUID string) error { return execSQL(ctx, db, ` UPDATE agent_definition SET retired_at = NOW(), updated_at = NOW() WHERE namespace = $1 AND agent_name = $2 - AND retired_at IS NULL AND agent_uid IS DISTINCT FROM $3::text - `, namespace, name, uid) + AND retired_at IS NULL AND agent_uid IS DISTINCT FROM NULLIF($3, '') + `, namespace, name, retainedUID) } // ListUnreferencedRuntimeRevisions lists revisions unused by active Agents, @@ -204,7 +233,7 @@ func getAvailableRuntimeRevisionForUpdate(ctx context.Context, tx pgx.Tx, revisi } return queryOne(ctx, tx, ` SELECT revision, namespace, agent_name, agent_uid, - source_snapshot, egress_destinations, credentials, actor_template_atespace, actor_template_name, actor_template_uid, + source_snapshot, egress_destinations, token_passthrough_origins, credentials, actor_template_atespace, actor_template_name, actor_template_uid, agent_card, deleted_at FROM agent_runtime_revision WHERE revision = $1 `, pgx.RowToStructByName[runtimeRevisionRow], revision) } @@ -227,14 +256,14 @@ func lockRuntimeRevisionForReference(ctx context.Context, tx pgx.Tx, revision, k type runtimeRevisionRecord struct { RuntimeArtifact - SourceSnapshot json.RawMessage - EgressDestinations []string - Credentials []egress.Credential + SourceSnapshot json.RawMessage + EgressDestinations []string + TokenPassthroughOrigins []string + Credentials []egress.Credential } -// recordRuntimeRevision writes immutable common inputs with mutable artifact -// status. The caller writes its typed extension and promotion pointer in this -// transaction, holding preparation locks before taking the revision lock. +// recordRuntimeRevision writes immutable common inputs and mutable artifact status. +// The caller writes its typed extension in the same transaction. func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevisionRecord) error { if revision.Credentials == nil { revision.Credentials = []egress.Credential{} @@ -242,14 +271,17 @@ func recordRuntimeRevision(ctx context.Context, tx pgx.Tx, revision runtimeRevis if revision.EgressDestinations == nil { revision.EgressDestinations = []string{} } + if revision.TokenPassthroughOrigins == nil { + revision.TokenPassthroughOrigins = []string{} + } result, err := tx.Exec(ctx, ` INSERT INTO runtime_revision (revision, kind, namespace, source_snapshot, egress_destinations, - actor_template_atespace, actor_template_name, actor_template_uid, credentials) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + actor_template_atespace, actor_template_name, actor_template_uid, credentials, token_passthrough_origins) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (revision) DO UPDATE SET actor_template_uid = EXCLUDED.actor_template_uid, updated_at = NOW() WHERE runtime_revision.deleted_at IS NULL AND runtime_revision.kind = EXCLUDED.kind `, revision.Revision, revision.Kind, revision.Namespace, revision.SourceSnapshot, revision.EgressDestinations, - revision.ActorTemplateAtespace, revision.ActorTemplateName, revision.ActorTemplateUID, revision.Credentials) + revision.ActorTemplateAtespace, revision.ActorTemplateName, revision.ActorTemplateUID, revision.Credentials, revision.TokenPassthroughOrigins) if err != nil { return fmt.Errorf("record runtime revision %s: %w", revision.Revision, err) } @@ -337,16 +369,17 @@ func (c *Client) DeleteRuntimeRevision(ctx context.Context, revision, actorTempl } type runtimeRevisionRow struct { - Revision string - Namespace string - AgentName string - AgentUID string - SourceSnapshot []byte - EgressDestinations []string - Credentials []egress.Credential - ActorTemplateAtespace string - ActorTemplateName string - ActorTemplateUID string - AgentCard []byte - DeletedAt *time.Time + Revision string + Namespace string + AgentName string + AgentUID string + SourceSnapshot []byte + EgressDestinations []string + TokenPassthroughOrigins []string + Credentials []egress.Credential + ActorTemplateAtespace string + ActorTemplateName string + ActorTemplateUID string + AgentCard []byte + DeletedAt *time.Time } diff --git a/go/core/internal/database/runtime_revisions_test.go b/go/core/internal/database/runtime_revisions_test.go index b12dfe45dc..928b7def9a 100644 --- a/go/core/internal/database/runtime_revisions_test.go +++ b/go/core/internal/database/runtime_revisions_test.go @@ -20,38 +20,48 @@ import ( ) func TestRetireAgentIdentities(t *testing.T) { - for _, test := range []struct { - name string - except *AgentDefinition - keep bool - }{ - {name: "all"}, - {name: "except current", except: &AgentDefinition{AgentUID: "assistant-uid"}, keep: true}, - {name: "replacement template", except: &AgentDefinition{AgentUID: "replacement-uid"}}, + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") + sessionFixture(t, client, ctx, "team-b", "other-namespace", "assistant", "kagent") + sessionFixture(t, client, ctx, "team-a", "other-template", "other", "kagent") + sessionFixture(t, client, ctx, "team-a", "other-agent", "another", "other") + for range 2 { + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) + } + revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) + require.NoError(t, err) + require.Len(t, revisions, 1, "retirement must stay within the requested namespace and name") + require.Equal(t, "revision", revisions[0].Revision) +} + +func TestListAgentEgressSourcesBoundsRetiredIdentities(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") + session, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "request") + require.NoError(t, err) + _, err = finishSessionOperation(ctx, client, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, "actor.example.com") + require.NoError(t, err) + for _, definition := range []AgentDefinition{ + {Namespace: "team-a", AgentName: "assistant", AgentUID: "replacement"}, + {Namespace: "team-a", AgentName: "unused", AgentUID: "unused"}, + {Namespace: "team-b", AgentName: "assistant", AgentUID: "other"}, } { - t.Run(test.name, func(t *testing.T) { - client := NewClient(setupTestDB(t), "public") - ctx := t.Context() - sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") - sessionFixture(t, client, ctx, "team-b", "other-namespace", "assistant", "kagent") - sessionFixture(t, client, ctx, "team-a", "other-template", "other", "kagent") - sessionFixture(t, client, ctx, "team-a", "other-agent", "another", "other") - for range 2 { - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", test.except)) - } - revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) - require.NoError(t, err) - if test.keep { - require.Empty(t, revisions) - session, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "session") - require.NoError(t, err) - require.Equal(t, "revision", session.GetPreparedRevision(), "the exception retains its last-good revision") - } else { - require.Len(t, revisions, 1, "retirement must stay within the requested namespace and names") - require.Equal(t, "revision", revisions[0].Revision) - } - }) + require.NoError(t, client.UpsertAgentDefinition(ctx, definition)) } + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "unused")) + sources, err := client.ListAgentEgressSources(ctx) + require.NoError(t, err) + require.Equal(t, []RuntimeEgressSource{ + {Kind: RuntimeEgressAgent, Namespace: "team-a", Name: "assistant"}, + {Kind: RuntimeEgressAgent, Namespace: "team-b", Name: "assistant"}, + }, sources, "include only active sources and retired identities that still have actors") + require.NoError(t, deleteSession(ctx, client, session.Id)) + sources, err = client.ListAgentEgressSources(ctx) + require.NoError(t, err) + require.Equal(t, []RuntimeEgressSource{{Kind: RuntimeEgressAgent, Namespace: "team-b", Name: "assistant"}}, sources) } func TestRuntimeRevisionCollectionAfterPairRetirement(t *testing.T) { @@ -67,7 +77,7 @@ func TestRuntimeRevisionCollectionAfterPairRetirement(t *testing.T) { _, err = client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) - err = client.RetireAgentIdentities(ctx, "team-a", "assistant", nil) + err = client.RetireAgentIdentities(ctx, "team-a", "assistant") require.NoError(t, err) revisions, err = client.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) @@ -97,7 +107,7 @@ func TestRuntimeRevisionCollectionPreservesSessionAndCheckpoint(t *testing.T) { require.NoError(t, err) _, err = markSessionReady(ctx, client, session.GetId(), "runtime") require.NoError(t, err) - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) assertRetained := func() { t.Helper() @@ -170,24 +180,35 @@ func TestRuntimeRevisionPairReplacement(t *testing.T) { revision.Revision = "new" revision.AgentUID = "replacement-uid" revision.ActorTemplateName = "new-actor-template" - require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, false)) + require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{})) pair := AgentDefinition{ Namespace: "team-a", AgentName: "assistant", AgentUID: revision.AgentUID, DesiredRevision: "new", } - require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) + upsertErr := client.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) request := newSessionRequest(uuid.NewString(), "assistant", "kagent", "") _, _, err = client.CreateSession(ctx, request, "session") require.ErrorIs(t, err, ErrNotFound, "a replacement must not select the previous template UID") - require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, true)) + require.NoError(t, client.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{Ready: true})) + session, _, err := client.CreateSession(ctx, request, "session") + require.NoError(t, err) + require.Equal(t, "new", session.GetPreparedRevision()) // Reconciliation of the same identity must preserve its last success while // a new desired revision is still preparing. pair.DesiredRevision = "pending" - require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) - session, _, err := client.CreateSession(ctx, request, "session") + upsertErr = client.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) + retried, created, err := client.CreateSession(ctx, request, "session") require.NoError(t, err) - require.Equal(t, "new", session.GetPreparedRevision()) + require.False(t, created) + require.Equal(t, session.GetId(), retried.GetId()) + require.Equal(t, "new", retried.GetPreparedRevision()) + fresh, created, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "pending") + require.NoError(t, err) + require.True(t, created) + require.Equal(t, "new", fresh.PreparedRevision) revisions, err := client.ListUnreferencedRuntimeRevisions(ctx) require.NoError(t, err) require.Len(t, revisions, 1) @@ -234,10 +255,10 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") - query := "FROM runtime_revision WHERE revision = $1 FOR UPDATE" + query := "FROM agent_definition p" if source != "session" { query = "FOR UPDATE OF r" - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) } barrier := &runtimeReferenceBarrier{query: query, afterQuery: referenceFirst, reached: make(chan struct{}), resume: make(chan struct{})} var resume sync.Once @@ -263,7 +284,8 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) pair.AgentUID = "replacement-uid" pair.DesiredRevision = "revision" } - created <- creating.UpsertAgentDefinition(ctx, pair) + upsertErr := creating.UpsertAgentDefinition(ctx, pair) + created <- upsertErr }() select { case <-barrier.reached: @@ -271,7 +293,18 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) t.Fatal(ctx.Err()) } if source == "session" { - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + if referenceFirst { + // Creation holds the definition lock until its reference commits, + // so retirement must follow creation before GC can claim it. + resume.Do(func() { close(barrier.resume) }) + require.NoError(t, <-created) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) + claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "revision") + require.NoError(t, err) + require.Nil(t, claimed) + return + } + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) } type claimResult struct { revision *RuntimeArtifact @@ -300,7 +333,11 @@ func TestRuntimeRevisionDeletionSerializesWithReferenceAcquisition(t *testing.T) require.NoError(t, result.err) require.NotNil(t, result.revision) resume.Do(func() { close(barrier.resume) }) - require.ErrorIs(t, <-created, ErrObjectDeleting) + if source == "session" { + require.ErrorIs(t, <-created, ErrNotFound) + } else { + require.ErrorIs(t, <-created, ErrObjectDeleting) + } } }) } @@ -314,17 +351,18 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") pair := AgentDefinition{ Namespace: "team-a", AgentName: "assistant", AgentUID: "assistant-uid", - DesiredRevision: "pending", + DesiredRevision: "revision", } - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) // A skipped finalization must leave last-good intact for reactivation. require.NoError(t, client.DeleteRuntimeRevision(ctx, "revision", "revision-actor-uid")) - require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) + upsertErr := client.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) session, _, err := client.CreateSession(ctx, newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), "session") require.NoError(t, err) require.Equal(t, "revision", session.GetPreparedRevision()) require.NoError(t, deleteSession(ctx, client, session.GetId())) - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) original, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "revision") @@ -332,12 +370,14 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.NotNil(t, claimed) // Reactivating a retired success pointer and adding a new desired edge are // both forbidden, as is overwriting a runtime while deletion is in flight. - require.ErrorIs(t, client.UpsertAgentDefinition(ctx, pair), ErrObjectDeleting) + upsertErr = client.UpsertAgentDefinition(ctx, pair) + require.ErrorIs(t, upsertErr, ErrObjectDeleting) pair.AgentUID = "replacement-uid" pair.DesiredRevision = "revision" - require.ErrorIs(t, client.UpsertAgentDefinition(ctx, pair), ErrObjectDeleting) - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, false), ErrObjectDeleting) - require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, true), ErrObjectDeleting) + upsertErr = client.UpsertAgentDefinition(ctx, pair) + require.ErrorIs(t, upsertErr, ErrObjectDeleting) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, RecordRevisionOptions{}), ErrObjectDeleting) + require.ErrorIs(t, client.RecordRuntimeRevision(ctx, *original, RecordRevisionOptions{Ready: true}), ErrObjectDeleting) // A new client rediscovers and retries the committed claim after a crash. restarted := NewClient(pool, "public") revisions, err := restarted.ListUnreferencedRuntimeRevisions(ctx) @@ -353,7 +393,7 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) require.ErrorIs(t, err, ErrNotFound) // The same digest can be prepared again once cleanup has completed. original.ActorTemplateUID = "recreated-actor-uid" - require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, false)) + require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, RecordRevisionOptions{})) newClaim, err := restarted.BeginRuntimeRevisionDeletion(ctx, "revision") require.NoError(t, err) require.NotNil(t, newClaim) @@ -361,12 +401,13 @@ func TestRuntimeRevisionClaimPreservesReferencesUntilFinalization(t *testing.T) _, err = restarted.GetRuntimeRevision(ctx, "revision") require.NoError(t, err, "a delayed collector must not finalize the new runtime") require.NoError(t, restarted.DeleteRuntimeRevision(ctx, "revision", "recreated-actor-uid")) - require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, false)) - require.NoError(t, restarted.UpsertAgentDefinition(ctx, pair)) + require.NoError(t, restarted.RecordRuntimeRevision(ctx, *original, RecordRevisionOptions{})) + upsertErr = restarted.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) } func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { - for _, operation := range []string{"reactivate", "record"} { + for _, operation := range []string{"reactivate", "promote"} { for _, finalizeFirst := range []bool{false, true} { t.Run(fmt.Sprintf("%s/finalize_first=%t", operation, finalizeFirst), func(t *testing.T) { pool := setupTestDB(t) @@ -374,9 +415,9 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() sessionFixture(t, client, ctx, "team-a", "revision", "assistant", "kagent") - require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant", nil)) - original, err := client.GetRuntimeRevision(ctx, "revision") + revision, err := client.GetRuntimeRevision(ctx, "revision") require.NoError(t, err) + require.NoError(t, client.RetireAgentIdentities(ctx, "team-a", "assistant")) claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "revision") require.NoError(t, err) require.NotNil(t, claimed) @@ -384,14 +425,14 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { barrier := &runtimeReferenceBarrier{ query: "FOR UPDATE OF r", reached: make(chan struct{}), resume: make(chan struct{}), } - if operation == "record" { - barrier.query = "INSERT INTO runtime_revision" + if operation == "promote" { + barrier.query = "FROM runtime_revision WHERE revision = $1 FOR UPDATE" } waitingQuery := "ORDER BY namespace, agent_uid" if finalizeFirst { barrier.query, barrier.afterQuery = "FROM runtime_revision WHERE revision = $1 FOR UPDATE", true waitingQuery = "INSERT INTO agent_definition" - if operation == "record" { + if operation == "promote" { waitingQuery = "FOR UPDATE OF p" } } @@ -412,11 +453,12 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { } created, deleted := make(chan error, 1), make(chan error, 1) create := func() { - if operation == "record" { - created <- creating.RecordRuntimeRevision(ctx, *original, true) + if operation == "promote" { + created <- creating.RecordRuntimeRevision(ctx, *revision, RecordRevisionOptions{Ready: true}) return } - created <- creating.UpsertAgentDefinition(ctx, pair) + upsertErr := creating.UpsertAgentDefinition(ctx, pair) + created <- upsertErr } finalize := func() { deleted <- deleting.DeleteRuntimeRevision(ctx, "revision", claimed.ActorTemplateUID) } if finalizeFirst { @@ -446,13 +488,14 @@ func TestRuntimeRevisionFinalizationSerializesWithPairWrites(t *testing.T) { require.ErrorIs(t, <-created, ErrObjectDeleting) } require.NoError(t, <-deleted, "finalization and pair writes must not deadlock") - require.NoError(t, client.UpsertAgentDefinition(ctx, pair)) + upsertErr := client.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) }) } } } -func TestRecordRuntimeRevisionPromotesOnlyCurrentActivePair(t *testing.T) { +func TestRecordReadyRuntimeRevisionOnlyCurrentActivePair(t *testing.T) { pool := setupTestDB(t) c := NewClient(pool, "public") ctx := t.Context() @@ -480,44 +523,54 @@ func TestRecordRuntimeRevisionPromotesOnlyCurrentActivePair(t *testing.T) { require.NoError(t, err) require.Equal(t, want, session.PreparedRevision) } - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, false)) + upsertErr := c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{})) assertAvailableRevision("") stored, err := c.GetRuntimeRevision(ctx, revision.Revision) require.NoError(t, err) require.Equal(t, "original", stored.AgentCard.Name) - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) assertAvailableRevision("first") pair.DesiredRevision = "second" - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) + upsertErr = c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) revision.Revision, revision.ActorTemplateName = "second", "second" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{})) + assertAvailableRevision("first") + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) assertAvailableRevision("second") // A delayed report for the old revision must not replace the newer ready revision. revision.Revision, revision.ActorTemplateName = "first", "first" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) assertAvailableRevision("second") pair.DesiredRevision = "third" - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) - require.NoError(t, c.RetireAgentIdentities(ctx, pair.Namespace, pair.AgentName, nil)) + upsertErr = c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) + require.NoError(t, c.RetireAgentIdentities(ctx, pair.Namespace, pair.AgentName)) revision.Revision, revision.ActorTemplateName = "third", "third" - require.NoError(t, c.RecordRuntimeRevision(ctx, revision, true)) + require.NoError(t, c.RecordRuntimeRevision(ctx, revision, RecordRevisionOptions{Ready: true})) assertAvailableRevision("") // Reviving the pair must retain the last success, not a report made while retired. - require.NoError(t, c.UpsertAgentDefinition(ctx, pair)) + upsertErr = c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) + assertAvailableRevision("second") + pair.DesiredRevision = "second" + upsertErr = c.UpsertAgentDefinition(ctx, pair) + require.NoError(t, upsertErr) assertAvailableRevision("second") } func TestRuntimeRevisionPersistsCredentialBindings(t *testing.T) { client := NewClient(setupTestDB(t), "public") - revision := RuntimeRevision{Revision: "credential-revision", Namespace: "team", AgentName: "agent", AgentUID: "agent", SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{"api.example.com"}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", Credentials: []egress.Credential{{Hostname: "api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}}} - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, false)) + revision := RuntimeRevision{Revision: "credential-revision", Namespace: "team", AgentName: "agent", AgentUID: "agent", SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{"https://api.example.com:443"}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", Credentials: []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}}} + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, RecordRevisionOptions{})) got, err := client.GetRuntimeRevision(t.Context(), revision.Revision) require.NoError(t, err) require.Equal(t, revision.Credentials, got.Credentials) revision.Credentials[0].URI = "ate-secret://k8s.io/default/team/other/token" - require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, false)) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), revision, RecordRevisionOptions{})) unchanged, err := client.GetRuntimeRevision(t.Context(), revision.Revision) require.NoError(t, err) require.Equal(t, got.Credentials, unchanged.Credentials, "revision credentials are immutable") @@ -527,10 +580,76 @@ func TestRuntimeRevisionRejectsMalformedStoredCredentials(t *testing.T) { revision, err := toRuntimeRevision(runtimeRevisionRow{ Revision: "bad-revision", Credentials: []egress.Credential{{ - Hostname: "*", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token", + Origin: "*", Header: "authorization", URI: "ate-secret://k8s.io/default/team/auth/token", }}, }) require.ErrorContains(t, err, "decode runtime revision bad-revision credentials") - require.ErrorContains(t, err, "exact DNS hostname") + require.ErrorContains(t, err, "exact HTTP(S) DNS origin") require.Nil(t, revision) } + +func TestRecordReadyRevisionRollsBackOnPromotionFailure(t *testing.T) { + for _, kind := range []string{"agent", "sandbox"} { + for _, existing := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/existing=%t", kind, existing), func(t *testing.T) { + pool := setupTestDB(t) + client := NewClient(pool, "public") + ctx := t.Context() + var record func(string, RecordRevisionOptions) error + var storedUID func() (string, error) + switch kind { + case "agent": + require.NoError(t, client.UpsertAgentDefinition(ctx, AgentDefinition{Namespace: "team", AgentName: "source", AgentUID: "uid", DesiredRevision: "revision"})) + record = func(uid string, options RecordRevisionOptions) error { + return client.RecordRuntimeRevision(ctx, RuntimeRevision{Revision: "revision", Namespace: "team", AgentName: "source", AgentUID: "uid", AgentCard: &a2apb.AgentCard{}, SourceSnapshot: []byte(`{}`), ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: uid}, options) + } + storedUID = func() (string, error) { + revision, err := client.GetRuntimeRevision(ctx, "revision") + if err != nil { + return "", err + } + return revision.ActorTemplateUID, nil + } + // Force the final write to fail after the revision has been recorded. + _, err := pool.Exec(ctx, `ALTER TABLE agent_definition ADD CONSTRAINT reject_promotion CHECK (latest_successful_revision IS NULL)`) + require.NoError(t, err) + t.Cleanup(func() { + if _, err := pool.Exec(context.Background(), `ALTER TABLE agent_definition DROP CONSTRAINT reject_promotion`); err != nil { + t.Errorf("remove test constraint: %v", err) + } + }) + case "sandbox": + require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team", SandboxTemplateName: "source", SandboxTemplateUID: "uid", DesiredRevision: "revision"})) + record = func(uid string, options RecordRevisionOptions) error { + return client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: "revision", Namespace: "team", ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: uid}, SandboxTemplateName: "source", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, options) + } + storedUID = func() (string, error) { + revision, err := client.GetSandboxRevision(ctx, "revision") + if err != nil { + return "", err + } + return revision.ActorTemplateUID, nil + } + _, err := pool.Exec(ctx, `ALTER TABLE sandbox_template_definition ADD CONSTRAINT reject_promotion CHECK (latest_successful_revision IS NULL)`) + require.NoError(t, err) + t.Cleanup(func() { + if _, err := pool.Exec(context.Background(), `ALTER TABLE sandbox_template_definition DROP CONSTRAINT reject_promotion`); err != nil { + t.Errorf("remove test constraint: %v", err) + } + }) + } + if existing { + require.NoError(t, record("original", RecordRevisionOptions{})) + } + require.ErrorContains(t, record("updated", RecordRevisionOptions{Ready: true}), "reject_promotion") + uid, err := storedUID() + if existing { + require.NoError(t, err) + require.Equal(t, "original", uid, "a failed promotion must roll back artifact updates") + } else { + require.ErrorIs(t, err, ErrNotFound, "a failed promotion must not leave a partially recorded revision") + } + }) + } + } +} diff --git a/go/core/internal/database/sandbox_revisions.go b/go/core/internal/database/sandbox_revisions.go index 5bee645014..6efbd1faee 100644 --- a/go/core/internal/database/sandbox_revisions.go +++ b/go/core/internal/database/sandbox_revisions.go @@ -6,6 +6,8 @@ import ( "fmt" "github.com/jackc/pgx/v5" + + "github.com/kagent-dev/kagent/go/core/internal/egress" ) type SandboxRevision struct { @@ -20,27 +22,33 @@ type SandboxTemplateDefinition struct { Namespace string SandboxTemplateName string SandboxTemplateUID string + EgressGrant egress.Access DesiredRevision string } // UpsertSandboxTemplateDefinition pins desired inputs before any backend mutation. A new // Kubernetes UID retires the former identity without inheriting its last success. +// Grants are canonicalized before writing; errors may include an uncertain commit outcome. func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition SandboxTemplateDefinition) error { + grants, err := canonicalStoredGrants(definition.EgressGrant) + if err != nil { + return fmt.Errorf("validate source grants: %w", err) + } return c.withTx(ctx, func(tx pgx.Tx) error { - if err := execSQL(ctx, tx, ` - UPDATE sandbox_template_definition SET retired_at = NOW(), updated_at = NOW() - WHERE namespace = $1 AND sandbox_template_name = $2 AND sandbox_template_uid <> $3 AND retired_at IS NULL - `, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID); err != nil { + if err := retireSandboxTemplateIdentities(ctx, tx, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID); err != nil { return err } if err := execSQL(ctx, tx, ` - INSERT INTO sandbox_template_definition (namespace, sandbox_template_name, sandbox_template_uid, desired_revision) - VALUES ($1, $2, $3, $4) + INSERT INTO sandbox_template_definition (namespace, sandbox_template_name, sandbox_template_uid, desired_revision, grant_destinations, grant_credentials) + VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (namespace, sandbox_template_uid) DO UPDATE SET - desired_revision = EXCLUDED.desired_revision, retired_at = NULL, updated_at = NOW() - `, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID, definition.DesiredRevision); err != nil { + sandbox_template_name = EXCLUDED.sandbox_template_name, desired_revision = EXCLUDED.desired_revision, + grant_destinations = EXCLUDED.grant_destinations, grant_credentials = EXCLUDED.grant_credentials, + retired_at = NULL, updated_at = NOW() + `, definition.Namespace, definition.SandboxTemplateName, definition.SandboxTemplateUID, definition.DesiredRevision, grants.Destinations, grants.Credentials); err != nil { return err } + revisions, err := queryMany(ctx, tx, ` SELECT r.revision, r.kind, r.namespace, r.actor_template_atespace, r.actor_template_name, r.actor_template_uid, r.deleted_at FROM runtime_revision r JOIN sandbox_template_definition p @@ -62,21 +70,46 @@ func (c *Client) UpsertSandboxTemplateDefinition(ctx context.Context, definition }) } +// GetSandboxEgressGrant returns the active identity's last admitted grants. +// Missing or retired identities return empty access; malformed stored grants fail. +func (c *Client) GetSandboxEgressGrant(ctx context.Context, namespace, uid string) (egress.Access, error) { + grants, err := queryOne(ctx, c.db, ` + SELECT grant_destinations AS destinations, grant_credentials AS credentials + FROM sandbox_template_definition WHERE namespace = $1 AND sandbox_template_uid = $2 AND retired_at IS NULL + `, pgx.RowToStructByName[egress.Access], namespace, uid) + return decodeEgressGrant(grants, err) +} + +// RetireSandboxTemplateIdentities retires active identities at this source name. +// Missing/already-retired identities are a no-op; retained actors lose live grants. func (c *Client) RetireSandboxTemplateIdentities(ctx context.Context, namespace, name string) error { - return execSQL(ctx, c.db, ` + return retireSandboxTemplateIdentities(ctx, c.db, namespace, name, "") +} + +// retireSandboxTemplateIdentities shares retirement with definition preparation +// in the caller's transaction. An empty retainedUID retires all active identities. +func retireSandboxTemplateIdentities(ctx context.Context, db dbExecutor, namespace, name, retainedUID string) error { + return execSQL(ctx, db, ` UPDATE sandbox_template_definition SET retired_at = NOW(), updated_at = NOW() WHERE namespace = $1 AND sandbox_template_name = $2 AND retired_at IS NULL - `, namespace, name) + AND sandbox_template_uid IS DISTINCT FROM NULLIF($3, '') + `, namespace, name, retainedUID) } -// RecordSandboxRevision promotes only the active identity still desiring this -// revision. A late preparation response cannot replace a newer successful input. -func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevision, ready bool) error { +// RecordSandboxRevision atomically stores immutable inputs and, when Ready, +// promotes the revision for its active source if still desired. Existing inputs +// are preserved; actor UID and update time refresh. Stale reports cannot replace +// the last successful revision. Deleting revisions return ErrObjectDeleting. +func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevision, options RecordRevisionOptions) error { return c.withTx(ctx, func(tx pgx.Tx) error { - if _, err := queryMany(ctx, tx, ` - SELECT 1 FROM sandbox_template_definition WHERE namespace = $1 AND sandbox_template_uid = $2 FOR UPDATE - `, pgx.RowTo[int], revision.Namespace, revision.SandboxTemplateUID); err != nil { - return err + if options.Ready { + // Match definition preparation and GC: source before revision. + if _, err := queryMany(ctx, tx, ` + SELECT 1 FROM sandbox_template_definition p + WHERE p.namespace = $1 AND p.sandbox_template_uid = $2 FOR UPDATE OF p + `, pgx.RowTo[int], revision.Namespace, revision.SandboxTemplateUID); err != nil { + return err + } } revision.Kind = runtimeKindSandbox if err := recordRuntimeRevision(ctx, tx, runtimeRevisionRecord{RuntimeArtifact: revision.RuntimeArtifact, SourceSnapshot: revision.SourceSnapshot}); err != nil { @@ -88,18 +121,21 @@ func (c *Client) RecordSandboxRevision(ctx context.Context, revision SandboxRevi `, revision.Revision, revision.SandboxTemplateName, revision.SandboxTemplateUID); err != nil { return err } - if !ready { + if !options.Ready { return nil } return execSQL(ctx, tx, ` UPDATE sandbox_template_definition p SET latest_successful_revision = r.revision, updated_at = NOW() FROM runtime_revision r JOIN sandbox_revision s USING (revision) - WHERE r.revision = $1 AND p.namespace = r.namespace AND p.sandbox_template_uid = s.sandbox_template_uid + WHERE r.revision = $1 AND p.namespace = $2 AND p.sandbox_template_uid = $3 + AND p.namespace = r.namespace AND p.sandbox_template_uid = s.sandbox_template_uid AND p.desired_revision = r.revision AND p.retired_at IS NULL - `, revision.Revision) + AND p.latest_successful_revision IS DISTINCT FROM r.revision + `, revision.Revision, revision.Namespace, revision.SandboxTemplateUID) }) } +// GetSandboxRevision reads immutable sandbox inputs; missing revisions return ErrNotFound. func (c *Client) GetSandboxRevision(ctx context.Context, revision string) (*SandboxRevision, error) { row, err := queryOne(ctx, c.db, ` SELECT r.revision, r.kind, r.namespace, r.actor_template_atespace, r.actor_template_name, r.actor_template_uid, diff --git a/go/core/internal/database/sandboxes.go b/go/core/internal/database/sandboxes.go index d758492026..412e21cc40 100644 --- a/go/core/internal/database/sandboxes.go +++ b/go/core/internal/database/sandboxes.go @@ -63,8 +63,9 @@ type SandboxCreateOptions struct { TTL time.Duration } -// CreateSandbox atomically reserves a sandbox and its prepared revision. A -// duplicate creator/requestID returns the original resource when inputs match. +// CreateSandbox atomically reserves a sandbox and its latest successful revision, +// or returns ErrNotFound if none is available. A duplicate creator/requestID +// returns the original resource when inputs match. // The boolean reports whether this call created the sandbox. func (c *Client) CreateSandbox(ctx context.Context, request *apiv1.Sandbox, requestID string, options SandboxCreateOptions) (*apiv1.Sandbox, bool, error) { existing, err := readSandboxRequest(ctx, c.db, request.GetCreator(), requestID) @@ -113,7 +114,8 @@ func readSandboxRequest(ctx context.Context, db dbExecutor, userID, requestID st } // insertSandbox reserves a new sandbox in CREATING/CREATE and pins the active -// template's latest successful revision. A duplicate creator/requestID returns +// template's latest successful revision. A missing prepared +// target returns ErrNotFound. A duplicate creator/requestID returns // pgx.ErrNoRows so the caller can roll back and read the winning transaction. func insertSandbox(ctx context.Context, tx pgx.Tx, request *apiv1.Sandbox, requestID string, options SandboxCreateOptions) (sandboxRow, error) { if options.TTL < time.Second || options.TTL > 24*time.Hour { @@ -124,7 +126,8 @@ func insertSandbox(ctx context.Context, tx pgx.Tx, request *apiv1.Sandbox, reque err := tx.QueryRow(ctx, ` SELECT latest_successful_revision, clock_timestamp() FROM sandbox_template_definition WHERE namespace = $1 AND sandbox_template_name = $2 AND sandbox_template_uid = $3 - AND retired_at IS NULL AND latest_successful_revision IS NOT NULL FOR UPDATE + AND retired_at IS NULL AND latest_successful_revision IS NOT NULL + FOR UPDATE `, request.GetSandboxTemplate().GetNamespace(), request.GetSandboxTemplate().GetName(), options.TemplateUID).Scan(&revision, &createdAt) if err != nil { return sandboxRow{}, notFoundOr(err) diff --git a/go/core/internal/database/sandboxes_test.go b/go/core/internal/database/sandboxes_test.go index 766118b448..40bed380c9 100644 --- a/go/core/internal/database/sandboxes_test.go +++ b/go/core/internal/database/sandboxes_test.go @@ -16,14 +16,58 @@ import ( func sandboxFixture(t *testing.T, client *Client, revision string) (*apiv1.Sandbox, SandboxCreateOptions) { t.Helper() ctx := t.Context() - require.NoError(t, client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision})) + upsertErr := client.UpsertSandboxTemplateDefinition(ctx, SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: revision}) + require.NoError(t, upsertErr) require.NoError(t, client.RecordSandboxRevision(ctx, SandboxRevision{RuntimeArtifact: RuntimeArtifact{Revision: revision, Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: revision, ActorTemplateUID: "actor-uid"}, - SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, true)) + SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", SourceSnapshot: []byte(`{}`)}, RecordRevisionOptions{Ready: true})) hash := sha256.Sum256([]byte("request")) return &apiv1.Sandbox{Id: uuid.NewString(), Creator: "alice", SandboxTemplate: &apiv1.ResourceReference{Namespace: "team-a", Name: "scratch"}, Name: "experiment"}, SandboxCreateOptions{RequestHash: hash[:], TemplateUID: "uid", TTL: time.Hour} } +func TestRecordReadySandboxRevisionOnlyCurrentActiveTemplate(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + ctx := t.Context() + request, options := sandboxFixture(t, client, "first") + definition := SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "uid", DesiredRevision: "second"} + upsertErr := client.UpsertSandboxTemplateDefinition(ctx, definition) + require.NoError(t, upsertErr) + revision, err := client.GetSandboxRevision(ctx, "first") + require.NoError(t, err) + revision.Revision, revision.ActorTemplateName = "second", "second" + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{})) + assertAvailable := func(want string) { + t.Helper() + input := proto.CloneOf(request) + input.Id = uuid.NewString() + instance, _, err := client.CreateSandbox(ctx, input, uuid.NewString(), options) + if want == "" { + require.ErrorIs(t, err, ErrNotFound) + return + } + require.NoError(t, err) + require.Equal(t, want, instance.PreparedRevision) + } + assertAvailable("first") + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{Ready: true})) + assertAvailable("second") + revision.Revision, revision.ActorTemplateName = "first", "first" + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{Ready: true})) + assertAvailable("second") + require.NoError(t, client.RetireSandboxTemplateIdentities(ctx, "team-a", "scratch")) + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{Ready: true})) + assertAvailable("") + upsertErr = client.UpsertSandboxTemplateDefinition(ctx, definition) + require.NoError(t, upsertErr) + assertAvailable("second") + revision.Revision, revision.ActorTemplateName = "unused", "unused" + require.NoError(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{})) + claimed, err := client.BeginRuntimeRevisionDeletion(ctx, "unused") + require.NoError(t, err) + require.NotNil(t, claimed) + require.ErrorIs(t, client.RecordSandboxRevision(ctx, *revision, RecordRevisionOptions{Ready: true}), ErrObjectDeleting) +} + func finishSandbox(t *testing.T, client *Client, id string, kind apiv1.RuntimeOperation) *apiv1.Sandbox { t.Helper() op, err := client.BeginSandboxOperation(t.Context(), id, kind) diff --git a/go/core/internal/database/scheduled_runs_test.go b/go/core/internal/database/scheduled_runs_test.go index 80aa0e82a9..ca835dc53d 100644 --- a/go/core/internal/database/scheduled_runs_test.go +++ b/go/core/internal/database/scheduled_runs_test.go @@ -393,7 +393,7 @@ func TestScheduledExecutionWaitsForPreparedRevision(t *testing.T) { db := setupTestDB(t) c := NewClient(db, "public") schedule, _ := createTestSchedule(t, c) - require.NoError(t, c.RetireAgentIdentities(t.Context(), "team-a", "report", nil)) + require.NoError(t, c.RetireAgentIdentities(t.Context(), "team-a", "report")) execution, err := c.TriggerScheduledRun(t.Context(), uuid.MustParse(schedule.Id), "alice", "manual") require.NoError(t, err) _, err = c.ReserveScheduledRunExecutionSession(t.Context(), uuid.MustParse(execution.Id), "alice") diff --git a/go/core/internal/database/sessions.go b/go/core/internal/database/sessions.go index dd0e4c496d..096f5ac661 100644 --- a/go/core/internal/database/sessions.go +++ b/go/core/internal/database/sessions.go @@ -53,11 +53,11 @@ func sameSessionRequest(session, request *apiv1.Session) bool { } // CreateSession atomically reserves a session, its conversation history, and the -// latest successful runtime revision. A repeated creator/requestID returns the existing -// session when the Agent matches, or ErrIdempotencyConflict otherwise. A -// deleted session returns ErrFailedPrecondition and keeps its request ID reserved. The -// boolean reports whether this call created the session; runtime provisioning belongs to -// the caller. +// latest successful revision, or returns ErrNotFound if none is available. A repeated +// creator/requestID returns the existing session when the Agent matches, or +// ErrIdempotencyConflict otherwise. A deleted session returns ErrFailedPrecondition +// and keeps its request ID reserved. The boolean reports whether this call created +// the session; runtime provisioning belongs to the caller. func (c *Client) CreateSession(ctx context.Context, request *apiv1.Session, requestID string) (*apiv1.Session, bool, error) { existing, err := readSessionRequest(ctx, c.db, request.GetCreator(), requestID) if err == nil { @@ -101,22 +101,23 @@ func (c *Client) CreateSession(ctx context.Context, request *apiv1.Session, requ } // insertSession reserves a new conversation in CREATING/CREATE and pins the active -// Agent's latest successful revision. Callers must supply a transaction so -// the history and session commit together. A missing prepared target returns ErrNotFound; -// a duplicate creator/requestID returns pgx.ErrNoRows for the caller to resolve. +// Agent's latest successful revision, locking the revision against cleanup. Callers must supply +// a transaction so the history and session commit together. A missing prepared target +// returns ErrNotFound; a duplicate creator/requestID returns pgx.ErrNoRows for the caller +// to resolve. func insertSession(ctx context.Context, db pgx.Tx, request *apiv1.Session, requestID string) (sessionRow, error) { type preparedRevision struct { Revision string DBTime time.Time } revision, err := queryOne(ctx, db, ` - SELECT r.revision, clock_timestamp() AS db_time + SELECT p.latest_successful_revision AS revision, clock_timestamp() AS db_time FROM agent_definition p - JOIN agent_runtime_revision r ON r.revision = p.latest_successful_revision - WHERE p.namespace = $1 AND p.agent_name = $2 AND p.retired_at IS NULL - `, pgx.RowToStructByName[preparedRevision], request.GetAgent().GetNamespace(), request.GetAgent().GetName()) + WHERE p.namespace = $1 AND p.agent_name = $2 AND p.retired_at IS NULL + AND p.latest_successful_revision IS NOT NULL + `, pgx.RowToStructByName[preparedRevision], request.GetAgent().GetNamespace(), request.GetAgent().GetName()) if err != nil { - return sessionRow{}, fmt.Errorf("get latest successful runtime revision: %w", notFoundOr(err)) + return sessionRow{}, fmt.Errorf("get latest prepared runtime revision: %w", notFoundOr(err)) } if _, err := getAvailableRuntimeRevisionForUpdate(ctx, db, revision.Revision); err != nil { return sessionRow{}, err diff --git a/go/core/internal/egress/credentials.go b/go/core/internal/egress/credentials.go index e0136cdfee..f226807c19 100644 --- a/go/core/internal/egress/credentials.go +++ b/go/core/internal/egress/credentials.go @@ -2,7 +2,6 @@ package egress import ( "fmt" - "net/netip" "net/url" "slices" "strings" @@ -14,28 +13,27 @@ import ( // Credential binds a Secret reference to an exact destination and HTTP header. // Values are fetched by the gateway and never enter a runtime revision. type Credential struct { - Hostname string `json:"hostname"` - Header string `json:"header"` - Prefix string `json:"prefix,omitempty"` - URI string `json:"uri"` + Origin string `json:"origin"` + Header string `json:"header"` + Prefix string `json:"prefix,omitempty"` + URI string `json:"uri"` } -// CanonicalCredentials validates and orders bindings. Substrate matches only -// hostnames, so two credentials for the same host and header cannot coexist. +// CanonicalCredentials validates and orders bindings by complete origin and header. func CanonicalCredentials(bindings []Credential) ([]Credential, error) { result := slices.Clone(bindings) for i := range result { c := &result[i] - c.Hostname = strings.TrimSuffix(strings.ToLower(c.Hostname), ".") - c.Header = strings.ToLower(c.Header) - _, ipErr := netip.ParseAddr(c.Hostname) - if ipErr == nil || len(validation.IsDNS1123Subdomain(c.Hostname)) != 0 { - return nil, fmt.Errorf("credential injection requires an exact DNS hostname, got %q", c.Hostname) + u, err := ParseOrigin(c.Origin) + if err != nil || strings.Contains(c.Origin, "*") { + return nil, fmt.Errorf("credential injection requires an exact HTTP(S) DNS origin, got %q", c.Origin) } - if !httpguts.ValidHeaderFieldName(c.Header) || !httpguts.ValidHeaderFieldValue(c.Prefix) { + c.Origin = u.String() + c.Header = strings.ToLower(c.Header) + if !validCredentialHeader(c.Header) || !httpguts.ValidHeaderFieldValue(c.Prefix) { return nil, fmt.Errorf("invalid credential injection header %q", c.Header) } - u, err := url.Parse(c.URI) + u, err = url.Parse(c.URI) if err != nil || u.Scheme != "ate-secret" || u.Host != "k8s.io" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { return nil, fmt.Errorf("invalid Kubernetes credential URI") } @@ -45,21 +43,33 @@ func CanonicalCredentials(bindings []Credential) ([]Credential, error) { } } slices.SortFunc(result, func(a, b Credential) int { - return strings.Compare(a.Hostname+"\x00"+a.Header, b.Hostname+"\x00"+b.Header) + return strings.Compare(a.Origin+"\x00"+a.Header, b.Origin+"\x00"+b.Header) }) for i := 1; i < len(result); i++ { a, b := result[i-1], result[i] - if a.Hostname == b.Hostname && a.Header == b.Header && a != b { - return nil, fmt.Errorf("conflicting credentials for %s header %s", a.Hostname, a.Header) + if a.Origin == b.Origin && a.Header == b.Header && a != b { + return nil, fmt.Errorf("conflicting credentials for %s header %s", a.Origin, a.Header) } } result = slices.Compact(result) counts := map[string]int{} for _, c := range result { - counts[c.Hostname]++ - if counts[c.Hostname] > 16 { - return nil, fmt.Errorf("credential injection supports at most 16 headers per hostname") + counts[c.Origin]++ + if counts[c.Origin] > 16 { + return nil, fmt.Errorf("credential injection supports at most 16 headers per origin") } } return result, nil } + +// validCredentialHeader excludes routing, framing, hop-by-hop and cookie headers. +func validCredentialHeader(header string) bool { + if len(header) > 256 || !httpguts.ValidHeaderFieldName(header) { + return false + } + switch strings.ToLower(header) { + case "host", "connection", "content-length", "transfer-encoding", "te", "trailer", "upgrade", "keep-alive", "proxy-authorization", "proxy-authenticate", "cookie": + return false + } + return true +} diff --git a/go/core/internal/egress/credentials_test.go b/go/core/internal/egress/credentials_test.go index e24254caaa..453d663065 100644 --- a/go/core/internal/egress/credentials_test.go +++ b/go/core/internal/egress/credentials_test.go @@ -7,17 +7,17 @@ import ( ) func TestCanonicalCredentials(t *testing.T) { - base := Credential{Hostname: "API.Example.com.", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"} + base := Credential{Origin: "https://API.Example.com.", Header: "Authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"} got, err := CanonicalCredentials([]Credential{base, base}) require.NoError(t, err) - require.Equal(t, []Credential{{Hostname: "api.example.com", Header: "authorization", Prefix: "Bearer ", URI: base.URI}}, got) - require.Equal(t, "API.Example.com.", base.Hostname) + require.Equal(t, []Credential{{Origin: "https://api.example.com:443", Header: "authorization", Prefix: "Bearer ", URI: base.URI}}, got) + require.Equal(t, "https://API.Example.com.", base.Origin) for _, test := range []struct { name string change func(*Credential) }{ - {"wildcard", func(c *Credential) { c.Hostname = "*.example.com" }}, - {"IP", func(c *Credential) { c.Hostname = "192.0.2.1" }}, + {"wildcard", func(c *Credential) { c.Origin = "https://*.example.com" }}, + {"IP", func(c *Credential) { c.Origin = "https://192.0.2.1" }}, {"header", func(c *Credential) { c.Header = "Authorization\r\nInjected" }}, {"prefix", func(c *Credential) { c.Prefix = "Bearer\n" }}, {"provider", func(c *Credential) { c.URI = "https://example.com/secret" }}, diff --git a/go/core/internal/egress/destination.go b/go/core/internal/egress/destination.go index ade6f203ed..eebe18251b 100644 --- a/go/core/internal/egress/destination.go +++ b/go/core/internal/egress/destination.go @@ -1,9 +1,15 @@ package egress import ( + "fmt" "net" + "net/netip" "net/url" + "slices" + "strconv" "strings" + + "k8s.io/apimachinery/pkg/util/validation" ) // Origin retains the protocol and destination port needed by Substrate's @@ -24,3 +30,78 @@ func Origin(u *url.URL) string { } return (&url.URL{Scheme: u.Scheme, Host: host}).String() } + +// ParseOrigin validates an HTTP(S) DNS origin and makes its port explicit. +// Wildcards replace one leftmost label and cannot cover an entire top-level domain. +func ParseOrigin(raw string) (*url.URL, error) { + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.Hostname() == "" || u.Path != "" || u.ForceQuery || u.RawQuery != "" || strings.Contains(raw, "#") || strings.HasSuffix(u.Host, ":") { + return nil, fmt.Errorf("invalid egress origin %q: expected scheme, DNS hostname and optional port", raw) + } + host := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") + dns := strings.TrimPrefix(host, "*.") + _, ipErr := netip.ParseAddr(dns) + if ipErr == nil || len(validation.IsDNS1123Subdomain(dns)) != 0 || (strings.HasPrefix(host, "*.") && !strings.Contains(dns, ".")) { + return nil, fmt.Errorf("invalid egress origin %q: expected a DNS name or leftmost-label wildcard", raw) + } + portText := u.Port() + if portText == "" { + portText = "80" + if u.Scheme == "https" { + portText = "443" + } + } + port, err := strconv.ParseUint(portText, 10, 16) + if err != nil || port == 0 { + return nil, fmt.Errorf("invalid egress port in %q", raw) + } + u.Host = net.JoinHostPort(host, strconv.FormatUint(port, 10)) + return u, nil +} + +// Access is a compiled network grant. It contains references, +// never Secret values. +type Access struct { + Destinations []string `json:"destinations,omitempty"` + Credentials []Credential `json:"credentials,omitempty"` +} + +// CanonicalAccess merges equivalent origins and validates destinations and +// credentials, including conflicts with canonical caller-token origins. +func CanonicalAccess(destinations []string, credentials []Credential, tokenPassthroughOrigins []string) (Access, error) { + result := Access{} + protocols := map[string]string{} + allowed := map[string]bool{} + for _, raw := range destinations { + u, err := ParseOrigin(raw) + if err != nil { + return Access{}, err + } + if scheme, exists := protocols[u.Host]; exists && scheme != u.Scheme { + return Access{}, fmt.Errorf("egress destination %q uses both HTTP and HTTPS on the same port", u.Host) + } + protocols[u.Host] = u.Scheme + allowed[u.String()] = true + } + for origin := range allowed { + result.Destinations = append(result.Destinations, origin) + } + slices.Sort(result.Destinations) + if len(result.Destinations) > 256 { + return Access{}, fmt.Errorf("egress policy exceeds 256 rules") + } + var err error + result.Credentials, err = CanonicalCredentials(credentials) + if err != nil { + return Access{}, err + } + for _, binding := range result.Credentials { + if !allowed[binding.Origin] { + return Access{}, fmt.Errorf("credential destination %q is not explicitly allowed", binding.Origin) + } + if slices.Contains(tokenPassthroughOrigins, binding.Origin) { + return Access{}, fmt.Errorf("destination %q cannot combine caller-token passthrough with gateway credentials", binding.Origin) + } + } + return result, nil +} diff --git a/go/core/internal/egress/destination_test.go b/go/core/internal/egress/destination_test.go index dc9fa6d812..20abd0d001 100644 --- a/go/core/internal/egress/destination_test.go +++ b/go/core/internal/egress/destination_test.go @@ -1,12 +1,58 @@ package egress import ( + "fmt" "net/url" "testing" "github.com/stretchr/testify/require" ) +func TestCanonicalAccess(t *testing.T) { + credential := Credential{Origin: "https://API.example.com", Header: "Authorization", URI: "ate-secret://k8s.io/default/team/auth/token"} + access, err := CanonicalAccess([]string{"https://*.example.com", "https://api.example.com:0443", "https://API.example.com.", "https://api.example.com:8443", "http://api.example.com"}, []Credential{credential, credential}, nil) + require.NoError(t, err) + require.Equal(t, []string{"http://api.example.com:80", "https://*.example.com:443", "https://api.example.com:443", "https://api.example.com:8443"}, access.Destinations) + require.Len(t, access.Credentials, 1) + require.Equal(t, "https://api.example.com:443", access.Credentials[0].Origin) + _, err = CanonicalAccess([]string{credential.Origin}, []Credential{credential}, []string{"https://api.example.com:443"}) + require.ErrorContains(t, err, "cannot combine caller-token passthrough with gateway credentials") + _, err = CanonicalAccess([]string{credential.Origin}, []Credential{credential}, []string{"https://api.example.com:8443", "http://api.example.com:443"}) + require.NoError(t, err, "caller tokens on different origins do not conflict") + for _, raw := range []string{"https://*", "https://*.com", "https://foo.*.com", "https://192.0.2.1", "https://[::1]", "https://user@api.example.com", "https://api.example.com/", "https://api.example.com?", "https://api.example.com#", "https://api.example.com:", "https://api.example.com:0", "https://api.example.com:65536", "ftp://api.example.com"} { + t.Run(raw, func(t *testing.T) { + _, err := ParseOrigin(raw) + require.Error(t, err) + }) + } + for _, allowed := range []string{"https://*.example.com", "http://api.example.com", "https://api.example.com:8443"} { + _, err := CanonicalAccess([]string{allowed}, []Credential{credential}, nil) + require.ErrorContains(t, err, "not explicitly allowed", "credential must require its exact origin") + } + _, err = CanonicalAccess([]string{"http://api.example.com:443", "https://api.example.com"}, nil, nil) + require.ErrorContains(t, err, "both HTTP and HTTPS") + var origins []string + for i := range 257 { + origins = append(origins, fmt.Sprintf("https://api%d.example.com", i)) + } + _, err = CanonicalAccess(origins, nil, nil) + require.ErrorContains(t, err, "256 rules") +} + +func TestCredentialsIsolatePortsAndRejectProtectedHeaders(t *testing.T) { + one := Credential{Origin: "https://api.example.com", Header: "Authorization", URI: "ate-secret://k8s.io/default/team/auth/token"} + two := one + two.Origin, two.URI = "https://api.example.com:8443", "ate-secret://k8s.io/default/team/other/token" + bindings, err := CanonicalCredentials([]Credential{one, two}) + require.NoError(t, err) + require.Len(t, bindings, 2) + for _, header := range []string{"hOsT", "Content-Length", "Connection", "Transfer-Encoding", "Cookie", "Proxy-Authorization"} { + one.Header = header + _, err := CanonicalCredentials([]Credential{one}) + require.Error(t, err) + } +} + func TestOrigin(t *testing.T) { for _, tt := range []struct{ raw, want string }{ {"https://API.Example.com./v1?key=private#fragment", "https://api.example.com:443"}, diff --git a/go/core/internal/egress/live.go b/go/core/internal/egress/live.go new file mode 100644 index 0000000000..fa223e0721 --- /dev/null +++ b/go/core/internal/egress/live.go @@ -0,0 +1,80 @@ +package egress + +import ( + "context" + "slices" +) + +// PolicyStore reads effective policy from persisted revision inputs and source grants. +// A missing or malformed policy is an error, never permission to retain old access. +type PolicyStore interface { + GetRuntimeEgressPolicy(context.Context, string) (Access, error) +} + +// EffectiveAccess retains the base and all grants compatible with this revision. +// Stored grants can conflict with an older actor's pinned base. +func EffectiveAccess(base, grants Access, tokenPassthroughOrigins []string) (Access, error) { + filtered, _ := FilterConflicts(base, grants, tokenPassthroughOrigins) + return MergeAccess(base, filtered, tokenPassthroughOrigins) +} + +// MergeAccess validates live grants against the pinned runtime's credentials and +// caller-token origins. Inputs are never modified. +func MergeAccess(base, grants Access, tokenPassthroughOrigins []string) (Access, error) { + return CanonicalAccess(append(slices.Clone(base.Destinations), grants.Destinations...), append(slices.Clone(base.Credentials), grants.Credentials...), tokenPassthroughOrigins) +} + +// IntersectAccess retains only destinations and complete credential bindings +// present in both inputs. Inputs contain canonical rules; b may be an invalid +// union of otherwise valid policies. Taking a subset of valid a remains valid. +func IntersectAccess(a, b Access) Access { + var result Access + for _, origin := range a.Destinations { + if slices.Contains(b.Destinations, origin) { + result.Destinations = append(result.Destinations, origin) + } + } + for _, credential := range a.Credentials { + if slices.Contains(result.Destinations, credential.Origin) && slices.Contains(b.Credentials, credential) { + result.Credentials = append(result.Credentials, credential) + } + } + return result +} + +// FilterConflicts retains grants compatible with a valid base and caller-token +// origins. Inputs must be individually canonical and valid. The base wins; +// sorted grant order determines which rules fit when the combined limits are +// exceeded. A conflicting credential does not remove its network destination. +// Neither input is modified; dropped origins are sorted and deduplicated. +func FilterConflicts(base, grants Access, tokenOrigins []string) (Access, []string) { + if _, err := MergeAccess(base, grants, tokenOrigins); err == nil { + return grants, nil + } + var result Access + var dropped []string + for _, origin := range grants.Destinations { + candidate := Access{Destinations: append(slices.Clone(result.Destinations), origin)} + if _, err := MergeAccess(base, candidate, tokenOrigins); err != nil { + dropped = append(dropped, origin) + } else { + result = candidate + } + } + for _, credential := range grants.Credentials { + if !slices.Contains(result.Destinations, credential.Origin) { + continue + } + candidate := Access{ + Destinations: result.Destinations, + Credentials: append(slices.Clone(result.Credentials), credential), + } + if _, err := MergeAccess(base, candidate, tokenOrigins); err != nil { + dropped = append(dropped, credential.Origin) + } else { + result = candidate + } + } + slices.Sort(dropped) + return result, slices.Compact(dropped) +} diff --git a/go/core/internal/egress/live_test.go b/go/core/internal/egress/live_test.go new file mode 100644 index 0000000000..e1899f8920 --- /dev/null +++ b/go/core/internal/egress/live_test.go @@ -0,0 +1,99 @@ +package egress + +import ( + "fmt" + "slices" + "testing" + + "github.com/stretchr/testify/require" +) + +func canonicalTestAccess(t *testing.T, destinations []string, credentials ...Credential) Access { + t.Helper() + access, err := CanonicalAccess(destinations, credentials, nil) + require.NoError(t, err) + return access +} + +func cloneAccess(a Access) Access { + return Access{Destinations: slices.Clone(a.Destinations), Credentials: slices.Clone(a.Credentials)} +} + +func TestIntersectAccess(t *testing.T) { + binding := Credential{Origin: "https://a.example.com", Header: "authorization", URI: "ate-secret://k8s.io/default/team/key/token"} + a := canonicalTestAccess(t, []string{"https://a.example.com:443", "https://removed.example.com"}, binding) + changed := a.Credentials[0] + changed.Prefix = "Bearer " + b := Access{Destinations: []string{"https://a.example.com:443", "https://new.example.com:443"}, Credentials: []Credential{changed, a.Credentials[0]}} + beforeA, beforeB := cloneAccess(a), cloneAccess(b) + require.Equal(t, Access{Destinations: []string{a.Destinations[0]}, Credentials: a.Credentials}, IntersectAccess(a, b)) + b.Credentials = []Credential{changed} + require.Empty(t, IntersectAccess(a, b).Credentials, "changed credential values are additions") + b.Credentials = beforeB.Credentials + require.Equal(t, beforeA, a) + require.Equal(t, beforeB, b) +} + +func TestFilterConflicts(t *testing.T) { + binding := Credential{Origin: "https://model.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team/model/token"} + base := canonicalTestAccess(t, []string{binding.Origin}, binding) + conflict := binding + conflict.URI = "ate-secret://k8s.io/default/team/other/token" + const extra = "https://tickets.example.com:443" + for _, test := range []struct { + name string + base, grants, want Access + tokens, dropped []string + }{ + {name: "additive", base: base, grants: canonicalTestAccess(t, []string{extra}), want: canonicalTestAccess(t, []string{extra})}, + {name: "duplicate", base: base, grants: base, want: base}, + {name: "credential conflict keeps network access", base: base, grants: canonicalTestAccess(t, []string{binding.Origin, extra}, conflict), + want: canonicalTestAccess(t, []string{binding.Origin, extra}), dropped: []string{binding.Origin}}, + {name: "protocol conflict keeps other grants", base: base, grants: canonicalTestAccess(t, []string{"http://model.example.com:443", extra}), + want: canonicalTestAccess(t, []string{extra}), dropped: []string{"http://model.example.com:443"}}, + {name: "token passthrough", base: canonicalTestAccess(t, []string{binding.Origin}), tokens: []string{binding.Origin}, + grants: canonicalTestAccess(t, []string{binding.Origin, extra}, binding), want: canonicalTestAccess(t, []string{binding.Origin, extra}), dropped: []string{binding.Origin}}, + } { + t.Run(test.name, func(t *testing.T) { + beforeBase, beforeGrants := cloneAccess(test.base), cloneAccess(test.grants) + filtered, dropped := FilterConflicts(test.base, test.grants, test.tokens) + require.Equal(t, test.want, filtered) + require.Equal(t, test.dropped, dropped) + effective, err := EffectiveAccess(test.base, test.grants, test.tokens) + require.NoError(t, err) + want, err := MergeAccess(test.base, test.want, test.tokens) + require.NoError(t, err) + require.Equal(t, want, effective) + require.Equal(t, beforeBase, test.base) + require.Equal(t, beforeGrants, test.grants) + }) + } +} + +func TestFilterConflictsLimits(t *testing.T) { + var destinations []string + for i := range 255 { + destinations = append(destinations, fmt.Sprintf("https://host%03d.example.com:443", i)) + } + base := canonicalTestAccess(t, destinations) + for _, input := range [][]string{ + {"https://z.example.com", "https://a.example.com"}, + {"https://a.example.com", "https://z.example.com"}, + } { + filtered, dropped := FilterConflicts(base, canonicalTestAccess(t, input), nil) + require.Equal(t, []string{"https://a.example.com:443"}, filtered.Destinations) + require.Equal(t, []string{"https://z.example.com:443"}, dropped) + } + var credentials []Credential + for i := range 16 { + credentials = append(credentials, Credential{Origin: "https://model.example.com:443", Header: fmt.Sprintf("x-header-%02d", i), URI: "ate-secret://k8s.io/default/team/auth/token"}) + } + base = canonicalTestAccess(t, []string{credentials[0].Origin}, credentials...) + extra := credentials[0] + extra.Header = "x-new" + grants := canonicalTestAccess(t, base.Destinations, extra) + filtered, dropped := FilterConflicts(base, grants, nil) + require.Empty(t, filtered.Credentials) + require.Equal(t, base.Destinations, filtered.Destinations) + require.Equal(t, base.Destinations, dropped) +} diff --git a/go/core/internal/grpcserver/scheduledrun_test.go b/go/core/internal/grpcserver/scheduledrun_test.go index 4fc98efd31..453442b661 100644 --- a/go/core/internal/grpcserver/scheduledrun_test.go +++ b/go/core/internal/grpcserver/scheduledrun_test.go @@ -180,12 +180,13 @@ func scheduledRunTestServer(t *testing.T) (*database.Client, apiv1.ScheduledRunS t.Cleanup(db.Close) store := database.NewClient(db, "public") pair := database.AgentDefinition{Namespace: "team", AgentName: "report", AgentUID: "template-uid", DesiredRevision: "scheduled-revision"} - require.NoError(t, store.UpsertAgentDefinition(t.Context(), pair)) + upsertErr := store.UpsertAgentDefinition(t.Context(), pair) + require.NoError(t, upsertErr) require.NoError(t, store.RecordRuntimeRevision(t.Context(), database.RuntimeRevision{ Revision: pair.DesiredRevision, Namespace: pair.Namespace, AgentName: pair.AgentName, AgentUID: pair.AgentUID, SourceSnapshot: []byte("{}"), AgentCard: &a2apb.AgentCard{}, EgressDestinations: []string{}, ActorTemplateAtespace: "team", ActorTemplateName: "runtime", ActorTemplateUID: "runtime-uid", - }, true)) + }, database.RecordRevisionOptions{Ready: true})) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) agent := &v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: "team", Name: "report"}, Spec: v1alpha3.AgentSpec{ diff --git a/go/core/internal/grpcserver/task_store_test.go b/go/core/internal/grpcserver/task_store_test.go index 5f1367897e..7ae3ad4cdb 100644 --- a/go/core/internal/grpcserver/task_store_test.go +++ b/go/core/internal/grpcserver/task_store_test.go @@ -637,11 +637,12 @@ func createTaskStoreSession(t *testing.T, store *database.Client) *apiv1.Session AgentCard: &a2apb.AgentCard{Name: "assistant"}, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "assistant-kagent-revision", ActorTemplateUID: "actor-template-uid", } - require.NoError(t, store.UpsertAgentDefinition(t.Context(), database.AgentDefinition{ + upsertErr := store.UpsertAgentDefinition(t.Context(), database.AgentDefinition{ Namespace: revision.Namespace, AgentName: revision.AgentName, AgentUID: revision.AgentUID, DesiredRevision: revision.Revision, - })) - require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision, true)) + }) + require.NoError(t, upsertErr) + require.NoError(t, store.RecordRuntimeRevision(t.Context(), revision, database.RecordRevisionOptions{Ready: true})) session, _, err := store.CreateSession(t.Context(), &apiv1.Session{ Id: uuid.NewString(), Creator: "alice", diff --git a/go/core/internal/service/sandbox/guest_test.go b/go/core/internal/service/sandbox/guest_test.go index c4c2bd2daf..6450166046 100644 --- a/go/core/internal/service/sandbox/guest_test.go +++ b/go/core/internal/service/sandbox/guest_test.go @@ -82,7 +82,7 @@ func (a *testActors) CreateActor(_ context.Context, space, name, templateSpace, Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}} return proto.CloneOf(a.actor), a.mutationErr } -func (a *testActors) EnsureActorEgressPolicy(context.Context, string, string, *ateapipb.EgressPolicy) error { +func (a *testActors) EnsureActorEgressPolicy(context.Context, string, string, string) error { a.mu.Lock() defer a.mu.Unlock() a.calls = append(a.calls, "policy") @@ -129,11 +129,12 @@ func guestFixture(t *testing.T) (*sandboxservice.Service, context.Context, *test require.NoError(t, err) t.Cleanup(pool.Close) store := database.NewClient(pool, "public") - require.NoError(t, store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"})) + upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"}) + require.NoError(t, upsertErr) require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - }, true)) + }, database.RecordRevisionOptions{Ready: true})) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&v1alpha3.SandboxTemplate{ diff --git a/go/core/internal/service/sandbox/service_test.go b/go/core/internal/service/sandbox/service_test.go index f3de53197d..f7b5801d87 100644 --- a/go/core/internal/service/sandbox/service_test.go +++ b/go/core/internal/service/sandbox/service_test.go @@ -45,7 +45,7 @@ type testActors struct { substrate.LifecycleClient mu sync.Mutex actor *ateapipb.Actor - egressPolicy *ateapipb.EgressPolicy + egressRevision string calls []string readErr error mutationErr error @@ -73,11 +73,11 @@ func (a *testActors) CreateActor(_ context.Context, space, name, templateSpace, Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}} return proto.CloneOf(a.actor), a.mutationErr } -func (a *testActors) EnsureActorEgressPolicy(_ context.Context, _, _ string, policy *ateapipb.EgressPolicy) error { +func (a *testActors) EnsureActorEgressPolicy(_ context.Context, _, _ string, revision string) error { a.mu.Lock() defer a.mu.Unlock() a.calls = append(a.calls, "policy") - a.egressPolicy = proto.CloneOf(policy) + a.egressRevision = revision return a.mutationErr } func (a *testActors) ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) { @@ -134,11 +134,12 @@ func serviceFixture(t *testing.T) (*Service, context.Context, *testActors) { require.NoError(t, err) t.Cleanup(pool.Close) store := database.NewClient(pool, "public") - require.NoError(t, store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"})) + upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: "revision"}) + require.NoError(t, upsertErr) require.NoError(t, store.RecordSandboxRevision(ctx, database.SandboxRevision{ RuntimeArtifact: database.RuntimeArtifact{Revision: "revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "revision", ActorTemplateUID: "revision-uid"}, SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), - }, true)) + }, database.RecordRevisionOptions{Ready: true})) scheme := runtime.NewScheme() require.NoError(t, v1alpha3.AddToScheme(scheme)) kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&v1alpha3.SandboxTemplate{ @@ -215,8 +216,7 @@ func TestSandboxLifecycleClientRetriesAfterRestart(t *testing.T) { } } require.Equal(t, 1, count, "retry must preserve the original Actor and files") - require.NotNil(t, actors.egressPolicy) - require.Empty(t, actors.egressPolicy.Rules) + require.NotEmpty(t, actors.egressRevision) }) } } @@ -285,5 +285,24 @@ func TestSandboxConflictingRequestCanRetryAfterActiveAttempt(t *testing.T) { require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, current.State) require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, current.Operation) - require.Equal(t, []string{"create", "policy", "resume", "suspend", "resume"}, actors.observedCalls()) + require.Equal(t, []string{"create", "policy", "resume", "suspend", "policy", "resume"}, actors.observedCalls()) +} + +func TestSandboxResolvesCurrentGrantsBeforeResume(t *testing.T) { + service, ctx, actors := serviceFixture(t) + revision := database.SandboxRevision{ + RuntimeArtifact: database.RuntimeArtifact{Revision: "policy-revision", Kind: "sandbox", Namespace: "team-a", ActorTemplateAtespace: "team-a", ActorTemplateName: "policy-runtime", ActorTemplateUID: "policy-runtime-uid"}, + SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", SourceSnapshot: []byte("{}"), + } + store := service.config.Store + upsertErr := store.UpsertSandboxTemplateDefinition(ctx, database.SandboxTemplateDefinition{Namespace: "team-a", SandboxTemplateName: "scratch", SandboxTemplateUID: "template-uid", DesiredRevision: revision.Revision}) + require.NoError(t, upsertErr) + require.NoError(t, store.RecordSandboxRevision(ctx, revision, database.RecordRevisionOptions{Ready: true})) + instance, err := service.Create(ctx, createRequest()) + require.NoError(t, err) + require.Equal(t, revision.Revision, instance.PreparedRevision) + require.Equal(t, revision.Revision, actors.egressRevision) + calls := actors.observedCalls() + require.Less(t, slices.Index(calls, "policy"), slices.Index(calls, "resume")) + require.Contains(t, calls, "policy") } diff --git a/go/core/internal/service/sandbox/transports_test.go b/go/core/internal/service/sandbox/transports_test.go index eca00727d0..0874a3f9f0 100644 --- a/go/core/internal/service/sandbox/transports_test.go +++ b/go/core/internal/service/sandbox/transports_test.go @@ -250,5 +250,5 @@ func TestSandboxTransports(t *testing.T) { require.NoError(t, err) _, err = client.DeleteSandbox(ctx, &apiv1.DeleteSandboxRequest{SandboxId: id}) require.NoError(t, err) - require.Equal(t, []string{"create", "policy", "resume", "suspend", "suspend", "resume", "suspend", "delete"}, actors.observedCalls()) + require.Equal(t, []string{"create", "policy", "resume", "suspend", "suspend", "policy", "resume", "suspend", "delete"}, actors.observedCalls()) } diff --git a/go/core/internal/service/sandbox/workflow.go b/go/core/internal/service/sandbox/workflow.go index e285492684..fc1df12f15 100644 --- a/go/core/internal/service/sandbox/workflow.go +++ b/go/core/internal/service/sandbox/workflow.go @@ -39,15 +39,11 @@ func (s *Service) run(ctx context.Context, id string, kind apiv1.RuntimeOperatio if err != nil { return nil, sandboxError(err) } - binding := substrate.ActorBinding{Atespace: revision.ActorTemplateAtespace, Name: substrate.ActorName(id), + binding := substrate.ActorBinding{Revision: revision.Revision, Atespace: revision.ActorTemplateAtespace, Name: substrate.ActorName(id), TemplateAtespace: revision.ActorTemplateAtespace, TemplateName: revision.ActorTemplateName} var creation *substrate.ActorCreation if kind == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { - policy, err := substrate.ActorEgressPolicy(binding.Atespace, nil, nil) - if err != nil { - return nil, sandboxError(err) - } - creation = &substrate.ActorCreation{EgressPolicy: policy, Resume: true} + creation = &substrate.ActorCreation{Resume: true} } prepare := substrate.PrepareActorTransition if op.ExecutorID != uuid.Nil { diff --git a/go/core/internal/service/session/workflow.go b/go/core/internal/service/session/workflow.go index 63a0517ead..805e073bb4 100644 --- a/go/core/internal/service/session/workflow.go +++ b/go/core/internal/service/session/workflow.go @@ -158,15 +158,11 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session tagName = "checkpoint-" + operation.SourceCheckpointID.String() } - binding := substrate.ActorBinding{Atespace: revision.ActorTemplateAtespace, Name: substrate.ActorName(session.Id), + binding := substrate.ActorBinding{Revision: revision.Revision, Atespace: revision.ActorTemplateAtespace, Name: substrate.ActorName(session.Id), TemplateAtespace: revision.ActorTemplateAtespace, TemplateName: revision.ActorTemplateName} var creation *substrate.ActorCreation if kind == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { - policy, err := substrate.ActorEgressPolicy(binding.Atespace, revision.EgressDestinations, revision.Credentials) - if err != nil { - return w.failPreparation(ctx, operation, err) - } - creation = &substrate.ActorCreation{EgressPolicy: policy} + creation = &substrate.ActorCreation{} if snapshot != nil { creation.Snapshot = &substrate.ActorSnapshot{Tag: &ateapipb.ObjectRef{Atespace: snapshot.Atespace, Name: tagName}, URI: snapshot.URI, ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA} } diff --git a/go/core/internal/service/session/workflow_test.go b/go/core/internal/service/session/workflow_test.go index ff9439c3a4..3f073eff0e 100644 --- a/go/core/internal/service/session/workflow_test.go +++ b/go/core/internal/service/session/workflow_test.go @@ -148,8 +148,9 @@ func lifecycleFixture(t *testing.T) (*lifecycleTestStore, *apiv1.Session) { AgentCard: &a2apb.AgentCard{Name: "assistant"}, EgressDestinations: []string{}, ActorTemplateAtespace: "team-a", ActorTemplateName: "assistant-kagent-revision", ActorTemplateUID: "actor-template-uid", } - require.NoError(t, client.UpsertAgentDefinition(t.Context(), database.AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision})) - require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision, true)) + upsertErr := client.UpsertAgentDefinition(t.Context(), database.AgentDefinition{Namespace: "team-a", AgentName: "assistant", AgentUID: "template-uid", DesiredRevision: revision.Revision}) + require.NoError(t, upsertErr) + require.NoError(t, client.RecordRuntimeRevision(t.Context(), *revision, database.RecordRevisionOptions{Ready: true})) session, _, err := client.CreateSession(t.Context(), &apiv1.Session{Id: uuid.NewString(), Creator: "alice", Agent: &apiv1.ResourceReference{Namespace: "team-a", Name: "assistant"}}, uuid.NewString()) require.NoError(t, err) return &lifecycleTestStore{Client: client, revision: revision}, session @@ -165,12 +166,12 @@ func (s *lifecycleTestStore) GetRuntimeRevision(context.Context, string) (*datab } type lifecycleTestActors struct { - mu sync.Mutex - actors map[string]*ateapipb.Actor - policyErr error - policy *ateapipb.EgressPolicy - policyActor string - policyCalls int + mu sync.Mutex + actors map[string]*ateapipb.Actor + policyErr error + policyRevision string + policyActor string + policyCalls int } func actorKey(atespace, name string) string { return atespace + "/" + name } @@ -284,11 +285,11 @@ func lifecycleForkFixture(t *testing.T, store *lifecycleTestStore, actors *lifec return fork, checkpoint.Id } -func (a *lifecycleTestActors) EnsureActorEgressPolicy(_ context.Context, atespace, name string, policy *ateapipb.EgressPolicy) error { +func (a *lifecycleTestActors) EnsureActorEgressPolicy(_ context.Context, atespace, name string, revision string) error { a.mu.Lock() defer a.mu.Unlock() a.policyActor = actorKey(atespace, name) - a.policy = proto.CloneOf(policy) + a.policyRevision = revision a.policyCalls++ return a.policyErr } @@ -304,27 +305,15 @@ func TestActorCreationRetainsEgressPolicyFailure(t *testing.T) { actors := &retryTestActors{lifecycleTestActors: base} workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second) callsBefore := base.policyCalls - store.revision.EgressDestinations = []string{"*"} - _, err := workflow.Create(t.Context(), session) - require.ErrorContains(t, err, "invalid egress destination") - require.Zero(t, actors.mutations.Load(), "validate the allowlist before issuing Actor creation") - require.Equal(t, callsBefore, base.policyCalls) - - store.revision.EgressDestinations = []string{"https://api.example.com:443", "https://other.example.com:8443"} - store.revision.Credentials = []egress.Credential{{Hostname: "api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team-a/auth/token"}} base.policyErr = context.DeadlineExceeded - _, err = workflow.Create(t.Context(), session) + _, err := workflow.Create(t.Context(), session) require.ErrorIs(t, err, context.DeadlineExceeded) current, err := store.GetSessionByID(t.Context(), session.Id) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_CREATING, current.State) require.Empty(t, current.A2AAuthority) require.Equal(t, actorKey("team-a", substrate.ActorName(session.Id)), base.policyActor) - require.Equal(t, &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "default"}, base.policy.Metadata) - require.Len(t, base.policy.Rules, 2) - require.Equal(t, &ateapipb.CredentialHeader{Header: "authorization", Prefix: "Bearer ", CredentialUri: "ate-secret://k8s.io/default/team-a/auth/token"}, base.policy.Rules[0].GetHttps().GetEffects().GetReplaceHeaders()[0]) - require.Equal(t, []string{"api.example.com"}, base.policy.Rules[0].GetHttps().GetHostnames()) - require.Equal(t, []string{"other.example.com"}, base.policy.Rules[1].GetHttps().GetHostnames()) + require.Equal(t, session.PreparedRevision, base.policyRevision) // Retry completes policy setup for the existing Actor before readiness. base.policyErr = nil @@ -349,7 +338,7 @@ func TestActorEgressPolicy(t *testing.T) { policy, err = substrate.ActorEgressPolicy("team-a", nil, nil) require.NoError(t, err) require.Empty(t, policy.Rules, "no destinations must deny all egress") - for _, destination := range []string{"", "*", "api.example.com", "https://api.example.com/path", "http://api.example.com:0", "http://api.example.com:65536", "http://user@api.example.com", "http://api.example.com?key=value", "http://*.example.com"} { + for _, destination := range []string{"", "*", "api.example.com", "https://api.example.com/path", "http://api.example.com:0", "http://api.example.com:65536", "http://user@api.example.com", "http://api.example.com?key=value"} { t.Run(destination, func(t *testing.T) { _, err := substrate.ActorEgressPolicy("team-a", []string{destination}, nil) require.Error(t, err) @@ -357,7 +346,7 @@ func TestActorEgressPolicy(t *testing.T) { } for _, destination := range []string{"http://192.0.2.1", "http://[2001:db8::1]", "http://[::ffff:192.0.2.1]"} { _, err := substrate.ActorEgressPolicy("team-a", []string{destination}, nil) - require.ErrorContains(t, err, "requires a DNS name") + require.ErrorContains(t, err, "expected a DNS name") } _, err = substrate.ActorEgressPolicy("team-a", []string{"http://api.example.com:8443", "https://api.example.com:8443"}, nil) require.ErrorContains(t, err, "both HTTP and HTTPS on the same port") @@ -365,15 +354,15 @@ func TestActorEgressPolicy(t *testing.T) { func TestActorEgressCredentialsRequireAllowedDestination(t *testing.T) { bindings := []egress.Credential{ - {Hostname: "api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}, - {Hostname: "api.example.com", Header: "x-api-key", URI: "ate-secret://k8s.io/default/team/auth/key"}, + {Origin: "https://api.example.com:8443", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}, + {Origin: "https://api.example.com:8443", Header: "x-api-key", URI: "ate-secret://k8s.io/default/team/auth/key"}, } _, err := substrate.ActorEgressPolicy("team", []string{"https://other.example.com"}, bindings) - require.ErrorContains(t, err, "is not allowed") + require.ErrorContains(t, err, "is not explicitly allowed") policy, err := substrate.ActorEgressPolicy("team", []string{"http://api.example.com:8080", "https://api.example.com:8443", "https://other.example.com"}, bindings) require.NoError(t, err) require.Len(t, policy.Rules, 3) - require.Len(t, policy.Rules[0].GetHttp().GetEffects().GetReplaceHeaders(), 2, "configured HTTP endpoints also require credential replacement") + require.Empty(t, policy.Rules[0].GetHttp().GetEffects().GetReplaceHeaders(), "credentials must not cross schemes or ports") require.Equal(t, []string{"api.example.com"}, policy.Rules[1].GetHttps().GetHostnames()) require.Equal(t, []int32{8443}, policy.Rules[1].GetHttps().GetPorts().GetNumbers()) require.Len(t, policy.Rules[1].GetHttps().GetEffects().GetReplaceHeaders(), 2) diff --git a/go/core/internal/substrate/config.go b/go/core/internal/substrate/config.go index 55939730fb..8699c200c3 100644 --- a/go/core/internal/substrate/config.go +++ b/go/core/internal/substrate/config.go @@ -1,9 +1,14 @@ package substrate -import "time" +import ( + "time" + + "github.com/kagent-dev/kagent/go/core/internal/egress" +) // Config holds connection settings for Agent Substrate ate-api. type Config struct { + EgressPolicies egress.PolicyStore // AteAPIEndpoint is a gRPC target (e.g. dns:///api.ate-system.svc:443). AteAPIEndpoint string // CAFile contains the CA certificates used to verify ate-api. diff --git a/go/core/internal/substrate/egress.go b/go/core/internal/substrate/egress.go index dd7d0be647..f09fbf6fae 100644 --- a/go/core/internal/substrate/egress.go +++ b/go/core/internal/substrate/egress.go @@ -3,6 +3,7 @@ package substrate import ( "context" "fmt" + "slices" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -10,25 +11,54 @@ import ( "google.golang.org/protobuf/proto" ) -// EnsureActorEgressPolicy can be retried after a lost create response. A prepared -// revision is immutable, so an existing policy must match; never accept or -// overwrite a different allowlist. Substrate deletes the policy with its Actor. -func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name string, policy *ateapipb.EgressPolicy) error { +// EnsureActorEgressPolicy installs current authorization without restarting the +// actor. Read the backend version before the controller's persisted inputs so +// concurrent applications cannot overwrite a newer policy with an old one. +// Conflicts reload both; lifecycle paths never translate Kubernetes. +// Substrate deletes the policy with its Actor. +func (c *Client) EnsureActorEgressPolicy(ctx context.Context, atespace, name, revision string) error { ctx, cancel := c.callCtx(ctx) defer cancel() - actor := actorRef(atespace, name) - _, err := c.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{ - Actor: actor, EgressPolicy: policy, - }) - if status.Code(err) != codes.AlreadyExists { - return err - } - existing, err := c.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) - if err != nil { - return err + if c.cfg.EgressPolicies == nil { + return fmt.Errorf("actor egress policy store is required") } - if !proto.Equal(&ateapipb.EgressPolicy{Rules: existing.GetRules()}, &ateapipb.EgressPolicy{Rules: policy.Rules}) { - return fmt.Errorf("existing Actor egress policy does not match the prepared revision") + actor := actorRef(atespace, name) + for range 5 { + existing, err := c.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) + if err != nil && status.Code(err) != codes.NotFound { + return err + } + access, err := c.cfg.EgressPolicies.GetRuntimeEgressPolicy(ctx, revision) + if err != nil { + return fmt.Errorf("load intended actor egress: %w", err) + } + policy, err := ActorEgressPolicy(atespace, access.Destinations, access.Credentials) + if err != nil { + return err + } + if existing == nil { + _, err = c.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{Actor: actor, EgressPolicy: policy}) + } else { + if !proto.Equal(&ateapipb.EgressPolicy{Rules: existing.GetRules()}, &ateapipb.EgressPolicy{Rules: policy.GetRules()}) { + policy.Metadata = proto.CloneOf(existing.GetMetadata()) + _, err = c.UpdateActorEgressPolicy(ctx, &ateapipb.UpdateActorEgressPolicyRequest{Actor: actor, EgressPolicy: policy}) + } + } + if status.Code(err) == codes.AlreadyExists || status.Code(err) == codes.Aborted { + continue + } + if err != nil { + return err + } + // A source update may have discovered this instance before its Actor + // existed. Recheck after applying so creation cannot miss that update. + latest, err := c.cfg.EgressPolicies.GetRuntimeEgressPolicy(ctx, revision) + if err != nil { + return fmt.Errorf("reload intended actor egress: %w", err) + } + if slices.Equal(access.Destinations, latest.Destinations) && slices.Equal(access.Credentials, latest.Credentials) { + return nil + } } - return nil + return status.Error(codes.Aborted, "actor egress policy changed during reconciliation") } diff --git a/go/core/internal/substrate/egress_test.go b/go/core/internal/substrate/egress_test.go index 79f4e2484b..4e35afb850 100644 --- a/go/core/internal/substrate/egress_test.go +++ b/go/core/internal/substrate/egress_test.go @@ -2,6 +2,7 @@ package substrate import ( "context" + "errors" "testing" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -10,28 +11,69 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + + "github.com/kagent-dev/kagent/go/core/internal/egress" ) -func TestEnsureActorEgressPolicyRetriesLostResponse(t *testing.T) { +type egressPolicyStoreFunc func(context.Context, string) (egress.Access, error) + +func (f egressPolicyStoreFunc) GetRuntimeEgressPolicy(ctx context.Context, revision string) (egress.Access, error) { + return f(ctx, revision) +} + +var _ egress.PolicyStore = egressPolicyStoreFunc(nil) + +func TestEnsureActorEgressPolicyUpdatesCurrentGrants(t *testing.T) { fake := &egressPolicyFake{createErr: context.DeadlineExceeded} - client := &Client{ControlClient: fake} - policy := &ateapipb.EgressPolicy{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "default"}, - Rules: []*ateapipb.EgressRule{ - {Http: &ateapipb.HTTPRule{Hostnames: []string{"api.example.com"}}}, - {Https: &ateapipb.HTTPSRule{Hostnames: []string{"api.example.com"}}}, - }, - } - require.ErrorIs(t, client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", policy), context.DeadlineExceeded) + access := egress.Access{Destinations: []string{"https://api.example.com:443"}} + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(_ context.Context, revision string) (egress.Access, error) { + require.Equal(t, "pinned-revision", revision) + return access, nil + })}} + reconcile := func() error { + return client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "pinned-revision") + } + require.ErrorIs(t, reconcile(), context.DeadlineExceeded) require.NotNil(t, fake.policy, "the server committed before the response was lost") fake.createErr = nil - require.NoError(t, client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", policy)) - require.NoError(t, client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", policy)) + require.NoError(t, reconcile()) + require.NoError(t, reconcile()) require.Equal(t, 1, fake.created) + require.Zero(t, fake.updated) require.Equal(t, actorRef("team-a", "actor"), fake.actor) - require.ErrorContains(t, client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", &ateapipb.EgressPolicy{}), "does not match") + + access = egress.Access{} + require.NoError(t, reconcile()) + require.Empty(t, fake.policy.Rules, "removing the grant revokes the existing actor") + require.Equal(t, 1, fake.updated) + access = egress.Access{Destinations: []string{"https://api.example.com:443"}, Credentials: []egress.Credential{{Origin: "https://api.example.com:443", Header: "authorization", URI: "ate-secret://k8s.io/default/team-a/auth/token", Prefix: "Bearer "}}} + require.NoError(t, reconcile()) + require.Equal(t, "Bearer ", fake.policy.Rules[0].GetHttps().GetEffects().GetReplaceHeaders()[0].Prefix) + access.Credentials[0].URI = "ate-secret://k8s.io/default/team-a/rotated/token" + require.NoError(t, reconcile()) + require.Equal(t, access.Credentials[0].URI, fake.policy.Rules[0].GetHttps().GetEffects().GetReplaceHeaders()[0].CredentialUri) fake.getErr = status.Error(codes.Unavailable, "unavailable") - require.Equal(t, codes.Unavailable, status.Code(client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", policy))) + require.Equal(t, codes.Unavailable, status.Code(reconcile())) +} + +func TestEnsureActorEgressPolicyReloadsAfterConflict(t *testing.T) { + fake := &egressPolicyFake{} + access := egress.Access{} + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Access, error) { return access, nil })}} + reconcile := func() error { + return client.EnsureActorEgressPolicy(t.Context(), "team-a", "actor", "revision") + } + require.NoError(t, reconcile()) + access.Destinations = []string{"https://revoked.example.com:443"} + fake.beforeUpdate = func() { + // Another reconciliation committed revocation after this attempt read the + // backend version. Retrying must resolve current grants, not replay the allow. + access = egress.Access{} + fake.policy.Metadata.Version++ + } + require.NoError(t, reconcile()) + require.Empty(t, fake.policy.Rules) + require.Zero(t, fake.updated) } type egressPolicyFake struct { @@ -39,7 +81,9 @@ type egressPolicyFake struct { policy *ateapipb.EgressPolicy actor *ateapipb.ObjectRef createErr, getErr error - created int + created, updated int + beforeUpdate func() + afterCreate func() } func (f *egressPolicyFake) CreateActorEgressPolicy(_ context.Context, req *ateapipb.CreateActorEgressPolicyRequest, _ ...grpc.CallOption) (*ateapipb.EgressPolicy, error) { @@ -51,10 +95,59 @@ func (f *egressPolicyFake) CreateActorEgressPolicy(_ context.Context, req *ateap f.policy.Metadata.Uid = "policy-uid" f.policy.Metadata.Version = 1 f.created++ - return f.policy, f.createErr + if f.afterCreate != nil { + f.afterCreate() + } + return proto.CloneOf(f.policy), f.createErr } func (f *egressPolicyFake) GetActorEgressPolicy(_ context.Context, req *ateapipb.GetActorEgressPolicyRequest, _ ...grpc.CallOption) (*ateapipb.EgressPolicy, error) { f.actor = req.Actor - return f.policy, f.getErr + if f.getErr != nil { + return nil, f.getErr + } + if f.policy == nil { + return nil, status.Error(codes.NotFound, "missing") + } + return proto.CloneOf(f.policy), nil +} + +func (f *egressPolicyFake) UpdateActorEgressPolicy(_ context.Context, req *ateapipb.UpdateActorEgressPolicyRequest, _ ...grpc.CallOption) (*ateapipb.EgressPolicy, error) { + if f.beforeUpdate != nil { + f.beforeUpdate() + f.beforeUpdate = nil + } + if req.EgressPolicy.Metadata.Version != f.policy.Metadata.Version || req.EgressPolicy.Metadata.Uid != f.policy.Metadata.Uid { + return nil, status.Error(codes.Aborted, "version conflict") + } + f.policy = proto.CloneOf(req.EgressPolicy) + f.policy.Metadata.Version++ + f.updated++ + return proto.CloneOf(f.policy), nil +} + +func TestEnsureActorEgressPolicyCatchesUpdateDuringCreation(t *testing.T) { + fake := &egressPolicyFake{} + access := egress.Access{Destinations: []string{"https://revoked.example.com:443"}} + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Access, error) { return access, nil })}} + fake.afterCreate = func() { + // The controller published revocation, but its actor discovery ran before + // creation. There is no concurrent Substrate write to cause a CAS conflict. + access = egress.Access{} + } + err := client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision") + require.NoError(t, err) + require.Empty(t, fake.policy.Rules) + require.Equal(t, 1, fake.created) + require.Equal(t, 1, fake.updated) +} + +func TestEnsureActorEgressPolicyRequiresPersistedPolicy(t *testing.T) { + fake := &egressPolicyFake{} + unavailable := errors.New("intended policy unavailable") + client := &Client{ControlClient: fake, cfg: Config{EgressPolicies: egressPolicyStoreFunc(func(context.Context, string) (egress.Access, error) { return egress.Access{}, unavailable })}} + err := client.EnsureActorEgressPolicy(t.Context(), "team", "actor", "revision") + require.ErrorIs(t, err, unavailable) + require.Zero(t, fake.created) + require.Zero(t, fake.updated) } diff --git a/go/core/internal/substrate/lifecycle.go b/go/core/internal/substrate/lifecycle.go index 169647eded..f42c40a7d9 100644 --- a/go/core/internal/substrate/lifecycle.go +++ b/go/core/internal/substrate/lifecycle.go @@ -16,7 +16,7 @@ type LifecycleClient interface { GetActor(context.Context, string, string) (*ateapipb.Actor, error) CreateActor(context.Context, string, string, string, string) (*ateapipb.Actor, error) CreateActorFromTag(context.Context, string, string, string, string, string, string) (*ateapipb.Actor, error) - EnsureActorEgressPolicy(context.Context, string, string, *ateapipb.EgressPolicy) error + EnsureActorEgressPolicy(context.Context, string, string, string) error ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) SuspendActor(context.Context, string, string) (*ateapipb.Actor, error) DeleteActor(context.Context, string, string) error @@ -25,6 +25,7 @@ type LifecycleClient interface { var _ LifecycleClient = (*Client)(nil) type ActorBinding struct { + Revision string Atespace string Name string TemplateAtespace string @@ -40,9 +41,8 @@ type ActorSnapshot struct { // ActorCreation describes runtime startup, independent of the owning API. // Agent creation stays suspended; a standalone guest starts serving immediately. type ActorCreation struct { - EgressPolicy *ateapipb.EgressPolicy - Snapshot *ActorSnapshot - Resume bool + Snapshot *ActorSnapshot + Resume bool } type ActorTransition struct { @@ -75,8 +75,8 @@ func prepareActorTransition(ctx context.Context, actors LifecycleClient, binding return nil, fmt.Errorf("invalid actor lifecycle operation %s", operation) } if operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { - if creation == nil || creation.EgressPolicy == nil { - return nil, fmt.Errorf("actor creation requires an egress policy") + if creation == nil || binding.Revision == "" { + return nil, fmt.Errorf("actor creation requires a prepared revision") } if snapshot := creation.Snapshot; snapshot != nil && (snapshot.Tag.GetAtespace() == "" || snapshot.Tag.GetName() == "" || snapshot.URI == "" || snapshot.ContentScope != ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA) { return nil, fmt.Errorf("actor restoration requires a retained DATA snapshot") @@ -96,7 +96,7 @@ func prepareActorTransition(ctx context.Context, actors LifecycleClient, binding if operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE && !reconcile { return nil, fmt.Errorf("refuse to adopt existing Actor %s/%s while creation is still pending", binding.Atespace, binding.Name) } - if !binding.matches(actor) { + if !binding.Matches(actor) { return nil, fmt.Errorf("actor %s/%s identity or template changed", binding.Atespace, binding.Name) } if operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME || operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND { @@ -144,7 +144,7 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio } } if err == nil { - err = actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, creation.EgressPolicy) + err = actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, binding.Revision) } if err == nil && creation.Resume && actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_RUNNING { actor, err = actors.ResumeActor(ctx, binding.Atespace, binding.Name) @@ -153,6 +153,9 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio } } case apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME: + if err := actors.EnsureActorEgressPolicy(ctx, binding.Atespace, binding.Name, binding.Revision); err != nil { + return err + } actor, err = actors.ResumeActor(ctx, binding.Atespace, binding.Name) if err == nil { err = transition.checkResult(actor, ateapipb.ActorState_ACTOR_STATE_RUNNING) @@ -192,13 +195,14 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio return nil } -func (b ActorBinding) matches(actor *ateapipb.Actor) bool { +// Matches checks that the actor belongs to the expected runtime and template. +func (b ActorBinding) Matches(actor *ateapipb.Actor) bool { return actor.GetMetadata().GetName() == b.Name && actor.GetMetadata().GetAtespace() == b.Atespace && actor.GetMetadata().GetUid() != "" && actor.GetActorTemplate().GetAtespace() == b.TemplateAtespace && actor.GetActorTemplate().GetName() == b.TemplateName } func (t *ActorTransition) checkResult(actor *ateapipb.Actor, state ateapipb.ActorState) error { - if !t.binding.matches(actor) || actor.GetStatus().GetState() != state || (t.actor != nil && actor.GetMetadata().GetUid() != t.actor.GetMetadata().GetUid()) { + if !t.binding.Matches(actor) || actor.GetStatus().GetState() != state || (t.actor != nil && actor.GetMetadata().GetUid() != t.actor.GetMetadata().GetUid()) { return fmt.Errorf("actor returned unexpected identity or state") } return nil diff --git a/go/core/internal/substrate/lifecycle_test.go b/go/core/internal/substrate/lifecycle_test.go index 6abfa64346..a4e1301b25 100644 --- a/go/core/internal/substrate/lifecycle_test.go +++ b/go/core/internal/substrate/lifecycle_test.go @@ -41,7 +41,7 @@ func (a *transitionActors) CreateActor(_ context.Context, space, name, templateS return proto.CloneOf(a.actor), nil } -func (a *transitionActors) EnsureActorEgressPolicy(context.Context, string, string, *ateapipb.EgressPolicy) error { +func (a *transitionActors) EnsureActorEgressPolicy(context.Context, string, string, string) error { a.calls = append(a.calls, "policy") return a.policyErr } @@ -88,9 +88,9 @@ func TestActorCreationStartupPolicy(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { actors := &transitionActors{policyErr: test.policyErr, resumeUID: test.resumeUID} - binding := ActorBinding{Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} + binding := ActorBinding{Revision: "revision", Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} transition, err := PrepareActorTransition(t.Context(), actors, binding, apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, - &ActorCreation{EgressPolicy: &ateapipb.EgressPolicy{}, Resume: test.resume}) + &ActorCreation{Resume: test.resume}) require.NoError(t, err) require.Empty(t, actors.calls, "preparation must not mutate the backend") err = ApplyActorTransition(t.Context(), actors, transition) @@ -118,8 +118,8 @@ func TestActorRetryContinuesExistingCreation(t *testing.T) { Status: &ateapipb.ActorStatus{State: state}, }} transition, err := PrepareActorRetry(t.Context(), actors, - ActorBinding{Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"}, - apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, &ActorCreation{EgressPolicy: &ateapipb.EgressPolicy{}, Resume: true}) + ActorBinding{Revision: "revision", Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"}, + apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, &ActorCreation{Resume: true}) require.NoError(t, err) require.NoError(t, ApplyActorTransition(t.Context(), actors, transition)) require.Equal(t, "original", actors.actor.Metadata.Uid) @@ -152,8 +152,8 @@ func TestActorTransitionAdmission(t *testing.T) { if test.mutate != nil { test.mutate(actors.actor) } - binding := ActorBinding{Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} - _, err = PrepareActorTransition(t.Context(), actors, binding, test.operation, &ActorCreation{EgressPolicy: &ateapipb.EgressPolicy{}}) + binding := ActorBinding{Revision: "revision", Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} + _, err = PrepareActorTransition(t.Context(), actors, binding, test.operation, &ActorCreation{}) require.Error(t, err) require.Empty(t, actors.calls) }) @@ -166,7 +166,7 @@ func TestActorDeleteStopsRunningCompute(t *testing.T) { require.NoError(t, err) actors.actor.Status.State = ateapipb.ActorState_ACTOR_STATE_RUNNING actors.calls = nil - binding := ActorBinding{Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} + binding := ActorBinding{Revision: "revision", Atespace: "team-a", Name: "instance", TemplateAtespace: "team-a", TemplateName: "revision"} transition, err := PrepareActorTransition(t.Context(), actors, binding, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE, nil) require.NoError(t, err) require.NoError(t, ApplyActorTransition(t.Context(), actors, transition)) @@ -191,7 +191,7 @@ func TestActorSuspendRequiresDataSnapshot(t *testing.T) { actors := &transitionActors{suspendSnapshot: test.snapshot} _, err := actors.CreateActor(t.Context(), "team", "instance", "team", "revision") require.NoError(t, err) - transition, err := PrepareActorTransition(t.Context(), actors, ActorBinding{Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"}, apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND, nil) + transition, err := PrepareActorTransition(t.Context(), actors, ActorBinding{Revision: "revision", Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"}, apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND, nil) require.NoError(t, err) require.ErrorContains(t, ApplyActorTransition(t.Context(), actors, transition), "external DATA snapshot") }) diff --git a/go/core/internal/substrate/policy.go b/go/core/internal/substrate/policy.go index 3fe905fb6e..9ab46ea20e 100644 --- a/go/core/internal/substrate/policy.go +++ b/go/core/internal/substrate/policy.go @@ -1,16 +1,9 @@ package substrate import ( - "fmt" - "net" - "net/netip" - "net/url" - "slices" "strconv" - "strings" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "k8s.io/apimachinery/pkg/util/validation" "github.com/kagent-dev/kagent/go/core/internal/egress" ) @@ -18,71 +11,29 @@ import ( // ActorEgressPolicy compiles HTTP(S) origins into an actor's default allowlist. // Credential bindings are already canonicalized by the store. func ActorEgressPolicy(atespace string, destinations []string, credentials []egress.Credential) (*ateapipb.EgressPolicy, error) { - type originTarget struct { - scheme, hostname string - port int32 - } - origins := map[string]originTarget{} - protocols := map[string]string{} - hosts := map[string]bool{} - for _, destination := range destinations { - u, err := url.Parse(destination) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.Hostname() == "" || u.ForceQuery || u.RawQuery != "" || u.Fragment != "" || u.Path != "" { - return nil, fmt.Errorf("invalid egress destination %q: expected an HTTP(S) origin", destination) - } - host := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") - if _, err := netip.ParseAddr(host); err == nil { - return nil, fmt.Errorf("egress destination %q requires a DNS name: Substrate does not support IP allowlists", destination) - } - if len(validation.IsDNS1123Subdomain(host)) != 0 { - return nil, fmt.Errorf("invalid egress destination %q", destination) - } - origin := egress.Origin(u) - u, err = url.Parse(origin) - if err != nil { - return nil, fmt.Errorf("invalid egress destination %q: %w", destination, err) - } - port, err := strconv.ParseUint(u.Port(), 10, 16) - if err != nil || port == 0 { - return nil, fmt.Errorf("invalid egress port in %q", destination) - } - u.Host = net.JoinHostPort(host, strconv.FormatUint(port, 10)) - if protocol, found := protocols[u.Host]; found && protocol != u.Scheme { - return nil, fmt.Errorf("egress destination %q uses both HTTP and HTTPS on the same port", u.Host) - } - protocols[u.Host] = u.Scheme - hosts[host] = true - origins[u.String()] = originTarget{scheme: u.Scheme, hostname: host, port: int32(port)} + access, err := egress.CanonicalAccess(destinations, credentials, nil) + if err != nil { + return nil, err } effects := map[string]*ateapipb.HttpRuleEffects{} - for _, binding := range credentials { - if !hosts[binding.Hostname] { - return nil, fmt.Errorf("credential destination %q is not allowed", binding.Hostname) - } - if effects[binding.Hostname] == nil { - effects[binding.Hostname] = &ateapipb.HttpRuleEffects{} + for _, binding := range access.Credentials { + if effects[binding.Origin] == nil { + effects[binding.Origin] = &ateapipb.HttpRuleEffects{} } - effects[binding.Hostname].ReplaceHeaders = append(effects[binding.Hostname].ReplaceHeaders, &ateapipb.CredentialHeader{Header: binding.Header, Prefix: binding.Prefix, CredentialUri: binding.URI}) + effects[binding.Origin].ReplaceHeaders = append(effects[binding.Origin].ReplaceHeaders, &ateapipb.CredentialHeader{Header: binding.Header, Prefix: binding.Prefix, CredentialUri: binding.URI}) } - names := make([]string, 0, len(origins)) - for origin := range origins { - names = append(names, origin) - } - slices.Sort(names) policy := &ateapipb.EgressPolicy{Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: "default"}} - for _, origin := range names { - target := origins[origin] - ports := &ateapipb.Ports{Numbers: []int32{target.port}} - // Keep exactly one rule per host and port so an overlapping rule cannot - // bypass credential replacement. - if target.scheme == "https" { - policy.Rules = append(policy.Rules, &ateapipb.EgressRule{Https: &ateapipb.HTTPSRule{Hostnames: []string{target.hostname}, Ports: ports, Effects: effects[target.hostname]}}) + for _, origin := range access.Destinations { + target, _ := egress.ParseOrigin(origin) // validated by CanonicalAccess + port, _ := strconv.ParseInt(target.Port(), 10, 32) + ports := &ateapipb.Ports{Numbers: []int32{int32(port)}} + // One rule per origin: exact matches win over wildcard rules, keeping + // credentials on their exact origin even when a wildcard also grants access. + if target.Scheme == "https" { + policy.Rules = append(policy.Rules, &ateapipb.EgressRule{Https: &ateapipb.HTTPSRule{Hostnames: []string{target.Hostname()}, Ports: ports, Effects: effects[origin]}}) } else { - policy.Rules = append(policy.Rules, &ateapipb.EgressRule{Http: &ateapipb.HTTPRule{Hostnames: []string{target.hostname}, Ports: ports, Effects: effects[target.hostname]}}) + policy.Rules = append(policy.Rules, &ateapipb.EgressRule{Http: &ateapipb.HTTPRule{Hostnames: []string{target.Hostname()}, Ports: ports, Effects: effects[origin]}}) } } - if len(policy.Rules) > 256 { - return nil, fmt.Errorf("egress policy exceeds 256 rules") - } return policy, nil } diff --git a/go/core/internal/substrate/policy_test.go b/go/core/internal/substrate/policy_test.go new file mode 100644 index 0000000000..525efb3b89 --- /dev/null +++ b/go/core/internal/substrate/policy_test.go @@ -0,0 +1,29 @@ +package substrate + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +func TestActorPolicyKeepsCredentialsOnExactOrigin(t *testing.T) { + policy, err := ActorEgressPolicy("team", []string{"https://*.example.com", "https://api.example.com:443", "https://api.example.com:8443", "http://api.example.com"}, []egress.Credential{ + {Origin: "https://api.example.com", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/team/auth/token"}, + {Origin: "https://api.example.com:8443", Header: "x-api-key", URI: "ate-secret://k8s.io/default/team/other/key"}, + }) + require.NoError(t, err) + require.Len(t, policy.Rules, 4) + require.Empty(t, policy.Rules[0].GetHttp().GetEffects().GetReplaceHeaders()) + require.Equal(t, []string{"*.example.com"}, policy.Rules[1].GetHttps().GetHostnames()) + require.Empty(t, policy.Rules[1].GetHttps().GetEffects().GetReplaceHeaders()) + for i, header := range []string{"authorization", "x-api-key"} { + rule := policy.Rules[i+2].GetHttps() + require.Equal(t, []string{"api.example.com"}, rule.GetHostnames()) + require.Len(t, rule.GetEffects().GetReplaceHeaders(), 1) + require.Equal(t, header, rule.GetEffects().GetReplaceHeaders()[0].Header) + } + require.Equal(t, []int32{443}, policy.Rules[2].GetHttps().GetPorts().GetNumbers()) + require.Equal(t, []int32{8443}, policy.Rules[3].GetHttps().GetPorts().GetNumbers()) +} diff --git a/go/core/internal/substrate/sandbox.go b/go/core/internal/substrate/sandbox.go index 5282acf36d..cfe27e2e28 100644 --- a/go/core/internal/substrate/sandbox.go +++ b/go/core/internal/substrate/sandbox.go @@ -9,6 +9,7 @@ import ( atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" @@ -25,7 +26,6 @@ type SandboxPolicy struct { type sandboxInputs struct { Kind string - Version int Namespace string Name string UID string @@ -67,17 +67,13 @@ func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1. if len(environment) > 32 { return nil, "", nil, fmt.Errorf("sandbox exceeds Substrate's 32 environment variables") } - // Namespace-scoped placement changes immutable template contents, so it - // needs a new revision even when the operator's inputs stay the same. - inputs := sandboxInputs{Kind: "sandbox", Version: 2, Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), Spec: template.Spec, Class: class, Policy: policy} + inputs := sandboxInputs{Kind: "sandbox", Namespace: template.Namespace, Name: template.Name, UID: string(template.UID), Spec: template.Spec, Class: class, Policy: policy} snapshot, err := json.Marshal(inputs) if err != nil { return nil, "", nil, fmt.Errorf("encode sandbox inputs: %w", err) } - digest := sha256.Sum256(snapshot) - revision := hex.EncodeToString(digest[:]) result := &ateapipb.ActorTemplate{ - Metadata: &ateapipb.ResourceMetadata{Atespace: template.Namespace, Name: "sandbox-" + revision[:40]}, + Metadata: &ateapipb.ResourceMetadata{Atespace: template.Namespace}, SandboxConfig: config, WorkerSelector: workerSelectorForPool(types.NamespacedName{Namespace: template.Namespace, Name: template.Spec.Substrate.WorkerPoolRef.Name}), Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{{Name: "cpu", Quantity: policy.CPU}, {Name: "memory", Quantity: policy.Memory}}}, @@ -99,5 +95,22 @@ func SandboxActorTemplate(template *v1alpha3.SandboxTemplate, class atev1alpha1. OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, }, } + // Hash the compiled template and source identity. Live policy references + // remain in provenance but do not affect immutable runtime contents. + compiled, err := proto.MarshalOptions{Deterministic: true}.Marshal(result) + if err != nil { + return nil, "", nil, fmt.Errorf("encode sandbox template: %w", err) + } + identity, err := json.Marshal(struct { + Name string + UID string + Template []byte + }{template.Name, string(template.UID), compiled}) + if err != nil { + return nil, "", nil, fmt.Errorf("encode sandbox revision: %w", err) + } + digest := sha256.Sum256(identity) + revision := hex.EncodeToString(digest[:]) + result.Metadata.Name = "sandbox-" + revision[:40] return result, revision, snapshot, nil } diff --git a/go/core/internal/substrate/sandbox_test.go b/go/core/internal/substrate/sandbox_test.go index 88edb45c88..4c2f30ff64 100644 --- a/go/core/internal/substrate/sandbox_test.go +++ b/go/core/internal/substrate/sandbox_test.go @@ -1,6 +1,7 @@ package substrate import ( + "encoding/json" "strings" "testing" @@ -35,6 +36,13 @@ func TestSandboxPreparationPinsCompleteInputs(t *testing.T) { _, same, _, err := SandboxActorTemplate(template, atev1alpha1.SandboxClassGvisor, policy) require.NoError(t, err) require.Equal(t, digest, same) + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "live"}} + _, unchanged, source, err := SandboxActorTemplate(template, "", policy) + require.NoError(t, err) + require.Equal(t, digest, unchanged, "policy references are live authorization") + var recorded sandboxInputs + require.NoError(t, json.Unmarshal(source, &recorded)) + require.Equal(t, template.Spec, recorded.Spec, "provenance retains the complete source spec") for _, change := range []struct { name string mutate func(*v1alpha3.SandboxTemplate, *SandboxPolicy) diff --git a/go/core/internal/translator/byo/compiler.go b/go/core/internal/translator/byo/compiler.go index c8b173811f..53f90f7fb9 100644 --- a/go/core/internal/translator/byo/compiler.go +++ b/go/core/internal/translator/byo/compiler.go @@ -70,7 +70,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build revision provenance: %w", err) } - environment, credentials, err := translator.CompileCredentials(input, compiled.Models, environment) + environment, credentials, tokenPassthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment) if err != nil { return nil, err } @@ -84,7 +84,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) Environment: environment, ConfigJSON: configJSON, AgentCard: card, SkillImages: compiled.SkillImages, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - Credentials: credentials, Provenance: provenance, EgressDestinations: slices.Compact(compiled.Egress), + TokenPassthroughOrigins: tokenPassthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: slices.Compact(compiled.Egress), }}, nil } diff --git a/go/core/internal/translator/claude/compiler.go b/go/core/internal/translator/claude/compiler.go index 59714932a7..d5bb297486 100644 --- a/go/core/internal/translator/claude/compiler.go +++ b/go/core/internal/translator/claude/compiler.go @@ -110,7 +110,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build Claude revision provenance: %w", err) } - environment, credentials, err := translator.CompileCredentials(input, nil, environment) + environment, credentials, tokenPassthroughOrigins, err := translator.CompileCredentials(input, nil, environment) if err != nil { return nil, err } @@ -127,9 +127,9 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) SkillImages: skills.Images, Environment: environment, ConfigJSON: configJSON, AgentCard: card, - WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, - SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - Credentials: credentials, Provenance: provenance, EgressDestinations: egress, + WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, + SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, + TokenPassthroughOrigins: tokenPassthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: egress, }, Warnings: mcp.warnings, }, nil diff --git a/go/core/internal/translator/codex/compiler.go b/go/core/internal/translator/codex/compiler.go index 9323d220fe..4300916d8f 100644 --- a/go/core/internal/translator/codex/compiler.go +++ b/go/core/internal/translator/codex/compiler.go @@ -130,7 +130,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build Codex revision provenance: %w", err) } - environment, credentials, err := translator.CompileCredentials(input, nil, environment) + environment, credentials, tokenPassthroughOrigins, err := translator.CompileCredentials(input, nil, environment) if err != nil { return nil, err } @@ -146,7 +146,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) SkillImages: skills.Images, Environment: environment, ConfigJSON: configJSON, AgentCard: card, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - Credentials: credentials, Provenance: provenance, EgressDestinations: egress, + TokenPassthroughOrigins: tokenPassthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: egress, }, Warnings: mcp.warnings, }, nil diff --git a/go/core/internal/translator/collections.go b/go/core/internal/translator/collections.go index 02d07639fc..46563043f6 100644 --- a/go/core/internal/translator/collections.go +++ b/go/core/internal/translator/collections.go @@ -11,6 +11,7 @@ import ( // Collections contains every typed input used while compiling a revision. // Production supplies informer-backed collections; tests supply KRT mocks. type Collections struct { + EgressPolicies krt.Collection[*v1alpha3.EgressPolicy] Harnesses krt.Collection[*v1alpha3.Harness] AgentTemplates krt.Collection[*v1alpha3.AgentTemplate] ResolvedModelConfigs krt.Collection[ResolvedModelConfig] diff --git a/go/core/internal/translator/compiler.go b/go/core/internal/translator/compiler.go index 9a1770d9f1..cc7d8f2195 100644 --- a/go/core/internal/translator/compiler.go +++ b/go/core/internal/translator/compiler.go @@ -124,7 +124,7 @@ func (c *Compiler) CompileAgent(ctx context.Context, agent *v1alpha3.Agent) (*Co return nil, fmt.Errorf("resolve Harness %q: not found", harnessKey) } harness := harnessConfiguration(*foundHarness) - result, err := c.compileConfiguration(ctx, agent.Name, harness, template) + result, err := c.compileConfiguration(ctx, agent, harness, template) if err != nil { return nil, err } @@ -141,9 +141,9 @@ func (c *Compiler) CompileAgent(ctx context.Context, agent *v1alpha3.Agent) (*Co return result, nil } -// compileConfiguration compiles resolved configuration for the named Agent. +// compileConfiguration resolves inputs and compiles configuration for the Agent. // The Agent name owns runtime identity; template and Harness names are provenance. -func (c *Compiler) compileConfiguration(ctx context.Context, agentName string, harness *HarnessConfiguration, template *TemplateConfiguration) (*CompileResult, error) { +func (c *Compiler) compileConfiguration(ctx context.Context, agent *v1alpha3.Agent, harness *HarnessConfiguration, template *TemplateConfiguration) (*CompileResult, error) { harnessCompiler := c.harnessCompilers[harnessType(harness)] if harnessCompiler == nil { return nil, NewValidationError("Harness runtime is not supported by any compiler") @@ -156,7 +156,7 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agentName string, h if err != nil { return nil, err } - input.AgentName = agentName + input.AgentName = agent.Name image, err := harnessImage(harness, c.images) if err != nil { return nil, err @@ -170,7 +170,7 @@ func (c *Compiler) compileConfiguration(ctx context.Context, agentName string, h if workerPool == nil { return nil, &WorkerPoolNotFoundError{WorkerPool: workerKey} } - result.AgentName = agentName + result.AgentName = agent.Name result.Image = image result.SandboxClass = (*workerPool).Spec.SandboxClass // Shared agents and plugin selections can contribute the same image. diff --git a/go/core/internal/translator/compiler_test.go b/go/core/internal/translator/compiler_test.go index 3b712f52f4..8ce2daebb1 100644 --- a/go/core/internal/translator/compiler_test.go +++ b/go/core/internal/translator/compiler_test.go @@ -227,6 +227,7 @@ func mockCollections(t *testing.T, objects ...any) translator.Collections { t.Helper() mock := krttest.NewMock(t, objects) collections := translator.Collections{ + EgressPolicies: krttest.GetMockCollection[*v1alpha3.EgressPolicy](mock), AgentTemplates: krttest.GetMockCollection[*v1alpha3.AgentTemplate](mock), Harnesses: krttest.GetMockCollection[*v1alpha3.Harness](mock), RemoteMCPServers: krttest.GetMockCollection[*v1alpha3.RemoteMCPServer](mock), @@ -405,6 +406,14 @@ func TestCompileAgentStructuredOutput(t *testing.T) { require.Empty(t, config.Agent.SubAgents[0].SessionDbUrl) require.Contains(t, string(revision.Provenance), `"kind":"ConfigMap"`) require.Equal(t, []string{"application/json"}, revision.AgentCard.DefaultOutputModes) + original, err := revision.Digest() + require.NoError(t, err) + configMap.Data["answer.json"] = `{"type":"object","properties":{"changed":{"type":"string"}}}` + changed, err := compiler(t, modelConfig(), configMap, child, harness).CompileAgent(t.Context(), inlineAgent(harness, root)) + require.NoError(t, err) + updated, err := changed.Digest() + require.NoError(t, err) + require.NotEqual(t, original, updated, "resolved ConfigMap contents must still affect compiled runtime identity") } func TestResolveModelConfigFoundryEndpoint(t *testing.T) { @@ -682,7 +691,10 @@ func TestCompileAgentSharedADKConfig(t *testing.T) { runtimeHarness.Spec.BYO = &v1alpha3.BYOHarness{} runtimeHarness.Spec.Workload.Command = []string{"/app"} } - revision, err := compiler(t, modelConfig(), child, runtimeHarness, remoteMCPServer("search", "https://search.example.com/mcp")).CompileAgent(context.Background(), inlineAgent(runtimeHarness, root)) + policy, secret := egressPolicy() + agent := inlineAgent(runtimeHarness, root) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + revision, err := compiler(t, modelConfig(), child, runtimeHarness, policy, secret, remoteMCPServer("search", "https://search.example.com/mcp")).CompileAgent(context.Background(), agent) require.NoError(t, err) var config configv1.ADKConfig require.NoError(t, runtimeconfig.Unmarshal(revision.ConfigJSON, &config)) @@ -692,6 +704,7 @@ func TestCompileAgentSharedADKConfig(t *testing.T) { require.Equal(t, "research carefully", config.Agent.SubAgents[0].Instruction) require.Equal(t, []string{"lookup"}, config.Agent.SubAgents[0].HttpTools[0].Tools) require.Equal(t, "review", config.Agent.SubAgents[0].AgentPlugins.Skills[0].Name) + require.NotContains(t, revision.EgressDestinations, "https://tickets.example.com:443") require.Contains(t, revision.EgressDestinations, "https://search.example.com:443") require.NotContains(t, revision.EgressDestinations, "https://ghcr.io:443") require.Equal(t, []translator.SkillImage{translator.SkillImage(child.Spec.Skills[0].Source.OCI)}, revision.SkillImages) diff --git a/go/core/internal/translator/credentials.go b/go/core/internal/translator/credentials.go index 29ac153c2e..99fce4bbfe 100644 --- a/go/core/internal/translator/credentials.go +++ b/go/core/internal/translator/credentials.go @@ -20,8 +20,9 @@ const CredentialPlaceholder = "kagent-credential-injected" // CompileCredentials replaces secret-backed environment values with inert SDK // placeholders and compiles their destination-scoped gateway bindings. Models // outside the agent tree (such as memory embeddings) are supplied separately. -func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, environment []corev1.EnvVar) ([]corev1.EnvVar, []egress.Credential, error) { +func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, environment []corev1.EnvVar) ([]corev1.EnvVar, []egress.Credential, []string, error) { var bindings []egress.Credential + var tokenPassthroughOrigins []string boundModels := map[string]bool{} boundMCP := map[string]bool{} bind := func(rawURL, header, prefix, namespace, name, key string) error { @@ -30,7 +31,7 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, return NewValidationError("credential destination must be an absolute HTTP(S) URL without user information or fragment") } uri := "ate-secret://k8s.io/default/" + namespace + "/" + name + "/" + key - bindings = append(bindings, egress.Credential{Hostname: u.Hostname(), Header: header, Prefix: prefix, URI: uri}) + bindings = append(bindings, egress.Credential{Origin: egress.Origin(u), Header: header, Prefix: prefix, URI: uri}) return nil } models := append([]*ResolvedModelConfig(nil), extraModels...) @@ -57,7 +58,7 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, return nil } if err := visit(input.Root); err != nil { - return nil, nil, err + return nil, nil, nil, err } for _, resolved := range models { model := resolved.Config @@ -82,14 +83,10 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, } } if err := bind(endpoint, header, prefix, model.Namespace, model.Spec.APIKeySecret, key); err != nil { - return nil, nil, err + return nil, nil, nil, err } boundModels[name+"\x00"+model.Spec.APIKeySecret+"\x00"+key] = true } - bindings, err := egress.CanonicalCredentials(bindings) - if err != nil { - return nil, nil, NewValidationError("%v", err) - } for _, resolved := range models { if !resolved.Config.Spec.APIKeyPassthrough { continue @@ -97,15 +94,23 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, _, endpoint, _, _ := modelCredentialTarget(resolved) u, err := url.Parse(endpoint) if err != nil { - return nil, nil, NewValidationError("invalid passthrough credential destination") + return nil, nil, nil, NewValidationError("invalid passthrough credential destination") } - hostname := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") - for _, binding := range bindings { - if binding.Hostname == hostname { - return nil, nil, NewValidationError("destination %q cannot combine caller-token passthrough with gateway credentials", hostname) - } + origin, err := egress.ParseOrigin(egress.Origin(u)) + if err != nil { + return nil, nil, nil, NewValidationError("invalid passthrough credential destination: %v", err) } + tokenPassthroughOrigins = append(tokenPassthroughOrigins, origin.String()) } + var destinations []string + for _, binding := range bindings { + destinations = append(destinations, binding.Origin) + } + access, err := egress.CanonicalAccess(destinations, bindings, tokenPassthroughOrigins) + if err != nil { + return nil, nil, nil, NewValidationError("%v", err) + } + result := append([]corev1.EnvVar(nil), environment...) for i, variable := range result { if variable.ValueFrom == nil { @@ -114,11 +119,11 @@ func CompileCredentials(input *HarnessInput, extraModels []*ResolvedModelConfig, ref := variable.ValueFrom.SecretKeyRef isMCP := strings.HasPrefix(variable.Name, "KAGENT_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CODEX_MCP_CREDENTIAL_") || strings.HasPrefix(variable.Name, "KAGENT_CLAUDE_MCP_CREDENTIAL_") if ref == nil || (!boundModels[variable.Name+"\x00"+ref.Name+"\x00"+ref.Key] && (!isMCP || !boundMCP[ref.Name+"\x00"+ref.Key])) { - return nil, nil, NewValidationError("environment credential %q cannot use gateway header injection; local signing and arbitrary secret environment variables are unsupported", variable.Name) + return nil, nil, nil, NewValidationError("environment credential %q cannot use gateway header injection; local signing and arbitrary secret environment variables are unsupported", variable.Name) } result[i].Value, result[i].ValueFrom = CredentialPlaceholder, nil } - return result, bindings, nil + return result, access.Credentials, tokenPassthroughOrigins, nil } func modelCredentialTarget(resolved *ResolvedModelConfig) (name, endpoint, header, prefix string) { diff --git a/go/core/internal/translator/credentials_test.go b/go/core/internal/translator/credentials_test.go index ba106e67b3..752362029d 100644 --- a/go/core/internal/translator/credentials_test.go +++ b/go/core/internal/translator/credentials_test.go @@ -34,13 +34,13 @@ func TestCompileCredentialDestinations(t *testing.T) { } input := credentialInput(test.spec) environment := []corev1.EnvVar{credentialEnv(test.env, "auth", test.spec.APIKeySecretKey)} - got, bindings, err := CompileCredentials(input, nil, environment) + got, bindings, _, err := CompileCredentials(input, nil, environment) require.NoError(t, err) require.Equal(t, CredentialPlaceholder, got[0].Value) require.Nil(t, got[0].ValueFrom) require.NotNil(t, environment[0].ValueFrom, "compilation must not mutate inputs") require.Len(t, bindings, 1) - require.Equal(t, test.host, bindings[0].Hostname) + require.Equal(t, "https://"+test.host+":443", bindings[0].Origin) require.Equal(t, test.header, bindings[0].Header) require.Equal(t, test.prefix, bindings[0].Prefix) require.Equal(t, "ate-secret://k8s.io/default/team/auth/"+test.spec.APIKeySecretKey, bindings[0].URI) @@ -54,30 +54,33 @@ func TestCompileCredentialsRejectsConflictingSharedModels(t *testing.T) { spec.APIKeySecret = "child" input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(spec).Root}} // The ADK environment is deduplicated, but every model still needs its own binding. - _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) + environment, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) require.ErrorContains(t, err, "conflicting credentials") + require.Nil(t, environment) input.Root.Shared[0].Agent.ResolvedModelConfig.Config.Spec.OpenAI = &v1alpha3.OpenAIConfig{BaseURL: "https://child.example.com/v1"} - _, bindings, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) + _, bindings, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "child", "token")}) require.NoError(t, err) require.Len(t, bindings, 2) } func TestCompileCredentialsRejectsLocalSecrets(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{}) - _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("AWS_SECRET_ACCESS_KEY", "auth", "token")}) + environment, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("AWS_SECRET_ACCESS_KEY", "auth", "token")}) require.ErrorContains(t, err, "cannot use gateway header injection") + require.Nil(t, environment) } func TestCompileCredentialsPreservesPassthrough(t *testing.T) { input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: true}) - _, bindings, err := CompileCredentials(input, nil, nil) + _, bindings, _, err := CompileCredentials(input, nil, nil) require.NoError(t, err) require.Empty(t, bindings) input.Root.Shared = []AgentInputBinding{{Agent: credentialInput(v1alpha3.ModelConfigSpec{ Provider: v1alpha3.ModelProviderOpenAI, APIKeySecret: "auth", APIKeySecretKey: "token", }).Root}} - _, _, err = CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}) + environment, _, _, err := CompileCredentials(input, nil, []corev1.EnvVar{credentialEnv("OPENAI_API_KEY", "auth", "token")}) require.ErrorContains(t, err, "cannot combine caller-token passthrough") + require.Nil(t, environment) } func credentialInput(spec v1alpha3.ModelConfigSpec) *HarnessInput { @@ -170,3 +173,12 @@ func TestModelCredentialTargetOllamaCloud(t *testing.T) { }) } } + +func TestPassthroughCredentialUsesCanonicalOrigin(t *testing.T) { + input := credentialInput(v1alpha3.ModelConfigSpec{Provider: v1alpha3.ModelProviderOpenAI, APIKeyPassthrough: true, OpenAI: &v1alpha3.OpenAIConfig{BaseURL: "https://API.OPENAI.COM.:0443/v1"}}) + environment, credentials, origins, err := CompileCredentials(input, nil, nil) + require.NoError(t, err) + require.Empty(t, environment) + require.Empty(t, credentials) + require.Equal(t, []string{"https://api.openai.com:443"}, origins) +} diff --git a/go/core/internal/translator/egress.go b/go/core/internal/translator/egress.go new file mode 100644 index 0000000000..3f8b5271ef --- /dev/null +++ b/go/core/internal/translator/egress.go @@ -0,0 +1,136 @@ +package translator + +import ( + "fmt" + "slices" + "strings" + + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation" + + v1alpha3 "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" +) + +// EgressGrantIssue describes one policy resolution error. +type EgressGrantIssue struct { + Policy string + Reason string + Message string +} + +// EgressGrantResolution contains the union of individually valid policies. +// Admission checks conflicts across this union and the desired revision's base. +type EgressGrantResolution struct { + Access egress.Access + Issues []EgressGrantIssue +} + +// CompileEgressPolicies resolves each same-namespace policy independently. +// KRT tracks policy and Secret dependencies. An unresolved policy contributes +// no grants, without removing grants supplied by the remaining policies. +func CompileEgressPolicies(ctx krt.HandlerContext, collections Collections, namespace string, refs []corev1.LocalObjectReference) EgressGrantResolution { + return compileEgressPolicies(namespace, refs, func(name string) (*v1alpha3.EgressPolicy, error) { + policy := krt.FetchOne(ctx, collections.EgressPolicies, krt.FilterKey(namespace+"/"+name)) + if policy == nil { + return nil, apierrors.NewNotFound(schema.GroupResource{Group: "api.kagent.dev", Resource: "egresspolicies"}, name) + } + return *policy, nil + }, func(name string) (*corev1.Secret, error) { + secret := krt.FetchOne(ctx, collections.Secrets, krt.FilterKey(namespace+"/"+name)) + if secret == nil { + return nil, apierrors.NewNotFound(corev1.Resource("secrets"), name) + } + return *secret, nil + }) +} + +func compileEgressPolicies(namespace string, refs []corev1.LocalObjectReference, getPolicy func(string) (*v1alpha3.EgressPolicy, error), getSecret func(string) (*corev1.Secret, error)) EgressGrantResolution { + var result EgressGrantResolution + var invalidRefs string + if len(refs) > 16 { + invalidRefs = "at most 16 egressPolicyRefs are supported" + } + for _, ref := range refs { + if len(validation.IsDNS1123Subdomain(ref.Name)) != 0 { + invalidRefs = fmt.Sprintf("invalid EgressPolicy reference %q", ref.Name) + break + } + } + for _, ref := range refs { + issue := EgressGrantIssue{Policy: namespace + "/" + ref.Name, Reason: v1alpha3.EgressPoliciesReasonInvalidEgressPolicy} + if invalidRefs != "" { + issue.Message = invalidRefs + result.Issues = append(result.Issues, issue) + continue + } + policy, err := getPolicy(ref.Name) + if err != nil || !policy.DeletionTimestamp.IsZero() { + if err == nil || apierrors.IsNotFound(err) { + issue.Reason = v1alpha3.EgressPoliciesReasonNotFound + } + issue.Message = "EgressPolicy is being deleted" + if err != nil { + issue.Message = err.Error() + } + result.Issues = append(result.Issues, issue) + continue + } + access, err := compileEgressPolicy(namespace, policy, getSecret) + if err != nil { + issue.Message = err.Error() + result.Issues = append(result.Issues, issue) + continue + } + result.Access.Destinations = append(result.Access.Destinations, access.Destinations...) + result.Access.Credentials = append(result.Access.Credentials, access.Credentials...) + } + slices.SortFunc(result.Issues, func(a, b EgressGrantIssue) int { return strings.Compare(a.Policy, b.Policy) }) + return result +} + +func compileEgressPolicy(namespace string, policy *v1alpha3.EgressPolicy, getSecret func(string) (*corev1.Secret, error)) (egress.Access, error) { + if len(policy.Spec.Destinations) == 0 || len(policy.Spec.Destinations) > 64 { + return egress.Access{}, NewValidationError("requires 1 to 64 destinations") + } + var destinations []string + var credentials []egress.Credential + for _, destination := range policy.Spec.Destinations { + origin, err := egress.ParseOrigin(destination.Origin) + if err != nil { + return egress.Access{}, err + } + destinations = append(destinations, origin.String()) + auth := destination.Authentication + if auth == nil { + continue + } + if strings.Contains(origin.Host, "*") { + return egress.Access{}, NewValidationError("authentication requires an exact HTTP(S) origin") + } + secret := auth.SecretKeyRef + binding := egress.Credential{Origin: origin.String(), Header: auth.Header} + if auth.Prefix != nil { + binding.Prefix = *auth.Prefix + } + if secret.Name == "" || secret.Key == "" || (secret.Optional != nil && *secret.Optional) { + return egress.Access{}, NewValidationError("requires a non-optional Secret name and key") + } + binding.URI = "ate-secret://k8s.io/default/" + namespace + "/" + secret.Name + "/" + secret.Key + if _, err := egress.CanonicalCredentials([]egress.Credential{binding}); err != nil { + return egress.Access{}, err + } + found, err := getSecret(secret.Name) + if err != nil { + return egress.Access{}, fmt.Errorf("resolve Secret %q: %w", secret.Name, err) + } + if _, ok := found.Data[secret.Key]; !ok { + return egress.Access{}, NewValidationError("Secret %q is missing key %q", secret.Name, secret.Key) + } + credentials = append(credentials, binding) + } + return egress.CanonicalAccess(destinations, credentials, nil) +} diff --git a/go/core/internal/translator/egress_test.go b/go/core/internal/translator/egress_test.go new file mode 100644 index 0000000000..ca53c00058 --- /dev/null +++ b/go/core/internal/translator/egress_test.go @@ -0,0 +1,267 @@ +package translator_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "istio.io/istio/pkg/kube/krt" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/egress" + "github.com/kagent-dev/kagent/go/core/internal/translator" +) + +func egressPolicy() (*v1alpha3.EgressPolicy, *corev1.Secret) { + return &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "tickets"}, Spec: v1alpha3.EgressPolicySpec{ + Destinations: []v1alpha3.EgressDestination{{Origin: "https://tickets.example.com", Authentication: &v1alpha3.EgressAuthentication{ + Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "ticket-auth"}, Key: "token"}, + }}}, + }}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "ticket-auth"}, Data: map[string][]byte{"token": []byte("private-ticket-token")}} +} + +func TestEgressPolicyCredentialPrefixes(t *testing.T) { + for _, tc := range []struct { + name string + header string + prefix *string + want string + }{ + {name: "bearer token", header: "Authorization", prefix: new("Bearer "), want: "Bearer "}, + {name: "API key", header: "X-API-Key"}, + {name: "empty prefix", header: "X-API-Key", prefix: new("")}, + {name: "custom prefix", header: "Authorization", prefix: new("Token "), want: "Token "}, + {name: "literal prefix", header: "Authorization", prefix: new("Token"), want: "Token"}, + } { + t.Run(tc.name, func(t *testing.T) { + policy, secret := egressPolicy() + auth := policy.Spec.Destinations[0].Authentication + auth.Header, auth.Prefix = tc.header, tc.prefix + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", []corev1.LocalObjectReference{{Name: policy.Name}}) + require.Empty(t, resolution.Issues) + access := resolution.Access + wantHeader := "authorization" + if tc.header == "X-API-Key" { + wantHeader = "x-api-key" + } + require.Equal(t, []egress.Credential{{Origin: "https://tickets.example.com:443", Header: wantHeader, Prefix: tc.want, URI: "ate-secret://k8s.io/default/test/ticket-auth/token"}}, access.Credentials) + }) + } +} + +func TestEgressPolicyHTTPCredentials(t *testing.T) { + policy, secret := egressPolicy() + policy.Spec.Destinations[0].Origin = "http://tickets.example.com:8080" + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", []corev1.LocalObjectReference{{Name: policy.Name}}) + require.Empty(t, resolution.Issues) + access := resolution.Access + require.Equal(t, []string{"http://tickets.example.com:8080"}, access.Destinations) + require.Equal(t, []egress.Credential{{Origin: "http://tickets.example.com:8080", Header: "authorization", Prefix: "Bearer ", URI: "ate-secret://k8s.io/default/test/ticket-auth/token"}}, access.Credentials) +} + +func TestAgentEgressPoliciesDoNotChangeRuntimeAcrossHarnesses(t *testing.T) { + for _, kind := range []string{"kagent", "codex", "claude", "byo"} { + t.Run(kind, func(t *testing.T) { + policy, secret := egressPolicy() + harness := &v1alpha3.Harness{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "runner"}, Spec: v1alpha3.HarnessSpec{ + Workload: v1alpha3.HarnessWorkload{Image: new("runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, + Substrate: v1alpha3.RuntimeSubstratePolicy{WorkerPoolRef: corev1.LocalObjectReference{Name: "default"}, SnapshotPolicy: v1alpha3.RuntimeSnapshotPolicy{Location: "snapshots"}}, + }} + switch kind { + case "kagent": + harness.Spec.Kagent = &v1alpha3.KagentHarness{} + case "codex": + harness.Spec.Codex = &v1alpha3.CodexHarness{} + case "claude": + harness.Spec.Claude = &v1alpha3.ClaudeHarness{} + case "byo": + harness.Spec.BYO = &v1alpha3.BYOHarness{} + } + model := modelConfig() + if kind == "codex" { + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = secret.Name, "token" + model.Spec.OpenAI = &v1alpha3.OpenAIConfig{APIFormat: new(v1alpha3.OpenAIAPIFormatResponses)} + } + if kind == "claude" { + model.Spec.Provider, model.Spec.Model = v1alpha3.ModelProviderAnthropic, "claude-sonnet-4-5" + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = secret.Name, "token" + } + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "helper"}, Spec: v1alpha3.AgentTemplateSpec{ModelConfig: &corev1.LocalObjectReference{Name: model.Name}, SystemPrompt: "help"}} + agent := inlineAgent(harness, template) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + compile := func() *translator.CompileResult { + result, err := compiler(t, model, harness, policy, secret).CompileAgent(t.Context(), agent) + require.NoError(t, err) + return result + } + result := compile() + var recorded struct{ Spec v1alpha3.AgentSpec } + require.NoError(t, json.Unmarshal(result.Provenance, &recorded)) + require.Equal(t, agent.Spec, recorded.Spec, "record the full source spec, including live policy references") + require.NotContains(t, result.EgressDestinations, "https://tickets.example.com:443") + require.NotEmpty(t, result.EgressDestinations, "preserve inferred integration access") + for _, binding := range result.Credentials { + require.NotEqual(t, "https://tickets.example.com:443", binding.Origin) + } + encoded, err := json.Marshal(result) + require.NoError(t, err) + require.NotContains(t, string(encoded), string(secret.Data["token"])) + original, err := result.Digest() + require.NoError(t, err) + secret.Data["token"], secret.UID = []byte("rotated-token"), "replacement" + rotated, err := compile().Digest() + require.NoError(t, err) + require.Equal(t, original, rotated) + policy.Spec.Destinations = append(policy.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://registry.npmjs.org"}) + changed := compile() + updated, err := changed.Digest() + require.NoError(t, err) + require.Equal(t, original, updated) + agent.Spec.EgressPolicyRefs = nil + unreferenced, err := compile().Digest() + require.NoError(t, err) + require.Equal(t, original, unreferenced) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: policy.Name}} + require.NotContains(t, result.EgressDestinations, "https://registry.npmjs.org:443", "previous revision remains pinned") + // Conflicting and missing policies are independent of runtime compilation. + policy.Spec.Destinations[0].Origin = "https://api.openai.com" + auth := policy.Spec.Destinations[0].Authentication + if kind == "claude" { + policy.Spec.Destinations[0].Origin = "https://api.anthropic.com" + auth.Header, auth.Prefix = "x-api-key", nil + } + auth.SecretKeyRef.Key = "alternate" + secret.Data["alternate"] = []byte("different-token") + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = secret.Name, "token" + _, err = compiler(t, model, harness, policy, secret).CompileAgent(t.Context(), agent) + require.NoError(t, err) + withConflict, err := compile().Digest() + require.NoError(t, err) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: "missing"}} + missing, err := compile().Digest() + require.NoError(t, err) + require.Equal(t, withConflict, missing) + // Codex and Claude reject caller-token passthrough independently. + if kind == "claude" || kind == "codex" { + return + } + model.Spec.APIKeySecret, model.Spec.APIKeySecretKey = "", "" + model.Spec.APIKeyPassthrough = true + _, err = compiler(t, model, harness, policy, secret).CompileAgent(t.Context(), agent) + require.NoError(t, err) + }) + } +} + +func TestEgressPolicyResolutionFailsClosed(t *testing.T) { + for _, name := range []string{"missing policy", "other namespace", "missing secret", "missing key", "optional secret", "wildcard auth", "empty auth", "missing header", "protected header", "invalid prefix", "conflicting auth", "conflicting prefix"} { + t.Run(name, func(t *testing.T) { + policy, secret := egressPolicy() + objects := []any{policy, secret} + refs := []corev1.LocalObjectReference{{Name: policy.Name}} + switch name { + case "missing policy": + objects = []any{secret} + case "other namespace": + policy.Namespace = "other" + case "missing secret": + objects = []any{policy} + case "missing key": + secret.Data = nil + case "optional secret": + policy.Spec.Destinations[0].Authentication.SecretKeyRef.Optional = new(true) + case "wildcard auth": + policy.Spec.Destinations[0].Origin = "https://*.example.com" + case "empty auth": + policy.Spec.Destinations[0].Authentication = &v1alpha3.EgressAuthentication{} + case "missing header": + policy.Spec.Destinations[0].Authentication.Header = "" + case "protected header": + policy.Spec.Destinations[0].Authentication.Header = "Host" + case "invalid prefix": + policy.Spec.Destinations[0].Authentication.Prefix = new("Bearer\r\nInjected: ") + case "conflicting auth", "conflicting prefix": + other := policy.DeepCopy() + other.Name = "other" + if name == "conflicting prefix" { + other.Spec.Destinations[0].Authentication.Prefix = new("Token ") + } else { + other.Spec.Destinations[0].Authentication.SecretKeyRef.Key = "other" + secret.Data["other"] = []byte("different") + } + objects = append(objects, other) + refs = append(refs, corev1.LocalObjectReference{Name: other.Name}) + } + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, objects...), "test", refs) + if name == "conflicting auth" || name == "conflicting prefix" { + require.Empty(t, resolution.Issues, "individually valid policies reach admission") + _, err := egress.CanonicalAccess(resolution.Access.Destinations, resolution.Access.Credentials, nil) + require.Error(t, err) + } else { + require.Len(t, resolution.Issues, 1) + require.Empty(t, resolution.Access.Destinations) + } + }) + } +} + +func TestEgressPoliciesMergeCredentialsAndOrigins(t *testing.T) { + policy, secret := egressPolicy() + other := policy.DeepCopy() + other.Name = "registries" + other.Spec.Destinations = append(other.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://*.npmjs.org"}, v1alpha3.EgressDestination{ + Origin: "https://tickets.example.com:8443", Authentication: &v1alpha3.EgressAuthentication{Header: "X-API-Key", SecretKeyRef: policy.Spec.Destinations[0].Authentication.SecretKeyRef}, + }) + collections := mockCollections(t, policy, other, secret) + refs := []corev1.LocalObjectReference{{Name: policy.Name}, {Name: other.Name}} + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, collections, "test", refs) + require.Empty(t, resolution.Issues) + access, err := egress.CanonicalAccess(resolution.Access.Destinations, resolution.Access.Credentials, nil) + require.NoError(t, err) + require.Len(t, access.Destinations, 3) + require.Len(t, access.Credentials, 2) + require.Equal(t, "x-api-key", access.Credentials[1].Header) + refs[0], refs[1] = refs[1], refs[0] + reversed := translator.CompileEgressPolicies(krt.TestingDummyContext{}, collections, "test", refs) + require.Empty(t, reversed.Issues) + canonical, err := egress.CanonicalAccess(reversed.Access.Destinations, reversed.Access.Credentials, nil) + require.NoError(t, err) + require.Equal(t, access, canonical) +} + +func TestEgressPoliciesResolveIndependently(t *testing.T) { + policy, secret := egressPolicy() + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", + []corev1.LocalObjectReference{{Name: "z-missing"}, {Name: policy.Name}, {Name: "a-missing"}}) + require.Equal(t, []string{"https://tickets.example.com:443"}, resolution.Access.Destinations) + require.Len(t, resolution.Access.Credentials, 1) + require.Len(t, resolution.Issues, 2) + require.Equal(t, "test/a-missing", resolution.Issues[0].Policy) + require.Equal(t, v1alpha3.EgressPoliciesReasonNotFound, resolution.Issues[0].Reason) + require.Equal(t, "test/z-missing", resolution.Issues[1].Policy) +} + +func TestInvalidEgressReferencesRevokeAll(t *testing.T) { + for _, name := range []string{"too many", "invalid name"} { + t.Run(name, func(t *testing.T) { + policy, secret := egressPolicy() + refs := []corev1.LocalObjectReference{{Name: policy.Name}} + if name == "invalid name" { + refs = append(refs, corev1.LocalObjectReference{Name: "Invalid!"}) + } else { + for range 16 { + refs = append(refs, corev1.LocalObjectReference{Name: policy.Name}) + } + } + resolution := translator.CompileEgressPolicies(krt.TestingDummyContext{}, mockCollections(t, policy, secret), "test", refs) + require.Empty(t, resolution.Access.Destinations) + require.Empty(t, resolution.Access.Credentials) + require.Len(t, resolution.Issues, len(refs)) + for _, issue := range resolution.Issues { + require.Equal(t, v1alpha3.EgressPoliciesReasonInvalidEgressPolicy, issue.Reason) + } + }) + } +} diff --git a/go/core/internal/translator/foundry_test.go b/go/core/internal/translator/foundry_test.go index eeb1491ce3..b2198fc2ed 100644 --- a/go/core/internal/translator/foundry_test.go +++ b/go/core/internal/translator/foundry_test.go @@ -90,7 +90,7 @@ func TestCompileFoundryEndpoint(t *testing.T) { } require.Contains(t, revision.Environment, corev1.EnvVar{Name: "FOUNDRY_ENDPOINT", Value: endpoint}) require.Len(t, revision.Credentials, 1) - require.Equal(t, host, revision.Credentials[0].Hostname) + require.Equal(t, endpoint+":443", revision.Credentials[0].Origin) require.Equal(t, "api-key", revision.Credentials[0].Header) require.Equal(t, "ate-secret://k8s.io/default/test/foundry-auth/token", revision.Credentials[0].URI) require.Contains(t, revision.EgressDestinations, "https://"+host+":443") diff --git a/go/core/internal/translator/kagent/compiler.go b/go/core/internal/translator/kagent/compiler.go index 18549acb5f..3593770b7e 100644 --- a/go/core/internal/translator/kagent/compiler.go +++ b/go/core/internal/translator/kagent/compiler.go @@ -63,7 +63,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) if err != nil { return nil, fmt.Errorf("build revision provenance: %w", err) } - environment, credentials, err := translator.CompileCredentials(input, compiled.Models, environment) + environment, credentials, tokenPassthroughOrigins, err := translator.CompileCredentials(input, compiled.Models, environment) if err != nil { return nil, err } @@ -77,7 +77,7 @@ func (c *Compiler) Compile(ctx context.Context, input *translator.HarnessInput) Environment: environment, ConfigJSON: configJSON, AgentCard: card, SkillImages: compiled.SkillImages, WorkerPoolName: harness.Spec.Substrate.WorkerPoolRef.Name, SnapshotLocation: harness.Spec.Substrate.SnapshotPolicy.Location, - Credentials: credentials, Provenance: provenance, EgressDestinations: compiled.Egress, + TokenPassthroughOrigins: tokenPassthroughOrigins, Credentials: credentials, Provenance: provenance, EgressDestinations: compiled.Egress, }}, nil } diff --git a/go/core/internal/translator/revision.go b/go/core/internal/translator/revision.go index 4772cb3bf9..37863518dc 100644 --- a/go/core/internal/translator/revision.go +++ b/go/core/internal/translator/revision.go @@ -30,9 +30,8 @@ func (id RevisionID) Short() string { return hex.EncodeToString(id[:shortRevisio // IsZero reports whether compilation has not produced an identity. func (id RevisionID) IsZero() bool { return id == RevisionID{} } -// CompileResult contains one immutable runtime revision and the non-blocking -// diagnostics produced while compiling it. Diagnostics are deliberately kept -// outside Revision because they do not describe runtime behavior. +// CompileResult contains immutable runtime inputs and non-blocking diagnostics. +// Diagnostics do not enter the revision's identity. type CompileResult struct { Revision Warnings []string @@ -61,11 +60,13 @@ type Revision struct { SandboxClass atev1alpha1.SandboxClass SnapshotLocation string - // Provenance identifies non-secret Kubernetes inputs. Gateway-fetched - // credential values are deliberately excluded from revision identity. + // Provenance records non-secret source configuration for diagnostics. Revision + // identity comes from compiled runtime inputs, not this source snapshot. Provenance json.RawMessage // Credentials contains references resolved by the egress gateway. Credentials []egress.Credential + // TokenPassthroughOrigins cannot receive gateway credentials, including live grants. + TokenPassthroughOrigins []string // EgressDestinations is the HTTP(S) origin allowlist required by this revision. EgressDestinations []string } @@ -92,21 +93,21 @@ func (r *Revision) Digest() (RevisionID, error) { return RevisionID{}, fmt.Errorf("unsupported sandbox class %q", sandboxClass) } raw, err := json.Marshal(struct { - AgentName string `json:"agentName"` - AgentUID string `json:"agentUID"` - Namespace string `json:"namespace"` - Image string `json:"image"` - Command []string `json:"command,omitempty"` - Args []string `json:"args,omitempty"` - Environment []corev1.EnvVar `json:"environment"` - ConfigJSON json.RawMessage `json:"config"` - SkillImages []SkillImage `json:"skillImages,omitempty"` - WorkerPoolName string `json:"workerPoolName"` - SnapshotLocation string `json:"snapshotLocation"` - Provenance json.RawMessage `json:"provenance"` - Credentials []egress.Credential `json:"credentials,omitempty"` - EgressDestinations []string `json:"egressDestinations"` - SandboxClass atev1alpha1.SandboxClass `json:"sandboxClass"` + AgentName string `json:"agentName"` + AgentUID string `json:"agentUID"` + Namespace string `json:"namespace"` + Image string `json:"image"` + Command []string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Environment []corev1.EnvVar `json:"environment"` + ConfigJSON json.RawMessage `json:"config"` + SkillImages []SkillImage `json:"skillImages,omitempty"` + WorkerPoolName string `json:"workerPoolName"` + SnapshotLocation string `json:"snapshotLocation"` + Credentials []egress.Credential `json:"credentials,omitempty"` + TokenPassthroughOrigins []string `json:"tokenPassthroughOrigins,omitempty"` + EgressDestinations []string `json:"egressDestinations"` + SandboxClass atev1alpha1.SandboxClass `json:"sandboxClass"` // Placement now matches the pool's namespace as well as its name. // Hash it explicitly so existing immutable templates do not conflict after upgrades. @@ -115,8 +116,8 @@ func (r *Revision) Digest() (RevisionID, error) { AgentName: r.AgentName, AgentUID: r.AgentUID, Namespace: r.Namespace, Image: r.Image, Command: r.Command, Args: r.Args, Environment: r.Environment, ConfigJSON: r.ConfigJSON, SkillImages: r.SkillImages, - WorkerPoolName: r.WorkerPoolName, SnapshotLocation: r.SnapshotLocation, Provenance: r.Provenance, - Credentials: r.Credentials, EgressDestinations: r.EgressDestinations, + WorkerPoolName: r.WorkerPoolName, SnapshotLocation: r.SnapshotLocation, + TokenPassthroughOrigins: r.TokenPassthroughOrigins, Credentials: r.Credentials, EgressDestinations: r.EgressDestinations, SandboxClass: sandboxClass, WorkerPoolNamespace: r.Namespace, }) if err != nil { diff --git a/go/core/internal/translator/revision_test.go b/go/core/internal/translator/revision_test.go index ecde2c53c5..e817061e39 100644 --- a/go/core/internal/translator/revision_test.go +++ b/go/core/internal/translator/revision_test.go @@ -49,7 +49,7 @@ func TestRevisionDigestReplacesNameOnlyWorkerPoolTemplate(t *testing.T) { require.Equal(t, id, repeated) } -func TestRevisionDigestIncludesProvenance(t *testing.T) { +func TestRevisionDigestExcludesProvenance(t *testing.T) { revision := &Revision{Namespace: "agents", AgentName: "helper", Provenance: []byte(`[{"kind":"ConfigMap","hash":"first"}]`)} first, err := revision.Digest() if err != nil { @@ -60,8 +60,8 @@ func TestRevisionDigestIncludesProvenance(t *testing.T) { if err != nil { t.Fatal(err) } - if first == second { - t.Fatal("configuration change did not change runtime revision") + if first != second { + t.Fatal("source-only change altered runtime revision") } if len(first.Short()) != 12 || !strings.HasPrefix(first.String(), first.Short()) { t.Fatalf("short revision %q is not a prefix of %q", first.Short(), first.String()) diff --git a/go/core/pkg/app/app.go b/go/core/pkg/app/app.go index 13bfd5fe13..347a4fb1ec 100644 --- a/go/core/pkg/app/app.go +++ b/go/core/pkg/app/app.go @@ -277,6 +277,7 @@ func Run(ctx context.Context, input *configv1.ControllerConfig, opts Options) er return err } actors, err := substrate.Dial(ctx, substrate.Config{ + EgressPolicies: store, AteAPIEndpoint: cfg.GetSubstrate().GetApiEndpoint(), CAFile: cfg.GetSubstrate().GetCaFile(), ClientCertFile: cfg.GetSubstrate().GetClientCertFile(), diff --git a/go/core/pkg/migrations/core/000001_initial.sql b/go/core/pkg/migrations/core/000001_initial.sql index d04ebba522..27b0c5e819 100644 --- a/go/core/pkg/migrations/core/000001_initial.sql +++ b/go/core/pkg/migrations/core/000001_initial.sql @@ -34,6 +34,7 @@ CREATE TABLE runtime_revision ( source_snapshot JSONB NOT NULL, egress_destinations TEXT[] NOT NULL DEFAULT '{}', credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(credentials) = 'array'), + token_passthrough_origins TEXT[] NOT NULL DEFAULT '{}', actor_template_atespace TEXT CONSTRAINT runtime_revision_actor_template_namespace_not_null NOT NULL, actor_template_name TEXT NOT NULL, actor_template_uid TEXT NOT NULL DEFAULT '', @@ -52,6 +53,7 @@ CREATE TABLE agent_revision ( agent_card BYTEA NOT NULL, FOREIGN KEY (revision) REFERENCES runtime_revision(revision) ON DELETE CASCADE ); +CREATE INDEX agent_revision_source_idx ON agent_revision (agent_name, revision); CREATE TABLE sandbox_revision ( revision TEXT PRIMARY KEY, @@ -59,11 +61,14 @@ CREATE TABLE sandbox_revision ( sandbox_template_uid TEXT NOT NULL, FOREIGN KEY (revision) REFERENCES runtime_revision(revision) ON DELETE CASCADE ); +CREATE INDEX sandbox_revision_source_idx ON sandbox_revision (sandbox_template_name, revision); CREATE TABLE agent_definition ( namespace TEXT NOT NULL, agent_name TEXT NOT NULL, agent_uid TEXT NOT NULL, + grant_destinations TEXT[] NOT NULL DEFAULT '{}', + grant_credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(grant_credentials) = 'array'), desired_revision TEXT NOT NULL, latest_successful_revision TEXT REFERENCES agent_revision(revision) ON DELETE RESTRICT, retired_at TIMESTAMPTZ, @@ -78,6 +83,8 @@ CREATE TABLE sandbox_template_definition ( namespace TEXT NOT NULL, sandbox_template_name TEXT NOT NULL, sandbox_template_uid TEXT NOT NULL, + grant_destinations TEXT[] NOT NULL DEFAULT '{}', + grant_credentials JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(grant_credentials) = 'array'), desired_revision TEXT NOT NULL, latest_successful_revision TEXT REFERENCES sandbox_revision(revision) ON DELETE RESTRICT, retired_at TIMESTAMPTZ, @@ -135,6 +142,8 @@ CREATE TABLE runtime_instance ( ); CREATE INDEX runtime_instance_owner_idx ON runtime_instance (kind, user_id, id) WHERE state <> 'RUNTIME_STATE_DELETED'; +CREATE INDEX runtime_instance_revision_idx ON runtime_instance (prepared_revision, id) + WHERE state <> 'RUNTIME_STATE_DELETED' AND operation <> 'RUNTIME_OPERATION_DELETE'; CREATE TABLE session ( id UUID PRIMARY KEY, diff --git a/go/core/pkg/migrations/cross_track_test.go b/go/core/pkg/migrations/cross_track_test.go index ab4401efc4..13f8585722 100644 --- a/go/core/pkg/migrations/cross_track_test.go +++ b/go/core/pkg/migrations/cross_track_test.go @@ -184,7 +184,7 @@ var schemaQualifiedChecks = []sqlCheck{ {"search_path", regexp.MustCompile(`(?i)\bsearch_path\b`)}, {"SET SCHEMA", regexp.MustCompile(`(?i)\bSET\s+SCHEMA\b`)}, {"schema-qualified table", regexp.MustCompile(`(?i)\b(?:CREATE|ALTER|DROP)\s+TABLE\s+(?:IF\s+(?:NOT\s+)?EXISTS\s+)?\w+\.\w+`)}, - {"schema-qualified index target", regexp.MustCompile(`(?i)\bON\s+\w+\.\w+`)}, + {"schema-qualified index target", regexp.MustCompile(`(?i)\bCREATE\s+(?:UNIQUE\s+)?INDEX\b[^;]*?\bON\s+(?:ONLY\s+)?\w+\.\w+`)}, {"schema-qualified reference", regexp.MustCompile(`(?i)\bREFERENCES\s+\w+\.\w+`)}, } @@ -240,12 +240,15 @@ func TestSchemaViolations(t *testing.T) { }{ {"unqualified table", `CREATE TABLE IF NOT EXISTS eval_set (id TEXT);`, nil}, {"unqualified index", `CREATE INDEX IF NOT EXISTS i ON eval_set(id);`, nil}, + {"join condition", `CREATE VIEW live AS SELECT a.id FROM actors a JOIN revisions r ON a.revision = r.id;`, nil}, + {"index followed by join", `CREATE INDEX i ON actors(id); CREATE VIEW live AS SELECT a.id FROM actors a JOIN revisions r ON a.revision = r.id;`, nil}, {"schema in comment", `-- create table myschema.foo here` + "\n" + `CREATE TABLE IF NOT EXISTS foo (id TEXT);`, nil}, {"qualified table", `CREATE TABLE myschema.eval_set (id TEXT);`, []string{"schema-qualified table"}}, {"create schema", `CREATE SCHEMA IF NOT EXISTS myschema;`, []string{"CREATE SCHEMA"}}, {"set search_path", `SET search_path TO myschema;`, []string{"search_path"}}, {"set schema", `ALTER TABLE foo SET SCHEMA myschema;`, []string{"SET SCHEMA"}}, {"qualified index target", `CREATE INDEX i ON myschema.foo(id);`, []string{"schema-qualified index target"}}, + {"qualified concurrent index target", `CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS i ON ONLY myschema.foo(id);`, []string{"schema-qualified index target"}}, {"qualified reference", `ALTER TABLE foo ADD CONSTRAINT fk FOREIGN KEY (b) REFERENCES myschema.bar(id);`, []string{"schema-qualified reference"}}, } for _, tt := range tests { diff --git a/go/core/pkg/migrations/runner_test.go b/go/core/pkg/migrations/runner_test.go index 6348ccea19..019b448a56 100644 --- a/go/core/pkg/migrations/runner_test.go +++ b/go/core/pkg/migrations/runner_test.go @@ -323,6 +323,14 @@ func TestBuiltinMigrationsRoundTrip(t *testing.T) { t.Fatalf("%s versions = %v, want %v", source.Name, versions, expected) } } + if !testColumnExists(t, dsn, "runtime_revision", "token_passthrough_origins") || !testColumnExists(t, dsn, "agent_runtime_revision", "token_passthrough_origins") { + t.Fatal("runtime and agent revisions must expose passthrough origins") + } + for _, table := range []string{"agent_definition", "sandbox_template_definition"} { + if !testColumnExists(t, dsn, table, "grant_destinations") || !testColumnExists(t, dsn, table, "grant_credentials") { + t.Fatalf("%s source grants are missing", table) + } + } if !testColumnExists(t, dsn, "session", "deletion_reason") { t.Fatal("session deletion reason is missing") } diff --git a/go/core/test/e2e/README.md b/go/core/test/e2e/README.md index f59a3e04e4..0d99b43814 100644 --- a/go/core/test/e2e/README.md +++ b/go/core/test/e2e/README.md @@ -238,6 +238,22 @@ selection quality. The registry must accept pushes from the test runner and be reachable by Substrate workers through their existing registry rewrite. Private-registry credentials are not required. +`TestSessionEgressPolicyUpdatesLiveActors` loads the same OCI skill and runs +the same HTTP probe script through each harness's shell tool. During a single +active invocation, it adds a policy reference and revokes the destination, +checking actual script output and destination requests without replacing the +actor. It also checks that deleting a policy while suspended revokes access +after resume and in a checkpoint fork. Runtime configuration stays on the same +prepared revision. New sessions continue using that revision with current grants +even when a policy cannot resolve. `TestSandboxEgressPolicyUpdatesLiveActor` exercises live +grant, revocation, and resume through the standalone sandbox process API. + +`TestEgressPolicyCredentialDelivery` runs the same skill on every harness against +an HTTP server that rejects requests unless both Authorization (with a Bearer +prefix) and X-API-Key (without a prefix) match the Secret. The script has only +inert placeholders. The test and its harness subtests run in parallel. Actor +identity, policy checks, and credential injection use the normal Substrate path. + `TestSessionHTTPInteraction` discovers an Agent's Agent Card, invokes the advertised JSON-RPC interface, streams a second turn, and checks task persistence across HTTP and gRPC. `TestSessionHTTPResubscribeAndCancel` diff --git a/go/core/test/e2e/a2a_http_test.go b/go/core/test/e2e/a2a_http_test.go index fb386fdc48..eeae1f04f6 100644 --- a/go/core/test/e2e/a2a_http_test.go +++ b/go/core/test/e2e/a2a_http_test.go @@ -246,8 +246,6 @@ func TestSessionHTTPPushFromNonleader(t *testing.T) { func preparePushLeaderHandoff(t *testing.T) (string, func()) { t.Helper() kube := interactionKubeClient(t) - require.NoError(t, appsv1.AddToScheme(kube.Scheme())) - require.NoError(t, coordinationv1.AddToScheme(kube.Scheme())) deployments := &appsv1.DeploymentList{} require.NoError(t, kube.List(t.Context(), deployments, ctrlclient.InNamespace("kagent"), ctrlclient.MatchingLabels{"app.kubernetes.io/component": "controller"})) require.Len(t, deployments.Items, 1) diff --git a/go/core/test/e2e/egress_credentials_test.go b/go/core/test/e2e/egress_credentials_test.go new file mode 100644 index 0000000000..5573ed6a2e --- /dev/null +++ b/go/core/test/e2e/egress_credentials_test.go @@ -0,0 +1,97 @@ +package e2e_test + +import ( + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kagent-dev/kagent/go/api/v1alpha3" +) + +func TestEgressPolicyCredentialDelivery(t *testing.T) { + t.Parallel() + target := interactionTarget(t) + kube := interactionKubeClient(t) + const token = "explicit-policy-test-token" + destination := startAuthenticatedEgressDestination(t, token) + image := publishSkillImage(t) + forEachHarness(t, func(t *testing.T, harness testHarness) { + t.Parallel() + modelURL, model, runs := startEgressSkillMock(t, harness.skillsRoot(), destination, "credentials") + config := harness.createModel(t, kube, modelURL, nil) + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-credentials-", Namespace: "kagent", Labels: harness.labels()}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: config.Name}, SystemPrompt: "Load the egress skill and run its probe.", + Skills: []v1alpha3.AgentTemplateSkill{{Name: "egress", Source: v1alpha3.ArtifactSource{OCI: image, Path: "egress"}}}, + }} + createAndWaitInteractionTemplate(t, harness, kube, template) + fixture := newInteractionFixtureForTemplate(t, harness, target, template.Name) + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-header-", Namespace: "kagent"}, StringData: map[string]string{"token": token}} + require.NoError(t, kube.Create(t.Context(), secret)) + t.Cleanup(func() { assert.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), secret))) }) + ref := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secret.Name}, Key: "token"} + var refs []corev1.LocalObjectReference + for _, authentication := range []*v1alpha3.EgressAuthentication{ + {Header: "Authorization", Prefix: new("Bearer "), SecretKeyRef: ref}, + {Header: "X-API-Key", SecretKeyRef: ref}, + } { + policy := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-header-", Namespace: "kagent"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: destination, Authentication: authentication}}}} + require.NoError(t, kube.Create(t.Context(), policy)) + t.Cleanup(func() { assert.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), policy))) }) + refs = append(refs, corev1.LocalObjectReference{Name: policy.Name}) + } + run := runs("credentials") + finish := runEgressSkill(t, fixture, model, run, "initial_DENIED", "authenticated_ALLOWED") + run.expect(t, "initial", "DENIED") + agent := &v1alpha3.Agent{} + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKey{Namespace: "kagent", Name: template.Name}, agent)) + agent.Spec.EgressPolicyRefs = refs + require.NoError(t, kube.Update(t.Context(), agent)) + run.setPhase("authenticated") + run.expect(t, "authenticated", "ALLOWED") + // The HTTP server returns success only when both placeholders have been + // replaced from the Secret, with and without a prefix respectively. + finish() + }) +} + +func startAuthenticatedEgressDestination(t *testing.T, token string) string { + t.Helper() + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer "+token || r.Header.Get("X-API-Key") != token { + http.Error(w, "credential injection required", http.StatusForbidden) + return + } + _, _ = io.WriteString(w, "EGRESS_DESTINATION_AUTHENTICATED\n") + })) + require.NoError(t, server.Listener.Close()) + listener, err := net.Listen("tcp", "0.0.0.0:0") + require.NoError(t, err) + server.Listener = listener + origin := reachableServerURL(t, "http://"+listener.Addr().String(), "") + parsed, err := url.Parse(origin) + require.NoError(t, err) + server.Start() + t.Cleanup(server.Close) + // Prove a successful fetch cannot be produced by a missing/unchanged header. + for _, value := range []string{"", "kagent-credential-injected"} { + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://"+net.JoinHostPort("127.0.0.1", parsed.Port()), nil) + require.NoError(t, err) + request.Header.Set("Authorization", "Bearer "+value) + request.Header.Set("X-API-Key", value) + response, err := server.Client().Do(request) + require.NoError(t, err) + require.NoError(t, response.Body.Close()) + require.Equal(t, http.StatusForbidden, response.StatusCode) + } + return origin +} diff --git a/go/core/test/e2e/egress_test.go b/go/core/test/e2e/egress_test.go index 0088ccea35..075ecf00f5 100644 --- a/go/core/test/e2e/egress_test.go +++ b/go/core/test/e2e/egress_test.go @@ -2,74 +2,511 @@ package e2e_test import ( "context" + "encoding/json" + "fmt" + "io" "net" "net/http" "net/http/httptest" + "net/http/httputil" "net/url" + "strings" + "sync" "sync/atomic" "testing" + "time" a2atype "github.com/a2aproject/a2a-go/v2/a2a" + a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" + "github.com/a2aproject/a2a-go/v2/a2apb/v1/pbconv" + guestpb "github.com/agent-substrate/env/proto/ateenv/v1alpha" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/google/uuid" + "github.com/kagent-dev/mockllm" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/api/v1alpha3" + "github.com/kagent-dev/kagent/go/core/internal/substrate" ) -func TestSessionEgressDeniesUnconfiguredDestination(t *testing.T) { +// Every harness executes this same foreground skill through its normal shell +// tool. The test changes policy while the invocation is waiting for the probe. +const egressProbeScript = `#!/bin/sh +control=$1 +target=$2 +run=$3 +echo "EGRESS_PROBE_STARTED_$run" +for i in $(seq 1 900); do + phase=$(wget -T 5 -qO- "$control/phase") || exit 1 + if [ "$phase" = stop ]; then + echo "EGRESS_PROBE_FINISHED_$run" + wget -T 5 -qO- "$control/stopped" >/dev/null + exit + fi + if wget -T 3 -qO- --header 'Authorization: Bearer kagent-credential-injected' --header 'X-API-Key: kagent-credential-injected' "$target/?phase=$phase"; then + result=ALLOWED + else + result=DENIED + fi + echo "EGRESS_RESULT_${phase}_$result" + wget -T 5 -qO- "$control/result?phase=$phase&result=$result" >/dev/null || exit 1 + sleep 0.2 +done +exit 1 +` + +func TestSessionEgressPolicyUpdatesLiveActors(t *testing.T) { t.Parallel() + target := interactionTarget(t) + image := publishSkillImage(t) forEachHarness(t, func(t *testing.T, harness testHarness) { t.Parallel() - target := interactionTarget(t) - var reachedDenied atomic.Bool - origin := startModelRecorder(t, startMockLLMServer(t, interactionMocks, "mocks/invoke_agent.json"), func([]byte) error { - reachedDenied.Store(true) - return nil - }) - denied, err := url.Parse(reachableModelURL(t, origin.URL)) - require.NoError(t, err) - // Give the destination a distinct hostname even when host mocks use - // host.docker.internal. The gateway allowlist is keyed by hostname. kube := interactionKubeClient(t) - alias := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{GenerateName: "denied-origin-", Namespace: "kagent"}, - Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeExternalName, ExternalName: denied.Hostname()}, + destination, requests := startEgressDestination(t) + modelURL, model, runs := startEgressSkillMock(t, harness.skillsRoot(), destination, "live", "checkpoint", "fresh", "resumed", "fork") + config := harness.createModel(t, kube, modelURL, nil) + template := &v1alpha3.AgentTemplate{ObjectMeta: metav1.ObjectMeta{GenerateName: "live-egress-", Namespace: "kagent", Labels: harness.labels()}, Spec: v1alpha3.AgentTemplateSpec{ + ModelConfig: &corev1.LocalObjectReference{Name: config.Name}, SystemPrompt: "Load the egress skill and run its probe.", + Skills: []v1alpha3.AgentTemplateSkill{{Name: "egress", Source: v1alpha3.ArtifactSource{OCI: image, Path: "egress"}}}, + }} + createAndWaitInteractionTemplate(t, harness, kube, template) + fixture := newInteractionFixtureForTemplate(t, harness, target, template.Name) + actor, err := findSubstrateActor(fixture.ctx, fixture.system, "kagent", substrate.ActorName(fixture.sessionID)) + require.NoError(t, err) + actorUID := actor.GetMetadata().GetUid() + agent := &v1alpha3.Agent{} + key := ctrlclient.ObjectKey{Namespace: "kagent", Name: template.Name} + require.NoError(t, kube.Get(t.Context(), key, agent)) + revision := agent.Status.LatestSuccessfulRevision + grant := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "live-access-", Namespace: "kagent"}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: destination}}}} + require.NoError(t, kube.Create(t.Context(), grant)) + t.Cleanup(func() { require.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), grant))) }) + setGrant := func(origin string) { + t.Helper() + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKeyFromObject(grant), grant)) + grant.Spec.Destinations[0].Origin = origin + require.NoError(t, kube.Update(t.Context(), grant)) } - require.NoError(t, kube.Create(t.Context(), alias)) - t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), alias)) }) - denied.Host = net.JoinHostPort(alias.Name+".kagent.svc.cluster.local", denied.Port()) - // First prove the destination is reachable and speaks this harness's - // model protocol when explicitly configured. - control := newInteractionFixture(t, harness, target, denied.String()) - _, _, task := control.send(t, "What is 2+2?") - require.Equal(t, a2atype.TaskStateCompleted, task.Status.State) - require.Contains(t, taskText(task), "The answer is 4.") - require.True(t, reachedDenied.Load()) - reachedDenied.Store(false) + run := runs("live") + finish := runEgressSkill(t, fixture, model, run, "initial_DENIED", "allowed_ALLOWED", "conflict_ALLOWED", "fixed_ALLOWED", "revoked_DENIED") + run.expect(t, "initial", "DENIED") + require.Zero(t, requests("live_initial"), "ungranted requests reached the destination") + require.NoError(t, kube.Get(t.Context(), key, agent)) + agent.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: grant.Name}} + require.NoError(t, kube.Update(t.Context(), agent)) + run.setPhase("allowed") + run.expect(t, "allowed", "ALLOWED") + require.Positive(t, requests("live_allowed")) + waitEgressCondition := func(reason string) { + t.Helper() + require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, agent); err != nil { + return false, err + } + condition := apimeta.FindStatusCondition(agent.Status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied) + return condition != nil && condition.Reason == reason && + condition.ObservedGeneration == agent.Generation && + apimeta.IsStatusConditionTrue(agent.Status.Conditions, v1alpha3.AgentConditionReady), nil + })) + require.Equal(t, revision, agent.Status.LatestSuccessfulRevision) + } + waitEgressCondition(v1alpha3.EgressPoliciesReasonApplied) + // The opposite protocol on the model's exact host/port conflicts with + // pinned access. Existing grants must keep working in this invocation. + modelOrigin, err := url.Parse(modelURL) + require.NoError(t, err) + require.Equal(t, "http", modelOrigin.Scheme) + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKeyFromObject(grant), grant)) + grant.Spec.Destinations = append(grant.Spec.Destinations, v1alpha3.EgressDestination{Origin: "https://" + modelOrigin.Host}) + require.NoError(t, kube.Update(t.Context(), grant)) + waitEgressCondition(v1alpha3.EgressPoliciesReasonConflict) + require.True(t, apimeta.IsStatusConditionFalse(agent.Status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied)) + run.setPhase("conflict") + run.expect(t, "conflict", "ALLOWED") + require.Positive(t, requests("live_conflict")) + require.NoError(t, kube.Get(t.Context(), ctrlclient.ObjectKeyFromObject(grant), grant)) + grant.Spec.Destinations = grant.Spec.Destinations[:1] + require.NoError(t, kube.Update(t.Context(), grant)) + waitEgressCondition(v1alpha3.EgressPoliciesReasonApplied) + require.True(t, apimeta.IsStatusConditionTrue(agent.Status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied)) + run.setPhase("fixed") + run.expect(t, "fixed", "ALLOWED") - var reachedAllowed atomic.Bool - server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - reachedAllowed.Store(true) - // The model host is allowed, but its redirect must not expand the - // Actor's policy to another reachable, unconfigured destination. - redirect := *denied - redirect.Path, redirect.RawQuery = r.URL.Path, r.URL.RawQuery - http.Redirect(w, r, redirect.String(), http.StatusTemporaryRedirect) + setGrant("https://unused.example.com") + run.setPhase("revoked") + run.expect(t, "revoked", "DENIED") + revokedCount := requests("live_revoked") + require.Never(t, func() bool { return requests("live_revoked") != revokedCount }, time.Second, 100*time.Millisecond) + actor, err = findSubstrateActor(fixture.ctx, fixture.system, "kagent", substrate.ActorName(fixture.sessionID)) + require.NoError(t, err) + require.Equal(t, actorUID, actor.GetMetadata().GetUid(), "live updates must not replace the actor") + require.Equal(t, ateapipb.ActorState_ACTOR_STATE_RUNNING, actor.GetStatus().GetState()) + finish() + assertActorSuspended(t, fixture) + + // Save a checkpoint while the source is granted access, then revoke it while + // suspended. Neither resume nor a DATA fork may resurrect that old grant. + setGrant(destination) + run = runs("checkpoint") + finish = runEgressSkill(t, fixture, model, run, "initial_ALLOWED") + run.expect(t, "initial", "ALLOWED") + task := finish() + assertActorSuspended(t, fixture) + checkpoint := createCheckpoint(t, fixture.ctx, fixture.checkpoints, &apiv1.CreateCheckpointRequest{SessionId: fixture.sessionID, RequestId: uuid.NewString(), ExpectedHeadTaskId: string(task.ID)}).GetCheckpoint() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(fixture.ctx), time.Minute) + defer cancel() + _, err := fixture.checkpoints.DeleteCheckpoint(ctx, &apiv1.DeleteCheckpointRequest{CheckpointId: checkpoint.Id}) + require.NoError(t, err) + }) + require.NoError(t, kube.Delete(t.Context(), grant)) + require.NoError(t, wait.PollUntilContextTimeout(t.Context(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, agent); err != nil { + return false, err + } + condition := apimeta.FindStatusCondition(agent.Status.Conditions, v1alpha3.AgentConditionEgressPoliciesApplied) + return condition != nil && condition.Reason == v1alpha3.EgressPoliciesReasonNotFound && apimeta.IsStatusConditionTrue(agent.Status.Conditions, v1alpha3.AgentConditionReady), nil })) - _ = server.Listener.Close() - listener, err := net.Listen("tcp", "0.0.0.0:0") - if err != nil { - t.Fatal(err) - } - server.Listener = listener - server.Start() - t.Cleanup(server.Close) - fixture := newInteractionFixture(t, harness, target, reachableModelURL(t, server.URL)) - _, _, task = fixture.send(t, "What is 2+2?") - if !reachedAllowed.Load() { - t.Fatal("the configured model destination was not reached") + require.Equal(t, revision, agent.Status.LatestSuccessfulRevision, "policy edits must not replace the runtime revision") + created, err := fixture.sessions.CreateSession(fixture.ctx, &apiv1.CreateSessionRequest{Agent: &apiv1.ResourceReference{Namespace: key.Namespace, Name: key.Name}, RequestId: uuid.NewString()}) + require.NoError(t, err, "unresolved policy must not block the last successful runtime") + require.Equal(t, revision, created.Session.PreparedRevision) + fresh := *fixture + fresh.sessionID, fresh.contextID = created.Session.Id, created.Session.ContextId + t.Cleanup(func() { + require.NoError(t, deleteIdleSession(context.WithoutCancel(fresh.ctx), fresh.sessions, fresh.sessionID)) + }) + run = runs("fresh") + finish = runEgressSkill(t, &fresh, model, run, "initial_DENIED") + run.expect(t, "initial", "DENIED") + require.Zero(t, requests("fresh_initial")) + finish() + run = runs("resumed") + finish = runEgressSkill(t, fixture, model, run, "initial_DENIED") + run.expect(t, "initial", "DENIED") + require.Zero(t, requests("resumed_initial")) + finish() + assertActorSuspended(t, fixture) + actor, err = findSubstrateActor(fixture.ctx, fixture.system, "kagent", substrate.ActorName(fixture.sessionID)) + require.NoError(t, err) + require.Equal(t, actorUID, actor.GetMetadata().GetUid()) + forked, err := fixture.checkpoints.ForkSession(fixture.ctx, &apiv1.ForkSessionRequest{CheckpointId: checkpoint.Id, RequestId: uuid.NewString()}) + require.NoError(t, err) + fork := *fixture + fork.sessionID = forked.Session.Id + fork.contextID = forked.Session.ContextId + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(fork.ctx), time.Minute) + defer cancel() + require.NoError(t, deleteIdleSession(ctx, fork.sessions, fork.sessionID)) + }) + run = runs("fork") + finish = runEgressSkill(t, &fork, model, run, "initial_DENIED") + run.expect(t, "initial", "DENIED") + require.Zero(t, requests("fork_initial")) + finish() + }) +} + +type egressProbeRun struct { + name string + proxy *httputil.ReverseProxy + mu sync.Mutex + phase string + lastPhase, lastResult string + consecutive int + stopped chan struct{} + stopOnce sync.Once +} + +func (e *egressProbeRun) setPhase(phase string) { + e.mu.Lock() + defer e.mu.Unlock() + e.phase = e.name + "_" + phase +} +func (e *egressProbeRun) expect(t *testing.T, phase, result string) { + t.Helper() + require.Eventually(t, func() bool { + e.mu.Lock() + defer e.mu.Unlock() + return e.lastPhase == e.name+"_"+phase && e.lastResult == result && e.consecutive >= 3 + }, 90*time.Second, 100*time.Millisecond, "%s %s did not become %s", e.name, phase, result) +} +func (e *egressProbeRun) finish(ctx context.Context) error { + e.mu.Lock() + e.phase = "stop" + e.mu.Unlock() + select { + case <-e.stopped: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func runEgressSkill(t *testing.T, fixture *interactionFixture, model *modelRecorder, run *egressProbeRun, expectedResults ...string) func() *a2atype.Task { + t.Helper() + start := len(model.Requests("", "")) + _, request := newMessageRequest(t, "Run egress skill "+run.name+".") + request.Message.ContextId = fixture.sessionID + request.Tenant = fixture.tenant + type response struct { + message *a2apb.SendMessageResponse + err error + } + done := make(chan response, 1) + ctx, cancel := context.WithCancel(fixture.ctx) + t.Cleanup(func() { + cancel() + if t.Failed() { + for _, request := range model.Requests("", "")[start:] { + last, err := lastEgressModelInput(request.Body) + if err == nil && len(last) != 0 { + t.Logf("egress %s model input: %s", run.name, last) + } + } } - require.Equal(t, a2atype.TaskStateFailed, task.Status.State, "redirect task text = %q", taskText(task)) - require.False(t, reachedDenied.Load(), "redirect reached an unconfigured destination") }) + go func() { r, err := sendMessageWithRetry(ctx, fixture.client, request); done <- response{r, err} }() + return func() *a2atype.Task { + t.Helper() + stopCtx, stop := context.WithTimeout(ctx, 15*time.Second) + defer stop() + require.NoError(t, run.finish(stopCtx)) + var result response + select { + case result = <-done: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + require.NoError(t, result.err) + parsed, err := pbconv.FromProtoSendMessageResponse(result.message) + require.NoError(t, err) + task, ok := parsed.(*a2atype.Task) + require.True(t, ok) + require.Equal(t, a2atype.TaskStateCompleted, task.Status.State, taskText(task)) + require.Contains(t, taskText(task), "EGRESS_"+run.name+"_DONE") + // Check actual tool output in fresh model inputs, not the model's fixed text + // or conversation history copied into a resumed session or checkpoint fork. + var outputs []byte + for _, request := range model.Requests("", "")[start:] { + last, err := lastEgressModelInput(request.Body) + require.NoError(t, err) + outputs = append(outputs, last...) + } + for _, marker := range []string{"SKILL_BODY_EGRESS", "EGRESS_PROBE_STARTED_" + run.name, "EGRESS_PROBE_FINISHED_" + run.name} { + require.Contains(t, string(outputs), marker) + } + for _, result := range expectedResults { + require.Contains(t, string(outputs), "EGRESS_RESULT_"+run.name+"_"+result) + } + return task + } +} + +func lastEgressModelInput(raw []byte) (json.RawMessage, error) { + if len(raw) == 0 { + return nil, nil + } + var body struct { + Messages []json.RawMessage `json:"messages"` + Input []json.RawMessage `json:"input"` + } + if err := json.Unmarshal(raw, &body); err != nil { + return nil, err + } + messages := append(body.Messages, body.Input...) + if len(messages) == 0 { + return nil, nil + } + return messages[len(messages)-1], nil +} + +func startEgressDestination(t *testing.T) (string, func(string) int) { + t.Helper() + var mu sync.Mutex + requests := map[string]int{} + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requests[r.URL.Query().Get("phase")]++ + mu.Unlock() + _, _ = io.WriteString(w, "EGRESS_DESTINATION_REACHED\n") + })) + _ = server.Listener.Close() + listener, err := net.Listen("tcp", "0.0.0.0:0") + require.NoError(t, err) + server.Listener = listener + server.Start() + t.Cleanup(server.Close) + destination, err := url.Parse(reachableServerURL(t, server.URL, "")) + require.NoError(t, err) + // Keep a distinct origin even on Docker Desktop, where host mocks otherwise + // share host.docker.internal with the implicitly allowed model endpoint. + kube := interactionKubeClient(t) + alias := &corev1.Service{ObjectMeta: metav1.ObjectMeta{GenerateName: "egress-origin-", Namespace: "kagent"}, Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeExternalName, ExternalName: destination.Hostname()}} + require.NoError(t, kube.Create(t.Context(), alias)) + t.Cleanup(func() { require.NoError(t, kube.Delete(context.Background(), alias)) }) + destination.Host = net.JoinHostPort(alias.Name+".kagent.svc.cluster.local", destination.Port()) + return destination.String(), func(phase string) int { mu.Lock(); defer mu.Unlock(); return requests[phase] } +} + +func startEgressSkillMock(t *testing.T, root, destination string, runNames ...string) (string, *modelRecorder, func(string) *egressProbeRun) { + t.Helper() + require.NotEmpty(t, runNames) + var active atomic.Pointer[egressProbeRun] + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + run := active.Load() + if run == nil { + http.Error(w, "no active probe", http.StatusServiceUnavailable) + return + } + if strings.HasPrefix(r.URL.Path, "/egress/") { + if !strings.HasPrefix(r.URL.Path, "/egress/"+run.name+"/") { + http.Error(w, "old probe", http.StatusGone) + return + } + switch { + case strings.HasSuffix(r.URL.Path, "/phase"): + run.mu.Lock() + phase := run.phase + run.mu.Unlock() + _, _ = io.WriteString(w, phase) + case strings.HasSuffix(r.URL.Path, "/result"): + phase, result := r.URL.Query().Get("phase"), r.URL.Query().Get("result") + run.mu.Lock() + if phase == run.lastPhase && result == run.lastResult { + run.consecutive++ + } else { + run.lastPhase, run.lastResult, run.consecutive = phase, result, 1 + } + run.mu.Unlock() + case strings.HasSuffix(r.URL.Path, "/stopped"): + run.stopOnce.Do(func() { close(run.stopped) }) + default: + http.NotFound(w, r) + } + return + } + run.proxy.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + recorder := startModelRecorder(t, server.URL, nil) + modelURL := reachableModelURL(t, recorder.URL) + origin := strings.TrimSuffix(modelURL, "/v1") + raw, err := interactionMocks.ReadFile("mocks/invoke_egress_skill.json") + require.NoError(t, err) + runs := map[string]*egressProbeRun{} + for _, phase := range runNames { + command := fmt.Sprintf("sh %s/egress/probe.sh '%s/egress/%s' '%s' %s", root, origin, phase, destination, phase) + replacements := []string{"PHASE", phase, "ROOT", root} + args, err := json.Marshal(struct { + Command string `json:"command"` + }{command}) + require.NoError(t, err) + encodedArgs, err := json.Marshal(string(args)) + require.NoError(t, err) + codexArgs, err := json.Marshal(struct { + Command string `json:"cmd"` + YieldTimeMS int `json:"yield_time_ms"` + }{command, 30000}) + require.NoError(t, err) + encodedCodexArgs, err := json.Marshal(string(codexArgs)) + require.NoError(t, err) + encodedCommand, err := json.Marshal(command) + require.NoError(t, err) + replacements = append(replacements, `"COMMAND_ARGUMENTS"`, string(encodedArgs), `"CODEX_COMMAND_ARGUMENTS"`, string(encodedCodexArgs), `"COMMAND"`, string(encodedCommand)) + var config mockllm.Config + require.NoError(t, json.Unmarshal([]byte(strings.NewReplacer(replacements...).Replace(string(raw))), &config)) + upstream, err := url.Parse(startMockLLMConfig(t, config)) + require.NoError(t, err) + run := &egressProbeRun{name: phase, proxy: httputil.NewSingleHostReverseProxy(upstream), stopped: make(chan struct{})} + run.setPhase("initial") + runs[phase] = run + } + active.Store(runs[runNames[0]]) + return modelURL, recorder, func(name string) *egressProbeRun { + t.Helper() + run := runs[name] + require.NotNil(t, run, "unregistered egress probe %q", name) + active.Store(run) + return run + } +} + +func TestSandboxEgressPolicyUpdatesLiveActor(t *testing.T) { + t.Parallel() + f := newSandboxFixture(t) + kube := interactionKubeClient(t) + destination, requests := startEgressDestination(t) + instance := f.create(t, 5*time.Minute) + actor, err := findSubstrateActor(f.ctx, f.system, f.template.Namespace, substrate.ActorName(instance.Id)) + require.NoError(t, err) + actorUID := actor.GetMetadata().GetUid() + probe := func(phase string) string { + t.Helper() + command := fmt.Sprintf("if wget -T 3 -qO- '%s/?phase=%s'; then printf ALLOWED; else printf DENIED; fi", destination, phase) + process, err := f.processes.StartProcess(f.guestContext(instance.Id), &guestpb.StartProcessRequest{Command: []string{"sh", "-c", command}}) + require.NoError(t, err) + outputs, err := f.processes.StreamProcessOutputs(f.guestContext(instance.Id), &guestpb.StreamProcessOutputsRequest{ProcessId: process.ProcessId, Follow: true}) + require.NoError(t, err) + var result strings.Builder + for { + chunk, err := outputs.Recv() + if err == io.EOF { + break + } + require.NoError(t, err) + if chunk.Source == guestpb.OutputSource_OUTPUT_SOURCE_STDOUT { + result.Write(chunk.Data) + } + } + return result.String() + } + require.Contains(t, probe("initial"), "DENIED") + require.Zero(t, requests("initial")) + grant := &v1alpha3.EgressPolicy{ObjectMeta: metav1.ObjectMeta{GenerateName: "sandbox-access-", Namespace: f.template.Namespace}, Spec: v1alpha3.EgressPolicySpec{Destinations: []v1alpha3.EgressDestination{{Origin: destination}}}} + require.NoError(t, kube.Create(f.ctx, grant)) + t.Cleanup(func() { require.NoError(t, ctrlclient.IgnoreNotFound(kube.Delete(context.Background(), grant))) }) + template := &v1alpha3.SandboxTemplate{} + key := ctrlclient.ObjectKey{Namespace: f.template.Namespace, Name: f.template.Name} + require.NoError(t, kube.Get(f.ctx, key, template)) + template.Spec.EgressPolicyRefs = []corev1.LocalObjectReference{{Name: grant.Name}} + require.NoError(t, kube.Update(f.ctx, template)) + require.Eventually(t, func() bool { return strings.Contains(probe("allowed"), "ALLOWED") }, time.Minute, time.Second) + require.Positive(t, requests("allowed")) + require.NoError(t, kube.Delete(f.ctx, grant)) + require.Eventually(t, func() bool { return strings.Contains(probe("revoked"), "DENIED") }, time.Minute, time.Second) + require.NoError(t, wait.PollUntilContextTimeout(f.ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + if err := kube.Get(ctx, key, template); err != nil { + return false, err + } + condition := apimeta.FindStatusCondition(template.Status.Conditions, v1alpha3.SandboxTemplateConditionEgressPoliciesApplied) + return condition != nil && condition.Reason == v1alpha3.EgressPoliciesReasonNotFound && apimeta.IsStatusConditionTrue(template.Status.Conditions, "Ready"), nil + })) + original := instance + instance = f.create(t, 5*time.Minute) + require.Equal(t, original.PreparedRevision, instance.PreparedRevision) + require.Contains(t, probe("fresh"), "DENIED") + require.Zero(t, requests("fresh")) + instance = original + _, err = f.client.SuspendSandbox(f.ctx, &apiv1.SuspendSandboxRequest{SandboxId: instance.Id}) + require.NoError(t, err) + _, err = f.client.ResumeSandbox(f.ctx, &apiv1.ResumeSandboxRequest{SandboxId: instance.Id}) + require.NoError(t, err) + require.Contains(t, probe("resumed"), "DENIED") + require.Zero(t, requests("resumed")) + current, err := f.client.GetSandbox(f.ctx, &apiv1.GetSandboxRequest{SandboxId: instance.Id}) + require.NoError(t, err) + require.Equal(t, instance.PreparedRevision, current.Sandbox.PreparedRevision) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, current.Sandbox.State) + actor, err = findSubstrateActor(f.ctx, f.system, f.template.Namespace, substrate.ActorName(instance.Id)) + require.NoError(t, err) + require.Equal(t, actorUID, actor.GetMetadata().GetUid()) } diff --git a/go/core/test/e2e/harness_test.go b/go/core/test/e2e/harness_test.go index fe86992f59..cb45e61592 100644 --- a/go/core/test/e2e/harness_test.go +++ b/go/core/test/e2e/harness_test.go @@ -6,7 +6,10 @@ import ( ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + adkconfig "github.com/kagent-dev/kagent/go/adk/pkg/config" "github.com/kagent-dev/kagent/go/api/v1alpha3" + claudeconfig "github.com/kagent-dev/kagent/go/harness/claude/config" + codexconfig "github.com/kagent-dev/kagent/go/harness/codex/config" ) type testHarness struct { @@ -56,3 +59,14 @@ func (h testHarness) createModel(t *testing.T, kube ctrlclient.Client, modelURL return nil } } + +func (h testHarness) skillsRoot() string { + switch h.name { + case codexE2EHarness: + return codexconfig.SkillsRoot("/data") + case claudeE2EHarness: + return claudeconfig.SkillsRoot("/data") + default: + return adkconfig.DefaultSkillsRoot + } +} diff --git a/go/core/test/e2e/interaction_test.go b/go/core/test/e2e/interaction_test.go index 1fefbeef47..e0169dd08d 100644 --- a/go/core/test/e2e/interaction_test.go +++ b/go/core/test/e2e/interaction_test.go @@ -37,7 +37,6 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -50,7 +49,7 @@ import ( ) //go:embed mocks/invoke_agent.json mocks/invoke_golang_hitl_ask_user.json mocks/invoke_mcp_agent.json mocks/invoke_shared_agent.json mocks/invoke_structured_output.json mocks/invoke_native_ask_user.json -//go:embed mocks/invoke_mounted_skills.json +//go:embed mocks/invoke_mounted_skills.json mocks/invoke_egress_skill.json var interactionMocks embed.FS const structuredOutputSchema = `{"type":"object","properties":{"answer":{"type":"integer"},"explanation":{"type":"string"}},"required":["answer","explanation"],"additionalProperties":false}` @@ -1243,17 +1242,7 @@ func interactionKubeClient(t *testing.T) ctrlclient.Client { if err != nil { t.Fatalf("load Kubernetes config: %v", err) } - clientScheme := k8sruntime.NewScheme() - if err := corev1.AddToScheme(clientScheme); err != nil { - t.Fatalf("register Kubernetes core API: %v", err) - } - if err := discoveryv1.AddToScheme(clientScheme); err != nil { - t.Fatalf("register discovery API: %v", err) - } - if err := v1alpha3.AddToScheme(clientScheme); err != nil { - t.Fatalf("register kagent API: %v", err) - } - kube, err := ctrlclient.New(cfg, ctrlclient.Options{Scheme: clientScheme}) + kube, err := ctrlclient.New(cfg, ctrlclient.Options{Scheme: suiteKubeScheme}) if err != nil { t.Fatalf("create Kubernetes client: %v", err) } diff --git a/go/core/test/e2e/main_test.go b/go/core/test/e2e/main_test.go index 252d6bfa74..945816a0a0 100644 --- a/go/core/test/e2e/main_test.go +++ b/go/core/test/e2e/main_test.go @@ -8,16 +8,28 @@ import ( "os" "testing" + appsv1 "k8s.io/api/apps/v1" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" + "github.com/kagent-dev/kagent/go/api/v1alpha3" kagentenv "github.com/kagent-dev/kagent/go/core/pkg/env" "github.com/kagent-dev/kagent/go/pkg/logging" ) var suiteTraceReceiver *otlpTraceReceiver +var suiteKubeScheme = runtime.NewScheme() func TestMain(m *testing.M) { ctrl.SetLogger(logging.AsLogr(slog.Default())) + schemeBuilder := runtime.SchemeBuilder{corev1.AddToScheme, discoveryv1.AddToScheme, appsv1.AddToScheme, coordinationv1.AddToScheme, v1alpha3.AddToScheme} + if err := schemeBuilder.AddToScheme(suiteKubeScheme); err != nil { + fmt.Fprintf(os.Stderr, "register Kubernetes APIs: %v\n", err) + os.Exit(1) + } address := kagentenv.E2EOTLPListenAddress.Get() if address == "" { os.Exit(m.Run()) diff --git a/go/core/test/e2e/mocks/invoke_egress_skill.json b/go/core/test/e2e/mocks/invoke_egress_skill.json new file mode 100644 index 0000000000..3042e44924 --- /dev/null +++ b/go/core/test/e2e/mocks/invoke_egress_skill.json @@ -0,0 +1,306 @@ +{ + "openai": [ + { + "name": "skill_PHASE_discover", + "match": { + "match_type": "contains", + "message": { + "role": "user", + "content": "Run egress skill PHASE." + } + }, + "response": { + "id": "skill_PHASE_discover", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-4.1-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_load_egress_PHASE", + "type": "function", + "function": { + "name": "load_skill", + "arguments": "{\"name\": \"egress\"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ] + } + }, + { + "name": "skill_PHASE_execute", + "match": { + "match_type": "contains", + "message": { + "role": "tool", + "content": "SKILL_BODY_EGRESS" + } + }, + "response": { + "id": "skill_PHASE_execute", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-4.1-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_skill_PHASE", + "type": "function", + "function": { + "name": "bash", + "arguments": "COMMAND_ARGUMENTS" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ] + } + }, + { + "name": "skill_PHASE_result", + "match": { + "match_type": "contains", + "message": { + "role": "tool", + "content": "EGRESS_PROBE_FINISHED_PHASE", + "tool_call_id": "call_skill_PHASE" + } + }, + "response": { + "id": "skill_PHASE_result", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-4.1-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "EGRESS_PHASE_DONE" + }, + "finish_reason": "stop" + } + ] + } + } + ], + "openai_response": [ + { + "name": "skill_PHASE_discover", + "match": { + "match_type": "contains", + "input": "Run egress skill PHASE." + }, + "response": { + "id": "skill_PHASE_discover", + "object": "response", + "created_at": 1677652290, + "model": "gpt-5.2-codex", + "output": [ + { + "id": "fc_load_egress_PHASE", + "type": "function_call", + "status": "completed", + "call_id": "call_load_egress_PHASE", + "name": "exec_command", + "arguments": "{\"cmd\": \"cat ROOT/egress/SKILL.md\"}" + } + ], + "parallel_tool_calls": false, + "tools": [], + "usage": { + "input_tokens": 10, + "output_tokens": 8, + "total_tokens": 18 + } + } + }, + { + "name": "skill_PHASE_execute", + "match": { + "match_type": "contains", + "input": "SKILL_BODY_EGRESS" + }, + "response": { + "id": "skill_PHASE_execute", + "object": "response", + "created_at": 1677652290, + "model": "gpt-5.2-codex", + "output": [ + { + "id": "fc_skill_PHASE", + "type": "function_call", + "status": "completed", + "call_id": "call_skill_PHASE", + "name": "exec_command", + "arguments": "CODEX_COMMAND_ARGUMENTS" + } + ], + "parallel_tool_calls": false, + "tools": [], + "usage": { + "input_tokens": 10, + "output_tokens": 8, + "total_tokens": 18 + } + } + }, + { + "name": "skill_PHASE_result", + "match": { + "match_type": "contains", + "input": "EGRESS_PROBE_FINISHED_PHASE" + }, + "response": { + "id": "skill_PHASE_result", + "object": "response", + "created_at": 1677652291, + "model": "gpt-5.2-codex", + "output": [ + { + "id": "msg_skill_PHASE_done", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "EGRESS_PHASE_DONE", + "annotations": [] + } + ] + } + ], + "parallel_tool_calls": false, + "tools": [], + "usage": { + "input_tokens": 14, + "output_tokens": 5, + "total_tokens": 19 + } + } + } + ], + "anthropic": [ + { + "name": "skill_PHASE_discover", + "match": { + "match_type": "contains", + "message": { + "role": "user", + "content": [ + { + "type": "text", + "text": "Run egress skill PHASE." + } + ] + } + }, + "response": { + "id": "skill_PHASE_discover", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_load_egress_PHASE", + "name": "Skill", + "input": { + "skill": "egress" + } + } + ], + "model": "claude-sonnet-4-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 10, + "output_tokens": 4 + } + } + }, + { + "name": "skill_PHASE_execute", + "match": { + "match_type": "contains", + "message": { + "role": "user", + "content": [ + { + "type": "text", + "text": "SKILL_BODY_EGRESS" + } + ] + } + }, + "response": { + "id": "skill_PHASE_execute", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_skill_PHASE", + "name": "Bash", + "input": { + "command": "COMMAND" + } + } + ], + "model": "claude-sonnet-4-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 10, + "output_tokens": 4 + } + } + }, + { + "name": "skill_PHASE_result", + "match": { + "match_type": "contains", + "message": { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_skill_PHASE" + } + ] + } + }, + "response": { + "id": "skill_PHASE_result", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "EGRESS_PHASE_DONE" + } + ], + "model": "claude-sonnet-4-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 14, + "output_tokens": 4 + } + } + } + ] +} diff --git a/go/core/test/e2e/runtime_revision_test.go b/go/core/test/e2e/runtime_revision_test.go index 888ad5ca2b..268d0fc860 100644 --- a/go/core/test/e2e/runtime_revision_test.go +++ b/go/core/test/e2e/runtime_revision_test.go @@ -68,11 +68,12 @@ func TestRuntimeRevisionLifecycle(t *testing.T) { } // No FailedPrecondition retry: Ready must already be backed by a persisted // successful revision when the first create request arrives. - created, err := sessions.CreateSession(ctx, request(templateName)) + createRequest := request(templateName) + created, err := sessions.CreateSession(ctx, createRequest) require.NoError(t, err) source := created.GetSession() t.Cleanup(func() { deleteSession(source.GetId()) }) - sourceTaskID := send(source.GetId()) + send(source.GetId()) // Observe the actual runtime through the public inventory API before deleting // references, so an empty response cannot falsely prove cleanup later. @@ -111,11 +112,18 @@ func TestRuntimeRevisionLifecycle(t *testing.T) { return false, nil })) fallback, err := sessions.CreateSession(ctx, request(templateName)) - require.NoError(t, err, "an invalid edit must preserve the current UID's last-good runtime") - t.Cleanup(func() { deleteSession(fallback.GetSession().GetId()) }) + require.NoError(t, err, "an invalid edit must preserve the last successful runtime for fresh sessions") + fallbackID := fallback.GetSession().GetId() + t.Cleanup(func() { deleteSession(fallbackID) }) + require.NotEqual(t, source.GetId(), fallbackID) require.Equal(t, source.GetPreparedRevision(), fallback.GetSession().GetPreparedRevision()) - send(fallback.GetSession().GetId()) - deleteSession(fallback.GetSession().GetId()) + send(fallbackID) + deleteSession(fallbackID) + retried, err := sessions.CreateSession(ctx, createRequest) + require.NoError(t, err, "creation retries must retain the existing session's pinned runtime") + require.Equal(t, source.GetId(), retried.GetSession().GetId()) + require.Equal(t, source.GetPreparedRevision(), retried.GetSession().GetPreparedRevision()) + sourceTaskID := send(source.GetId()) checkpointResponse := createCheckpoint(t, ctx, checkpoints, &apiv1.CreateCheckpointRequest{SessionId: source.GetId(), RequestId: uuid.NewString(), ExpectedHeadTaskId: sourceTaskID}) checkpointID := checkpointResponse.GetCheckpoint().GetId() @@ -132,19 +140,21 @@ func TestRuntimeRevisionLifecycle(t *testing.T) { agent := &v1alpha3.Agent{ObjectMeta: metav1.ObjectMeta{Namespace: template.Namespace, Name: template.Name}} require.NoError(t, kube.Get(ctx, ctrlclient.ObjectKeyFromObject(agent), agent)) require.NoError(t, kube.Delete(ctx, agent)) - // Wait for the controller to retire the pair, using the same public create - // path as a client. Dispose of any session created before retirement wins. + // Kubernetes deletion reaches the database through the controller. Wait for + // retirement, releasing any sessions created before that event is observed + // so they cannot keep the runtime alive during the GC assertion below. require.NoError(t, wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - response, err := sessions.CreateSession(ctx, request(templateName)) + created, err := sessions.CreateSession(ctx, request(templateName)) if status.Code(err) == codes.FailedPrecondition { return true, nil } if err != nil { return false, err } - deleteSession(response.GetSession().GetId()) - return false, nil - })) + id := created.GetSession().GetId() + t.Cleanup(func() { deleteSession(id) }) + return false, deleteIdleSession(ctx, sessions, id) + }), "Agent deletion must eventually block fresh sessions") forked, err := checkpoints.ForkSession(ctx, &apiv1.ForkSessionRequest{CheckpointId: checkpointID, RequestId: uuid.NewString()}) require.NoError(t, err, "a checkpoint must retain runnable inputs after its source and template are deleted") forkID := forked.GetSession().GetId() diff --git a/go/core/test/e2e/session_expiration_test.go b/go/core/test/e2e/session_expiration_test.go index a045e2b468..9ab841c561 100644 --- a/go/core/test/e2e/session_expiration_test.go +++ b/go/core/test/e2e/session_expiration_test.go @@ -92,7 +92,6 @@ func TestSessionIdleExpiration(t *testing.T) { func setSessionExpirationPolicy(t *testing.T, target, ttl, pollInterval string) { t.Helper() kube := interactionKubeClient(t) - require.NoError(t, appsv1.AddToScheme(kube.Scheme())) deployments := &appsv1.DeploymentList{} require.NoError(t, kube.List(t.Context(), deployments, ctrlclient.InNamespace("kagent"), ctrlclient.MatchingLabels{"app.kubernetes.io/component": "controller"})) require.Len(t, deployments.Items, 1) diff --git a/go/core/test/e2e/skills_test.go b/go/core/test/e2e/skills_test.go index de8ca4d81d..d88920d4bc 100644 --- a/go/core/test/e2e/skills_test.go +++ b/go/core/test/e2e/skills_test.go @@ -36,13 +36,7 @@ func TestOCISkillsSurviveSuspension(t *testing.T) { image := publishSkillImage(t) forEachHarness(t, func(t *testing.T, harness testHarness) { t.Parallel() - root := "/skills" - switch harness.name { - case codexE2EHarness: - root = "/data/codex/skills" - case claudeE2EHarness: - root = "/data/generated/claude/.claude/skills" - } + root := harness.skillsRoot() model, setPhase := startMountedSkillsMock(t, root) kube := interactionKubeClient(t) config := harness.createModel(t, kube, reachableModelURL(t, model.URL), nil) @@ -112,6 +106,8 @@ func publishSkillImage(t *testing.T) string { body string mode int64 }{ + {"egress/SKILL.md", "---\nname: egress\ndescription: Probe live HTTP egress permissions.\n---\nSKILL_BODY_EGRESS: Run probe.sh against the requested control and destination URLs.\n", 0o644}, + {"egress/probe.sh", egressProbeScript, 0o755}, {"standalone/SKILL.md", "---\nname: standalone\ndescription: Execute the standalone mounted skill.\n---\nSKILL_BODY_STANDALONE: Run run.sh with the requested phase.\n", 0o644}, {"standalone/run.sh", "#!/bin/sh\nprintf 'mounted-standalone-%s:' \"$1\"\n", 0o755}, {"plugin/plugin.json", `{"$schema":"https://agent-plugins.org/schemas/1.0.0/plugin.schema.json","name":"mounted-test"}`, 0o644}, diff --git a/go/harness/claude/config/paths.go b/go/harness/claude/config/paths.go new file mode 100644 index 0000000000..2b1953ba29 --- /dev/null +++ b/go/harness/claude/config/paths.go @@ -0,0 +1,8 @@ +package config + +import "path/filepath" + +// SkillsRoot is the directory of selected skills in the generated Claude plugin. +func SkillsRoot(dataDir string) string { + return filepath.Join(dataDir, "generated", "claude", ".claude", "skills") +} diff --git a/go/harness/claude/internal/adapter/adapter.go b/go/harness/claude/internal/adapter/adapter.go index b6067d9803..1945cee735 100644 --- a/go/harness/claude/internal/adapter/adapter.go +++ b/go/harness/claude/internal/adapter/adapter.go @@ -59,7 +59,7 @@ func New(ctx context.Context, input Input) (*driver.ProcessDriver, error) { } var pluginDirs []string if cfg.SkillResources != nil { - skillsDir := filepath.Join(skillRoot, ".claude", "skills") + skillsDir := config.SkillsRoot(input.DurableDir) if err := utils.EnsurePrivateDir(skillsDir); err != nil { return nil, fmt.Errorf("prepare generated Claude skills directory: %w", err) } diff --git a/go/harness/codex/config/paths.go b/go/harness/codex/config/paths.go new file mode 100644 index 0000000000..0b76ba467a --- /dev/null +++ b/go/harness/codex/config/paths.go @@ -0,0 +1,8 @@ +package config + +import "path/filepath" + +// SkillsRoot is the directory where Codex discovers installed skills. +func SkillsRoot(dataDir string) string { + return filepath.Join(dataDir, "codex", "skills") +} diff --git a/go/harness/codex/internal/adapter/adapter.go b/go/harness/codex/internal/adapter/adapter.go index 1552695aa5..d92bb6ca02 100644 --- a/go/harness/codex/internal/adapter/adapter.go +++ b/go/harness/codex/internal/adapter/adapter.go @@ -41,7 +41,8 @@ func New(ctx context.Context, input Input) (*driver.ProcessDriver, error) { return nil, fmt.Errorf("workspace and durable directories must be absolute paths") } codexHome := filepath.Join(input.DurableDir, "codex") - for _, directory := range []string{input.Workspace, codexHome, filepath.Join(codexHome, "agents"), filepath.Join(codexHome, "skills")} { + skillsRoot := config.SkillsRoot(input.DurableDir) + for _, directory := range []string{input.Workspace, codexHome, filepath.Join(codexHome, "agents"), skillsRoot} { if err := utils.EnsurePrivateDir(directory); err != nil { return nil, fmt.Errorf("prepare Codex directory %q: %w", directory, err) } @@ -49,13 +50,13 @@ func New(ctx context.Context, input Input) (*driver.ProcessDriver, error) { if err := reconcileGeneratedDir(filepath.Join(codexHome, "agents"), agentFileNames(cfg.Agents)); err != nil { return nil, fmt.Errorf("reconcile Codex agents: %w", err) } - if err := reconcileGeneratedDir(filepath.Join(codexHome, "skills"), nil); err != nil { + if err := reconcileGeneratedDir(skillsRoot, nil); err != nil { return nil, fmt.Errorf("reconcile Codex skills: %w", err) } if cfg.SkillResources != nil { materialized, err := agentplugins.Materialize(ctx, *cfg.SkillResources, agentplugins.Paths{ Packages: filepath.Join(codexHome, "packages"), - Skills: filepath.Join(codexHome, "skills"), + Skills: skillsRoot, }) if err != nil { return nil, fmt.Errorf("materialize Codex skills: %w", err) diff --git a/helm/kagent-crds/templates/api.kagent.dev_agents.yaml b/helm/kagent-crds/templates/api.kagent.dev_agents.yaml index bdc7bbab3e..6ba119ed9f 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_agents.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_agents.yaml @@ -52,6 +52,36 @@ spec: are local to the Agent's namespace, including references nested in inline template specs. properties: + egressPolicyRefs: + description: |- + EgressPolicyRefs grants additional access to the whole runtime, including + shared subagents. Policies resolve in this Agent's namespace. Integration + access inferred from models and tools is always included. Changes update + existing Sessions without a new invocation or runtime restart. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: egressPolicyRefs requires non-empty names + rule: self.all(ref, has(ref.name) && ref.name != '') harnessRef: description: |- HarnessRef selects the platform-managed runtime configuration that executes @@ -487,6 +517,11 @@ spec: description: AgentStatus reports compilation and preparation of an Agent. properties: conditions: + description: |- + Conditions report runtime readiness and independent egress grant admission. + EgressPoliciesApplied is False for EgressPolicyNotFound, InvalidEgressPolicy, + or EgressGrantConflict; Applied means all grants are admitted. Grant issues + do not affect Ready. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -541,7 +576,7 @@ spec: - status - type type: object - maxItems: 4 + maxItems: 5 type: array x-kubernetes-list-map-keys: - type diff --git a/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml b/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml new file mode 100644 index 0000000000..3945fc758b --- /dev/null +++ b/helm/kagent-crds/templates/api.kagent.dev_egresspolicies.yaml @@ -0,0 +1,140 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.22.0 + name: egresspolicies.api.kagent.dev +spec: + group: api.kagent.dev + names: + categories: + - kagent + kind: EgressPolicy + listKind: EgressPolicyList + plural: egresspolicies + singular: egresspolicy + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: |- + EgressPolicy is a reusable grant of additional network access. Workloads in its + namespace select it explicitly; changes update their existing runtime Actors. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EgressPolicySpec grants additional HTTP(S) access to workloads + in its namespace. + properties: + destinations: + description: Destinations are additional origins the workload may + reach. + items: + description: EgressDestination grants access to one HTTP(S) origin, + including all its paths. + properties: + authentication: + description: |- + Authentication replaces a placeholder header at the gateway. Requests + must send the header; its credential value is never supplied to the runtime. + properties: + header: + description: |- + Header is the request header carrying the credential, for example + Authorization or X-API-Key. + maxLength: 256 + pattern: ^[!#$%&'*+.^_|~0-9A-Za-z-]+$ + type: string + x-kubernetes-validations: + - message: this header cannot carry an egress credential + rule: '!(self.lowerAscii() in [''host'', ''connection'', + ''content-length'', ''transfer-encoding'', ''te'', ''trailer'', + ''upgrade'', ''keep-alive'', ''proxy-authorization'', + ''proxy-authenticate'', ''cookie''])' + prefix: + description: |- + Prefix is prepended verbatim to the Secret value, for example "Bearer ". + Include any separating space. Omitted or empty means no prefix. + maxLength: 256 + pattern: ^[^\x00-\x08\x0a-\x1f\x7f]*$ + type: string + secretKeyRef: + description: SecretKeyRef selects a required key in the + EgressPolicy's namespace. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: secretKeyRef requires a name and key + rule: has(self.name) && self.name != '' && self.key != + '' + - message: egress credentials cannot be optional + rule: '!has(self.optional) || !self.optional' + required: + - header + - secretKeyRef + type: object + origin: + description: |- + Origin is a scheme, DNS hostname, and optional port. A wildcard replaces + exactly one leftmost DNS label, for example https://*.example.com. + maxLength: 270 + pattern: ^https?://(\*\.([a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?\.)+|([a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?\.)*)[a-z]([-a-z0-9]{0,61}[a-z0-9])?(:([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?$ + type: string + required: + - origin + type: object + x-kubernetes-validations: + - message: authentication requires an exact HTTP(S) origin + rule: '!has(self.authentication) || !self.origin.contains(''*'')' + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - origin + x-kubernetes-list-type: map + required: + - destinations + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/helm/kagent-crds/templates/api.kagent.dev_remotemcpservers.yaml b/helm/kagent-crds/templates/api.kagent.dev_remotemcpservers.yaml index 0ade50f33d..5f78ca86b9 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_remotemcpservers.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_remotemcpservers.yaml @@ -128,10 +128,15 @@ spec: description: type: string headersFrom: + description: |- + HeadersFrom supplies request headers. Secret-backed headers are injected by + the egress gateway and cannot use routing, framing, hop-by-hop, + proxy-authentication, or Cookie headers. At most 64 headers may be supplied. items: description: ValueRef represents a configuration value properties: name: + maxLength: 256 type: string value: type: string @@ -161,9 +166,20 @@ spec: - name type: object x-kubernetes-validations: + - fieldPath: .name + message: Secret-backed headers require a valid HTTP header name + rule: '!has(self.valueFrom) || self.valueFrom.type != ''Secret'' + || self.name.matches(''^[!#$%&\\x27*+.^_`|~0-9A-Za-z-]+$'')' + - fieldPath: .name + message: this header cannot carry an egress credential + rule: '!has(self.valueFrom) || self.valueFrom.type != ''Secret'' + || !(self.name.lowerAscii() in [''host'', ''connection'', ''content-length'', + ''transfer-encoding'', ''te'', ''trailer'', ''upgrade'', ''keep-alive'', + ''proxy-authorization'', ''proxy-authenticate'', ''cookie''])' - message: exactly one of the fields in [value valueFrom] must be set rule: (has(self.value)?1:0)+(has(self.valueFrom)?1:0) == 1 + maxItems: 64 type: array protocol: default: STREAMABLE_HTTP diff --git a/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml b/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml index 4749ca0147..bf2fd9509d 100644 --- a/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml +++ b/helm/kagent-crds/templates/api.kagent.dev_sandboxtemplates.yaml @@ -51,6 +51,34 @@ spec: description: SandboxTemplateSpec defines standalone sandbox environment configuration. properties: + egressPolicyRefs: + description: |- + EgressPolicyRefs grants additional access in this template's namespace. + Changes update the access of existing sandboxes without restarting them. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: egressPolicyRefs requires non-empty names + rule: self.all(ref, has(ref.name) && ref.name != '') env: description: |- Env supplies runtime environment defaults. @@ -132,7 +160,11 @@ spec: template inputs. properties: conditions: - description: Conditions describe whether a prepared revision is available. + description: |- + Conditions report runtime readiness and independent egress grant admission. + EgressPoliciesApplied is False for EgressPolicyNotFound, InvalidEgressPolicy, + or EgressGrantConflict; Applied means all grants are admitted. Grant issues + do not affect Ready. items: description: Condition contains details for one aspect of the current state of this API Resource. diff --git a/helm/kagent/templates/rbac/getter-role.yaml b/helm/kagent/templates/rbac/getter-role.yaml index c136b629ec..007b3fbab0 100644 --- a/helm/kagent/templates/rbac/getter-role.yaml +++ b/helm/kagent/templates/rbac/getter-role.yaml @@ -4,6 +4,7 @@ resources: - harnesses - agents + - egresspolicies - sandboxtemplates - agenttemplates - modelconfigs diff --git a/helm/kagent/tests/rbac_test.yaml b/helm/kagent/tests/rbac_test.yaml index ef6d6debb2..4e5a82d805 100644 --- a/helm/kagent/tests/rbac_test.yaml +++ b/helm/kagent/tests/rbac_test.yaml @@ -140,9 +140,18 @@ tests: - equal: path: rules[0].apiGroups value: [api.kagent.dev] + - equal: + path: rules[0].resources + value: [harnesses, agents, egresspolicies, sandboxtemplates, agenttemplates, modelconfigs, modelproviderconfigs, remotemcpservers] + template: rbac/getter-role.yaml + - equal: + path: rules[0].verbs + value: [get, list, watch] + template: rbac/getter-role.yaml - equal: path: rules[0].resources value: [harnesses, agents, sandboxtemplates, agenttemplates, modelconfigs, modelproviderconfigs, remotemcpservers] + template: rbac/writer-role.yaml - it: should grant namespaced agent access in the new group set: @@ -154,9 +163,18 @@ tests: - equal: path: rules[0].apiGroups value: [api.kagent.dev] + - equal: + path: rules[0].resources + value: [harnesses, agents, egresspolicies, sandboxtemplates, agenttemplates, modelconfigs, modelproviderconfigs, remotemcpservers] + template: rbac/getter-role.yaml + - equal: + path: rules[0].verbs + value: [get, list, watch] + template: rbac/getter-role.yaml - equal: path: rules[0].resources value: [harnesses, agents, sandboxtemplates, agenttemplates, modelconfigs, modelproviderconfigs, remotemcpservers] + template: rbac/writer-role.yaml - it: should have correct labels on all resources templates: diff --git a/ui/src/api/domain/agents.ts b/ui/src/api/domain/agents.ts index 2ee45c74a6..e818b80df4 100644 --- a/ui/src/api/domain/agents.ts +++ b/ui/src/api/domain/agents.ts @@ -8,6 +8,7 @@ export type AgentTemplateSource = export interface AgentSpec { template: AgentTemplateSource; harnessRef: { name: string }; + egressPolicyRefs?: { name: string }[]; } /** One condition the controller recorded for an Agent. */ export interface AgentCondition { diff --git a/ui/src/pages/agents/AgentFormPage.tsx b/ui/src/pages/agents/AgentFormPage.tsx index 17eb636b01..44bc160331 100644 --- a/ui/src/pages/agents/AgentFormPage.tsx +++ b/ui/src/pages/agents/AgentFormPage.tsx @@ -117,6 +117,7 @@ function AgentForm({ agent }: { agent?: Agent }) { setSaving(true); setFailure(undefined); const agentSpec = { + ...(spec?.egressPolicyRefs ? { egressPolicyRefs: spec.egressPolicyRefs } : {}), template: templateSource === "reference" ? { ref: { name: templateRef! } } : { inline: specFromDraft(inlineTemplate, spec?.template.inline) },