Sync upstream asynchronous snapshots and sandbox runtime updates

.github/workflows/helm-e2e.yaml

@@ -65,7 +65,7 @@ jobs:
         # the chart composes {registry}/{repository}/{component}, so the local
         # registry serves each image where the default repository expects it --
         # the same path-preserving rule a production mirror follows.
-        for component in ateapi atecontroller atelet podcertcontroller atenet objectstore-plugin credential-provider/kubernetes-secrets; do
+        for component in ateapi ate-idp-server atecontroller atelet podcertcontroller atenet objectstore-plugin credential-provider/kubernetes-secrets; do
           KO_DOCKER_REPO="localhost:5001/kagent-dev/substrate/${component##*/}" \
             ./hack/run-tool.sh ko build --bare --tags helm-e2e \
               --platform linux/amd64 "./cmd/${component}"

.github/workflows/pr-workflow.yaml

@@ -58,24 +58,46 @@ jobs:
     # we only test on amd64, make sure everything still builds for arm64
     - name: build for arm64
       run: GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build ./...
-    # gotestsum wraps go test to emit JUnit; the go test arguments are unchanged.
+    # gotestsum wraps go test to emit JUnit and the go test -json stream.
+    # the log only gets a line per package and failed test output, the full
+    # output is in the uploaded json
+    # NOTE: no -v with gotestsum, it replaces the -test.v=test2json that -json
+    # sets, which marks t.Error output for the failure report below
     - name: unit tests
-      run: hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/unit.xml"
-        --jsonfile "${ARTIFACTS}/unit.json" --format standard-verbose
-        -- -race -v ./...
+      run: |
+        go -C tools/junittool run . register "${ARTIFACTS}/unit.xml"
+        hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/unit.xml" \
+          --jsonfile "${ARTIFACTS}/unit.json" \
+          --format pkgname-and-test-fails --format-hide-empty-pkg \
+          -- -race ./...
     # tools/apitool has its own module so we need to run it explicitly.
     # run-tool.sh resolves the repo root itself, so it works from the subdir.
     - name: apitool tests
       working-directory: tools/apitool
-      run: ../../hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/apitool.xml"
-        --jsonfile "${ARTIFACTS}/apitool.json" --format standard-verbose
-        -- -race -v ./...
+      run: |
+        go -C ../junittool run . register "${ARTIFACTS}/apitool.xml"
+        ../../hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/apitool.xml" \
+          --jsonfile "${ARTIFACTS}/apitool.json" \
+          --format pkgname-and-test-fails --format-hide-empty-pkg \
+          -- -race ./...
+    # tools/junittool has its own module too
+    - name: junittool tests
+      working-directory: tools/junittool
+      run: |
+        go run . register "${ARTIFACTS}/junittool.xml"
+        ../../hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/junittool.xml" \
+          --jsonfile "${ARTIFACTS}/junittool.json" \
+          --format pkgname-and-test-fails --format-hide-empty-pkg \
+          -- -race ./...
     # Kept apart from the root module so it can move to its own repository.
     - name: gcp-secret-manager credential provider tests
       working-directory: internal/plugins/gcp-secret-manager
-      run: ../../../hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/gcp-secret-manager.xml"
-        --jsonfile "${ARTIFACTS}/gcp-secret-manager.json" --format standard-verbose
-        -- -race -v ./...
+      run: |
+        go -C ../../../tools/junittool run . register "${ARTIFACTS}/gcp-secret-manager.xml"
+        ../../../hack/run-tool.sh gotestsum --junitfile "${ARTIFACTS}/gcp-secret-manager.xml" \
+          --jsonfile "${ARTIFACTS}/gcp-secret-manager.json" \
+          --format pkgname-and-test-fails --format-hide-empty-pkg \
+          -- -race ./...
     # The root verifiers stop at the module's go.mod, so it gets its own run:
     # gofmt, go vet and the tidy check, then golangci-lint with the root config.
     - name: gcp-secret-manager credential provider verify
@@ -104,12 +126,16 @@ jobs:
     - name: root-gated tests
       env:
         ROOT_JUNIT_FILE: ${{ github.workspace }}/_artifacts/root.xml
-      run: hack/run-root-tests.sh -race -v
+      run: hack/run-root-tests.sh -race
     # An exit code cannot distinguish "all passed" from "none ran". A registered
     # file that was never written fails too.
     - name: Require tests to have run
       if: always()
       run: go -C tools/junittool run . verify -manifest "${ARTIFACTS}/expected-junit.txt"
+    # list each failed test and its error in the job summary and as annotations
+    - name: Summarize test failures
+      if: always()
+      run: go -C tools/junittool run . report "${ARTIFACTS}"
     - name: Upload test results
       if: always()
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
@@ -253,7 +279,7 @@ jobs:
       env:
         E2E_PARALLELISM: 8
         E2E_JUNIT_FILE: ${{ github.workspace }}/_artifacts/e2e-gvisor${{ env.LANE_SUFFIX }}.xml
-      run: hack/run-e2e-kind.sh -v -args --no-color --preview="${ATE_PREVIEW}"
+      run: hack/run-e2e-kind.sh -args --no-color --preview="${ATE_PREVIEW}"
     - name: Focus the node on micro-VM
       run: hack/e2e-focus-sandbox-class.sh microvm
     - name: Run E2E tests (micro-VM)
@@ -263,14 +289,17 @@ jobs:
         E2E_SANDBOX_CLASS: microvm
         E2E_PARALLELISM: 8
         E2E_JUNIT_FILE: ${{ github.workspace }}/_artifacts/e2e-microvm${{ env.LANE_SUFFIX }}.xml
-      run: hack/run-e2e-kind.sh -v -args --no-color --preview="${ATE_PREVIEW}"
+      run: hack/run-e2e-kind.sh -args --no-color --preview="${ATE_PREVIEW}"
     # Every lane must have reported tests; a `-run` filter that stops matching
     # otherwise leaves its lane green and empty. The set comes from the manifest
     # each lane registers itself in, so adding a lane needs no change here, and
     # run-e2e.sh refuses to run in CI without E2E_JUNIT_FILE.
     - name: Require every lane to have run tests
       if: always()
       run: go -C tools/junittool run . verify -manifest "${ARTIFACTS}/expected-junit.txt"
+    - name: Summarize test failures
+      if: always()
+      run: go -C tools/junittool run . report "${ARTIFACTS}"
     - name: Save Go build cache
       # Main only, so PRs restore the latest main build rather than each
       # storing its own copy.

AGENTS.md

@@ -102,7 +102,7 @@ The following is what Substrate currently offers.
 Keep this up to date when updating AGENTS.md.
 
 - **Workload Isolation**: The project uses `gVisor` (`runsc`) for sandboxing and security isolation of workloads on pods.
-- **JWTs and JWKs**: Signing (`internal/localjwtauthority`), verification (`internal/oidcjwt`), and JWK set publishing (`internal/oidcdiscovery`) are built on the standard library's crypto packages. Extend those packages rather than adding go-jose or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses.
+- **JWTs and JWKs**: Signing and JWK set encoding (`internal/localjwtauthority`) and verification (`internal/oidcjwt`) are built on the standard library's crypto packages. Extend those packages rather than adding go-jose or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses.
 - **Ingress Authentication**: `atenet-router --ingress-auth-mode=static-mtls` requires clients to present a SPIFFE certificate signed by `--ingress-client-ca-file` whose ID is in `--ingress-allowed-spiffe-ids`; that list is the whole authorization policy, so any listed client may reach any Actor. The default, `deprecated-insecure`, checks a certificate only when a client presents one, and also serves plaintext listeners that accept any client. Install with it via `ate-setup --ingress-auth-mode=static-mtls` (`manifests/ate-install/components/router-static-mtls`); the allowlist lives in `manifests/ate-install/atenet-router.yaml`.
 - **Control API Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), on by default. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): with enforcement on, an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/authorization.md` for roles and known gaps.
 

README.md

@@ -268,6 +268,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit
 * `cmd/ateapi`: The core control plane API server exposing gRPC endpoints to manage actor and worker lifecycles.
 * `cmd/atelet`: A node-level DaemonSet that supervises physical worker pods, coordinates snapshotting, and manages state transfers.
 * `cmd/atecontroller`: A Kubernetes controller that reconciles WorkerPool custom resources.
+* `cmd/ate-idp-server`: Serves the OpenID discovery document and key set that relying parties use to verify actor JWTs.
 * `cmd/atenet`: A combined networking controller providing Envoy routing and proxy sidecars.
 * `cmd/ateworker-gvisor`: An interior-pod helper running inside sandboxed worker pods to execute `runsc` checkpoint and restore commands.
 * `cmd/ateworker-microvm`: The micro-VM peer of `ateworker-gvisor`, running actors as cloud-hypervisor VMs.

benchmarking/README.md

@@ -118,13 +118,14 @@ Four flags control the optional post-run measurements described in
 [Benchmark output files](#benchmark-output-files):
 
 * `--cluster-facts` / `--no-cluster-facts`: read node capacity and worker pod
-  count from the Kubernetes API once the run ends, to derive density frontiers.
+  count from the Kubernetes API once the run ends, to derive density frontiers,
+  and worker pod restarts and evictions around the run, for `oom_events`.
   On by default. Pass `--no-cluster-facts` to skip Kubernetes API discovery.
 * `--prometheus-url`: the Prometheus to harvest server-side telemetry from.
   Defaults to the in-cluster service installed by
   [Optional: Prometheus + Grafana](#optional-prometheus--grafana).
 * `--atelet-lag-s`: how long to wait after the run before reading the
-  atelet's snapshot and active actor metrics. Defaults to 70.
+  atelet's metrics. Defaults to 70.
 * `--phase-breakdown` / `--no-phase-breakdown`: read the atelet and worker pod
   logs once the run ends and add the suspend/resume phase percentiles to
   `stats.jsonl`. On by default; needs `pods/log` access in `ate-system` and
@@ -240,6 +241,35 @@ When the run ends, shutdown cancels cycles still in flight. Calls cut short this
 nothing, and the interrupted task records no `TaskCEL` or `TaskWallClock` row, so the end of
 a run does not show up as a burst of failures.
 
+#### Sweperf Log Lines
+
+The boomer log also carries per-task facts for post-processing such as the goodput script in
+`tools/goodput`, which decides from them whether a task was good. The benchmark judges
+nothing. Durations are in milliseconds.
+
+* `Completed sweperf cycle`: `actor`, `cycle`, `cel_ms` (0 when the execute failed),
+  `wall_ms` (resume start to cycle end), `ok` (the execute succeeded, and so did the suspend
+  if there was one), `resume_ms` (as on the `ResumeActor` row; omitted for a no-op resume)
+  and `ttfe_ms` (resume start to the asynchronous ack; omitted without one, and for a no-op
+  resume even when its execute got one).
+* `ResumeActor failed in step`: also carries `wall_ms`. The cycle is retried on the next
+  iteration, and the failed attempt's time counts toward the task's `wall_ms`.
+* `sweperf task record`, one per finished task: `v` (1), `user` (boomer goroutine ID, unique
+  per boomer process), `actor`, `template`, `cycles` and `steps` (the cycle and trace step
+  counts the task ran, resolved when it started, so a mid-run change to
+  `--sweperf-num-cycles` or `--sweperf-total-steps` applies from the next task; `cycles` is
+  capped at `steps`), `task_start` (when the session started) and `end`, `startup_ms`
+  (`CreateActor` until the sandbox answers `/status`), `cel_ms` and `wall_ms` (as in `TaskCEL`
+  and `TaskWallClock`), `think_ms` (think time between the task's cycles), `resume_max_ms`
+  (slowest non-no-op resume), `ttfe_max_ms` (slowest `ttfe_ms`; omitted when no cycle after
+  the no-op resume got an asynchronous ack, so always for a one-cycle task), `retries`
+  (failed resumes), `failed`, and the first failure's `fail_step` (`resume`, `execute` or
+  `suspend`) and `fail_cycle`.
+* `sweperf on_start failed`: also carries `user`, to tie a failed start to its user.
+* `sweperf shutdown started`: the worker began tearing down. Startups and cycles in flight
+  are canceled: interrupted tasks log no task record, and startup failures logged after it
+  belong to the teardown, not the run.
+
 ### Agent-Session Benchmark
 
 The agent-session benchmark (`--user-class agentsession`) emulates a fleet of
@@ -312,8 +342,12 @@ deployment fails loudly instead of showing up as OOM-flaky steps.
   gap gets ±20% jitter so sessions don't move in lockstep.
 * `--resume-mode implicit|explicit` — implicit (default) lets the parked
   first request wake the actor; explicit issues ResumeActor before traffic.
-* `--lifecycle-mode suspend|pause` — durable suspend (default) or node-local
-  pause between steps.
+* `--lifecycle-mode suspend|pause|none` — durable suspend (default),
+  node-local pause between steps, or `none` to keep actors resident in memory
+  between steps without hibernating.
+* `--agentsession-snapshot-steps` — comma-separated 1-based step numbers after
+  which the actor is hibernated (for example, `3,10,20,30,40,50,61`); between
+  all other steps the actor stays resident.
 
 #### Writing an agent-session script
 
@@ -493,13 +527,14 @@ warm-up until they have all joined at the spawn rate.
 
 * `ResumeActorCold`: Latency of each measured resume. A resume that finds its
   actor already running fails, as no cold resume happened.
-* `PauseActor`: Latency of the pause that makes the actor cold again.
-* `PauseActorRecovery`: After a failed cycle, the pause that makes the actor
-  cold for the next one. It is load on the server, so it is recorded, but
-  apart from `PauseActor`. A recovery pause that finds the actor already
-  paused counts as a success.
-* `CreateActor`, `ResumeActorWarmup`, `PauseActorWarmup`: A user's warm-up of
-  its actor, in the step that added the user.
+* `SuspendActorAsync`: Latency of the async SuspendActor that makes the actor
+  cold again.
+* `SuspendActorAsyncRecovery`: After a failed cycle, the async SuspendActor that
+  makes the actor cold for the next one. It is load on the server, so it is
+  recorded, but apart from `SuspendActorAsync`. A recovery suspend that finds
+  the actor already suspended counts as a success.
+* `CreateActor`, `ResumeActorWarmup`, `SuspendActorAsyncWarmup`: A user's
+  warm-up of its actor, in the step that added the user.
 
 ### Isolate Mode
 
@@ -696,6 +731,49 @@ actually did, independent of what the load generator reported.
   nodes: the collector keeps an exited atelet's last value for about 5
   minutes, so a restart can briefly count twice, and an atelet scraped both
   directly and through a collector counts twice.
+* `working_set.node`, `worker`, `atelet`: memory working set in GiB from
+  cAdvisor on the nodes hosting a worker pod, per worker pod (every actor on it
+  included), and per atelet on those nodes. Each has a percentile `summary`
+  (plus `max`) over the steady-state window, `count` of series seen in it, and
+  a `timeseries` every 10s over the whole run with the sum (`total_gb`) and
+  largest (`max_gb`). A spike shorter than 10s can be missed.
+* `working_set.actor`, `per_actor`: the same shape for the actors' own working
+  set as the atelet reports it (`ate_actor_stats_memory_working_set_bytes`),
+  per atelet with every template summed, and the average per measured actor
+  (`ate_actor_stats_sampled_actors`) on each atelet; `count` is atelet
+  processes, keyed like `active_actors`. As `per_actor` is an average per
+  atelet, its `max` is the highest atelet average, not the largest actor, and
+  its `total_gb` has no meaning. A gVisor actor is read from its sandbox cgroup
+  on the host, a microVM actor from inside the guest. As with `active_actors`,
+  suspended actors fall out, but through the telemetry-meter a series that
+  stops being exported keeps its last value for up to 5m. Missing samples are
+  skipped, not counted as 0. The atelet samples every
+  `--actor-stats-poll-interval` (default 1m), so these move in steps of a
+  minute or more. `actor` matches one worker pod's actors only when the node
+  runs one worker pod.
+* `oom_events.node`, `pod`: OOM kills over the run, from cAdvisor's
+  `container_oom_events_total`. `node` counts only global OOMs. `pod` counts
+  kills inside a worker pod (actor sandboxes and the worker container) and is
+  best effort: a non-zero value is real, but a kill whose cgroup is gone before
+  the scrape is missed. A node-allocatable OOM (`oom_memcg=/kubepods.slice`)
+  isn't global, so it isn't in `node`, and `pod` usually misses it; the kubelet
+  evictions that often come with it show in `evicted`. OOMs inside a microVM
+  guest are invisible and show up as task failures. Memory PSI (`node_psi` /
+  `pod_psi`) is the early warning.
+* `oom_events.worker_restarts`, `worker_oomkilled_pods`, `evicted`,
+  `worker_pods_lost`: from the Kubernetes API, read before and after the run.
+  Restarts on a pod deleted during the run are lost. `worker_oomkilled_pods`
+  counts pods whose last restart was `OOMKilled`, not kills; a group kill of
+  the whole worker (`memory.oom.group=1`) shows here. `evicted` is kubelet
+  eviction under node pressure (usually memory), not a kernel OOM kill. It also
+  counts replacement pods the kubelet refused to start while the node was under
+  pressure, as they get the same `Evicted` reason. It comes from `Evicted`
+  events, which last about 1h, so a run longer than that can undercount.
+  `worker_pods_lost` is pods gone by the end for any reason (eviction,
+  scale-down, recycling), not an OOM count; a pod created and deleted within
+  the run isn't counted. A node OOM can also show as a worker `OOMKilled`
+  restart, so never add the fields. A field is `null` when its Kubernetes read
+  fails, and all are with `--no-cluster-facts`.
 
 Every distribution reports p50, p90, p95 and p99 over the steady-state window.
 The steady-state window runs from the first to the last Locust sample at 90% or
@@ -708,14 +786,15 @@ every 10s.
 
 The atelet exports before Prometheus scrapes it, so the harvest waits
 `--atelet-lag-s` seconds (default 70, enough for the OTel SDK's 60s default
-export and a 10s scrape) and reads the snapshot and `active_actors` windows
-half that late, stamping `active_actors` points back by the same amount. The
-window ends at the last full-load sample, so teardown suspends are left out.
-
-`metadata.start_ts` and `end_ts` bound the whole run, which the packing and
-`active_actors` timeseries cover. `steady_start_ts` and `steady_end_ts` bound
-the steady-state window. A flat subset of the numbers also goes into a
-`server_summary` row in `stats.jsonl`.
+export and a 10s scrape) and reads the snapshot, `active_actors` and actor
+working set windows half that late, stamping `active_actors` and actor
+working set points back by the same amount. The window ends at the last
+full-load sample, so teardown suspends are left out.
+
+`metadata.start_ts` and `end_ts` bound the whole run, which the packing,
+`active_actors` and `working_set` timeseries cover. `steady_start_ts` and
+`steady_end_ts` bound the steady-state window. A flat subset of the numbers
+also goes into a `server_summary` row in `stats.jsonl`.
 
 Neither the Kubernetes API nor Prometheus is required. If either is unreachable,
 or discovery was skipped, the affected fields are written as `null` and the run